Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix broken yt likes in comments #628

Merged
merged 16 commits into from
May 29, 2021
Merged
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ public class CommentsInfoItem extends InfoItem {
@Nullable
private DateWrapper uploadDate;
private int likeCount;
private String textualLikeCount;
private boolean heartedByUploader;
private boolean pinned;

Expand Down Expand Up @@ -89,6 +90,14 @@ public void setLikeCount(int likeCount) {
this.likeCount = likeCount;
}

public String getTextualLikeCount() {
return textualLikeCount;
}

public void setTextualLikeCount(String textualLikeCount) {
this.textualLikeCount = textualLikeCount;
}

public void setHeartedByUploader(boolean isHeartedByUploader) {
this.heartedByUploader = isHeartedByUploader;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,59 +3,95 @@
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.utils.Utils;

import javax.annotation.Nullable;

public interface CommentsInfoItemExtractor extends InfoItemExtractor {

/**
* Return the like count of the comment, or -1 if it's unavailable
* Return the like count of the comment, or -1 if it's unavailable<br/>
*
* NOTE: Currently only implemented for YT {@link YoutubeCommentsInfoItemExtractor#getLikeCount()}
* with limitations (only approximate like count is returned)
*
* @see StreamExtractor#getLikeCount()
*/
int getLikeCount() throws ParsingException;
default int getLikeCount() throws ParsingException {
return -1;
}

/**
* The unmodified like count given by the service<br/>
*
* It may be language dependent
*/
default String getTextualLikeCount() throws ParsingException {
return Utils.EMPTY_STRING;
}

/**
* The text of the comment
*/
String getCommentText() throws ParsingException;
default String getCommentText() throws ParsingException {
return Utils.EMPTY_STRING;
}

/**
* The upload date given by the service, unmodified
*
* @see StreamExtractor#getTextualUploadDate()
*/
String getTextualUploadDate() throws ParsingException;
default String getTextualUploadDate() throws ParsingException {
return Utils.EMPTY_STRING;
}

/**
* The upload date wrapped with DateWrapper class
*
* @see StreamExtractor#getUploadDate()
*/
@Nullable
DateWrapper getUploadDate() throws ParsingException;
default DateWrapper getUploadDate() throws ParsingException {
return null;
}

String getCommentId() throws ParsingException;
default String getCommentId() throws ParsingException {
return Utils.EMPTY_STRING;
}

String getUploaderUrl() throws ParsingException;
default String getUploaderUrl() throws ParsingException {
return Utils.EMPTY_STRING;
}

String getUploaderName() throws ParsingException;
default String getUploaderName() throws ParsingException {
return Utils.EMPTY_STRING;
}

String getUploaderAvatarUrl() throws ParsingException;
default String getUploaderAvatarUrl() throws ParsingException {
return Utils.EMPTY_STRING;
}

/**
* Whether the comment has been hearted by the uploader
*/
boolean isHeartedByUploader() throws ParsingException;
default boolean isHeartedByUploader() throws ParsingException {
return false;
}

/**
* Whether the comment is pinned
*/
boolean isPinned() throws ParsingException;
default boolean isPinned() throws ParsingException {
return false;
}

/**
* Whether the uploader is verified by the service
*/
boolean isUploaderVerified() throws ParsingException;
default boolean isUploaderVerified() throws ParsingException {
return false;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@

import java.util.ArrayList;
import java.util.List;
import java.util.Vector;

public class CommentsInfoItemsCollector extends InfoItemsCollector<CommentsInfoItem, CommentsInfoItemExtractor> {

Expand Down Expand Up @@ -65,6 +64,11 @@ public CommentsInfoItem extract(CommentsInfoItemExtractor extractor) throws Pars
} catch (Exception e) {
addError(e);
}
try {
resultItem.setTextualLikeCount(extractor.getTextualLikeCount());
} catch (Exception e) {
addError(e);
}
try {
resultItem.setThumbnailUrl(extractor.getThumbnailUrl());
} catch (Exception e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,6 @@
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;

import javax.annotation.Nullable;

public class BandcampCommentsInfoItemExtractor implements CommentsInfoItemExtractor {

Expand All @@ -32,39 +29,11 @@ public String getThumbnailUrl() throws ParsingException {
return writing.getElementsByClass("thumb").attr("src");
}

@Override
public int getLikeCount() {
return -1;
}

@Override
public String getCommentText() {
return writing.getElementsByClass("text").first().ownText();
}

@Override
public String getTextualUploadDate() {
return "";
}

@Nullable
@Override
public DateWrapper getUploadDate() {
return null;
}

@Override
public String getCommentId() {
return "";
}

@Override
public String getUploaderUrl() {
//return writing.getElementsByClass("name").attr("href");
// Fan links cannot be opened
return "";
}

@Override
public String getUploaderName() throws ParsingException {
return writing.getElementsByClass("name").first().text();
Expand All @@ -74,19 +43,4 @@ public String getUploaderName() throws ParsingException {
public String getUploaderAvatarUrl() {
return writing.getElementsByClass("thumb").attr("src");
}

@Override
public boolean isHeartedByUploader() {
return false;
}

@Override
public boolean isPinned() {
return false;
}

@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,6 @@ public DateWrapper getUploadDate() throws ParsingException {
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
}

@Override
public int getLikeCount() {
return -1;
}

@Override
public String getCommentText() throws ParsingException {
final String htmlText = JsonUtils.getString(item, "text");
Expand Down Expand Up @@ -89,21 +84,6 @@ public String getUploaderAvatarUrl() {
return baseUrl + value;
}

@Override
public boolean isHeartedByUploader() throws ParsingException {
return false;
}

@Override
public boolean isPinned() throws ParsingException {
return false;
}

@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}

@Override
public String getUploaderName() throws ParsingException {
return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,16 +38,6 @@ public String getUploaderAvatarUrl() {
return json.getObject("user").getString("avatar_url");
}

@Override
public boolean isHeartedByUploader() throws ParsingException {
return false;
}

@Override
public boolean isPinned() throws ParsingException {
return false;
}

@Override
public boolean isUploaderVerified() throws ParsingException {
return json.getObject("user").getBoolean("verified");
Expand All @@ -69,11 +59,6 @@ public DateWrapper getUploadDate() throws ParsingException {
return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualUploadDate()));
}

@Override
public int getLikeCount() {
return -1;
}

@Override
public String getName() throws ParsingException {
return json.getObject("user").getString("permalink");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;

import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
Expand Down Expand Up @@ -70,12 +71,70 @@ public DateWrapper getUploadDate() throws ParsingException {
}
}

/**
* @implNote The method is parsing internally a localized string.<br/>
* <ul>
* <li>
* More than >1k likes will result in an inaccurate number
* </li>
* <li>
* This will fail for other languages than English.
* However as long as the Extractor only uses "en-GB"
* (as seen in {@link org.schabi.newpipe.extractor.services.youtube.YoutubeService#SUPPORTED_LANGUAGES})
* everything will work fine.
* </li>
* </ul>
* <br/>
* Consider using {@link #getTextualLikeCount()}
*/
@Override
public int getLikeCount() throws ParsingException {
// This may return a language dependent version, e.g. in German: 3,3 Mio
final String textualLikeCount = getTextualLikeCount();
try {
if (Utils.isBlank(textualLikeCount)) {
return 0;
}

return (int) Utils.mixedNumberWordToLong(textualLikeCount);
} catch (Exception e) {
throw new ParsingException("Unexpected error while converting textual like count to like count", e);
}
}

@Override
public String getTextualLikeCount() throws ParsingException {
/*
* Example results as of 2021-05-20:
* Language = English
* 3.3M
* 48K
* 1.4K
* 270K
* 19
* 6
*
* Language = German
* 3,3 Mio
* 48.189
* 1419
* 270.984
* 19
* 6
*/
try {
return json.getInt("likeCount");
// If a comment has no likes voteCount is not set
if (!json.has("voteCount")) {
return EMPTY_STRING;
}

final JsonObject voteCountObj = JsonUtils.getObject(json, "voteCount");
if (voteCountObj.isEmpty()) {
return EMPTY_STRING;
}
return getTextFromObject(voteCountObj);
} catch (Exception e) {
throw new ParsingException("Could not get like count", e);
throw new ParsingException("Could not get vote count", e);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@

import java.io.IOException;

import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.Bandcamp;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;

public class BandcampCommentsExtractorTest {

Expand Down Expand Up @@ -47,6 +47,7 @@ public void testGetCommentsAllData() throws IOException, ExtractionException {
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertEquals(-1, c.getLikeCount());
assertTrue(Utils.isBlank(c.getTextualLikeCount()));
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,8 @@ public void testGetCommentsAllData() throws IOException, ExtractionException {
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() != -1);
assertEquals(-1, c.getLikeCount());
assertTrue(Utils.isBlank(c.getTextualLikeCount()));
}
}

Expand Down
Loading