Merge pull request #293 from Stypox/fix-soundcloud

[SoundCloud] Fix by migrating to `api-v2` and other improvements

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java

@@ -44,7 +44,7 @@ public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException,
     }
 
 
-    private void computNextPageAndStreams() throws IOException, ExtractionException {
+    private void computeNextPageAndStreams() throws IOException, ExtractionException {
         collector = new StreamInfoItemsCollector(getServiceId());
 
         String apiUrl = "https://api-v2.soundcloud.com/charts" +
@@ -69,7 +69,7 @@ private void computNextPageAndStreams() throws IOException, ExtractionException
     @Override
     public String getNextPageUrl() throws IOException, ExtractionException {
         if (nextPageUrl == null) {
-            computNextPageAndStreams();
+            computeNextPageAndStreams();
         }
         return nextPageUrl;
     }
@@ -78,7 +78,7 @@ public String getNextPageUrl() throws IOException, ExtractionException {
     @Override
     public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
         if (collector == null) {
-            computNextPageAndStreams();
+            computeNextPageAndStreams();
         }
         return new InfoItemsPage<>(collector, getNextPageUrl());
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -108,7 +108,7 @@ static Calendar parseDate(String textualUploadDate) throws ParsingException {
      * See https://developers.soundcloud.com/docs/api/reference#resolve
      */
     public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException {
-        String apiUrl = "https://api.soundcloud.com/resolve"
+        String apiUrl = "https://api-v2.soundcloud.com/resolve"
                 + "?url=" + URLEncoder.encode(url, "UTF-8")
                 + "&client_id=" + clientId();
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java

@@ -1,8 +1,11 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
+import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
+
+import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -12,16 +15,20 @@
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 
-import javax.annotation.Nonnull;
 import java.io.IOException;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
 @SuppressWarnings("WeakerAccess")
 public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
+    private static final int streamsPerRequestedPage = 15;
+
     private String playlistId;
     private JsonObject playlist;
 
-    private StreamInfoItemsCollector streamInfoItemsCollector = null;
-    private String nextPageUrl = null;
+    private StreamInfoItemsCollector streamInfoItemsCollector;
+    private String nextPageUrl;
 
     public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
         super(service, linkHandler);
@@ -31,7 +38,7 @@ public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler lin
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
 
         playlistId = getLinkHandler().getId();
-        String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId +
+        String apiUrl = "https://api-v2.soundcloud.com/playlists/" + playlistId +
                 "?client_id=" + SoundcloudParsingHelper.clientId() +
                 "&representation=compact";
 
@@ -55,6 +62,7 @@ public String getName() {
         return playlist.getString("title");
     }
 
+    @Nullable
     @Override
     public String getThumbnailUrl() {
         String artworkUrl = playlist.getString("artwork_url");
@@ -64,21 +72,20 @@ public String getThumbnailUrl() {
             // if it also fails, return null
             try {
                 final InfoItemsPage<StreamInfoItem> infoItems = getInitialPage();
-                if (infoItems.getItems().isEmpty()) return null;
 
                 for (StreamInfoItem item : infoItems.getItems()) {
-                    final String thumbnailUrl = item.getThumbnailUrl();
-                    if (thumbnailUrl == null || thumbnailUrl.isEmpty()) continue;
-
-                    String thumbnailUrlBetterResolution = thumbnailUrl.replace("large.jpg", "crop.jpg");
-                    return thumbnailUrlBetterResolution;
+                    artworkUrl = item.getThumbnailUrl();
+                    if (artworkUrl != null && !artworkUrl.isEmpty()) break;
                 }
             } catch (Exception ignored) {
             }
+
+            if (artworkUrl == null) {
+                return null;
+            }
         }
 
-        String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
-        return artworkUrlBetterResolution;
+        return artworkUrl.replace("large.jpg", "crop.jpg");
     }
 
     @Override
@@ -110,27 +117,42 @@ public long getStreamCount() {
     @Override
     public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
         if (streamInfoItemsCollector == null) {
-            computeStreamsAndNextPageUrl();
+            computeInitialTracksAndNextPageUrl();
         }
-        return new InfoItemsPage<>(streamInfoItemsCollector, getNextPageUrl());
+        return new InfoItemsPage<>(streamInfoItemsCollector, nextPageUrl);
     }
 
-    private void computeStreamsAndNextPageUrl() throws ExtractionException, IOException {
+    private void computeInitialTracksAndNextPageUrl() throws IOException, ExtractionException {
         streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
+        StringBuilder nextPageUrlBuilder = new StringBuilder("https://api-v2.soundcloud.com/tracks?client_id=");
+        nextPageUrlBuilder.append(SoundcloudParsingHelper.clientId());
+        nextPageUrlBuilder.append("&ids=");
+
+        JsonArray tracks = playlist.getArray("tracks");
+        for (Object o : tracks) {
+            if (o instanceof JsonObject) {
+                JsonObject track = (JsonObject) o;
+                if (track.has("title")) { // i.e. if full info is available
+                    streamInfoItemsCollector.commit(new SoundcloudStreamInfoItemExtractor(track));
+                } else {
+                    // %09d would be enough, but a 0 before the number does not create problems, so let's be sure
+                    nextPageUrlBuilder.append(String.format("%010d,", track.getInt("id")));
+                }
+            }
+        }
 
-        // Note the "api", NOT "api-v2"
-        String apiUrl = "https://api.soundcloud.com/playlists/" + getId() + "/tracks"
-                + "?client_id=" + SoundcloudParsingHelper.clientId()
-                + "&limit=20"
-                + "&linked_partitioning=1";
-
-        nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl);
+        nextPageUrlBuilder.setLength(nextPageUrlBuilder.length() - 1); // remove trailing ,
+        nextPageUrl = nextPageUrlBuilder.toString();
+        if (nextPageUrl.endsWith("&ids")) {
+            // there are no other videos
+            nextPageUrl = "";
+        }
     }
 
     @Override
     public String getNextPageUrl() throws IOException, ExtractionException {
         if (nextPageUrl == null) {
-            computeStreamsAndNextPageUrl();
+            computeInitialTracksAndNextPageUrl();
         }
         return nextPageUrl;
     }
@@ -141,9 +163,36 @@ public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException,
             throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
         }
 
+        // see computeInitialTracksAndNextPageUrl
+        final int lengthFirstPartOfUrl = ("https://api-v2.soundcloud.com/tracks?client_id="
+                + SoundcloudParsingHelper.clientId()
+                + "&ids=").length();
+        final int lengthOfEveryStream = 11;
+
+        String currentPageUrl, nextUrl;
+        int lengthMaxStreams = lengthFirstPartOfUrl + lengthOfEveryStream * streamsPerRequestedPage;
+        if (pageUrl.length() <= lengthMaxStreams) {
+            currentPageUrl = pageUrl; // fetch every remaining video, there are less than the max
+            nextUrl = ""; // afterwards the list is complete
+        } else {
+            currentPageUrl = pageUrl.substring(0, lengthMaxStreams);
+            nextUrl = pageUrl.substring(0, lengthFirstPartOfUrl) + pageUrl.substring(lengthMaxStreams);
+        }
+
         StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl);
+        String response = NewPipe.getDownloader().get(currentPageUrl, getExtractorLocalization()).responseBody();
+
+        try {
+            JsonArray tracks = JsonParser.array().from(response);
+            for (Object track : tracks) {
+                if (track instanceof JsonObject) {
+                    collector.commit(new SoundcloudStreamInfoItemExtractor((JsonObject) track));
+                }
+            }
+        } catch (JsonParserException e) {
+            throw new ParsingException("Could not parse json response", e);
+        }
 
-        return new InfoItemsPage<>(collector, nextPageUrl);
+        return new InfoItemsPage<>(collector, nextUrl);
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java

@@ -4,6 +4,7 @@
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
+
 import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
@@ -13,9 +14,15 @@
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
-import org.schabi.newpipe.extractor.stream.*;
+import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.Description;
+import org.schabi.newpipe.extractor.stream.StreamExtractor;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.stream.StreamType;
+import org.schabi.newpipe.extractor.stream.SubtitlesStream;
+import org.schabi.newpipe.extractor.stream.VideoStream;
 
-import javax.annotation.Nonnull;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
@@ -24,6 +31,8 @@
 import java.util.List;
 import java.util.Locale;
 
+import javax.annotation.Nonnull;
+
 public class SoundcloudStreamExtractor extends StreamExtractor {
     private JsonObject track;
 
@@ -55,14 +64,14 @@ public String getName() {
 
     @Nonnull
     @Override
-    public String getTextualUploadDate() {
-        return track.getString("created_at");
+    public String getTextualUploadDate() throws ParsingException {
+        return track.getString("created_at").replace("T"," ").replace("Z", "");
     }
 
     @Nonnull
     @Override
     public DateWrapper getUploadDate() throws ParsingException {
-        return new DateWrapper(SoundcloudParsingHelper.parseDate(getTextualUploadDate()));
+        return new DateWrapper(SoundcloudParsingHelper.parseDate(track.getString("created_at")));
     }
 
     @Nonnull
@@ -146,24 +155,13 @@ public List<AudioStream> getAudioStreams() throws IOException, ExtractionExcepti
         List<AudioStream> audioStreams = new ArrayList<>();
         Downloader dl = NewPipe.getDownloader();
 
-        String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId())
-                + "?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());
-
-        String response = dl.get(apiUrl, getExtractorLocalization()).responseBody();
-        JsonObject responseObject;
-        try {
-            responseObject = JsonParser.object().from(response);
-        } catch (JsonParserException e) {
-            throw new ParsingException("Could not parse json response", e);
-        }
-
         // Streams can be streamable and downloadable - or explicitly not.
         // For playing the track, it is only necessary to have a streamable track.
         // If this is not the case, this track might not be published yet.
-        if (!responseObject.getBoolean("streamable")) return audioStreams;
+        if (!track.getBoolean("streamable")) return audioStreams;
 
         try {
-            JsonArray transcodings = responseObject.getObject("media").getArray("transcodings");
+            JsonArray transcodings = track.getObject("media").getArray("transcodings");
 
             // get information about what stream formats are available
             for (Object transcoding : transcodings) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java

@@ -36,7 +36,7 @@ public List<SubscriptionItem> fromChannelUrl(String channelUrl) throws IOExcepti
             throw new InvalidSourceException(e);
         }
 
-        String apiUrl = "https://api.soundcloud.com/users/" + id + "/followings"
+        String apiUrl = "https://api-v2.soundcloud.com/users/" + id + "/followings"
                 + "?client_id=" + SoundcloudParsingHelper.clientId()
                 + "&limit=200";
         ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(service.getServiceId());

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java

@@ -24,6 +24,8 @@ public void assertThatHardcodedClientIdIsValid() throws Exception {
     public void resolveUrlWithEmbedPlayerTest() throws Exception {
         Assert.assertEquals("https://soundcloud.com/trapcity", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/26057743"));
         Assert.assertEquals("https://soundcloud.com/nocopyrightsounds", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/16069159"));
+        Assert.assertEquals("https://soundcloud.com/trapcity", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api-v2.soundcloud.com/users/26057743"));
+        Assert.assertEquals("https://soundcloud.com/nocopyrightsounds", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api-v2.soundcloud.com/users/16069159"));
     }
 
     @Test