From 1ad7e5a6bee28c5332a72c26caebcef2e1945f93 Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Sat, 23 Jan 2021 18:17:35 +0100 Subject: [PATCH] Support SoundCloud HLS by using a workaround This commit tries to support SoundCloud HLS streams by parsing M3U manifests, get the last segment URL (in order to get track length) and request a segment URL equals to track's duration so it's a single URL. --- .../extractors/SoundcloudStreamExtractor.java | 76 ++++++++++++++----- 1 file changed, 56 insertions(+), 20 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 62e79cb2d1..dc79d05d0c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -11,7 +11,6 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; -import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; @@ -33,11 +32,14 @@ import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.annotation.Nonnull; import javax.annotation.Nullable; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; +import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class SoundcloudStreamExtractor extends StreamExtractor { @@ -179,7 +181,7 @@ public String getHlsUrl() { @Override public List getAudioStreams() throws IOException, ExtractionException { - List audioStreams = new ArrayList<>(); + final List audioStreams = new ArrayList<>(); final Downloader dl = NewPipe.getDownloader(); // Streams can be streamable and downloadable - or explicitly not. @@ -190,50 +192,84 @@ public List getAudioStreams() throws IOException, ExtractionExcepti try { final JsonArray transcodings = track.getObject("media").getArray("transcodings"); - // get information about what stream formats are available - for (Object transcoding : transcodings) { - + // Get information about what stream formats are available + for (final Object transcoding : transcodings) { final JsonObject t = (JsonObject) transcoding; String url = t.getString("url"); + final String mediaUrl; + final MediaFormat mediaFormat; + final int bitrate; if (!isNullOrEmpty(url)) { + if (t.getString("preset").contains("mp3")) { + mediaFormat = MediaFormat.MP3; + bitrate = 128; + } else if (t.getString("preset").contains("opus")) { + mediaFormat = MediaFormat.OPUS; + bitrate = 64; + } else { + continue; + } + + // TODO: move this to a separate method to generate valid urls when needed (e.g. resuming a paused stream) - // We can only play the mp3 format, but not handle m3u playlists / streams. - // what about Opus? - if (t.getString("preset").contains("mp3") - && t.getObject("format").getString("protocol").equals("progressive")) { + if (t.getObject("format").getString("protocol").equals("progressive")) { // This url points to the endpoint which generates a unique and short living url to the stream. - // TODO: move this to a separate method to generate valid urls when needed (e.g. resuming a paused stream) url += "?client_id=" + SoundcloudParsingHelper.clientId(); final String res = dl.get(url).responseBody(); try { JsonObject mp3UrlObject = JsonParser.object().from(res); // Links in this file are also only valid for a short period. - audioStreams.add(new AudioStream(mp3UrlObject.getString("url"), - MediaFormat.MP3, 128)); - } catch (JsonParserException e) { + mediaUrl = mp3UrlObject.getString("url"); + } catch (final JsonParserException e) { + throw new ParsingException("Could not parse streamable url", e); + } + } else if (t.getObject("format").getString("protocol").equals("hls")) { + // This url points to the endpoint which generates a unique and short living url to the stream. + url += "?client_id=" + SoundcloudParsingHelper.clientId(); + final String res = dl.get(url).responseBody(); + + try { + final JsonObject mp3HlsUrlObject = JsonParser.object().from(res); + // Links in this file are also only valid for a short period. + + // Parsing the HLS manifest to get a single file by requesting a range equal to 0-track_length + final String hlsManifestResponse = dl.get(mp3HlsUrlObject.getString("url")).responseBody(); + final List hlsRangesList = new ArrayList<>(); + final Matcher regex = Pattern.compile("((https?):((//)|(\\\\))+[\\w\\d:#@%/;$()~_?+-=\\\\.&]*)") + .matcher(hlsManifestResponse); + + while (regex.find()) { + hlsRangesList.add(hlsManifestResponse.substring(regex.start(0), regex.end(0))); + } + + final String hlsLastRangeUrl = hlsRangesList.get(hlsRangesList.size() - 1); + final String[] hlsLastRangeUrlArray = hlsLastRangeUrl.split("/"); + + mediaUrl = HTTPS + hlsLastRangeUrlArray[2] + "/media/0/" + hlsLastRangeUrlArray[5] + "/" + hlsLastRangeUrlArray[6]; + } catch (final JsonParserException e) { throw new ParsingException("Could not parse streamable url", e); } + } else { + continue; } + + audioStreams.add(new AudioStream(mediaUrl, mediaFormat, bitrate)); } } - } catch (NullPointerException e) { + } catch (final NullPointerException e) { throw new ExtractionException("Could not get SoundCloud's track audio url", e); } - if (audioStreams.isEmpty()) { - throw new ContentNotSupportedException("HLS audio streams are not yet supported"); - } - return audioStreams; } private static String urlEncode(String value) { try { return URLEncoder.encode(value, "UTF-8"); - } catch (UnsupportedEncodingException e) { + } catch (final UnsupportedEncodingException e) { throw new IllegalStateException(e); } } @@ -334,4 +370,4 @@ public List getStreamSegments() { public List getMetaInfo() { return Collections.emptyList(); } -} +} \ No newline at end of file