|
|
|
|
@ -6,11 +6,15 @@ import com.github.kiulian.downloader.cipher.Cipher; |
|
|
|
|
import com.github.kiulian.downloader.cipher.CipherFactory; |
|
|
|
|
import com.github.kiulian.downloader.downloader.Downloader; |
|
|
|
|
import com.github.kiulian.downloader.downloader.YoutubeCallback; |
|
|
|
|
import com.github.kiulian.downloader.downloader.request.RequestPlaylistInfo; |
|
|
|
|
import com.github.kiulian.downloader.downloader.request.RequestVideoInfo; |
|
|
|
|
import com.github.kiulian.downloader.downloader.request.RequestWebpage; |
|
|
|
|
import com.github.kiulian.downloader.downloader.response.Response; |
|
|
|
|
import com.github.kiulian.downloader.downloader.response.ResponseImpl; |
|
|
|
|
import com.github.kiulian.downloader.extractor.Extractor; |
|
|
|
|
import com.github.kiulian.downloader.model.playlist.PlaylistDetails; |
|
|
|
|
import com.github.kiulian.downloader.model.playlist.PlaylistInfo; |
|
|
|
|
import com.github.kiulian.downloader.model.playlist.PlaylistVideoDetails; |
|
|
|
|
import com.github.kiulian.downloader.model.subtitles.SubtitlesInfo; |
|
|
|
|
import com.github.kiulian.downloader.model.videos.VideoDetails; |
|
|
|
|
import com.github.kiulian.downloader.model.videos.VideoInfo; |
|
|
|
|
@ -27,6 +31,7 @@ import java.io.UnsupportedEncodingException; |
|
|
|
|
import java.net.URLDecoder; |
|
|
|
|
import java.util.ArrayList; |
|
|
|
|
import java.util.Collections; |
|
|
|
|
import java.util.LinkedList; |
|
|
|
|
import java.util.List; |
|
|
|
|
import java.util.concurrent.ExecutorService; |
|
|
|
|
import java.util.concurrent.Future; |
|
|
|
|
@ -258,7 +263,6 @@ public class ParserImpl implements Parser { |
|
|
|
|
throw new YoutubeException.BadPageException("deciphering is required but no js url"); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
boolean hasVideo = itag.isVideo() || json.has("size") || json.has("width"); |
|
|
|
|
boolean hasAudio = itag.isAudio() || json.has("audioQuality"); |
|
|
|
|
if (hasVideo && hasAudio) return new VideoWithAudioFormat(json, isAdaptive, clientVersion); |
|
|
|
|
@ -292,4 +296,146 @@ public class ParserImpl implements Parser { |
|
|
|
|
} |
|
|
|
|
return subtitlesInfo; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
|
public Response<PlaylistInfo> parsePlaylist(RequestPlaylistInfo request) { |
|
|
|
|
if (request.isAsync()) { |
|
|
|
|
ExecutorService executorService = config.getExecutorService(); |
|
|
|
|
Future<PlaylistInfo> result = executorService.submit(() -> parsePlaylist(request.getPlaylistId(), request.getCallback())); |
|
|
|
|
return ResponseImpl.fromFuture(result); |
|
|
|
|
} |
|
|
|
|
try { |
|
|
|
|
PlaylistInfo result = parsePlaylist(request.getPlaylistId(), request.getCallback()); |
|
|
|
|
return ResponseImpl.from(result); |
|
|
|
|
} catch (YoutubeException e) { |
|
|
|
|
return ResponseImpl.error(e); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private PlaylistInfo parsePlaylist(String playlistId, YoutubeCallback<PlaylistInfo> callback) throws YoutubeException { |
|
|
|
|
String htmlUrl = "https://www.youtube.com/playlist?list=" + playlistId; |
|
|
|
|
Response<String> response = downloader.downloadWebpage(new RequestWebpage(htmlUrl)); |
|
|
|
|
if (!response.ok()) { |
|
|
|
|
YoutubeException e = new YoutubeException.DownloadException(String.format("Could not load url: %s, exception: %s", htmlUrl, response.error().getMessage())); |
|
|
|
|
if (callback != null) callback.onError(e); |
|
|
|
|
throw e; |
|
|
|
|
} |
|
|
|
|
String html = response.data(); |
|
|
|
|
JsonObject initialData; |
|
|
|
|
try { |
|
|
|
|
initialData = extractor.extractInitialDataFromHtml(html); |
|
|
|
|
} catch (YoutubeException e) { |
|
|
|
|
if (callback != null) callback.onError(e); |
|
|
|
|
throw e; |
|
|
|
|
} |
|
|
|
|
if (!initialData.has("metadata")) { |
|
|
|
|
throw new YoutubeException.BadPageException("Invalid initial data json"); |
|
|
|
|
} |
|
|
|
|
String title = initialData.getAsJsonObject("metadata").getAsJsonObject("playlistMetadataRenderer").get("title").getAsString(); |
|
|
|
|
PlaylistDetails playlistDetails = new PlaylistDetails(playlistId, title); |
|
|
|
|
List<PlaylistVideoDetails> videos; |
|
|
|
|
try { |
|
|
|
|
videos = parsePlaylistVideos(initialData); |
|
|
|
|
} catch (YoutubeException e) { |
|
|
|
|
if (callback != null) callback.onError(e); |
|
|
|
|
throw e; |
|
|
|
|
} |
|
|
|
|
return new PlaylistInfo(playlistDetails, videos); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private List<PlaylistVideoDetails> parsePlaylistVideos(JsonObject initialData) throws YoutubeException { |
|
|
|
|
JsonObject content; |
|
|
|
|
try { |
|
|
|
|
content = initialData.getAsJsonObject("contents") |
|
|
|
|
.getAsJsonObject("twoColumnBrowseResultsRenderer") |
|
|
|
|
.getAsJsonArray("tabs").get(0).getAsJsonObject() |
|
|
|
|
.getAsJsonObject("tabRenderer") |
|
|
|
|
.getAsJsonObject("content") |
|
|
|
|
.getAsJsonObject("sectionListRenderer") |
|
|
|
|
.getAsJsonArray("contents").get(0).getAsJsonObject() |
|
|
|
|
.getAsJsonObject("itemSectionRenderer") |
|
|
|
|
.getAsJsonArray("contents").get(0).getAsJsonObject() |
|
|
|
|
.getAsJsonObject("playlistVideoListRenderer"); |
|
|
|
|
} catch (NullPointerException e) { |
|
|
|
|
throw new YoutubeException.BadPageException("Playlist initial data not found"); |
|
|
|
|
} |
|
|
|
|
List<PlaylistVideoDetails> videos = new LinkedList<>(); |
|
|
|
|
JsonObject context = initialData.getAsJsonObject("responseContext"); |
|
|
|
|
String clientVersion = extractor.extractClientVersionFromContext(context); |
|
|
|
|
populatePlaylist(content, videos, clientVersion); |
|
|
|
|
return videos; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private void populatePlaylist(JsonObject content, List<PlaylistVideoDetails> videos, String clientVersion) throws YoutubeException { |
|
|
|
|
JsonArray contents; |
|
|
|
|
if (content.has("contents")) { // parse first items (up to 100)
|
|
|
|
|
contents = content.getAsJsonArray("contents"); |
|
|
|
|
} else if (content.has("continuationItems")) { // parse continuationItems
|
|
|
|
|
contents = content.getAsJsonArray("continuationItems"); |
|
|
|
|
} else if (content.has("continuations")) { // load continuation
|
|
|
|
|
JsonObject nextContinuationData = content.getAsJsonArray("continuations").get(0).getAsJsonObject().getAsJsonObject("nextContinuationData"); |
|
|
|
|
String continuation = nextContinuationData.get("continuation").getAsString(); |
|
|
|
|
String ctp = nextContinuationData.get("clickTrackingParams").getAsString(); |
|
|
|
|
loadPlaylistContinuation(continuation, ctp, videos, clientVersion); |
|
|
|
|
return; |
|
|
|
|
} else { // nothing found
|
|
|
|
|
return; |
|
|
|
|
} |
|
|
|
|
for (int i = 0; i < contents.size(); i++) { |
|
|
|
|
JsonObject contentsItem = contents.get(i).getAsJsonObject(); |
|
|
|
|
if (contentsItem.has("playlistVideoRenderer")) { |
|
|
|
|
videos.add(new PlaylistVideoDetails(contentsItem.getAsJsonObject("playlistVideoRenderer"))); |
|
|
|
|
} else { |
|
|
|
|
if (contentsItem.has("continuationItemRenderer")) { |
|
|
|
|
JsonObject continuationEndpoint = contentsItem.getAsJsonObject("continuationItemRenderer").getAsJsonObject("continuationEndpoint"); |
|
|
|
|
String continuation = continuationEndpoint.getAsJsonObject("continuationCommand").get("token").getAsString(); |
|
|
|
|
String ctp = continuationEndpoint.get("clickTrackingParams").getAsString(); |
|
|
|
|
loadPlaylistContinuation(continuation, ctp, videos, clientVersion); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private void loadPlaylistContinuation(String continuation, String ctp, List<PlaylistVideoDetails> videos, String clientVersion) throws YoutubeException { |
|
|
|
|
JsonObject client = new JsonObject(); |
|
|
|
|
client.addProperty("clientName", "WEB"); |
|
|
|
|
client.addProperty("clientVersion", "2.20201021.03.00"); |
|
|
|
|
|
|
|
|
|
JsonObject context = new JsonObject(); |
|
|
|
|
context.add("client", client); |
|
|
|
|
|
|
|
|
|
JsonObject clickTracking = new JsonObject(); |
|
|
|
|
clickTracking.addProperty("clickTrackingParams", ctp); |
|
|
|
|
|
|
|
|
|
JsonObject body = new JsonObject(); |
|
|
|
|
body.add("context", context); |
|
|
|
|
body.add("clickTracking", clickTracking); |
|
|
|
|
body.addProperty("continuation", continuation); |
|
|
|
|
|
|
|
|
|
String url = "https://www.youtube.com/youtubei/v1/browse?key=" + ANDROID_APIKEY; |
|
|
|
|
RequestWebpage request = new RequestWebpage(url, "POST", body.toString()) |
|
|
|
|
.header("X-YouTube-Client-Name", "1") |
|
|
|
|
.header("X-YouTube-Client-Version", clientVersion) |
|
|
|
|
.header("Content-Type", "application/json"); |
|
|
|
|
|
|
|
|
|
Response<String> response = downloader.downloadWebpage(request); |
|
|
|
|
if (!response.ok()) { |
|
|
|
|
throw new YoutubeException.DownloadException(String.format("Could not load url: %s, exception: %s", url, response.error().getMessage())); |
|
|
|
|
} |
|
|
|
|
String html = response.data(); |
|
|
|
|
try { |
|
|
|
|
JsonObject content; |
|
|
|
|
JsonObject jsonResponse = JsonParser.parseString(html).getAsJsonObject(); |
|
|
|
|
if (jsonResponse.has("continuationContents")) { |
|
|
|
|
content = jsonResponse.getAsJsonObject("continuationContents").getAsJsonObject("playlistVideoListContinuation"); |
|
|
|
|
} else { |
|
|
|
|
content = jsonResponse.getAsJsonArray("onResponseReceivedActions").get(0).getAsJsonObject().getAsJsonObject("appendContinuationItemsAction"); |
|
|
|
|
} |
|
|
|
|
populatePlaylist(content, videos, clientVersion); |
|
|
|
|
} catch (YoutubeException e) { |
|
|
|
|
throw e; |
|
|
|
|
} catch (Exception e) { |
|
|
|
|
throw new YoutubeException.BadPageException("Could not parse playlist continuation json"); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
FongMi
2 years ago