diff --git a/app/src/main/java/com/fongmi/android/tv/model/SiteViewModel.java b/app/src/main/java/com/fongmi/android/tv/model/SiteViewModel.java index 395966041..9d093bc6a 100644 --- a/app/src/main/java/com/fongmi/android/tv/model/SiteViewModel.java +++ b/app/src/main/java/com/fongmi/android/tv/model/SiteViewModel.java @@ -19,7 +19,6 @@ import com.fongmi.android.tv.bean.Url; import com.fongmi.android.tv.bean.Vod; import com.fongmi.android.tv.exception.ExtractException; import com.fongmi.android.tv.player.Source; -import com.fongmi.android.tv.player.extractor.Thunder; import com.fongmi.android.tv.utils.ResUtil; import com.fongmi.android.tv.utils.Sniffer; import com.github.catvod.crawler.Spider; @@ -32,12 +31,9 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.Iterator; -import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import okhttp3.Call; @@ -123,7 +119,7 @@ public class SiteViewModel extends ViewModel { VodConfig.get().setRecent(site); Result result = Result.fromJson(detailContent); if (!result.getList().isEmpty()) result.getList().get(0).setVodFlags(); - if (!result.getList().isEmpty()) checkThunder(result.getList().get(0).getVodFlags()); + if (!result.getList().isEmpty()) Source.get().parse(result.getList().get(0).getVodFlags()); return result; } else if (site.isEmpty() && "push_agent".equals(key)) { Vod vod = new Vod(); @@ -131,7 +127,7 @@ public class SiteViewModel extends ViewModel { vod.setVodName(id); vod.setVodPic("https://pic.rmb.bdstatic.com/bjh/1d0b02d0f57f0a42201f92caba5107ed.jpeg"); vod.setVodFlags(Flag.create(ResUtil.getString(R.string.push), ResUtil.getString(R.string.play), id)); - checkThunder(vod.getVodFlags()); + Source.get().parse(vod.getVodFlags()); return Result.vod(vod); } else { ArrayMap params = new ArrayMap<>(); @@ -141,7 +137,7 @@ public class SiteViewModel extends ViewModel { SpiderDebug.log(detailContent); Result result = Result.fromType(site.getType(), detailContent); if (!result.getList().isEmpty()) result.getList().get(0).setVodFlags(); - if (!result.getList().isEmpty()) checkThunder(result.getList().get(0).getVodFlags()); + if (!result.getList().isEmpty()) Source.get().parse(result.getList().get(0).getVodFlags()); return result; } }); @@ -269,28 +265,6 @@ public class SiteViewModel extends ViewModel { return result; } - private void checkThunder(List flags) throws Exception { - for (Flag flag : flags) { - ExecutorService executor = Executors.newFixedThreadPool(Constant.THREAD_POOL * 2); - for (Future> future : executor.invokeAll(getThunder(flag), 30, TimeUnit.SECONDS)) flag.getEpisodes().addAll(future.get()); - executor.shutdownNow(); - } - } - - private List getThunder(Flag flag) { - List items = new ArrayList<>(); - Iterator iterator = flag.getEpisodes().iterator(); - while (iterator.hasNext()) addThunder(iterator, items); - return items; - } - - private void addThunder(Iterator iterator, List items) { - String url = iterator.next().getUrl(); - if (!Sniffer.isThunder(url)) return; - items.add(Thunder.Parser.get(url)); - iterator.remove(); - } - private void post(Site site, Result result) { if (result.getList().isEmpty()) return; for (Vod vod : result.getList()) vod.setSite(site); diff --git a/app/src/main/java/com/fongmi/android/tv/player/Source.java b/app/src/main/java/com/fongmi/android/tv/player/Source.java index 86b9cce8b..eab75b1b9 100644 --- a/app/src/main/java/com/fongmi/android/tv/player/Source.java +++ b/app/src/main/java/com/fongmi/android/tv/player/Source.java @@ -1,6 +1,9 @@ package com.fongmi.android.tv.player; +import com.fongmi.android.tv.Constant; import com.fongmi.android.tv.bean.Channel; +import com.fongmi.android.tv.bean.Episode; +import com.fongmi.android.tv.bean.Flag; import com.fongmi.android.tv.bean.Result; import com.fongmi.android.tv.player.extractor.Force; import com.fongmi.android.tv.player.extractor.JianPian; @@ -12,7 +15,13 @@ import com.fongmi.android.tv.player.extractor.Youtube; import com.fongmi.android.tv.utils.UrlUtil; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; public class Source { @@ -44,6 +53,28 @@ public class Source { return null; } + private void addCallable(Iterator iterator, List>> items) { + String url = iterator.next().getUrl(); + if (Thunder.Parser.match(url)) { + items.add(Thunder.Parser.get(url)); + iterator.remove(); + } else if (Youtube.Parser.match(url)) { + items.add(Youtube.Parser.get(url)); + iterator.remove(); + } + } + + public void parse(List flags) throws Exception { + for (Flag flag : flags) { + ExecutorService executor = Executors.newFixedThreadPool(Constant.THREAD_POOL * 2); + List>> items = new ArrayList<>(); + Iterator iterator = flag.getEpisodes().iterator(); + while (iterator.hasNext()) addCallable(iterator, items); + for (Future> future : executor.invokeAll(items, 30, TimeUnit.SECONDS)) flag.getEpisodes().addAll(future.get()); + executor.shutdownNow(); + } + } + public String fetch(Result result) throws Exception { String url = result.getUrl().v(); Extractor extractor = getExtractor(url); diff --git a/app/src/main/java/com/fongmi/android/tv/player/extractor/Thunder.java b/app/src/main/java/com/fongmi/android/tv/player/extractor/Thunder.java index e7cf2e91d..28cdac666 100644 --- a/app/src/main/java/com/fongmi/android/tv/player/extractor/Thunder.java +++ b/app/src/main/java/com/fongmi/android/tv/player/extractor/Thunder.java @@ -7,7 +7,6 @@ import com.fongmi.android.tv.bean.Episode; import com.fongmi.android.tv.exception.ExtractException; import com.fongmi.android.tv.player.Source; import com.fongmi.android.tv.utils.Download; -import com.fongmi.android.tv.utils.Sniffer; import com.fongmi.android.tv.utils.UrlUtil; import com.github.catvod.utils.Path; import com.github.catvod.utils.Util; @@ -22,6 +21,7 @@ import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.concurrent.Callable; +import java.util.regex.Pattern; public class Thunder implements Source.Extractor { @@ -70,9 +70,14 @@ public class Thunder implements Source.Extractor { public static class Parser implements Callable> { + private static final Pattern THUNDER = Pattern.compile("(magnet|thunder|ed2k):.*"); private final String url; private int time; + public static boolean match(String url) { + return THUNDER.matcher(url).find() || isTorrent(url); + } + public static Parser get(String url) { return new Parser(url); } @@ -86,9 +91,13 @@ public class Thunder implements Source.Extractor { time += 10; } + private static boolean isTorrent(String url) { + return !url.startsWith("magnet") && url.split(";")[0].endsWith(".torrent"); + } + @Override public List call() { - boolean torrent = Sniffer.isTorrent(url); + boolean torrent = isTorrent(url); List episodes = new ArrayList<>(); GetTaskId taskId = XLTaskHelper.get().parse(url, Path.thunder(Util.md5(url))); if (!torrent && !taskId.getRealUrl().startsWith("magnet")) return Arrays.asList(Episode.create(taskId.getFileName(), taskId.getRealUrl())); diff --git a/app/src/main/java/com/fongmi/android/tv/player/extractor/Youtube.java b/app/src/main/java/com/fongmi/android/tv/player/extractor/Youtube.java index 216ccfe53..2df7a8671 100644 --- a/app/src/main/java/com/fongmi/android/tv/player/extractor/Youtube.java +++ b/app/src/main/java/com/fongmi/android/tv/player/extractor/Youtube.java @@ -1,18 +1,25 @@ package com.fongmi.android.tv.player.extractor; +import android.net.Uri; import android.util.Base64; +import com.fongmi.android.tv.bean.Episode; import com.fongmi.android.tv.player.Source; import com.github.catvod.net.OkHttp; import com.github.kiulian.downloader.YoutubeDownloader; +import com.github.kiulian.downloader.downloader.request.RequestPlaylistInfo; import com.github.kiulian.downloader.downloader.request.RequestVideoInfo; +import com.github.kiulian.downloader.model.playlist.PlaylistInfo; +import com.github.kiulian.downloader.model.playlist.PlaylistVideoDetails; import com.github.kiulian.downloader.model.videos.VideoInfo; import com.github.kiulian.downloader.model.videos.formats.AudioFormat; import com.github.kiulian.downloader.model.videos.formats.Format; import com.github.kiulian.downloader.model.videos.formats.VideoFormat; +import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.concurrent.Callable; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -21,9 +28,9 @@ public class Youtube implements Source.Extractor { private static final String MPD = "\n" + "\n" + "%s\n" + "%s\n" + "\n" + ""; private static final String ADAPT = "\n" + "\n" + "\n" + "%s\n" + "\n" + "\n" + "\n" + "\n" + ""; private static final Pattern PATTERN = Pattern.compile("(?<=watch\\?v=|youtu.be/|/shorts/|/live/)([\\w-]{11})"); - private YoutubeDownloader downloader; + private static YoutubeDownloader downloader; - private YoutubeDownloader getDownloader() { + private static YoutubeDownloader getDownloader() { return downloader = downloader == null ? new YoutubeDownloader(OkHttp.client()) : downloader; } @@ -82,4 +89,31 @@ public class Youtube implements Source.Extractor { @Override public void exit() { } + + public static class Parser implements Callable> { + + private final String url; + + public static boolean match(String url) { + return PATTERN.matcher(url).find() && url.contains("list="); + } + + public static Parser get(String url) { + return new Parser(url); + } + + public Parser(String url) { + this.url = url; + } + + @Override + public List call() { + List episodes = new ArrayList<>(); + String id = Uri.parse(url).getQueryParameter("list"); + RequestPlaylistInfo request = new RequestPlaylistInfo(id); + PlaylistInfo info = getDownloader().getPlaylistInfo(request).data(); + for (PlaylistVideoDetails video : info.videos()) episodes.add(Episode.create(video.title(), "https://www.youtube.com/watch?v=" + video.videoId())); + return episodes; + } + } } diff --git a/app/src/main/java/com/fongmi/android/tv/utils/Sniffer.java b/app/src/main/java/com/fongmi/android/tv/utils/Sniffer.java index dc97eaa9c..c61587971 100644 --- a/app/src/main/java/com/fongmi/android/tv/utils/Sniffer.java +++ b/app/src/main/java/com/fongmi/android/tv/utils/Sniffer.java @@ -18,15 +18,6 @@ public class Sniffer { public static final Pattern CLICKER = Pattern.compile("\\[a=cr:(\\{.*?\\})\\/](.*?)\\[\\/a]"); public static final Pattern AI_PUSH = Pattern.compile("(http|https|rtmp|rtsp|smb|ftp|thunder|magnet|ed2k|mitv|tvbox-xg|jianpian|video):[^\\s]+", Pattern.MULTILINE); public static final Pattern SNIFFER = Pattern.compile("http((?!http).){12,}?\\.(m3u8|mp4|mkv|flv|mp3|m4a|aac|mpd)\\?.*|http((?!http).){12,}\\.(m3u8|mp4|mkv|flv|mp3|m4a|aac|mpd)|http((?!http).)*?video/tos*|http((?!http).)*?obj/tos*"); - public static final Pattern THUNDER = Pattern.compile("(magnet|thunder|ed2k):.*"); - - public static boolean isThunder(String url) { - return THUNDER.matcher(url).find() || isTorrent(url); - } - - public static boolean isTorrent(String url) { - return !url.startsWith("magnet") && url.split(";")[0].endsWith(".torrent"); - } public static String getUrl(String text) { if (Json.valid(text)) return text; diff --git a/youtube/src/main/java/com/github/kiulian/downloader/YoutubeDownloader.java b/youtube/src/main/java/com/github/kiulian/downloader/YoutubeDownloader.java index 13a64926b..6848c76c1 100644 --- a/youtube/src/main/java/com/github/kiulian/downloader/YoutubeDownloader.java +++ b/youtube/src/main/java/com/github/kiulian/downloader/YoutubeDownloader.java @@ -3,9 +3,11 @@ package com.github.kiulian.downloader; import com.github.kiulian.downloader.cipher.CachedCipherFactory; import com.github.kiulian.downloader.downloader.Downloader; import com.github.kiulian.downloader.downloader.DownloaderImpl; +import com.github.kiulian.downloader.downloader.request.RequestPlaylistInfo; import com.github.kiulian.downloader.downloader.request.RequestVideoInfo; import com.github.kiulian.downloader.downloader.response.Response; import com.github.kiulian.downloader.extractor.ExtractorImpl; +import com.github.kiulian.downloader.model.playlist.PlaylistInfo; import com.github.kiulian.downloader.model.videos.VideoInfo; import com.github.kiulian.downloader.parser.Parser; import com.github.kiulian.downloader.parser.ParserImpl; @@ -28,4 +30,8 @@ public class YoutubeDownloader { public Response getVideoInfo(RequestVideoInfo request) { return parser.parseVideo(request); } + + public Response getPlaylistInfo(RequestPlaylistInfo request) { + return parser.parsePlaylist(request); + } } diff --git a/youtube/src/main/java/com/github/kiulian/downloader/downloader/request/RequestPlaylistInfo.java b/youtube/src/main/java/com/github/kiulian/downloader/downloader/request/RequestPlaylistInfo.java new file mode 100644 index 000000000..051ff943a --- /dev/null +++ b/youtube/src/main/java/com/github/kiulian/downloader/downloader/request/RequestPlaylistInfo.java @@ -0,0 +1,16 @@ +package com.github.kiulian.downloader.downloader.request; + +import com.github.kiulian.downloader.model.playlist.PlaylistInfo; + +public class RequestPlaylistInfo extends Request { + + private final String playlistId; + + public RequestPlaylistInfo(String playlistId) { + this.playlistId = playlistId; + } + + public String getPlaylistId() { + return playlistId; + } +} \ No newline at end of file diff --git a/youtube/src/main/java/com/github/kiulian/downloader/extractor/Extractor.java b/youtube/src/main/java/com/github/kiulian/downloader/extractor/Extractor.java index df21c1ac6..6a06c82d2 100644 --- a/youtube/src/main/java/com/github/kiulian/downloader/extractor/Extractor.java +++ b/youtube/src/main/java/com/github/kiulian/downloader/extractor/Extractor.java @@ -5,9 +5,13 @@ import com.google.gson.JsonObject; public interface Extractor { + JsonObject extractInitialDataFromHtml(String html) throws YoutubeException; + JsonObject extractPlayerConfigFromHtml(String html) throws YoutubeException; String extractJsUrlFromConfig(JsonObject config, String videoId) throws YoutubeException; String extractClientVersionFromContext(JsonObject context); + + int extractIntegerFromText(String text); } diff --git a/youtube/src/main/java/com/github/kiulian/downloader/extractor/ExtractorImpl.java b/youtube/src/main/java/com/github/kiulian/downloader/extractor/ExtractorImpl.java index b29e929b7..2339ca67d 100644 --- a/youtube/src/main/java/com/github/kiulian/downloader/extractor/ExtractorImpl.java +++ b/youtube/src/main/java/com/github/kiulian/downloader/extractor/ExtractorImpl.java @@ -23,6 +23,12 @@ public class ExtractorImpl implements Extractor { Pattern.compile("ytInitialPlayerResponse\\s*=\\s*(\\{.+?\\})\\s*\\;") ); + private static final List YT_INITIAL_DATA_PATTERNS = Arrays.asList( + Pattern.compile("window\\[\"ytInitialData\"\\] = (\\{.*?\\});"), + Pattern.compile("ytInitialData = (\\{.*?\\});") + ); + + private static final Pattern TEXT_NUMBER_REGEX = Pattern.compile("[0-9]+[0-9, ']*"); private static final Pattern ASSETS_JS_REGEX = Pattern.compile("\"assets\":.+?\"js\":\\s*\"([^\"]+)\""); private static final Pattern EMB_JS_REGEX = Pattern.compile("\"jsUrl\":\\s*\"([^\"]+)\""); @@ -32,6 +38,25 @@ public class ExtractorImpl implements Extractor { this.downloader = downloader; } + @Override + public JsonObject extractInitialDataFromHtml(String html) throws YoutubeException { + String ytInitialData = null; + for (Pattern pattern : YT_INITIAL_DATA_PATTERNS) { + Matcher matcher = pattern.matcher(html); + if (matcher.find()) { + ytInitialData = matcher.group(1); + } + } + if (ytInitialData == null) { + throw new YoutubeException.BadPageException("Could not find initial data on web page"); + } + try { + return JsonParser.parseString(ytInitialData).getAsJsonObject(); + } catch (Exception e) { + throw new YoutubeException.BadPageException("Initial data contains invalid json"); + } + } + @Override public JsonObject extractPlayerConfigFromHtml(String html) throws YoutubeException { String ytPlayerConfig = null; @@ -102,4 +127,11 @@ public class ExtractorImpl implements Extractor { } return DEFAULT_CLIENT_VERSION; } + + @Override + public int extractIntegerFromText(String text) { + Matcher matcher = TEXT_NUMBER_REGEX.matcher(text); + if (matcher.find()) return Integer.parseInt(matcher.group(0).replaceAll("[, ']", "")); + return 0; + } } diff --git a/youtube/src/main/java/com/github/kiulian/downloader/model/AbstractListVideoDetails.java b/youtube/src/main/java/com/github/kiulian/downloader/model/AbstractListVideoDetails.java new file mode 100644 index 000000000..6396c2299 --- /dev/null +++ b/youtube/src/main/java/com/github/kiulian/downloader/model/AbstractListVideoDetails.java @@ -0,0 +1,17 @@ +package com.github.kiulian.downloader.model; + +import com.google.gson.JsonObject; + +public class AbstractListVideoDetails extends AbstractVideoDetails { + + public AbstractListVideoDetails(JsonObject json) { + super(json); + author = Utils.parseRuns(json.getAsJsonObject("shortBylineText")); + JsonObject jsonTitle = json.getAsJsonObject("title"); + if (jsonTitle.has("simpleText")) { + title = jsonTitle.get("simpleText").getAsString(); + } else { + title = Utils.parseRuns(jsonTitle); + } + } +} \ No newline at end of file diff --git a/youtube/src/main/java/com/github/kiulian/downloader/model/AbstractVideoDetails.java b/youtube/src/main/java/com/github/kiulian/downloader/model/AbstractVideoDetails.java index 082c8d90e..33c243fa7 100644 --- a/youtube/src/main/java/com/github/kiulian/downloader/model/AbstractVideoDetails.java +++ b/youtube/src/main/java/com/github/kiulian/downloader/model/AbstractVideoDetails.java @@ -28,6 +28,10 @@ public abstract class AbstractVideoDetails { if (json.has("lengthSeconds")) lengthSeconds = json.get("lengthSeconds").getAsInt(); } + public String videoId() { + return videoId; + } + public String title() { return title; } diff --git a/youtube/src/main/java/com/github/kiulian/downloader/model/Utils.java b/youtube/src/main/java/com/github/kiulian/downloader/model/Utils.java index 3c9a53230..7919d1319 100644 --- a/youtube/src/main/java/com/github/kiulian/downloader/model/Utils.java +++ b/youtube/src/main/java/com/github/kiulian/downloader/model/Utils.java @@ -19,6 +19,24 @@ public class Utils { } } + public static String parseRuns(JsonObject container) { + if (container == null) { + return null; + } + JsonArray runs = container.getAsJsonArray("runs"); + if (runs == null) { + return null; + } else if (runs.size() == 1) { + return runs.get(0).getAsJsonObject().get("text").getAsString(); + } else { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < runs.size(); i++) { + builder.append(runs.get(i).getAsJsonObject().get("text").getAsString()); + } + return builder.toString(); + } + } + public static List parseThumbnails(JsonObject container) { if (container == null) { return null; diff --git a/youtube/src/main/java/com/github/kiulian/downloader/model/playlist/PlaylistDetails.java b/youtube/src/main/java/com/github/kiulian/downloader/model/playlist/PlaylistDetails.java new file mode 100644 index 000000000..f13d52121 --- /dev/null +++ b/youtube/src/main/java/com/github/kiulian/downloader/model/playlist/PlaylistDetails.java @@ -0,0 +1,20 @@ +package com.github.kiulian.downloader.model.playlist; + +public class PlaylistDetails { + + private String playlistId; + private String title; + + public PlaylistDetails(String playlistId, String title) { + this.playlistId = playlistId; + this.title = title; + } + + public String playlistId() { + return playlistId; + } + + public String title() { + return title; + } +} \ No newline at end of file diff --git a/youtube/src/main/java/com/github/kiulian/downloader/model/playlist/PlaylistInfo.java b/youtube/src/main/java/com/github/kiulian/downloader/model/playlist/PlaylistInfo.java new file mode 100644 index 000000000..bd9b6f423 --- /dev/null +++ b/youtube/src/main/java/com/github/kiulian/downloader/model/playlist/PlaylistInfo.java @@ -0,0 +1,35 @@ +package com.github.kiulian.downloader.model.playlist; + +import com.github.kiulian.downloader.model.Filter; + +import java.util.List; + +public class PlaylistInfo { + + private PlaylistDetails details; + private List videos; + + public PlaylistInfo(PlaylistDetails details, List videos) { + this.details = details; + this.videos = videos; + } + + public PlaylistDetails details() { + return details; + } + + public List videos() { + return videos; + } + + public PlaylistVideoDetails findVideoById(String videoId) { + for (PlaylistVideoDetails video : videos) { + if (video.videoId().equals(videoId)) return video; + } + return null; + } + + public List findVideos(Filter filter) { + return filter.select(videos); + } +} \ No newline at end of file diff --git a/youtube/src/main/java/com/github/kiulian/downloader/model/playlist/PlaylistVideoDetails.java b/youtube/src/main/java/com/github/kiulian/downloader/model/playlist/PlaylistVideoDetails.java new file mode 100644 index 000000000..db8bd3d92 --- /dev/null +++ b/youtube/src/main/java/com/github/kiulian/downloader/model/playlist/PlaylistVideoDetails.java @@ -0,0 +1,35 @@ +package com.github.kiulian.downloader.model.playlist; + +import com.github.kiulian.downloader.model.AbstractListVideoDetails; +import com.google.gson.JsonObject; + +public class PlaylistVideoDetails extends AbstractListVideoDetails { + + private int index; + private boolean isPlayable; + + public PlaylistVideoDetails(JsonObject json) { + super(json); + if (!thumbnails().isEmpty()) { + // Otherwise, contains "/hqdefault.jpg?" + isLive = thumbnails().get(0).contains("/hqdefault_live.jpg?"); + } + if (json.has("index")) { + index = json.getAsJsonObject("index").get("simpleText").getAsInt(); + } + isPlayable = json.get("isPlayable").getAsBoolean(); + } + + @Override + protected boolean isDownloadable() { + return isPlayable && super.isDownloadable(); + } + + public int index() { + return index; + } + + public boolean isPlayable() { + return isPlayable; + } +} \ No newline at end of file diff --git a/youtube/src/main/java/com/github/kiulian/downloader/parser/Parser.java b/youtube/src/main/java/com/github/kiulian/downloader/parser/Parser.java index 7822d0f49..729b240b8 100644 --- a/youtube/src/main/java/com/github/kiulian/downloader/parser/Parser.java +++ b/youtube/src/main/java/com/github/kiulian/downloader/parser/Parser.java @@ -1,10 +1,14 @@ package com.github.kiulian.downloader.parser; +import com.github.kiulian.downloader.downloader.request.RequestPlaylistInfo; import com.github.kiulian.downloader.downloader.request.RequestVideoInfo; import com.github.kiulian.downloader.downloader.response.Response; +import com.github.kiulian.downloader.model.playlist.PlaylistInfo; import com.github.kiulian.downloader.model.videos.VideoInfo; public interface Parser { Response parseVideo(RequestVideoInfo request); + + Response parsePlaylist(RequestPlaylistInfo request); } diff --git a/youtube/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java b/youtube/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java index 68d9572b9..1f42452b4 100644 --- a/youtube/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java +++ b/youtube/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java @@ -6,11 +6,15 @@ import com.github.kiulian.downloader.cipher.Cipher; import com.github.kiulian.downloader.cipher.CipherFactory; import com.github.kiulian.downloader.downloader.Downloader; import com.github.kiulian.downloader.downloader.YoutubeCallback; +import com.github.kiulian.downloader.downloader.request.RequestPlaylistInfo; import com.github.kiulian.downloader.downloader.request.RequestVideoInfo; import com.github.kiulian.downloader.downloader.request.RequestWebpage; import com.github.kiulian.downloader.downloader.response.Response; import com.github.kiulian.downloader.downloader.response.ResponseImpl; import com.github.kiulian.downloader.extractor.Extractor; +import com.github.kiulian.downloader.model.playlist.PlaylistDetails; +import com.github.kiulian.downloader.model.playlist.PlaylistInfo; +import com.github.kiulian.downloader.model.playlist.PlaylistVideoDetails; import com.github.kiulian.downloader.model.subtitles.SubtitlesInfo; import com.github.kiulian.downloader.model.videos.VideoDetails; import com.github.kiulian.downloader.model.videos.VideoInfo; @@ -27,6 +31,7 @@ import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedList; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -258,7 +263,6 @@ public class ParserImpl implements Parser { throw new YoutubeException.BadPageException("deciphering is required but no js url"); } } - boolean hasVideo = itag.isVideo() || json.has("size") || json.has("width"); boolean hasAudio = itag.isAudio() || json.has("audioQuality"); if (hasVideo && hasAudio) return new VideoWithAudioFormat(json, isAdaptive, clientVersion); @@ -292,4 +296,146 @@ public class ParserImpl implements Parser { } return subtitlesInfo; } + + @Override + public Response parsePlaylist(RequestPlaylistInfo request) { + if (request.isAsync()) { + ExecutorService executorService = config.getExecutorService(); + Future result = executorService.submit(() -> parsePlaylist(request.getPlaylistId(), request.getCallback())); + return ResponseImpl.fromFuture(result); + } + try { + PlaylistInfo result = parsePlaylist(request.getPlaylistId(), request.getCallback()); + return ResponseImpl.from(result); + } catch (YoutubeException e) { + return ResponseImpl.error(e); + } + } + + private PlaylistInfo parsePlaylist(String playlistId, YoutubeCallback callback) throws YoutubeException { + String htmlUrl = "https://www.youtube.com/playlist?list=" + playlistId; + Response response = downloader.downloadWebpage(new RequestWebpage(htmlUrl)); + if (!response.ok()) { + YoutubeException e = new YoutubeException.DownloadException(String.format("Could not load url: %s, exception: %s", htmlUrl, response.error().getMessage())); + if (callback != null) callback.onError(e); + throw e; + } + String html = response.data(); + JsonObject initialData; + try { + initialData = extractor.extractInitialDataFromHtml(html); + } catch (YoutubeException e) { + if (callback != null) callback.onError(e); + throw e; + } + if (!initialData.has("metadata")) { + throw new YoutubeException.BadPageException("Invalid initial data json"); + } + String title = initialData.getAsJsonObject("metadata").getAsJsonObject("playlistMetadataRenderer").get("title").getAsString(); + PlaylistDetails playlistDetails = new PlaylistDetails(playlistId, title); + List videos; + try { + videos = parsePlaylistVideos(initialData); + } catch (YoutubeException e) { + if (callback != null) callback.onError(e); + throw e; + } + return new PlaylistInfo(playlistDetails, videos); + } + + private List parsePlaylistVideos(JsonObject initialData) throws YoutubeException { + JsonObject content; + try { + content = initialData.getAsJsonObject("contents") + .getAsJsonObject("twoColumnBrowseResultsRenderer") + .getAsJsonArray("tabs").get(0).getAsJsonObject() + .getAsJsonObject("tabRenderer") + .getAsJsonObject("content") + .getAsJsonObject("sectionListRenderer") + .getAsJsonArray("contents").get(0).getAsJsonObject() + .getAsJsonObject("itemSectionRenderer") + .getAsJsonArray("contents").get(0).getAsJsonObject() + .getAsJsonObject("playlistVideoListRenderer"); + } catch (NullPointerException e) { + throw new YoutubeException.BadPageException("Playlist initial data not found"); + } + List videos = new LinkedList<>(); + JsonObject context = initialData.getAsJsonObject("responseContext"); + String clientVersion = extractor.extractClientVersionFromContext(context); + populatePlaylist(content, videos, clientVersion); + return videos; + } + + private void populatePlaylist(JsonObject content, List videos, String clientVersion) throws YoutubeException { + JsonArray contents; + if (content.has("contents")) { // parse first items (up to 100) + contents = content.getAsJsonArray("contents"); + } else if (content.has("continuationItems")) { // parse continuationItems + contents = content.getAsJsonArray("continuationItems"); + } else if (content.has("continuations")) { // load continuation + JsonObject nextContinuationData = content.getAsJsonArray("continuations").get(0).getAsJsonObject().getAsJsonObject("nextContinuationData"); + String continuation = nextContinuationData.get("continuation").getAsString(); + String ctp = nextContinuationData.get("clickTrackingParams").getAsString(); + loadPlaylistContinuation(continuation, ctp, videos, clientVersion); + return; + } else { // nothing found + return; + } + for (int i = 0; i < contents.size(); i++) { + JsonObject contentsItem = contents.get(i).getAsJsonObject(); + if (contentsItem.has("playlistVideoRenderer")) { + videos.add(new PlaylistVideoDetails(contentsItem.getAsJsonObject("playlistVideoRenderer"))); + } else { + if (contentsItem.has("continuationItemRenderer")) { + JsonObject continuationEndpoint = contentsItem.getAsJsonObject("continuationItemRenderer").getAsJsonObject("continuationEndpoint"); + String continuation = continuationEndpoint.getAsJsonObject("continuationCommand").get("token").getAsString(); + String ctp = continuationEndpoint.get("clickTrackingParams").getAsString(); + loadPlaylistContinuation(continuation, ctp, videos, clientVersion); + } + } + } + } + + private void loadPlaylistContinuation(String continuation, String ctp, List videos, String clientVersion) throws YoutubeException { + JsonObject client = new JsonObject(); + client.addProperty("clientName", "WEB"); + client.addProperty("clientVersion", "2.20201021.03.00"); + + JsonObject context = new JsonObject(); + context.add("client", client); + + JsonObject clickTracking = new JsonObject(); + clickTracking.addProperty("clickTrackingParams", ctp); + + JsonObject body = new JsonObject(); + body.add("context", context); + body.add("clickTracking", clickTracking); + body.addProperty("continuation", continuation); + + String url = "https://www.youtube.com/youtubei/v1/browse?key=" + ANDROID_APIKEY; + RequestWebpage request = new RequestWebpage(url, "POST", body.toString()) + .header("X-YouTube-Client-Name", "1") + .header("X-YouTube-Client-Version", clientVersion) + .header("Content-Type", "application/json"); + + Response response = downloader.downloadWebpage(request); + if (!response.ok()) { + throw new YoutubeException.DownloadException(String.format("Could not load url: %s, exception: %s", url, response.error().getMessage())); + } + String html = response.data(); + try { + JsonObject content; + JsonObject jsonResponse = JsonParser.parseString(html).getAsJsonObject(); + if (jsonResponse.has("continuationContents")) { + content = jsonResponse.getAsJsonObject("continuationContents").getAsJsonObject("playlistVideoListContinuation"); + } else { + content = jsonResponse.getAsJsonArray("onResponseReceivedActions").get(0).getAsJsonObject().getAsJsonObject("appendContinuationItemsAction"); + } + populatePlaylist(content, videos, clientVersion); + } catch (YoutubeException e) { + throw e; + } catch (Exception e) { + throw new YoutubeException.BadPageException("Could not parse playlist continuation json"); + } + } }