From 099b1ece4f0dbc1b0f4a1860b29b1368a2cf8ec7 Mon Sep 17 00:00:00 2001 From: FongMi Date: Sat, 7 Jan 2023 14:59:00 +0800 Subject: [PATCH] Support spider manualVideoCheck --- .../java/com/fongmi/android/tv/api/JsLoader.java | 2 +- .../java/com/fongmi/android/tv/bean/Result.java | 10 ++++++++++ .../fongmi/android/tv/model/SiteViewModel.java | 1 + .../com/fongmi/android/tv/player/ParseTask.java | 12 ++++++++---- .../android/tv/ui/custom/CustomWebView.java | 15 +++++++++++++-- .../java/com/fongmi/android/tv/utils/Sniffer.java | 3 --- .../java/com/fongmi/android/tv/utils/Utils.java | 1 - .../java/com/github/catvod/crawler/Spider.java | 8 ++++++++ drpy/src/main/java/com/hiker/drpy/Spider.java | 9 ++++++--- 9 files changed, 47 insertions(+), 14 deletions(-) diff --git a/app/src/main/java/com/fongmi/android/tv/api/JsLoader.java b/app/src/main/java/com/fongmi/android/tv/api/JsLoader.java index 64c596f7b..52b9e0be1 100644 --- a/app/src/main/java/com/fongmi/android/tv/api/JsLoader.java +++ b/app/src/main/java/com/fongmi/android/tv/api/JsLoader.java @@ -37,7 +37,7 @@ public class JsLoader { return spider; } catch (Throwable e) { e.printStackTrace(); - return null; + return new Spider(); } } } diff --git a/app/src/main/java/com/fongmi/android/tv/bean/Result.java b/app/src/main/java/com/fongmi/android/tv/bean/Result.java index 737b91135..e480f3160 100644 --- a/app/src/main/java/com/fongmi/android/tv/bean/Result.java +++ b/app/src/main/java/com/fongmi/android/tv/bean/Result.java @@ -51,6 +51,8 @@ public class Result { private String flag; @SerializedName("url") private String url; + @SerializedName("key") + private String key; @SerializedName("subs") private List subs; @@ -165,6 +167,14 @@ public class Result { this.url = url; } + public String getKey() { + return TextUtils.isEmpty(key) ? "" : key; + } + + public void setKey(String key) { + this.key = key; + } + public List getSubs() { return subs == null ? Collections.emptyList() : subs; } diff --git a/app/src/main/java/com/fongmi/android/tv/model/SiteViewModel.java b/app/src/main/java/com/fongmi/android/tv/model/SiteViewModel.java index 896885ed7..9fd15e065 100644 --- a/app/src/main/java/com/fongmi/android/tv/model/SiteViewModel.java +++ b/app/src/main/java/com/fongmi/android/tv/model/SiteViewModel.java @@ -138,6 +138,7 @@ public class SiteViewModel extends ViewModel { SpiderDebug.log(playerContent); Result result = Result.objectFrom(playerContent); if (result.getFlag().isEmpty()) result.setFlag(flag); + result.setKey(key); return result; } else if (site.getType() == 4) { ArrayMap params = new ArrayMap<>(); diff --git a/app/src/main/java/com/fongmi/android/tv/player/ParseTask.java b/app/src/main/java/com/fongmi/android/tv/player/ParseTask.java index 28cccf7cc..44b0dc142 100644 --- a/app/src/main/java/com/fongmi/android/tv/player/ParseTask.java +++ b/app/src/main/java/com/fongmi/android/tv/player/ParseTask.java @@ -39,7 +39,7 @@ public class ParseTask { public ParseTask run(Result result, boolean useParse) { setParse(result, useParse); - executor.execute(() -> doInBackground(result.getUrl(), result.getFlag())); + executor.execute(() -> doInBackground(result.getKey(), result.getUrl(), result.getFlag())); return this; } @@ -50,14 +50,14 @@ public class ParseTask { if (parse == null) parse = Parse.get(0, result.getPlayUrl(), result.getHeader()); } - private void doInBackground(String webUrl, String flag) { + private void doInBackground(String key, String webUrl, String flag) { if (webUrl.startsWith("magnet:")) { onParseError(); return; } switch (parse.getType()) { case 0: //嗅探 - App.post(() -> startWeb(parse.getUrl() + webUrl, parse.getHeaders(), callback)); + App.post(() -> startWeb(key, parse.getUrl() + webUrl, parse.getHeaders(), callback)); break; case 1: //Json jsonParse(webUrl); @@ -108,7 +108,11 @@ public class ParseTask { } private void startWeb(String url, Map headers, Callback callback) { - if (webView != null) webView.start(url, headers, callback); + startWeb("", url, headers, callback); + } + + private void startWeb(String key, String url, Map headers, Callback callback) { + if (webView != null) webView.start(key, url, headers, callback); } private void onParseSuccess(Map headers, String url, String from) { diff --git a/app/src/main/java/com/fongmi/android/tv/ui/custom/CustomWebView.java b/app/src/main/java/com/fongmi/android/tv/ui/custom/CustomWebView.java index 33fb7c193..18111a069 100644 --- a/app/src/main/java/com/fongmi/android/tv/ui/custom/CustomWebView.java +++ b/app/src/main/java/com/fongmi/android/tv/ui/custom/CustomWebView.java @@ -17,8 +17,10 @@ import androidx.annotation.Nullable; import com.fongmi.android.tv.App; import com.fongmi.android.tv.api.ApiConfig; +import com.fongmi.android.tv.bean.Site; import com.fongmi.android.tv.player.ParseTask; import com.fongmi.android.tv.utils.Utils; +import com.github.catvod.crawler.Spider; import java.io.ByteArrayInputStream; import java.util.Arrays; @@ -31,6 +33,7 @@ public class CustomWebView extends WebView { private ParseTask.Callback callback; private WebResourceResponse empty; private List keys; + private String key; private String ads; private int retry; @@ -63,10 +66,11 @@ public class CustomWebView extends WebView { } } - public void start(String url, Map headers, ParseTask.Callback callback) { + public void start(String key, String url, Map headers, ParseTask.Callback callback) { this.callback = callback; setUserAgent(headers); loadUrl(url, headers); + this.key = key; retry = 0; } @@ -80,7 +84,7 @@ public class CustomWebView extends WebView { if (ads.contains(host)) return empty; App.post(mTimer, 15 * 1000); Map headers = request.getRequestHeaders(); - if (Utils.isVideoFormat(url, headers)) post(headers, url); + if (isVideoFormat(url, headers)) post(headers, url); return super.shouldInterceptRequest(view, request); } @@ -96,6 +100,13 @@ public class CustomWebView extends WebView { }; } + private boolean isVideoFormat(String url, Map headers) { + Site site = ApiConfig.get().getSite(key); + Spider spider = ApiConfig.get().getCSP(site); + if (spider.manualVideoCheck()) return spider.isVideoFormat(url); + return Utils.isVideoFormat(url, headers); + } + private final Runnable mTimer = new Runnable() { @Override public void run() { diff --git a/app/src/main/java/com/fongmi/android/tv/utils/Sniffer.java b/app/src/main/java/com/fongmi/android/tv/utils/Sniffer.java index ac9645a3d..5f4d606fe 100644 --- a/app/src/main/java/com/fongmi/android/tv/utils/Sniffer.java +++ b/app/src/main/java/com/fongmi/android/tv/utils/Sniffer.java @@ -4,8 +4,6 @@ import java.util.regex.Pattern; public class Sniffer { - public static final Pattern BLOCK = Pattern.compile("http((?!http).)*renren.pro(.*).demo.m3u8"); - public static final Pattern RULE = Pattern.compile( "http((?!http).){12,}?\\.(m3u8|mp4|flv|avi|mkv|rm|wmv|mpg|m4a|mp3)\\?.*|" + "http((?!http).){12,}\\.(m3u8|mp4|flv|avi|mkv|rm|wmv|mpg|m4a|mp3)|" + @@ -15,7 +13,6 @@ public class Sniffer { public static final Pattern CUSTOM = Pattern.compile( "http((?!http).)*?xg.php\\?id=|" + "http((?!http).)*?/m3/(.*)\\.css|" + - "http((?!http).)*renren.pro(.*).m3u8|" + "http((?!http).)*?_playback/\\?video_id=|" + "http((?!http).)*huoshan.com(.*)?item_id=|" + "http((?!http).)*qianpailive.com(.*)&vid=|" + diff --git a/app/src/main/java/com/fongmi/android/tv/utils/Utils.java b/app/src/main/java/com/fongmi/android/tv/utils/Utils.java index 6ee138cda..5ad650318 100644 --- a/app/src/main/java/com/fongmi/android/tv/utils/Utils.java +++ b/app/src/main/java/com/fongmi/android/tv/utils/Utils.java @@ -72,7 +72,6 @@ public class Utils { } public static boolean isVideoFormat(String url, Map headers) { - if (Sniffer.BLOCK.matcher(url).find()) return false; if (Sniffer.CUSTOM.matcher(url).find()) return true; if (headers.containsKey("Accept") && headers.get("Accept").startsWith("image")) return false; if (url.contains("url=http") || url.contains(".js") || url.contains(".css") || url.contains(".html")) return false; diff --git a/catvod/src/main/java/com/github/catvod/crawler/Spider.java b/catvod/src/main/java/com/github/catvod/crawler/Spider.java index 48bc5e113..9f6b1024d 100644 --- a/catvod/src/main/java/com/github/catvod/crawler/Spider.java +++ b/catvod/src/main/java/com/github/catvod/crawler/Spider.java @@ -37,4 +37,12 @@ public abstract class Spider { public String playerContent(String flag, String id, List vipFlags) throws Exception { return ""; } + + public boolean isVideoFormat(String url) { + return false; + } + + public boolean manualVideoCheck() { + return false; + } } diff --git a/drpy/src/main/java/com/hiker/drpy/Spider.java b/drpy/src/main/java/com/hiker/drpy/Spider.java index 047fa1c99..d1026aaf3 100644 --- a/drpy/src/main/java/com/hiker/drpy/Spider.java +++ b/drpy/src/main/java/com/hiker/drpy/Spider.java @@ -20,12 +20,15 @@ import java.util.concurrent.Future; public class Spider extends com.github.catvod.crawler.Spider { - private final ExecutorService executor; - private final String key; - private final String api; + private ExecutorService executor; + private String key; + private String api; private QuickJSContext ctx; private JSObject jsObject; + public Spider() { + } + public Spider(String api) { this.executor = Executors.newSingleThreadExecutor(); this.key = "__" + UUID.randomUUID().toString().replace("-", "") + "__";