diff --git a/app/src/main/java/com/github/tvbox/osc/util/parser/JsonParallel.java b/app/src/main/java/com/github/tvbox/osc/util/parser/JsonParallel.java new file mode 100644 index 00000000..f5b2ddb5 --- /dev/null +++ b/app/src/main/java/com/github/tvbox/osc/util/parser/JsonParallel.java @@ -0,0 +1,129 @@ +package com.github.tvbox.osc.util.parser; +import android.util.Base64; +import com.github.catvod.crawler.SpiderDebug; +import org.json.JSONObject; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import okhttp3.Call; +import okhttp3.Headers; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; + +/** + * 并发解析，直到获得第一个结果 + */ +public class JsonParallel { + + public static JSONObject parse(LinkedHashMap jx, String url) { + try { + if (jx != null && jx.size() > 0) { + OkHttpClient client = new OkHttpClient(); + // 使用线程池并发处理各个任务 + ExecutorService executorService = Executors.newFixedThreadPool(6); + CompletionService completionService = new ExecutorCompletionService<>(executorService); + List> futures = new ArrayList<>(); + + // 遍历所有的解析配置 + for (final String jxName : jx.keySet()) { + final String parseUrl = jx.get(jxName); + futures.add(completionService.submit(new Callable() { + @Override + public JSONObject call() { + try { + // 获取请求头，并从中取出实际url + HashMap reqHeaders = JsonParallel.getReqHeader(parseUrl); + String realUrl = reqHeaders.get("url"); + reqHeaders.remove("url"); + SpiderDebug.log(realUrl + url); + Headers headers = Headers.of(reqHeaders); + Request request = new Request.Builder() + .url(realUrl + url) + .headers(headers) + .tag("ParseTag") + .build(); + + Call call = client.newCall(request); + Response response = call.execute(); + String json = response.body().string(); + + JSONObject taskResult = Utils.jsonParse(url, json); + taskResult.put("jxFrom", jxName); + SpiderDebug.log(taskResult.toString()); + return taskResult; + } catch (Throwable th) { + SpiderDebug.log(th); + return null; + } + } + })); + } + + JSONObject pTaskResult = null; + for (int i = 0; i < futures.size(); ++i) { + Future completed = completionService.take(); + try { + pTaskResult = completed.get(); + if (pTaskResult != null) { + client.dispatcher().cancelAll(); + for (Future future : futures) { + try { + future.cancel(true); + } catch (Throwable t) { + SpiderDebug.log(t); + } + } + futures.clear(); + break; + } + } catch (Throwable th) { + SpiderDebug.log(th); + } + } + executorService.shutdownNow(); + if (pTaskResult != null) + return pTaskResult; + } + } catch (Throwable th) { + SpiderDebug.log(th); + } + return new JSONObject(); + } + + public static HashMap getReqHeader(String url) { + HashMap reqHeaders = new HashMap<>(); + reqHeaders.put("url", url); + if (url.contains("cat_ext")) { + try { + int start = url.indexOf("cat_ext="); + int end = url.indexOf("&", start); + String ext = url.substring(start + 8, end); + ext = new String(Base64.decode(ext, Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP)); + String newUrl = url.substring(0, start) + url.substring(end + 1); + JSONObject jsonObject = new JSONObject(ext); + if (jsonObject.has("header")) { + JSONObject headerJson = jsonObject.optJSONObject("header"); + Iterator keys = headerJson.keys(); + while (keys.hasNext()) { + String key = keys.next(); + reqHeaders.put(key, headerJson.optString(key, "")); + } + } + reqHeaders.put("url", newUrl); + } catch (Throwable th) { + + } + } + return reqHeaders; + } +} diff --git a/app/src/main/java/com/github/tvbox/osc/util/parser/SuperParse.java b/app/src/main/java/com/github/tvbox/osc/util/parser/SuperParse.java new file mode 100644 index 00000000..94690811 --- /dev/null +++ b/app/src/main/java/com/github/tvbox/osc/util/parser/SuperParse.java @@ -0,0 +1,236 @@ +package com.github.tvbox.osc.util.parser; + +import android.util.Base64; + +import com.github.catvod.crawler.SpiderDebug; +import org.json.JSONArray; +import org.json.JSONObject; + +import java.io.ByteArrayInputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +public class SuperParse { + public static HashMap> flagWebJx = new HashMap<>(); + static HashMap> configs = null; + + public static JSONObject parse(LinkedHashMap> jx, String flag, String url) { + try { + // 初始化全局配置（configs）一次 + if (configs == null) { + configs = new HashMap<>(); + for (Map.Entry> entry : jx.entrySet()) { + String key = entry.getKey(); + HashMap parseBean = entry.getValue(); + if (parseBean == null) { + continue; + } + String type = parseBean.get("type"); + if (type == null) { + continue; + } + if ("1".equals(type) || "0".equals(type)) { + try { + String ext = parseBean.get("ext"); + if (ext == null) { + continue; + } + JSONArray flagsArray = new JSONObject(ext).getJSONArray("flag"); + for (int j = 0; j < flagsArray.length(); j++) { + String flagKey = flagsArray.getString(j); + ArrayList flagJx = configs.get(flagKey); + if (flagJx == null) { + flagJx = new ArrayList<>(); + configs.put(flagKey, flagJx); + } + flagJx.add(key); + } + } catch (Exception e) { + SpiderDebug.log(e); + } + } + } + } + + // 根据配置构建 jsonJx 和 webJx + LinkedHashMap jsonJx = new LinkedHashMap<>(); + ArrayList webJx = new ArrayList<>(); + List targetKeys = configs.get(flag); + + if (targetKeys != null && !targetKeys.isEmpty()) { + for (String key : targetKeys) { + HashMap parseBean = jx.get(key); + if (parseBean == null) { + continue; + } + String type = parseBean.get("type"); + if (type == null) { + continue; + } + if ("1".equals(type)) { + String urlValue = parseBean.get("url"); + String ext = parseBean.get("ext"); + if (urlValue != null && ext != null) { + jsonJx.put(key, mixUrl(urlValue, ext)); + } + } else if ("0".equals(type)) { + String urlValue = parseBean.get("url"); + if (urlValue != null) { + webJx.add(urlValue); + } + } + } + } else { + for (Map.Entry> entry : jx.entrySet()) { + String key = entry.getKey(); + HashMap parseBean = entry.getValue(); + if (parseBean == null) { + continue; + } + String type = parseBean.get("type"); + if (type == null) { + continue; + } + if ("1".equals(type)) { + String urlValue = parseBean.get("url"); + String ext = parseBean.get("ext"); + if (urlValue != null && ext != null) { + jsonJx.put(key, mixUrl(urlValue, ext)); + } + } else if ("0".equals(type)) { + String urlValue = parseBean.get("url"); + if (urlValue != null) { + webJx.add(urlValue); + } + } + } + } + // 缓存 webview 解析的地址 + if (!webJx.isEmpty()) { + flagWebJx.put(flag, webJx); + } + ExecutorService exec = Executors.newFixedThreadPool(2); + CompletionService cs = new ExecutorCompletionService<>(exec); + List> tasks = new ArrayList<>(); + + tasks.add(cs.submit(new Callable() { + @Override + public JSONObject call() { + return JsonParallel.parse(jsonJx, url); + } + })); + if (!webJx.isEmpty()) { + tasks.add(cs.submit(new Callable() { + @Override + public JSONObject call() { + JSONObject webResult = new JSONObject(); + String encodedUrl = Base64.encodeToString(url.getBytes(), + Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP); + try { + webResult.put("url", "proxy://go=MixWeb&flag=" + flag + "&url=" + encodedUrl); + webResult.put("parse", 1); + webResult.put("ua", Utils.UaWinChrome); + } catch (Exception e) { + SpiderDebug.log(e); + } + return webResult; + } + })); + } + JSONObject result = null; + for (int i = 0, n = tasks.size(); i < n; i++) { + try { + Future future = cs.take(); + JSONObject res = future.get(); + if (res != null && res.has("url")) { + result = res; + break; + } + } catch (Exception e) { + SpiderDebug.log(e); + } + } + for (Future future : tasks) { + future.cancel(true); + } + exec.shutdownNow(); + if (result != null) { + return result; + } + } catch (Exception e) { + SpiderDebug.log(e); + } + return new JSONObject(); + } + + private static String mixUrl(String url, String ext) { + if (ext.trim().length() > 0) { + int idx = url.indexOf("?"); + if (idx > 0) { + return url.substring(0, idx + 1) + "cat_ext=" + Base64.encodeToString(ext.getBytes(), Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP) + "&" + url.substring(idx + 1); + } + } + return url; + } + + public static Object[] loadHtml(String flag, String url) { + try { + url = new String(Base64.decode(url, Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP), "UTF-8"); + String html = "\n" + + "\n" + + "\n" + + "\n" + + "解析\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + ""; + + StringBuilder jxs = new StringBuilder(); + if (flagWebJx.containsKey(flag)) { + ArrayList jxUrls = flagWebJx.get(flag); + for (int i = 0; i < jxUrls.size(); i++) { + jxs.append("\""); + jxs.append(jxUrls.get(i)); + jxs.append("\""); + if (i < jxUrls.size() - 1) { + jxs.append(","); + } + } + } + html = html.replace("#url#", url).replace("#jxs#", jxs.toString()); + Object[] result = new Object[3]; + result[0] = 200; + result[1] = "text/html; charset=\"UTF-8\""; + ByteArrayInputStream baos = new ByteArrayInputStream(html.toString().getBytes("UTF-8")); + result[2] = baos; + return result; + } catch (Throwable th) { + th.printStackTrace(); + } + return null; + } +} diff --git a/app/src/main/java/com/github/tvbox/osc/util/parser/Utils.java b/app/src/main/java/com/github/tvbox/osc/util/parser/Utils.java new file mode 100644 index 00000000..c881a6c2 --- /dev/null +++ b/app/src/main/java/com/github/tvbox/osc/util/parser/Utils.java @@ -0,0 +1,115 @@ +package com.github.tvbox.osc.util.parser; + +import android.annotation.TargetApi; +import android.os.Build; +import android.webkit.ValueCallback; +import android.webkit.WebView; + +import org.json.JSONException; +import org.json.JSONObject; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Pattern; + +@TargetApi(Build.VERSION_CODES.KITKAT) +public class Utils { + + public static final Pattern RULE = Pattern.compile("http((?!http).){12,}?\\.(m3u8|mp4|flv|avi|mkv|rm|wmv|mpg|m4a|mp3)\\?.*|http((?!http).){12,}\\.(m3u8|mp4|flv|avi|mkv|rm|wmv|mpg|m4a|mp3)|http((?!http).)*?video/tos*"); + public static final String UaWinChrome = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.54 Safari/537.36"; + public static final String UaMobile = "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Mobile/15E148 Safari/604.1"; + + public static boolean isVip(String url) { + List hosts = Arrays.asList("iqiyi.com", "v.qq.com", "youku.com", "le.com", "tudou.com", "mgtv.com", "sohu.com", "acfun.cn", "bilibili.com", "baofeng.com", "pptv.com"); + for (String host : hosts) if (url.contains(host)) return true; + return false; + } + + public static boolean isVideoFormat(String url) { + if (url.contains("url=http") || url.contains(".js") || url.contains(".css") || url.contains(".html")) return false; + return RULE.matcher(url).find(); + } + + public static String substring(String text) { + return substring(text, 1); + } + + public static String substring(String text, int num) { + if (text != null && text.length() > num) { + return text.substring(0, text.length() - num); + } else { + return text; + } + } + + public static void loadUrl(WebView webView, String script) { + loadUrl(webView, script, null); + } + + public static void loadUrl(WebView webView, String script, ValueCallback callback) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) webView.evaluateJavascript(script, callback); + else webView.loadUrl(script); + } + + + + public static boolean isBlackVodUrl(String input, String url) { + if (url.contains("973973.xyz") || url.contains(".fit:")) + return true; + return false; + } + + public static JSONObject fixJsonVodHeader(JSONObject headers, String input, String url) throws JSONException { + if (headers == null) + headers = new JSONObject(); + if (input.contains("www.mgtv.com")) { + headers.put("Referer", " "); + headers.put("User-Agent", " Mozilla/5.0"); + } else if (url.contains("titan.mgtv")) { + headers.put("Referer", " "); + headers.put("User-Agent", " Mozilla/5.0"); + } else if (input.contains("bilibili")) { + headers.put("Referer", " https://www.bilibili.com/"); + headers.put("User-Agent", " " + UaWinChrome); + } + return headers; + } + + public static JSONObject jsonParse(String input, String json) throws JSONException { + JSONObject jsonPlayData = new JSONObject(json); + String url; + if (jsonPlayData.has("data")) { + url = jsonPlayData.getJSONObject("data").getString("url"); + } else { + url = jsonPlayData.getString("url"); + } + if (url.startsWith("//")) { + url = "https:" + url; + } + if (!url.startsWith("http")) { + return null; + } + if (url.equals(input)) { + if (isVip(url) || !isVideoFormat(url)) { + return null; + } + } + if (Utils.isBlackVodUrl(input, url)) { + return null; + } + JSONObject headers = new JSONObject(); + String ua = jsonPlayData.optString("user-agent", ""); + if (ua.trim().length() > 0) { + headers.put("User-Agent", " " + ua); + } + String referer = jsonPlayData.optString("referer", ""); + if (referer.trim().length() > 0) { + headers.put("Referer", " " + referer); + } + + headers = Utils.fixJsonVodHeader(headers, input, url); + JSONObject taskResult = new JSONObject(); + taskResult.put("header", headers); + taskResult.put("url", url); + return taskResult; + } +}