hg
/
JY-TVBoxOS
mirror of https://github.com/q215613905/TVBoxOS.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

内置超级解析;

Browse Source
pull/144/head
jun 1 year ago
parent a9b5b36f3e
commit 1c8c740f29
3 changed files with 480 additions and 0 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 129
      app/src/main/java/com/github/tvbox/osc/util/parser/JsonParallel.java
  2. 236
      app/src/main/java/com/github/tvbox/osc/util/parser/SuperParse.java
  3. 115
      app/src/main/java/com/github/tvbox/osc/util/parser/Utils.java

129
app/src/main/java/com/github/tvbox/osc/util/parser/JsonParallel.java
Unescape View File

@ -0,0 +1,129 @@
package com.github.tvbox.osc.util.parser;
import android.util.Base64;
import com.github.catvod.crawler.SpiderDebug;
import org.json.JSONObject;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import okhttp3.Call;
import okhttp3.Headers;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
/**
* 并发解析，直到获得第一个结果
*/
public class JsonParallel {
public static JSONObject parse(LinkedHashMap<String, String> jx, String url) {
try {
if (jx != null && jx.size() > 0) {
OkHttpClient client = new OkHttpClient();
// 使用线程池并发处理各个任务
ExecutorService executorService = Executors.newFixedThreadPool(6);
CompletionService<JSONObject> completionService = new ExecutorCompletionService<>(executorService);
List<Future<JSONObject>> futures = new ArrayList<>();
// 遍历所有的解析配置
for (final String jxName : jx.keySet()) {
final String parseUrl = jx.get(jxName);
futures.add(completionService.submit(new Callable<JSONObject>() {
@Override
public JSONObject call() {
try {
// 获取请求头，并从中取出实际url
HashMap<String, String> reqHeaders = JsonParallel.getReqHeader(parseUrl);
String realUrl = reqHeaders.get("url");
reqHeaders.remove("url");
SpiderDebug.log(realUrl + url);
Headers headers = Headers.of(reqHeaders);
Request request = new Request.Builder()
.url(realUrl + url)
.headers(headers)
.tag("ParseTag")
.build();
Call call = client.newCall(request);
Response response = call.execute();
String json = response.body().string();
JSONObject taskResult = Utils.jsonParse(url, json);
taskResult.put("jxFrom", jxName);
SpiderDebug.log(taskResult.toString());
return taskResult;
} catch (Throwable th) {
SpiderDebug.log(th);
return null;
}
}
}));
}
JSONObject pTaskResult = null;
for (int i = 0; i < futures.size(); ++i) {
Future<JSONObject> completed = completionService.take();
try {
pTaskResult = completed.get();
if (pTaskResult != null) {
client.dispatcher().cancelAll();
for (Future<JSONObject> future : futures) {
try {
future.cancel(true);
} catch (Throwable t) {
SpiderDebug.log(t);
}
}
futures.clear();
break;
}
} catch (Throwable th) {
SpiderDebug.log(th);
}
}
executorService.shutdownNow();
if (pTaskResult != null)
return pTaskResult;
}
} catch (Throwable th) {
SpiderDebug.log(th);
}
return new JSONObject();
}
public static HashMap<String, String> getReqHeader(String url) {
HashMap<String, String> reqHeaders = new HashMap<>();
reqHeaders.put("url", url);
if (url.contains("cat_ext")) {
try {
int start = url.indexOf("cat_ext=");
int end = url.indexOf("&", start);
String ext = url.substring(start + 8, end);
ext = new String(Base64.decode(ext, Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP));
String newUrl = url.substring(0, start) + url.substring(end + 1);
JSONObject jsonObject = new JSONObject(ext);
if (jsonObject.has("header")) {
JSONObject headerJson = jsonObject.optJSONObject("header");
Iterator<String> keys = headerJson.keys();
while (keys.hasNext()) {
String key = keys.next();
reqHeaders.put(key, headerJson.optString(key, ""));
}
}
reqHeaders.put("url", newUrl);
} catch (Throwable th) {
}
}
return reqHeaders;
}
}

236
app/src/main/java/com/github/tvbox/osc/util/parser/SuperParse.java
Unescape View File

@ -0,0 +1,236 @@
package com.github.tvbox.osc.util.parser;
import android.util.Base64;
import com.github.catvod.crawler.SpiderDebug;
import org.json.JSONArray;
import org.json.JSONObject;
import java.io.ByteArrayInputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
public class SuperParse {
public static HashMap<String, ArrayList<String>> flagWebJx = new HashMap<>();
static HashMap<String, ArrayList<String>> configs = null;
public static JSONObject parse(LinkedHashMap<String, HashMap<String, String>> jx, String flag, String url) {
try {
// 初始化全局配置（configs）一次
if (configs == null) {
configs = new HashMap<>();
for (Map.Entry<String, HashMap<String, String>> entry : jx.entrySet()) {
String key = entry.getKey();
HashMap<String, String> parseBean = entry.getValue();
if (parseBean == null) {
continue;
}
String type = parseBean.get("type");
if (type == null) {
continue;
}
if ("1".equals(type) || "0".equals(type)) {
try {
String ext = parseBean.get("ext");
if (ext == null) {
continue;
}
JSONArray flagsArray = new JSONObject(ext).getJSONArray("flag");
for (int j = 0; j < flagsArray.length(); j++) {
String flagKey = flagsArray.getString(j);
ArrayList<String> flagJx = configs.get(flagKey);
if (flagJx == null) {
flagJx = new ArrayList<>();
configs.put(flagKey, flagJx);
}
flagJx.add(key);
}
} catch (Exception e) {
SpiderDebug.log(e);
}
}
}
}
// 根据配置构建 jsonJx 和 webJx
LinkedHashMap<String, String> jsonJx = new LinkedHashMap<>();
ArrayList<String> webJx = new ArrayList<>();
List<String> targetKeys = configs.get(flag);
if (targetKeys != null && !targetKeys.isEmpty()) {
for (String key : targetKeys) {
HashMap<String, String> parseBean = jx.get(key);
if (parseBean == null) {
continue;
}
String type = parseBean.get("type");
if (type == null) {
continue;
}
if ("1".equals(type)) {
String urlValue = parseBean.get("url");
String ext = parseBean.get("ext");
if (urlValue != null && ext != null) {
jsonJx.put(key, mixUrl(urlValue, ext));
}
} else if ("0".equals(type)) {
String urlValue = parseBean.get("url");
if (urlValue != null) {
webJx.add(urlValue);
}
}
}
} else {
for (Map.Entry<String, HashMap<String, String>> entry : jx.entrySet()) {
String key = entry.getKey();
HashMap<String, String> parseBean = entry.getValue();
if (parseBean == null) {
continue;
}
String type = parseBean.get("type");
if (type == null) {
continue;
}
if ("1".equals(type)) {
String urlValue = parseBean.get("url");
String ext = parseBean.get("ext");
if (urlValue != null && ext != null) {
jsonJx.put(key, mixUrl(urlValue, ext));
}
} else if ("0".equals(type)) {
String urlValue = parseBean.get("url");
if (urlValue != null) {
webJx.add(urlValue);
}
}
}
}
// 缓存 webview 解析的地址
if (!webJx.isEmpty()) {
flagWebJx.put(flag, webJx);
}
ExecutorService exec = Executors.newFixedThreadPool(2);
CompletionService<JSONObject> cs = new ExecutorCompletionService<>(exec);
List<Future<JSONObject>> tasks = new ArrayList<>();
tasks.add(cs.submit(new Callable<JSONObject>() {
@Override
public JSONObject call() {
return JsonParallel.parse(jsonJx, url);
}
}));
if (!webJx.isEmpty()) {
tasks.add(cs.submit(new Callable<JSONObject>() {
@Override
public JSONObject call() {
JSONObject webResult = new JSONObject();
String encodedUrl = Base64.encodeToString(url.getBytes(),
Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP);
try {
webResult.put("url", "proxy://go=MixWeb&flag=" + flag + "&url=" + encodedUrl);
webResult.put("parse", 1);
webResult.put("ua", Utils.UaWinChrome);
} catch (Exception e) {
SpiderDebug.log(e);
}
return webResult;
}
}));
}
JSONObject result = null;
for (int i = 0, n = tasks.size(); i < n; i++) {
try {
Future<JSONObject> future = cs.take();
JSONObject res = future.get();
if (res != null && res.has("url")) {
result = res;
break;
}
} catch (Exception e) {
SpiderDebug.log(e);
}
}
for (Future<JSONObject> future : tasks) {
future.cancel(true);
}
exec.shutdownNow();
if (result != null) {
return result;
}
} catch (Exception e) {
SpiderDebug.log(e);
}
return new JSONObject();
}
private static String mixUrl(String url, String ext) {
if (ext.trim().length() > 0) {
int idx = url.indexOf("?");
if (idx > 0) {
return url.substring(0, idx + 1) + "cat_ext=" + Base64.encodeToString(ext.getBytes(), Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP) + "&" + url.substring(idx + 1);
}
}
return url;
}
public static Object[] loadHtml(String flag, String url) {
try {
url = new String(Base64.decode(url, Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP), "UTF-8");
String html = "\n" +
"<!doctype html>\n" +
"<html>\n" +
"<head>\n" +
"<title>解析</title>\n" +
"<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />\n" +
"<meta http-equiv=\"X-UA-Compatible\" content=\"IE=EmulateIE10\" />\n" +
"<meta name=\"renderer\" content=\"webkit|ie-comp|ie-stand\">\n" +
"<meta name=\"viewport\" content=\"width=device-width\">\n" +
"</head>\n" +
"<body>\n" +
"<script>\n" +
"var apiArray=[#jxs#];\n" +
"var urlPs=\"#url#\";\n" +
"var iframeHtml=\"\";\n" +
"for(var i=0;i<apiArray.length;i++){\n" +
"var URL=apiArray[i]+urlPs;\n" +
"iframeHtml=iframeHtml+\"<iframe sandbox='allow-scripts allow-same-origin allow-forms' frameborder='0' allowfullscreen='true' webkitallowfullscreen='true' mozallowfullscreen='true' src=\"+URL+\"></iframe>\";\n" +
"}\n" +
"document.write(iframeHtml);\n" +
"</script>\n" +
"</body>\n" +
"</html>";
StringBuilder jxs = new StringBuilder();
if (flagWebJx.containsKey(flag)) {
ArrayList<String> jxUrls = flagWebJx.get(flag);
for (int i = 0; i < jxUrls.size(); i++) {
jxs.append("\"");
jxs.append(jxUrls.get(i));
jxs.append("\"");
if (i < jxUrls.size() - 1) {
jxs.append(",");
}
}
}
html = html.replace("#url#", url).replace("#jxs#", jxs.toString());
Object[] result = new Object[3];
result[0] = 200;
result[1] = "text/html; charset=\"UTF-8\"";
ByteArrayInputStream baos = new ByteArrayInputStream(html.toString().getBytes("UTF-8"));
result[2] = baos;
return result;
} catch (Throwable th) {
th.printStackTrace();
}
return null;
}
}

115
app/src/main/java/com/github/tvbox/osc/util/parser/Utils.java
Unescape View File

@ -0,0 +1,115 @@
package com.github.tvbox.osc.util.parser;
import android.annotation.TargetApi;
import android.os.Build;
import android.webkit.ValueCallback;
import android.webkit.WebView;
import org.json.JSONException;
import org.json.JSONObject;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Pattern;
@TargetApi(Build.VERSION_CODES.KITKAT)
public class Utils {
public static final Pattern RULE = Pattern.compile("http((?!http).){12,}?\\.(m3u8|mp4|flv|avi|mkv|rm|wmv|mpg|m4a|mp3)\\?.*|http((?!http).){12,}\\.(m3u8|mp4|flv|avi|mkv|rm|wmv|mpg|m4a|mp3)|http((?!http).)*?video/tos*");
public static final String UaWinChrome = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.54 Safari/537.36";
public static final String UaMobile = "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Mobile/15E148 Safari/604.1";
public static boolean isVip(String url) {
List<String> hosts = Arrays.asList("iqiyi.com", "v.qq.com", "youku.com", "le.com", "tudou.com", "mgtv.com", "sohu.com", "acfun.cn", "bilibili.com", "baofeng.com", "pptv.com");
for (String host : hosts) if (url.contains(host)) return true;
return false;
}
public static boolean isVideoFormat(String url) {
if (url.contains("url=http") || url.contains(".js") || url.contains(".css") || url.contains(".html")) return false;
return RULE.matcher(url).find();
}
public static String substring(String text) {
return substring(text, 1);
}
public static String substring(String text, int num) {
if (text != null && text.length() > num) {
return text.substring(0, text.length() - num);
} else {
return text;
}
}
public static void loadUrl(WebView webView, String script) {
loadUrl(webView, script, null);
}
public static void loadUrl(WebView webView, String script, ValueCallback<String> callback) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) webView.evaluateJavascript(script, callback);
else webView.loadUrl(script);
}
public static boolean isBlackVodUrl(String input, String url) {
if (url.contains("973973.xyz") || url.contains(".fit:"))
return true;
return false;
}
public static JSONObject fixJsonVodHeader(JSONObject headers, String input, String url) throws JSONException {
if (headers == null)
headers = new JSONObject();
if (input.contains("www.mgtv.com")) {
headers.put("Referer", " ");
headers.put("User-Agent", " Mozilla/5.0");
} else if (url.contains("titan.mgtv")) {
headers.put("Referer", " ");
headers.put("User-Agent", " Mozilla/5.0");
} else if (input.contains("bilibili")) {
headers.put("Referer", " https://www.bilibili.com/");
headers.put("User-Agent", " " + UaWinChrome);
}
return headers;
}
public static JSONObject jsonParse(String input, String json) throws JSONException {
JSONObject jsonPlayData = new JSONObject(json);
String url;
if (jsonPlayData.has("data")) {
url = jsonPlayData.getJSONObject("data").getString("url");
} else {
url = jsonPlayData.getString("url");
}
if (url.startsWith("//")) {
url = "https:" + url;
}
if (!url.startsWith("http")) {
return null;
}
if (url.equals(input)) {
if (isVip(url) || !isVideoFormat(url)) {
return null;
}
}
if (Utils.isBlackVodUrl(input, url)) {
return null;
}
JSONObject headers = new JSONObject();
String ua = jsonPlayData.optString("user-agent", "");
if (ua.trim().length() > 0) {
headers.put("User-Agent", " " + ua);
}
String referer = jsonPlayData.optString("referer", "");
if (referer.trim().length() > 0) {
headers.put("Referer", " " + referer);
}
headers = Utils.fixJsonVodHeader(headers, input, url);
JSONObject taskResult = new JSONObject();
taskResult.put("header", headers);
taskResult.put("url", url);
return taskResult;
}
}
Write Preview
Loading…
Cancel
Save