hg
/
BeartvSpider
mirror of https://github.com/FongMi/CatVodSpider.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Clean code

Browse Source
pull/69/head
FongMi 3 years ago
parent 0ae4f4d3ca
commit f1eb933dab
7 changed files with 1 additions and 296 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 65
      app/src/main/java/com/github/catvod/parser/JsonBasic.java
  2. 71
      app/src/main/java/com/github/catvod/parser/JsonParallel.java
  3. 35
      app/src/main/java/com/github/catvod/parser/JsonSequence.java
  4. 88
      app/src/main/java/com/github/catvod/parser/MixDemo.java
  5. 36
      app/src/main/java/com/github/catvod/utils/Utils.java
  6. BIN
      jar/custom_spider.jar
  7. 2
      jar/custom_spider.jar.md5

65
app/src/main/java/com/github/catvod/parser/JsonBasic.java
Unescape View File

@ -1,65 +0,0 @@
package com.github.catvod.parser;
import android.util.Base64;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.net.OkHttp;
import com.github.catvod.utils.Utils;
import org.json.JSONObject;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Set;
public class JsonBasic {
public static JSONObject parse(LinkedHashMap<String, String> jx, String url) {
SpiderDebug.log("Load Json Parse Basic...");
if (jx.isEmpty()) return new JSONObject();
Set<String> jxNames = jx.keySet();
for (String jxName : jxNames) {
String parseUrl = jx.get(jxName);
HashMap<String, String> reqHeaders = getReqHeader(parseUrl);
try {
String realUrl = reqHeaders.get("url");
reqHeaders.remove("url");
SpiderDebug.log(realUrl + url);
String json = OkHttp.string(realUrl + url, reqHeaders);
JSONObject taskResult = Utils.jsonParse(url, json);
if (taskResult == null) continue;
taskResult.put("jxFrom", jxName);
SpiderDebug.log(taskResult.toString());
return taskResult;
} catch (Exception ignored) {
}
}
return new JSONObject();
}
public static HashMap<String, String> getReqHeader(String url) {
HashMap<String, String> reqHeaders = new HashMap<>();
reqHeaders.put("url", url);
if (!url.contains("cat_ext")) return reqHeaders;
try {
int start = url.indexOf("cat_ext=");
int end = url.indexOf("&", start);
String ext = url.substring(start + 8, end);
ext = new String(Base64.decode(ext, Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP));
String newUrl = url.substring(0, start) + url.substring(end + 1);
JSONObject jsonObject = new JSONObject(ext);
if (jsonObject.has("header")) {
JSONObject headerJson = jsonObject.optJSONObject("header");
Iterator<String> keys = headerJson.keys();
while (keys.hasNext()) {
String key = keys.next();
reqHeaders.put(key, headerJson.optString(key, ""));
}
}
reqHeaders.put("url", newUrl);
} catch (Exception ignored) {
}
return reqHeaders;
}
}

71
app/src/main/java/com/github/catvod/parser/JsonParallel.java
Unescape View File

@ -1,71 +0,0 @@
package com.github.catvod.parser;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.net.OkHttp;
import com.github.catvod.utils.Utils;
import org.json.JSONObject;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
public class JsonParallel {
private static final String ParseOKTag = "p_json_parse";
public static JSONObject parse(LinkedHashMap<String, String> jx, String url) {
if (jx.isEmpty()) return new JSONObject();
ExecutorService service = Executors.newFixedThreadPool(3);
CompletionService<JSONObject> completionService = new ExecutorCompletionService<>(service);
List<Future<JSONObject>> futures = new ArrayList<>();
Set<String> jxNames = jx.keySet();
for (String jxName : jxNames) {
String parseUrl = jx.get(jxName);
futures.add(completionService.submit(() -> {
try {
HashMap<String, String> reqHeaders = JsonBasic.getReqHeader(parseUrl);
String realUrl = reqHeaders.get("url");
reqHeaders.remove("url");
SpiderDebug.log(realUrl + url);
String json = OkHttp.string(realUrl + url, ParseOKTag, reqHeaders);
JSONObject taskResult = Utils.jsonParse(url, json);
taskResult.put("jxFrom", jxName);
SpiderDebug.log(taskResult.toString());
return taskResult;
} catch (Exception ignored) {
return null;
}
}));
}
JSONObject pTaskResult = null;
for (int i = 0; i < futures.size(); ++i) {
try {
Future<JSONObject> completed = completionService.take();
pTaskResult = completed.get();
if (pTaskResult != null) {
OkHttp.cancel(ParseOKTag);
for (int j = 0; j < futures.size(); j++) {
try {
futures.get(j).cancel(true);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
futures.clear();
break;
}
} catch (Exception ignored) {
}
}
service.shutdownNow();
return pTaskResult != null ? pTaskResult : new JSONObject();
}
}

35
app/src/main/java/com/github/catvod/parser/JsonSequence.java
Unescape View File

@ -1,35 +0,0 @@
package com.github.catvod.parser;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.net.OkHttp;
import com.github.catvod.utils.Utils;
import org.json.JSONObject;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Set;
public class JsonSequence {
public static JSONObject parse(LinkedHashMap<String, String> jx, String url) {
if (jx.isEmpty()) return new JSONObject();
Set<String> jxNames = jx.keySet();
for (String jxName : jxNames) {
try {
String parseUrl = jx.get(jxName);
HashMap<String, String> reqHeaders = JsonBasic.getReqHeader(parseUrl);
String realUrl = reqHeaders.get("url");
reqHeaders.remove("url");
SpiderDebug.log(realUrl + url);
String json = OkHttp.string(realUrl + url, reqHeaders);
JSONObject taskResult = Utils.jsonParse(url, json);
if (taskResult == null) continue;
taskResult.put("jxFrom", jxName);
return taskResult;
} catch (Exception ignored) {
}
}
return new JSONObject();
}
}

88
app/src/main/java/com/github/catvod/parser/MixDemo.java
Unescape View File

@ -1,88 +0,0 @@
package com.github.catvod.parser;
import android.util.Base64;
import org.json.JSONArray;
import org.json.JSONObject;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
public class MixDemo {
private static final HashMap<String, ArrayList<String>> flagWebJx = new HashMap<>();
private static HashMap<String, ArrayList<String>> configs = null;
private static void setConfigs(LinkedHashMap<String, HashMap<String, String>> jx) {
configs = new HashMap<>();
for (String key : jx.keySet()) {
HashMap<String, String> parseBean = jx.get(key);
String type = parseBean.get("type");
if (type.equals("1") || type.equals("0")) {
try {
JSONArray flags = new JSONObject(parseBean.get("ext")).getJSONArray("flag");
for (int j = 0; j < flags.length(); j++) {
String flagKey = flags.getString(j);
ArrayList<String> flagJx = configs.get(flagKey);
if (flagJx == null) {
flagJx = new ArrayList<>();
configs.put(flagKey, flagJx);
}
flagJx.add(key);
}
} catch (Exception ignored) {
}
}
}
}
public static JSONObject parse(LinkedHashMap<String, HashMap<String, String>> jx, String nameMe, String flag, String url) {
try {
if (configs == null) setConfigs(jx);
LinkedHashMap<String, String> jsonJx = new LinkedHashMap<>();
ArrayList<String> webJx = new ArrayList<>();
ArrayList<String> flagJx = configs.get(flag);
if (flagJx != null && !flagJx.isEmpty()) {
for (int i = 0; i < flagJx.size(); i++) {
String key = flagJx.get(i);
HashMap<String, String> parseBean = jx.get(key);
String type = parseBean.get("type");
if (type.equals("1")) {
jsonJx.put(key, mixUrl(parseBean.get("url"), parseBean.get("ext")));
} else if (type.equals("0")) {
webJx.add(parseBean.get("url"));
}
}
} else {
for (String key : jx.keySet()) {
HashMap<String, String> parseBean = jx.get(key);
String type = parseBean.get("type");
if (type.equals("1")) {
jsonJx.put(key, mixUrl(parseBean.get("url"), parseBean.get("ext")));
} else if (type.equals("0")) {
webJx.add(parseBean.get("url"));
}
}
}
if (!webJx.isEmpty()) flagWebJx.put(flag, webJx);
JSONObject jsonResult = JsonParallel.parse(jsonJx, url);
if (jsonResult.has("url")) return jsonResult;
if (!webJx.isEmpty()) {
JSONObject webResult = new JSONObject();
webResult.put("url", "proxy://do=parseMix&flag=" + flag + "&url=" + Base64.encodeToString(url.getBytes(), Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP));
webResult.put("parse", 1);
return webResult;
}
} catch (Exception ignored) {
}
return new JSONObject();
}
private static String mixUrl(String url, String ext) {
if (ext.trim().isEmpty()) return url;
int index = url.indexOf("?");
if (index == -1) return url;
return url.substring(0, index + 1) + "cat_ext=" + Base64.encodeToString(ext.getBytes(), Base64.DEFAULT | Base64.URL_SAFE | Base64.NO_WRAP) + "&" + url.substring(index + 1);
}
}

36
app/src/main/java/com/github/catvod/utils/Utils.java
Unescape View File

@ -13,9 +13,6 @@ import android.webkit.WebViewClient;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.spider.Init;
import org.json.JSONException;
import org.json.JSONObject;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.util.Arrays;
@ -72,39 +69,6 @@ public class Utils {
return src;
}
public static JSONObject fixJsonVodHeader(JSONObject headers, String input, String url) throws JSONException {
if (headers == null) headers = new JSONObject();
if (input.contains("www.mgtv.com")) {
headers.put("Referer", "");
headers.put("User-Agent", "Mozilla/5.0");
} else if (url.contains("titan.mgtv")) {
headers.put("Referer", "");
headers.put("User-Agent", "Mozilla/5.0");
} else if (input.contains("bilibili")) {
headers.put("Referer", "https://www.bilibili.com/");
headers.put("User-Agent", CHROME);
}
return headers;
}
public static JSONObject jsonParse(String input, String json) throws JSONException {
JSONObject jsonPlayData = new JSONObject(json);
String url = jsonPlayData.getString("url");
if (url.startsWith("//")) url = "https:" + url;
if (!url.startsWith("http")) return null;
if (url.equals(input)) if (isVip(url) || !isVideoFormat(url)) return null;
JSONObject headers = new JSONObject();
String ua = jsonPlayData.optString("user-agent", "");
if (ua.trim().length() > 0) headers.put("User-Agent", ua);
String referer = jsonPlayData.optString("referer", "");
if (referer.trim().length() > 0) headers.put("Referer", referer);
headers = Utils.fixJsonVodHeader(headers, input, url);
JSONObject taskResult = new JSONObject();
taskResult.put("header", headers);
taskResult.put("url", url);
return taskResult;
}
public static String substring(String text) {
return substring(text, 1);
}

BIN
jar/custom_spider.jar
View File

Binary file not shown.

2
jar/custom_spider.jar.md5
Unescape View File

@ -1 +1 @@
384628eab024a2a40a57835d2f0684d8
216774d6bcf8c060124d3562267c7d89

Write Preview
Loading…
Cancel
Save