Merge pull request #568 from okcaptain/dev

Dev

app/build.gradle

@@ -13,8 +13,8 @@ android {
         minSdk 21
         //noinspection ExpiredTargetSdkVersion
         targetSdk 28
-        versionCode 246
-        versionName "2.4.6"
+        versionCode 247
+        versionName "2.4.7"
         javaCompileOptions {
             annotationProcessorOptions {
                 arguments = ["room.schemaLocation": "$projectDir/schemas".toString()]

app/src/main/java/com/fongmi/android/tv/api/config/LiveConfig.java

@@ -1,5 +1,6 @@
 package com.fongmi.android.tv.api.config;
 
+import android.net.Uri;
 import android.text.TextUtils;
 
 import com.fongmi.android.tv.App;
@@ -25,6 +26,7 @@ import com.github.catvod.utils.Json;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
 
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -128,13 +130,22 @@ public class LiveConfig {
     }
 
     private void parseText(String text, Callback callback) {
-        Live live = new Live(config.getUrl()).sync();
+        Live live = new Live(parseName(config.getUrl()), config.getUrl()).sync();
         LiveParser.text(live, text);
         lives.add(live);
         setHome(live, true);
         App.post(callback::success);
     }
 
+    private String parseName(String url) {
+        Uri uri = Uri.parse(url);
+        if ("file".equals(uri.getScheme())) return new File(url).getName();
+        if (uri.getLastPathSegment() != null) return uri.getLastPathSegment();
+        if (uri.getQuery() != null) return uri.getQuery();
+        if (uri.getHost() != null) return uri.getHost();
+        return url;
+    }
+
     private void checkJson(JsonObject object, Callback callback) {
         if (object.has("msg") && callback != null) {
             App.post(() -> callback.error(object.get("msg").getAsString()));

app/src/main/java/com/fongmi/android/tv/bean/Live.java

@@ -1,6 +1,5 @@
 package com.fongmi.android.tv.bean;
 
-import android.net.Uri;
 import android.text.TextUtils;
 
 import androidx.annotation.NonNull;
@@ -22,7 +21,6 @@ import com.google.gson.annotations.JsonAdapter;
 import com.google.gson.annotations.SerializedName;
 import com.google.gson.reflect.TypeToken;
 
-import java.io.File;
 import java.lang.reflect.Type;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -137,11 +135,6 @@ public class Live {
     public Live() {
     }
 
-    public Live(String url) {
-        this.name = url.startsWith("file") ? new File(url).getName() : Uri.parse(url).getLastPathSegment();
-        this.url = url;
-    }
-
     public Live(@NonNull String name, String url) {
         this.name = name;
         this.url = url;

app/src/main/java/com/fongmi/android/tv/player/ParseJob.java

@@ -154,7 +154,7 @@ public class ParseJob implements ParseCallback {
     }
 
     private void checkResult(Map<String, String> headers, String url, String from, boolean error) {
-        if (isPass(headers, url)) {
+        if (url.length() > 40) {
             onParseSuccess(headers, url, from);
         } else if (error) {
             onParseError();
@@ -168,15 +168,6 @@ public class ParseJob implements ParseCallback {
         else onParseSuccess(result.getHeaders(), result.getUrl().v(), result.getJxFrom());
     }
 
-    private boolean isPass(Map<String, String> headers, String url) {
-        try {
-            if (url.length() < 40) return false;
-            return OkHttp.newCall(url, Headers.of(headers)).execute().code() == 200;
-        } catch (Exception e) {
-            return false;
-        }
-    }
-
     private void startWeb(List<Parse> items, String webUrl) {
         StringBuilder sb = new StringBuilder();
         for (Parse item : items) sb.append(item.getUrl()).append(";");
@@ -197,7 +188,7 @@ public class ParseJob implements ParseCallback {
 
     private Map<String, String> getHeader(JsonObject object) {
         Map<String, String> headers = new HashMap<>();
-        for (Map.Entry<String, JsonElement> entry : object.entrySet()) if (entry.getKey().equalsIgnoreCase(HttpHeaders.USER_AGENT) || entry.getKey().equalsIgnoreCase(HttpHeaders.REFERER) || entry.getKey().equalsIgnoreCase("ua")) headers.put(UrlUtil.fixHeader(entry.getKey()), object.get(entry.getKey()).getAsString());
+        for (Map.Entry<String, JsonElement> entry : object.entrySet()) if (!entry.getValue().isJsonNull() && (entry.getKey().equalsIgnoreCase(HttpHeaders.USER_AGENT) || entry.getKey().equalsIgnoreCase(HttpHeaders.REFERER) || entry.getKey().equalsIgnoreCase("ua"))) headers.put(UrlUtil.fixHeader(entry.getKey()), entry.getValue().getAsString());
         if (headers.isEmpty()) return parse.getHeaders();
         return headers;
     }

catvod/src/main/java/com/github/catvod/utils/Json.java

@@ -2,8 +2,6 @@ package com.github.catvod.utils;
 
 import android.text.TextUtils;
 
-import androidx.collection.ArrayMap;
-
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
@@ -82,13 +80,6 @@ public class Json {
         return map;
     }
 
-    public static ArrayMap<String, String> toArrayMap(JsonElement element) {
-        ArrayMap<String, String> map = new ArrayMap<>();
-        JsonObject object = safeObject(element);
-        for (Map.Entry<String, JsonElement> entry : object.entrySet()) map.put(entry.getKey(), safeString(object, entry.getKey()));
-        return map;
-    }
-
     public static JsonObject toObject(Map<String, String> map) {
         JsonObject object = new JsonObject();
         for (String key : map.keySet()) object.addProperty(key, map.get(key));

pyramid/src/main/python/base/spider.py

@@ -13,6 +13,9 @@ from com.github.catvod import Proxy
 class Spider(metaclass=ABCMeta):
     _instance = None
 
+    def __init__(self):
+        self.extend = ''
+
     def __new__(cls, *args, **kwargs):
         if cls._instance:
             return cls._instance
@@ -24,26 +27,21 @@ class Spider(metaclass=ABCMeta):
     def init(self, extend=""):
         pass
 
-    @abstractmethod
     def homeContent(self, filter):
         pass
 
     def homeVideoContent(self):
         pass
 
-    @abstractmethod
     def categoryContent(self, tid, pg, filter, extend):
         pass
 
-    @abstractmethod
     def detailContent(self, ids):
         pass
 
-    @abstractmethod
     def searchContent(self, key, quick, pg="1"):
         pass
 
-    @abstractmethod
     def playerContent(self, flag, id, vipFlags):
         pass
 
@@ -104,9 +102,21 @@ class Spider(metaclass=ABCMeta):
     def html(self, content):
         return etree.HTML(content)
 
+    def str2json(str):
+        return json.loads(str)
+
+    def json2str(str):
+        return json.dumps(str, ensure_ascii=False)
+
     def getProxyUrl(self, local=True):
         return f'{Proxy.getUrl(local)}?do=py'
 
+    def log(self, msg):
+        if isinstance(msg, dict) or isinstance(msg, list):
+            print(json.dumps(msg, ensure_ascii=False))
+        else:
+            print(f'{msg}')
+
     def getCache(self, key):
         value = self.fetch(f'http://127.0.0.1:{Proxy.getPort()}/cache?do=get&key={key}', timeout=5).text
         if len(value) > 0:

quickjs/src/main/java/com/fongmi/quickjs/crawler/Spider.java

@@ -140,7 +140,7 @@ public class Spider extends com.github.catvod.crawler.Spider {
 
     private void initializeJS() throws Exception {
         submit(() -> {
-            if (ctx == null) createCtx();
+            createCtx();
             createObj();
             return null;
         }).get();
@@ -167,14 +167,13 @@ public class Spider extends com.github.catvod.crawler.Spider {
     }
 
     private void createObj() {
-        String jsEval = "__jsEvalReturn";
         String spider = "__JS_SPIDER__";
         String global = "globalThis." + spider;
         String content = Module.get().fetch(api);
-        if (content.startsWith("//bb")) ctx.execute(Module.get().bb(content));
-        else ctx.evaluateModule(content.replace(spider, global), api);
+        boolean bb = content.startsWith("//bb");
+        cat = bb || content.contains("__jsEvalReturn");
+        if (!bb) ctx.evaluateModule(content.replace(spider, global), api);
         ctx.evaluateModule(String.format(Asset.read("js/lib/spider.js"), api));
-        if (content.startsWith("//bb") || content.contains(jsEval)) cat = true;
         jsObject = (JSObject) ctx.getProperty(ctx.getGlobalObject(), spider);
     }