py add download dependence

app/src/main/java/com/fongmi/android/tv/api/loader/PyLoader.java

@@ -42,8 +42,8 @@ public class PyLoader {
     public Spider getSpider(String key, String api, String ext) {
         try {
             if (spiders.containsKey(key)) return spiders.get(key);
-            Method method = loader.getClass().getMethod("spider", Context.class, String.class, String.class);
-            Spider spider = (Spider) method.invoke(loader, App.get(), key, api);
+            Method method = loader.getClass().getMethod("spider", Context.class, String.class);
+            Spider spider = (Spider) method.invoke(loader, App.get(), api);
             spider.init(App.get(), ext);
             spiders.put(key, spider);
             return spider;

catvod/src/main/java/com/github/catvod/utils/Path.java

@@ -129,6 +129,10 @@ public class Path {
         return new File(js(), name);
     }
 
+    public static File py(String name) {
+        return new File(py(), name);
+    }
+
     public static File jar(String name) {
         return new File(jar(), Util.md5(name).concat(".jar"));
     }

pyramid/src/main/java/com/undcover/freedom/pyramid/Loader.java

@@ -12,19 +12,17 @@ import com.github.catvod.utils.Path;
 public class Loader {
 
     private PyObject app;
-    private String cache;
 
     @Keep
     private void init(Context context) {
         if (!Python.isStarted()) Python.start(new AndroidPlatform(context));
         app = Python.getInstance().getModule("app");
-        cache = Path.py().getAbsolutePath();
     }
 
     @Keep
-    public Spider spider(Context context, String key, String api) {
+    public Spider spider(Context context, String api) {
         if (app == null) init(context);
-        PyObject obj = app.callAttr("spider", cache, key, api);
-        return new Spider(app, obj);
+        PyObject obj = app.callAttr("spider", Path.py().getAbsolutePath(), api);
+        return new Spider(app, obj, api);
     }
 }

pyramid/src/main/java/com/undcover/freedom/pyramid/Spider.java

@@ -4,6 +4,8 @@ import android.content.Context;
 
 import com.chaquo.python.PyObject;
 import com.github.catvod.Proxy;
+import com.github.catvod.utils.Path;
+import com.github.catvod.utils.UriUtil;
 import com.github.catvod.utils.Util;
 import com.google.gson.Gson;
 
@@ -16,12 +18,14 @@ public class Spider extends com.github.catvod.crawler.Spider {
 
     private final PyObject app;
     private final PyObject obj;
+    private final String api;
     private final Gson gson;
 
-    public Spider(PyObject app, PyObject obj) {
+    public Spider(PyObject app, PyObject obj, String api) {
         this.gson = new Gson();
         this.app = app;
         this.obj = obj;
+        this.api = api;
     }
 
     @Override
@@ -31,6 +35,8 @@ public class Spider extends com.github.catvod.crawler.Spider {
 
     @Override
     public void init(Context context, String extend) {
+        List<PyObject> items = app.callAttr("getDependence", obj).asList();
+        for (PyObject item : items) download(item + ".py");
         app.callAttr("init", obj, extend);
     }
 
@@ -108,6 +114,12 @@ public class Spider extends com.github.catvod.crawler.Spider {
         }
     }
 
+    private void download(String name) {
+        String path = Path.py(name).getAbsolutePath();
+        String url = UriUtil.resolve(api, name);
+        app.callAttr("download", path, url);
+    }
+
     private String replaceProxy(String content) {
         return content.replace("http://127.0.0.1:UndCover/proxy", Proxy.getUrl(true));
     }

pyramid/src/main/python/app.py

@@ -4,22 +4,23 @@ from importlib.machinery import SourceFileLoader
 import json
 
 
-def spider(cache, key, api):
+def spider(cache, api):
     name = os.path.basename(api)
     path = cache + '/' + name
-    downloadFile(path, api)
+    download(path, api)
+    name = name.split('.')[0]
     return SourceFileLoader(name, path).load_module().Spider()
 
 
-def downloadFile(name, api):
+def download(path, api):
     if api.startswith('http'):
-        writeFile(name, redirect(api).content)
+        writeFile(path, redirect(api).content)
     else:
-        writeFile(name, str.encode(api))
+        writeFile(path, str.encode(api))
 
 
-def writeFile(name, content):
-    with open(name, 'wb') as f:
+def writeFile(path, content):
+    with open(path, 'wb') as f:
         f.write(content)
 
 
@@ -35,6 +36,16 @@ def str2json(content):
     return json.loads(content)
 
 
+def getDependence(ru):
+    result = ru.getDependence()
+    return result
+
+
+def getName(ru):
+    result = ru.getName()
+    return result
+
+
 def init(ru, extend):
     ru.init(extend)
 

pyramid/src/main/python/base/spider.py

@@ -1,7 +1,9 @@
 import re
+import os
 import json
 import requests
 from lxml import etree
+from com.chaquo.python import Python
 from abc import abstractmethod, ABCMeta
 from importlib.machinery import SourceFileLoader
 
@@ -69,15 +71,17 @@ class Spider(metaclass=ABCMeta):
     def getDependence(self):
         return []
 
-    def regStr(self, src, reg, group=1):
-        m = re.search(reg, src)
-        src = ''
-        if m:
-            src = m.group(group)
-        return src
+    def loadSpider(self, name):
+        return self.loadModule(name).Spider()
 
-    def str2json(self, str):
-        return json.loads(str)
+    def loadModule(self, name):
+        cache_dir = Python.getPlatform().getApplication().getCacheDir().getAbsolutePath()
+        path = os.path.join(os.path.join(cache_dir, 'py'),  f'{name}.py')
+        return SourceFileLoader(name, path).load_module()
+
+    def removeHtmlTags(self, src):
+        clean = re.compile('<.*?>')
+        return re.sub(clean, '', src)
 
     def cleanText(self, src):
         clean = re.sub('[\U0001F600-\U0001F64F\U0001F300-\U0001F5FF\U0001F680-\U0001F6FF\U0001F1E0-\U0001F1FF]', '', src)
@@ -88,25 +92,10 @@ class Spider(metaclass=ABCMeta):
         rsp.encoding = 'utf-8'
         return rsp
 
-    def post(self, url, params=None, data, cookies=None, headers=None, timeout=5, verify=True, stream=False, allow_redirects = True):
-        rsp = requests.post(url, params=params, data=data, cookies=cookies, headers=headers, timeout=timeout, verify=verify, stream=stream, allow_redirects=allow_redirects)
-        rsp.encoding = 'utf-8'
-        return rsp
-
-    def postJson(self, url, params=None, json, cookies=None, headers=None, timeout=5, verify=True, stream=False, allow_redirects = True):
-        rsp = requests.post(url, params=params, json=json, cookies=cookies, headers=headers, timeout=timeout, verify=verify, stream=stream, allow_redirects=allow_redirects)
+    def post(self, url, params=None, data=None, json=None, cookies=None, headers=None, timeout=5, verify=True, stream=False, allow_redirects = True):
+        rsp = requests.post(url, params=params, data=data, json=json, cookies=cookies, headers=headers, timeout=timeout, verify=verify, stream=stream, allow_redirects=allow_redirects)
         rsp.encoding = 'utf-8'
         return rsp
 
     def html(self, content):
         return etree.HTML(content)
-
-    def xpText(self, root, expr):
-        ele = root.xpath(expr)
-        if len(ele) == 0:
-            return ''
-        else:
-            return ele[0]
-
-    def loadModule(self, name, path):
-        return SourceFileLoader(name, path).load_module()