From bafe21e19af20ff4dbf6befe8cca41df4ea9ab20 Mon Sep 17 00:00:00 2001 From: okjack Date: Wed, 17 Jul 2024 17:48:20 +0800 Subject: [PATCH] py add download dependence --- .../android/tv/api/loader/PyLoader.java | 4 +- .../java/com/github/catvod/utils/Path.java | 4 ++ .../com/undcover/freedom/pyramid/Loader.java | 8 ++-- .../com/undcover/freedom/pyramid/Spider.java | 14 ++++++- pyramid/src/main/python/app.py | 25 ++++++++---- pyramid/src/main/python/base/spider.py | 39 +++++++------------ 6 files changed, 54 insertions(+), 40 deletions(-) diff --git a/app/src/main/java/com/fongmi/android/tv/api/loader/PyLoader.java b/app/src/main/java/com/fongmi/android/tv/api/loader/PyLoader.java index 08626af60..942d565f1 100644 --- a/app/src/main/java/com/fongmi/android/tv/api/loader/PyLoader.java +++ b/app/src/main/java/com/fongmi/android/tv/api/loader/PyLoader.java @@ -42,8 +42,8 @@ public class PyLoader { public Spider getSpider(String key, String api, String ext) { try { if (spiders.containsKey(key)) return spiders.get(key); - Method method = loader.getClass().getMethod("spider", Context.class, String.class, String.class); - Spider spider = (Spider) method.invoke(loader, App.get(), key, api); + Method method = loader.getClass().getMethod("spider", Context.class, String.class); + Spider spider = (Spider) method.invoke(loader, App.get(), api); spider.init(App.get(), ext); spiders.put(key, spider); return spider; diff --git a/catvod/src/main/java/com/github/catvod/utils/Path.java b/catvod/src/main/java/com/github/catvod/utils/Path.java index fc09e2932..7ced5cb5c 100644 --- a/catvod/src/main/java/com/github/catvod/utils/Path.java +++ b/catvod/src/main/java/com/github/catvod/utils/Path.java @@ -129,6 +129,10 @@ public class Path { return new File(js(), name); } + public static File py(String name) { + return new File(py(), name); + } + public static File jar(String name) { return new File(jar(), Util.md5(name).concat(".jar")); } diff --git a/pyramid/src/main/java/com/undcover/freedom/pyramid/Loader.java b/pyramid/src/main/java/com/undcover/freedom/pyramid/Loader.java index 53be6d712..8532d886e 100644 --- a/pyramid/src/main/java/com/undcover/freedom/pyramid/Loader.java +++ b/pyramid/src/main/java/com/undcover/freedom/pyramid/Loader.java @@ -12,19 +12,17 @@ import com.github.catvod.utils.Path; public class Loader { private PyObject app; - private String cache; @Keep private void init(Context context) { if (!Python.isStarted()) Python.start(new AndroidPlatform(context)); app = Python.getInstance().getModule("app"); - cache = Path.py().getAbsolutePath(); } @Keep - public Spider spider(Context context, String key, String api) { + public Spider spider(Context context, String api) { if (app == null) init(context); - PyObject obj = app.callAttr("spider", cache, key, api); - return new Spider(app, obj); + PyObject obj = app.callAttr("spider", Path.py().getAbsolutePath(), api); + return new Spider(app, obj, api); } } diff --git a/pyramid/src/main/java/com/undcover/freedom/pyramid/Spider.java b/pyramid/src/main/java/com/undcover/freedom/pyramid/Spider.java index b0cca3d99..f3ce1dcb2 100644 --- a/pyramid/src/main/java/com/undcover/freedom/pyramid/Spider.java +++ b/pyramid/src/main/java/com/undcover/freedom/pyramid/Spider.java @@ -4,6 +4,8 @@ import android.content.Context; import com.chaquo.python.PyObject; import com.github.catvod.Proxy; +import com.github.catvod.utils.Path; +import com.github.catvod.utils.UriUtil; import com.github.catvod.utils.Util; import com.google.gson.Gson; @@ -16,12 +18,14 @@ public class Spider extends com.github.catvod.crawler.Spider { private final PyObject app; private final PyObject obj; + private final String api; private final Gson gson; - public Spider(PyObject app, PyObject obj) { + public Spider(PyObject app, PyObject obj, String api) { this.gson = new Gson(); this.app = app; this.obj = obj; + this.api = api; } @Override @@ -31,6 +35,8 @@ public class Spider extends com.github.catvod.crawler.Spider { @Override public void init(Context context, String extend) { + List items = app.callAttr("getDependence", obj).asList(); + for (PyObject item : items) download(item + ".py"); app.callAttr("init", obj, extend); } @@ -108,6 +114,12 @@ public class Spider extends com.github.catvod.crawler.Spider { } } + private void download(String name) { + String path = Path.py(name).getAbsolutePath(); + String url = UriUtil.resolve(api, name); + app.callAttr("download", path, url); + } + private String replaceProxy(String content) { return content.replace("http://127.0.0.1:UndCover/proxy", Proxy.getUrl(true)); } diff --git a/pyramid/src/main/python/app.py b/pyramid/src/main/python/app.py index 6e5db1fea..d39ac70fc 100644 --- a/pyramid/src/main/python/app.py +++ b/pyramid/src/main/python/app.py @@ -4,22 +4,23 @@ from importlib.machinery import SourceFileLoader import json -def spider(cache, key, api): +def spider(cache, api): name = os.path.basename(api) path = cache + '/' + name - downloadFile(path, api) + download(path, api) + name = name.split('.')[0] return SourceFileLoader(name, path).load_module().Spider() -def downloadFile(name, api): +def download(path, api): if api.startswith('http'): - writeFile(name, redirect(api).content) + writeFile(path, redirect(api).content) else: - writeFile(name, str.encode(api)) + writeFile(path, str.encode(api)) -def writeFile(name, content): - with open(name, 'wb') as f: +def writeFile(path, content): + with open(path, 'wb') as f: f.write(content) @@ -35,6 +36,16 @@ def str2json(content): return json.loads(content) +def getDependence(ru): + result = ru.getDependence() + return result + + +def getName(ru): + result = ru.getName() + return result + + def init(ru, extend): ru.init(extend) diff --git a/pyramid/src/main/python/base/spider.py b/pyramid/src/main/python/base/spider.py index 1944c942c..9c69d5010 100644 --- a/pyramid/src/main/python/base/spider.py +++ b/pyramid/src/main/python/base/spider.py @@ -1,7 +1,9 @@ import re +import os import json import requests from lxml import etree +from com.chaquo.python import Python from abc import abstractmethod, ABCMeta from importlib.machinery import SourceFileLoader @@ -69,15 +71,17 @@ class Spider(metaclass=ABCMeta): def getDependence(self): return [] - def regStr(self, src, reg, group=1): - m = re.search(reg, src) - src = '' - if m: - src = m.group(group) - return src + def loadSpider(self, name): + return self.loadModule(name).Spider() - def str2json(self, str): - return json.loads(str) + def loadModule(self, name): + cache_dir = Python.getPlatform().getApplication().getCacheDir().getAbsolutePath() + path = os.path.join(os.path.join(cache_dir, 'py'), f'{name}.py') + return SourceFileLoader(name, path).load_module() + + def removeHtmlTags(self, src): + clean = re.compile('<.*?>') + return re.sub(clean, '', src) def cleanText(self, src): clean = re.sub('[\U0001F600-\U0001F64F\U0001F300-\U0001F5FF\U0001F680-\U0001F6FF\U0001F1E0-\U0001F1FF]', '', src) @@ -88,25 +92,10 @@ class Spider(metaclass=ABCMeta): rsp.encoding = 'utf-8' return rsp - def post(self, url, params=None, data, cookies=None, headers=None, timeout=5, verify=True, stream=False, allow_redirects = True): - rsp = requests.post(url, params=params, data=data, cookies=cookies, headers=headers, timeout=timeout, verify=verify, stream=stream, allow_redirects=allow_redirects) - rsp.encoding = 'utf-8' - return rsp - - def postJson(self, url, params=None, json, cookies=None, headers=None, timeout=5, verify=True, stream=False, allow_redirects = True): - rsp = requests.post(url, params=params, json=json, cookies=cookies, headers=headers, timeout=timeout, verify=verify, stream=stream, allow_redirects=allow_redirects) + def post(self, url, params=None, data=None, json=None, cookies=None, headers=None, timeout=5, verify=True, stream=False, allow_redirects = True): + rsp = requests.post(url, params=params, data=data, json=json, cookies=cookies, headers=headers, timeout=timeout, verify=verify, stream=stream, allow_redirects=allow_redirects) rsp.encoding = 'utf-8' return rsp def html(self, content): return etree.HTML(content) - - def xpText(self, root, expr): - ele = root.xpath(expr) - if len(ele) == 0: - return '' - else: - return ele[0] - - def loadModule(self, name, path): - return SourceFileLoader(name, path).load_module()