fix python bug

8 months ago · c6d2c53958
parent d4c02c99e2
commit c6d2c53958
4 changed files with 27 additions and 9 deletions
--- a/app/src/python/java/com/github/catvod/crawler/pyLoader.java
+++ b/app/src/python/java/com/github/catvod/crawler/pyLoader.java
@ -72,6 +72,7 @@ public class pyLoader implements IPyLoader {

    @Override
    public Object[] proxyInvoke(Map<String, String> params){
+        if(recentPyApi==null)return null;
        LOG.i("echo-recentPyApi" + recentPyApi);
        try {
            PythonSpider originalSpider = (PythonSpider) getSpider(MD5.string2MD5(recentPyApi), recentPyApi,"");
--- a/app/src/python/java/com/undcover/freedom/pyramid/PythonSpider.java
+++ b/app/src/python/java/com/undcover/freedom/pyramid/PythonSpider.java
@ -39,6 +39,11 @@ public class PythonSpider extends Spider {
        this.name = name;
    }

+    @Override
+    public void init(Context context) {
+        app.callAttr("init", pySpider);
+    }
+
    public void init(Context context, String url) {
        app = PythonLoader.getInstance().pyApp;
        PyObject retValue = app.callAttr("downloadPlugin", cachePath, url);
@ -46,6 +51,7 @@ public class PythonSpider extends Spider {
        String extInfo = uri.getQueryParameter("extend");
        if (null == extInfo) extInfo = "";
        String path = retValue.toString();
+        Log.i("PyLoader", "echo-init path: " +path);
        File file = new File(path);
        if (file.exists()) {
            pySpider = app.callAttr("loadFromDisk", path);
@ -53,8 +59,10 @@ public class PythonSpider extends Spider {
            List<PyObject> poList = app.callAttr("getDependence", pySpider).asList();
            for (PyObject po : poList) {
                String api = po.toString();
+                Log.i("PyLoader", "echo-init api: " +api);
                String depUrl = PythonLoader.getInstance().getUrlByApi(api);
                if (!depUrl.isEmpty()) {
+                    Log.i("PyLoader", "echo-init depUrl: " +depUrl);
                    String tmpPath = app.callAttr("downloadPlugin", cachePath, depUrl).toString();
                    if (!new File(tmpPath).exists()) {
                        PyToast.showCancelableToast(api + "加载失败!");
--- a/pyramid/src/python/app.py
+++ b/pyramid/src/python/app.py
@ -5,6 +5,8 @@ import requests
 from importlib.machinery import SourceFileLoader  ### 导入这个模块
 from urllib import parse
 import json
+import sys
+sys.dont_write_bytecode = True

 def createFile(file_path):
    if os.path.exists(file_path) is False:
@ -84,7 +86,6 @@ def init(ru,extend):
        if sp != None:
            sp.setExtendInfo(sParam[key])
            spoList.append(sp)
-#     ru.setExtendInfo(extend)
    ru.init(extend)

 def homeContent(ru,filter):
--- a/pyramid/src/python/base/spider.py
+++ b/pyramid/src/python/base/spider.py
@ -97,10 +97,18 @@ class Spider(metaclass=ABCMeta):
        rsp.encoding = 'utf-8'
        return rsp

-    def post(self, url, params=None, data=None, json=None, cookies=None, headers=None, timeout=5, verify=True,
-             stream=False, allow_redirects=True):
-        rsp = requests.post(url, params=params, data=data, json=json, cookies=cookies, headers=headers, timeout=timeout,
-                            verify=verify, stream=stream, allow_redirects=allow_redirects)
+    def _clean_header_value(self, value):
+        cleaned = value.strip()
+        if cleaned.startswith("<!DOCTYPE html>"):
+            cleaned = "DefaultValue"  # 根据需求替换为合适的默认值
+        return cleaned
+
+    def post(self, url, params=None, data=None, json=None, cookies=None, headers=None, timeout=5, verify=True, stream=False, allow_redirects=True):
+        # 如果 headers 不为 None，则对其进行预处理
+        if headers:
+            headers = {k: self._clean_header_value(v) for k, v in headers.items()}
+
+        rsp = requests.post(url, params=params, data=data, json=json, cookies=cookies, headers=headers, timeout=timeout, verify=verify, stream=stream, allow_redirects=allow_redirects)
        rsp.encoding = 'utf-8'
        return rsp

@ -114,7 +122,7 @@ class Spider(metaclass=ABCMeta):
        return json.dumps(str, ensure_ascii=False)

    def getProxyUrl(self, local=True):
-        return f'{Proxy.getUrl(local)}?do=py'
+        return f'{Proxy.getUrl(self,local)}?do=py'

    def log(self, msg):
        if isinstance(msg, dict) or isinstance(msg, list):
@ -123,7 +131,7 @@ class Spider(metaclass=ABCMeta):
            print(f'{msg}')

    def getCache(self, key):
-        value = self.fetch(f'http://127.0.0.1:{Proxy.getPort()}/cache?do=get&key={key}', timeout=5).text
+        value = self.fetch(f'http://127.0.0.1:{Proxy.getPort(self)}/cache?do=get&key={key}', timeout=5).text
        if len(value) > 0:
            if value.startswith('{') and value.endswith('}') or value.startswith('[') and value.endswith(']'):
                value = json.loads(value)
@ -143,9 +151,9 @@ class Spider(metaclass=ABCMeta):
        if len(value) > 0:
            if type(value) == dict or type(value) == list:
                value = json.dumps(value, ensure_ascii=False)
-        r = self.post(f'http://127.0.0.1:{Proxy.getPort()}/cache?do=set&key={key}', data={"value": value}, timeout=5)
+        r = self.post(f'http://127.0.0.1:{Proxy.getPort(self)}/cache?do=set&key={key}', data={"value": value}, timeout=5)
        return 'succeed' if r.status_code == 200 else 'failed'

    def delCache(self, key):
-        r = self.fetch(f'http://127.0.0.1:{Proxy.getPort()}/cache?do=del&key={key}', timeout=5)
+        r = self.fetch(f'http://127.0.0.1:{Proxy.getPort(self)}/cache?do=del&key={key}', timeout=5)
        return 'succeed' if r.status_code == 200 else 'failed'