|
|
|
|
@ -4,12 +4,21 @@ from importlib.machinery import SourceFileLoader |
|
|
|
|
import json |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def create_file(file_path): |
|
|
|
|
if os.path.exists(file_path) is False: |
|
|
|
|
os.makedirs(file_path) |
|
|
|
|
def spider(cache, key, api): |
|
|
|
|
name = os.path.basename(api) |
|
|
|
|
path = cache + '/' + name |
|
|
|
|
downloadFile(path, api) |
|
|
|
|
return SourceFileLoader(name, path).load_module().Spider() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def downloadFile(name, api): |
|
|
|
|
if api.startswith('http'): |
|
|
|
|
writeFile(name, redirect(api).content) |
|
|
|
|
else: |
|
|
|
|
writeFile(name, str.encode(api)) |
|
|
|
|
|
|
|
|
|
def write_file(name, content): |
|
|
|
|
|
|
|
|
|
def writeFile(name, content): |
|
|
|
|
with open(name, 'wb') as f: |
|
|
|
|
f.write(content) |
|
|
|
|
|
|
|
|
|
@ -22,20 +31,6 @@ def redirect(url): |
|
|
|
|
return rsp |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def download_file(name, api): |
|
|
|
|
if api.startswith('http'): |
|
|
|
|
write_file(name, redirect(api).content) |
|
|
|
|
else: |
|
|
|
|
write_file(name, str.encode(api)) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def init_py(cache, key, api): |
|
|
|
|
name = os.path.basename(api) |
|
|
|
|
path = cache + '/' + name |
|
|
|
|
download_file(path, api) |
|
|
|
|
return SourceFileLoader(name, path).load_module().Spider() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def str2json(content): |
|
|
|
|
return json.loads(content) |
|
|
|
|
|
|
|
|
|
|