hg
/
BeartvSpider
mirror of https://github.com/FongMi/CatVodSpider.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support gson

Browse Source
pull/1/head
FongMi 4 years ago
parent 53c90c2bdd
commit ffdbe8b88a
7 changed files with 530 additions and 4 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 6
      app/proguard-rules.pro
  2. 28
      app/src/main/java/com/github/catvod/bean/Class.java
  3. 37
      app/src/main/java/com/github/catvod/bean/Result.java
  4. 22
      app/src/main/java/com/github/catvod/bean/Vod.java
  5. 439
      app/src/main/java/com/github/catvod/spider/XPath.java
  6. BIN
      jar/custom_spider.jar
  7. 2
      jar/custom_spider.jar.md5

6
app/proguard-rules.pro vendored
Unescape View File

@ -11,7 +11,7 @@
-keepattributes Signature
-keepattributes *Annotation*
-dontwarn sun.misc.**
-keep class com.google.gson.examples.android.model.** { <fields>; }
-keep class com.google.gson.**{*;}
-keep class * extends com.google.gson.TypeAdapter
-keep class * implements com.google.gson.TypeAdapterFactory
-keep class * implements com.google.gson.JsonSerializer
@ -21,5 +21,5 @@
-keep,allowobfuscation,allowshrinking class * extends com.google.gson.reflect.TypeToken
# OkHttp
-dontwarn okhttp3.**
-keep class okhttp3.** { *; }
-keep class okio.**{*;}
-keep class okhttp3.**{*;}

28
app/src/main/java/com/github/catvod/bean/Class.java
Unescape View File

@ -0,0 +1,28 @@
package com.github.catvod.bean;
import com.google.gson.annotations.SerializedName;
public class Class {
@SerializedName("type_id")
private String typeId;
@SerializedName("type_name")
private String typeName;
public Class(String typeId, String typeName) {
this.typeId = typeId;
this.typeName = typeName;
}
public String getTypeId() {
return typeId;
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (!(obj instanceof Class)) return false;
Class it = (Class) obj;
return getTypeId().equals(it.getTypeId());
}
}

37
app/src/main/java/com/github/catvod/bean/Result.java
Unescape View File

@ -0,0 +1,37 @@
package com.github.catvod.bean;
import com.google.gson.Gson;
import com.google.gson.annotations.SerializedName;
import org.jetbrains.annotations.NotNull;
import org.json.JSONObject;
import java.util.List;
public class Result {
@SerializedName("class")
private List<Class> classes;
@SerializedName("list")
private List<Vod> list;
@SerializedName("filters")
private JSONObject filters;
public void setClasses(List<Class> classes) {
this.classes = classes;
}
public void setList(List<Vod> list) {
this.list = list;
}
public void setFilters(JSONObject filters) {
this.filters = filters;
}
@NotNull
@Override
public String toString() {
return new Gson().toJson(this);
}
}

22
app/src/main/java/com/github/catvod/bean/Vod.java
Unescape View File

@ -0,0 +1,22 @@
package com.github.catvod.bean;
import com.google.gson.annotations.SerializedName;
public class Vod {
@SerializedName("vod_id")
private String vodId;
@SerializedName("vod_name")
private String vodName;
@SerializedName("vod_pic")
private String vodPic;
@SerializedName("vod_remarks")
private String vodRemarks;
public Vod(String vodId, String vodName, String vodPic, String vodRemarks) {
this.vodId = vodId;
this.vodName = vodName;
this.vodPic = vodPic;
this.vodRemarks = vodRemarks;
}
}

439
app/src/main/java/com/github/catvod/spider/XPath.java
Unescape View File

@ -0,0 +1,439 @@
package com.github.catvod.spider;
import android.content.Context;
import android.text.TextUtils;
import com.github.catvod.bean.Class;
import com.github.catvod.bean.Result;
import com.github.catvod.bean.Vod;
import com.github.catvod.crawler.Spider;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.net.OkHttpUtil;
import com.github.catvod.utils.Misc;
import com.github.catvod.utils.Trans;
import com.github.catvod.xpath.XPathRule;
import org.json.JSONArray;
import org.json.JSONObject;
import org.seimicrawler.xpath.JXDocument;
import org.seimicrawler.xpath.JXNode;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Set;
public class XPath extends Spider {
protected String ext = null;
protected XPathRule rule = null;
protected HashMap<String, String> getHeaders() {
HashMap<String, String> headers = new HashMap<>();
headers.put("User-Agent", rule.getUa().isEmpty() ? Misc.CHROME : rule.getUa());
return headers;
}
@Override
public void init(Context context, String extend) {
super.init(context, extend);
this.ext = extend;
}
@Override
public String homeContent(boolean filter) {
try {
fetchRule();
Result result = new Result();
List<Vod> videos = new ArrayList<>();
List<Class> classes = new ArrayList<>();
if (rule.getCateManual().size() > 0) {
Set<String> keys = rule.getCateManual().keySet();
for (String k : keys) classes.add(new Class(k, rule.getCateManual().get(k)));
}
try {
String webUrl = rule.getHomeUrl();
JXDocument doc = JXDocument.create(fetch(webUrl));
if (rule.getCateManual().size() == 0) {
List<JXNode> navNodes = doc.selN(rule.getCateNode());
for (int i = 0; i < navNodes.size(); i++) {
String name = navNodes.get(i).selOne(rule.getCateName()).asString().trim();
name = rule.getCateNameR(name);
String id = navNodes.get(i).selOne(rule.getCateId()).asString().trim();
id = rule.getCateIdR(id);
classes.add(new Class(id, name));
}
}
if (!rule.getHomeVodNode().isEmpty()) {
try {
List<JXNode> vodNodes = doc.selN(rule.getHomeVodNode());
for (int i = 0; i < vodNodes.size(); i++) {
String name = vodNodes.get(i).selOne(rule.getHomeVodName()).asString().trim();
name = rule.getHomeVodNameR(name);
String id = vodNodes.get(i).selOne(rule.getHomeVodId()).asString().trim();
id = rule.getHomeVodIdR(id);
String pic = vodNodes.get(i).selOne(rule.getHomeVodImg()).asString().trim();
pic = rule.getHomeVodImgR(pic);
pic = Misc.fixUrl(webUrl, pic);
String mark = "";
if (!rule.getHomeVodMark().isEmpty()) {
try {
mark = vodNodes.get(i).selOne(rule.getHomeVodMark()).asString().trim();
mark = rule.getHomeVodMarkR(mark);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
videos.add(new Vod(id, name, pic, mark));
}
} catch (Exception e) {
SpiderDebug.log(e);
}
}
} catch (Exception e) {
SpiderDebug.log(e);
}
result.setList(videos);
result.setClasses(classes);
result.setFilters(filter && rule.getFilter() != null ? rule.getFilter() : null);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
return "";
}
}
@Override
public String homeVideoContent() {
try {
fetchRule();
} catch (Exception e) {
SpiderDebug.log(e);
}
return "";
}
protected String categoryUrl(String tid, String pg, boolean filter, HashMap<String, String> extend) {
return rule.getCateUrl().replace("{cateId}", tid).replace("{catePg}", pg);
}
@Override
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) {
try {
fetchRule();
String webUrl = categoryUrl(tid, pg, filter, extend);
JSONArray videos = new JSONArray();
JXDocument doc = JXDocument.create(fetch(webUrl));
List<JXNode> vodNodes = doc.selN(rule.getCateVodNode());
for (int i = 0; i < vodNodes.size(); i++) {
String name = vodNodes.get(i).selOne(rule.getCateVodName()).asString().trim();
name = rule.getCateVodNameR(name);
String id = vodNodes.get(i).selOne(rule.getCateVodId()).asString().trim();
id = rule.getCateVodIdR(id);
String pic = vodNodes.get(i).selOne(rule.getCateVodImg()).asString().trim();
pic = rule.getCateVodImgR(pic);
pic = Misc.fixUrl(webUrl, pic);
String mark = "";
if (!rule.getCateVodMark().isEmpty()) {
try {
mark = vodNodes.get(i).selOne(rule.getCateVodMark()).asString().trim();
mark = rule.getCateVodMarkR(mark);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
JSONObject v = new JSONObject();
v.put("vod_id", id);
v.put("vod_name", Trans.get(name));
v.put("vod_pic", pic);
v.put("vod_remarks", Trans.get(mark));
videos.put(v);
}
JSONObject result = new JSONObject();
result.put("page", pg);
result.put("pagecount", Integer.MAX_VALUE);
result.put("limit", 90);
result.put("total", Integer.MAX_VALUE);
result.put("list", videos);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
}
return "";
}
protected void detailContentExt(String content, JSONObject vod) {
}
@Override
public String detailContent(List<String> ids) {
try {
fetchRule();
String webUrl = rule.getDetailUrl().replace("{vid}", ids.get(0));
String webContent = fetch(webUrl);
JXDocument doc = JXDocument.create(webContent);
JXNode vodNode = doc.selNOne(rule.getDetailNode());
String cover = "", title = "", desc = "", category = "", area = "", year = "", remark = "", director = "", actor = "";
title = vodNode.selOne(rule.getDetailName()).asString().trim();
title = rule.getDetailNameR(title);
if (!rule.getDetailImg().isEmpty()) {
try {
cover = vodNode.selOne(rule.getDetailImg()).asString().trim();
cover = rule.getDetailImgR(cover);
cover = Misc.fixUrl(webUrl, cover);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailCate().isEmpty()) {
try {
category = vodNode.selOne(rule.getDetailCate()).asString().trim();
category = rule.getDetailCateR(category);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailYear().isEmpty()) {
try {
year = vodNode.selOne(rule.getDetailYear()).asString().trim();
year = rule.getDetailYearR(year);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailArea().isEmpty()) {
try {
area = vodNode.selOne(rule.getDetailArea()).asString().trim();
area = rule.getDetailAreaR(area);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailMark().isEmpty()) {
try {
remark = vodNode.selOne(rule.getDetailMark()).asString().trim();
remark = rule.getDetailMarkR(remark);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailActor().isEmpty()) {
try {
actor = vodNode.selOne(rule.getDetailActor()).asString().trim();
actor = rule.getDetailActorR(actor);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailDirector().isEmpty()) {
try {
director = vodNode.selOne(rule.getDetailDirector()).asString().trim();
director = rule.getDetailDirectorR(director);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
if (!rule.getDetailDesc().isEmpty()) {
try {
desc = vodNode.selOne(rule.getDetailDesc()).asString().trim();
desc = rule.getDetailDescR(desc);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
JSONObject vod = new JSONObject();
vod.put("vod_id", ids.get(0));
vod.put("vod_name", Trans.get(title));
vod.put("vod_pic", cover);
vod.put("type_name", Trans.get(category));
vod.put("vod_year", year);
vod.put("vod_area", Trans.get(area));
vod.put("vod_remarks", Trans.get(remark));
vod.put("vod_actor", Trans.get(actor));
vod.put("vod_director", Trans.get(director));
vod.put("vod_content", Trans.get(desc));
ArrayList<String> playFrom = new ArrayList<>();
List<JXNode> fromNodes = doc.selN(rule.getDetailFromNode());
for (int i = 0; i < fromNodes.size(); i++) {
String name = fromNodes.get(i).selOne(rule.getDetailFromName()).asString().trim();
name = rule.getDetailFromNameR(name);
playFrom.add(name);
}
ArrayList<String> playList = new ArrayList<>();
List<JXNode> urlListNodes = doc.selN(rule.getDetailUrlNode());
for (int i = 0; i < urlListNodes.size(); i++) {
List<JXNode> urlNodes = urlListNodes.get(i).sel(rule.getDetailUrlSubNode());
List<String> vodItems = new ArrayList<>();
for (int j = 0; j < urlNodes.size(); j++) {
String name = urlNodes.get(j).selOne(rule.getDetailUrlName()).asString().trim();
name = rule.getDetailUrlNameR(name);
String id = urlNodes.get(j).selOne(rule.getDetailUrlId()).asString().trim();
id = rule.getDetailUrlIdR(id);
vodItems.add(Trans.get(name) + "$" + id);
}
// 排除播放列表为空的播放源
if (vodItems.size() == 0 && playFrom.size() > i) {
playFrom.set(i, "");
}
playList.add(TextUtils.join("#", vodItems));
}
// 排除播放列表为空的播放源
for (int i = playFrom.size() - 1; i >= 0; i--) {
if (playFrom.get(i).isEmpty())
playFrom.remove(i);
}
for (int i = playList.size() - 1; i >= 0; i--) {
if (playList.get(i).isEmpty())
playList.remove(i);
}
for (int i = playList.size() - 1; i >= 0; i--) {
if (i >= playFrom.size())
playList.remove(i);
}
String vod_play_from = TextUtils.join("$$$", playFrom);
String vod_play_url = TextUtils.join("$$$", playList);
vod.put("vod_play_from", Trans.get(vod_play_from));
vod.put("vod_play_url", vod_play_url);
detailContentExt(webContent, vod);
JSONObject result = new JSONObject();
JSONArray list = new JSONArray();
list.put(vod);
result.put("list", list);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e.getMessage());
for (StackTraceElement traceEl : e.getStackTrace()) {
SpiderDebug.log(traceEl.toString());
}
}
return "";
}
@Override
public String playerContent(String flag, String id, List<String> vipFlags) {
try {
fetchRule();
String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id);
SpiderDebug.log(webUrl);
JSONObject result = new JSONObject();
result.put("parse", "1");
if (!rule.getPlayUa().isEmpty()) {
result.put("ua", rule.getPlayUa());
}
result.put("url", webUrl);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
}
return "";
}
@Override
public String searchContent(String key, boolean quick) {
try {
fetchRule();
if (rule.getSearchUrl().isEmpty()) {
return "";
}
String webUrl = rule.getSearchUrl().replace("{wd}", URLEncoder.encode(key));
String webContent = fetch(webUrl);
JSONObject result = new JSONObject();
JSONArray videos = new JSONArray();
// add maccms suggest search api support
if (rule.getSearchVodNode().startsWith("json:")) {
String[] node = rule.getSearchVodNode().substring(5).split(">");
JSONObject data = new JSONObject(webContent);
for (int i = 0; i < node.length; i++) {
if (i == node.length - 1) {
JSONArray vodArray = data.getJSONArray(node[i]);
for (int j = 0; j < vodArray.length(); j++) {
JSONObject vod = vodArray.getJSONObject(j);
String name = vod.optString(rule.getSearchVodName()).trim();
name = rule.getSearchVodNameR(name);
String id = vod.optString(rule.getSearchVodId()).trim();
id = rule.getSearchVodIdR(id);
String pic = vod.optString(rule.getSearchVodImg()).trim();
pic = rule.getSearchVodImgR(pic);
pic = Misc.fixUrl(webUrl, pic);
String mark = vod.optString(rule.getSearchVodMark()).trim();
mark = rule.getSearchVodMarkR(mark);
JSONObject v = new JSONObject();
v.put("vod_id", id);
v.put("vod_name", Trans.get(name));
v.put("vod_pic", pic);
v.put("vod_remarks", Trans.get(mark));
videos.put(v);
}
} else {
data = data.getJSONObject(node[i]);
}
}
} else {
JXDocument doc = JXDocument.create(webContent);
List<JXNode> vodNodes = doc.selN(rule.getSearchVodNode());
for (int i = 0; i < vodNodes.size(); i++) {
String name = vodNodes.get(i).selOne(rule.getSearchVodName()).asString().trim();
name = rule.getSearchVodNameR(name);
String id = vodNodes.get(i).selOne(rule.getSearchVodId()).asString().trim();
id = rule.getSearchVodIdR(id);
String pic = vodNodes.get(i).selOne(rule.getSearchVodImg()).asString().trim();
pic = rule.getSearchVodImgR(pic);
pic = Misc.fixUrl(webUrl, pic);
String mark = "";
if (!rule.getCateVodMark().isEmpty()) {
try {
mark = vodNodes.get(i).selOne(rule.getSearchVodMark()).asString().trim();
mark = rule.getSearchVodMarkR(mark);
} catch (Exception e) {
SpiderDebug.log(e);
}
}
JSONObject v = new JSONObject();
v.put("vod_id", id);
v.put("vod_name", Trans.get(name));
v.put("vod_pic", pic);
v.put("vod_remarks", Trans.get(mark));
videos.put(v);
}
}
result.put("list", videos);
return result.toString();
} catch (
Exception e) {
SpiderDebug.log(e);
}
return "";
}
protected void fetchRule() {
if (rule == null) {
if (ext != null) {
if (ext.startsWith("http")) {
String json = OkHttpUtil.string(ext);
rule = XPathRule.fromJson(json);
loadRuleExt(json);
} else {
rule = XPathRule.fromJson(ext);
loadRuleExt(ext);
}
}
}
}
protected void loadRuleExt(String json) {
}
protected String fetch(String webUrl) {
SpiderDebug.log(webUrl);
return OkHttpUtil.string(webUrl, getHeaders());
}
}

BIN
jar/custom_spider.jar
View File

Binary file not shown.

2
jar/custom_spider.jar.md5
Unescape View File

@ -1 +1 @@
e79ee1c89df18b03332b1f342ca44d2f
1012798d28ebd672ba6007e6d6fab951

Write Preview
Loading…
Cancel
Save