hg
/
BeartvSpider
mirror of https://github.com/FongMi/CatVodSpider.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use gson for xpath

Browse Source
pull/1/head
FongMi 4 years ago
parent ffdbe8b88a
commit ce3aa70ea3
3 changed files with 123 additions and 73 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 18
      app/src/main/java/com/github/catvod/bean/Result.java
  2. 67
      app/src/main/java/com/github/catvod/bean/Vod.java
  3. 111
      app/src/main/java/com/github/catvod/spider/XPath.java

18
app/src/main/java/com/github/catvod/bean/Result.java
Unescape View File

@ -16,6 +16,12 @@ public class Result {
private List<Vod> list;
@SerializedName("filters")
private JSONObject filters;
@SerializedName("parse")
private String parse;
@SerializedName("url")
private String url;
@SerializedName("ua")
private String ua;
public void setClasses(List<Class> classes) {
this.classes = classes;
@ -29,6 +35,18 @@ public class Result {
this.filters = filters;
}
public void setParse(String parse) {
this.parse = parse;
}
public void setUrl(String url) {
this.url = url;
}
public void setUa(String ua) {
this.ua = ua;
}
@NotNull
@Override
public String toString() {

67
app/src/main/java/com/github/catvod/bean/Vod.java
Unescape View File

@ -4,6 +4,8 @@ import com.google.gson.annotations.SerializedName;
public class Vod {
@SerializedName("type_name")
private String typeName;
@SerializedName("vod_id")
private String vodId;
@SerializedName("vod_name")
@ -12,6 +14,23 @@ public class Vod {
private String vodPic;
@SerializedName("vod_remarks")
private String vodRemarks;
@SerializedName("vod_year")
private String vodYear;
@SerializedName("vod_area")
private String vodArea;
@SerializedName("vod_actor")
private String vodActor;
@SerializedName("vod_director")
private String vodDirector;
@SerializedName("vod_content")
private String vodContent;
@SerializedName("vod_play_from")
private String vodPlayFrom;
@SerializedName("vod_play_url")
private String vodPlayUrl;
public Vod() {
}
public Vod(String vodId, String vodName, String vodPic, String vodRemarks) {
this.vodId = vodId;
@ -19,4 +38,52 @@ public class Vod {
this.vodPic = vodPic;
this.vodRemarks = vodRemarks;
}
public void setTypeName(String typeName) {
this.typeName = typeName;
}
public void setVodId(String vodId) {
this.vodId = vodId;
}
public void setVodName(String vodName) {
this.vodName = vodName;
}
public void setVodPic(String vodPic) {
this.vodPic = vodPic;
}
public void setVodRemarks(String vodRemarks) {
this.vodRemarks = vodRemarks;
}
public void setVodYear(String vodYear) {
this.vodYear = vodYear;
}
public void setVodArea(String vodArea) {
this.vodArea = vodArea;
}
public void setVodActor(String vodActor) {
this.vodActor = vodActor;
}
public void setVodDirector(String vodDirector) {
this.vodDirector = vodDirector;
}
public void setVodContent(String vodContent) {
this.vodContent = vodContent;
}
public void setVodPlayFrom(String vodPlayFrom) {
this.vodPlayFrom = vodPlayFrom;
}
public void setVodPlayUrl(String vodPlayUrl) {
this.vodPlayUrl = vodPlayUrl;
}
}

111
app/src/main/java/com/github/catvod/spider/XPath.java
Unescape View File

@ -10,7 +10,6 @@ import com.github.catvod.crawler.Spider;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.net.OkHttpUtil;
import com.github.catvod.utils.Misc;
import com.github.catvod.utils.Trans;
import com.github.catvod.xpath.XPathRule;
import org.json.JSONArray;
@ -20,6 +19,7 @@ import org.seimicrawler.xpath.JXNode;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Set;
@ -123,7 +123,7 @@ public class XPath extends Spider {
try {
fetchRule();
String webUrl = categoryUrl(tid, pg, filter, extend);
JSONArray videos = new JSONArray();
List<Vod> videos = new ArrayList<>();
JXDocument doc = JXDocument.create(fetch(webUrl));
List<JXNode> vodNodes = doc.selN(rule.getCateVodNode());
for (int i = 0; i < vodNodes.size(); i++) {
@ -143,27 +143,18 @@ public class XPath extends Spider {
SpiderDebug.log(e);
}
}
JSONObject v = new JSONObject();
v.put("vod_id", id);
v.put("vod_name", Trans.get(name));
v.put("vod_pic", pic);
v.put("vod_remarks", Trans.get(mark));
videos.put(v);
videos.add(new Vod(id, name, pic, mark));
}
JSONObject result = new JSONObject();
result.put("page", pg);
result.put("pagecount", Integer.MAX_VALUE);
result.put("limit", 90);
result.put("total", Integer.MAX_VALUE);
result.put("list", videos);
Result result = new Result();
result.setList(videos);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
return "";
}
return "";
}
protected void detailContentExt(String content, JSONObject vod) {
protected void detailContentExt(String content, Vod vod) {
}
@ -175,12 +166,9 @@ public class XPath extends Spider {
String webContent = fetch(webUrl);
JXDocument doc = JXDocument.create(webContent);
JXNode vodNode = doc.selNOne(rule.getDetailNode());
String cover = "", title = "", desc = "", category = "", area = "", year = "", remark = "", director = "", actor = "";
title = vodNode.selOne(rule.getDetailName()).asString().trim();
title = rule.getDetailNameR(title);
if (!rule.getDetailImg().isEmpty()) {
try {
cover = vodNode.selOne(rule.getDetailImg()).asString().trim();
@ -247,17 +235,17 @@ public class XPath extends Spider {
}
}
JSONObject vod = new JSONObject();
vod.put("vod_id", ids.get(0));
vod.put("vod_name", Trans.get(title));
vod.put("vod_pic", cover);
vod.put("type_name", Trans.get(category));
vod.put("vod_year", year);
vod.put("vod_area", Trans.get(area));
vod.put("vod_remarks", Trans.get(remark));
vod.put("vod_actor", Trans.get(actor));
vod.put("vod_director", Trans.get(director));
vod.put("vod_content", Trans.get(desc));
Vod vod = new Vod();
vod.setVodId(ids.get(0));
vod.setVodName(title);
vod.setVodPic(cover);
vod.setTypeName(category);
vod.setVodYear(year);
vod.setVodArea(area);
vod.setVodRemarks(remark);
vod.setVodActor(actor);
vod.setVodDirector(director);
vod.setVodContent(desc);
ArrayList<String> playFrom = new ArrayList<>();
List<JXNode> fromNodes = doc.selN(rule.getDetailFromNode());
@ -277,15 +265,13 @@ public class XPath extends Spider {
name = rule.getDetailUrlNameR(name);
String id = urlNodes.get(j).selOne(rule.getDetailUrlId()).asString().trim();
id = rule.getDetailUrlIdR(id);
vodItems.add(Trans.get(name) + "$" + id);
vodItems.add(name + "$" + id);
}
// 排除播放列表为空的播放源
if (vodItems.size() == 0 && playFrom.size() > i) {
playFrom.set(i, "");
}
playList.add(TextUtils.join("#", vodItems));
}
// 排除播放列表为空的播放源
for (int i = playFrom.size() - 1; i >= 0; i--) {
if (playFrom.get(i).isEmpty())
playFrom.remove(i);
@ -300,21 +286,16 @@ public class XPath extends Spider {
}
String vod_play_from = TextUtils.join("$$$", playFrom);
String vod_play_url = TextUtils.join("$$$", playList);
vod.put("vod_play_from", Trans.get(vod_play_from));
vod.put("vod_play_url", vod_play_url);
vod.setVodPlayFrom(vod_play_from);
vod.setVodPlayUrl(vod_play_url);
detailContentExt(webContent, vod);
JSONObject result = new JSONObject();
JSONArray list = new JSONArray();
list.put(vod);
result.put("list", list);
Result result = new Result();
result.setList(Arrays.asList(vod));
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e.getMessage());
for (StackTraceElement traceEl : e.getStackTrace()) {
SpiderDebug.log(traceEl.toString());
}
SpiderDebug.log(e);
return "";
}
return "";
}
@Override
@ -323,31 +304,26 @@ public class XPath extends Spider {
fetchRule();
String webUrl = rule.getPlayUrl().isEmpty() ? id : rule.getPlayUrl().replace("{playUrl}", id);
SpiderDebug.log(webUrl);
JSONObject result = new JSONObject();
result.put("parse", "1");
if (!rule.getPlayUa().isEmpty()) {
result.put("ua", rule.getPlayUa());
}
result.put("url", webUrl);
Result result = new Result();
result.setParse("1");
if (!rule.getPlayUa().isEmpty()) result.setUa(rule.getPlayUa());
result.setUrl(webUrl);
return result.toString();
} catch (Exception e) {
SpiderDebug.log(e);
return "";
}
return "";
}
@Override
public String searchContent(String key, boolean quick) {
try {
fetchRule();
if (rule.getSearchUrl().isEmpty()) {
return "";
}
if (rule.getSearchUrl().isEmpty()) return "";
String webUrl = rule.getSearchUrl().replace("{wd}", URLEncoder.encode(key));
String webContent = fetch(webUrl);
JSONObject result = new JSONObject();
JSONArray videos = new JSONArray();
// add maccms suggest search api support
Result result = new Result();
List<Vod> videos = new ArrayList<>();
if (rule.getSearchVodNode().startsWith("json:")) {
String[] node = rule.getSearchVodNode().substring(5).split(">");
JSONObject data = new JSONObject(webContent);
@ -365,12 +341,7 @@ public class XPath extends Spider {
pic = Misc.fixUrl(webUrl, pic);
String mark = vod.optString(rule.getSearchVodMark()).trim();
mark = rule.getSearchVodMarkR(mark);
JSONObject v = new JSONObject();
v.put("vod_id", id);
v.put("vod_name", Trans.get(name));
v.put("vod_pic", pic);
v.put("vod_remarks", Trans.get(mark));
videos.put(v);
videos.add(new Vod(id, name, pic, mark));
}
} else {
data = data.getJSONObject(node[i]);
@ -396,21 +367,15 @@ public class XPath extends Spider {
SpiderDebug.log(e);
}
}
JSONObject v = new JSONObject();
v.put("vod_id", id);
v.put("vod_name", Trans.get(name));
v.put("vod_pic", pic);
v.put("vod_remarks", Trans.get(mark));
videos.put(v);
videos.add(new Vod(id, name, pic, mark));
}
}
result.put("list", videos);
result.setList(videos);
return result.toString();
} catch (
Exception e) {
} catch (Exception e) {
SpiderDebug.log(e);
return "";
}
return "";
}
protected void fetchRule() {

Write Preview
Loading…
Cancel
Save