hg
/
BeartvSpider
mirror of https://github.com/FongMi/CatVodSpider.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'pr/40'

Browse Source
pull/69/head
FongMi 2 years ago
parent d4291d7d47 14b806271d
commit 4dad5a865e
2 changed files with 398 additions and 0 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 145
      app/src/main/java/com/github/catvod/spider/Douban.java
  2. 253
      app/src/main/java/com/github/catvod/spider/Xb6v.java

145
app/src/main/java/com/github/catvod/spider/Douban.java
Unescape View File

@ -0,0 +1,145 @@
package com.github.catvod.spider;
import android.content.Context;
import com.github.catvod.bean.Class;
import com.github.catvod.bean.Result;
import com.github.catvod.bean.Vod;
import com.github.catvod.crawler.Spider;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.net.OkHttp;
import com.github.catvod.utils.Utils;
import com.google.gson.JsonParser;
import org.json.JSONArray;
import org.json.JSONObject;
import java.net.URLEncoder;
import java.util.*;
public class Douban extends Spider {
private final String hostURL = "https://frodo.douban.com/api/v2";
private final String apikey = "?apikey=0ac44ae016490db2204ce0a042db2916";
private String extend;
private Map<String, String> getHeader() {
Map<String, String> header = new HashMap<>();
header.put("Host", "frodo.douban.com");
header.put("Connection", "Keep-Alive");
header.put("Referer", "https://servicewechat.com/wx2f9b06c1de1ccfca/84/page-frame.html");
header.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat");
return header;
}
@Override
public void init(Context context, String extend) throws Exception {
super.init(context, extend);
this.extend = extend;
}
@Override
public String homeContent(boolean filter) throws Exception {
List<Class> classes = new ArrayList<>();
List<String> typeIds = Arrays.asList("hot_gaia", "tv_hot", "show_hot", "movie", "tv", "rank_list_movie", "rank_list_tv");
List<String> typeNames = Arrays.asList("热门电影", "热播剧集", "热播综艺", "电影筛选", "电视筛选", "电影榜单", "电视剧榜单");
for (int i = 0; i < typeIds.size(); i++) classes.add(new Class(typeIds.get(i), typeNames.get(i)));
String recommendURL = "http://api.douban.com/api/v2/subject_collection/subject_real_time_hotest/items" + apikey;
JSONObject jsonObject = new JSONObject(OkHttp.string(recommendURL, getHeader()));
JSONArray items = jsonObject.optJSONArray("subject_collection_items");
return Result.string(classes, parseVodListFromJSONArray(items), filter ? JsonParser.parseString(OkHttp.string(extend)) : null);
}
@Override
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) throws Exception {
HashMap<String, String> ext = new HashMap<>();
if (extend != null && extend.size() > 0) {
ext.putAll(extend);
}
String sort = ext.get("sort") == null ? "T" : ext.get("sort");
String tags = URLEncoder.encode(getTags(ext));
int start = (Integer.parseInt(pg) - 1) * 20;
String cateURL;
String itemKey = "items";
switch (tid) {
case "hot_gaia":
sort = ext.get("sort") == null ? "recommend" : ext.get("sort");
String area = ext.get("area") == null ? "全部" : ext.get("area");
sort = sort + "&area=" + URLEncoder.encode(area);
cateURL = hostURL + "/movie/hot_gaia" + apikey + "&sort=" + sort + "&start=" + start + "&count=20";
break;
case "tv_hot":
String type = ext.get("type") == null ? "tv_hot" : ext.get("type");
cateURL = hostURL + "/subject_collection/" + type + "/items" + apikey + "&start=" + start + "&count=20";
itemKey = "subject_collection_items";
break;
case "show_hot":
String showType = ext.get("type") == null ? "show_hot" : ext.get("type");
cateURL = hostURL + "/subject_collection/" + showType + "/items" + apikey + "&start=" + start + "&count=20";
itemKey = "subject_collection_items";
break;
case "tv":
cateURL = hostURL + "/tv/recommend" + apikey + "&sort=" + sort + "&tags=" + tags + "&start=" + start + "&count=20";
break;
case "rank_list_movie":
String rankMovieType = ext.get("榜单") == null ? "movie_real_time_hotest" : ext.get("榜单");
cateURL = hostURL + "/subject_collection/" + rankMovieType + "/items" + apikey + "&start=" + start + "&count=20";
itemKey = "subject_collection_items";
break;
case "rank_list_tv":
String rankTVType = ext.get("榜单") == null ? "tv_real_time_hotest" : ext.get("榜单");
cateURL = hostURL + "/subject_collection/" + rankTVType + "/items" + apikey + "&start=" + start + "&count=20";
itemKey = "subject_collection_items";
break;
default:
cateURL = hostURL + "/movie/recommend" + apikey + "&sort=" + sort + "&tags=" + tags + "&start=" + start + "&count=20";
}
JSONObject jsonObject = new JSONObject(OkHttp.string(cateURL, getHeader()));
JSONArray items = jsonObject.getJSONArray(itemKey);
List<Vod> list = parseVodListFromJSONArray(items);
int page = Integer.parseInt(pg), count = Integer.MAX_VALUE, limit = 20, total = Integer.MAX_VALUE;
return Result.get().vod(list).page(page, count, limit, total).string();
}
private List<Vod> parseVodListFromJSONArray(JSONArray items) throws Exception {
List<Vod> list = new ArrayList<>();
for (int i = 0; i < items.length(); i++) {
JSONObject item = items.getJSONObject(i);
String vodId = "msearch:" + item.optString("id");
String name = item.optString("title");
String pic = getPic(item);
String remark = getRating(item);
list.add(new Vod(vodId, name, pic, remark));
}
return list;
}
private String getRating(JSONObject item) {
try {
return "评分：" + item.getJSONObject("rating").optString("value");
} catch (Exception e) {
SpiderDebug.log(e);
}
return "";
}
private String getPic(JSONObject item) {
try {
return item.getJSONObject("pic").optString("normal") + "@Referer=https://api.douban.com/@User-Agent=" + Utils.CHROME;
} catch (Exception e) {
SpiderDebug.log(e);
}
return "";
}
private String getTags(HashMap<String, String> ext) {
try {
StringBuilder tags = new StringBuilder();
for (String key : ext.keySet()) {
if (key.equals("sort")) continue;
tags.append(ext.get(key)).append(",");
}
return tags.substring(0, tags.lastIndexOf(","));
} catch (Exception e) {
// SpiderDebug.log(e);
}
return "";
}
}

253
app/src/main/java/com/github/catvod/spider/Xb6v.java
Unescape View File

@ -0,0 +1,253 @@
package com.github.catvod.spider;
import android.text.TextUtils;
import com.github.catvod.bean.Class;
import com.github.catvod.bean.Filter;
import com.github.catvod.bean.Result;
import com.github.catvod.bean.Vod;
import com.github.catvod.crawler.Spider;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.net.OkHttp;
import com.github.catvod.utils.Utils;
import okhttp3.FormBody;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* @author zhixc
* 新版6V电影网
*/
public class Xb6v extends Spider {
private final String siteURL = "http://www.xb6v.com";
private String nextSearchURLPrefix;
private String nextSearchURLSuffix;
private Map<String, String> getHeader() {
Map<String, String> header = new HashMap<>();
header.put("User-Agent", Utils.CHROME);
header.put("Referer", siteURL + "/");
return header;
}
private Map<String, String> getDetailHeader() {
Map<String, String> header = new HashMap<>();
header.put("User-Agent", Utils.CHROME);
return header;
}
@Override
public String homeContent(boolean filter) throws Exception {
List<Class> classes = new ArrayList<>();
String html = OkHttp.string(siteURL, getHeader());
Document doc = Jsoup.parse(html);
Elements elements = doc.select("#menus > li > a");
LinkedHashMap<String, List<Filter>> filters = new LinkedHashMap<>();
for (int i = 0; i < elements.size(); i++) {
if (i < 2 || i == elements.size() - 1) continue;
Element e = elements.get(i);
String typeId = e.attr("href");
String typeName = e.text();
if (typeName.equals("电视剧")) {
List<Filter.Value> values = new ArrayList<>();
values.add(new Filter.Value("不限", ""));
for (Element a : e.nextElementSibling().select("a")) {
values.add(new Filter.Value(a.text(), a.attr("href").replaceAll(typeId, "")));
}
List<Filter> filterList = new ArrayList<>();
filterList.add(new Filter("cateId", "类型", values));
filters.put(typeId, filterList);
}
classes.add(new Class(typeId, typeName));
}
return Result.string(classes, parseVodListFromDoc(doc), filters);
}
private List<Vod> parseVodListFromDoc(Document doc) {
Elements items = doc.select("#post_container .post_hover");
List<Vod> list = new ArrayList<>();
for (Element item : items) {
Element element = item.select("[class=zoom]").get(0);
String vodId = element.attr("href");
String name = element.attr("title").replaceAll("</?[^>]+>", "");
String pic = element.select("img").attr("src");
String remark = item.select("[rel=category tag]").text();
list.add(new Vod(vodId, name, pic, remark));
}
return list;
}
@Override
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) throws Exception {
HashMap<String, String> ext = new HashMap<>();
if (extend != null && extend.size() > 0) {
ext.putAll(extend);
}
String cateId = ext.get("cateId") == null ? "" : ext.get("cateId");
String cateURL = siteURL + tid + cateId;
if (!pg.equals("1")) cateURL += "index_" + pg + ".html";
String html = OkHttp.string(cateURL, getHeader());
Document doc = Jsoup.parse(html);
String href = doc.select(".pagination > a").last().attr("href");
int page = Integer.parseInt(pg);
int count = Integer.parseInt(getStrByRegex(Pattern.compile("index_(.*?).html"), href));
int limit = 18;
Elements items = doc.select("#post_container .post_hover");
int total = page == count ? (page - 1) * limit + items.size() : count * limit;
return Result.get().vod(parseVodListFromDoc(doc)).page(page, count, limit, total).string();
}
@Override
public String detailContent(List<String> ids) throws Exception {
String vodId = ids.get(0);
String detailURL = siteURL + vodId;
String html = OkHttp.string(detailURL, getDetailHeader());
Document doc = Jsoup.parse(html);
Elements sourceList = doc.select("#post_content");
String circuitName = "磁力线路";
Map<String, String> playMap = new LinkedHashMap<>();
int i = 0;
for (Element source : sourceList) {
Elements aList = source.select("table a");
List<String> vodItems = new ArrayList<>();
for (Element a : aList) {
String episodeURL = a.attr("href");
String episodeName = a.text();
if (!episodeURL.toLowerCase().startsWith("magnet")) continue;
vodItems.add(episodeName + "$" + episodeURL);
}
if (vodItems.size() > 0) {
i++;
playMap.put(circuitName + i, TextUtils.join("#", vodItems));
}
}
String partHTML = doc.select(".context").html();
String name = doc.select(".article_container > h1").text();
String pic = doc.select("#post_content img").attr("src");
String typeName = getStrByRegex(Pattern.compile("◎类　　别　(.*?)<br>"), partHTML);
if (typeName.equals("")) typeName = doc.select("[rel=category tag]").text();
String year = getStrByRegex(Pattern.compile("◎年　　代　(.*?)<br>"), partHTML);
if (year.equals("")) year = getStrByRegex(Pattern.compile("首播:(.*?)<br>"), partHTML);
String area = getStrByRegex(Pattern.compile("◎产　　地　(.*?)<br>"), partHTML);
if (area.equals("")) area = getStrByRegex(Pattern.compile("地区:(.*?)<br>"), partHTML);
String remark = getStrByRegex(Pattern.compile("◎上映日期　(.*?)<br>"), partHTML);
String actor = getActorOrDirector(Pattern.compile("◎演　　员　(.*?)</p>"), partHTML);
if (actor.equals("")) actor = getActorOrDirector(Pattern.compile("◎主　　演　(.*?)</p>"), partHTML);
if (actor.equals("")) actor = getActorOrDirector(Pattern.compile("主演:(.*?)<br>"), partHTML);
String director = getActorOrDirector(Pattern.compile("◎导　　演　(.*?)<br>"), partHTML);
if (director.equals("")) director = getActorOrDirector(Pattern.compile("导演:(.*?)<br>"), partHTML);
String description = getDescription(Pattern.compile("◎简　　介(.*?)<hr>", Pattern.CASE_INSENSITIVE | Pattern.DOTALL), partHTML);
if (description.equals("")) description = getDescription(Pattern.compile("简介(.*?)</p>", Pattern.CASE_INSENSITIVE | Pattern.DOTALL), partHTML);
Vod vod = new Vod();
vod.setVodId(ids.get(0));
vod.setVodName(name);
vod.setVodPic(pic);
vod.setTypeName(typeName);
vod.setVodYear(year);
vod.setVodArea(area);
vod.setVodRemarks(remark);
vod.setVodActor(actor);
vod.setVodDirector(director);
vod.setVodContent(description);
if (playMap.size() > 0) {
vod.setVodPlayFrom(TextUtils.join("$$$", playMap.keySet()));
vod.setVodPlayUrl(TextUtils.join("$$$", playMap.values()));
}
return Result.string(vod);
}
private String getStrByRegex(Pattern pattern, String str) {
try {
Matcher matcher = pattern.matcher(str);
if (matcher.find()) {
return matcher.group(1).trim();
}
} catch (Exception e) {
SpiderDebug.log(e);
}
return "";
}
private String getActorOrDirector(Pattern pattern, String str) {
return getStrByRegex(pattern, str)
.replaceAll("<br>", "")
.replaceAll("&nbsp;", "")
.replaceAll("&amp;", "")
.replaceAll("middot;", "・")
.replaceAll("　　　　　", ",")
.replaceAll("　　　　 　", ",");
}
private String getDescription(Pattern pattern, String str) {
return getStrByRegex(pattern, str)
.replaceAll("</?[^>]+>", "")
.replaceAll("\n", "")
.replaceAll("　　　　", "")
.replaceAll("&amp;", "")
.replaceAll("middot;", "・")
.replaceAll("ldquo;", "【")
.replaceAll("rdquo;", "】");
}
@Override
public String searchContent(String key, boolean quick) throws Exception {
return searchContent(key, quick, "1");
}
@Override
public String searchContent(String key, boolean quick, String pg) throws Exception {
String searchURL = siteURL + "/e/search/index.php";
String html;
if (pg.equals("1")) {
RequestBody formBody = new FormBody.Builder()
.add("show", "title")
.add("tempid", "1")
.add("tbname", "article")
.add("mid", "1")
.add("dopost", "search")
.add("submit", "")
.addEncoded("keyboard", key)
.build();
Request request = new Request.Builder()
.url(searchURL)
.addHeader("User-Agent", Utils.CHROME)
.addHeader("Origin", siteURL)
.addHeader("Referer", siteURL + "/")
.post(formBody)
.build();
Response response = OkHttp.client().newCall(request).execute();
if (response.body() == null) return "";
initNextSearchURL(response);
html = response.body().string();
response.close();
} else {
int page = Integer.parseInt(pg) - 1;
searchURL = nextSearchURLPrefix + page + nextSearchURLSuffix;
html = OkHttp.string(searchURL, getHeader());
}
return Result.string(parseVodListFromDoc(Jsoup.parse(html)));
}
private void initNextSearchURL(Response response) {
String[] split = String.valueOf(response.request().url()).split("\\?searchid=");
nextSearchURLPrefix = split[0] + "index.php?page=";
nextSearchURLSuffix = "&searchid=" + split[1];
}
@Override
public String playerContent(String flag, String id, List<String> vipFlags) throws Exception {
return Result.get().url(id).string();
}
}
Write Preview
Loading…
Cancel
Save