Merge pull request #90 from LuckySe7ens/main

add living spider
pull/91/head
FongMi 11 months ago committed by GitHub
commit 58db696fa5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 220
      app/src/main/java/com/github/catvod/spider/Living.java
  2. 545
      app/src/main/java/com/github/catvod/utils/LZString.java

@ -0,0 +1,220 @@
package com.github.catvod.spider;
import android.content.Context;
import android.text.TextUtils;
import android.util.Base64;
import com.github.catvod.bean.Class;
import com.github.catvod.bean.Filter;
import com.github.catvod.bean.Result;
import com.github.catvod.bean.Vod;
import com.github.catvod.crawler.Spider;
import com.github.catvod.crawler.SpiderDebug;
import com.github.catvod.net.OkHttp;
import com.github.catvod.utils.LZString;
import com.github.catvod.utils.Util;
import org.json.JSONException;
import org.json.JSONObject;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* @author leospring
* 聚合直播
*/
public class Living extends Spider {
private String host = "https://lemonlive.deno.dev";
private String cookie = "";
@Override
public void init(Context context, String extend) throws Exception {
if (!TextUtils.isEmpty(extend)) {
host = extend;
}
}
@Override
public String homeContent(boolean filter) throws Exception {
List<Class> classList = new ArrayList<>();
LinkedHashMap<String, List<Filter>> filters = new LinkedHashMap<>();
classList.add(new Class("huya", "虎牙"));
classList.add(new Class("douyu", "斗鱼"));
classList.add(new Class("douyin", "抖音"));
classList.add(new Class("bilibili", "哔哩哔哩"));
classList.add(new Class("cc", "网易CC"));
List<Filter> huyaFilterList = new ArrayList<>();
List<Filter.Value> huyaVals = new ArrayList<>();
huyaVals.add(new Filter.Value("网游", "1"));
huyaVals.add(new Filter.Value("手游", "3"));
huyaVals.add(new Filter.Value("娱乐", "8"));
huyaVals.add(new Filter.Value("单机", "2"));
huyaFilterList.add(new Filter("type", "分类", huyaVals));
filters.put("huya", huyaFilterList);
List<Filter> douyuFilterList = new ArrayList<>();
List<Filter.Value> douyuVals = new ArrayList<>();
douyuVals.add(new Filter.Value("网游竞技", "PCgame"));
douyuVals.add(new Filter.Value("单机热游", "djry"));
douyuVals.add(new Filter.Value("手游休闲", "syxx"));
douyuVals.add(new Filter.Value("娱乐天地", "yl"));
douyuVals.add(new Filter.Value("颜值", "yz"));
douyuVals.add(new Filter.Value("科技文化", "kjwh"));
douyuVals.add(new Filter.Value("语言互动", "yp"));
douyuFilterList.add(new Filter("type", "分类", douyuVals));
filters.put("douyu", douyuFilterList);
List<Filter> douyinFilterList = new ArrayList<>();
List<Filter.Value> douyinVals = new ArrayList<>();
douyinVals.add(new Filter.Value("竞技游戏", "2"));
douyinVals.add(new Filter.Value("射击游戏", "1"));
douyinVals.add(new Filter.Value("单机游戏", "3"));
douyinVals.add(new Filter.Value("棋牌游戏", "4"));
douyinVals.add(new Filter.Value("休闲益智", "5"));
douyinVals.add(new Filter.Value("角色扮演", "6"));
douyinVals.add(new Filter.Value("策略卡牌", "7"));
douyinVals.add(new Filter.Value("娱乐天地", "10000"));
douyinVals.add(new Filter.Value("科技文化", "10001"));
douyinFilterList.add(new Filter("type", "分类", douyinVals));
filters.put("douyin", douyinFilterList);
List<Filter> biliFilterList = new ArrayList<>();
List<Filter.Value> biliVals = new ArrayList<>();
biliVals.add(new Filter.Value("网游", "2"));
biliVals.add(new Filter.Value("手游", "3"));
biliVals.add(new Filter.Value("单机游戏", "6"));
biliVals.add(new Filter.Value("娱乐", "1"));
biliVals.add(new Filter.Value("电台", "5"));
biliVals.add(new Filter.Value("虚拟主播", "9"));
biliVals.add(new Filter.Value("聊天室", "14"));
biliVals.add(new Filter.Value("生活", "10"));
biliVals.add(new Filter.Value("知识", "11"));
biliVals.add(new Filter.Value("赛事", "13"));
biliVals.add(new Filter.Value("互动玩法", "15"));
biliFilterList.add(new Filter("type", "分类", biliVals));
filters.put("bilibili", biliFilterList);
List<Filter> ccFilterList = new ArrayList<>();
List<Filter.Value> ccVals = new ArrayList<>();
ccVals.add(new Filter.Value("网游", "1"));
ccVals.add(new Filter.Value("手游", "2"));
ccVals.add(new Filter.Value("竞技", "4"));
ccVals.add(new Filter.Value("综艺", "5"));
ccFilterList.add(new Filter("type", "分类", ccVals));
filters.put("cc", ccFilterList);
return Result.string(classList, filters);
}
@Override
public String categoryContent(String tid, String pg, boolean filter, HashMap<String, String> extend) throws Exception {
if (!tid.contains("_")) {
String url = host + "/api/" + tid + "/getCategories";
JSONObject json = request(url);
String type = extend.get("type");
if (TextUtils.isEmpty(type)) type = json.optJSONArray("data").optJSONObject(0).optString("id");
List<Vod> vodList = new ArrayList<>();
for (int i = 0; i < json.optJSONArray("data").length(); i++) {
JSONObject data = json.optJSONArray("data").optJSONObject(i);
if (type.equals(data.optString("id"))) {
for (int j = 0; j < data.optJSONArray("list").length(); j++) {
JSONObject item = data.optJSONArray("list").optJSONObject(j);
vodList.add(new Vod(tid + "_" + item.optString("cid"), item.optString("name"), item.optString("pic"),data.optString("name"), true));
}
}
}
return Result.string(vodList);
} else {
String[] split = tid.split("_");
String url = host + "/api/" + split[0] + "/getCategoryRooms?id=" + split[1] + "&pid=" + (split[0].equals("bilibili") ? "2":"1") + "&page=" + pg;
if (!TextUtils.isEmpty(cookie)) url = url + "&cookie=" + URLDecoder.decode(cookie, "UTF-8");
JSONObject json = request(url);
if (!TextUtils.isEmpty(json.optJSONObject("data").optString("cookie"))) {
cookie = json.optJSONObject("data").optString("cookie");
}
List<Vod> vodList = new ArrayList<>();
for (int i = 0; i < json.optJSONObject("data").optJSONArray("list").length(); i++) {
JSONObject data = json.optJSONObject("data").optJSONArray("list").optJSONObject(i);
vodList.add(new Vod(split[0] + "_" + data.optString("roomId"), data.optString("title"), data.optString("cover"), data.optString("nickname")));
}
return Result.string(vodList);
}
}
@Override
public String detailContent(List<String> ids) throws Exception {
String[] split = ids.get(0).split("_");
String url = host + "/api/" + split[0] + "/getRoomDetail?id=" + split[1];
JSONObject json = request(url).optJSONObject("data");
Vod vod = new Vod();
vod.setVodId(json.optString("roomId"));
vod.setVodName(json.optString("title"));
vod.setVodArea(json.optString("online"));
vod.setVodDirector(json.optString("siteId"));
vod.setVodActor(json.optString("nickname"));
vod.setVodPic(json.optString("cover"));
vod.setVodContent(json.optString("url"));
vod.setTypeName(json.optString("category"));
JSONObject info = json.optJSONObject("info");
String params = "";
if (info != null) {
params = getHuyaParam(info.optString("name"), info.optString("code"));
}
List<String> fromList = new ArrayList<>();
List<String> playList = new ArrayList<>();
for (int i = 0; i < json.optJSONArray("stream").length(); i++) {
JSONObject data = json.optJSONArray("stream").optJSONObject(i);
fromList.add(data.optString("name"));
List<String> nameUrls = new ArrayList<>();
for (int j = 0; j < data.optJSONArray("lines").length(); j++) {
JSONObject urls = data.optJSONArray("lines").optJSONObject(j);
String playUrl = urls.optString("url") + params;
nameUrls.add(urls.optString("name") + "$" + playUrl);
}
playList.add(TextUtils.join("#", nameUrls));
}
vod.setVodPlayFrom(TextUtils.join("$$$", fromList));
vod.setVodPlayUrl(TextUtils.join("$$$", playList));
return Result.string(vod);
}
@Override
public String playerContent(String flag, String id, List<String> vipFlags) throws Exception {
String url = id;
if (!url.startsWith("http")) url = "https:" + url;
return Result.get().url(url).toString();
}
private String getHuyaParam(String name, String code) throws UnsupportedEncodingException {
String N = "1063681129617";
long currentTimeMillis = System.currentTimeMillis();
String i = String.valueOf(currentTimeMillis % 10000000000L * 1000 + (long)(Math.random() * 4294967295L));
String r = code.split("fs=")[1].split("&")[0];
String s = Long.toHexString((currentTimeMillis / 1000) | 21600);
String f = String.valueOf(currentTimeMillis + Long.parseLong(N));
String fmPart = code.split("fm=")[1].split("&")[0];
String c = new String(Base64.decode(URLDecoder.decode(fmPart, "UTF-8"), Base64.NO_WRAP)).split("_")[0];
String u = Util.MD5(f + "|tars_mp|102");
return String.format("&wsSecret=%s&uuid=%s&wsTime=%s&uid=%s&seqid=%s&fs=%s&ctype=tars_mp&t=102&ver=1&sv=2401310321",
Util.MD5(c + "_" + N + "_" + name + "_" + u + "_" + s),
i,
s,
N,
f,
r);
}
private JSONObject request(String url) throws JSONException {
String str = OkHttp.string(url, Map.of("sec-fetch-site", "same-origin"));
String result = LZString.decompressFromBase64(str.replaceAll(" ", ""));
//SpiderDebug.log("result==" + result);
return new JSONObject(result);
}
}

@ -0,0 +1,545 @@
package com.github.catvod.utils;
/*
* LZString4Java By Rufus Huang
* https://github.com/rufushuang/lz-string4java
* MIT License
*
* Port from original JavaScript version by pieroxy
* https://github.com/pieroxy/lz-string
*/
import java.util.*;
public class LZString {
private static char[] keyStrBase64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".toCharArray();
private static char[] keyStrUriSafe = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-$".toCharArray();
private static Map<char[], Map<Character, Integer>> baseReverseDic = new HashMap<char[], Map<Character, Integer>>();
private static char getBaseValue(char[] alphabet, Character character) {
Map<Character, Integer> map = baseReverseDic.get(alphabet);
if (map == null) {
map = new HashMap<Character, Integer>();
baseReverseDic.put(alphabet, map);
for (int i = 0; i < alphabet.length; i++) {
map.put(alphabet[i], i);
}
}
return (char) map.get(character).intValue();
}
public static String compressToBase64(String input) {
if (input == null)
return "";
String res = LZString._compress(input, 6, new CompressFunctionWrapper() {
@Override
public char doFunc(int a) {
return keyStrBase64[a];
}
});
switch (res.length() % 4) { // To produce valid Base64
default: // When could this happen ?
case 0:
return res;
case 1:
return res + "===";
case 2:
return res + "==";
case 3:
return res + "=";
}
}
public static String decompressFromBase64(final String inputStr) {
if (inputStr == null)
return "";
if (inputStr.equals(""))
return null;
return LZString._decompress(inputStr.length(), 32, new DecompressFunctionWrapper() {
@Override
public char doFunc(int index) {
return getBaseValue(keyStrBase64, inputStr.charAt(index));
}
});
}
public static String compressToUTF16(String input) {
if (input == null)
return "";
return LZString._compress(input, 15, new CompressFunctionWrapper() {
@Override
public char doFunc(int a) {
return fc(a + 32);
}
}) + " ";
}
public static String decompressFromUTF16(final String compressedStr) {
if (compressedStr == null)
return "";
if (compressedStr.isEmpty())
return null;
return LZString._decompress(compressedStr.length(), 16384, new DecompressFunctionWrapper() {
@Override
public char doFunc(int index) {
return (char) (compressedStr.charAt(index) - 32);
}
});
}
//TODO: java has no Uint8Array type, what can we do?
public static String compressToEncodedURIComponent(String input) {
if (input == null)
return "";
return LZString._compress(input, 6, new CompressFunctionWrapper() {
@Override
public char doFunc(int a) {
return keyStrUriSafe[a];
}
});
}
public static String decompressFromEncodedURIComponent(String inputStr) {
if (inputStr == null) return "";
if (inputStr.isEmpty()) return null;
final String urlEncodedInputStr = inputStr.replace(' ', '+');
return LZString._decompress(urlEncodedInputStr.length(), 32, new DecompressFunctionWrapper() {
@Override
public char doFunc(int index) {
return getBaseValue(keyStrUriSafe, urlEncodedInputStr.charAt(index));
}
});
}
private static abstract class CompressFunctionWrapper {
public abstract char doFunc(int i);
}
public static String compress(String uncompressed) {
return LZString._compress(uncompressed, 16, new CompressFunctionWrapper() {
@Override
public char doFunc(int a) {
return fc(a);
}
});
}
private static String _compress(String uncompressedStr, int bitsPerChar, CompressFunctionWrapper getCharFromInt) {
if (uncompressedStr == null) return "";
int i, value;
Map<String, Integer> context_dictionary = new HashMap<String, Integer>();
Set<String> context_dictionaryToCreate = new HashSet<String>();
String context_c = "";
String context_wc = "";
String context_w = "";
int context_enlargeIn = 2; // Compensate for the first entry which should not count
int context_dictSize = 3;
int context_numBits = 2;
StringBuilder context_data = new StringBuilder(uncompressedStr.length() / 3);
int context_data_val = 0;
int context_data_position = 0;
int ii;
for (ii = 0; ii < uncompressedStr.length(); ii += 1) {
context_c = String.valueOf(uncompressedStr.charAt(ii));
if (!context_dictionary.containsKey(context_c)) {
context_dictionary.put(context_c, context_dictSize++);
context_dictionaryToCreate.add(context_c);
}
context_wc = context_w + context_c;
if (context_dictionary.containsKey(context_wc)) {
context_w = context_wc;
} else {
if (context_dictionaryToCreate.contains(context_w)) {
if (context_w.charAt(0) < 256) {
for (i = 0; i < context_numBits; i++) {
context_data_val = (context_data_val << 1);
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.append(getCharFromInt.doFunc(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
}
value = context_w.charAt(0);
for (i = 0; i < 8; i++) {
context_data_val = (context_data_val << 1) | (value & 1);
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.append(getCharFromInt.doFunc(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
value = value >> 1;
}
} else {
value = 1;
for (i = 0; i < context_numBits; i++) {
context_data_val = (context_data_val << 1) | value;
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.append(getCharFromInt.doFunc(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
value = 0;
}
value = context_w.charAt(0);
for (i = 0; i < 16; i++) {
context_data_val = (context_data_val << 1) | (value & 1);
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.append(getCharFromInt.doFunc(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
value = value >> 1;
}
}
context_enlargeIn--;
if (context_enlargeIn == 0) {
context_enlargeIn = powerOf2(context_numBits);
context_numBits++;
}
context_dictionaryToCreate.remove(context_w);
} else {
value = context_dictionary.get(context_w);
for (i = 0; i < context_numBits; i++) {
context_data_val = (context_data_val << 1) | (value & 1);
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.append(getCharFromInt.doFunc(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
value = value >> 1;
}
}
context_enlargeIn--;
if (context_enlargeIn == 0) {
context_enlargeIn = powerOf2(context_numBits);
context_numBits++;
}
// Add wc to the dictionary.
context_dictionary.put(context_wc, context_dictSize++);
context_w = context_c;
}
}
// Output the code for w.
if (!context_w.isEmpty()) {
if (context_dictionaryToCreate.contains(context_w)) {
if (context_w.charAt(0) < 256) {
for (i = 0; i < context_numBits; i++) {
context_data_val = (context_data_val << 1);
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.append(getCharFromInt.doFunc(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
}
value = context_w.charAt(0);
for (i = 0; i < 8; i++) {
context_data_val = (context_data_val << 1) | (value & 1);
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.append(getCharFromInt.doFunc(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
value = value >> 1;
}
} else {
value = 1;
for (i = 0; i < context_numBits; i++) {
context_data_val = (context_data_val << 1) | value;
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.append(getCharFromInt.doFunc(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
value = 0;
}
value = context_w.charAt(0);
for (i = 0; i < 16; i++) {
context_data_val = (context_data_val << 1) | (value & 1);
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.append(getCharFromInt.doFunc(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
value = value >> 1;
}
}
context_enlargeIn--;
if (context_enlargeIn == 0) {
context_enlargeIn = powerOf2(context_numBits);
context_numBits++;
}
context_dictionaryToCreate.remove(context_w);
} else {
value = context_dictionary.get(context_w);
for (i = 0; i < context_numBits; i++) {
context_data_val = (context_data_val << 1) | (value & 1);
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.append(getCharFromInt.doFunc(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
value = value >> 1;
}
}
context_enlargeIn--;
if (context_enlargeIn == 0) {
context_enlargeIn = powerOf2(context_numBits);
context_numBits++;
}
}
// Mark the end of the stream
value = 2;
for (i = 0; i < context_numBits; i++) {
context_data_val = (context_data_val << 1) | (value & 1);
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.append(getCharFromInt.doFunc(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
value = value >> 1;
}
// Flush the last char
while (true) {
context_data_val = (context_data_val << 1);
if (context_data_position == bitsPerChar - 1) {
context_data.append(getCharFromInt.doFunc(context_data_val));
break;
}
else
context_data_position++;
}
return context_data.toString();
}
private static abstract class DecompressFunctionWrapper {
public abstract char doFunc(int i);
}
protected static class DecData {
public char val;
public int position;
public int index;
}
public static String f(int i) {
return String.valueOf((char) i);
}
public static char fc(int i) {
return (char) i;
}
public static String decompress(final String compressed) {
if (compressed == null)
return "";
if (compressed.isEmpty())
return null;
return LZString._decompress(compressed.length(), 32768, new DecompressFunctionWrapper() {
@Override
public char doFunc(int i) {
return compressed.charAt(i);
}
});
}
private static String _decompress(int length, int resetValue, DecompressFunctionWrapper getNextValue) {
List<String> dictionary = new ArrayList<String>();
// TODO: is next an unused variable in original lz-string?
@SuppressWarnings("unused")
int next;
int enlargeIn = 4;
int dictSize = 4;
int numBits = 3;
String entry = "";
StringBuilder result = new StringBuilder();
String w;
int bits, resb; int maxpower, power;
String c = null;
DecData data = new DecData();
data.val = getNextValue.doFunc(0);
data.position = resetValue;
data.index = 1;
for (int i = 0; i < 3; i += 1) {
dictionary.add(i, f(i));
}
bits = 0;
maxpower = (int) powerOf2(2);
power = 1;
while (power != maxpower) {
resb = data.val & data.position;
data.position >>= 1;
if (data.position == 0) {
data.position = resetValue;
data.val = getNextValue.doFunc(data.index++);
}
bits |= (resb > 0 ? 1 : 0) * power;
power <<= 1;
}
switch (next = bits) {
case 0:
bits = 0;
maxpower = (int) powerOf2(8);
power=1;
while (power != maxpower) {
resb = data.val & data.position;
data.position >>= 1;
if (data.position == 0) {
data.position = resetValue;
data.val = getNextValue.doFunc(data.index++);
}
bits |= (resb>0 ? 1 : 0) * power;
power <<= 1;
}
c = f(bits);
break;
case 1:
bits = 0;
maxpower = powerOf2(16);
power=1;
while (power!=maxpower) {
resb = data.val & data.position;
data.position >>= 1;
if (data.position == 0) {
data.position = resetValue;
data.val = getNextValue.doFunc(data.index++);
}
bits |= (resb>0 ? 1 : 0) * power;
power <<= 1;
}
c = f(bits);
break;
case 2:
return "";
}
dictionary.add(3, c);
w = c;
result.append(w);
while (true) {
if (data.index > length) {
return "";
}
bits = 0;
maxpower = powerOf2(numBits);
power=1;
while (power!=maxpower) {
resb = data.val & data.position;
data.position >>= 1;
if (data.position == 0) {
data.position = resetValue;
data.val = getNextValue.doFunc(data.index++);
}
bits |= (resb>0 ? 1 : 0) * power;
power <<= 1;
}
// TODO: very strange here, c above is as char/string, here further is a int, rename "c" in the switch as "cc"
int cc;
switch (cc = bits) {
case 0:
bits = 0;
maxpower = powerOf2(8);
power=1;
while (power!=maxpower) {
resb = data.val & data.position;
data.position >>= 1;
if (data.position == 0) {
data.position = resetValue;
data.val = getNextValue.doFunc(data.index++);
}
bits |= (resb>0 ? 1 : 0) * power;
power <<= 1;
}
dictionary.add(dictSize++, f(bits));
cc = dictSize-1;
enlargeIn--;
break;
case 1:
bits = 0;
maxpower = powerOf2(16);
power=1;
while (power!=maxpower) {
resb = data.val & data.position;
data.position >>= 1;
if (data.position == 0) {
data.position = resetValue;
data.val = getNextValue.doFunc(data.index++);
}
bits |= (resb>0 ? 1 : 0) * power;
power <<= 1;
}
dictionary.add(dictSize++, f(bits));
cc = dictSize-1;
enlargeIn--;
break;
case 2:
return result.toString();
}
if (enlargeIn == 0) {
enlargeIn = powerOf2(numBits);
numBits++;
}
if (cc < dictionary.size() && dictionary.get(cc) != null) {
entry = dictionary.get(cc);
} else {
if (cc == dictSize) {
entry = w + w.charAt(0);
} else {
return null;
}
}
result.append(entry);
// Add w+entry[0] to the dictionary.
dictionary.add(dictSize++, w + entry.charAt(0));
enlargeIn--;
w = entry;
if (enlargeIn == 0) {
enlargeIn = powerOf2(numBits);
numBits++;
}
}
}
private static int powerOf2(int power) {
return 1 << power;
}
}
Loading…
Cancel
Save