多页搜索
This commit is contained in:
+53
-38
@@ -28,6 +28,7 @@ interface ResolvedLegadoConfig {
|
|||||||
const DEFAULT_TIMEOUT_MS = Number(process.env.LEGADO_TIMEOUT_MS || process.env.OPDS_TIMEOUT_MS || 20000);
|
const DEFAULT_TIMEOUT_MS = Number(process.env.LEGADO_TIMEOUT_MS || process.env.OPDS_TIMEOUT_MS || 20000);
|
||||||
const MAX_TEXT_BYTES = Number(process.env.LEGADO_MAX_TEXT_BYTES || 3 * 1024 * 1024);
|
const MAX_TEXT_BYTES = Number(process.env.LEGADO_MAX_TEXT_BYTES || 3 * 1024 * 1024);
|
||||||
const LEGADO_CACHE_VERSION = 'v5';
|
const LEGADO_CACHE_VERSION = 'v5';
|
||||||
|
const DEFAULT_LEGADO_SEARCH_PAGES = Number(process.env.LEGADO_SEARCH_PAGES || 5);
|
||||||
const textCache = new Map<string, { expiresAt: number; data: string }>();
|
const textCache = new Map<string, { expiresAt: number; data: string }>();
|
||||||
const searchCache = new Map<string, { expiresAt: number; data: BookListItem[] }>();
|
const searchCache = new Map<string, { expiresAt: number; data: BookListItem[] }>();
|
||||||
const detailCache = new Map<string, { expiresAt: number; data: BookDetail }>();
|
const detailCache = new Map<string, { expiresAt: number; data: BookDetail }>();
|
||||||
@@ -453,45 +454,59 @@ export class LegadoClient {
|
|||||||
const cached = searchCache.get(cacheKey);
|
const cached = searchCache.get(cacheKey);
|
||||||
if (cached && cached.expiresAt > Date.now()) return { source, results: cached.data };
|
if (cached && cached.expiresAt > Date.now()) return { source, results: cached.data };
|
||||||
|
|
||||||
const targetUrl = buildUrlFromTemplate(rule.searchUrl, source, q, 1);
|
|
||||||
const html = await fetchText(source, targetUrl);
|
|
||||||
const results: BookListItem[] = [];
|
const results: BookListItem[] = [];
|
||||||
const json = parseJsonMaybe(html);
|
const seen = new Set<string>();
|
||||||
if (json && ruleIsJson(rule.ruleSearch.bookList)) {
|
for (let page = 1; page <= DEFAULT_LEGADO_SEARCH_PAGES; page += 1) {
|
||||||
const items = selectJsonItems(json, rule.ruleSearch.bookList);
|
const targetUrl = buildUrlFromTemplate(rule.searchUrl, source, q, page);
|
||||||
items.forEach((item) => {
|
const html = await fetchText(source, targetUrl);
|
||||||
const detailHref = readJsonRule(item, rule.ruleSearch?.bookUrl, source, targetUrl);
|
let pageCount = 0;
|
||||||
const title = readJsonRule(item, rule.ruleSearch?.name, source, targetUrl);
|
const json = parseJsonMaybe(html);
|
||||||
if (!title && !detailHref) return;
|
if (json && ruleIsJson(rule.ruleSearch.bookList)) {
|
||||||
const cover = readJsonRule(item, rule.ruleSearch?.coverUrl, source, targetUrl);
|
const items = selectJsonItems(json, rule.ruleSearch.bookList);
|
||||||
results.push(makeItem(source, {
|
pageCount = items.length;
|
||||||
id: jsonPrimitiveToString(item?.id) || undefined,
|
items.forEach((item) => {
|
||||||
title,
|
const detailHref = readJsonRule(item, rule.ruleSearch?.bookUrl, source, targetUrl);
|
||||||
author: readJsonRule(item, rule.ruleSearch?.author, source, targetUrl),
|
const title = readJsonRule(item, rule.ruleSearch?.name, source, targetUrl);
|
||||||
summary: readJsonRule(item, rule.ruleSearch?.intro, source, targetUrl),
|
if (!title && !detailHref) return;
|
||||||
cover: cover || undefined,
|
const cover = readJsonRule(item, rule.ruleSearch?.coverUrl, source, targetUrl);
|
||||||
detailHref,
|
const itemId = jsonPrimitiveToString(item?.id) || undefined;
|
||||||
tags: readJsonRule(item, rule.ruleSearch?.kind, source, targetUrl).split(/[,,\s]+/).filter(Boolean),
|
const dedupeKey = itemId || detailHref || `${title}|${cover}`;
|
||||||
}));
|
if (dedupeKey && seen.has(dedupeKey)) return;
|
||||||
});
|
if (dedupeKey) seen.add(dedupeKey);
|
||||||
} else {
|
results.push(makeItem(source, {
|
||||||
const $ = cheerio.load(html);
|
id: itemId,
|
||||||
const items = selectElements($, $.root(), rule.ruleSearch.bookList);
|
title,
|
||||||
items.each((_, element) => {
|
author: readJsonRule(item, rule.ruleSearch?.author, source, targetUrl),
|
||||||
const root = $(element);
|
summary: readJsonRule(item, rule.ruleSearch?.intro, source, targetUrl),
|
||||||
const detailHref = readValue($, root, rule.ruleSearch?.bookUrl, targetUrl);
|
cover: cover || undefined,
|
||||||
const title = readValue($, root, rule.ruleSearch?.name, targetUrl);
|
detailHref,
|
||||||
if (!title && !detailHref) return;
|
tags: readJsonRule(item, rule.ruleSearch?.kind, source, targetUrl).split(/[,,\s]+/).filter(Boolean),
|
||||||
const cover = readValue($, root, rule.ruleSearch?.coverUrl, targetUrl);
|
}));
|
||||||
results.push(makeItem(source, {
|
});
|
||||||
title,
|
} else {
|
||||||
author: readValue($, root, rule.ruleSearch?.author, targetUrl),
|
const $ = cheerio.load(html);
|
||||||
summary: readValue($, root, rule.ruleSearch?.intro, targetUrl),
|
const items = selectElements($, $.root(), rule.ruleSearch.bookList);
|
||||||
cover: cover || undefined,
|
pageCount = items.length;
|
||||||
detailHref,
|
items.each((_, element) => {
|
||||||
tags: readValue($, root, rule.ruleSearch?.kind, targetUrl).split(/[,,\s]+/).filter(Boolean),
|
const root = $(element);
|
||||||
}));
|
const detailHref = readValue($, root, rule.ruleSearch?.bookUrl, targetUrl);
|
||||||
});
|
const title = readValue($, root, rule.ruleSearch?.name, targetUrl);
|
||||||
|
if (!title && !detailHref) return;
|
||||||
|
const cover = readValue($, root, rule.ruleSearch?.coverUrl, targetUrl);
|
||||||
|
const dedupeKey = detailHref || `${title}|${cover}`;
|
||||||
|
if (dedupeKey && seen.has(dedupeKey)) return;
|
||||||
|
if (dedupeKey) seen.add(dedupeKey);
|
||||||
|
results.push(makeItem(source, {
|
||||||
|
title,
|
||||||
|
author: readValue($, root, rule.ruleSearch?.author, targetUrl),
|
||||||
|
summary: readValue($, root, rule.ruleSearch?.intro, targetUrl),
|
||||||
|
cover: cover || undefined,
|
||||||
|
detailHref,
|
||||||
|
tags: readValue($, root, rule.ruleSearch?.kind, targetUrl).split(/[,,\s]+/).filter(Boolean),
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (pageCount === 0) break;
|
||||||
}
|
}
|
||||||
searchCache.set(cacheKey, { data: results, expiresAt: Date.now() + cacheTTL });
|
searchCache.set(cacheKey, { data: results, expiresAt: Date.now() + cacheTTL });
|
||||||
return { source, results };
|
return { source, results };
|
||||||
|
|||||||
Reference in New Issue
Block a user