diff --git a/plugins/multisrc/readnovelfull/sources.json b/plugins/multisrc/readnovelfull/sources.json index fe4c613ac..fc093afaa 100644 --- a/plugins/multisrc/readnovelfull/sources.json +++ b/plugins/multisrc/readnovelfull/sources.json @@ -56,9 +56,8 @@ "noAjax": true, "noPages": ["sort/most-popular"], "pageAsPath": true, - "chapterListing": "api/chapterlist.php", - "chapterParam": "aid", - "versionIncrements": 2 + "chapterListPaginated": true, + "versionIncrements": 3 } }, { diff --git a/plugins/multisrc/readnovelfull/template.ts b/plugins/multisrc/readnovelfull/template.ts index af5d7eda4..17dce3c7b 100644 --- a/plugins/multisrc/readnovelfull/template.ts +++ b/plugins/multisrc/readnovelfull/template.ts @@ -25,6 +25,7 @@ type ReadNovelFullOptions = { noPages?: string[]; pageAsPath?: boolean; customJs?: string; + chapterListPaginated?: boolean; }; export type ReadNovelFullMetadata = { @@ -143,6 +144,135 @@ export class ReadNovelFullPlugin implements Plugin.PluginBase { return novels; } + // =========================================================================== + // HELPERS (chapter-list pagination) + // =========================================================================== + + parseChapterListFragment( + html: string, + startIndex: number, + ): Plugin.ChapterItem[] { + const chapters: Plugin.ChapterItem[] = []; + let tempChapter: Partial = {}; + let i = startIndex; + let inChapter = false; + + const parser = new Parser({ + onopentag: (name, attribs) => { + if (name === 'a' && attribs.href) { + i++; + inChapter = true; + tempChapter.name = attribs.title || `Chapter ${i}`; + tempChapter.releaseTime = null; + tempChapter.chapterNumber = i; + tempChapter.path = attribs.href.startsWith('/') + ? attribs.href.substring(1) + : attribs.href; + } + }, + onclosetag: name => { + if (name === 'a' && inChapter) { + if (tempChapter.name && tempChapter.path) { + chapters.push({ ...tempChapter } as Plugin.ChapterItem); + } + tempChapter = {}; + inChapter = false; + } + }, + }); + + parser.write(html); + parser.end(); + return chapters; + } + + async fetchAllChaptersViaJsonAjax( + novelPath: string, + ): Promise { + const pageSize = 40; + const rateLimitState = { backoffUntil: 0 }; + + const first = await this.fetchChapterPageWithRetry( + novelPath, + 1, + pageSize, + rateLimitState, + ); + if (!first) { + return []; + } + + const allChapters: Plugin.ChapterItem[] = [...first.chapters]; + let fetchedPages = 1; + + if (first.totalPages > 1) { + for (let page = 2; page <= first.totalPages; page++) { + const result = await this.fetchChapterPageWithRetry( + novelPath, + page, + pageSize, + rateLimitState, + ); + if (result) { + allChapters.push(...result.chapters); + fetchedPages++; + } + } + } + + return allChapters; + } + + private async fetchChapterPageWithRetry( + novelPath: string, + page: number, + pageSize: number, + rateLimitState: { backoffUntil: number }, + ): Promise<{ totalPages: number; chapters: Plugin.ChapterItem[] } | null> { + const url = `${this.site}${novelPath}?ajax=chapters&page=${page}&pageSize=${pageSize}`; + const maxAttempts = 3; + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + // Wait out any rate-limit cooldown before sending. + const waitMs = rateLimitState.backoffUntil - Date.now(); + if (waitMs > 0) { + await this.sleep(waitMs); + } + + try { + const result = await fetchApi(url); + if (result.ok) { + const json = await result.json(); + const chapters = this.parseChapterListFragment( + json.html || '', + (page - 1) * pageSize, + ); + await this.sleep(150); // pacing delay — confirmed safe (zero 429s) across multiple runs/novels + return { totalPages: json.totalPage || 1, chapters }; + } + + if (result.status === 429) { + const retryAfterHeader = result.headers?.get?.('Retry-After'); + const retryAfterMs = retryAfterHeader + ? Number(retryAfterHeader) * 1000 + : 3000 * (attempt + 1); // fallback: 3s, 6s, 9s + const cooldownUntil = Date.now() + retryAfterMs; + // Only extend the cooldown, never shorten it + if (cooldownUntil > rateLimitState.backoffUntil) { + rateLimitState.backoffUntil = cooldownUntil; + } + } else { + await this.sleep(800 * (attempt + 1)); + } + } catch (err) { + await this.sleep(800 * (attempt + 1)); + } + } + + return null; + } + // ============================================================================================ + async popularNovels( pageNo: number, { @@ -499,7 +629,9 @@ export class ReadNovelFullPlugin implements Plugin.PluginBase { parser.write(body); parser.end(); - if (this.options.noAjax && chapters.length > 0 && !totalChapter) { + if (this.options.chapterListPaginated) { + novel.chapters = await this.fetchAllChaptersViaJsonAjax(novelPath); + } else if (this.options.noAjax && chapters.length > 0 && !totalChapter) { novel.chapters = chapters; } else if (novelId !== null) { const chapterListing = @@ -525,6 +657,7 @@ export class ReadNovelFullPlugin implements Plugin.PluginBase { } else { chaptersUrl = `${this.site}${chapterListing}?${params.toString()}`; } + const ajaxResult = await fetchApi(chaptersUrl, fetchOptions); if (!ajaxResult.ok) { console.error(`Failed to fetch chapters: ${ajaxResult.status}`); @@ -637,7 +770,7 @@ export class ReadNovelFullPlugin implements Plugin.PluginBase { '>': '>', '"': '"', "'": ''', - ' ': ' ', + ' ': ' ', '\u200C': '', // this is probably a breaking change, report if paragraphs look weird }; const escapeHtml = (text: string) =>