From 4fe0726ed4e74348d6df5ec86170b3c89d5a5768 Mon Sep 17 00:00:00 2001 From: Batorian Date: Wed, 2 Jul 2025 23:53:19 +0200 Subject: [PATCH 01/35] adding custom fetchOptions --- src/plugins/english/novelupdates.ts | 59 +++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index e44e74331..5b98a587e 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -4,9 +4,21 @@ import { Filters, FilterTypes } from '@libs/filterInputs'; import { Plugin } from '@typings/plugin'; class NovelUpdates implements Plugin.PluginBase { + private fetchOptions = { + headers: { + 'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0', + 'Accept': + 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,us;q=0.5', + 'DNT': '1', // Do Not Track + 'Upgrade-Insecure-Requests': '1', // Upgrade-Insecure-Requests + }, + }; + id = 'novelupdates'; name = 'Novel Updates'; - version = '0.9.1'; + version = '0.10.0'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -562,6 +574,42 @@ class NovelUpdates implements Plugin.PluginBase { } break; } + // Last edited in 0.10.0 by Batorian - 02/07/2025 + case 'stellarrealm': { + // Remove ad-related bloat elements + bloatElements = [ + '.my-6.py-4.border-t.border-b.border-border\\/20.ad-container', + ]; + bloatElements.forEach(tag => loadedCheerio(tag).remove()); + + // Extract the data-page attribute from
+ const dataPage = loadedCheerio('#app').attr('data-page'); + if (!dataPage) { + throw new Error('data-page attribute not found on stellarrealm.net'); + } + + // Parse the JSON from data-page + const pageData = JSON.parse(dataPage) as { + component: string; + props: { + chapter: { + id: number; + title: string; + content: string; + }; + }; + }; + + chapterTitle = pageData.props.chapter.title; + chapterContent = pageData.props.chapter.content; + + // Clean up content (e.g., remove inline styles or scripts if needed) + const chapterCheerio = parseHTML(chapterContent); + chapterCheerio('script, style').remove(); + chapterContent = chapterCheerio.html()!; + + break; + } // Last edited in 0.9.0 by Batorian - 19/03/2025 case 'tinytranslation': { bloatElements = [ @@ -665,7 +713,7 @@ class NovelUpdates implements Plugin.PluginBase { async parseChapter(chapterPath: string): Promise { let chapterText; - const result = await fetchApi(this.site + chapterPath); + const result = await fetchApi(this.site + chapterPath, this.fetchOptions); const body = await result.text(); const url = result.url; const domainParts = url.toLowerCase().split('/')[2].split('.'); @@ -712,7 +760,12 @@ class NovelUpdates implements Plugin.PluginBase { ); // Manually set WordPress flag for known sites - const manualWordPress = ['etherreads', 'greenztl2', 'soafp']; + const manualWordPress = [ + 'etherreads', + 'greenztl2', + 'noicetranslations', + 'soafp', + ]; if (!isWordPress && domainParts.some(wp => manualWordPress.includes(wp))) { isWordPress = true; } From 02578335a234e1bbcdccdefb094859bed27cbbc9 Mon Sep 17 00:00:00 2001 From: Batorian Date: Wed, 2 Jul 2025 23:56:55 +0200 Subject: [PATCH 02/35] change version to test --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8138d79c0..89751a370 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "lnreader-plugins", - "version": "3.0.0", + "version": "3.1.0", "description": "Plugins repo for LNReader", "main": "index.js", "type": "module", From 83dada8524d1dd2cb89a6d30e155ce78ae438320 Mon Sep 17 00:00:00 2001 From: Batorian Date: Thu, 3 Jul 2025 00:18:51 +0200 Subject: [PATCH 03/35] add more fetchOptions and debugging --- src/plugins/english/novelupdates.ts | 474 +++++++++++++++------------- 1 file changed, 251 insertions(+), 223 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 5b98a587e..271f16bc6 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -18,7 +18,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.0'; + version = '0.10.1'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -275,7 +275,9 @@ class NovelUpdates implements Plugin.PluginBase { const url = `${chapterPath}/__data.json?x-sveltekit-invalidated=001`; try { // Fetch the chapter's data in JSON format - const json = await fetchApi(url).then(r => r.json()); + const json = await fetchApi(url, this.fetchOptions).then(r => + r.json(), + ); const nodes = json.nodes; const data = nodes .filter((node: { type: string }) => node.type === 'data') @@ -347,7 +349,7 @@ class NovelUpdates implements Plugin.PluginBase { // Get the chapter link from the main page const url = loadedCheerio('article > p > a').first().attr('href')!; if (url) { - const result = await fetchApi(url); + const result = await fetchApi(url, this.fetchOptions); const body = await result.text(); loadedCheerio = parseHTML(body); } @@ -443,7 +445,7 @@ class NovelUpdates implements Plugin.PluginBase { case 'raeitranslations': { const parts = chapterPath.split('/'); const url = `${parts[0]}//api.${parts[2]}/api/chapters/single?id=${parts[3]}&num=${parts[4]}`; - const json = await fetchApi(url).then(r => r.json()); + const json = await fetchApi(url, this.fetchOptions).then(r => r.json()); const titleElement = `Chapter ${json.currentChapter.chapTag}`; chapterTitle = json.currentChapter.chapTitle ? `${titleElement} - ${json.currentChapter.chapTitle}` @@ -491,7 +493,7 @@ class NovelUpdates implements Plugin.PluginBase { const chapterId = chapterPath.split('/').pop(); chapterTitle = `Chapter ${chapterId}`; const url = `${chapterPath.split('chapter')[0]}txt/${chapterId}.txt`; - chapterContent = await fetchApi(url) + chapterContent = await fetchApi(url, this.fetchOptions) .then(r => r.text()) .then(text => { // Split text into sentences based on newline characters @@ -554,7 +556,7 @@ class NovelUpdates implements Plugin.PluginBase { // Get the chapter link from the main page const url = loadedCheerio('.entry-content a').attr('href')!; if (url) { - const result = await fetchApi(url); + const result = await fetchApi(url, this.fetchOptions); const body = await result.text(); loadedCheerio = parseHTML(body); } @@ -635,7 +637,7 @@ class NovelUpdates implements Plugin.PluginBase { // Get the chapter link from the main page const url = loadedCheerio('.entry-content a').attr('href')!; if (url) { - const result = await fetchApi(chapterPath + url); + const result = await fetchApi(chapterPath + url, this.fetchOptions); const body = await result.text(); loadedCheerio = parseHTML(body); } @@ -682,8 +684,10 @@ class NovelUpdates implements Plugin.PluginBase { case 'yoru': { const chapterId = chapterPath.split('/').pop(); const url = `https://pxp-main-531j.onrender.com/api/v1/book_chapters/${chapterId}/content`; - const json = await fetchApi(url).then(r => r.json()); - chapterText = await fetchApi(json).then(r => r.text()); + const json = await fetchApi(url, this.fetchOptions).then(r => r.json()); + chapterText = await fetchApi(json, this.fetchOptions).then(r => + r.text(), + ); break; } // Last edited in 0.9.0 by Batorian - 19/03/2025 @@ -712,244 +716,268 @@ class NovelUpdates implements Plugin.PluginBase { async parseChapter(chapterPath: string): Promise { let chapterText; + try { + const requestUrl = this.site + chapterPath; + console.log('Request URL:', requestUrl); + console.log('Fetch Options:', this.fetchOptions); - const result = await fetchApi(this.site + chapterPath, this.fetchOptions); - const body = await result.text(); - const url = result.url; - const domainParts = url.toLowerCase().split('/')[2].split('.'); - - const loadedCheerio = parseHTML(body); + const result = await fetchApi(requestUrl, this.fetchOptions); + console.log('Response Status:', result.status); - // Handle CAPTCHA cases - const blockedTitles = [ - 'bot verification', - 'just a moment...', - 'redirecting...', - 'un instant...', - 'you are being redirected...', - ]; - const title = loadedCheerio('title').text().trim().toLowerCase(); - if (blockedTitles.includes(title)) { - throw new Error('Captcha detected, please open in webview.'); - } + const headersObj: Record = {}; + result.headers.forEach((value, key) => { + headersObj[key] = value; + }); + console.log('Response Headers:', headersObj); - // Check if chapter url is wrong or site is down - if (!result.ok) { - throw new Error( - `Failed to fetch ${result.url}: ${result.status} ${result.statusText}`, - ); - } + console.log('Redirected URL:', result.url); - // Detect platforms - let isBlogspot = ['blogspot', 'blogger'].some(keyword => - [ - loadedCheerio('meta[name="google-adsense-platform-domain"]').attr( - 'content', - ), - loadedCheerio('meta[name="generator"]').attr('content'), - ].some(meta => meta?.toLowerCase().includes(keyword)), - ); + if (!result.ok) { + throw new Error(`HTTP error: ${result.status} ${result.statusText}`); + } - let isWordPress = ['wordpress', 'site kit by google'].some(keyword => - [ - loadedCheerio('#dcl_comments-js-extra').html(), - loadedCheerio('meta[name="generator"]').attr('content'), - loadedCheerio('.powered-by').text(), - loadedCheerio('footer').text(), - ].some(meta => meta?.toLowerCase().includes(keyword)), - ); + const body = await result.text(); + console.log('Response Body (first 500 chars):', body.substring(0, 500)); - // Manually set WordPress flag for known sites - const manualWordPress = [ - 'etherreads', - 'greenztl2', - 'noicetranslations', - 'soafp', - ]; - if (!isWordPress && domainParts.some(wp => manualWordPress.includes(wp))) { - isWordPress = true; - } + const loadedCheerio = parseHTML(body); + const title = loadedCheerio('title').text().trim().toLowerCase(); + console.log('Page Title:', title); - // Handle outlier sites - const outliers = [ - 'anotivereads', - 'arcanetranslations', - 'asuratls', - 'darkstartranslations', - 'fictionread', - 'helscans', - 'infinitenoveltranslations', - 'mirilu', - 'novelworldtranslations', - 'sacredtexttranslations', - 'stabbingwithasyringe', - 'tinytranslation', - 'vampiramtl', - 'zetrotranslation', - ]; - if (domainParts.some(d => outliers.includes(d))) { - isWordPress = false; - isBlogspot = false; - } + const blockedTitles = [ + 'bot verification', + 'just a moment...', + 'redirecting...', + 'un instant...', + 'you are being redirected...', + ]; + if (blockedTitles.includes(title)) { + throw new Error('Captcha detected, please open in webview.'); + } - // Last edited in 0.9.0 - 19/03/2025 - /** - * Blogspot sites: - * - ¼-Assed - * - AsuraTls (Outlier) - * - FictionRead (Outlier) - * - Novel World Translations (Outlier) - * - SacredText TL (Outlier) - * - Toasteful - * - * WordPress sites: - * - Anomlaously Creative (Outlier) - * - Arcane Translations (Outlier) - * - Blossom Translation - * - Darkstar Translations (Outlier) - * - Dumahs Translations - * - ElloMTL - * - Femme Fables - * - Gadgetized Panda Translation - * - Gem Novels - * - Goblinslate - * - GreenzTL - * - Hel Scans (Outlier) - * - ippotranslations - * - JATranslations - * - Light Novels Translations - * - Mirilu - Novel Reader Attempts Translating (Outlier) - * - Neosekai Translations - * - Shanghai Fantasy - * - Soafp (Manually added) - * - Stabbing with a Syringe (Outlier) - * - StoneScape - * - TinyTL (Outlier) - * - VampiraMTL (Outlier) - * - Wonder Novels - * - Yong Library - * - Zetro Translation (Outlier) - */ + const url = result.url; + const domainParts = url.toLowerCase().split('/')[2].split('.'); - // Fetch chapter content based on detected platform - if (!isWordPress && !isBlogspot) { - chapterText = await this.getChapterBody(loadedCheerio, domainParts, url); - } else { - const bloatElements = isBlogspot - ? ['.button-container', '.ChapterNav', '.ch-bottom', '.separator'] - : [ - '.ad', - '.author-avatar', - '.chapter-warning', - '.entry-meta', - '.ezoic-ad', - '.mb-center', - '.modern-footnotes-footnote__note', - '.patreon-widget', - '.post-cats', - '.pre-bar', - '.sharedaddy', - '.sidebar', - '.swg-button-v2-light', - '.wp-block-buttons', - //'.wp-block-columns', - '.wp-dark-mode-switcher', - '.wp-next-post-navi', - '#hpk', - '#jp-post-flair', - '#textbox', - ]; + // Detect platforms + let isBlogspot = ['blogspot', 'blogger'].some(keyword => + [ + loadedCheerio('meta[name="google-adsense-platform-domain"]').attr( + 'content', + ), + loadedCheerio('meta[name="generator"]').attr('content'), + ].some(meta => meta?.toLowerCase().includes(keyword)), + ); - bloatElements.forEach(tag => loadedCheerio(tag).remove()); + let isWordPress = ['wordpress', 'site kit by google'].some(keyword => + [ + loadedCheerio('#dcl_comments-js-extra').html(), + loadedCheerio('meta[name="generator"]').attr('content'), + loadedCheerio('.powered-by').text(), + loadedCheerio('footer').text(), + ].some(meta => meta?.toLowerCase().includes(keyword)), + ); - // Extract title - const titleSelectors = isBlogspot - ? ['.entry-title', '.post-title', 'head title'] - : [ - '.entry-title', - '.chapter__title', - '.title-content', - '.wp-block-post-title', - '.title_story', - '#chapter-heading', - 'head title', - 'h1:first-of-type', - 'h2:first-of-type', - '.active', - ]; - let chapterTitle = titleSelectors - .map(sel => loadedCheerio(sel).first().text()) - .find(text => text); + // Manually set WordPress flag for known sites + const manualWordPress = [ + 'etherreads', + 'greenztl2', + 'noicetranslations', + 'soafp', + ]; + if ( + !isWordPress && + domainParts.some(wp => manualWordPress.includes(wp)) + ) { + isWordPress = true; + } - // Extract subtitle (if any) - const chapterSubtitle = - loadedCheerio('.cat-series').first().text() || - loadedCheerio('h1.leading-none ~ span').first().text(); - if (chapterSubtitle) chapterTitle = chapterSubtitle; + // Handle outlier sites + const outliers = [ + 'anotivereads', + 'arcanetranslations', + 'asuratls', + 'darkstartranslations', + 'fictionread', + 'helscans', + 'infinitenoveltranslations', + 'mirilu', + 'novelworldtranslations', + 'sacredtexttranslations', + 'stabbingwithasyringe', + 'tinytranslation', + 'vampiramtl', + 'zetrotranslation', + ]; + if (domainParts.some(d => outliers.includes(d))) { + isWordPress = false; + isBlogspot = false; + } - // Extract content - const contentSelectors = isBlogspot - ? ['.content-post', '.entry-content', '.post-body'] - : [ - '.chapter__content', - '.entry-content', - '.text_story', - '.post-content', - '.contenta', - '.single_post', - '.main-content', - '.reader-content', - '#content', - '#the-content', - 'article.post', - ]; - const chapterContent = contentSelectors - .map(sel => loadedCheerio(sel).html()!) - .find(html => html); + // Last edited in 0.9.0 - 19/03/2025 + /** + * Blogspot sites: + * - ¼-Assed + * - AsuraTls (Outlier) + * - FictionRead (Outlier) + * - Novel World Translations (Outlier) + * - SacredText TL (Outlier) + * - Toasteful + * + * WordPress sites: + * - Anomlaously Creative (Outlier) + * - Arcane Translations (Outlier) + * - Blossom Translation + * - Darkstar Translations (Outlier) + * - Dumahs Translations + * - ElloMTL + * - Femme Fables + * - Gadgetized Panda Translation + * - Gem Novels + * - Goblinslate + * - GreenzTL + * - Hel Scans (Outlier) + * - ippotranslations + * - JATranslations + * - Light Novels Translations + * - Mirilu - Novel Reader Attempts Translating (Outlier) + * - Neosekai Translations + * - Shanghai Fantasy + * - Soafp (Manually added) + * - Stabbing with a Syringe (Outlier) + * - StoneScape + * - TinyTL (Outlier) + * - VampiraMTL (Outlier) + * - Wonder Novels + * - Yong Library + * - Zetro Translation (Outlier) + */ - if (chapterTitle) { - chapterText = `

${chapterTitle}



${chapterContent}`; + // Fetch chapter content based on detected platform + if (!isWordPress && !isBlogspot) { + chapterText = await this.getChapterBody( + loadedCheerio, + domainParts, + url, + ); } else { - chapterText = chapterContent; - } - } + const bloatElements = isBlogspot + ? ['.button-container', '.ChapterNav', '.ch-bottom', '.separator'] + : [ + '.ad', + '.author-avatar', + '.chapter-warning', + '.entry-meta', + '.ezoic-ad', + '.mb-center', + '.modern-footnotes-footnote__note', + '.patreon-widget', + '.post-cats', + '.pre-bar', + '.sharedaddy', + '.sidebar', + '.swg-button-v2-light', + '.wp-block-buttons', + //'.wp-block-columns', + '.wp-dark-mode-switcher', + '.wp-next-post-navi', + '#hpk', + '#jp-post-flair', + '#textbox', + ]; - // Fallback content extraction - if (!chapterText) { - ['nav', 'header', 'footer', '.hidden'].forEach(tag => - loadedCheerio(tag).remove(), - ); - chapterText = loadedCheerio('body').html()!; - } + bloatElements.forEach(tag => loadedCheerio(tag).remove()); - // Convert relative URLs to absolute - chapterText = chapterText.replace( - /href="\//g, - `href="${this.getLocation(result.url)}/`, - ); + // Extract title + const titleSelectors = isBlogspot + ? ['.entry-title', '.post-title', 'head title'] + : [ + '.entry-title', + '.chapter__title', + '.title-content', + '.wp-block-post-title', + '.title_story', + '#chapter-heading', + 'head title', + 'h1:first-of-type', + 'h2:first-of-type', + '.active', + ]; + let chapterTitle = titleSelectors + .map(sel => loadedCheerio(sel).first().text()) + .find(text => text); - // Process images - const chapterCheerio = parseHTML(chapterText); - chapterCheerio('noscript').remove(); + // Extract subtitle (if any) + const chapterSubtitle = + loadedCheerio('.cat-series').first().text() || + loadedCheerio('h1.leading-none ~ span').first().text(); + if (chapterSubtitle) chapterTitle = chapterSubtitle; - chapterCheerio('img').each((_, el) => { - const $el = chapterCheerio(el); + // Extract content + const contentSelectors = isBlogspot + ? ['.content-post', '.entry-content', '.post-body'] + : [ + '.chapter__content', + '.entry-content', + '.text_story', + '.post-content', + '.contenta', + '.single_post', + '.main-content', + '.reader-content', + '#content', + '#the-content', + 'article.post', + ]; + const chapterContent = contentSelectors + .map(sel => loadedCheerio(sel).html()!) + .find(html => html); - // Only update if the lazy-loaded attribute exists - if ($el.attr('data-lazy-src')) { - $el.attr('src', $el.attr('data-lazy-src')); - } - if ($el.attr('data-lazy-srcset')) { - $el.attr('srcset', $el.attr('data-lazy-srcset')); + if (chapterTitle) { + chapterText = `

${chapterTitle}



${chapterContent}`; + } else { + chapterText = chapterContent; + } } - // Remove lazy-loading class if it exists - if ($el.hasClass('lazyloaded')) { - $el.removeClass('lazyloaded'); + // Fallback content extraction + if (!chapterText) { + ['nav', 'header', 'footer', '.hidden'].forEach(tag => + loadedCheerio(tag).remove(), + ); + chapterText = loadedCheerio('body').html()!; } - }); - return chapterCheerio.html()!; + // Convert relative URLs to absolute + chapterText = chapterText.replace( + /href="\//g, + `href="${this.getLocation(result.url)}/`, + ); + + // Process images + const chapterCheerio = parseHTML(chapterText); + chapterCheerio('noscript').remove(); + + chapterCheerio('img').each((_, el) => { + const $el = chapterCheerio(el); + + // Only update if the lazy-loaded attribute exists + if ($el.attr('data-lazy-src')) { + $el.attr('src', $el.attr('data-lazy-src')); + } + if ($el.attr('data-lazy-srcset')) { + $el.attr('srcset', $el.attr('data-lazy-srcset')); + } + + // Remove lazy-loading class if it exists + if ($el.hasClass('lazyloaded')) { + $el.removeClass('lazyloaded'); + } + }); + + return chapterCheerio.html()!; + } catch (error) { + console.error('Fetch Error:', error); + throw new Error(`Network request failed: ${error}`); + } } async searchNovels( From d02288b736838045404fe7b719ac15584830f34f Mon Sep 17 00:00:00 2001 From: Batorian Date: Thu, 3 Jul 2025 00:30:38 +0200 Subject: [PATCH 04/35] enhance debugging --- src/plugins/english/novelupdates.ts | 56 +++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 271f16bc6..ffdc7166c 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -18,7 +18,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.1'; + version = '0.10.2'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -721,19 +721,50 @@ class NovelUpdates implements Plugin.PluginBase { console.log('Request URL:', requestUrl); console.log('Fetch Options:', this.fetchOptions); - const result = await fetchApi(requestUrl, this.fetchOptions); - console.log('Response Status:', result.status); + // Perform a HEAD request to get the redirected URL + let redirectedUrl; + try { + const headResponse = await fetchApi(requestUrl, { + ...this.fetchOptions, + method: 'HEAD', + }); + redirectedUrl = headResponse.url; + console.log('Redirected URL:', redirectedUrl); + } catch (error) { + const err = error as any; + console.error('HEAD Request Error:', { + message: err.message, + stack: err.stack, + }); + throw new Error(`Failed to resolve redirect: ${err.message}`); + } - const headersObj: Record = {}; - result.headers.forEach((value, key) => { - headersObj[key] = value; - }); - console.log('Response Headers:', headersObj); + // Enhance headers for the target site + const enhancedFetchOptions = { + headers: { + ...this.fetchOptions.headers, + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + 'Accept-Encoding': 'gzip, deflate, br', + 'Referer': 'https://www.novelupdates.com/', + }, + }; - console.log('Redirected URL:', result.url); + // Fetch the redirected URL + const result = await fetchApi(redirectedUrl, enhancedFetchOptions); + console.log('Response Status:', result.status); + console.log( + 'Response Headers:', + Object.fromEntries(result.headers.entries()), + ); + console.log('Final URL:', result.url); if (!result.ok) { - throw new Error(`HTTP error: ${result.status} ${result.statusText}`); + throw new Error( + `HTTP error: ${result.status} ${result.statusText} (URL: ${result.url})`, + ); } const body = await result.text(); @@ -751,7 +782,10 @@ class NovelUpdates implements Plugin.PluginBase { 'you are being redirected...', ]; if (blockedTitles.includes(title)) { - throw new Error('Captcha detected, please open in webview.'); + console.log('Falling back to webview for URL:', redirectedUrl); + throw new Error( + `Captcha detected, please open in webview: ${redirectedUrl}`, + ); } const url = result.url; From 325912450ff77f068c7f5118ef02d4eb6ae455d8 Mon Sep 17 00:00:00 2001 From: Batorian Date: Thu, 3 Jul 2025 00:33:45 +0200 Subject: [PATCH 05/35] fix --- src/plugins/english/novelupdates.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index ffdc7166c..2d021b72f 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -18,7 +18,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.2'; + version = '0.10.3'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -755,10 +755,12 @@ class NovelUpdates implements Plugin.PluginBase { // Fetch the redirected URL const result = await fetchApi(redirectedUrl, enhancedFetchOptions); console.log('Response Status:', result.status); - console.log( - 'Response Headers:', - Object.fromEntries(result.headers.entries()), - ); + + const headersObj: Record = {}; + result.headers.forEach((value, key) => { + headersObj[key] = value; + }); + console.log('Response Headers:', headersObj); console.log('Final URL:', result.url); if (!result.ok) { From 0baaab257dfac45c5c648e54a5d21cd056f4459a Mon Sep 17 00:00:00 2001 From: Batorian Date: Thu, 3 Jul 2025 00:51:31 +0200 Subject: [PATCH 06/35] fix --- src/plugins/english/novelupdates.ts | 30 ++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 2d021b72f..f47d4b673 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -18,7 +18,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.3'; + version = '0.10.4'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -747,7 +747,7 @@ class NovelUpdates implements Plugin.PluginBase { 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'none', 'Sec-Fetch-User': '?1', - 'Accept-Encoding': 'gzip, deflate, br', + 'Accept-Encoding': 'gzip, deflate', // Exclude Brotli to ensure compatibility 'Referer': 'https://www.novelupdates.com/', }, }; @@ -769,8 +769,32 @@ class NovelUpdates implements Plugin.PluginBase { ); } + // Check for compression + const contentEncoding = headersObj['content-encoding']?.toLowerCase(); + console.log('Content-Encoding:', contentEncoding); + const body = await result.text(); - console.log('Response Body (first 500 chars):', body.substring(0, 500)); + // Check if the response looks like binary/compressed + const isBinary = /[\x00-\x08\x0E-\x1F\x80-\xFF]/.test( + body.substring(0, 100), + ); + console.log('Is Response Binary:', isBinary); + console.log('Response Body (first 1000 chars):', body.substring(0, 1000)); + + // Early CAPTCHA detection + if ( + isBinary || + body.includes('cf-chl-bypass') || + body.includes('beacon.min.js') + ) { + console.log( + 'Cloudflare challenge detected, falling back to webview for URL:', + redirectedUrl, + ); + throw new Error( + `Cloudflare challenge detected, please open in webview: ${redirectedUrl}`, + ); + } const loadedCheerio = parseHTML(body); const title = loadedCheerio('title').text().trim().toLowerCase(); From 4fc583acdadf9329cc074320fb939ef00ab262ed Mon Sep 17 00:00:00 2001 From: Batorian Date: Thu, 3 Jul 2025 00:54:55 +0200 Subject: [PATCH 07/35] Revert "fix" This reverts commit 0baaab257dfac45c5c648e54a5d21cd056f4459a. --- src/plugins/english/novelupdates.ts | 30 +++-------------------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index f47d4b673..2d021b72f 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -18,7 +18,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.4'; + version = '0.10.3'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -747,7 +747,7 @@ class NovelUpdates implements Plugin.PluginBase { 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'none', 'Sec-Fetch-User': '?1', - 'Accept-Encoding': 'gzip, deflate', // Exclude Brotli to ensure compatibility + 'Accept-Encoding': 'gzip, deflate, br', 'Referer': 'https://www.novelupdates.com/', }, }; @@ -769,32 +769,8 @@ class NovelUpdates implements Plugin.PluginBase { ); } - // Check for compression - const contentEncoding = headersObj['content-encoding']?.toLowerCase(); - console.log('Content-Encoding:', contentEncoding); - const body = await result.text(); - // Check if the response looks like binary/compressed - const isBinary = /[\x00-\x08\x0E-\x1F\x80-\xFF]/.test( - body.substring(0, 100), - ); - console.log('Is Response Binary:', isBinary); - console.log('Response Body (first 1000 chars):', body.substring(0, 1000)); - - // Early CAPTCHA detection - if ( - isBinary || - body.includes('cf-chl-bypass') || - body.includes('beacon.min.js') - ) { - console.log( - 'Cloudflare challenge detected, falling back to webview for URL:', - redirectedUrl, - ); - throw new Error( - `Cloudflare challenge detected, please open in webview: ${redirectedUrl}`, - ); - } + console.log('Response Body (first 500 chars):', body.substring(0, 500)); const loadedCheerio = parseHTML(body); const title = loadedCheerio('title').text().trim().toLowerCase(); From d4cee67343193dc6672f24438b2c71399532cc92 Mon Sep 17 00:00:00 2001 From: Batorian Date: Thu, 3 Jul 2025 00:56:28 +0200 Subject: [PATCH 08/35] fix --- src/plugins/english/novelupdates.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 2d021b72f..53e8c2838 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -747,7 +747,7 @@ class NovelUpdates implements Plugin.PluginBase { 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'none', 'Sec-Fetch-User': '?1', - 'Accept-Encoding': 'gzip, deflate, br', + 'Accept-Encoding': 'gzip, deflate', 'Referer': 'https://www.novelupdates.com/', }, }; From 9ef69ce119d835cd32f27edc06e773e127c5ba61 Mon Sep 17 00:00:00 2001 From: Batorian Date: Thu, 3 Jul 2025 01:06:50 +0200 Subject: [PATCH 09/35] add looping --- src/plugins/english/novelupdates.ts | 52 +++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 53e8c2838..45b552b76 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -18,7 +18,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.3'; + version = '0.10.4'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -752,24 +752,54 @@ class NovelUpdates implements Plugin.PluginBase { }, }; - // Fetch the redirected URL - const result = await fetchApi(redirectedUrl, enhancedFetchOptions); - console.log('Response Status:', result.status); + // Retry fetch with delay to handle intermittent failures + let result; + const maxRetries = 3; + let attempt = 0; + while (attempt < maxRetries) { + try { + console.log(`Fetch Attempt ${attempt + 1} for URL:`, redirectedUrl); + result = await fetchApi(redirectedUrl, enhancedFetchOptions); + console.log('Response Status:', result.status); + + if (result.status === 0) { + throw new Error('Invalid response status: 0'); + } + break; // Success, exit retry loop + } catch (error) { + const err = error as any; + attempt++; + console.error(`Fetch Attempt ${attempt} Error:`, { + message: err.message, + stack: err.stack, + }); + if (attempt === maxRetries) { + console.log( + 'Max retries reached, falling back to webview for URL:', + redirectedUrl, + ); + throw new Error( + `Network request failed after ${maxRetries} attempts, please open in webview.`, + ); + } + await new Promise(resolve => setTimeout(resolve, 2000)); // 2-second delay between retries + } + } const headersObj: Record = {}; - result.headers.forEach((value, key) => { + result?.headers.forEach((value, key) => { headersObj[key] = value; }); console.log('Response Headers:', headersObj); - console.log('Final URL:', result.url); + console.log('Final URL:', result?.url); - if (!result.ok) { + if (!result?.ok) { throw new Error( - `HTTP error: ${result.status} ${result.statusText} (URL: ${result.url})`, + `HTTP error: ${result?.status} ${result?.statusText} (URL: ${result?.url})`, ); } - const body = await result.text(); + const body = await result?.text(); console.log('Response Body (first 500 chars):', body.substring(0, 500)); const loadedCheerio = parseHTML(body); @@ -790,8 +820,8 @@ class NovelUpdates implements Plugin.PluginBase { ); } - const url = result.url; - const domainParts = url.toLowerCase().split('/')[2].split('.'); + const url = result?.url; + const domainParts = url?.toLowerCase().split('/')[2].split('.'); // Detect platforms let isBlogspot = ['blogspot', 'blogger'].some(keyword => From de88c1c9d68fdfbe3eb3571fd6208f40621cd12e Mon Sep 17 00:00:00 2001 From: Batorian Date: Thu, 3 Jul 2025 01:28:38 +0200 Subject: [PATCH 10/35] GET instead of HEAD request --- src/plugins/english/novelupdates.ts | 64 +++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 45b552b76..dccb8f9b8 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -18,7 +18,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.4'; + version = '0.10.5'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -721,22 +721,53 @@ class NovelUpdates implements Plugin.PluginBase { console.log('Request URL:', requestUrl); console.log('Fetch Options:', this.fetchOptions); - // Perform a HEAD request to get the redirected URL + const maxRetries = 3; + + // Perform a GET request to get the redirected URL let redirectedUrl; - try { - const headResponse = await fetchApi(requestUrl, { - ...this.fetchOptions, - method: 'HEAD', - }); - redirectedUrl = headResponse.url; - console.log('Redirected URL:', redirectedUrl); - } catch (error) { - const err = error as any; - console.error('HEAD Request Error:', { - message: err.message, - stack: err.stack, - }); - throw new Error(`Failed to resolve redirect: ${err.message}`); + let headAttempt = 0; + + while (headAttempt < maxRetries) { + try { + console.log( + `Attempt ${headAttempt + 1} to resolve redirect for:`, + requestUrl, + ); + + // Use a GET request. It is much more reliable than HEAD. + const resolveRedirectResponse = await fetchApi(requestUrl, { + ...this.fetchOptions, + method: 'GET', // Use GET instead of HEAD + }); + + // The .url property will contain the final URL after all redirects + redirectedUrl = resolveRedirectResponse.url; + console.log('Successfully resolved Redirected URL:', redirectedUrl); + break; // Success, exit the loop + } catch (error) { + const err = error as any; + headAttempt++; + console.error(`Redirect Resolution Attempt ${headAttempt} Error:`, { + message: err.message, + stack: err.stack, + }); + + if (headAttempt >= maxRetries) { + console.error('Max retries reached for redirect resolution.'); + throw new Error( + `Failed to resolve redirect for ${requestUrl} after ${maxRetries} attempts: ${err.message}`, + ); + } + // Wait before retrying + await new Promise(resolve => setTimeout(resolve, 1500)); + } + } + + // If redirectedUrl is still not set after retries, throw a definitive error. + if (!redirectedUrl) { + throw new Error( + 'Could not determine the final URL after multiple attempts.', + ); } // Enhance headers for the target site @@ -754,7 +785,6 @@ class NovelUpdates implements Plugin.PluginBase { // Retry fetch with delay to handle intermittent failures let result; - const maxRetries = 3; let attempt = 0; while (attempt < maxRetries) { try { From 5c6f12321f9e27e0b4c452057fa3cc2f84ec0a72 Mon Sep 17 00:00:00 2001 From: Batorian Date: Thu, 3 Jul 2025 01:39:04 +0200 Subject: [PATCH 11/35] Rewrite --- src/plugins/english/novelupdates.ts | 119 ++++++++++------------------ 1 file changed, 40 insertions(+), 79 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index dccb8f9b8..28b3c0f37 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -18,7 +18,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.5'; + version = '0.10.6'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -719,82 +719,49 @@ class NovelUpdates implements Plugin.PluginBase { try { const requestUrl = this.site + chapterPath; console.log('Request URL:', requestUrl); - console.log('Fetch Options:', this.fetchOptions); - const maxRetries = 3; - - // Perform a GET request to get the redirected URL - let redirectedUrl; - let headAttempt = 0; - - while (headAttempt < maxRetries) { - try { - console.log( - `Attempt ${headAttempt + 1} to resolve redirect for:`, - requestUrl, - ); - - // Use a GET request. It is much more reliable than HEAD. - const resolveRedirectResponse = await fetchApi(requestUrl, { - ...this.fetchOptions, - method: 'GET', // Use GET instead of HEAD - }); - - // The .url property will contain the final URL after all redirects - redirectedUrl = resolveRedirectResponse.url; - console.log('Successfully resolved Redirected URL:', redirectedUrl); - break; // Success, exit the loop - } catch (error) { - const err = error as any; - headAttempt++; - console.error(`Redirect Resolution Attempt ${headAttempt} Error:`, { - message: err.message, - stack: err.stack, - }); - - if (headAttempt >= maxRetries) { - console.error('Max retries reached for redirect resolution.'); - throw new Error( - `Failed to resolve redirect for ${requestUrl} after ${maxRetries} attempts: ${err.message}`, - ); - } - // Wait before retrying - await new Promise(resolve => setTimeout(resolve, 1500)); - } - } - - // If redirectedUrl is still not set after retries, throw a definitive error. - if (!redirectedUrl) { - throw new Error( - 'Could not determine the final URL after multiple attempts.', - ); - } - - // Enhance headers for the target site + // --- CORRECTED LOGIC: Build the full headers immediately --- + // These headers are needed from the very first request to pass anti-bot checks. const enhancedFetchOptions = { + ...this.fetchOptions, // Make sure your base options are included + method: 'GET', // We are only doing GET requests now headers: { ...this.fetchOptions.headers, + 'Accept': + 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'Accept-Encoding': 'gzip, deflate', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'none', 'Sec-Fetch-User': '?1', - 'Accept-Encoding': 'gzip, deflate', + 'Upgrade-Insecure-Requests': '1', + // This Referer is critical for many sites that check where traffic comes from. 'Referer': 'https://www.novelupdates.com/', }, }; + console.log('Using Enhanced Fetch Options:', enhancedFetchOptions); - // Retry fetch with delay to handle intermittent failures + // --- A single, robust retry loop for the entire fetch process --- let result; + const maxRetries = 3; let attempt = 0; while (attempt < maxRetries) { try { - console.log(`Fetch Attempt ${attempt + 1} for URL:`, redirectedUrl); - result = await fetchApi(redirectedUrl, enhancedFetchOptions); + console.log(`Fetch Attempt ${attempt + 1} for URL:`, requestUrl); + + // The single fetch call now uses the *original* URL and the *enhanced* options. + // fetch will handle the redirects for us. + result = await fetchApi(requestUrl, enhancedFetchOptions); + console.log('Response Status:', result.status); + console.log('Final URL after redirects:', result.url); // This is your 'redirectedUrl' - if (result.status === 0) { - throw new Error('Invalid response status: 0'); + if (!result.ok) { + throw new Error( + `HTTP error: ${result.status} ${result.statusText}`, + ); } + break; // Success, exit retry loop } catch (error) { const err = error as any; @@ -803,36 +770,30 @@ class NovelUpdates implements Plugin.PluginBase { message: err.message, stack: err.stack, }); - if (attempt === maxRetries) { + if (attempt >= maxRetries) { console.log( 'Max retries reached, falling back to webview for URL:', - redirectedUrl, + requestUrl, ); throw new Error( - `Network request failed after ${maxRetries} attempts, please open in webview.`, + `Network request failed after ${maxRetries} attempts. Please open in webview.`, ); } - await new Promise(resolve => setTimeout(resolve, 2000)); // 2-second delay between retries + await new Promise(resolve => setTimeout(resolve, 2000)); // 2-second delay } } + // Since the loop only breaks on success, 'result' must be a valid response here. const headersObj: Record = {}; result?.headers.forEach((value, key) => { headersObj[key] = value; }); console.log('Response Headers:', headersObj); - console.log('Final URL:', result?.url); - - if (!result?.ok) { - throw new Error( - `HTTP error: ${result?.status} ${result?.statusText} (URL: ${result?.url})`, - ); - } const body = await result?.text(); - console.log('Response Body (first 500 chars):', body.substring(0, 500)); + console.log('Response Body (first 500 chars):', body?.substring(0, 500)); - const loadedCheerio = parseHTML(body); + const loadedCheerio = parseHTML(body || ''); const title = loadedCheerio('title').text().trim().toLowerCase(); console.log('Page Title:', title); @@ -843,10 +804,10 @@ class NovelUpdates implements Plugin.PluginBase { 'un instant...', 'you are being redirected...', ]; - if (blockedTitles.includes(title)) { - console.log('Falling back to webview for URL:', redirectedUrl); + if (blockedTitles.some(blocked => title.includes(blocked))) { + console.log('Falling back to webview for URL:', result?.url); throw new Error( - `Captcha detected, please open in webview: ${redirectedUrl}`, + `Captcha or bot-check detected, please open in webview: ${result?.url}`, ); } @@ -881,7 +842,7 @@ class NovelUpdates implements Plugin.PluginBase { ]; if ( !isWordPress && - domainParts.some(wp => manualWordPress.includes(wp)) + domainParts?.some(wp => manualWordPress.includes(wp)) ) { isWordPress = true; } @@ -903,7 +864,7 @@ class NovelUpdates implements Plugin.PluginBase { 'vampiramtl', 'zetrotranslation', ]; - if (domainParts.some(d => outliers.includes(d))) { + if (domainParts?.some(d => outliers.includes(d))) { isWordPress = false; isBlogspot = false; } @@ -951,8 +912,8 @@ class NovelUpdates implements Plugin.PluginBase { if (!isWordPress && !isBlogspot) { chapterText = await this.getChapterBody( loadedCheerio, - domainParts, - url, + domainParts || [], + url || '', ); } else { const bloatElements = isBlogspot @@ -1045,7 +1006,7 @@ class NovelUpdates implements Plugin.PluginBase { // Convert relative URLs to absolute chapterText = chapterText.replace( /href="\//g, - `href="${this.getLocation(result.url)}/`, + `href="${this.getLocation(result?.url || '')}/`, ); // Process images From 7b762a031e17b993375dcfd318b8f0120ba86618 Mon Sep 17 00:00:00 2001 From: Batorian Date: Thu, 3 Jul 2025 01:41:24 +0200 Subject: [PATCH 12/35] Revert "Rewrite" This reverts commit 5c6f12321f9e27e0b4c452057fa3cc2f84ec0a72. --- src/plugins/english/novelupdates.ts | 119 ++++++++++++++++++---------- 1 file changed, 79 insertions(+), 40 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 28b3c0f37..dccb8f9b8 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -18,7 +18,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.6'; + version = '0.10.5'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -719,49 +719,82 @@ class NovelUpdates implements Plugin.PluginBase { try { const requestUrl = this.site + chapterPath; console.log('Request URL:', requestUrl); + console.log('Fetch Options:', this.fetchOptions); - // --- CORRECTED LOGIC: Build the full headers immediately --- - // These headers are needed from the very first request to pass anti-bot checks. + const maxRetries = 3; + + // Perform a GET request to get the redirected URL + let redirectedUrl; + let headAttempt = 0; + + while (headAttempt < maxRetries) { + try { + console.log( + `Attempt ${headAttempt + 1} to resolve redirect for:`, + requestUrl, + ); + + // Use a GET request. It is much more reliable than HEAD. + const resolveRedirectResponse = await fetchApi(requestUrl, { + ...this.fetchOptions, + method: 'GET', // Use GET instead of HEAD + }); + + // The .url property will contain the final URL after all redirects + redirectedUrl = resolveRedirectResponse.url; + console.log('Successfully resolved Redirected URL:', redirectedUrl); + break; // Success, exit the loop + } catch (error) { + const err = error as any; + headAttempt++; + console.error(`Redirect Resolution Attempt ${headAttempt} Error:`, { + message: err.message, + stack: err.stack, + }); + + if (headAttempt >= maxRetries) { + console.error('Max retries reached for redirect resolution.'); + throw new Error( + `Failed to resolve redirect for ${requestUrl} after ${maxRetries} attempts: ${err.message}`, + ); + } + // Wait before retrying + await new Promise(resolve => setTimeout(resolve, 1500)); + } + } + + // If redirectedUrl is still not set after retries, throw a definitive error. + if (!redirectedUrl) { + throw new Error( + 'Could not determine the final URL after multiple attempts.', + ); + } + + // Enhance headers for the target site const enhancedFetchOptions = { - ...this.fetchOptions, // Make sure your base options are included - method: 'GET', // We are only doing GET requests now headers: { ...this.fetchOptions.headers, - 'Accept': - 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', - 'Accept-Encoding': 'gzip, deflate', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'none', 'Sec-Fetch-User': '?1', - 'Upgrade-Insecure-Requests': '1', - // This Referer is critical for many sites that check where traffic comes from. + 'Accept-Encoding': 'gzip, deflate', 'Referer': 'https://www.novelupdates.com/', }, }; - console.log('Using Enhanced Fetch Options:', enhancedFetchOptions); - // --- A single, robust retry loop for the entire fetch process --- + // Retry fetch with delay to handle intermittent failures let result; - const maxRetries = 3; let attempt = 0; while (attempt < maxRetries) { try { - console.log(`Fetch Attempt ${attempt + 1} for URL:`, requestUrl); - - // The single fetch call now uses the *original* URL and the *enhanced* options. - // fetch will handle the redirects for us. - result = await fetchApi(requestUrl, enhancedFetchOptions); - + console.log(`Fetch Attempt ${attempt + 1} for URL:`, redirectedUrl); + result = await fetchApi(redirectedUrl, enhancedFetchOptions); console.log('Response Status:', result.status); - console.log('Final URL after redirects:', result.url); // This is your 'redirectedUrl' - if (!result.ok) { - throw new Error( - `HTTP error: ${result.status} ${result.statusText}`, - ); + if (result.status === 0) { + throw new Error('Invalid response status: 0'); } - break; // Success, exit retry loop } catch (error) { const err = error as any; @@ -770,30 +803,36 @@ class NovelUpdates implements Plugin.PluginBase { message: err.message, stack: err.stack, }); - if (attempt >= maxRetries) { + if (attempt === maxRetries) { console.log( 'Max retries reached, falling back to webview for URL:', - requestUrl, + redirectedUrl, ); throw new Error( - `Network request failed after ${maxRetries} attempts. Please open in webview.`, + `Network request failed after ${maxRetries} attempts, please open in webview.`, ); } - await new Promise(resolve => setTimeout(resolve, 2000)); // 2-second delay + await new Promise(resolve => setTimeout(resolve, 2000)); // 2-second delay between retries } } - // Since the loop only breaks on success, 'result' must be a valid response here. const headersObj: Record = {}; result?.headers.forEach((value, key) => { headersObj[key] = value; }); console.log('Response Headers:', headersObj); + console.log('Final URL:', result?.url); + + if (!result?.ok) { + throw new Error( + `HTTP error: ${result?.status} ${result?.statusText} (URL: ${result?.url})`, + ); + } const body = await result?.text(); - console.log('Response Body (first 500 chars):', body?.substring(0, 500)); + console.log('Response Body (first 500 chars):', body.substring(0, 500)); - const loadedCheerio = parseHTML(body || ''); + const loadedCheerio = parseHTML(body); const title = loadedCheerio('title').text().trim().toLowerCase(); console.log('Page Title:', title); @@ -804,10 +843,10 @@ class NovelUpdates implements Plugin.PluginBase { 'un instant...', 'you are being redirected...', ]; - if (blockedTitles.some(blocked => title.includes(blocked))) { - console.log('Falling back to webview for URL:', result?.url); + if (blockedTitles.includes(title)) { + console.log('Falling back to webview for URL:', redirectedUrl); throw new Error( - `Captcha or bot-check detected, please open in webview: ${result?.url}`, + `Captcha detected, please open in webview: ${redirectedUrl}`, ); } @@ -842,7 +881,7 @@ class NovelUpdates implements Plugin.PluginBase { ]; if ( !isWordPress && - domainParts?.some(wp => manualWordPress.includes(wp)) + domainParts.some(wp => manualWordPress.includes(wp)) ) { isWordPress = true; } @@ -864,7 +903,7 @@ class NovelUpdates implements Plugin.PluginBase { 'vampiramtl', 'zetrotranslation', ]; - if (domainParts?.some(d => outliers.includes(d))) { + if (domainParts.some(d => outliers.includes(d))) { isWordPress = false; isBlogspot = false; } @@ -912,8 +951,8 @@ class NovelUpdates implements Plugin.PluginBase { if (!isWordPress && !isBlogspot) { chapterText = await this.getChapterBody( loadedCheerio, - domainParts || [], - url || '', + domainParts, + url, ); } else { const bloatElements = isBlogspot @@ -1006,7 +1045,7 @@ class NovelUpdates implements Plugin.PluginBase { // Convert relative URLs to absolute chapterText = chapterText.replace( /href="\//g, - `href="${this.getLocation(result?.url || '')}/`, + `href="${this.getLocation(result.url)}/`, ); // Process images From 2fe7d066d3f0d5d1074fab5c7471136777cf5fa8 Mon Sep 17 00:00:00 2001 From: Batorian Date: Thu, 3 Jul 2025 01:42:04 +0200 Subject: [PATCH 13/35] Revert "GET instead of HEAD request" This reverts commit de88c1c9d68fdfbe3eb3571fd6208f40621cd12e. --- src/plugins/english/novelupdates.ts | 64 ++++++++--------------------- 1 file changed, 17 insertions(+), 47 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index dccb8f9b8..45b552b76 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -18,7 +18,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.5'; + version = '0.10.4'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -721,53 +721,22 @@ class NovelUpdates implements Plugin.PluginBase { console.log('Request URL:', requestUrl); console.log('Fetch Options:', this.fetchOptions); - const maxRetries = 3; - - // Perform a GET request to get the redirected URL + // Perform a HEAD request to get the redirected URL let redirectedUrl; - let headAttempt = 0; - - while (headAttempt < maxRetries) { - try { - console.log( - `Attempt ${headAttempt + 1} to resolve redirect for:`, - requestUrl, - ); - - // Use a GET request. It is much more reliable than HEAD. - const resolveRedirectResponse = await fetchApi(requestUrl, { - ...this.fetchOptions, - method: 'GET', // Use GET instead of HEAD - }); - - // The .url property will contain the final URL after all redirects - redirectedUrl = resolveRedirectResponse.url; - console.log('Successfully resolved Redirected URL:', redirectedUrl); - break; // Success, exit the loop - } catch (error) { - const err = error as any; - headAttempt++; - console.error(`Redirect Resolution Attempt ${headAttempt} Error:`, { - message: err.message, - stack: err.stack, - }); - - if (headAttempt >= maxRetries) { - console.error('Max retries reached for redirect resolution.'); - throw new Error( - `Failed to resolve redirect for ${requestUrl} after ${maxRetries} attempts: ${err.message}`, - ); - } - // Wait before retrying - await new Promise(resolve => setTimeout(resolve, 1500)); - } - } - - // If redirectedUrl is still not set after retries, throw a definitive error. - if (!redirectedUrl) { - throw new Error( - 'Could not determine the final URL after multiple attempts.', - ); + try { + const headResponse = await fetchApi(requestUrl, { + ...this.fetchOptions, + method: 'HEAD', + }); + redirectedUrl = headResponse.url; + console.log('Redirected URL:', redirectedUrl); + } catch (error) { + const err = error as any; + console.error('HEAD Request Error:', { + message: err.message, + stack: err.stack, + }); + throw new Error(`Failed to resolve redirect: ${err.message}`); } // Enhance headers for the target site @@ -785,6 +754,7 @@ class NovelUpdates implements Plugin.PluginBase { // Retry fetch with delay to handle intermittent failures let result; + const maxRetries = 3; let attempt = 0; while (attempt < maxRetries) { try { From 554bd23f55aa060fcfc540181056449393826d76 Mon Sep 17 00:00:00 2001 From: Batorian Date: Thu, 3 Jul 2025 01:48:33 +0200 Subject: [PATCH 14/35] Rewrite --- src/plugins/english/novelupdates.ts | 261 +++++++++++++--------------- 1 file changed, 119 insertions(+), 142 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 45b552b76..72e4c184b 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -18,7 +18,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.4'; + version = '0.10.5'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -718,26 +718,13 @@ class NovelUpdates implements Plugin.PluginBase { let chapterText; try { const requestUrl = this.site + chapterPath; - console.log('Request URL:', requestUrl); - console.log('Fetch Options:', this.fetchOptions); // Perform a HEAD request to get the redirected URL - let redirectedUrl; - try { - const headResponse = await fetchApi(requestUrl, { - ...this.fetchOptions, - method: 'HEAD', - }); - redirectedUrl = headResponse.url; - console.log('Redirected URL:', redirectedUrl); - } catch (error) { - const err = error as any; - console.error('HEAD Request Error:', { - message: err.message, - stack: err.stack, - }); - throw new Error(`Failed to resolve redirect: ${err.message}`); - } + const headResponse = await fetchApi(requestUrl, { + ...this.fetchOptions, + method: 'HEAD', + }); + const redirectedUrl = headResponse.url; // Enhance headers for the target site const enhancedFetchOptions = { @@ -753,58 +740,18 @@ class NovelUpdates implements Plugin.PluginBase { }; // Retry fetch with delay to handle intermittent failures - let result; - const maxRetries = 3; - let attempt = 0; - while (attempt < maxRetries) { - try { - console.log(`Fetch Attempt ${attempt + 1} for URL:`, redirectedUrl); - result = await fetchApi(redirectedUrl, enhancedFetchOptions); - console.log('Response Status:', result.status); - - if (result.status === 0) { - throw new Error('Invalid response status: 0'); - } - break; // Success, exit retry loop - } catch (error) { - const err = error as any; - attempt++; - console.error(`Fetch Attempt ${attempt} Error:`, { - message: err.message, - stack: err.stack, - }); - if (attempt === maxRetries) { - console.log( - 'Max retries reached, falling back to webview for URL:', - redirectedUrl, - ); - throw new Error( - `Network request failed after ${maxRetries} attempts, please open in webview.`, - ); - } - await new Promise(resolve => setTimeout(resolve, 2000)); // 2-second delay between retries - } - } - - const headersObj: Record = {}; - result?.headers.forEach((value, key) => { - headersObj[key] = value; - }); - console.log('Response Headers:', headersObj); - console.log('Final URL:', result?.url); + const result = await this.fetchWithRetry( + redirectedUrl, + enhancedFetchOptions, + ); if (!result?.ok) { - throw new Error( - `HTTP error: ${result?.status} ${result?.statusText} (URL: ${result?.url})`, - ); + throw new Error(`HTTP error: ${result?.status} ${result?.statusText}`); } const body = await result?.text(); - console.log('Response Body (first 500 chars):', body.substring(0, 500)); - const loadedCheerio = parseHTML(body); const title = loadedCheerio('title').text().trim().toLowerCase(); - console.log('Page Title:', title); const blockedTitles = [ 'bot verification', @@ -814,7 +761,6 @@ class NovelUpdates implements Plugin.PluginBase { 'you are being redirected...', ]; if (blockedTitles.includes(title)) { - console.log('Falling back to webview for URL:', redirectedUrl); throw new Error( `Captcha detected, please open in webview: ${redirectedUrl}`, ); @@ -925,83 +871,7 @@ class NovelUpdates implements Plugin.PluginBase { url, ); } else { - const bloatElements = isBlogspot - ? ['.button-container', '.ChapterNav', '.ch-bottom', '.separator'] - : [ - '.ad', - '.author-avatar', - '.chapter-warning', - '.entry-meta', - '.ezoic-ad', - '.mb-center', - '.modern-footnotes-footnote__note', - '.patreon-widget', - '.post-cats', - '.pre-bar', - '.sharedaddy', - '.sidebar', - '.swg-button-v2-light', - '.wp-block-buttons', - //'.wp-block-columns', - '.wp-dark-mode-switcher', - '.wp-next-post-navi', - '#hpk', - '#jp-post-flair', - '#textbox', - ]; - - bloatElements.forEach(tag => loadedCheerio(tag).remove()); - - // Extract title - const titleSelectors = isBlogspot - ? ['.entry-title', '.post-title', 'head title'] - : [ - '.entry-title', - '.chapter__title', - '.title-content', - '.wp-block-post-title', - '.title_story', - '#chapter-heading', - 'head title', - 'h1:first-of-type', - 'h2:first-of-type', - '.active', - ]; - let chapterTitle = titleSelectors - .map(sel => loadedCheerio(sel).first().text()) - .find(text => text); - - // Extract subtitle (if any) - const chapterSubtitle = - loadedCheerio('.cat-series').first().text() || - loadedCheerio('h1.leading-none ~ span').first().text(); - if (chapterSubtitle) chapterTitle = chapterSubtitle; - - // Extract content - const contentSelectors = isBlogspot - ? ['.content-post', '.entry-content', '.post-body'] - : [ - '.chapter__content', - '.entry-content', - '.text_story', - '.post-content', - '.contenta', - '.single_post', - '.main-content', - '.reader-content', - '#content', - '#the-content', - 'article.post', - ]; - const chapterContent = contentSelectors - .map(sel => loadedCheerio(sel).html()!) - .find(html => html); - - if (chapterTitle) { - chapterText = `

${chapterTitle}



${chapterContent}`; - } else { - chapterText = chapterContent; - } + chapterText = this.extractPlatformContent(loadedCheerio, isBlogspot); } // Fallback content extraction @@ -1046,6 +916,113 @@ class NovelUpdates implements Plugin.PluginBase { } } + private async fetchWithRetry( + url: string, + options: any, + maxRetries = 3, + ): Promise { + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + const result = await fetchApi(url, options); + if (result.status === 0) { + throw new Error('Invalid response status: 0'); + } + return result; + } catch (error) { + if (attempt === maxRetries) { + throw new Error( + `Network request failed after ${maxRetries} attempts, please open in webview.`, + ); + } + await new Promise(resolve => setTimeout(resolve, 2000)); // 2-second delay between retries + } + } + throw new Error('Unreachable'); + } + + private extractPlatformContent( + loadedCheerio: any, + isBlogspot: boolean, + ): string { + const bloatElements = isBlogspot + ? ['.button-container', '.ChapterNav', '.ch-bottom', '.separator'] + : [ + '.ad', + '.author-avatar', + '.chapter-warning', + '.entry-meta', + '.ezoic-ad', + '.mb-center', + '.modern-footnotes-footnote__note', + '.patreon-widget', + '.post-cats', + '.pre-bar', + '.sharedaddy', + '.sidebar', + '.swg-button-v2-light', + '.wp-block-buttons', + //'.wp-block-columns', + '.wp-dark-mode-switcher', + '.wp-next-post-navi', + '#hpk', + '#jp-post-flair', + '#textbox', + ]; + + bloatElements.forEach(tag => loadedCheerio(tag).remove()); + + // Extract title + const titleSelectors = isBlogspot + ? ['.entry-title', '.post-title', 'head title'] + : [ + '.entry-title', + '.chapter__title', + '.title-content', + '.wp-block-post-title', + '.title_story', + '#chapter-heading', + 'head title', + 'h1:first-of-type', + 'h2:first-of-type', + '.active', + ]; + let chapterTitle = titleSelectors + .map(sel => loadedCheerio(sel).first().text()) + .find(text => text); + + // Extract subtitle (if any) + const chapterSubtitle = + loadedCheerio('.cat-series').first().text() || + loadedCheerio('h1.leading-none ~ span').first().text(); + if (chapterSubtitle) chapterTitle = chapterSubtitle; + + // Extract content + const contentSelectors = isBlogspot + ? ['.content-post', '.entry-content', '.post-body'] + : [ + '.chapter__content', + '.entry-content', + '.text_story', + '.post-content', + '.contenta', + '.single_post', + '.main-content', + '.reader-content', + '#content', + '#the-content', + 'article.post', + ]; + const chapterContent = contentSelectors + .map(sel => loadedCheerio(sel).html()!) + .find(html => html); + + if (chapterTitle) { + return `

${chapterTitle}



${chapterContent}`; + } else { + return chapterContent; + } + } + async searchNovels( searchTerm: string, page: number, From 3f28fc845a13e51089f6ca70d869ee27397a7ffe Mon Sep 17 00:00:00 2001 From: Batorian Date: Thu, 3 Jul 2025 01:53:46 +0200 Subject: [PATCH 15/35] Add debugging again --- src/plugins/english/novelupdates.ts | 66 +++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 12 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 72e4c184b..fdfdec262 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -18,7 +18,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.5'; + version = '0.10.6'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -718,13 +718,26 @@ class NovelUpdates implements Plugin.PluginBase { let chapterText; try { const requestUrl = this.site + chapterPath; + console.log('Request URL:', requestUrl); + console.log('Fetch Options:', this.fetchOptions); // Perform a HEAD request to get the redirected URL - const headResponse = await fetchApi(requestUrl, { - ...this.fetchOptions, - method: 'HEAD', - }); - const redirectedUrl = headResponse.url; + let redirectedUrl; + try { + const headResponse = await fetchApi(requestUrl, { + ...this.fetchOptions, + method: 'HEAD', + }); + redirectedUrl = headResponse.url; + console.log('Redirected URL:', redirectedUrl); + } catch (error) { + const err = error as any; + console.error('HEAD Request Error:', { + message: err.message, + stack: err.stack, + }); + throw new Error(`Failed to resolve redirect: ${err.message}`); + } // Enhance headers for the target site const enhancedFetchOptions = { @@ -745,13 +758,25 @@ class NovelUpdates implements Plugin.PluginBase { enhancedFetchOptions, ); + const headersObj: Record = {}; + result?.headers.forEach((value, key) => { + headersObj[key] = value; + }); + console.log('Response Headers:', headersObj); + console.log('Final URL:', result?.url); + if (!result?.ok) { - throw new Error(`HTTP error: ${result?.status} ${result?.statusText}`); + throw new Error( + `HTTP error: ${result?.status} ${result?.statusText} (URL: ${result?.url})`, + ); } const body = await result?.text(); + console.log('Response Body (first 500 chars):', body.substring(0, 500)); + const loadedCheerio = parseHTML(body); const title = loadedCheerio('title').text().trim().toLowerCase(); + console.log('Page Title:', title); const blockedTitles = [ 'bot verification', @@ -761,6 +786,7 @@ class NovelUpdates implements Plugin.PluginBase { 'you are being redirected...', ]; if (blockedTitles.includes(title)) { + console.log('Falling back to webview for URL:', redirectedUrl); throw new Error( `Captcha detected, please open in webview: ${redirectedUrl}`, ); @@ -921,15 +947,31 @@ class NovelUpdates implements Plugin.PluginBase { options: any, maxRetries = 3, ): Promise { + let result; for (let attempt = 1; attempt <= maxRetries; attempt++) { try { - const result = await fetchApi(url, options); - if (result.status === 0) { - throw new Error('Invalid response status: 0'); + console.log(`Fetch Attempt ${attempt} for URL:`, url); + result = await fetchApi(url, options); + console.log('Response Status:', result?.status); + + if (!result || result.status === 0 || result.status === undefined) { + throw new Error( + 'Invalid response status: ' + (result?.status || 'undefined'), + ); } - return result; + break; // Success, exit retry loop } catch (error) { + const err = error as any; + console.error(`Fetch Attempt ${attempt} Error:`, { + message: err.message, + stack: err.stack, + }); + if (attempt === maxRetries) { + console.log( + 'Max retries reached, falling back to webview for URL:', + url, + ); throw new Error( `Network request failed after ${maxRetries} attempts, please open in webview.`, ); @@ -937,7 +979,7 @@ class NovelUpdates implements Plugin.PluginBase { await new Promise(resolve => setTimeout(resolve, 2000)); // 2-second delay between retries } } - throw new Error('Unreachable'); + return result!; } private extractPlatformContent( From bc1f061a3a285a7dcc43f57e4f80bf31d6f24b32 Mon Sep 17 00:00:00 2001 From: Batorian Date: Thu, 3 Jul 2025 02:00:26 +0200 Subject: [PATCH 16/35] Update --- src/plugins/english/novelupdates.ts | 42 ++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index fdfdec262..128b0cd1b 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -18,7 +18,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.6'; + version = '0.10.7'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -722,21 +722,37 @@ class NovelUpdates implements Plugin.PluginBase { console.log('Fetch Options:', this.fetchOptions); // Perform a HEAD request to get the redirected URL - let redirectedUrl; + let redirectedUrl = requestUrl; // Default fallback try { const headResponse = await fetchApi(requestUrl, { ...this.fetchOptions, method: 'HEAD', }); - redirectedUrl = headResponse.url; - console.log('Redirected URL:', redirectedUrl); + + // Check if we got a valid response + if ( + headResponse && + headResponse.status && + headResponse.status >= 200 && + headResponse.status < 600 + ) { + redirectedUrl = headResponse.url || requestUrl; + console.log('Redirected URL:', redirectedUrl); + } else { + console.log( + 'HEAD request returned invalid status, using original URL:', + headResponse?.status, + ); + redirectedUrl = requestUrl; + } } catch (error) { const err = error as any; console.error('HEAD Request Error:', { message: err.message, stack: err.stack, }); - throw new Error(`Failed to resolve redirect: ${err.message}`); + console.log('HEAD request failed, using original URL as fallback'); + redirectedUrl = requestUrl; // Use original URL as fallback instead of throwing } // Enhance headers for the target site @@ -953,18 +969,30 @@ class NovelUpdates implements Plugin.PluginBase { console.log(`Fetch Attempt ${attempt} for URL:`, url); result = await fetchApi(url, options); console.log('Response Status:', result?.status); + console.log('Response OK:', result?.ok); - if (!result || result.status === 0 || result.status === undefined) { + // Check for valid response + if (!result) { + throw new Error('No response received'); + } + + if (result.status === 0) { throw new Error( - 'Invalid response status: ' + (result?.status || 'undefined'), + 'Network error: status 0 (possibly CORS or network issue)', ); } + + if (result.status < 200 || result.status > 599) { + throw new Error(`Invalid status code: ${result.status}`); + } + break; // Success, exit retry loop } catch (error) { const err = error as any; console.error(`Fetch Attempt ${attempt} Error:`, { message: err.message, stack: err.stack, + status: result?.status, }); if (attempt === maxRetries) { From fb3582274e80ed48255c10cf8354da5ad43324da Mon Sep 17 00:00:00 2001 From: Batorian Date: Thu, 3 Jul 2025 02:01:59 +0200 Subject: [PATCH 17/35] Revert "Update" This reverts commit bc1f061a3a285a7dcc43f57e4f80bf31d6f24b32. --- src/plugins/english/novelupdates.ts | 42 +++++------------------------ 1 file changed, 7 insertions(+), 35 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 128b0cd1b..fdfdec262 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -18,7 +18,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.7'; + version = '0.10.6'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -722,37 +722,21 @@ class NovelUpdates implements Plugin.PluginBase { console.log('Fetch Options:', this.fetchOptions); // Perform a HEAD request to get the redirected URL - let redirectedUrl = requestUrl; // Default fallback + let redirectedUrl; try { const headResponse = await fetchApi(requestUrl, { ...this.fetchOptions, method: 'HEAD', }); - - // Check if we got a valid response - if ( - headResponse && - headResponse.status && - headResponse.status >= 200 && - headResponse.status < 600 - ) { - redirectedUrl = headResponse.url || requestUrl; - console.log('Redirected URL:', redirectedUrl); - } else { - console.log( - 'HEAD request returned invalid status, using original URL:', - headResponse?.status, - ); - redirectedUrl = requestUrl; - } + redirectedUrl = headResponse.url; + console.log('Redirected URL:', redirectedUrl); } catch (error) { const err = error as any; console.error('HEAD Request Error:', { message: err.message, stack: err.stack, }); - console.log('HEAD request failed, using original URL as fallback'); - redirectedUrl = requestUrl; // Use original URL as fallback instead of throwing + throw new Error(`Failed to resolve redirect: ${err.message}`); } // Enhance headers for the target site @@ -969,30 +953,18 @@ class NovelUpdates implements Plugin.PluginBase { console.log(`Fetch Attempt ${attempt} for URL:`, url); result = await fetchApi(url, options); console.log('Response Status:', result?.status); - console.log('Response OK:', result?.ok); - // Check for valid response - if (!result) { - throw new Error('No response received'); - } - - if (result.status === 0) { + if (!result || result.status === 0 || result.status === undefined) { throw new Error( - 'Network error: status 0 (possibly CORS or network issue)', + 'Invalid response status: ' + (result?.status || 'undefined'), ); } - - if (result.status < 200 || result.status > 599) { - throw new Error(`Invalid status code: ${result.status}`); - } - break; // Success, exit retry loop } catch (error) { const err = error as any; console.error(`Fetch Attempt ${attempt} Error:`, { message: err.message, stack: err.stack, - status: result?.status, }); if (attempt === maxRetries) { From e52decdfb18faaa55752db37e459c4cc44f4d2d9 Mon Sep 17 00:00:00 2001 From: Batorian Date: Tue, 8 Jul 2025 11:09:24 +0200 Subject: [PATCH 18/35] update stellar realm --- src/plugins/english/novelupdates.ts | 79 +++++++++++++++++++---------- 1 file changed, 51 insertions(+), 28 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index fdfdec262..66945c85e 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -18,7 +18,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.6'; + version = '0.10.7'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -578,38 +578,61 @@ class NovelUpdates implements Plugin.PluginBase { } // Last edited in 0.10.0 by Batorian - 02/07/2025 case 'stellarrealm': { - // Remove ad-related bloat elements - bloatElements = [ - '.my-6.py-4.border-t.border-b.border-border\\/20.ad-container', - ]; - bloatElements.forEach(tag => loadedCheerio(tag).remove()); - - // Extract the data-page attribute from
- const dataPage = loadedCheerio('#app').attr('data-page'); - if (!dataPage) { - throw new Error('data-page attribute not found on stellarrealm.net'); - } + // Modular extraction inspired by W2e + const extractStellarRealmContent = (cheerioInstance: CheerioAPI) => { + // Remove ad-related bloat elements + const bloatElements = ['.ad-container', 'script', 'style']; + bloatElements.forEach(tag => cheerioInstance(tag).remove()); + + // Extract the data-page attribute from
+ const dataPage = cheerioInstance('#app').attr('data-page'); + if (!dataPage) { + throw new Error('data-page attribute not found on Stellar Realm.'); + } - // Parse the JSON from data-page - const pageData = JSON.parse(dataPage) as { - component: string; - props: { - chapter: { - id: number; - title: string; - content: string; + // Parse the JSON from data-page + let pageData; + try { + pageData = JSON.parse(dataPage) as { + component: string; + props: { + chapter: { + id: number; + title: string; + content: string; + }; + }; }; - }; - }; + } catch (e) { + throw new Error( + 'Failed to parse data-page JSON for Stellar Realm.', + ); + } - chapterTitle = pageData.props.chapter.title; - chapterContent = pageData.props.chapter.content; + let chapterTitle = pageData.props.chapter.title; + let chapterContent = pageData.props.chapter.content; - // Clean up content (e.g., remove inline styles or scripts if needed) - const chapterCheerio = parseHTML(chapterContent); - chapterCheerio('script, style').remove(); - chapterContent = chapterCheerio.html()!; + // Clean up content (remove inline styles/scripts if needed) + const chapterCheerio = parseHTML(chapterContent); + chapterCheerio('script, style').remove(); + chapterContent = chapterCheerio.html()!; + // Return formatted HTML + return `

${chapterTitle}



${chapterContent}`; + }; + + try { + chapterText = extractStellarRealmContent(loadedCheerio); + } catch (err) { + // Fallback: try to extract whatever is in #app or body + let fallbackContent = + loadedCheerio('#app').html() || loadedCheerio('body').html() || ''; + // Remove scripts/styles + const fallbackCheerio = parseHTML(fallbackContent); + fallbackCheerio('script, style').remove(); + fallbackContent = fallbackCheerio.html()!; + chapterText = fallbackContent || 'Unable to extract chapter content.'; + } break; } // Last edited in 0.9.0 by Batorian - 19/03/2025 From 4a3de3926f3ebe2e7970acb362244377663128af Mon Sep 17 00:00:00 2001 From: Batorian Date: Tue, 8 Jul 2025 11:17:48 +0200 Subject: [PATCH 19/35] try adding fallback for HEAD request --- src/plugins/english/novelupdates.ts | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 66945c85e..38869b072 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -18,7 +18,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.7'; + version = '0.10.8'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -745,13 +745,26 @@ class NovelUpdates implements Plugin.PluginBase { console.log('Fetch Options:', this.fetchOptions); // Perform a HEAD request to get the redirected URL - let redirectedUrl; + let redirectedUrl: string | undefined = undefined; + let headResponse: any = undefined; try { - const headResponse = await fetchApi(requestUrl, { + headResponse = await fetchApi(requestUrl, { ...this.fetchOptions, method: 'HEAD', }); - redirectedUrl = headResponse.url; + // Only use the response if status is in [200, 599] + if ( + headResponse && + typeof headResponse.status === 'number' && + headResponse.status >= 200 && + headResponse.status <= 599 && + headResponse.url + ) { + redirectedUrl = headResponse.url; + } else { + // HEAD failed or returned invalid status, fallback to GET + redirectedUrl = requestUrl; + } console.log('Redirected URL:', redirectedUrl); } catch (error) { const err = error as any; @@ -759,7 +772,8 @@ class NovelUpdates implements Plugin.PluginBase { message: err.message, stack: err.stack, }); - throw new Error(`Failed to resolve redirect: ${err.message}`); + // Fallback to GET if HEAD fails + redirectedUrl = requestUrl; } // Enhance headers for the target site @@ -777,7 +791,7 @@ class NovelUpdates implements Plugin.PluginBase { // Retry fetch with delay to handle intermittent failures const result = await this.fetchWithRetry( - redirectedUrl, + redirectedUrl || requestUrl, enhancedFetchOptions, ); From 61e19a3c1a3405108f4b24d314b2d3f7690067ae Mon Sep 17 00:00:00 2001 From: Batorian Date: Tue, 8 Jul 2025 11:24:26 +0200 Subject: [PATCH 20/35] remove HEAD fetch --- src/plugins/english/novelupdates.ts | 64 +++++------------------------ 1 file changed, 11 insertions(+), 53 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 38869b072..58a7b704a 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -13,12 +13,18 @@ class NovelUpdates implements Plugin.PluginBase { 'Accept-Language': 'en-US,us;q=0.5', 'DNT': '1', // Do Not Track 'Upgrade-Insecure-Requests': '1', // Upgrade-Insecure-Requests + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + 'Accept-Encoding': 'gzip, deflate', + 'Referer': 'https://www.novelupdates.com/', }, }; id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.8'; + version = '0.10.9'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -744,56 +750,8 @@ class NovelUpdates implements Plugin.PluginBase { console.log('Request URL:', requestUrl); console.log('Fetch Options:', this.fetchOptions); - // Perform a HEAD request to get the redirected URL - let redirectedUrl: string | undefined = undefined; - let headResponse: any = undefined; - try { - headResponse = await fetchApi(requestUrl, { - ...this.fetchOptions, - method: 'HEAD', - }); - // Only use the response if status is in [200, 599] - if ( - headResponse && - typeof headResponse.status === 'number' && - headResponse.status >= 200 && - headResponse.status <= 599 && - headResponse.url - ) { - redirectedUrl = headResponse.url; - } else { - // HEAD failed or returned invalid status, fallback to GET - redirectedUrl = requestUrl; - } - console.log('Redirected URL:', redirectedUrl); - } catch (error) { - const err = error as any; - console.error('HEAD Request Error:', { - message: err.message, - stack: err.stack, - }); - // Fallback to GET if HEAD fails - redirectedUrl = requestUrl; - } - - // Enhance headers for the target site - const enhancedFetchOptions = { - headers: { - ...this.fetchOptions.headers, - 'Sec-Fetch-Dest': 'document', - 'Sec-Fetch-Mode': 'navigate', - 'Sec-Fetch-Site': 'none', - 'Sec-Fetch-User': '?1', - 'Accept-Encoding': 'gzip, deflate', - 'Referer': 'https://www.novelupdates.com/', - }, - }; - - // Retry fetch with delay to handle intermittent failures - const result = await this.fetchWithRetry( - redirectedUrl || requestUrl, - enhancedFetchOptions, - ); + // Always use GET, let the server handle redirects + const result = await this.fetchWithRetry(requestUrl, this.fetchOptions); const headersObj: Record = {}; result?.headers.forEach((value, key) => { @@ -823,9 +781,9 @@ class NovelUpdates implements Plugin.PluginBase { 'you are being redirected...', ]; if (blockedTitles.includes(title)) { - console.log('Falling back to webview for URL:', redirectedUrl); + console.log('Falling back to webview for URL:', requestUrl); throw new Error( - `Captcha detected, please open in webview: ${redirectedUrl}`, + `Captcha detected, please open in webview: ${requestUrl}`, ); } From dd367fff363e967de4b7ce6453167bf80ce44c92 Mon Sep 17 00:00:00 2001 From: Batorian Date: Tue, 8 Jul 2025 11:34:27 +0200 Subject: [PATCH 21/35] add redirect follow and randomize request url --- src/plugins/english/novelupdates.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 58a7b704a..22100e4bd 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -6,13 +6,10 @@ import { Plugin } from '@typings/plugin'; class NovelUpdates implements Plugin.PluginBase { private fetchOptions = { headers: { - 'User-Agent': - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,us;q=0.5', - 'DNT': '1', // Do Not Track - 'Upgrade-Insecure-Requests': '1', // Upgrade-Insecure-Requests + 'Upgrade-Insecure-Requests': '1', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'none', @@ -20,6 +17,7 @@ class NovelUpdates implements Plugin.PluginBase { 'Accept-Encoding': 'gzip, deflate', 'Referer': 'https://www.novelupdates.com/', }, + redirect: 'follow', }; id = 'novelupdates'; @@ -750,8 +748,15 @@ class NovelUpdates implements Plugin.PluginBase { console.log('Request URL:', requestUrl); console.log('Fetch Options:', this.fetchOptions); + // Add a random query parameter to bypass cache/network issues + const urlWithBypass = + requestUrl + (requestUrl.includes('?') ? '&' : '?') + 't=' + Date.now(); + // Always use GET, let the server handle redirects - const result = await this.fetchWithRetry(requestUrl, this.fetchOptions); + const result = await this.fetchWithRetry( + urlWithBypass, + this.fetchOptions, + ); const headersObj: Record = {}; result?.headers.forEach((value, key) => { From 7eae88fe0ead65de61b99d9741e951758f1357f5 Mon Sep 17 00:00:00 2001 From: Batorian Date: Tue, 8 Jul 2025 11:47:44 +0200 Subject: [PATCH 22/35] try storing cookies --- src/plugins/english/novelupdates.ts | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 22100e4bd..06e3970cb 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -948,12 +948,42 @@ class NovelUpdates implements Plugin.PluginBase { maxRetries = 3, ): Promise { let result; + let storedCookies = ''; for (let attempt = 1; attempt <= maxRetries; attempt++) { try { console.log(`Fetch Attempt ${attempt} for URL:`, url); + + // Add Cookie header if we have stored cookies + if (storedCookies) { + options.headers = { + ...options.headers, + Cookie: storedCookies, + }; + } + result = await fetchApi(url, options); console.log('Response Status:', result?.status); + // Extract and store cookies from set-cookie header + let setCookie: string | null | undefined = undefined; + if (result?.headers?.get) { + setCookie = result.headers.get('set-cookie'); + } + if (setCookie) { + // If multiple cookies, join them with '; ' + if (Array.isArray(setCookie)) { + storedCookies = setCookie + .map((c: string) => c.split(';')[0]) + .join('; '); + } else { + storedCookies = setCookie + .split(',') + .map((c: string) => c.split(';')[0]) + .join('; '); + } + console.log('Stored Cookies:', storedCookies); + } + if (!result || result.status === 0 || result.status === undefined) { throw new Error( 'Invalid response status: ' + (result?.status || 'undefined'), From 5f3316e18bf86d76c2002e9fb6ade3e2b2d13fdd Mon Sep 17 00:00:00 2001 From: Batorian Date: Tue, 8 Jul 2025 11:58:35 +0200 Subject: [PATCH 23/35] try random user agent --- src/plugins/english/novelupdates.ts | 56 ++++++++++++----------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 06e3970cb..9ef1f326c 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -22,7 +22,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.9'; + version = '0.10.10'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -741,6 +741,18 @@ class NovelUpdates implements Plugin.PluginBase { return chapterText; } + // Helper to generate a random User-Agent string + getRandomUserAgent() { + const userAgents = [ + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1', + 'Mozilla/5.0 (Linux; Android 11; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36', + ]; + return userAgents[Math.floor(Math.random() * userAgents.length)]; + } + async parseChapter(chapterPath: string): Promise { let chapterText; try { @@ -752,10 +764,19 @@ class NovelUpdates implements Plugin.PluginBase { const urlWithBypass = requestUrl + (requestUrl.includes('?') ? '&' : '?') + 't=' + Date.now(); + // Clone fetchOptions and randomize User-Agent for this request + const fetchOptionsWithRandomUA = { + ...this.fetchOptions, + headers: { + ...this.fetchOptions.headers, + 'User-Agent': this.getRandomUserAgent(), + }, + }; + // Always use GET, let the server handle redirects const result = await this.fetchWithRetry( urlWithBypass, - this.fetchOptions, + fetchOptionsWithRandomUA, ); const headersObj: Record = {}; @@ -948,42 +969,11 @@ class NovelUpdates implements Plugin.PluginBase { maxRetries = 3, ): Promise { let result; - let storedCookies = ''; for (let attempt = 1; attempt <= maxRetries; attempt++) { try { console.log(`Fetch Attempt ${attempt} for URL:`, url); - - // Add Cookie header if we have stored cookies - if (storedCookies) { - options.headers = { - ...options.headers, - Cookie: storedCookies, - }; - } - result = await fetchApi(url, options); console.log('Response Status:', result?.status); - - // Extract and store cookies from set-cookie header - let setCookie: string | null | undefined = undefined; - if (result?.headers?.get) { - setCookie = result.headers.get('set-cookie'); - } - if (setCookie) { - // If multiple cookies, join them with '; ' - if (Array.isArray(setCookie)) { - storedCookies = setCookie - .map((c: string) => c.split(';')[0]) - .join('; '); - } else { - storedCookies = setCookie - .split(',') - .map((c: string) => c.split(';')[0]) - .join('; '); - } - console.log('Stored Cookies:', storedCookies); - } - if (!result || result.status === 0 || result.status === undefined) { throw new Error( 'Invalid response status: ' + (result?.status || 'undefined'), From a626959d926de97c958d3bd6e49074317de9ccb5 Mon Sep 17 00:00:00 2001 From: Batorian Date: Tue, 8 Jul 2025 12:02:54 +0200 Subject: [PATCH 24/35] revert parse chapter changes --- src/plugins/english/novelupdates.ts | 533 +++++++++++----------------- 1 file changed, 213 insertions(+), 320 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 9ef1f326c..171b9ddfc 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -22,7 +22,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.10'; + version = '0.10.11'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -741,348 +741,241 @@ class NovelUpdates implements Plugin.PluginBase { return chapterText; } - // Helper to generate a random User-Agent string - getRandomUserAgent() { - const userAgents = [ - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Linux; Android 11; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36', - ]; - return userAgents[Math.floor(Math.random() * userAgents.length)]; - } - async parseChapter(chapterPath: string): Promise { let chapterText; - try { - const requestUrl = this.site + chapterPath; - console.log('Request URL:', requestUrl); - console.log('Fetch Options:', this.fetchOptions); - - // Add a random query parameter to bypass cache/network issues - const urlWithBypass = - requestUrl + (requestUrl.includes('?') ? '&' : '?') + 't=' + Date.now(); - - // Clone fetchOptions and randomize User-Agent for this request - const fetchOptionsWithRandomUA = { - ...this.fetchOptions, - headers: { - ...this.fetchOptions.headers, - 'User-Agent': this.getRandomUserAgent(), - }, - }; - // Always use GET, let the server handle redirects - const result = await this.fetchWithRetry( - urlWithBypass, - fetchOptionsWithRandomUA, - ); - - const headersObj: Record = {}; - result?.headers.forEach((value, key) => { - headersObj[key] = value; - }); - console.log('Response Headers:', headersObj); - console.log('Final URL:', result?.url); - - if (!result?.ok) { - throw new Error( - `HTTP error: ${result?.status} ${result?.statusText} (URL: ${result?.url})`, - ); - } - - const body = await result?.text(); - console.log('Response Body (first 500 chars):', body.substring(0, 500)); - - const loadedCheerio = parseHTML(body); - const title = loadedCheerio('title').text().trim().toLowerCase(); - console.log('Page Title:', title); + const result = await fetchApi(this.site + chapterPath); + const body = await result.text(); + const url = result.url; + const domainParts = url.toLowerCase().split('/')[2].split('.'); - const blockedTitles = [ - 'bot verification', - 'just a moment...', - 'redirecting...', - 'un instant...', - 'you are being redirected...', - ]; - if (blockedTitles.includes(title)) { - console.log('Falling back to webview for URL:', requestUrl); - throw new Error( - `Captcha detected, please open in webview: ${requestUrl}`, - ); - } + const loadedCheerio = parseHTML(body); - const url = result?.url; - const domainParts = url?.toLowerCase().split('/')[2].split('.'); + // Handle CAPTCHA cases + const blockedTitles = [ + 'bot verification', + 'just a moment...', + 'redirecting...', + 'un instant...', + 'you are being redirected...', + ]; + const title = loadedCheerio('title').text().trim().toLowerCase(); + if (blockedTitles.includes(title)) { + throw new Error('Captcha detected, please open in webview.'); + } - // Detect platforms - let isBlogspot = ['blogspot', 'blogger'].some(keyword => - [ - loadedCheerio('meta[name="google-adsense-platform-domain"]').attr( - 'content', - ), - loadedCheerio('meta[name="generator"]').attr('content'), - ].some(meta => meta?.toLowerCase().includes(keyword)), + // Check if chapter url is wrong or site is down + if (!result.ok) { + throw new Error( + `Failed to fetch ${result.url}: ${result.status} ${result.statusText}`, ); + } - let isWordPress = ['wordpress', 'site kit by google'].some(keyword => - [ - loadedCheerio('#dcl_comments-js-extra').html(), - loadedCheerio('meta[name="generator"]').attr('content'), - loadedCheerio('.powered-by').text(), - loadedCheerio('footer').text(), - ].some(meta => meta?.toLowerCase().includes(keyword)), - ); + // Detect platforms + let isBlogspot = ['blogspot', 'blogger'].some(keyword => + [ + loadedCheerio('meta[name="google-adsense-platform-domain"]').attr( + 'content', + ), + loadedCheerio('meta[name="generator"]').attr('content'), + ].some(meta => meta?.toLowerCase().includes(keyword)), + ); - // Manually set WordPress flag for known sites - const manualWordPress = [ - 'etherreads', - 'greenztl2', - 'noicetranslations', - 'soafp', - ]; - if ( - !isWordPress && - domainParts.some(wp => manualWordPress.includes(wp)) - ) { - isWordPress = true; - } + let isWordPress = ['wordpress', 'site kit by google'].some(keyword => + [ + loadedCheerio('#dcl_comments-js-extra').html(), + loadedCheerio('meta[name="generator"]').attr('content'), + loadedCheerio('.powered-by').text(), + loadedCheerio('footer').text(), + ].some(meta => meta?.toLowerCase().includes(keyword)), + ); - // Handle outlier sites - const outliers = [ - 'anotivereads', - 'arcanetranslations', - 'asuratls', - 'darkstartranslations', - 'fictionread', - 'helscans', - 'infinitenoveltranslations', - 'mirilu', - 'novelworldtranslations', - 'sacredtexttranslations', - 'stabbingwithasyringe', - 'tinytranslation', - 'vampiramtl', - 'zetrotranslation', - ]; - if (domainParts.some(d => outliers.includes(d))) { - isWordPress = false; - isBlogspot = false; - } + // Manually set WordPress flag for known sites + const manualWordPress = ['etherreads', 'greenztl2', 'soafp']; + if (!isWordPress && domainParts.some(wp => manualWordPress.includes(wp))) { + isWordPress = true; + } - // Last edited in 0.9.0 - 19/03/2025 - /** - * Blogspot sites: - * - ¼-Assed - * - AsuraTls (Outlier) - * - FictionRead (Outlier) - * - Novel World Translations (Outlier) - * - SacredText TL (Outlier) - * - Toasteful - * - * WordPress sites: - * - Anomlaously Creative (Outlier) - * - Arcane Translations (Outlier) - * - Blossom Translation - * - Darkstar Translations (Outlier) - * - Dumahs Translations - * - ElloMTL - * - Femme Fables - * - Gadgetized Panda Translation - * - Gem Novels - * - Goblinslate - * - GreenzTL - * - Hel Scans (Outlier) - * - ippotranslations - * - JATranslations - * - Light Novels Translations - * - Mirilu - Novel Reader Attempts Translating (Outlier) - * - Neosekai Translations - * - Shanghai Fantasy - * - Soafp (Manually added) - * - Stabbing with a Syringe (Outlier) - * - StoneScape - * - TinyTL (Outlier) - * - VampiraMTL (Outlier) - * - Wonder Novels - * - Yong Library - * - Zetro Translation (Outlier) - */ + // Handle outlier sites + const outliers = [ + 'anotivereads', + 'arcanetranslations', + 'asuratls', + 'darkstartranslations', + 'fictionread', + 'helscans', + 'infinitenoveltranslations', + 'mirilu', + 'novelworldtranslations', + 'sacredtexttranslations', + 'stabbingwithasyringe', + 'tinytranslation', + 'vampiramtl', + 'zetrotranslation', + ]; + if (domainParts.some(d => outliers.includes(d))) { + isWordPress = false; + isBlogspot = false; + } - // Fetch chapter content based on detected platform - if (!isWordPress && !isBlogspot) { - chapterText = await this.getChapterBody( - loadedCheerio, - domainParts, - url, - ); - } else { - chapterText = this.extractPlatformContent(loadedCheerio, isBlogspot); - } + // Last edited in 0.9.0 - 19/03/2025 + /** + * Blogspot sites: + * - ¼-Assed + * - AsuraTls (Outlier) + * - FictionRead (Outlier) + * - Novel World Translations (Outlier) + * - SacredText TL (Outlier) + * - Toasteful + * + * WordPress sites: + * - Anomlaously Creative (Outlier) + * - Arcane Translations (Outlier) + * - Blossom Translation + * - Darkstar Translations (Outlier) + * - Dumahs Translations + * - ElloMTL + * - Femme Fables + * - Gadgetized Panda Translation + * - Gem Novels + * - Goblinslate + * - GreenzTL + * - Hel Scans (Outlier) + * - ippotranslations + * - JATranslations + * - Light Novels Translations + * - Mirilu - Novel Reader Attempts Translating (Outlier) + * - Neosekai Translations + * - Shanghai Fantasy + * - Soafp (Manually added) + * - Stabbing with a Syringe (Outlier) + * - StoneScape + * - TinyTL (Outlier) + * - VampiraMTL (Outlier) + * - Wonder Novels + * - Yong Library + * - Zetro Translation (Outlier) + */ + + // Fetch chapter content based on detected platform + if (!isWordPress && !isBlogspot) { + chapterText = await this.getChapterBody(loadedCheerio, domainParts, url); + } else { + const bloatElements = isBlogspot + ? ['.button-container', '.ChapterNav', '.ch-bottom', '.separator'] + : [ + '.ad', + '.author-avatar', + '.chapter-warning', + '.entry-meta', + '.ezoic-ad', + '.mb-center', + '.modern-footnotes-footnote__note', + '.patreon-widget', + '.post-cats', + '.pre-bar', + '.sharedaddy', + '.sidebar', + '.swg-button-v2-light', + '.wp-block-buttons', + //'.wp-block-columns', + '.wp-dark-mode-switcher', + '.wp-next-post-navi', + '#hpk', + '#jp-post-flair', + '#textbox', + ]; + + bloatElements.forEach(tag => loadedCheerio(tag).remove()); + + // Extract title + const titleSelectors = isBlogspot + ? ['.entry-title', '.post-title', 'head title'] + : [ + '.entry-title', + '.chapter__title', + '.title-content', + '.wp-block-post-title', + '.title_story', + '#chapter-heading', + 'head title', + 'h1:first-of-type', + 'h2:first-of-type', + '.active', + ]; + let chapterTitle = titleSelectors + .map(sel => loadedCheerio(sel).first().text()) + .find(text => text); + + // Extract subtitle (if any) + const chapterSubtitle = + loadedCheerio('.cat-series').first().text() || + loadedCheerio('h1.leading-none ~ span').first().text(); + if (chapterSubtitle) chapterTitle = chapterSubtitle; + + // Extract content + const contentSelectors = isBlogspot + ? ['.content-post', '.entry-content', '.post-body'] + : [ + '.chapter__content', + '.entry-content', + '.text_story', + '.post-content', + '.contenta', + '.single_post', + '.main-content', + '.reader-content', + '#content', + '#the-content', + 'article.post', + ]; + const chapterContent = contentSelectors + .map(sel => loadedCheerio(sel).html()!) + .find(html => html); - // Fallback content extraction - if (!chapterText) { - ['nav', 'header', 'footer', '.hidden'].forEach(tag => - loadedCheerio(tag).remove(), - ); - chapterText = loadedCheerio('body').html()!; + if (chapterTitle) { + chapterText = `

${chapterTitle}



${chapterContent}`; + } else { + chapterText = chapterContent; } - - // Convert relative URLs to absolute - chapterText = chapterText.replace( - /href="\//g, - `href="${this.getLocation(result.url)}/`, - ); - - // Process images - const chapterCheerio = parseHTML(chapterText); - chapterCheerio('noscript').remove(); - - chapterCheerio('img').each((_, el) => { - const $el = chapterCheerio(el); - - // Only update if the lazy-loaded attribute exists - if ($el.attr('data-lazy-src')) { - $el.attr('src', $el.attr('data-lazy-src')); - } - if ($el.attr('data-lazy-srcset')) { - $el.attr('srcset', $el.attr('data-lazy-srcset')); - } - - // Remove lazy-loading class if it exists - if ($el.hasClass('lazyloaded')) { - $el.removeClass('lazyloaded'); - } - }); - - return chapterCheerio.html()!; - } catch (error) { - console.error('Fetch Error:', error); - throw new Error(`Network request failed: ${error}`); } - } - private async fetchWithRetry( - url: string, - options: any, - maxRetries = 3, - ): Promise { - let result; - for (let attempt = 1; attempt <= maxRetries; attempt++) { - try { - console.log(`Fetch Attempt ${attempt} for URL:`, url); - result = await fetchApi(url, options); - console.log('Response Status:', result?.status); - if (!result || result.status === 0 || result.status === undefined) { - throw new Error( - 'Invalid response status: ' + (result?.status || 'undefined'), - ); - } - break; // Success, exit retry loop - } catch (error) { - const err = error as any; - console.error(`Fetch Attempt ${attempt} Error:`, { - message: err.message, - stack: err.stack, - }); - - if (attempt === maxRetries) { - console.log( - 'Max retries reached, falling back to webview for URL:', - url, - ); - throw new Error( - `Network request failed after ${maxRetries} attempts, please open in webview.`, - ); - } - await new Promise(resolve => setTimeout(resolve, 2000)); // 2-second delay between retries - } + // Fallback content extraction + if (!chapterText) { + ['nav', 'header', 'footer', '.hidden'].forEach(tag => + loadedCheerio(tag).remove(), + ); + chapterText = loadedCheerio('body').html()!; } - return result!; - } - private extractPlatformContent( - loadedCheerio: any, - isBlogspot: boolean, - ): string { - const bloatElements = isBlogspot - ? ['.button-container', '.ChapterNav', '.ch-bottom', '.separator'] - : [ - '.ad', - '.author-avatar', - '.chapter-warning', - '.entry-meta', - '.ezoic-ad', - '.mb-center', - '.modern-footnotes-footnote__note', - '.patreon-widget', - '.post-cats', - '.pre-bar', - '.sharedaddy', - '.sidebar', - '.swg-button-v2-light', - '.wp-block-buttons', - //'.wp-block-columns', - '.wp-dark-mode-switcher', - '.wp-next-post-navi', - '#hpk', - '#jp-post-flair', - '#textbox', - ]; + // Convert relative URLs to absolute + chapterText = chapterText.replace( + /href="\//g, + `href="${this.getLocation(result.url)}/`, + ); - bloatElements.forEach(tag => loadedCheerio(tag).remove()); + // Process images + const chapterCheerio = parseHTML(chapterText); + chapterCheerio('noscript').remove(); - // Extract title - const titleSelectors = isBlogspot - ? ['.entry-title', '.post-title', 'head title'] - : [ - '.entry-title', - '.chapter__title', - '.title-content', - '.wp-block-post-title', - '.title_story', - '#chapter-heading', - 'head title', - 'h1:first-of-type', - 'h2:first-of-type', - '.active', - ]; - let chapterTitle = titleSelectors - .map(sel => loadedCheerio(sel).first().text()) - .find(text => text); + chapterCheerio('img').each((_, el) => { + const $el = chapterCheerio(el); - // Extract subtitle (if any) - const chapterSubtitle = - loadedCheerio('.cat-series').first().text() || - loadedCheerio('h1.leading-none ~ span').first().text(); - if (chapterSubtitle) chapterTitle = chapterSubtitle; + // Only update if the lazy-loaded attribute exists + if ($el.attr('data-lazy-src')) { + $el.attr('src', $el.attr('data-lazy-src')); + } + if ($el.attr('data-lazy-srcset')) { + $el.attr('srcset', $el.attr('data-lazy-srcset')); + } - // Extract content - const contentSelectors = isBlogspot - ? ['.content-post', '.entry-content', '.post-body'] - : [ - '.chapter__content', - '.entry-content', - '.text_story', - '.post-content', - '.contenta', - '.single_post', - '.main-content', - '.reader-content', - '#content', - '#the-content', - 'article.post', - ]; - const chapterContent = contentSelectors - .map(sel => loadedCheerio(sel).html()!) - .find(html => html); + // Remove lazy-loading class if it exists + if ($el.hasClass('lazyloaded')) { + $el.removeClass('lazyloaded'); + } + }); - if (chapterTitle) { - return `

${chapterTitle}



${chapterContent}`; - } else { - return chapterContent; - } + return chapterCheerio.html()!; } async searchNovels( From 042009a337f0a3e4f48e62baa381d2deceb350b6 Mon Sep 17 00:00:00 2001 From: Batorian Date: Tue, 8 Jul 2025 12:08:22 +0200 Subject: [PATCH 25/35] adding debug logs --- src/plugins/english/novelupdates.ts | 450 +++++++++++++++------------- 1 file changed, 240 insertions(+), 210 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 171b9ddfc..22df112f8 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -22,7 +22,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.11'; + version = '0.10.12'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -743,239 +743,269 @@ class NovelUpdates implements Plugin.PluginBase { async parseChapter(chapterPath: string): Promise { let chapterText; + try { + const requestUrl = this.site + chapterPath; + console.log('Request URL:', requestUrl); - const result = await fetchApi(this.site + chapterPath); - const body = await result.text(); - const url = result.url; - const domainParts = url.toLowerCase().split('/')[2].split('.'); + const result = await fetchApi(requestUrl); + console.log('Response Status:', result.status); + console.log('Final URL:', result.url); + if (result.headers && typeof result.headers.forEach === 'function') { + const headersObj: Record = {}; + result.headers.forEach((value, key) => { + headersObj[key] = value; + }); + console.log('Response Headers:', headersObj); + } - const loadedCheerio = parseHTML(body); + const body = await result.text(); + console.log('Response Body (first 500 chars):', body.substring(0, 500)); + const url = result.url; + const domainParts = url.toLowerCase().split('/')[2].split('.'); - // Handle CAPTCHA cases - const blockedTitles = [ - 'bot verification', - 'just a moment...', - 'redirecting...', - 'un instant...', - 'you are being redirected...', - ]; - const title = loadedCheerio('title').text().trim().toLowerCase(); - if (blockedTitles.includes(title)) { - throw new Error('Captcha detected, please open in webview.'); - } + const loadedCheerio = parseHTML(body); + + // Handle CAPTCHA cases + const blockedTitles = [ + 'bot verification', + 'just a moment...', + 'redirecting...', + 'un instant...', + 'you are being redirected...', + ]; + const title = loadedCheerio('title').text().trim().toLowerCase(); + console.log('Page Title:', title); + if (blockedTitles.includes(title)) { + console.log('Falling back to webview for URL:', requestUrl); + throw new Error('Captcha detected, please open in webview.'); + } + + // Check if chapter url is wrong or site is down + if (!result.ok) { + console.log('Fetch failed:', result.status, result.statusText); + throw new Error( + `Failed to fetch ${result.url}: ${result.status} ${result.statusText}`, + ); + } - // Check if chapter url is wrong or site is down - if (!result.ok) { - throw new Error( - `Failed to fetch ${result.url}: ${result.status} ${result.statusText}`, + // Detect platforms + let isBlogspot = ['blogspot', 'blogger'].some(keyword => + [ + loadedCheerio('meta[name="google-adsense-platform-domain"]').attr( + 'content', + ), + loadedCheerio('meta[name="generator"]').attr('content'), + ].some(meta => meta?.toLowerCase().includes(keyword)), ); - } - // Detect platforms - let isBlogspot = ['blogspot', 'blogger'].some(keyword => - [ - loadedCheerio('meta[name="google-adsense-platform-domain"]').attr( - 'content', - ), - loadedCheerio('meta[name="generator"]').attr('content'), - ].some(meta => meta?.toLowerCase().includes(keyword)), - ); + let isWordPress = ['wordpress', 'site kit by google'].some(keyword => + [ + loadedCheerio('#dcl_comments-js-extra').html(), + loadedCheerio('meta[name="generator"]').attr('content'), + loadedCheerio('.powered-by').text(), + loadedCheerio('footer').text(), + ].some(meta => meta?.toLowerCase().includes(keyword)), + ); - let isWordPress = ['wordpress', 'site kit by google'].some(keyword => - [ - loadedCheerio('#dcl_comments-js-extra').html(), - loadedCheerio('meta[name="generator"]').attr('content'), - loadedCheerio('.powered-by').text(), - loadedCheerio('footer').text(), - ].some(meta => meta?.toLowerCase().includes(keyword)), - ); + // Manually set WordPress flag for known sites + const manualWordPress = ['etherreads', 'greenztl2', 'soafp']; + if ( + !isWordPress && + domainParts.some(wp => manualWordPress.includes(wp)) + ) { + isWordPress = true; + } - // Manually set WordPress flag for known sites - const manualWordPress = ['etherreads', 'greenztl2', 'soafp']; - if (!isWordPress && domainParts.some(wp => manualWordPress.includes(wp))) { - isWordPress = true; - } + // Handle outlier sites + const outliers = [ + 'anotivereads', + 'arcanetranslations', + 'asuratls', + 'darkstartranslations', + 'fictionread', + 'helscans', + 'infinitenoveltranslations', + 'mirilu', + 'novelworldtranslations', + 'sacredtexttranslations', + 'stabbingwithasyringe', + 'tinytranslation', + 'vampiramtl', + 'zetrotranslation', + ]; + if (domainParts.some(d => outliers.includes(d))) { + isWordPress = false; + isBlogspot = false; + } - // Handle outlier sites - const outliers = [ - 'anotivereads', - 'arcanetranslations', - 'asuratls', - 'darkstartranslations', - 'fictionread', - 'helscans', - 'infinitenoveltranslations', - 'mirilu', - 'novelworldtranslations', - 'sacredtexttranslations', - 'stabbingwithasyringe', - 'tinytranslation', - 'vampiramtl', - 'zetrotranslation', - ]; - if (domainParts.some(d => outliers.includes(d))) { - isWordPress = false; - isBlogspot = false; - } + // Last edited in 0.9.0 - 19/03/2025 + /** + * Blogspot sites: + * - ¼-Assed + * - AsuraTls (Outlier) + * - FictionRead (Outlier) + * - Novel World Translations (Outlier) + * - SacredText TL (Outlier) + * - Toasteful + * + * WordPress sites: + * - Anomlaously Creative (Outlier) + * - Arcane Translations (Outlier) + * - Blossom Translation + * - Darkstar Translations (Outlier) + * - Dumahs Translations + * - ElloMTL + * - Femme Fables + * - Gadgetized Panda Translation + * - Gem Novels + * - Goblinslate + * - GreenzTL + * - Hel Scans (Outlier) + * - ippotranslations + * - JATranslations + * - Light Novels Translations + * - Mirilu - Novel Reader Attempts Translating (Outlier) + * - Neosekai Translations + * - Shanghai Fantasy + * - Soafp (Manually added) + * - Stabbing with a Syringe (Outlier) + * - StoneScape + * - TinyTL (Outlier) + * - VampiraMTL (Outlier) + * - Wonder Novels + * - Yong Library + * - Zetro Translation (Outlier) + */ - // Last edited in 0.9.0 - 19/03/2025 - /** - * Blogspot sites: - * - ¼-Assed - * - AsuraTls (Outlier) - * - FictionRead (Outlier) - * - Novel World Translations (Outlier) - * - SacredText TL (Outlier) - * - Toasteful - * - * WordPress sites: - * - Anomlaously Creative (Outlier) - * - Arcane Translations (Outlier) - * - Blossom Translation - * - Darkstar Translations (Outlier) - * - Dumahs Translations - * - ElloMTL - * - Femme Fables - * - Gadgetized Panda Translation - * - Gem Novels - * - Goblinslate - * - GreenzTL - * - Hel Scans (Outlier) - * - ippotranslations - * - JATranslations - * - Light Novels Translations - * - Mirilu - Novel Reader Attempts Translating (Outlier) - * - Neosekai Translations - * - Shanghai Fantasy - * - Soafp (Manually added) - * - Stabbing with a Syringe (Outlier) - * - StoneScape - * - TinyTL (Outlier) - * - VampiraMTL (Outlier) - * - Wonder Novels - * - Yong Library - * - Zetro Translation (Outlier) - */ + // Fetch chapter content based on detected platform + if (!isWordPress && !isBlogspot) { + chapterText = await this.getChapterBody( + loadedCheerio, + domainParts, + url, + ); + } else { + const bloatElements = isBlogspot + ? ['.button-container', '.ChapterNav', '.ch-bottom', '.separator'] + : [ + '.ad', + '.author-avatar', + '.chapter-warning', + '.entry-meta', + '.ezoic-ad', + '.mb-center', + '.modern-footnotes-footnote__note', + '.patreon-widget', + '.post-cats', + '.pre-bar', + '.sharedaddy', + '.sidebar', + '.swg-button-v2-light', + '.wp-block-buttons', + //'.wp-block-columns', + '.wp-dark-mode-switcher', + '.wp-next-post-navi', + '#hpk', + '#jp-post-flair', + '#textbox', + ]; - // Fetch chapter content based on detected platform - if (!isWordPress && !isBlogspot) { - chapterText = await this.getChapterBody(loadedCheerio, domainParts, url); - } else { - const bloatElements = isBlogspot - ? ['.button-container', '.ChapterNav', '.ch-bottom', '.separator'] - : [ - '.ad', - '.author-avatar', - '.chapter-warning', - '.entry-meta', - '.ezoic-ad', - '.mb-center', - '.modern-footnotes-footnote__note', - '.patreon-widget', - '.post-cats', - '.pre-bar', - '.sharedaddy', - '.sidebar', - '.swg-button-v2-light', - '.wp-block-buttons', - //'.wp-block-columns', - '.wp-dark-mode-switcher', - '.wp-next-post-navi', - '#hpk', - '#jp-post-flair', - '#textbox', - ]; + bloatElements.forEach(tag => loadedCheerio(tag).remove()); - bloatElements.forEach(tag => loadedCheerio(tag).remove()); + // Extract title + const titleSelectors = isBlogspot + ? ['.entry-title', '.post-title', 'head title'] + : [ + '.entry-title', + '.chapter__title', + '.title-content', + '.wp-block-post-title', + '.title_story', + '#chapter-heading', + 'head title', + 'h1:first-of-type', + 'h2:first-of-type', + '.active', + ]; + let chapterTitle = titleSelectors + .map(sel => loadedCheerio(sel).first().text()) + .find(text => text); - // Extract title - const titleSelectors = isBlogspot - ? ['.entry-title', '.post-title', 'head title'] - : [ - '.entry-title', - '.chapter__title', - '.title-content', - '.wp-block-post-title', - '.title_story', - '#chapter-heading', - 'head title', - 'h1:first-of-type', - 'h2:first-of-type', - '.active', - ]; - let chapterTitle = titleSelectors - .map(sel => loadedCheerio(sel).first().text()) - .find(text => text); + // Extract subtitle (if any) + const chapterSubtitle = + loadedCheerio('.cat-series').first().text() || + loadedCheerio('h1.leading-none ~ span').first().text(); + if (chapterSubtitle) chapterTitle = chapterSubtitle; - // Extract subtitle (if any) - const chapterSubtitle = - loadedCheerio('.cat-series').first().text() || - loadedCheerio('h1.leading-none ~ span').first().text(); - if (chapterSubtitle) chapterTitle = chapterSubtitle; + // Extract content + const contentSelectors = isBlogspot + ? ['.content-post', '.entry-content', '.post-body'] + : [ + '.chapter__content', + '.entry-content', + '.text_story', + '.post-content', + '.contenta', + '.single_post', + '.main-content', + '.reader-content', + '#content', + '#the-content', + 'article.post', + ]; + const chapterContent = contentSelectors + .map(sel => loadedCheerio(sel).html()!) + .find(html => html); - // Extract content - const contentSelectors = isBlogspot - ? ['.content-post', '.entry-content', '.post-body'] - : [ - '.chapter__content', - '.entry-content', - '.text_story', - '.post-content', - '.contenta', - '.single_post', - '.main-content', - '.reader-content', - '#content', - '#the-content', - 'article.post', - ]; - const chapterContent = contentSelectors - .map(sel => loadedCheerio(sel).html()!) - .find(html => html); + if (chapterTitle) { + chapterText = `

${chapterTitle}



${chapterContent}`; + } else { + chapterText = chapterContent; + } + } - if (chapterTitle) { - chapterText = `

${chapterTitle}



${chapterContent}`; - } else { - chapterText = chapterContent; + // Fallback content extraction + if (!chapterText) { + console.log('Fallback: extracting from body'); + ['nav', 'header', 'footer', '.hidden'].forEach(tag => + loadedCheerio(tag).remove(), + ); + chapterText = loadedCheerio('body').html()!; } - } - // Fallback content extraction - if (!chapterText) { - ['nav', 'header', 'footer', '.hidden'].forEach(tag => - loadedCheerio(tag).remove(), + // Convert relative URLs to absolute + chapterText = chapterText.replace( + /href="\//g, + `href="${this.getLocation(result.url)}/`, ); - chapterText = loadedCheerio('body').html()!; - } - // Convert relative URLs to absolute - chapterText = chapterText.replace( - /href="\//g, - `href="${this.getLocation(result.url)}/`, - ); - - // Process images - const chapterCheerio = parseHTML(chapterText); - chapterCheerio('noscript').remove(); + // Process images + const chapterCheerio = parseHTML(chapterText); + chapterCheerio('noscript').remove(); - chapterCheerio('img').each((_, el) => { - const $el = chapterCheerio(el); + chapterCheerio('img').each((_, el) => { + const $el = chapterCheerio(el); - // Only update if the lazy-loaded attribute exists - if ($el.attr('data-lazy-src')) { - $el.attr('src', $el.attr('data-lazy-src')); - } - if ($el.attr('data-lazy-srcset')) { - $el.attr('srcset', $el.attr('data-lazy-srcset')); - } + // Only update if the lazy-loaded attribute exists + if ($el.attr('data-lazy-src')) { + $el.attr('src', $el.attr('data-lazy-src')); + } + if ($el.attr('data-lazy-srcset')) { + $el.attr('srcset', $el.attr('data-lazy-srcset')); + } - // Remove lazy-loading class if it exists - if ($el.hasClass('lazyloaded')) { - $el.removeClass('lazyloaded'); - } - }); + // Remove lazy-loading class if it exists + if ($el.hasClass('lazyloaded')) { + $el.removeClass('lazyloaded'); + } + }); - return chapterCheerio.html()!; + console.log('Returning chapter HTML'); + return chapterCheerio.html()!; + } catch (error) { + console.error('Fetch Error:', error); + throw new Error(`Network request failed: ${error}`); + } } async searchNovels( From 454e0b7b46217334186b2a39f63fc8a8f81861f8 Mon Sep 17 00:00:00 2001 From: Batorian Date: Tue, 8 Jul 2025 12:15:53 +0200 Subject: [PATCH 26/35] add fetch options --- src/plugins/english/novelupdates.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 22df112f8..0dd920a1c 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -22,7 +22,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.12'; + version = '0.10.13'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -747,7 +747,7 @@ class NovelUpdates implements Plugin.PluginBase { const requestUrl = this.site + chapterPath; console.log('Request URL:', requestUrl); - const result = await fetchApi(requestUrl); + const result = await fetchApi(requestUrl, this.fetchOptions); console.log('Response Status:', result.status); console.log('Final URL:', result.url); if (result.headers && typeof result.headers.forEach === 'function') { From 01c887121c2efe2f9eb281467151a63bf6e20b84 Mon Sep 17 00:00:00 2001 From: Batorian Date: Tue, 8 Jul 2025 12:23:35 +0200 Subject: [PATCH 27/35] add redirect url --- src/plugins/english/novelupdates.ts | 36 +++++++++++++++++++---------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 0dd920a1c..bc0ed7b8f 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -22,7 +22,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.13'; + version = '0.10.14'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -747,20 +747,28 @@ class NovelUpdates implements Plugin.PluginBase { const requestUrl = this.site + chapterPath; console.log('Request URL:', requestUrl); - const result = await fetchApi(requestUrl, this.fetchOptions); - console.log('Response Status:', result.status); - console.log('Final URL:', result.url); - if (result.headers && typeof result.headers.forEach === 'function') { + const result = await fetchApi(requestUrl, { + ...this.fetchOptions, + method: 'HEAD', + }); + const redirectUrl = result.url; + const finalResult = await fetchApi(redirectUrl, this.fetchOptions); + console.log('Response Status:', finalResult.status); + console.log('Final URL:', finalResult.url); + if ( + finalResult.headers && + typeof finalResult.headers.forEach === 'function' + ) { const headersObj: Record = {}; - result.headers.forEach((value, key) => { + finalResult.headers.forEach((value, key) => { headersObj[key] = value; }); console.log('Response Headers:', headersObj); } - const body = await result.text(); + const body = await finalResult.text(); console.log('Response Body (first 500 chars):', body.substring(0, 500)); - const url = result.url; + const url = finalResult.url; const domainParts = url.toLowerCase().split('/')[2].split('.'); const loadedCheerio = parseHTML(body); @@ -781,10 +789,14 @@ class NovelUpdates implements Plugin.PluginBase { } // Check if chapter url is wrong or site is down - if (!result.ok) { - console.log('Fetch failed:', result.status, result.statusText); + if (!finalResult.ok) { + console.log( + 'Fetch failed:', + finalResult.status, + finalResult.statusText, + ); throw new Error( - `Failed to fetch ${result.url}: ${result.status} ${result.statusText}`, + `Failed to fetch ${finalResult.url}: ${finalResult.status} ${finalResult.statusText}`, ); } @@ -976,7 +988,7 @@ class NovelUpdates implements Plugin.PluginBase { // Convert relative URLs to absolute chapterText = chapterText.replace( /href="\//g, - `href="${this.getLocation(result.url)}/`, + `href="${this.getLocation(finalResult.url)}/`, ); // Process images From 52c254652231e7cf1d267406a02d039745fcd15e Mon Sep 17 00:00:00 2001 From: Batorian Date: Tue, 8 Jul 2025 12:41:04 +0200 Subject: [PATCH 28/35] add noice translations support --- src/plugins/english/novelupdates.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index bc0ed7b8f..e0bf4f9cd 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -580,7 +580,7 @@ class NovelUpdates implements Plugin.PluginBase { } break; } - // Last edited in 0.10.0 by Batorian - 02/07/2025 + // Last edited in 0.10.0 by Batorian - 08/07/2025 case 'stellarrealm': { // Modular extraction inspired by W2e const extractStellarRealmContent = (cheerioInstance: CheerioAPI) => { @@ -820,7 +820,12 @@ class NovelUpdates implements Plugin.PluginBase { ); // Manually set WordPress flag for known sites - const manualWordPress = ['etherreads', 'greenztl2', 'soafp']; + const manualWordPress = [ + 'etherreads', + 'greenztl2', + 'noicetranslations', + 'soafp', + ]; if ( !isWordPress && domainParts.some(wp => manualWordPress.includes(wp)) @@ -850,7 +855,7 @@ class NovelUpdates implements Plugin.PluginBase { isBlogspot = false; } - // Last edited in 0.9.0 - 19/03/2025 + // Last edited in 0.10.0 - 08/07/2025 /** * Blogspot sites: * - ¼-Assed @@ -878,6 +883,7 @@ class NovelUpdates implements Plugin.PluginBase { * - Light Novels Translations * - Mirilu - Novel Reader Attempts Translating (Outlier) * - Neosekai Translations + * - Noice Translations * - Shanghai Fantasy * - Soafp (Manually added) * - Stabbing with a Syringe (Outlier) From 21b49d9a751dab5f05d4e3b346976f103b643f7f Mon Sep 17 00:00:00 2001 From: Batorian Date: Tue, 8 Jul 2025 12:44:37 +0200 Subject: [PATCH 29/35] revert HEAD request to fix captcha errors --- src/plugins/english/novelupdates.ts | 36 ++++++++++------------------- 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index e0bf4f9cd..b374b64de 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -22,7 +22,7 @@ class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.14'; + version = '0.10.15'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -747,28 +747,20 @@ class NovelUpdates implements Plugin.PluginBase { const requestUrl = this.site + chapterPath; console.log('Request URL:', requestUrl); - const result = await fetchApi(requestUrl, { - ...this.fetchOptions, - method: 'HEAD', - }); - const redirectUrl = result.url; - const finalResult = await fetchApi(redirectUrl, this.fetchOptions); - console.log('Response Status:', finalResult.status); - console.log('Final URL:', finalResult.url); - if ( - finalResult.headers && - typeof finalResult.headers.forEach === 'function' - ) { + const result = await fetchApi(requestUrl, this.fetchOptions); + console.log('Response Status:', result.status); + console.log('Final URL:', result.url); + if (result.headers && typeof result.headers.forEach === 'function') { const headersObj: Record = {}; - finalResult.headers.forEach((value, key) => { + result.headers.forEach((value, key) => { headersObj[key] = value; }); console.log('Response Headers:', headersObj); } - const body = await finalResult.text(); + const body = await result.text(); console.log('Response Body (first 500 chars):', body.substring(0, 500)); - const url = finalResult.url; + const url = result.url; const domainParts = url.toLowerCase().split('/')[2].split('.'); const loadedCheerio = parseHTML(body); @@ -789,14 +781,10 @@ class NovelUpdates implements Plugin.PluginBase { } // Check if chapter url is wrong or site is down - if (!finalResult.ok) { - console.log( - 'Fetch failed:', - finalResult.status, - finalResult.statusText, - ); + if (!result.ok) { + console.log('Fetch failed:', result.status, result.statusText); throw new Error( - `Failed to fetch ${finalResult.url}: ${finalResult.status} ${finalResult.statusText}`, + `Failed to fetch ${result.url}: ${result.status} ${result.statusText}`, ); } @@ -994,7 +982,7 @@ class NovelUpdates implements Plugin.PluginBase { // Convert relative URLs to absolute chapterText = chapterText.replace( /href="\//g, - `href="${this.getLocation(finalResult.url)}/`, + `href="${this.getLocation(result.url)}/`, ); // Process images From 0d4a92e297f55690fa214e73d539f9b0855cbfd2 Mon Sep 17 00:00:00 2001 From: Batorian Date: Tue, 8 Jul 2025 12:50:46 +0200 Subject: [PATCH 30/35] add user agent --- src/plugins/english/novelupdates.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index b374b64de..7b97b5197 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -6,23 +6,21 @@ import { Plugin } from '@typings/plugin'; class NovelUpdates implements Plugin.PluginBase { private fetchOptions = { headers: { + 'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:137.0) Gecko/20100101 Firefox/137.0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,us;q=0.5', - 'Upgrade-Insecure-Requests': '1', - 'Sec-Fetch-Dest': 'document', - 'Sec-Fetch-Mode': 'navigate', - 'Sec-Fetch-Site': 'none', - 'Sec-Fetch-User': '?1', - 'Accept-Encoding': 'gzip, deflate', - 'Referer': 'https://www.novelupdates.com/', + 'Referer': 'https://www.novelupdates.com/', // Referer + 'DNT': '1', // Do Not Track + 'Upgrade-Insecure-Requests': '1', // Upgrade-Insecure-Requests }, redirect: 'follow', }; id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.15'; + version = '0.10.16'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; From 5741c8a78ffe5bf77741ca7337d2b95e05546bd0 Mon Sep 17 00:00:00 2001 From: Batorian Date: Tue, 8 Jul 2025 13:03:06 +0200 Subject: [PATCH 31/35] remove fetch options and add head request --- src/plugins/english/novelupdates.ts | 50 +++++++++++++---------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 7b97b5197..f35aea9ad 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -4,23 +4,9 @@ import { Filters, FilterTypes } from '@libs/filterInputs'; import { Plugin } from '@typings/plugin'; class NovelUpdates implements Plugin.PluginBase { - private fetchOptions = { - headers: { - 'User-Agent': - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:137.0) Gecko/20100101 Firefox/137.0', - 'Accept': - 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', - 'Accept-Language': 'en-US,us;q=0.5', - 'Referer': 'https://www.novelupdates.com/', // Referer - 'DNT': '1', // Do Not Track - 'Upgrade-Insecure-Requests': '1', // Upgrade-Insecure-Requests - }, - redirect: 'follow', - }; - id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.16'; + version = '0.10.17'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -277,9 +263,7 @@ class NovelUpdates implements Plugin.PluginBase { const url = `${chapterPath}/__data.json?x-sveltekit-invalidated=001`; try { // Fetch the chapter's data in JSON format - const json = await fetchApi(url, this.fetchOptions).then(r => - r.json(), - ); + const json = await fetchApi(url).then(r => r.json()); const nodes = json.nodes; const data = nodes .filter((node: { type: string }) => node.type === 'data') @@ -351,7 +335,7 @@ class NovelUpdates implements Plugin.PluginBase { // Get the chapter link from the main page const url = loadedCheerio('article > p > a').first().attr('href')!; if (url) { - const result = await fetchApi(url, this.fetchOptions); + const result = await fetchApi(url); const body = await result.text(); loadedCheerio = parseHTML(body); } @@ -447,7 +431,7 @@ class NovelUpdates implements Plugin.PluginBase { case 'raeitranslations': { const parts = chapterPath.split('/'); const url = `${parts[0]}//api.${parts[2]}/api/chapters/single?id=${parts[3]}&num=${parts[4]}`; - const json = await fetchApi(url, this.fetchOptions).then(r => r.json()); + const json = await fetchApi(url).then(r => r.json()); const titleElement = `Chapter ${json.currentChapter.chapTag}`; chapterTitle = json.currentChapter.chapTitle ? `${titleElement} - ${json.currentChapter.chapTitle}` @@ -495,7 +479,7 @@ class NovelUpdates implements Plugin.PluginBase { const chapterId = chapterPath.split('/').pop(); chapterTitle = `Chapter ${chapterId}`; const url = `${chapterPath.split('chapter')[0]}txt/${chapterId}.txt`; - chapterContent = await fetchApi(url, this.fetchOptions) + chapterContent = await fetchApi(url) .then(r => r.text()) .then(text => { // Split text into sentences based on newline characters @@ -558,7 +542,7 @@ class NovelUpdates implements Plugin.PluginBase { // Get the chapter link from the main page const url = loadedCheerio('.entry-content a').attr('href')!; if (url) { - const result = await fetchApi(url, this.fetchOptions); + const result = await fetchApi(url); const body = await result.text(); loadedCheerio = parseHTML(body); } @@ -662,7 +646,7 @@ class NovelUpdates implements Plugin.PluginBase { // Get the chapter link from the main page const url = loadedCheerio('.entry-content a').attr('href')!; if (url) { - const result = await fetchApi(chapterPath + url, this.fetchOptions); + const result = await fetchApi(chapterPath + url); const body = await result.text(); loadedCheerio = parseHTML(body); } @@ -709,10 +693,8 @@ class NovelUpdates implements Plugin.PluginBase { case 'yoru': { const chapterId = chapterPath.split('/').pop(); const url = `https://pxp-main-531j.onrender.com/api/v1/book_chapters/${chapterId}/content`; - const json = await fetchApi(url, this.fetchOptions).then(r => r.json()); - chapterText = await fetchApi(json, this.fetchOptions).then(r => - r.text(), - ); + const json = await fetchApi(url).then(r => r.json()); + chapterText = await fetchApi(json).then(r => r.text()); break; } // Last edited in 0.9.0 by Batorian - 19/03/2025 @@ -745,7 +727,19 @@ class NovelUpdates implements Plugin.PluginBase { const requestUrl = this.site + chapterPath; console.log('Request URL:', requestUrl); - const result = await fetchApi(requestUrl, this.fetchOptions); + // HEAD fetch to get the final redirect URL + let headUrl = requestUrl; + try { + const headResult = await fetchApi(requestUrl, { method: 'HEAD' }); + if (headResult && headResult.url) { + headUrl = headResult.url; + console.log('HEAD redirect URL:', headUrl); + } + } catch (e) { + console.log('HEAD fetch failed, using original URL'); + } + + const result = await fetchApi(headUrl); console.log('Response Status:', result.status); console.log('Final URL:', result.url); if (result.headers && typeof result.headers.forEach === 'function') { From bf0c8a9cc61a11076cca1ff530e5d7c900a46870 Mon Sep 17 00:00:00 2001 From: Batorian Date: Tue, 8 Jul 2025 13:14:34 +0200 Subject: [PATCH 32/35] update WP detection --- src/plugins/english/novelupdates.ts | 69 ++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 16 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index f35aea9ad..0e9bbe77a 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -6,7 +6,7 @@ import { Plugin } from '@typings/plugin'; class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.17'; + version = '0.10.18'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -790,27 +790,64 @@ class NovelUpdates implements Plugin.PluginBase { ].some(meta => meta?.toLowerCase().includes(keyword)), ); - let isWordPress = ['wordpress', 'site kit by google'].some(keyword => - [ - loadedCheerio('#dcl_comments-js-extra').html(), - loadedCheerio('meta[name="generator"]').attr('content'), - loadedCheerio('.powered-by').text(), - loadedCheerio('footer').text(), - ].some(meta => meta?.toLowerCase().includes(keyword)), - ); + // Improved WordPress detection with debug logs + let isWordPress = false; + const wpEvidence: string[] = []; + const wpChecks = [ + { + label: 'meta[name="generator"]', + value: loadedCheerio('meta[name="generator"]').attr('content'), + }, + { + label: '#dcl_comments-js-extra', + value: loadedCheerio('#dcl_comments-js-extra').html(), + }, + { label: '.powered-by', value: loadedCheerio('.powered-by').text() }, + { label: 'footer', value: loadedCheerio('footer').text() }, + { label: 'body class', value: loadedCheerio('body').attr('class') }, + { + label: 'wp-content in html', + value: loadedCheerio.html().includes('wp-content') + ? 'wp-content found' + : '', + }, + { + label: 'window._wpemojiSettings', + value: loadedCheerio.html().includes('_wpemojiSettings') + ? '_wpemojiSettings found' + : '', + }, + ]; + for (const check of wpChecks) { + if ( + check.value && + typeof check.value === 'string' && + (check.value.toLowerCase().includes('wordpress') || + check.value.toLowerCase().includes('site kit by google') || + check.value.toLowerCase().includes('wp-content') || + check.value.toLowerCase().includes('wpemoji')) + ) { + isWordPress = true; + wpEvidence.push(`${check.label}: ${check.value}`); + } + } + if (isWordPress) { + console.log('WordPress detected! Evidence:', wpEvidence); + } else { + console.log( + 'WordPress NOT detected. Evidence checked:', + wpChecks.map(c => `${c.label}: ${c.value}`), + ); + } // Manually set WordPress flag for known sites - const manualWordPress = [ - 'etherreads', - 'greenztl2', - 'noicetranslations', - 'soafp', - ]; + const manualWordPress = ['etherreads', 'greenztl2', 'soafp']; if ( !isWordPress && domainParts.some(wp => manualWordPress.includes(wp)) ) { isWordPress = true; + console.log('WordPress manually set for domain:', domainParts); } // Handle outlier sites @@ -850,7 +887,7 @@ class NovelUpdates implements Plugin.PluginBase { * - Arcane Translations (Outlier) * - Blossom Translation * - Darkstar Translations (Outlier) - * - Dumahs Translations + * - Dumah's Translations * - ElloMTL * - Femme Fables * - Gadgetized Panda Translation From 4c529e9c01de95f38e56ab49966d91727928fdf2 Mon Sep 17 00:00:00 2001 From: Batorian Date: Tue, 8 Jul 2025 13:29:22 +0200 Subject: [PATCH 33/35] edit WP content --- src/plugins/english/novelupdates.ts | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 0e9bbe77a..5ad496e48 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -6,7 +6,7 @@ import { Plugin } from '@typings/plugin'; class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.18'; + version = '0.10.19'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -224,20 +224,6 @@ class NovelUpdates implements Plugin.PluginBase { break; } // Last edited in 0.9.0 by Batorian - 19/03/2025 - case 'darkstartranslations': { - chapterTitle = loadedCheerio('ol.breadcrumb li').last().text().trim(); - chapterContent = loadedCheerio('.text-left').html()!; - // Load the extracted chapter content into Cheerio - const chapterCheerio = parseHTML(chapterContent); - // Add an empty row (extra
) after each
element - chapterCheerio('br').each((_, el) => { - chapterCheerio(el).after('
'); // Add one more
for an empty row - }); - // Get the updated content - chapterContent = chapterCheerio.html(); - break; - } - // Last edited in 0.9.0 by Batorian - 19/03/2025 case 'fictionread': { bloatElements = [ '.content > style', @@ -855,7 +841,6 @@ class NovelUpdates implements Plugin.PluginBase { 'anotivereads', 'arcanetranslations', 'asuratls', - 'darkstartranslations', 'fictionread', 'helscans', 'infinitenoveltranslations', @@ -886,7 +871,6 @@ class NovelUpdates implements Plugin.PluginBase { * - Anomlaously Creative (Outlier) * - Arcane Translations (Outlier) * - Blossom Translation - * - Darkstar Translations (Outlier) * - Dumah's Translations * - ElloMTL * - Femme Fables @@ -895,6 +879,7 @@ class NovelUpdates implements Plugin.PluginBase { * - Goblinslate * - GreenzTL * - Hel Scans (Outlier) + * - Hiraeth Translation * - ippotranslations * - JATranslations * - Light Novels Translations @@ -951,6 +936,7 @@ class NovelUpdates implements Plugin.PluginBase { const titleSelectors = isBlogspot ? ['.entry-title', '.post-title', 'head title'] : [ + 'li.active', '.entry-title', '.chapter__title', '.title-content', From d249aa4dd1719208d26507f07f71fe4f691030ee Mon Sep 17 00:00:00 2001 From: Batorian Date: Tue, 8 Jul 2025 13:50:36 +0200 Subject: [PATCH 34/35] update Blogspot and remove manual WP --- src/plugins/english/novelupdates.ts | 71 ++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 5ad496e48..92f0d6856 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -6,7 +6,7 @@ import { Plugin } from '@typings/plugin'; class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.19'; + version = '0.10.20'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/'; @@ -767,14 +767,52 @@ class NovelUpdates implements Plugin.PluginBase { } // Detect platforms - let isBlogspot = ['blogspot', 'blogger'].some(keyword => - [ - loadedCheerio('meta[name="google-adsense-platform-domain"]').attr( - 'content', - ), - loadedCheerio('meta[name="generator"]').attr('content'), - ].some(meta => meta?.toLowerCase().includes(keyword)), - ); + let isBlogspot = false; + const blogspotEvidence: string[] = []; + const blogspotChecks = [ + { + label: 'meta[name="google-adsense-platform-domain"]', + value: loadedCheerio( + 'meta[name="google-adsense-platform-domain"]', + ).attr('content'), + }, + { + label: 'meta[name="generator"]', + value: loadedCheerio('meta[name="generator"]').attr('content'), + }, + { label: 'body class', value: loadedCheerio('body').attr('class') }, + { + label: 'blogspot in html', + value: loadedCheerio.html().includes('blogspot') + ? 'blogspot found' + : '', + }, + { + label: 'blogger in html', + value: loadedCheerio.html().includes('blogger') + ? 'blogger found' + : '', + }, + ]; + for (const check of blogspotChecks) { + if ( + check.value && + typeof check.value === 'string' && + (check.value.toLowerCase().includes('blogspot') || + check.value.toLowerCase().includes('blogger')) + ) { + isBlogspot = true; + blogspotEvidence.push(`${check.label}: ${check.value}`); + } + } + if (isBlogspot) { + console.log('Blogspot detected! Evidence:', blogspotEvidence); + } else { + console.log( + 'Blogspot NOT detected. Evidence checked:', + blogspotChecks.map(c => `${c.label}: ${c.value}`), + ); + } // Improved WordPress detection with debug logs let isWordPress = false; @@ -826,16 +864,6 @@ class NovelUpdates implements Plugin.PluginBase { ); } - // Manually set WordPress flag for known sites - const manualWordPress = ['etherreads', 'greenztl2', 'soafp']; - if ( - !isWordPress && - domainParts.some(wp => manualWordPress.includes(wp)) - ) { - isWordPress = true; - console.log('WordPress manually set for domain:', domainParts); - } - // Handle outlier sites const outliers = [ 'anotivereads', @@ -873,11 +901,12 @@ class NovelUpdates implements Plugin.PluginBase { * - Blossom Translation * - Dumah's Translations * - ElloMTL + * - Ether Reads * - Femme Fables * - Gadgetized Panda Translation * - Gem Novels * - Goblinslate - * - GreenzTL + * - GreenzTL (Taken down) * - Hel Scans (Outlier) * - Hiraeth Translation * - ippotranslations @@ -887,7 +916,7 @@ class NovelUpdates implements Plugin.PluginBase { * - Neosekai Translations * - Noice Translations * - Shanghai Fantasy - * - Soafp (Manually added) + * - Soafp * - Stabbing with a Syringe (Outlier) * - StoneScape * - TinyTL (Outlier) From 01af01fbbc5e9c8f72fca4035c1b34779bd8080f Mon Sep 17 00:00:00 2001 From: Batorian Date: Tue, 8 Jul 2025 13:53:16 +0200 Subject: [PATCH 35/35] revert version --- package.json | 2 +- src/plugins/english/novelupdates.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 89751a370..8138d79c0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "lnreader-plugins", - "version": "3.1.0", + "version": "3.0.0", "description": "Plugins repo for LNReader", "main": "index.js", "type": "module", diff --git a/src/plugins/english/novelupdates.ts b/src/plugins/english/novelupdates.ts index 92f0d6856..6aeb553c3 100644 --- a/src/plugins/english/novelupdates.ts +++ b/src/plugins/english/novelupdates.ts @@ -6,7 +6,7 @@ import { Plugin } from '@typings/plugin'; class NovelUpdates implements Plugin.PluginBase { id = 'novelupdates'; name = 'Novel Updates'; - version = '0.10.20'; + version = '0.10.0'; icon = 'src/en/novelupdates/icon.png'; customCSS = 'src/en/novelupdates/customCSS.css'; site = 'https://www.novelupdates.com/';