From 4e9b1f6ab0b9c18869e9fbe5885cb69899b5db26 Mon Sep 17 00:00:00 2001 From: Eltik <76538547+Eltik@users.noreply.github.com> Date: Wed, 27 Sep 2023 08:50:42 -0400 Subject: [PATCH] fix: Proxy checking + request.ts --- anify-backend/src/helper/request.ts | 124 ++++++------ .../src/proxies/impl/checkProxies.ts | 188 ++++++++---------- anify-backend/src/scripts/checkProxies.ts | 8 +- 3 files changed, 156 insertions(+), 164 deletions(-) diff --git a/anify-backend/src/helper/request.ts b/anify-backend/src/helper/request.ts index 72bac01..f858a83 100644 --- a/anify-backend/src/helper/request.ts +++ b/anify-backend/src/helper/request.ts @@ -23,69 +23,81 @@ export default class Http { } static async request(url: string, config: RequestInit = {}, proxyRequest = true, requests = 0, customProxy: string | undefined = undefined): Promise { - try { - if (proxyRequest) { - const proxyUrl = customProxy || this.getRandomUnbannedProxy(); - if (!proxyUrl) { - return Promise.reject(new Error("No unbanned proxies available")); - } + return new Promise(async (resolve, reject) => { + try { + if (proxyRequest) { + const proxyUrl = customProxy || this.getRandomUnbannedProxy(); + if (!proxyUrl) { + return Promise.reject(new Error("No unbanned proxies available")); + } + + const modifyUrl = `${proxyUrl}/${url}`; - const modifyUrl = `${proxyUrl}/${url}`; - - const controller = new AbortController(); - const id = setTimeout(() => { - this.updateBannedProxies(proxyUrl); - controller.abort(); - }, 5000); - - try { - config = { - ...config, - headers: { - ...config.headers, - Origin: "https://anify.tv", - }, - }; - const response = await fetch(modifyUrl, { signal: controller.signal, ...config }).catch( - (err) => - ({ - ok: false, - status: 500, - statusText: "Timeout", - json: () => Promise.resolve({ error: err }), - }) as Response, - ); - - if (!response.ok) { + const controller = new AbortController(); + const id = setTimeout(() => { this.updateBannedProxies(proxyUrl); - } + controller.abort(); + }, 5000); + + controller.signal.addEventListener("abort", () => { + console.log(colors.red(`http://${proxyUrl} aborted.`)); + + return resolve({ + ok: false, + status: 500, + statusText: "Timeout", + } as Response); + }); - if (response.statusText === "Timeout") { - if (requests >= 3) { - console.log(colors.red("Request timed out. Retried 3 times. Aborting...")); - return response; + try { + config = { + ...config, + headers: { + ...config.headers, + Origin: "https://anify.tv", + }, + }; + const response = await fetch(modifyUrl, { signal: controller.signal, ...config }).catch( + (err) => + ({ + ok: false, + status: 500, + statusText: "Timeout", + json: () => Promise.resolve({ error: err }), + }) as Response, + ); + + if (!response.ok) { + this.updateBannedProxies(proxyUrl); } - return this.request(url, config, proxyRequest, requests + 1); - } + if (response.statusText === "Timeout") { + if (requests >= 3) { + console.log(colors.red("Request timed out. Retried 3 times. Aborting...")); + return response; + } + + return this.request(url, config, proxyRequest, requests + 1); + } - clearTimeout(id); - return response; - } catch (error) { - console.log(proxyUrl); - clearTimeout(id); - throw error; + clearTimeout(id); + return resolve(response); + } catch (error) { + console.log(proxyUrl); + clearTimeout(id); + throw error; + } + } else { + return fetch(url, config); } - } else { - return fetch(url, config); + } catch (e) { + console.log(e); + return resolve({ + ok: false, + status: 500, + statusText: "Internal Server Error", + } as Response); } - } catch (e) { - console.log(e); - return { - ok: false, - status: 500, - statusText: "Internal Server Error", - } as Response; - } + }); } } diff --git a/anify-backend/src/proxies/impl/checkProxies.ts b/anify-backend/src/proxies/impl/checkProxies.ts index f5858d0..0092a1b 100644 --- a/anify-backend/src/proxies/impl/checkProxies.ts +++ b/anify-backend/src/proxies/impl/checkProxies.ts @@ -4,6 +4,11 @@ import ChunkedExecutor from "../executor"; import { CORS_PROXIES } from ".."; import { ANIME_PROVIDERS, BASE_PROVIDERS, MANGA_PROVIDERS, META_PROVIDERS } from "../../mappings"; import { Format, Type } from "../../types/enums"; +import BaseProvider from "../../mappings/impl/base"; +import AnimeProvider from "../../mappings/impl/anime"; +import MangaProvider from "../../mappings/impl/manga"; +import InformationProvider from "../../mappings/impl/information"; +import MetaProvider from "../../mappings/impl/meta"; const toCheck: string[] = []; @@ -38,21 +43,13 @@ export async function checkCorsProxies(): Promise { }) .filter((obj) => obj.port != 8080); - const chunkSize = 25; - const perChunkCallback = (chunk: IP[]) => { - console.log(colors.gray(`Checking ${chunk.length} proxies...`)); - }; - - const perResultsCallback = (result: (string | undefined)[]) => { - const ips = result.filter(isString); - goodIps.push(...ips); - console.log(colors.green(`${ips.length} proxies are good!`)); + for (const ip of ips) { + const result = await makeRequest(ip); + if (result) goodIps.push(result) && console.log(colors.green(result + " passed!")); + else console.log(colors.red(`${ip.ip}:${ip.port} failed.`)); Bun.write("./goodProxies.json", JSON.stringify(goodIps, null, 4)); - }; - - const executor = new ChunkedExecutor(ips, chunkSize, makeRequest, perChunkCallback, perResultsCallback); - await executor.execute(); + } console.log(colors.gray("Finished checking proxies.")); @@ -63,113 +60,90 @@ export async function checkCorsProxies(): Promise { } async function makeRequest(ip: IP): Promise { - const controller = new AbortController(); - - console.log(colors.gray("Checking ") + `${ip.ip}:${ip.port}` + colors.gray(".") + colors.gray(" (Timeout: 5 seconds)")); - - setTimeout(() => { - controller.abort(); - }, 5000); - - try { - const response = await fetch(`http://${ip.ip}:${ip.port}/iscorsneeded`, { - signal: controller.signal, - }).catch( - (err) => - ({ - ok: false, - status: 500, - statusText: "Timeout", - json: () => Promise.resolve({ error: err }), - }) as Response, - ); - if (response.status === 200 && (await response.text()) === "no") { - let isOkay = true; - - console.log(colors.yellow("Testing ") + `${ip.ip}:${ip.port}` + colors.yellow(".")); - - for (const provider of BASE_PROVIDERS) { - console.log(colors.gray("Testing ") + provider.id + colors.gray(".")); - - provider.customProxy = `http://${ip.ip}:${ip.port}`; - - const providerResponse = await provider.search("Mushoku Tensei", provider.formats.includes(Format.TV) ? Type.ANIME : Type.MANGA, provider.formats, 0, 10).catch(() => { - return undefined; - }); - - provider.customProxy = undefined; - - if (!providerResponse) { - console.log(colors.red(`${provider.id} failed.`)); - isOkay = false; - break; + return new Promise(async (resolve, reject) => { + const controller = new AbortController(); + + console.log(colors.gray("Checking ") + `${ip.ip}:${ip.port}` + colors.gray(".") + colors.gray(" (Timeout: 5 seconds)")); + + setTimeout(() => { + controller.abort(); + }, 5000); + + controller.signal.addEventListener("abort", () => { + console.log(colors.red(`http://${ip.ip}:${ip.port} aborted.`)); + + return resolve(undefined); + }); + + try { + const response = await fetch(`http://${ip.ip}:${ip.port}/iscorsneeded`, { + signal: controller.signal, + }).catch( + (err) => + ({ + ok: false, + status: 500, + statusText: "Timeout", + json: () => Promise.resolve({ error: err }), + }) as Response, + ); + + if (response.status === 200 && (await response.text()) === "no") { + console.log(colors.green(`http://${ip.ip}:${ip.port} is a CORS proxy.`)); + + // Check all providers + let isOkay = true; + for (const provider of BASE_PROVIDERS) { + console.log(colors.gray("Testing ") + provider.id + colors.gray(".")); + + provider.customProxy = `http://${ip.ip}:${ip.port}`; + + const providerResponse = await provider.search("Mushoku Tensei", provider.formats.includes(Format.TV) ? Type.ANIME : Type.MANGA, provider.formats, 0, 10).catch(() => { + return undefined; + }); + + provider.customProxy = undefined; + + if (!providerResponse) { + console.log(colors.red(`${provider.id} failed.`)); + isOkay = false; + break; + } } - } - if (isOkay) { - console.log(colors.yellow("Base providers passed.")); - } else { - console.log(colors.red("Base providers failed.")); - return undefined; - } + if (isOkay) console.log(colors.yellow("Base providers passed.")); + else return undefined; - for (const provider of ANIME_PROVIDERS) { - console.log(colors.gray("Testing ") + provider.id + colors.gray(".")); + for (const provider of ANIME_PROVIDERS) { + console.log(colors.gray("Testing ") + provider.id + colors.gray(".")); - provider.customProxy = `http://${ip.ip}:${ip.port}`; + provider.customProxy = `http://${ip.ip}:${ip.port}`; - const providerResponse = await provider.search("Mushoku Tensei").catch(() => { - return undefined; - }); + const providerResponse = await provider.search("Mushoku Tensei").catch(() => { + return undefined; + }); - provider.customProxy = undefined; + provider.customProxy = undefined; - if (!providerResponse) { - console.log(colors.red(`${provider.id} failed.`)); - isOkay = false; - break; + if (!providerResponse) { + console.log(colors.red(`${provider.id} failed.`)); + isOkay = false; + break; + } } - } - if (isOkay) { - console.log(colors.yellow("Anime providers passed.")); - } else { - console.log(colors.red("Anime providers failed.")); - return undefined; - } + if (isOkay) console.log(colors.yellow("Anime providers passed.")); + else return undefined; - for (const provider of MANGA_PROVIDERS) { - console.log(colors.gray("Testing ") + provider.id + colors.gray(".")); - - provider.customProxy = `http://${ip.ip}:${ip.port}`; - - const providerResponse = await provider.search("Mushoku Tensei").catch(() => { - return undefined; - }); - - provider.customProxy = undefined; - - if (!providerResponse) { - console.log(colors.red(`${provider.id} failed.`)); - isOkay = false; - break; - } - } - - if (isOkay) { - console.log(colors.yellow("Manga providers passed.")); - return ip.ip + ":" + ip.port; + return resolve(ip.ip + ":" + ip.port); } else { - console.log(colors.red("Manga providers failed.")); - return undefined; + console.log(colors.red(`${ip.ip}:${ip.port} is not a CORS proxy.`)); + return resolve(undefined); } - } else { - console.log(colors.red(`${ip.ip}:${ip.port} is not a CORS proxy.`)); - return undefined; + } catch (error) { + return resolve(undefined); } - } catch (error) { - return undefined; - } + }); } interface IP { diff --git a/anify-backend/src/scripts/checkProxies.ts b/anify-backend/src/scripts/checkProxies.ts index d1cdd7e..85646c9 100644 --- a/anify-backend/src/scripts/checkProxies.ts +++ b/anify-backend/src/scripts/checkProxies.ts @@ -1,3 +1,9 @@ import { checkCorsProxies } from "../proxies/impl/checkProxies"; -checkCorsProxies().then(console.log); +checkCorsProxies().then((data) => { + // Hang infinitely + console.log(data); + console.log("Successfully checked CORS proxies!"); + + setInterval(() => {}, 1000); +});