Skip to content

Commit

Permalink
fix: Proxy checking + request.ts
Browse files Browse the repository at this point in the history
  • Loading branch information
@Eltik
Eltik committed Sep 27, 2023
1 parent dd3a58f commit 4e9b1f6
Show file tree
Hide file tree
Showing 3 changed files with 156 additions and 164 deletions.
124 changes: 68 additions & 56 deletions anify-backend/src/helper/request.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,69 +23,81 @@ export default class Http {
}

static async request(url: string, config: RequestInit = {}, proxyRequest = true, requests = 0, customProxy: string | undefined = undefined): Promise<Response> {
try {
if (proxyRequest) {
const proxyUrl = customProxy || this.getRandomUnbannedProxy();
if (!proxyUrl) {
return Promise.reject(new Error("No unbanned proxies available"));
}
return new Promise(async (resolve, reject) => {
try {
if (proxyRequest) {
const proxyUrl = customProxy || this.getRandomUnbannedProxy();
if (!proxyUrl) {
return Promise.reject(new Error("No unbanned proxies available"));
}

const modifyUrl = `${proxyUrl}/${url}`;

const modifyUrl = `${proxyUrl}/${url}`;

const controller = new AbortController();
const id = setTimeout(() => {
this.updateBannedProxies(proxyUrl);
controller.abort();
}, 5000);

try {
config = {
...config,
headers: {
...config.headers,
Origin: "https://anify.tv",
},
};
const response = await fetch(modifyUrl, { signal: controller.signal, ...config }).catch(
(err) =>
({
ok: false,
status: 500,
statusText: "Timeout",
json: () => Promise.resolve({ error: err }),
}) as Response,
);

if (!response.ok) {
const controller = new AbortController();
const id = setTimeout(() => {
this.updateBannedProxies(proxyUrl);
}
controller.abort();
}, 5000);

controller.signal.addEventListener("abort", () => {
console.log(colors.red(`http://${proxyUrl} aborted.`));

return resolve({
ok: false,
status: 500,
statusText: "Timeout",
} as Response);
});

if (response.statusText === "Timeout") {
if (requests >= 3) {
console.log(colors.red("Request timed out. Retried 3 times. Aborting..."));
return response;
try {
config = {
...config,
headers: {
...config.headers,
Origin: "https://anify.tv",
},
};
const response = await fetch(modifyUrl, { signal: controller.signal, ...config }).catch(
(err) =>
({
ok: false,
status: 500,
statusText: "Timeout",
json: () => Promise.resolve({ error: err }),
}) as Response,
);

if (!response.ok) {
this.updateBannedProxies(proxyUrl);
}

return this.request(url, config, proxyRequest, requests + 1);
}
if (response.statusText === "Timeout") {
if (requests >= 3) {
console.log(colors.red("Request timed out. Retried 3 times. Aborting..."));
return response;
}

return this.request(url, config, proxyRequest, requests + 1);
}

clearTimeout(id);
return response;
} catch (error) {
console.log(proxyUrl);
clearTimeout(id);
throw error;
clearTimeout(id);
return resolve(response);
} catch (error) {
console.log(proxyUrl);
clearTimeout(id);
throw error;
}
} else {
return fetch(url, config);
}
} else {
return fetch(url, config);
} catch (e) {
console.log(e);
return resolve({
ok: false,
status: 500,
statusText: "Internal Server Error",
} as Response);
}
} catch (e) {
console.log(e);
return {
ok: false,
status: 500,
statusText: "Internal Server Error",
} as Response;
}
});
}
}
188 changes: 81 additions & 107 deletions anify-backend/src/proxies/impl/checkProxies.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ import ChunkedExecutor from "../executor";
import { CORS_PROXIES } from "..";
import { ANIME_PROVIDERS, BASE_PROVIDERS, MANGA_PROVIDERS, META_PROVIDERS } from "../../mappings";
import { Format, Type } from "../../types/enums";
import BaseProvider from "../../mappings/impl/base";
import AnimeProvider from "../../mappings/impl/anime";
import MangaProvider from "../../mappings/impl/manga";
import InformationProvider from "../../mappings/impl/information";
import MetaProvider from "../../mappings/impl/meta";

const toCheck: string[] = [];

Expand Down Expand Up @@ -38,21 +43,13 @@ export async function checkCorsProxies(): Promise<string[]> {
})
.filter((obj) => obj.port != 8080);

const chunkSize = 25;
const perChunkCallback = (chunk: IP[]) => {
console.log(colors.gray(`Checking ${chunk.length} proxies...`));
};

const perResultsCallback = (result: (string | undefined)[]) => {
const ips = result.filter(isString);
goodIps.push(...ips);
console.log(colors.green(`${ips.length} proxies are good!`));
for (const ip of ips) {
const result = await makeRequest(ip);
if (result) goodIps.push(result) && console.log(colors.green(result + " passed!"));
else console.log(colors.red(`${ip.ip}:${ip.port} failed.`));

Bun.write("./goodProxies.json", JSON.stringify(goodIps, null, 4));
};

const executor = new ChunkedExecutor<IP, string | undefined>(ips, chunkSize, makeRequest, perChunkCallback, perResultsCallback);
await executor.execute();
}

console.log(colors.gray("Finished checking proxies."));

Expand All @@ -63,113 +60,90 @@ export async function checkCorsProxies(): Promise<string[]> {
}

async function makeRequest(ip: IP): Promise<string | undefined> {
const controller = new AbortController();

console.log(colors.gray("Checking ") + `${ip.ip}:${ip.port}` + colors.gray(".") + colors.gray(" (Timeout: 5 seconds)"));

setTimeout(() => {
controller.abort();
}, 5000);

try {
const response = await fetch(`http://${ip.ip}:${ip.port}/iscorsneeded`, {
signal: controller.signal,
}).catch(
(err) =>
({
ok: false,
status: 500,
statusText: "Timeout",
json: () => Promise.resolve({ error: err }),
}) as Response,
);
if (response.status === 200 && (await response.text()) === "no") {
let isOkay = true;

console.log(colors.yellow("Testing ") + `${ip.ip}:${ip.port}` + colors.yellow("."));

for (const provider of BASE_PROVIDERS) {
console.log(colors.gray("Testing ") + provider.id + colors.gray("."));

provider.customProxy = `http://${ip.ip}:${ip.port}`;

const providerResponse = await provider.search("Mushoku Tensei", provider.formats.includes(Format.TV) ? Type.ANIME : Type.MANGA, provider.formats, 0, 10).catch(() => {
return undefined;
});

provider.customProxy = undefined;

if (!providerResponse) {
console.log(colors.red(`${provider.id} failed.`));
isOkay = false;
break;
return new Promise(async (resolve, reject) => {
const controller = new AbortController();

console.log(colors.gray("Checking ") + `${ip.ip}:${ip.port}` + colors.gray(".") + colors.gray(" (Timeout: 5 seconds)"));

setTimeout(() => {
controller.abort();
}, 5000);

controller.signal.addEventListener("abort", () => {
console.log(colors.red(`http://${ip.ip}:${ip.port} aborted.`));

return resolve(undefined);
});

try {
const response = await fetch(`http://${ip.ip}:${ip.port}/iscorsneeded`, {
signal: controller.signal,
}).catch(
(err) =>
({
ok: false,
status: 500,
statusText: "Timeout",
json: () => Promise.resolve({ error: err }),
}) as Response,
);

if (response.status === 200 && (await response.text()) === "no") {
console.log(colors.green(`http://${ip.ip}:${ip.port} is a CORS proxy.`));

// Check all providers
let isOkay = true;
for (const provider of BASE_PROVIDERS) {
console.log(colors.gray("Testing ") + provider.id + colors.gray("."));

provider.customProxy = `http://${ip.ip}:${ip.port}`;

const providerResponse = await provider.search("Mushoku Tensei", provider.formats.includes(Format.TV) ? Type.ANIME : Type.MANGA, provider.formats, 0, 10).catch(() => {
return undefined;
});

provider.customProxy = undefined;

if (!providerResponse) {
console.log(colors.red(`${provider.id} failed.`));
isOkay = false;
break;
}
}
}

if (isOkay) {
console.log(colors.yellow("Base providers passed."));
} else {
console.log(colors.red("Base providers failed."));
return undefined;
}
if (isOkay) console.log(colors.yellow("Base providers passed."));
else return undefined;

for (const provider of ANIME_PROVIDERS) {
console.log(colors.gray("Testing ") + provider.id + colors.gray("."));
for (const provider of ANIME_PROVIDERS) {
console.log(colors.gray("Testing ") + provider.id + colors.gray("."));

provider.customProxy = `http://${ip.ip}:${ip.port}`;
provider.customProxy = `http://${ip.ip}:${ip.port}`;

const providerResponse = await provider.search("Mushoku Tensei").catch(() => {
return undefined;
});
const providerResponse = await provider.search("Mushoku Tensei").catch(() => {
return undefined;
});

provider.customProxy = undefined;
provider.customProxy = undefined;

if (!providerResponse) {
console.log(colors.red(`${provider.id} failed.`));
isOkay = false;
break;
if (!providerResponse) {
console.log(colors.red(`${provider.id} failed.`));
isOkay = false;
break;
}
}
}

if (isOkay) {
console.log(colors.yellow("Anime providers passed."));
} else {
console.log(colors.red("Anime providers failed."));
return undefined;
}
if (isOkay) console.log(colors.yellow("Anime providers passed."));
else return undefined;

for (const provider of MANGA_PROVIDERS) {
console.log(colors.gray("Testing ") + provider.id + colors.gray("."));

provider.customProxy = `http://${ip.ip}:${ip.port}`;

const providerResponse = await provider.search("Mushoku Tensei").catch(() => {
return undefined;
});

provider.customProxy = undefined;

if (!providerResponse) {
console.log(colors.red(`${provider.id} failed.`));
isOkay = false;
break;
}
}

if (isOkay) {
console.log(colors.yellow("Manga providers passed."));
return ip.ip + ":" + ip.port;
return resolve(ip.ip + ":" + ip.port);
} else {
console.log(colors.red("Manga providers failed."));
return undefined;
console.log(colors.red(`${ip.ip}:${ip.port} is not a CORS proxy.`));
return resolve(undefined);
}
} else {
console.log(colors.red(`${ip.ip}:${ip.port} is not a CORS proxy.`));
return undefined;
} catch (error) {
return resolve(undefined);
}
} catch (error) {
return undefined;
}
});
}

interface IP {
Expand Down
8 changes: 7 additions & 1 deletion anify-backend/src/scripts/checkProxies.ts
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
import { checkCorsProxies } from "../proxies/impl/checkProxies";

checkCorsProxies().then(console.log);
checkCorsProxies().then((data) => {
// Hang infinitely
console.log(data);
console.log("Successfully checked CORS proxies!");

setInterval(() => {}, 1000);
});

0 comments on commit 4e9b1f6

Please sign in to comment.