fix: 9anime/TMDB and crawling script

Eltik · Sep 28, 2023 · 8aa5962 · 8aa5962
1 parent 110fc03
commit 8aa5962
Show file tree

Hide file tree

Showing 4 changed files with 53 additions and 42 deletions.
diff --git a/anify-backend/src/mappings/impl/anime/nineanime.ts b/anify-backend/src/mappings/impl/anime/nineanime.ts
@@ -5,6 +5,9 @@ import { load } from "cheerio";
 import { env } from "../../../env";
 import { Format, Formats, StreamingServers, SubType } from "../../../types/enums";
 import Extractor from "../../../helper/extractor";
+import Http from "../../../helper/request";
+import { isString } from "../../../helper";
+import vm from "node:vm";
 
 export default class NineAnime extends AnimeProvider {
     override rateLimit = 250;
@@ -262,13 +265,12 @@ export default class NineAnime extends AnimeProvider {
         return await (await this.request(`${this.resolver}/decrypt?query=${encodeURIComponent(query)}&apikey=${this.resolverKey}`, {}, false)).json();
     }
 
-    /*
     // This bypass works. However because it sends requests very quickly in a short amount of time, it causes proxies to get banned very quickly.
     override async request(url: string, options: RequestInit = {}, proxyRequest = true): Promise<Response> {
         if (url.includes(this.resolver ?? "")) {
             return Http.request("ANIME", url, options, false);
         }
-        const proxy = proxyRequest ? ((this.customProxy?.length ?? 0) > 0 ? this.customProxy : Http.getRandomUnbannedProxy()) : undefined;
+        const proxy = proxyRequest ? ((this.customProxy?.length ?? 0) > 0 ? this.customProxy : Http.getRandomUnbannedProxy("ANIME")) : undefined;
 
         const headers = {
             "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Mobile/15E148 Safari/604.1",
@@ -319,7 +321,7 @@ export default class NineAnime extends AnimeProvider {
         const code = EvalDecode("${data2}");
         data = code;
         `,
-            context
+            context,
         );
 
         const kMatch = context.data.match(/var k='([^']+)'/);
@@ -347,13 +349,12 @@ export default class NineAnime extends AnimeProvider {
 
         const cookies = req3.headers.get("set-cookie");
 
-        console.log(await req3.text())
+        console.log(await req3.text());
 
         return Http.request("ANIME", url, { headers: { Cookie: cookies ?? "" }, ...options }, proxyRequest, 0, proxy);
 
         //return Http.request(url, { headers: { Cookie: cookies?.join("; ") ?? "" }, ...options }, proxyRequest, 0, proxy);
     }
-    */
 
     /*
     The waf page evals this:

diff --git a/anify-backend/src/mappings/impl/information/tmdb.ts b/anify-backend/src/mappings/impl/information/tmdb.ts
@@ -33,40 +33,46 @@ export default class TMDB extends InformationProvider<Anime | Manga, AnimeInfo |
         if (!data) return undefined;
 
         if (data.ok) {
-            const info = await data.json();
-
-            return {
-                id: tmdbId,
-                title: {
-                    english: info.name,
-                    romaji: null,
-                    native: info.original_name,
-                },
-                currentEpisode: info.last_episode_to_air?.episode_number,
-                trailer: null,
-                duration: info.episode_run_time[0] ?? null,
-                color: null,
-                bannerImage: info.backdrop_path ? `https://image.tmdb.org/t/p/w500${info.backdrop_path}` : null,
-                coverImage: info.poster_path ? `https://image.tmdb.org/t/p/w500${info.poster_path}` : null,
-                status: null,
-                format: Format.UNKNOWN,
-                season: Season.UNKNOWN,
-                synonyms: [],
-                description: info.overview,
-                year: info.first_air_date ? new Date(info.first_air_date).getFullYear() : 0,
-                totalEpisodes: info.number_of_episodes,
-                genres: info.genres?.map((genre: { id: number; name: string }) => genre.name),
-                rating: info.vote_average,
-                popularity: info.popularity,
-                countryOfOrigin: info.origin_country[0] ?? null,
-                tags: [],
-                relations: [],
-                artwork: [],
-                characters: [],
-                totalChapters: null,
-                totalVolumes: null,
-                type: media.type,
-            } as AnimeInfo;
+            try {
+                const info = await data.json();
+
+                if (!info) return undefined;
+
+                return {
+                    id: tmdbId,
+                    title: {
+                        english: info.name,
+                        romaji: null,
+                        native: info.original_name,
+                    },
+                    currentEpisode: info.last_episode_to_air?.episode_number,
+                    trailer: null,
+                    duration: info.episode_run_time[0] ?? null,
+                    color: null,
+                    bannerImage: info.backdrop_path ? `https://image.tmdb.org/t/p/w500${info.backdrop_path}` : null,
+                    coverImage: info.poster_path ? `https://image.tmdb.org/t/p/w500${info.poster_path}` : null,
+                    status: null,
+                    format: Format.UNKNOWN,
+                    season: Season.UNKNOWN,
+                    synonyms: [],
+                    description: info.overview,
+                    year: info.first_air_date ? new Date(info.first_air_date).getFullYear() : 0,
+                    totalEpisodes: info.number_of_episodes,
+                    genres: info.genres?.map((genre: { id: number; name: string }) => genre.name),
+                    rating: info.vote_average,
+                    popularity: info.popularity,
+                    countryOfOrigin: info.origin_country[0] ?? null,
+                    tags: [],
+                    relations: [],
+                    artwork: [],
+                    characters: [],
+                    totalChapters: null,
+                    totalVolumes: null,
+                    type: media.type,
+                } as AnimeInfo;
+            } catch (e) {
+                return undefined;
+            }
         }
 
         return undefined;

diff --git a/anify-backend/src/scripts/crawl.ts b/anify-backend/src/scripts/crawl.ts
@@ -90,4 +90,10 @@ async function before() {
 }
 
 const type = process.argv.slice(2)?.toString()?.toLowerCase() ?? "anime";
-crawl(type === "anime" ? Type.ANIME : type === "manga" ? Type.MANGA : Type.ANIME, type === "anime" ? [Format.TV, Format.TV_SHORT, Format.MOVIE, Format.OVA, Format.ONA, Format.SPECIAL, Format.MUSIC] : type === "manga" ? [Format.MANGA, Format.ONE_SHOT] : type === "novel" ? [Format.NOVEL] : [Format.TV, Format.TV_SHORT, Format.MOVIE, Format.OVA, Format.ONA, Format.SPECIAL, Format.MUSIC]);
+crawl(type === "anime" ? Type.ANIME : type === "manga" ? Type.MANGA : Type.ANIME, type === "anime" ? [Format.TV, Format.TV_SHORT, Format.MOVIE, Format.OVA, Format.ONA, Format.SPECIAL, Format.MUSIC] : type === "manga" ? [Format.MANGA, Format.ONE_SHOT] : type === "novel" ? [Format.NOVEL] : [Format.TV, Format.TV_SHORT, Format.MOVIE, Format.OVA, Format.ONA, Format.SPECIAL, Format.MUSIC]).then((data) => {
+    // Hang infinitely
+    console.log(data);
+    console.log("Successfully crawled!");
+
+    setInterval(() => {}, 1000);
+});
diff --git a/anify-backend/src/server/lib/keys.ts b/anify-backend/src/server/lib/keys.ts
@@ -8,7 +8,5 @@ export const apiKeyMiddleware = async (req: Request): Promise<boolean> => {
         return false;
     }
 
-    await redis.incr(`apikeys:${key}:requests`);
-
     return true;
 };