fix: being blocked by Cloudflare Turnstile (#150)

WFCD · Jan 23, 2025 · 1a3611d · 1a3611d
1 parent 424db4c
commit 1a3611d
Show file tree

Hide file tree

Showing 8 changed files with 8,901 additions and 1,071 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -1,7 +1,7 @@
 name: Build
 on:
   schedule:
-    - cron: "55 */4 * * *"
+    - cron: '55 */4 * * *'
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -11,20 +11,21 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-      with:
-        token: ${{ secrets.GH_TOKEN }}
-    - uses: actions/setup-node@v4
-      with:
-        node-version-file: '.nvmrc'
-    - run: npm ci
-    - run: npm run build
-    - run: npm test
-    - run: git checkout -- package-lock.json #prevent package-lock.json-only feat changes
-    - uses: stefanzweifel/git-auto-commit-action@v5
-      with:
-        skip_dirty_check: false
-        commit_message: 'feat(patchlogs): new patchlogs'
-        commit_user_name: wfcd-bot-boi
-        commit_user_email: botboi@warframestat.us
-        commit_author: wfcd-bot-boi <botboi@warframestat.us>
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GH_TOKEN }}
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+      - run: npm ci
+      - run: npm i
+      - run: npm run build
+      - run: npm test
+      - run: git checkout -- package-lock.json #prevent package-lock.json-only feat changes
+      - uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          skip_dirty_check: false
+          commit_message: 'feat(patchlogs): new patchlogs'
+          commit_user_name: wfcd-bot-boi
+          commit_user_email: botboi@warframestat.us
+          commit_author: wfcd-bot-boi <botboi@warframestat.us>
diff --git a/.github/workflows/static.yaml b/.github/workflows/static.yaml
@@ -22,48 +22,49 @@ jobs:
     needs: [install]
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-node@v4
-      with:
-        node-version-file: '.nvmrc'
-    - uses: actions/cache/restore@v4
-      with:
-       path: node_modules/
-       key: ${{ runner.os }}-${{ github.run_id }}${{ github.run_number }}
-    - run: npm run lint
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+      - uses: actions/cache/restore@v4
+        with:
+          path: node_modules/
+          key: ${{ runner.os }}-${{ github.run_id }}${{ github.run_number }}
+      - run: npm run lint
   build:
     name: Build
     needs: [install]
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-node@v4
-      with:
-        node-version-file: '.nvmrc'
-    - uses: actions/cache/restore@v4
-      with:
-        path: node_modules/
-        key: ${{ runner.os }}-${{ github.run_id }}${{ github.run_number }}
-    - run: npm run build
-    - uses: actions/cache/save@v4
-      with:
-        path: |
-          data/patchlogs.json
-          node_modules/
-        key: ${{ runner.os }}-${{ github.run_id }}${{ github.run_number }}
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+      - run: npm i
+      - uses: actions/cache/restore@v4
+        with:
+          path: node_modules/
+          key: ${{ runner.os }}-${{ github.run_id }}${{ github.run_number }}
+      - run: npm run build
+      - uses: actions/cache/save@v4
+        with:
+          path: |
+            data/patchlogs.json
+            node_modules/
+          key: ${{ runner.os }}-${{ github.run_id }}${{ github.run_number }}
   test:
     name: Test
     runs-on: ubuntu-latest
     needs: [build, lint]
     steps:
-    - uses: actions/checkout@v4
-    - uses: actions/cache/restore@v4
-      with:
-        path: |
-          data/patchlogs.json
-          node_modules/
-        key: ${{ runner.os }}-${{ github.run_id }}${{ github.run_number }}
-    - uses: actions/setup-node@v4
-      with:
-        node-version-file: '.nvmrc'
-    - run: npm test
+      - uses: actions/checkout@v4
+      - uses: actions/cache/restore@v4
+        with:
+          path: |
+            data/patchlogs.json
+            node_modules/
+          key: ${{ runner.os }}-${{ github.run_id }}${{ github.run_number }}
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+      - run: npm test
diff --git a/.nvmrc b/.nvmrc
@@ -1 +1 @@
-lts/iron
+lts/jod
diff --git a/build/scraper.js b/build/scraper.js
@@ -1,6 +1,8 @@
 import { load } from 'cheerio';
+import puppeteer from 'puppeteer-extra';
+import StealthPlugin from 'puppeteer-extra-plugin-stealth';
 
-import cache from '../data/patchlogs.json' assert { type: 'json' };
+import cache from '../data/patchlogs.json' with { type: 'json' };
 
 import ProgressBar from './progress.js';
 import sleep from './sleep.js';
@@ -39,7 +41,27 @@ class Scraper {
   }
 
   async #fetch(url = baseUrl) {
-    return (await fetch(url)).text();
+    let browser;
+
+    try {
+      browser = await puppeteer
+        .use(StealthPlugin())
+        .launch({ headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox'] });
+
+      const page = await browser.newPage();
+
+      await page.goto(url, {
+        waitUntil: ['networkidle0', 'domcontentloaded'],
+        timeout: 30000, // 30 second timeout
+      });
+
+      return await page.content();
+    } catch (err) {
+      console.error('Failed to fetch page:', err);
+      throw err;
+    } finally {
+      await browser.close();
+    }
   }
 
   /**