trying to fix nodegyp issues

.github/workflows/release-web.yml

@@ -39,8 +39,8 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
-            GITHUB_TOKEN_ARG=${{ secrets.GITHUB_TOKEN }}
-            VANILLA_DATA_ARG=${{ secrets.VANILLA_DATA }}
-            FORGE_DATA_ARG=${{ secrets.FORGE_DATA }}
-            CURSEFORGE_TOKEN_ARG=${{ secrets.CURSEFORGE_TOKEN }}
-            MODRINTH_TOKEN_ARG=${{ secrets.MODRINTH_TOKEN }}
+            GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
+            VANILLA_DATA=${{ secrets.VANILLA_DATA }}
+            FORGE_DATA=${{ secrets.FORGE_DATA }}
+            CURSEFORGE_TOKEN=${{ secrets.CURSEFORGE_DOWNLOAD_TOKEN }}
+            MODRINTH_TOKEN=${{ secrets.MODRINTH_TOKEN }}

web/apps/next/Dockerfile

@@ -1,12 +1,12 @@
-FROM node:20-alpine AS base
+FROM node:20 AS base
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 RUN corepack enable
 ENV CI=true
 
 FROM base AS builder
-RUN apk add --no-cache libc6-compat
-RUN apk update
+#RUN apk add --no-cache libc6-compat
+#RUN apk update
 # Set working directory
 WORKDIR /app
 RUN pnpm add -g turbo
@@ -16,14 +16,30 @@ RUN turbo prune --scope=next --docker
 # Add lockfile and package.json's of isolated subworkspace
 FROM base AS installer
 
-ENV GITHUB_TOKEN=$GITHUB_TOKEN_ARG
-ENV VANILLA_DATA=$VANILLA_DATA_ARG
-ENV FORGE_DATA=$FORGE_DATA_ARG
-ENV CURSEFORGE_TOKEN=$CURSEFORGE_TOKEN_ARG
-ENV MODRINTH_TOKEN=$MODRINTH_TOKEN_ARG
+ENV NODE_GYP_FORCE_PYTHON=/usr/local/bin/python3.9
 
-RUN apk add --no-cache libc6-compat
-RUN apk update
+# required dependencies for node-canvas
+RUN apt-get update
+RUN apt-get install -y software-properties-common build-essential libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev pkg-config python3-launchpadlib
+
+RUN add-apt-repository -y ppa:deadsnakes/ppa
+RUN apt-get update
+RUN apt-get install -y python3.9
+RUN ls /usr/local/bin
+RUN whereis python3.9
+RUN /usr/local/bin/python3.9 -v
+
+ARG GITHUB_TOKEN
+ARG VANILLA_DATA
+ARG FORGE_DATA
+ARG CURSEFORGE_TOKEN
+ARG MODRINTH_TOKEN
+
+ENV GITHUB_TOKEN=$GITHUB_TOKEN
+ENV VANILLA_DATA=$VANILLA_DATA
+ENV FORGE_DATA=$FORGE_DATA
+ENV CURSEFORGE_TOKEN=$CURSEFORGE_TOKEN
+ENV MODRINTH_TOKEN=$MODRINTH_TOKEN
 WORKDIR /app
 
 # First install the dependencies (as they change less often)
@@ -36,7 +52,7 @@ RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install -r --workspace-ro
 COPY --from=builder /app/out/full/ .
 RUN pnpm turbo run build --filter=next...
 
-FROM base AS runner
+FROM node:20-alpine AS runner
 WORKDIR /app
 
 # Don't run production as root

web/package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "scripts": {
-    "build": "turbo run build",
+    "build": "dotenv -- turbo run build",
     "dev": "dotenv -- turbo run dev",
     "lint": "turbo run lint",
     "test": "turbo run test",

web/packages/datagen/src/fetchers/modrinth.ts

@@ -24,7 +24,6 @@ export default class ModrinthFetcher extends UrlFetcher {
   }
 
   async getUrl(source: Source) {
-    if (!source.project) throw new Error("Project id missing");
     if (!source.file) throw new Error("File id missing");
 
     const version = await this.api.get<ModrinthVersion>(

web/packages/datagen/src/fetchers/url.ts

@@ -1,6 +1,17 @@
 import { UrlFetcher } from "./index.js";
 import { Source } from "../sources.js";
 import axios, { AxiosInstance } from "axios";
+import getEnv from "../env.js";
+
+function replaceEnv(string: string) {
+  const matches = string.match(/(\$\w+)($|\s)/g) ?? [];
+  return matches
+    .map((it) => it.slice(1))
+    .reduce((s, key) => {
+      const env = getEnv(key);
+      return s.replace(s, env);
+    }, string);
+}
 
 export default class AnyUrlFetcher extends UrlFetcher {
   private api = axios.create({ responseType: "stream" });
@@ -11,6 +22,6 @@ export default class AnyUrlFetcher extends UrlFetcher {
 
   async getUrl(source: Source) {
     if (!source.url) throw new Error("URL missing");
-    return source.url;
+    return replaceEnv(source.url);
   }
 }

web/packages/datagen/src/sourcesFetcher.ts

@@ -6,17 +6,6 @@ import CurseforgeFetcher from "./fetchers/curseforge.js";
 import FileFetcher from "./fetchers/file.js";
 import Fetcher from "./fetchers/index.js";
 import AnyUrlFetcher from "./fetchers/url.js";
-import getEnv from "./env.js";
-
-function replaceEnv(string: string) {
-  const matches = string.match(/(\$\w+)($|\s)/g) ?? [];
-  return matches
-    .map((it) => it.slice(1))
-    .reduce((s, key) => {
-      const env = getEnv(key);
-      return s.replace(s, env);
-    }, string);
-}
 
 function createFetchers() {
   const fetchers: Record<string, new () => Fetcher> = {
@@ -66,10 +55,9 @@ async function fetchSource(
     throw e;
   }
 
-  console.log(`Successfully fetched ${source.name}`);
-
   // wait because of a race condition
-  await new Promise((res) => setTimeout(res, 500));
+  await new Promise((res) => setTimeout(res, 3000));
+  console.log(`Successfully fetched ${source.name}`);
 }
 
 export default async function fetchSources() {