Large scale refactor of monorepo.

- Refactor the non-packages portion of the monorepo into ESM - Move internal packages to self-contained @internals/* packages, reference via pnpm monorepo support - Refactor package bundling to leverage verdaccio for a saner approach to package managemenet during tests - Expose local servers via tunnelmole to avoid issues with browserstack-local - Support index.min.js UMD exports for all models - Expose Upscaler CLI for managing monorepo scripts: - build:upscaler - build:models - build:core - scaffold:upscaler - scaffold:models - 5 test commands
thekevinscott · Sep 14, 2023 · 1b9d632 · 1b9d632
1 parent 2b5d019
commit 1b9d632
Show file tree

Hide file tree

Showing 439 changed files with 647,738 additions and 20,371 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -45,7 +45,7 @@ jobs:
         run: 'pnpm --filter="upscaler" build:only'
 
       - name: 'Validate'
-        run: 'pnpm --filter="upscaler" validate:build'
+        run: 'pnpm --filter="upscaler" validate'
 
   unit-browser-jest:
     name: 'Unit / Browser / Jest'
@@ -160,59 +160,21 @@ jobs:
         with:
           verbose: true
 
-  integration-browser-browserstack:
-    name: 'Integration / Browser / Browserstack'
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [16]
-    steps:
-      - name: 'BrowserStack Env Setup'  # Invokes the setup-env action
-        uses: browserstack/github-actions/setup-env@master
-        with:
-          username: ${{ secrets.BROWSERSTACK_USERNAME }}
-          access-key: ${{ secrets.BROWSERSTACK_ACCESS_KEY }}
-
-      - name: 'Checkout repository'
-        uses: actions/checkout@v2
-
-      - name: 'Setup PNPM with Node ${{ matrix.node-version }}'
-        uses: ./.github/actions/setup-pnpm
-        with:
-          node-version: ${{ matrix.node-version }}
 
-      - name: 'Setup DVC & Pull Models'
-        uses: ./.github/actions/setup-dvc
-        with:
-          gdrive_credentials_data: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}          
-
-      - name: 'Build Upscaler'
-        run: pnpm test:integration:browserstack --ci --verbose --skipTest --skipModelBuild --skipBundle
-
-      - name: 'Build Models'
-        run: pnpm test:integration:browserstack --ci --verbose --skipTest --skipUpscalerBuild --skipBundle
-
-      - name: 'Bundle'
-        run: pnpm test:integration:browserstack --ci --verbose --skipTest --skipUpscalerBuild --skipModelBuild
-
-      - name: 'Integration Tests'
-        run: pnpm test:integration:browserstack --ci --verbose --skipUpscalerBuild --skipModelBuild --skipBundle
-        env:
-          NODE_OPTIONS: --max_old_space_size=8192
-
-  integration-browser-local:
-    name: 'Integration / Browser / Local'
+  integration:
+    name: 'Integration'
     runs-on: ubuntu-latest
     strategy:
       matrix:
         node-version: [16]
+        test: [clientside, serverside, memory-leaks, models, browserstack]
     steps:
       - name: 'BrowserStack Env Setup'  # Invokes the setup-env action
         uses: browserstack/github-actions/setup-env@master
         with:
           username: ${{ secrets.BROWSERSTACK_USERNAME }}
           access-key: ${{ secrets.BROWSERSTACK_ACCESS_KEY }}
-
+      
       - name: 'Checkout repository'
         uses: actions/checkout@v2
 
@@ -226,153 +188,20 @@ jobs:
         with:
           gdrive_credentials_data: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}          
 
-      # - name: Setup tmate session
-      #   uses: mxschmitt/action-tmate@v3
-      #   with:
-      #     detached: true
-
       - name: 'Build Upscaler'
-        run: pnpm test:integration:browser --ci --verbose --skipTest --skipModelBuild --skipBundle
+        run: pnpm cli test integration ${{ matrix.test }} --ci -l verbose --skip-model-build --skip-bundle --skip-test
 
       - name: 'Build Models'
-        run: pnpm test:integration:browser --ci --verbose --skipTest --skipUpscalerBuild --skipBundle
+        run: pnpm cli test integration ${{ matrix.test }} --ci -l verbose --skip-upscaler --skip-bundle --skip-test
 
       - name: 'Bundle'
-        run: pnpm test:integration:browser --ci --verbose --skipTest --skipUpscalerBuild --skipModelBuild
+        run: pnpm cli test integration ${{ matrix.test }} --ci -l verbose --skip-upscaler --skip-model-build --skip-test
 
       - name: 'Integration Tests'
-        run: pnpm test:integration:browser --ci --verbose --skipUpscalerBuild --skipModelBuild --skipBundle
+        run: pnpm cli test integration ${{ matrix.test }} --ci -l verbose --skip-upscaler --skip-model-build --skip-bundle
         env:
           NODE_OPTIONS: --max_old_space_size=8192
 
-  integration-node-local-16:
-    name: 'Integration / Node / Local / v16'
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [16]
-    steps:
-      - name: 'Checkout repository'
-        uses: actions/checkout@v2
-
-      - name: 'Setup PNPM with Node ${{ matrix.node-version }}'
-        uses: ./.github/actions/setup-pnpm
-        with:
-          node-version: ${{ matrix.node-version }}
-
-      - name: 'Setup DVC & Pull Models'
-        uses: ./.github/actions/setup-dvc
-        with:
-          gdrive_credentials_data: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}          
-
-      - name: 'Build Upscaler'
-        run: pnpm test:integration:node --ci --verbose --skipTest --skipModelBuild --skipBundle
-
-      - name: 'Build Models'
-        run: pnpm test:integration:node --ci --verbose --skipTest --skipUpscalerBuild --skipBundle
-
-      - name: 'Bundle'
-        run: pnpm test:integration:node --ci --verbose --skipTest --skipUpscalerBuild --skipModelBuild
-
-      - name: 'Integration Tests'
-        run: pnpm test:integration:node --ci --verbose --skipUpscalerBuild --skipModelBuild --skipBundle
-
-  models-browser:
-    name: 'Models / Browser'
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [16]
-    steps:
-      - name: 'Checkout repository'
-        uses: actions/checkout@v2
-
-      - name: 'Setup PNPM with Node ${{ matrix.node-version }}'
-        uses: ./.github/actions/setup-pnpm
-        with:
-          node-version: ${{ matrix.node-version }}
-
-      - name: 'Setup DVC & Pull Models'
-        uses: ./.github/actions/setup-dvc
-        with:
-          gdrive_credentials_data: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}          
-
-      - name: 'Build Upscaler'
-        run: pnpm test:model --ci --verbose --skipTest --skipModelBuild --skipBundle
-
-      - name: 'Build Models'
-        run: pnpm test:model --ci --verbose --skipTest --skipUpscalerBuild --skipBundle
-
-      - name: 'Bundle'
-        run: pnpm test:model --ci --verbose --skipTest --skipUpscalerBuild --skipModelBuild
-
-      - name: 'Integration Tests'
-        run: pnpm test:model --platform browser --ci --verbose --skipUpscalerBuild --skipModelBuild --skipBundle
-
-  models-node:
-    name: 'Models / Node'
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [16]
-    steps:
-      - name: 'Checkout repository'
-        uses: actions/checkout@v2
-
-      - name: 'Setup PNPM with Node ${{ matrix.node-version }}'
-        uses: ./.github/actions/setup-pnpm
-        with:
-          node-version: ${{ matrix.node-version }}
-
-      - name: 'Setup DVC & Pull Models'
-        uses: ./.github/actions/setup-dvc
-        with:
-          gdrive_credentials_data: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}          
-
-      - name: 'Build Upscaler'
-        run: pnpm test:model --ci --verbose --skipTest --skipModelBuild --skipBundle
-
-      - name: 'Build Models'
-        run: pnpm test:model --ci --verbose --skipTest --skipUpscalerBuild --skipBundle
-
-      - name: 'Bundle'
-        run: pnpm test:model --ci --verbose --skipTest --skipUpscalerBuild --skipModelBuild
-
-      - name: 'Integration Tests'
-        run: pnpm test:model --platform node --ci --verbose --skipUpscalerBuild --skipModelBuild --skipBundle
-
-  memory-leaks:
-    name: 'Memory Leaks'
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [16]
-    steps:
-      - name: 'Checkout repository'
-        uses: actions/checkout@v2
-
-      - name: 'Setup PNPM with Node ${{ matrix.node-version }}'
-        uses: ./.github/actions/setup-pnpm
-        with:
-          node-version: ${{ matrix.node-version }}
-
-      - name: 'Setup DVC & Pull Models'
-        uses: ./.github/actions/setup-dvc
-        with:
-          gdrive_credentials_data: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}          
-
-      - name: 'Build Upscaler'
-        run: pnpm test:memory-leaks --ci --verbose --skipTest --skipModelBuild --skipBundle
-
-      - name: 'Build Models'
-        run: pnpm test:memory-leaks --ci --verbose --skipTest --skipUpscalerBuild --skipBundle
-
-      - name: 'Bundle'
-        run: pnpm test:memory-leaks --ci --verbose --skipTest --skipUpscalerBuild --skipModelBuild
-
-      - name: 'Memory Leak Tests'
-        run: pnpm test:memory-leaks --ci --verbose --skipUpscalerBuild --skipModelBuild --skipBundle
-
   build-docs:
     name: 'Build Documentation'
     runs-on: ubuntu-latest

diff --git a/.gitignore b/.gitignore
@@ -21,6 +21,7 @@ yarn-error.log
 .yarnrc
 .parcel-cache
 *.tsbuildinfo
+.npmrc
 .yarn
 .pnpm-debug.log
 umd-tmp

diff --git a/.npmrc b/.npmrc
diff --git a/dev/browser/basic/esm/index.html b/dev/browser/basic/esm/index.html
@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <title>UpscalerJS Development: Basic ESM</title>
+  <link rel="stylesheet" href="../../global.css" />
+</head>
+<body>
+  <script type="module" src="./index.ts"></script>
+</body>
+</html>
+
diff --git a/dev/browser/basic/esm/index.ts b/dev/browser/basic/esm/index.ts
@@ -0,0 +1,2 @@
+import Upscaler from 'upscaler';
+const upscaler = new Upscaler();
diff --git a/dev/browser/basic/index.html b/dev/browser/basic/index.html
@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <link rel="stylesheet" href="../global.css" />
+</head>
+<body>
+  <ul>
+    <li><a href="esm/">ESM</a></li>
+    <li><a href="umd/">UMD</a></li>
+  </ul>
+</body>
+</html>
diff --git a/dev/browser/basic/umd/index.html b/dev/browser/basic/umd/index.html
@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <title>UpscalerJS Development: Basic UMD</title>
+  <link rel="stylesheet" href="../../global.css" />
+</head>
+<body>
+  <script type="module" src="./index.ts"></script>
+  <script src="/node_modules/upscaler/node_modules/@upscalerjs/default-model/dist/umd/index.min.js"></script>
+  <script src="/node_modules/upscaler/dist/browser/umd/upscaler.min.js"></script>
+</body>
+</html>
+
diff --git a/dev/browser/basic/umd/index.ts b/dev/browser/basic/umd/index.ts
@@ -0,0 +1,3 @@
+const upscaler = new window['Upscaler']({
+  model: window['DefaultUpscalerJSModel'],
+});
diff --git a/dev/browser/index.html b/dev/browser/index.html
@@ -5,6 +5,7 @@
 </head>
 <body>
   <ul>
+    <li><a href="/basic/">Basic</a></li>
     <li><a href="/speed-tests/">Speed Tests</a></li>
     <li><a href="/specific-model/">Specific Model</a></li>
     <li><a href="/generate-homepage-gif/">Generate Homepage GIF</a></li>

diff --git a/dev/node/src/utils/upscaler.ts b/dev/node/src/utils/upscaler.ts
@@ -1,7 +1,6 @@
 import * as fs from 'fs-extra';
 import * as path from 'path';
 import { MODELS_DIR, UPSCALERJS_DIR } from './constants';
-import { ModelDefinitionFn } from "../../../../packages/core/src";
 
 export const isValidEnv = (env: string): env is ValidEnv => ['node', 'node-gpu'].includes(env);
 export type ValidEnv = 'node' | 'node-gpu';

diff --git a/docs/package.json b/docs/package.json
@@ -4,18 +4,18 @@
   "private": true,
   "scripts": {
     "build:dependencies": "pnpm --filter @upscalerjs/scripts model:build -a -o cjs -o esm -o umd && pnpm --filter upscaler build",
-    "build-api": "pnpm --filter @upscalerjs/scripts docs:build-api",
-    "build-api:watch": "nodemon -e ts --ignore '../packages/upscalerjs/**/*.generated.ts' --watch '../packages/core/**/*' --watch '../packages/upscalerjs/**/*' --watch '../scripts/package-scripts/docs/build-api.ts' -x \"pnpm run build-api\"",
-    "build-guides": "pnpm --filter @upscalerjs/scripts docs:build-guides",
-    "build-guides:watch": "nodemon -e md --watch '../examples/**/*' --watch '../scripts/package-scripts/docs/build-guides.ts' -x \"pnpm run build-guides\"",
-    "link-model-readmes": "pnpm --filter @upscalerjs/scripts docs:link-model-readmes",
-    "link-model-readmes:watch": "nodemon -e md -e mdx --watch '../models/**/*' --watch '../scripts/package-scripts/docs/link-model-readmes.ts' -x \"pnpm run link-model-readmes\"",
+    "write-api": "pnpm cli write docs api",
+    "write-api:watch": "nodemon -e ts --ignore '../packages/upscalerjs/**/*.generated.ts' --watch '../packages/core/**/*' --watch '../packages/upscalerjs/**/*' -x \"pnpm run write-api\"",
+    "write-guides": "pnpm cli write docs guides",
+    "write-guides:watch": "nodemon -e md --watch '../examples/**/*' -x \"pnpm run write-guides\"",
+    "write-model-readmes": "pnpm cli write docs models",
+    "write-model-readmes:watch": "nodemon -e md -e mdx --watch '../models/**/*' -x \"pnpm run write-model-readmes\"",
     "docusaurus": "docusaurus",
     "start": "pnpm run start:all",
-    "start:all": "concurrently \"pnpm run docusaurus:start\" \"pnpm run link-model-readmes:watch\" \"pnpm run build-guides:watch\" \"pnpm run build-api:watch\"",
+    "start:all": "concurrently \"pnpm run docusaurus:start\" \"pnpm run write-model-readmes:watch\" \"pnpm run write-guides:watch\" \"pnpm run write-api:watch\"",
     "docusaurus:start": "docusaurus start --host 0.0.0.0",
     "build:only": "docusaurus build",
-    "build": "concurrently \"pnpm build-guides --shouldClearMarkdown\" \"pnpm build-api --shouldClearMarkdown\" \"pnpm link-model-readmes --shouldClearMarkdown\" && pnpm tense-checks && pnpm build:dependencies && pnpm build:only",
+    "build": "concurrently \"pnpm write-guides --should-clear-markdown\" \"pnpm write-api --should-clear-markdown\" \"pnpm write-model-readmes --should-clear-markdown\" && pnpm tense-checks && pnpm build:dependencies && pnpm build:only",
     "swizzle": "docusaurus swizzle",
     "deploy": "docusaurus deploy",
     "clear": "docusaurus clear",

diff --git a/internals/bundlers/.gitignore b/internals/bundlers/.gitignore
@@ -0,0 +1 @@
+.storage
diff --git a/internals/bundlers/package.json b/internals/bundlers/package.json
@@ -0,0 +1,33 @@
+{
+  "name": "@internals/bundlers",
+  "private": true,
+  "type": "module",
+  "version": "0.1.0",
+  "description": "Test servers for testing UpscalerJS against",
+  "exports": {
+    ".": "./src/index.ts",
+    "./webpack": "./src/bundlers/webpack/src/index.ts",
+    "./esbuild": "./src/bundlers/esbuild/src/index.ts",
+    "./node": "./src/bundlers/node/src/index.ts",
+    "./umd": "./src/bundlers/umd/src/index.ts"
+  },
+  "author": "Kevin Scott",
+  "license": "MIT",
+  "dependencies": {
+    "fs-extra": "latest",
+    "@internals/common": "workspace:*",
+    "@internals/http-server": "workspace:*",
+    "@internals/registry": "workspace:*"
+  },
+  "scripts": {
+    "test": "vitest"
+  },
+  "engines": {
+    "node": ">=16.0.0"
+  },
+  "devDependencies": {
+    "vitest": "^0.34.2",
+    "@types/ejs": "^3.1.2",
+    "@types/js-yaml": "^4.0.5"
+  }
+}
diff --git a/test/lib/node/.gitignore → .../bundlers/src/bundlers/esbuild/.gitignore b/test/lib/node/.gitignore → .../bundlers/src/bundlers/esbuild/.gitignore
diff --git a/test/lib/esm-esbuild/src/index.html → ...undlers/esbuild/_templates/index.html.ejs b/test/lib/esm-esbuild/src/index.html → ...undlers/esbuild/_templates/index.html.ejs
@@ -1,8 +1,8 @@
 <html>
   <head>
     <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon"> 
+    <link rel="icon" href="data:;base64,iVBORw0KGgo=">
     <title>UpscalerJS Integration Test: ESM via ESBuild</title>
-    <link rel="icon" href="data:;base64,=">
     <style>
       h1 { font-size: 100px; }
     </style>

diff --git a/internals/bundlers/src/bundlers/esbuild/_templates/index.js.ejs b/internals/bundlers/src/bundlers/esbuild/_templates/index.js.ejs
@@ -0,0 +1,13 @@
+import * as tf from '@tensorflow/tfjs';
+import Upscaler from 'upscaler';
+
+/*** Auto-generated dependencies ***/
+<% dependencies.forEach(function(dependency){ %>
+import <%- dependency.hashedName %> from '<%- dependency.name %>';
+window['<%- dependency.name %>'] = <%- dependency.hashedName %>;
+<%  }); %>
+
+window.tf = tf;
+window.Upscaler = Upscaler;
+document.title = [document.title, 'Loaded'].join(' | ');
+document.body.querySelector('#output').innerHTML = document.title;
-Original file line number
+Diff line change
@@ Expand Up / @@ -21,6 +21,7 @@ yarn-error.log @@
     .yarnrc
     .parcel-cache
     *.tsbuildinfo
+    .npmrc
     .yarn
     .pnpm-debug.log
     umd-tmp
@@ Expand Down @@
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		import Upscaler from 'upscaler';
		const upscaler = new Upscaler();