enhance: reduce memory consumption (#3705)

* enhance: reduce memory consumption * Disable Yoga's caching * fix - align graphql-tools versions * .. * Fix website build * Invalidate cache * Work pls
ardatan · Mar 16, 2022 · 01bac6b · 01bac6b
1 parent ed38e0d
commit 01bac6b
Show file tree

Hide file tree

Showing 58 changed files with 393 additions and 276 deletions.
diff --git a/.changeset/heavy-horses-watch.md b/.changeset/heavy-horses-watch.md
@@ -0,0 +1,40 @@
+---
+'@graphql-mesh/cache-inmemory-lru': patch
+'@graphql-mesh/cli': patch
+'@graphql-mesh/config': patch
+'@graphql-mesh/graphql': patch
+'@graphql-mesh/json-schema': patch
+'@graphql-mesh/mongoose': patch
+'@graphql-mesh/mysql': patch
+'@graphql-mesh/neo4j': patch
+'@graphql-mesh/new-openapi': patch
+'@graphql-mesh/odata': patch
+'@graphql-mesh/openapi': patch
+'@graphql-mesh/raml': patch
+'@graphql-mesh/soap': patch
+'@graphql-mesh/thrift': patch
+'@omnigraph/json-schema': patch
+'@omnigraph/raml': patch
+'@graphql-mesh/merger-bare': patch
+'@graphql-mesh/merger-federation': patch
+'@graphql-mesh/merger-stitching': patch
+'@graphql-mesh/runtime': patch
+'@graphql-mesh/store': patch
+'@graphql-mesh/transform-cache': patch
+'@graphql-mesh/transform-encapsulate': patch
+'@graphql-mesh/transform-extend': patch
+'@graphql-mesh/transform-federation': patch
+'@graphql-mesh/transform-filter-schema': patch
+'@graphql-mesh/transform-mock': patch
+'@graphql-mesh/transform-naming-convention': patch
+'@graphql-mesh/transform-prefix': patch
+'@graphql-mesh/transform-prune': patch
+'@graphql-mesh/transform-rename': patch
+'@graphql-mesh/transform-replace-field': patch
+'@graphql-mesh/transform-snapshot': patch
+'@graphql-mesh/transform-type-merging': patch
+'@graphql-mesh/types': patch
+'@graphql-mesh/utils': patch
+---
+
+fix - align graphql-tools versions
diff --git a/.changeset/red-hairs-know.md b/.changeset/red-hairs-know.md
@@ -0,0 +1,14 @@
+---
+'@graphql-mesh/cache-inmemory-lru': minor
+'@graphql-mesh/cli': minor
+'@graphql-mesh/graphql': minor
+'@graphql-mesh/json-schema': minor
+'@graphql-mesh/new-openapi': minor
+'@graphql-mesh/raml': minor
+'@graphql-mesh/runtime': minor
+'@graphql-mesh/store': minor
+'@graphql-mesh/types': minor
+'@graphql-mesh/utils': minor
+---
+
+enhance: reduce memory consumption
diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml
@@ -44,9 +44,9 @@ jobs:
         uses: actions/cache@v2
         with:
           path: '**/node_modules'
-          key: ${{ runner.os }}-17-yarn-${{ hashFiles('yarn.lock') }}
+          key: ${{ runner.os }}-17-node-modules-${{ hashFiles('yarn.lock') }}
           restore-keys: |
-            ${{ runner.os }}-17-yarn-
+            ${{ runner.os }}-17-node-modules-
 
       - name: Install Dependencies using Yarn
         run: yarn install --ignore-engines && git checkout yarn.lock

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -35,12 +35,10 @@ jobs:
         uses: actions/cache@v2
         with:
           path: '**/node_modules'
-          key: ${{ runner.os }}-17-yarn-${{ hashFiles('yarn.lock') }}
+          key: ${{ runner.os }}-17-node-modules-${{ hashFiles('yarn.lock') }}
           restore-keys: |
-            ${{ runner.os }}-17-yarn-
+            ${{ runner.os }}-17-node-modules-
 
-      - name: Install Dependencies using Yarn
-        run: yarn install --ignore-engines && git checkout yarn.lock
       - name: Create Release Pull Request or Publish to npm
         id: changesets
         uses: changesets/action@v1

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -22,9 +22,9 @@ jobs:
         uses: actions/cache@v2
         with:
           path: '**/node_modules'
-          key: ${{ runner.os }}-17-yarn-${{ hashFiles('yarn.lock') }}
+          key: ${{ runner.os }}-17-node-modules-${{ hashFiles('yarn.lock') }}
           restore-keys: |
-            ${{ runner.os }}-17-yarn-
+            ${{ runner.os }}-17-node-modules-
 
       - name: Install Dependencies using Yarn
         run: yarn install --ignore-engines && git checkout yarn.lock
@@ -46,9 +46,9 @@ jobs:
         uses: actions/cache@v2
         with:
           path: '**/node_modules'
-          key: ${{ runner.os }}-17-yarn-${{ hashFiles('yarn.lock') }}
+          key: ${{ runner.os }}-17-node-modules-${{ hashFiles('yarn.lock') }}
           restore-keys: |
-            ${{ runner.os }}-17-yarn-
+            ${{ runner.os }}-17-node-modules-
 
       - name: Install Dependencies using Yarn
         run: yarn install --ignore-engines && git checkout yarn.lock
@@ -101,9 +101,9 @@ jobs:
         uses: actions/cache@v2
         with:
           path: '**/node_modules'
-          key: ${{ runner.os }}-${{matrix.node-version}}-yarn-${{ hashFiles('yarn.lock') }}
+          key: ${{ runner.os }}-${{matrix.node-version}}-node-modules-${{ hashFiles('yarn.lock') }}
           restore-keys: |
-            ${{ runner.os }}-${{matrix.node-version}}-yarn-
+            ${{ runner.os }}-${{matrix.node-version}}-node-modules-
       - name: Install Dependencies using Yarn
         run: yarn install --ignore-engines && git checkout yarn.lock
       - name: Cache Jest

diff --git a/examples/graphql-file-upload-example/package.json b/examples/graphql-file-upload-example/package.json
@@ -11,7 +11,7 @@
     "@graphql-mesh/cli": "0.61.0",
     "@graphql-mesh/graphql": "0.21.6",
     "concurrently": "5.3.0",
-    "@graphql-yoga/node": "0.1.0-beta.4",
+    "@graphql-yoga/node": "0.1.1-canary-cabb320.0",
     "sharp": "0.26.3",
     "graphql": "16.0.1"
   }

diff --git a/package.json b/package.json
@@ -20,7 +20,7 @@
     "prettier:check": "prettier --ignore-path .gitignore --check \"**/*.{ts,tsx,graphql,yml}\"",
     "test": "jest --forceExit --no-watchman",
     "test-and-build": "yarn build && yarn test",
-    "prerelease": "yarn build",
+    "prerelease": "yarn install && yarn build",
     "release": "changeset publish",
     "release:canary": "(node scripts/canary-release.js && yarn build && yarn changeset publish --tag $NPM_TAG) || echo Skipping Canary..."
   },

diff --git a/packages/cache/inmemory-lru/src/index.ts b/packages/cache/inmemory-lru/src/index.ts
@@ -1,10 +1,15 @@
-import { globalLruCache } from '@graphql-mesh/utils';
-import { KeyValueCache, KeyValueCacheSetOptions } from '@graphql-mesh/types';
+import { createLruCache, LRUCache } from '@graphql-mesh/utils';
+import { KeyValueCache, KeyValueCacheSetOptions, YamlConfig } from '@graphql-mesh/types';
 
 type CacheEntry<V> = { expiresAt: number; value: V };
 
 export default class InMemoryLRUCache<V = any> implements KeyValueCache<V> {
   private cacheIdentifier = Date.now();
+  private lruCache: LRUCache;
+
+  constructor(options?: YamlConfig.InMemoryLRUConfig) {
+    this.lruCache = createLruCache(options?.max, options?.ttl);
+  }
 
   private nextTick() {
     // Make sure this is scheduled for next tick because LRU Cache is synchronous
@@ -14,24 +19,24 @@ export default class InMemoryLRUCache<V = any> implements KeyValueCache<V> {
 
   async get(key: string) {
     await this.nextTick();
-    const entry: CacheEntry<V> = globalLruCache.get(`${this.cacheIdentifier}-${key}`);
+    const entry: CacheEntry<V> = this.lruCache.get(`${this.cacheIdentifier}-${key}`);
     if (entry?.expiresAt && Date.now() > entry.expiresAt) {
-      globalLruCache.delete(key);
+      this.lruCache.delete(key);
       return undefined;
     }
     return entry?.value;
   }
 
   async set(key: string, value: V, options?: KeyValueCacheSetOptions) {
     await this.nextTick();
-    globalLruCache.set(`${this.cacheIdentifier}-${key}`, {
+    this.lruCache.set(`${this.cacheIdentifier}-${key}`, {
       expiresAt: options?.ttl ? Date.now() + options.ttl * 1000 : Infinity,
       value,
     });
   }
 
   async delete(key: string) {
     await this.nextTick();
-    globalLruCache.delete(`${this.cacheIdentifier}-${key}`);
+    this.lruCache.delete(`${this.cacheIdentifier}-${key}`);
   }
 }
diff --git a/packages/cache/inmemory-lru/yaml-config.graphql b/packages/cache/inmemory-lru/yaml-config.graphql
@@ -4,4 +4,5 @@ type Cache {
 
 type InMemoryLRUConfig @md {
   max: Int
+  ttl: Int
 }
diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -48,13 +48,13 @@
     "@graphql-mesh/utils": "0.29.0",
     "@graphql-mesh/store": "0.3.29",
     "@graphql-mesh/types": "0.66.6",
-    "@graphql-tools/code-file-loader": "7.2.4",
-    "@graphql-tools/utils": "8.6.2",
-    "@graphql-tools/load": "7.5.2",
-    "@graphql-tools/graphql-file-loader": "7.3.4",
-    "@graphql-tools/wrap": "8.4.4",
+    "@graphql-tools/code-file-loader": "7.2.5",
+    "@graphql-tools/utils": "8.6.3",
+    "@graphql-tools/load": "7.5.3",
+    "@graphql-tools/graphql-file-loader": "7.3.5",
+    "@graphql-tools/wrap": "8.4.6",
     "@graphql-yoga/render-graphiql": "0.1.0-beta.1",
-    "@graphql-yoga/node": "0.1.0-beta.4",
+    "@graphql-yoga/node": "0.1.1-canary-cabb320.0",
     "body-parser": "1.19.2",
     "cookie-parser": "1.4.6",
     "cors": "2.8.5",

diff --git a/packages/cli/src/commands/serve/graphql-handler.ts b/packages/cli/src/commands/serve/graphql-handler.ts
@@ -10,6 +10,8 @@ function shouldRenderGraphiQL(req: IncomingMessage) {
 export const graphqlHandler = (mesh$: Promise<MeshInstance>): RequestHandler => {
   const yoga$ = mesh$.then(mesh =>
     createServer({
+      parserCache: false,
+      validationCache: false,
       plugins: [
         ...mesh.plugins,
         useExtendContext(({ req, res }) => ({

diff --git a/packages/cli/src/commands/ts-artifacts.ts b/packages/cli/src/commands/ts-artifacts.ts
@@ -293,9 +293,14 @@ export async function ${cliParams.builtMeshFactoryName}(): Promise<MeshInstance<
   return getMesh<MeshContext>(meshConfig);
 }
 
-export async function ${cliParams.builtMeshSDKFactoryName}<TContext = any>() {
-  const { execute } = await ${cliParams.builtMeshFactoryName}();
-  return getSdk<TContext>((document, variables, context) => execute(document, variables, context));
+export async function ${
+                cliParams.builtMeshSDKFactoryName
+              }<TGlobalContext = any, TOperationContext = any>(globalContext: TGlobalContext) {
+  const { execute, meshContext } = await ${cliParams.builtMeshFactoryName}();
+  return getSdk<TOperationContext>((document, variables, context) => execute(document, variables, ({
+    ...context,
+    ...meshContext,
+  })));
 }`;
 
               return {

diff --git a/packages/config/package.json b/packages/config/package.json
@@ -30,11 +30,11 @@
     "@graphql-mesh/types": "0.66.6",
     "@graphql-mesh/store": "0.3.29",
     "@graphql-mesh/utils": "0.29.0",
-    "@graphql-tools/merge": "8.2.3",
-    "@graphql-tools/utils": "8.6.2",
-    "@graphql-tools/load": "7.5.2",
-    "@graphql-tools/code-file-loader": "7.2.4",
-    "@graphql-tools/graphql-file-loader": "7.3.4",
+    "@graphql-tools/merge": "8.2.4",
+    "@graphql-tools/utils": "8.6.3",
+    "@graphql-tools/load": "7.5.3",
+    "@graphql-tools/code-file-loader": "7.2.5",
+    "@graphql-tools/graphql-file-loader": "7.3.5",
     "param-case": "3.0.4",
     "@envelop/core": "2.1.0",
     "tslib": "^2.3.1",

diff --git a/packages/handlers/graphql/package.json b/packages/handlers/graphql/package.json
@@ -26,10 +26,10 @@
     "@graphql-mesh/store": "0.3.29",
     "@graphql-mesh/utils": "0.29.0",
     "@graphql-mesh/types": "0.66.6",
-    "@graphql-tools/url-loader": "7.9.2",
-    "@graphql-tools/wrap": "8.4.4",
-    "@graphql-tools/delegate": "8.5.3",
-    "@graphql-tools/utils": "8.6.2",
+    "@graphql-tools/url-loader": "7.9.4",
+    "@graphql-tools/wrap": "8.4.6",
+    "@graphql-tools/delegate": "8.5.4",
+    "@graphql-tools/utils": "8.6.3",
     "tslib": "^2.3.1"
   },
   "publishConfig": {

diff --git a/packages/handlers/graphql/src/index.ts b/packages/handlers/graphql/src/index.ts
@@ -41,7 +41,7 @@ export default class GraphQLHandler implements MeshHandler {
     this.config = config;
     this.baseDir = baseDir;
     this.cache = cache;
-    this.nonExecutableSchema = store.proxy('schema.graphql', PredefinedProxyOptions.GraphQLSchemaWithDiffing);
+    this.nonExecutableSchema = store.proxy('introspectionSchema', PredefinedProxyOptions.GraphQLSchemaWithDiffing);
     this.importFn = importFn;
   }
 

diff --git a/packages/handlers/json-schema/package.json b/packages/handlers/json-schema/package.json
@@ -27,7 +27,7 @@
     "@graphql-mesh/types": "0.66.6",
     "@graphql-mesh/utils": "0.29.0",
     "@graphql-mesh/store": "0.3.29",
-    "@graphql-tools/utils": "8.6.2",
+    "@graphql-tools/utils": "8.6.3",
     "graphql-compose": "9.0.7",
     "@omnigraph/json-schema": "0.17.7",
     "tslib": "^2.3.1",

diff --git a/packages/handlers/json-schema/src/index.ts b/packages/handlers/json-schema/src/index.ts
@@ -22,7 +22,7 @@ export default class JsonSchemaHandler implements MeshHandler {
     this.baseDir = baseDir;
     this.cache = cache;
     this.pubsub = pubsub;
-    this.jsonSchema = store.proxy('jsonSchema.json', JsonSchemaWithDiff);
+    this.jsonSchema = store.proxy('jsonSchemaBundle', JsonSchemaWithDiff);
     this.logger = logger;
     this.fetch = getCachedFetch(this.cache);
   }

diff --git a/packages/handlers/mongoose/package.json b/packages/handlers/mongoose/package.json
@@ -33,7 +33,7 @@
     "graphql-compose-mongoose": "9.7.1",
     "graphql-compose-connection": "8.2.1",
     "graphql-compose-pagination": "8.3.0",
-    "@graphql-tools/stitching-directives": "2.2.2",
+    "@graphql-tools/stitching-directives": "2.2.4",
     "tslib": "^2.3.1"
   },
   "publishConfig": {

diff --git a/packages/handlers/mysql/package.json b/packages/handlers/mysql/package.json
@@ -35,7 +35,7 @@
     "mysql-utilities": "1.1.3",
     "graphql-scalars": "1.15.0",
     "tslib": "^2.3.1",
-    "@graphql-tools/utils": "8.6.2"
+    "@graphql-tools/utils": "8.6.3"
   },
   "devDependencies": {
     "@types/graphql-fields": "1.3.4",

diff --git a/packages/handlers/neo4j/package.json b/packages/handlers/neo4j/package.json
@@ -32,7 +32,7 @@
     "neo4j-driver": "4.4.3",
     "neo4j-graphql-js": "2.19.4",
     "tslib": "^2.3.1",
-    "@graphql-tools/utils": "8.6.2"
+    "@graphql-tools/utils": "8.6.3"
   },
   "publishConfig": {
     "access": "public",

diff --git a/packages/handlers/new-openapi/src/index.ts b/packages/handlers/new-openapi/src/index.ts
@@ -24,7 +24,7 @@ export default class OpenAPIHandler implements MeshHandler {
     this.config = config;
     this.baseDir = baseDir;
     this.fetch = getCachedFetch(cache);
-    this.bundleStoreProxy = store.proxy('oas-bundle.js', PredefinedProxyOptions.JsonWithoutValidation);
+    this.bundleStoreProxy = store.proxy('jsonSchemaBundle', PredefinedProxyOptions.JsonWithoutValidation);
     this.pubsub = pubsub;
     this.logger = logger;
   }

diff --git a/packages/handlers/odata/package.json b/packages/handlers/odata/package.json
@@ -30,7 +30,7 @@
     "@graphql-mesh/types": "0.66.6",
     "@graphql-mesh/store": "0.3.29",
     "@graphql-mesh/utils": "0.29.0",
-    "@graphql-tools/utils": "8.6.2",
+    "@graphql-tools/utils": "8.6.3",
     "http-string-parser": "0.0.6",
     "cross-undici-fetch": "0.1.25",
     "dataloader": "2.0.0",

diff --git a/packages/handlers/openapi/package.json b/packages/handlers/openapi/package.json
@@ -30,7 +30,7 @@
     "@graphql-mesh/types": "0.66.6",
     "@graphql-mesh/store": "0.3.29",
     "@graphql-mesh/utils": "0.29.0",
-    "@graphql-tools/utils": "8.6.2",
+    "@graphql-tools/utils": "8.6.3",
     "cross-undici-fetch": "0.1.25",
     "deep-equal": "2.0.5",
     "form-urlencoded": "6.0.6",

diff --git a/packages/handlers/raml/src/index.ts b/packages/handlers/raml/src/index.ts
@@ -16,7 +16,7 @@ export default class RAMLHandler implements MeshHandler {
     this.config = config;
     this.baseDir = baseDir;
     this.fetch = getCachedFetch(cache);
-    this.bundleStoreProxy = store.proxy('raml-bundle.js', PredefinedProxyOptions.JsonWithoutValidation);
+    this.bundleStoreProxy = store.proxy('jsonSchemaBundle', PredefinedProxyOptions.JsonWithoutValidation);
     this.pubsub = pubsub;
     this.logger = logger;
   }

diff --git a/packages/handlers/soap/package.json b/packages/handlers/soap/package.json
@@ -26,7 +26,7 @@
     "@graphql-mesh/types": "0.66.6",
     "@graphql-mesh/store": "0.3.29",
     "@graphql-mesh/utils": "0.29.0",
-    "@graphql-tools/utils": "8.6.2",
+    "@graphql-tools/utils": "8.6.3",
     "graphql-scalars": "1.15.0",
     "soap": "0.43.0",
     "tslib": "^2.3.1"

diff --git a/packages/handlers/thrift/package.json b/packages/handlers/thrift/package.json
@@ -33,7 +33,7 @@
     "@graphql-mesh/types": "0.66.6",
     "@graphql-mesh/store": "0.3.29",
     "@graphql-mesh/utils": "0.29.0",
-    "@graphql-tools/utils": "8.6.2",
+    "@graphql-tools/utils": "8.6.3",
     "fetchache": "0.1.1",
     "graphql-scalars": "1.15.0",
     "url-join": "4.0.1",

diff --git a/packages/loaders/json-schema/package.json b/packages/loaders/json-schema/package.json
@@ -32,7 +32,7 @@
     "json-machete": "0.5.17",
     "@graphql-mesh/types": "0.66.6",
     "@graphql-mesh/utils": "0.29.0",
-    "@graphql-tools/utils": "8.6.2",
+    "@graphql-tools/utils": "8.6.3",
     "to-json-schema": "0.2.5",
     "graphql-compose": "9.0.7",
     "graphql-scalars": "1.15.0",

diff --git a/packages/loaders/raml/package.json b/packages/loaders/raml/package.json
@@ -29,7 +29,7 @@
     "json-machete": "0.5.17",
     "to-json-schema": "0.2.5",
     "@graphql-mesh/utils": "0.29.0",
-    "@graphql-tools/utils": "8.6.2",
+    "@graphql-tools/utils": "8.6.3",
     "tslib": "^2.3.1",
     "change-case": "4.1.2"
   },