From 42e7ec6275d02297035f27295d70916983f1afda Mon Sep 17 00:00:00 2001 From: Zetazzz Date: Mon, 3 Jul 2023 14:30:01 +0800 Subject: [PATCH 1/4] add includes option --- .../src/encoding/proto/implements/decoder.ts | 7 - .../encoding/proto/implements/from-amino.ts | 5 - .../src/encoding/proto/implements/to-amino.ts | 5 - packages/ast/types/test-utils/index.d.ts | 4 + .../store/__snapshots__/store.test.ts.snap | 216 ++++++++++++++++++ packages/parser/__tests__/store/store.test.ts | 126 ++++++++++ packages/parser/src/store.ts | 26 ++- .../src/generators/create-amino-converters.ts | 4 - .../telescope/src/generators/create-types.ts | 2 - packages/types/src/telescope.ts | 5 + packages/types/types/telescope.d.ts | 4 + 11 files changed, 379 insertions(+), 25 deletions(-) create mode 100644 packages/parser/__tests__/store/__snapshots__/store.test.ts.snap create mode 100644 packages/parser/__tests__/store/store.test.ts diff --git a/packages/ast/src/encoding/proto/implements/decoder.ts b/packages/ast/src/encoding/proto/implements/decoder.ts index 2d8c64d47c..503de82765 100644 --- a/packages/ast/src/encoding/proto/implements/decoder.ts +++ b/packages/ast/src/encoding/proto/implements/decoder.ts @@ -51,13 +51,6 @@ export const createInterfaceDecoderHelper = ( // MARKED AS NOT DRY const allTypes: TypeUrlRef[] = typeRefs?.reduce((m, typeRef) => { - // check excludes - const packages = - context.pluginValue('prototypes.excluded.packages') ?? []; - const protos = context.pluginValue('prototypes.excluded.protos') ?? []; - const excluded = - packages.includes(typeRef.pkg) || protos.includes(typeRef.ref); - if (excluded) return m; return [...m, ...typeRef.types]; }, []) ?? []; diff --git a/packages/ast/src/encoding/proto/implements/from-amino.ts b/packages/ast/src/encoding/proto/implements/from-amino.ts index 01238856c9..5dc223b8ca 100644 --- a/packages/ast/src/encoding/proto/implements/from-amino.ts +++ b/packages/ast/src/encoding/proto/implements/from-amino.ts @@ -103,11 +103,6 @@ export const createInterfaceFromAminoHelper = ( // MARKED AS NOT DRY const allTypes: TypeUrlRef[] = typeRefs?.reduce((m, typeRef) => { - // check excludes - const packages = context.pluginValue('prototypes.excluded.packages') ?? []; - const protos = context.pluginValue('prototypes.excluded.protos') ?? []; - const excluded = packages.includes(typeRef.pkg) || protos.includes(typeRef.ref); - if (excluded) return m; return [...m, ...typeRef.types]; }, []) ?? []; diff --git a/packages/ast/src/encoding/proto/implements/to-amino.ts b/packages/ast/src/encoding/proto/implements/to-amino.ts index 302c34c8a4..654ccc05c0 100644 --- a/packages/ast/src/encoding/proto/implements/to-amino.ts +++ b/packages/ast/src/encoding/proto/implements/to-amino.ts @@ -96,11 +96,6 @@ export const createInterfaceToAminoHelper = ( // MARKED AS NOT DRY const allTypes: TypeUrlRef[] = typeRefs?.reduce((m, typeRef) => { - // check excludes - const packages = context.pluginValue('prototypes.excluded.packages') ?? []; - const protos = context.pluginValue('prototypes.excluded.protos') ?? []; - const excluded = packages.includes(typeRef.pkg) || protos.includes(typeRef.ref); - if (excluded) return m; return [...m, ...typeRef.types]; }, []) ?? []; diff --git a/packages/ast/types/test-utils/index.d.ts b/packages/ast/types/test-utils/index.d.ts index e8c06f7c84..cff8ae30e6 100644 --- a/packages/ast/types/test-utils/index.d.ts +++ b/packages/ast/types/test-utils/index.d.ts @@ -49,6 +49,10 @@ export declare const defaultTelescopeOptions: { packages?: string[]; protos?: string[]; }; + includes?: { + packages?: string[]; + protos?: string[]; + }; typingsFormat?: { customTypes?: { useCosmosSDKDec?: boolean; diff --git a/packages/parser/__tests__/store/__snapshots__/store.test.ts.snap b/packages/parser/__tests__/store/__snapshots__/store.test.ts.snap new file mode 100644 index 0000000000..34ebe3e2c2 --- /dev/null +++ b/packages/parser/__tests__/store/__snapshots__/store.test.ts.snap @@ -0,0 +1,216 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`cosmos/gov/v1beta1/gov.proto 1`] = ` +[ + "cosmos.base.v1beta1", + "cosmos.gov.v1beta1", + "cosmos_proto", + "gogoproto", + "google.protobuf", +] +`; + +exports[`cosmos/gov/v1beta1/gov.proto excluded 1`] = `[]`; + +exports[`cosmos/gov/v1beta1/gov.proto excluded dep 1`] = ` +[ + "cosmos.base.v1beta1", + "cosmos.gov.v1beta1", + "cosmos_proto", + "gogoproto", + "google.protobuf", +] +`; + +exports[`get all packages 1`] = ` +[ + "akash.audit.v1beta1", + "akash.audit.v1beta2", + "akash.base.v1beta1", + "akash.base.v1beta2", + "akash.cert.v1beta2", + "akash.deployment.v1beta1", + "akash.deployment.v1beta2", + "akash.escrow.v1beta1", + "akash.escrow.v1beta2", + "akash.inflation.v1beta2", + "akash.market.v1beta2", + "akash.provider.v1beta1", + "akash.provider.v1beta2", + "cosmos.app.v1alpha1", + "cosmos.auth.v1beta1", + "cosmos.authz.v1beta1", + "cosmos.bank.v1beta1", + "cosmos.base.abci.v1beta1", + "cosmos.base.kv.v1beta1", + "cosmos.base.query.v1beta1", + "cosmos.base.reflection.v1beta1", + "cosmos.base.reflection.v2alpha1", + "cosmos.base.snapshots.v1beta1", + "cosmos.base.store.v1beta1", + "cosmos.base.tendermint.v1beta1", + "cosmos.base.v1beta1", + "cosmos.capability.v1beta1", + "cosmos.crisis.v1beta1", + "cosmos.crypto.ed25519", + "cosmos.crypto.hd.v1", + "cosmos.crypto.keyring.v1", + "cosmos.crypto.multisig", + "cosmos.crypto.multisig.v1beta1", + "cosmos.crypto.secp256k1", + "cosmos.crypto.secp256r1", + "cosmos.distribution.v1beta1", + "cosmos.evidence.v1beta1", + "cosmos.feegrant.v1beta1", + "cosmos.genutil.v1beta1", + "cosmos.gov.v1", + "cosmos.gov.v1beta1", + "cosmos.group.v1", + "cosmos.mint.v1beta1", + "cosmos.msg.v1", + "cosmos.nft.v1beta1", + "cosmos.orm.module.v1alpha1", + "cosmos.orm.v1", + "cosmos.orm.v1alpha1", + "cosmos.params.v1beta1", + "cosmos.slashing.v1beta1", + "cosmos.staking.v1beta1", + "cosmos.tx.signing.v1beta1", + "cosmos.tx.v1beta1", + "cosmos.upgrade.v1beta1", + "cosmos.vesting.v1beta1", + "cosmos_proto", + "cosmwasm.wasm.v1", + "evmos.claims.v1", + "evmos.epochs.v1", + "evmos.erc20.v1", + "evmos.fees.v1", + "evmos.incentives.v1", + "evmos.inflation.v1", + "evmos.recovery.v1", + "evmos.vesting.v1", + "gogoproto", + "google.api", + "google.api.expr.conformance.v1alpha1", + "google.api.expr.v1alpha1", + "google.api.expr.v1beta1", + "google.api.servicecontrol.v1", + "google.api.servicecontrol.v2", + "google.api.servicemanagement.v1", + "google.api.serviceusage.v1", + "google.api.serviceusage.v1beta1", + "google.logging.type", + "google.logging.v2", + "google.longrunning", + "google.protobuf", + "google.protobuf.compiler", + "google.rpc", + "google.rpc.context", + "ibc.applications.transfer.v1", + "ibc.applications.transfer.v2", + "ibc.core.channel.v1", + "ibc.core.client.v1", + "ibc.core.commitment.v1", + "ibc.core.connection.v1", + "ibc.core.port.v1", + "ibc.core.types.v1", + "ibc.lightclients.localhost.v1", + "ibc.lightclients.solomachine.v1", + "ibc.lightclients.solomachine.v2", + "ibc.lightclients.tendermint.v1", + "ics23", + "osmosis.claim.v1beta1", + "osmosis.epochs.v1beta1", + "osmosis.gamm.poolmodels.balancer.v1beta1", + "osmosis.gamm.poolmodels.stableswap.v1beta1", + "osmosis.gamm.v1beta1", + "osmosis.gamm.v2", + "osmosis.ibcratelimit.v1beta1", + "osmosis.incentives", + "osmosis.lockup", + "osmosis.mint.v1beta1", + "osmosis.poolincentives.v1beta1", + "osmosis.store.v1beta1", + "osmosis.superfluid", + "osmosis.superfluid.v1beta1", + "osmosis.tokenfactory.v1beta1", + "osmosis.twap.v1beta1", + "osmosis.txfees.v1beta1", + "tendermint.abci", + "tendermint.crypto", + "tendermint.libs.bits", + "tendermint.p2p", + "tendermint.types", + "tendermint.version", +] +`; + +exports[`gov.proto & tx.proto & signing.proto 1`] = ` +[ + "cosmos.base.v1beta1", + "cosmos.crypto.multisig.v1beta1", + "cosmos.gov.v1beta1", + "cosmos.tx.signing.v1beta1", + "cosmos_proto", + "gogoproto", + "google.protobuf", + "osmosis.gamm.v1beta1", +] +`; + +exports[`gov.proto & tx.proto & signing.proto excluded 1`] = ` +[ + "cosmos.base.v1beta1", + "cosmos.crypto.multisig.v1beta1", + "cosmos.gov.v1beta1", + "cosmos.tx.signing.v1beta1", + "cosmos_proto", + "gogoproto", + "google.protobuf", +] +`; + +exports[`gov.proto & tx.proto 1`] = ` +[ + "cosmos.base.v1beta1", + "cosmos.gov.v1beta1", + "cosmos_proto", + "gogoproto", + "google.protobuf", + "osmosis.gamm.v1beta1", +] +`; + +exports[`osmosis excluded gamm 1`] = ` +[ + "cosmos.bank.v1beta1", + "cosmos.base.query.v1beta1", + "cosmos.base.v1beta1", + "cosmos.staking.v1beta1", + "cosmos_proto", + "gogoproto", + "google.api", + "google.protobuf", + "osmosis.claim.v1beta1", + "osmosis.epochs.v1beta1", + "osmosis.ibcratelimit.v1beta1", + "osmosis.incentives", + "osmosis.lockup", + "osmosis.mint.v1beta1", + "osmosis.poolincentives.v1beta1", + "osmosis.store.v1beta1", + "osmosis.superfluid", + "osmosis.superfluid.v1beta1", + "osmosis.tokenfactory.v1beta1", + "osmosis.twap.v1beta1", + "osmosis.txfees.v1beta1", +] +`; + +exports[`osmosis/gamm/v1beta1/tx.proto 1`] = ` +[ + "cosmos.base.v1beta1", + "gogoproto", + "osmosis.gamm.v1beta1", +] +`; diff --git a/packages/parser/__tests__/store/store.test.ts b/packages/parser/__tests__/store/store.test.ts new file mode 100644 index 0000000000..ed75488791 --- /dev/null +++ b/packages/parser/__tests__/store/store.test.ts @@ -0,0 +1,126 @@ +import { getTestProtoStore } from '../../test-utils'; + +it('get all packages', () => { + const store = getTestProtoStore(); + store.traverseAll(); + const pkgs = store.getPackages().sort(); + + expect(pkgs).toMatchSnapshot(); +}); + +it('cosmos/gov/v1beta1/gov.proto', () => { + const store = getTestProtoStore(); + store.options.prototypes!.includes = { + protos:["cosmos/gov/v1beta1/gov.proto"] + } + store.traverseAll(); + const pkgs = store.getPackages().sort(); + + expect(pkgs).toMatchSnapshot(); +}); + +it('cosmos/gov/v1beta1/gov.proto excluded', () => { + const store = getTestProtoStore(); + store.options.prototypes!.includes = { + protos:["cosmos/gov/v1beta1/gov.proto"] + } + store.options.prototypes!.excluded = { + protos:["cosmos/gov/v1beta1/gov.proto"] + } + store.traverseAll(); + const pkgs = store.getPackages().sort(); + + expect(pkgs).toMatchSnapshot(); +}); + +it('cosmos/gov/v1beta1/gov.proto excluded dep', () => { + const store = getTestProtoStore(); + store.options.prototypes!.includes = { + protos:["cosmos/gov/v1beta1/gov.proto"] + } + store.options.prototypes!.excluded = { + protos:["gogoproto/gogo.proto"] + } + store.traverseAll(); + const pkgs = store.getPackages().sort(); + + expect(pkgs).toMatchSnapshot(); +}); + +it('osmosis/gamm/v1beta1/tx.proto', () => { + const store = getTestProtoStore(); + store.options.prototypes!.includes = { + protos:["osmosis/gamm/v1beta1/tx.proto"] + } + store.traverseAll(); + const pkgs = store.getPackages().sort(); + + expect(pkgs).toMatchSnapshot(); +}); + +it('gov.proto & tx.proto', () => { + const store = getTestProtoStore(); + store.options.prototypes!.includes = { + protos:[ + "cosmos/gov/v1beta1/gov.proto", + "osmosis/gamm/v1beta1/tx.proto" + ] + } + store.traverseAll(); + const pkgs = store.getPackages().sort(); + + expect(pkgs).toMatchSnapshot(); +}); + +it('gov.proto & tx.proto & signing.proto', () => { + const store = getTestProtoStore(); + store.options.prototypes!.includes = { + protos:[ + "cosmos/gov/v1beta1/gov.proto", + "osmosis/gamm/v1beta1/tx.proto", + "cosmos/tx/signing/v1beta1/signing.proto" + ] + } + store.traverseAll(); + const pkgs = store.getPackages().sort(); + + expect(pkgs).toMatchSnapshot(); +}); + +it('gov.proto & tx.proto & signing.proto excluded', () => { + const store = getTestProtoStore(); + store.options.prototypes!.includes = { + protos:[ + "cosmos/gov/v1beta1/gov.proto", + "osmosis/gamm/v1beta1/tx.proto", + "cosmos/tx/signing/v1beta1/signing.proto" + ] + } + store.options.prototypes!.excluded = { + protos:[ + "**/gamm/**", + ] + } + store.traverseAll(); + const pkgs = store.getPackages().sort(); + + expect(pkgs).toMatchSnapshot(); +}); + +it('osmosis excluded gamm', () => { + const store = getTestProtoStore(); + store.options.prototypes!.includes = { + packages:[ + "osmosis.*" + ] + } + store.options.prototypes!.excluded = { + protos:[ + "**/gamm/**", + ] + } + store.traverseAll(); + const pkgs = store.getPackages().sort(); + + expect(pkgs).toMatchSnapshot(); +}); \ No newline at end of file diff --git a/packages/parser/src/store.ts b/packages/parser/src/store.ts index f7b3d7e3ec..b34a78eec8 100644 --- a/packages/parser/src/store.ts +++ b/packages/parser/src/store.ts @@ -3,7 +3,7 @@ import { parse } from '@cosmology/protobufjs'; import { readFileSync } from 'fs'; import { join, resolve as pathResolve } from 'path'; import { ALLOWED_RPC_SERVICES, ProtoDep, ProtoField, ProtoRef, ProtoServiceMethod, ProtoType, TelescopeOptions } from '@osmonauts/types'; -import { createTypeUrlTypeMap, getNestedProto, getPackageAndNestedFromStr } from './utils'; +import { createTypeUrlTypeMap, getNestedProto, getPackageAndNestedFromStr, isRefIncluded, isRefExcluded } from './utils'; import { parseFullyTraversedProtoImports, symbolsToImportNames, traverse } from './traverse'; import { lookupAny, lookupAnyFromImports } from './lookup'; import { defaultTelescopeOptions, TelescopeLogLevel, TraversalSymbol } from '@osmonauts/types'; @@ -165,6 +165,7 @@ export class ProtoStore { getPackages(): string[] { if (this.packages) return this.packages; + this.packages = this.getProtos().reduce((m, ref) => { return [...new Set([...m, ref.proto.package])]; }, []); @@ -207,8 +208,21 @@ export class ProtoStore { } traverseAll(): void { + let actualFiles = new Set(); + if (this._traversed) return; this.protos = this.getProtos().map((ref: ProtoRef) => { + // get included imported files + const isIncluded = isRefIncluded(ref, this.options.prototypes.includes) + const isExcluded = isRefExcluded(ref, this.options.prototypes.excluded) + + if(isIncluded && !isExcluded){ + actualFiles.add(ref.filename); + if(ref.proto.imports && ref.proto.imports.length){ + actualFiles = new Set([...actualFiles, ...ref.proto.imports]) + } + } + return { absolute: ref.absolute, filename: ref.filename, @@ -220,6 +234,10 @@ export class ProtoStore { // process import names this.protos = this.protos.map((ref: ProtoRef) => { + if(!actualFiles.has(ref.filename)){ + return null + } + const traversed = ref.traversed; const symbs = this._symbols .filter(f => f.ref === ref.filename); @@ -237,7 +255,11 @@ export class ProtoStore { ...ref, traversed }; - }); + }).filter(Boolean); + + //reset and recalculate pkgs and deps later + this.packages = null; + this.deps = null; this._traversed = true; } diff --git a/packages/telescope/src/generators/create-amino-converters.ts b/packages/telescope/src/generators/create-amino-converters.ts index 2ea1cfef83..7679acec7a 100644 --- a/packages/telescope/src/generators/create-amino-converters.ts +++ b/packages/telescope/src/generators/create-amino-converters.ts @@ -24,10 +24,6 @@ export const plugin = ( return; } - if (c.proto.isExcluded()) { - return; - } - const localname = bundler.getLocalFilename(c.ref, 'amino'); const filename = bundler.getFilename(localname); const ctx = bundler.getFreshContext(c); diff --git a/packages/telescope/src/generators/create-types.ts b/packages/telescope/src/generators/create-types.ts index eab54b2bbc..0ce81edaf9 100644 --- a/packages/telescope/src/generators/create-types.ts +++ b/packages/telescope/src/generators/create-types.ts @@ -22,8 +22,6 @@ export const plugin = ( bundler.contexts = baseProtos.map(ref => { const context = builder.context(ref); - if (isRefExcluded(ref, builder.options.prototypes?.excluded)) return; - parse(context); context.buildBase(); diff --git a/packages/types/src/telescope.ts b/packages/types/src/telescope.ts index 9719cc4854..b55023792c 100644 --- a/packages/types/src/telescope.ts +++ b/packages/types/src/telescope.ts @@ -62,6 +62,11 @@ interface TelescopeOpts { packages?: string[]; protos?: string[]; }; + includes?: { + packages?: string[]; + protos?: string[]; + }; + typingsFormat?: { customTypes?:{ useCosmosSDKDec?: boolean; diff --git a/packages/types/types/telescope.d.ts b/packages/types/types/telescope.d.ts index 5c7c29b9d5..db6be67cea 100644 --- a/packages/types/types/telescope.d.ts +++ b/packages/types/types/telescope.d.ts @@ -52,6 +52,10 @@ interface TelescopeOpts { packages?: string[]; protos?: string[]; }; + includes?: { + packages?: string[]; + protos?: string[]; + }; typingsFormat?: { customTypes?: { useCosmosSDKDec?: boolean; From 16b27572fdb9d1c31885436000baddfae76b74c3 Mon Sep 17 00:00:00 2001 From: Zetazzz Date: Mon, 3 Jul 2023 16:40:01 +0800 Subject: [PATCH 2/4] add build and nobuild to cli --- packages/parser/src/utils.ts | 6 +-- packages/telescope/src/commands/transpile.ts | 47 +++++++++++++++++++- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/packages/parser/src/utils.ts b/packages/parser/src/utils.ts index 8904e522dc..c01fc934b2 100644 --- a/packages/parser/src/utils.ts +++ b/packages/parser/src/utils.ts @@ -111,7 +111,7 @@ export const isRefIncluded = ( } // TODO consider deprecating `patterns` in favor of packages and protos supporting minimatch - if (include?.patterns?.some(pattern => minimatch(ref.filename, pattern))) { + if (include?.patterns?.some(pattern => Boolean(ref.filename) && minimatch(ref.filename, pattern))) { return true; } @@ -119,7 +119,7 @@ export const isRefIncluded = ( if (!globPattern.test(pkgName)) { return ref.proto.package === pkgName; } - return minimatch(ref.proto.package, pkgName) + return Boolean(ref.proto?.package) && minimatch(ref.proto.package, pkgName) }); if (pkgMatched) { @@ -130,7 +130,7 @@ export const isRefIncluded = ( if (!globPattern.test(protoName)) { return ref.filename === protoName; } - return minimatch(ref.filename, protoName) + return Boolean(ref.filename) && minimatch(ref.filename, protoName) }); if (protoMatched) { diff --git a/packages/telescope/src/commands/transpile.ts b/packages/telescope/src/commands/transpile.ts index 6bd8c8e582..535ba89e0a 100644 --- a/packages/telescope/src/commands/transpile.ts +++ b/packages/telescope/src/commands/transpile.ts @@ -29,6 +29,18 @@ export default async (argv) => { message: 'where is the output directory?', default: './src/codegen' }, + { + _: true, + type: 'path', + name: 'build', + message: 'which files to include. ex: osmosis/**/gamm/**/*.proto or cosmos/bank/v1beta1/bank.proto', + }, + { + _: true, + type: 'path', + name: 'nobuild', + message: 'which files to exclude. ex: osmosis/**/gamm/**/*.proto or cosmos/bank/v1beta1/bank.proto', + }, { type: 'confirm', name: 'includeAminos', @@ -52,6 +64,8 @@ export default async (argv) => { let { protoDirs, outPath, + build, + nobuild, includeAminos, includeLCDClients, includeRPCClients, @@ -61,7 +75,7 @@ export default async (argv) => { protoDirs = [protoDirs]; } - const options = { + const options: any = { aminoEncoding: { enabled: includeAminos }, @@ -73,6 +87,37 @@ export default async (argv) => { } }; + if(build || nobuild){ + if(!options.prototypes){ + options.prototypes = {} + } + + } + + if(build){ + if (!Array.isArray(build)) { + build = [build]; + } + + if(!options.prototypes.includes){ + options.prototypes.includes = {} + } + + options.prototypes.includes.protos = build; + } + + if(nobuild){ + if (!Array.isArray(nobuild)) { + nobuild = [nobuild]; + } + + if(!options.prototypes.excluded){ + options.prototypes.excluded = {} + } + + options.prototypes.excluded.protos = nobuild; + } + writeFileSync( process.cwd() + '/.telescope.json', JSON.stringify( From 11976a734c317883cbdd7ff07a83d6a9aaedbc32 Mon Sep 17 00:00:00 2001 From: Zetazzz Date: Mon, 3 Jul 2023 22:18:57 +0800 Subject: [PATCH 3/4] optimize isRefIncluded --- packages/parser/src/utils.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/parser/src/utils.ts b/packages/parser/src/utils.ts index c01fc934b2..cdf5312a83 100644 --- a/packages/parser/src/utils.ts +++ b/packages/parser/src/utils.ts @@ -111,26 +111,26 @@ export const isRefIncluded = ( } // TODO consider deprecating `patterns` in favor of packages and protos supporting minimatch - if (include?.patterns?.some(pattern => Boolean(ref.filename) && minimatch(ref.filename, pattern))) { + if (Boolean(ref.filename) && include?.patterns?.some(pattern => minimatch(ref.filename, pattern))) { return true; } - const pkgMatched = include?.packages?.some(pkgName => { + const pkgMatched = Boolean(ref.proto?.package) && include?.packages?.some(pkgName => { if (!globPattern.test(pkgName)) { return ref.proto.package === pkgName; } - return Boolean(ref.proto?.package) && minimatch(ref.proto.package, pkgName) + return minimatch(ref.proto.package, pkgName) }); if (pkgMatched) { return true; } - const protoMatched = include?.protos?.some(protoName => { + const protoMatched = Boolean(ref.filename) && include?.protos?.some(protoName => { if (!globPattern.test(protoName)) { return ref.filename === protoName; } - return Boolean(ref.filename) && minimatch(ref.filename, protoName) + return minimatch(ref.filename, protoName) }); if (protoMatched) { From 37cd2c7df4cb67ae7a867c66aa1ec5995e83aca6 Mon Sep 17 00:00:00 2001 From: Zetazzz Date: Tue, 4 Jul 2023 12:42:00 +0800 Subject: [PATCH 4/4] using resolver to fix deps bug --- .../store/__snapshots__/store.test.ts.snap | 6 +++++ .../__tests__/traverse/bad.traversal.test.ts | 2 +- packages/parser/src/store.ts | 24 ++++++++++++------- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/packages/parser/__tests__/store/__snapshots__/store.test.ts.snap b/packages/parser/__tests__/store/__snapshots__/store.test.ts.snap index 34ebe3e2c2..6d370596f8 100644 --- a/packages/parser/__tests__/store/__snapshots__/store.test.ts.snap +++ b/packages/parser/__tests__/store/__snapshots__/store.test.ts.snap @@ -186,6 +186,7 @@ exports[`osmosis excluded gamm 1`] = ` "cosmos.bank.v1beta1", "cosmos.base.query.v1beta1", "cosmos.base.v1beta1", + "cosmos.msg.v1", "cosmos.staking.v1beta1", "cosmos_proto", "gogoproto", @@ -204,13 +205,18 @@ exports[`osmosis excluded gamm 1`] = ` "osmosis.tokenfactory.v1beta1", "osmosis.twap.v1beta1", "osmosis.txfees.v1beta1", + "tendermint.crypto", + "tendermint.types", + "tendermint.version", ] `; exports[`osmosis/gamm/v1beta1/tx.proto 1`] = ` [ "cosmos.base.v1beta1", + "cosmos_proto", "gogoproto", + "google.protobuf", "osmosis.gamm.v1beta1", ] `; diff --git a/packages/parser/__tests__/traverse/bad.traversal.test.ts b/packages/parser/__tests__/traverse/bad.traversal.test.ts index 3c27daaab5..7d63665d01 100644 --- a/packages/parser/__tests__/traverse/bad.traversal.test.ts +++ b/packages/parser/__tests__/traverse/bad.traversal.test.ts @@ -36,7 +36,7 @@ message MyMessage { store.traverseAll(); } catch (e) { failed = true; - expect(e.message).toEqual('missing proto import gogoproto/gogo.proto') + expect(e.message).toEqual('Dependency Not Found gogoproto/gogo.proto') } expect(failed).toBe(true); }); diff --git a/packages/parser/src/store.ts b/packages/parser/src/store.ts index b34a78eec8..0d3c7a2a1a 100644 --- a/packages/parser/src/store.ts +++ b/packages/parser/src/store.ts @@ -16,6 +16,7 @@ import google_empty from './native/empty'; import google_field_mask from './native/field_mask'; import google_struct from './native/struct'; import google_wrappers from './native/wrappers'; +import { ProtoResolver } from './resolver'; const GOOGLE_PROTOS = [ ['google/protobuf/any.proto', google_any], @@ -208,18 +209,23 @@ export class ProtoStore { } traverseAll(): void { + if (this._traversed) return; + let actualFiles = new Set(); + let resolver = new ProtoResolver(this.getDeps()); - if (this._traversed) return; this.protos = this.getProtos().map((ref: ProtoRef) => { - // get included imported files - const isIncluded = isRefIncluded(ref, this.options.prototypes.includes) - const isExcluded = isRefExcluded(ref, this.options.prototypes.excluded) - - if(isIncluded && !isExcluded){ - actualFiles.add(ref.filename); - if(ref.proto.imports && ref.proto.imports.length){ - actualFiles = new Set([...actualFiles, ...ref.proto.imports]) + if( !actualFiles.has(ref.filename) ){ + // get included imported files + const isIncluded = isRefIncluded(ref, this.options.prototypes.includes) + const isExcluded = isRefExcluded(ref, this.options.prototypes.excluded) + + if(isIncluded && !isExcluded){ + const deps = resolver.resolve(ref.filename); + + for (const dep of deps) { + actualFiles.add(dep); + } } }