diff --git a/packages/ast/src/encoding/proto/implements/decoder.ts b/packages/ast/src/encoding/proto/implements/decoder.ts index 2d8c64d47c..503de82765 100644 --- a/packages/ast/src/encoding/proto/implements/decoder.ts +++ b/packages/ast/src/encoding/proto/implements/decoder.ts @@ -51,13 +51,6 @@ export const createInterfaceDecoderHelper = ( // MARKED AS NOT DRY const allTypes: TypeUrlRef[] = typeRefs?.reduce((m, typeRef) => { - // check excludes - const packages = - context.pluginValue('prototypes.excluded.packages') ?? []; - const protos = context.pluginValue('prototypes.excluded.protos') ?? []; - const excluded = - packages.includes(typeRef.pkg) || protos.includes(typeRef.ref); - if (excluded) return m; return [...m, ...typeRef.types]; }, []) ?? []; diff --git a/packages/ast/src/encoding/proto/implements/from-amino.ts b/packages/ast/src/encoding/proto/implements/from-amino.ts index 01238856c9..5dc223b8ca 100644 --- a/packages/ast/src/encoding/proto/implements/from-amino.ts +++ b/packages/ast/src/encoding/proto/implements/from-amino.ts @@ -103,11 +103,6 @@ export const createInterfaceFromAminoHelper = ( // MARKED AS NOT DRY const allTypes: TypeUrlRef[] = typeRefs?.reduce((m, typeRef) => { - // check excludes - const packages = context.pluginValue('prototypes.excluded.packages') ?? []; - const protos = context.pluginValue('prototypes.excluded.protos') ?? []; - const excluded = packages.includes(typeRef.pkg) || protos.includes(typeRef.ref); - if (excluded) return m; return [...m, ...typeRef.types]; }, []) ?? []; diff --git a/packages/ast/src/encoding/proto/implements/to-amino.ts b/packages/ast/src/encoding/proto/implements/to-amino.ts index 302c34c8a4..654ccc05c0 100644 --- a/packages/ast/src/encoding/proto/implements/to-amino.ts +++ b/packages/ast/src/encoding/proto/implements/to-amino.ts @@ -96,11 +96,6 @@ export const createInterfaceToAminoHelper = ( // MARKED AS NOT DRY const allTypes: TypeUrlRef[] = typeRefs?.reduce((m, typeRef) => { - // check excludes - const packages = context.pluginValue('prototypes.excluded.packages') ?? []; - const protos = context.pluginValue('prototypes.excluded.protos') ?? []; - const excluded = packages.includes(typeRef.pkg) || protos.includes(typeRef.ref); - if (excluded) return m; return [...m, ...typeRef.types]; }, []) ?? []; diff --git a/packages/ast/types/test-utils/index.d.ts b/packages/ast/types/test-utils/index.d.ts index e8c06f7c84..cff8ae30e6 100644 --- a/packages/ast/types/test-utils/index.d.ts +++ b/packages/ast/types/test-utils/index.d.ts @@ -49,6 +49,10 @@ export declare const defaultTelescopeOptions: { packages?: string[]; protos?: string[]; }; + includes?: { + packages?: string[]; + protos?: string[]; + }; typingsFormat?: { customTypes?: { useCosmosSDKDec?: boolean; diff --git a/packages/parser/__tests__/store/__snapshots__/store.test.ts.snap b/packages/parser/__tests__/store/__snapshots__/store.test.ts.snap new file mode 100644 index 0000000000..6d370596f8 --- /dev/null +++ b/packages/parser/__tests__/store/__snapshots__/store.test.ts.snap @@ -0,0 +1,222 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`cosmos/gov/v1beta1/gov.proto 1`] = ` +[ + "cosmos.base.v1beta1", + "cosmos.gov.v1beta1", + "cosmos_proto", + "gogoproto", + "google.protobuf", +] +`; + +exports[`cosmos/gov/v1beta1/gov.proto excluded 1`] = `[]`; + +exports[`cosmos/gov/v1beta1/gov.proto excluded dep 1`] = ` +[ + "cosmos.base.v1beta1", + "cosmos.gov.v1beta1", + "cosmos_proto", + "gogoproto", + "google.protobuf", +] +`; + +exports[`get all packages 1`] = ` +[ + "akash.audit.v1beta1", + "akash.audit.v1beta2", + "akash.base.v1beta1", + "akash.base.v1beta2", + "akash.cert.v1beta2", + "akash.deployment.v1beta1", + "akash.deployment.v1beta2", + "akash.escrow.v1beta1", + "akash.escrow.v1beta2", + "akash.inflation.v1beta2", + "akash.market.v1beta2", + "akash.provider.v1beta1", + "akash.provider.v1beta2", + "cosmos.app.v1alpha1", + "cosmos.auth.v1beta1", + "cosmos.authz.v1beta1", + "cosmos.bank.v1beta1", + "cosmos.base.abci.v1beta1", + "cosmos.base.kv.v1beta1", + "cosmos.base.query.v1beta1", + "cosmos.base.reflection.v1beta1", + "cosmos.base.reflection.v2alpha1", + "cosmos.base.snapshots.v1beta1", + "cosmos.base.store.v1beta1", + "cosmos.base.tendermint.v1beta1", + "cosmos.base.v1beta1", + "cosmos.capability.v1beta1", + "cosmos.crisis.v1beta1", + "cosmos.crypto.ed25519", + "cosmos.crypto.hd.v1", + "cosmos.crypto.keyring.v1", + "cosmos.crypto.multisig", + "cosmos.crypto.multisig.v1beta1", + "cosmos.crypto.secp256k1", + "cosmos.crypto.secp256r1", + "cosmos.distribution.v1beta1", + "cosmos.evidence.v1beta1", + "cosmos.feegrant.v1beta1", + "cosmos.genutil.v1beta1", + "cosmos.gov.v1", + "cosmos.gov.v1beta1", + "cosmos.group.v1", + "cosmos.mint.v1beta1", + "cosmos.msg.v1", + "cosmos.nft.v1beta1", + "cosmos.orm.module.v1alpha1", + "cosmos.orm.v1", + "cosmos.orm.v1alpha1", + "cosmos.params.v1beta1", + "cosmos.slashing.v1beta1", + "cosmos.staking.v1beta1", + "cosmos.tx.signing.v1beta1", + "cosmos.tx.v1beta1", + "cosmos.upgrade.v1beta1", + "cosmos.vesting.v1beta1", + "cosmos_proto", + "cosmwasm.wasm.v1", + "evmos.claims.v1", + "evmos.epochs.v1", + "evmos.erc20.v1", + "evmos.fees.v1", + "evmos.incentives.v1", + "evmos.inflation.v1", + "evmos.recovery.v1", + "evmos.vesting.v1", + "gogoproto", + "google.api", + "google.api.expr.conformance.v1alpha1", + "google.api.expr.v1alpha1", + "google.api.expr.v1beta1", + "google.api.servicecontrol.v1", + "google.api.servicecontrol.v2", + "google.api.servicemanagement.v1", + "google.api.serviceusage.v1", + "google.api.serviceusage.v1beta1", + "google.logging.type", + "google.logging.v2", + "google.longrunning", + "google.protobuf", + "google.protobuf.compiler", + "google.rpc", + "google.rpc.context", + "ibc.applications.transfer.v1", + "ibc.applications.transfer.v2", + "ibc.core.channel.v1", + "ibc.core.client.v1", + "ibc.core.commitment.v1", + "ibc.core.connection.v1", + "ibc.core.port.v1", + "ibc.core.types.v1", + "ibc.lightclients.localhost.v1", + "ibc.lightclients.solomachine.v1", + "ibc.lightclients.solomachine.v2", + "ibc.lightclients.tendermint.v1", + "ics23", + "osmosis.claim.v1beta1", + "osmosis.epochs.v1beta1", + "osmosis.gamm.poolmodels.balancer.v1beta1", + "osmosis.gamm.poolmodels.stableswap.v1beta1", + "osmosis.gamm.v1beta1", + "osmosis.gamm.v2", + "osmosis.ibcratelimit.v1beta1", + "osmosis.incentives", + "osmosis.lockup", + "osmosis.mint.v1beta1", + "osmosis.poolincentives.v1beta1", + "osmosis.store.v1beta1", + "osmosis.superfluid", + "osmosis.superfluid.v1beta1", + "osmosis.tokenfactory.v1beta1", + "osmosis.twap.v1beta1", + "osmosis.txfees.v1beta1", + "tendermint.abci", + "tendermint.crypto", + "tendermint.libs.bits", + "tendermint.p2p", + "tendermint.types", + "tendermint.version", +] +`; + +exports[`gov.proto & tx.proto & signing.proto 1`] = ` +[ + "cosmos.base.v1beta1", + "cosmos.crypto.multisig.v1beta1", + "cosmos.gov.v1beta1", + "cosmos.tx.signing.v1beta1", + "cosmos_proto", + "gogoproto", + "google.protobuf", + "osmosis.gamm.v1beta1", +] +`; + +exports[`gov.proto & tx.proto & signing.proto excluded 1`] = ` +[ + "cosmos.base.v1beta1", + "cosmos.crypto.multisig.v1beta1", + "cosmos.gov.v1beta1", + "cosmos.tx.signing.v1beta1", + "cosmos_proto", + "gogoproto", + "google.protobuf", +] +`; + +exports[`gov.proto & tx.proto 1`] = ` +[ + "cosmos.base.v1beta1", + "cosmos.gov.v1beta1", + "cosmos_proto", + "gogoproto", + "google.protobuf", + "osmosis.gamm.v1beta1", +] +`; + +exports[`osmosis excluded gamm 1`] = ` +[ + "cosmos.bank.v1beta1", + "cosmos.base.query.v1beta1", + "cosmos.base.v1beta1", + "cosmos.msg.v1", + "cosmos.staking.v1beta1", + "cosmos_proto", + "gogoproto", + "google.api", + "google.protobuf", + "osmosis.claim.v1beta1", + "osmosis.epochs.v1beta1", + "osmosis.ibcratelimit.v1beta1", + "osmosis.incentives", + "osmosis.lockup", + "osmosis.mint.v1beta1", + "osmosis.poolincentives.v1beta1", + "osmosis.store.v1beta1", + "osmosis.superfluid", + "osmosis.superfluid.v1beta1", + "osmosis.tokenfactory.v1beta1", + "osmosis.twap.v1beta1", + "osmosis.txfees.v1beta1", + "tendermint.crypto", + "tendermint.types", + "tendermint.version", +] +`; + +exports[`osmosis/gamm/v1beta1/tx.proto 1`] = ` +[ + "cosmos.base.v1beta1", + "cosmos_proto", + "gogoproto", + "google.protobuf", + "osmosis.gamm.v1beta1", +] +`; diff --git a/packages/parser/__tests__/store/store.test.ts b/packages/parser/__tests__/store/store.test.ts new file mode 100644 index 0000000000..ed75488791 --- /dev/null +++ b/packages/parser/__tests__/store/store.test.ts @@ -0,0 +1,126 @@ +import { getTestProtoStore } from '../../test-utils'; + +it('get all packages', () => { + const store = getTestProtoStore(); + store.traverseAll(); + const pkgs = store.getPackages().sort(); + + expect(pkgs).toMatchSnapshot(); +}); + +it('cosmos/gov/v1beta1/gov.proto', () => { + const store = getTestProtoStore(); + store.options.prototypes!.includes = { + protos:["cosmos/gov/v1beta1/gov.proto"] + } + store.traverseAll(); + const pkgs = store.getPackages().sort(); + + expect(pkgs).toMatchSnapshot(); +}); + +it('cosmos/gov/v1beta1/gov.proto excluded', () => { + const store = getTestProtoStore(); + store.options.prototypes!.includes = { + protos:["cosmos/gov/v1beta1/gov.proto"] + } + store.options.prototypes!.excluded = { + protos:["cosmos/gov/v1beta1/gov.proto"] + } + store.traverseAll(); + const pkgs = store.getPackages().sort(); + + expect(pkgs).toMatchSnapshot(); +}); + +it('cosmos/gov/v1beta1/gov.proto excluded dep', () => { + const store = getTestProtoStore(); + store.options.prototypes!.includes = { + protos:["cosmos/gov/v1beta1/gov.proto"] + } + store.options.prototypes!.excluded = { + protos:["gogoproto/gogo.proto"] + } + store.traverseAll(); + const pkgs = store.getPackages().sort(); + + expect(pkgs).toMatchSnapshot(); +}); + +it('osmosis/gamm/v1beta1/tx.proto', () => { + const store = getTestProtoStore(); + store.options.prototypes!.includes = { + protos:["osmosis/gamm/v1beta1/tx.proto"] + } + store.traverseAll(); + const pkgs = store.getPackages().sort(); + + expect(pkgs).toMatchSnapshot(); +}); + +it('gov.proto & tx.proto', () => { + const store = getTestProtoStore(); + store.options.prototypes!.includes = { + protos:[ + "cosmos/gov/v1beta1/gov.proto", + "osmosis/gamm/v1beta1/tx.proto" + ] + } + store.traverseAll(); + const pkgs = store.getPackages().sort(); + + expect(pkgs).toMatchSnapshot(); +}); + +it('gov.proto & tx.proto & signing.proto', () => { + const store = getTestProtoStore(); + store.options.prototypes!.includes = { + protos:[ + "cosmos/gov/v1beta1/gov.proto", + "osmosis/gamm/v1beta1/tx.proto", + "cosmos/tx/signing/v1beta1/signing.proto" + ] + } + store.traverseAll(); + const pkgs = store.getPackages().sort(); + + expect(pkgs).toMatchSnapshot(); +}); + +it('gov.proto & tx.proto & signing.proto excluded', () => { + const store = getTestProtoStore(); + store.options.prototypes!.includes = { + protos:[ + "cosmos/gov/v1beta1/gov.proto", + "osmosis/gamm/v1beta1/tx.proto", + "cosmos/tx/signing/v1beta1/signing.proto" + ] + } + store.options.prototypes!.excluded = { + protos:[ + "**/gamm/**", + ] + } + store.traverseAll(); + const pkgs = store.getPackages().sort(); + + expect(pkgs).toMatchSnapshot(); +}); + +it('osmosis excluded gamm', () => { + const store = getTestProtoStore(); + store.options.prototypes!.includes = { + packages:[ + "osmosis.*" + ] + } + store.options.prototypes!.excluded = { + protos:[ + "**/gamm/**", + ] + } + store.traverseAll(); + const pkgs = store.getPackages().sort(); + + expect(pkgs).toMatchSnapshot(); +}); \ No newline at end of file diff --git a/packages/parser/__tests__/traverse/bad.traversal.test.ts b/packages/parser/__tests__/traverse/bad.traversal.test.ts index 3c27daaab5..7d63665d01 100644 --- a/packages/parser/__tests__/traverse/bad.traversal.test.ts +++ b/packages/parser/__tests__/traverse/bad.traversal.test.ts @@ -36,7 +36,7 @@ message MyMessage { store.traverseAll(); } catch (e) { failed = true; - expect(e.message).toEqual('missing proto import gogoproto/gogo.proto') + expect(e.message).toEqual('Dependency Not Found gogoproto/gogo.proto') } expect(failed).toBe(true); }); diff --git a/packages/parser/src/store.ts b/packages/parser/src/store.ts index f7b3d7e3ec..0d3c7a2a1a 100644 --- a/packages/parser/src/store.ts +++ b/packages/parser/src/store.ts @@ -3,7 +3,7 @@ import { parse } from '@cosmology/protobufjs'; import { readFileSync } from 'fs'; import { join, resolve as pathResolve } from 'path'; import { ALLOWED_RPC_SERVICES, ProtoDep, ProtoField, ProtoRef, ProtoServiceMethod, ProtoType, TelescopeOptions } from '@osmonauts/types'; -import { createTypeUrlTypeMap, getNestedProto, getPackageAndNestedFromStr } from './utils'; +import { createTypeUrlTypeMap, getNestedProto, getPackageAndNestedFromStr, isRefIncluded, isRefExcluded } from './utils'; import { parseFullyTraversedProtoImports, symbolsToImportNames, traverse } from './traverse'; import { lookupAny, lookupAnyFromImports } from './lookup'; import { defaultTelescopeOptions, TelescopeLogLevel, TraversalSymbol } from '@osmonauts/types'; @@ -16,6 +16,7 @@ import google_empty from './native/empty'; import google_field_mask from './native/field_mask'; import google_struct from './native/struct'; import google_wrappers from './native/wrappers'; +import { ProtoResolver } from './resolver'; const GOOGLE_PROTOS = [ ['google/protobuf/any.proto', google_any], @@ -165,6 +166,7 @@ export class ProtoStore { getPackages(): string[] { if (this.packages) return this.packages; + this.packages = this.getProtos().reduce((m, ref) => { return [...new Set([...m, ref.proto.package])]; }, []); @@ -208,7 +210,25 @@ export class ProtoStore { traverseAll(): void { if (this._traversed) return; + + let actualFiles = new Set(); + let resolver = new ProtoResolver(this.getDeps()); + this.protos = this.getProtos().map((ref: ProtoRef) => { + if( !actualFiles.has(ref.filename) ){ + // get included imported files + const isIncluded = isRefIncluded(ref, this.options.prototypes.includes) + const isExcluded = isRefExcluded(ref, this.options.prototypes.excluded) + + if(isIncluded && !isExcluded){ + const deps = resolver.resolve(ref.filename); + + for (const dep of deps) { + actualFiles.add(dep); + } + } + } + return { absolute: ref.absolute, filename: ref.filename, @@ -220,6 +240,10 @@ export class ProtoStore { // process import names this.protos = this.protos.map((ref: ProtoRef) => { + if(!actualFiles.has(ref.filename)){ + return null + } + const traversed = ref.traversed; const symbs = this._symbols .filter(f => f.ref === ref.filename); @@ -237,7 +261,11 @@ export class ProtoStore { ...ref, traversed }; - }); + }).filter(Boolean); + + //reset and recalculate pkgs and deps later + this.packages = null; + this.deps = null; this._traversed = true; } diff --git a/packages/parser/src/utils.ts b/packages/parser/src/utils.ts index 8904e522dc..cdf5312a83 100644 --- a/packages/parser/src/utils.ts +++ b/packages/parser/src/utils.ts @@ -111,11 +111,11 @@ export const isRefIncluded = ( } // TODO consider deprecating `patterns` in favor of packages and protos supporting minimatch - if (include?.patterns?.some(pattern => minimatch(ref.filename, pattern))) { + if (Boolean(ref.filename) && include?.patterns?.some(pattern => minimatch(ref.filename, pattern))) { return true; } - const pkgMatched = include?.packages?.some(pkgName => { + const pkgMatched = Boolean(ref.proto?.package) && include?.packages?.some(pkgName => { if (!globPattern.test(pkgName)) { return ref.proto.package === pkgName; } @@ -126,7 +126,7 @@ export const isRefIncluded = ( return true; } - const protoMatched = include?.protos?.some(protoName => { + const protoMatched = Boolean(ref.filename) && include?.protos?.some(protoName => { if (!globPattern.test(protoName)) { return ref.filename === protoName; } diff --git a/packages/telescope/src/commands/transpile.ts b/packages/telescope/src/commands/transpile.ts index 6bd8c8e582..535ba89e0a 100644 --- a/packages/telescope/src/commands/transpile.ts +++ b/packages/telescope/src/commands/transpile.ts @@ -29,6 +29,18 @@ export default async (argv) => { message: 'where is the output directory?', default: './src/codegen' }, + { + _: true, + type: 'path', + name: 'build', + message: 'which files to include. ex: osmosis/**/gamm/**/*.proto or cosmos/bank/v1beta1/bank.proto', + }, + { + _: true, + type: 'path', + name: 'nobuild', + message: 'which files to exclude. ex: osmosis/**/gamm/**/*.proto or cosmos/bank/v1beta1/bank.proto', + }, { type: 'confirm', name: 'includeAminos', @@ -52,6 +64,8 @@ export default async (argv) => { let { protoDirs, outPath, + build, + nobuild, includeAminos, includeLCDClients, includeRPCClients, @@ -61,7 +75,7 @@ export default async (argv) => { protoDirs = [protoDirs]; } - const options = { + const options: any = { aminoEncoding: { enabled: includeAminos }, @@ -73,6 +87,37 @@ export default async (argv) => { } }; + if(build || nobuild){ + if(!options.prototypes){ + options.prototypes = {} + } + + } + + if(build){ + if (!Array.isArray(build)) { + build = [build]; + } + + if(!options.prototypes.includes){ + options.prototypes.includes = {} + } + + options.prototypes.includes.protos = build; + } + + if(nobuild){ + if (!Array.isArray(nobuild)) { + nobuild = [nobuild]; + } + + if(!options.prototypes.excluded){ + options.prototypes.excluded = {} + } + + options.prototypes.excluded.protos = nobuild; + } + writeFileSync( process.cwd() + '/.telescope.json', JSON.stringify( diff --git a/packages/telescope/src/generators/create-amino-converters.ts b/packages/telescope/src/generators/create-amino-converters.ts index 2ea1cfef83..7679acec7a 100644 --- a/packages/telescope/src/generators/create-amino-converters.ts +++ b/packages/telescope/src/generators/create-amino-converters.ts @@ -24,10 +24,6 @@ export const plugin = ( return; } - if (c.proto.isExcluded()) { - return; - } - const localname = bundler.getLocalFilename(c.ref, 'amino'); const filename = bundler.getFilename(localname); const ctx = bundler.getFreshContext(c); diff --git a/packages/telescope/src/generators/create-types.ts b/packages/telescope/src/generators/create-types.ts index eab54b2bbc..0ce81edaf9 100644 --- a/packages/telescope/src/generators/create-types.ts +++ b/packages/telescope/src/generators/create-types.ts @@ -22,8 +22,6 @@ export const plugin = ( bundler.contexts = baseProtos.map(ref => { const context = builder.context(ref); - if (isRefExcluded(ref, builder.options.prototypes?.excluded)) return; - parse(context); context.buildBase(); diff --git a/packages/types/src/telescope.ts b/packages/types/src/telescope.ts index 9719cc4854..b55023792c 100644 --- a/packages/types/src/telescope.ts +++ b/packages/types/src/telescope.ts @@ -62,6 +62,11 @@ interface TelescopeOpts { packages?: string[]; protos?: string[]; }; + includes?: { + packages?: string[]; + protos?: string[]; + }; + typingsFormat?: { customTypes?:{ useCosmosSDKDec?: boolean; diff --git a/packages/types/types/telescope.d.ts b/packages/types/types/telescope.d.ts index 5c7c29b9d5..db6be67cea 100644 --- a/packages/types/types/telescope.d.ts +++ b/packages/types/types/telescope.d.ts @@ -52,6 +52,10 @@ interface TelescopeOpts { packages?: string[]; protos?: string[]; }; + includes?: { + packages?: string[]; + protos?: string[]; + }; typingsFormat?: { customTypes?: { useCosmosSDKDec?: boolean;