From 9e5099d0213c68831c92bc37b2ee7e6392eb7f72 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Wed, 12 Jun 2024 19:34:09 -0700 Subject: [PATCH 1/3] json-patch-protos to allow inline customizations when SDK is lagging fixes --- .../proto.enum.custom.spec.ts.snap | 8 ++-- .../encoding/proto/proto.enum.custom.spec.ts | 12 ++++- packages/parser/package.json | 3 +- packages/parser/src/store.ts | 34 ++++++++------ packages/types/package.json | 3 +- packages/types/src/telescope.ts | 5 +++ packages/types/types/telescope.d.ts | 4 ++ packages/utils/src/utils.ts | 45 ++++++++++++++++++- packages/utils/types/utils.d.ts | 25 +++++++++++ yarn.lock | 5 +++ 10 files changed, 121 insertions(+), 23 deletions(-) diff --git a/packages/ast/src/encoding/proto/__snapshots__/proto.enum.custom.spec.ts.snap b/packages/ast/src/encoding/proto/__snapshots__/proto.enum.custom.spec.ts.snap index 7d0860a70..f1af1d841 100644 --- a/packages/ast/src/encoding/proto/__snapshots__/proto.enum.custom.spec.ts.snap +++ b/packages/ast/src/encoding/proto/__snapshots__/proto.enum.custom.spec.ts.snap @@ -22,13 +22,13 @@ exports[`createProtoEnumFromJSON 1`] = ` case "AccessTypeUnspecified": return AccessType.ACCESS_TYPE_UNSPECIFIED; case 1: - case "AccessTypeNobody": + case "NobodyAccess": return AccessType.ACCESS_TYPE_NOBODY; case 2: case "AccessTypeOnlyAddress": return AccessType.ACCESS_TYPE_ONLY_ADDRESS; case 3: - case "AccessTypeEverybody": + case "EverybodyAccess": return AccessType.ACCESS_TYPE_EVERYBODY; case -1: case "UNRECOGNIZED": @@ -44,11 +44,11 @@ exports[`createProtoEnumToJSON 1`] = ` case AccessType.ACCESS_TYPE_UNSPECIFIED: return "AccessTypeUnspecified"; case AccessType.ACCESS_TYPE_NOBODY: - return "AccessTypeNobody"; + return "NobodyAccess"; case AccessType.ACCESS_TYPE_ONLY_ADDRESS: return "AccessTypeOnlyAddress"; case AccessType.ACCESS_TYPE_EVERYBODY: - return "AccessTypeEverybody"; + return "EverybodyAccess"; case AccessType.UNRECOGNIZED: default: return "UNRECOGNIZED"; diff --git a/packages/ast/src/encoding/proto/proto.enum.custom.spec.ts b/packages/ast/src/encoding/proto/proto.enum.custom.spec.ts index 505609cf0..fb0db86d9 100644 --- a/packages/ast/src/encoding/proto/proto.enum.custom.spec.ts +++ b/packages/ast/src/encoding/proto/proto.enum.custom.spec.ts @@ -3,15 +3,23 @@ import { createProtoEnumFromJSON, createProtoEnumToJSON, } from "./enums"; -import { getNestedProto } from "@cosmology/utils"; +import { getNestedProto, convertProtoPathToNestedJSONPath as convert } from "@cosmology/utils"; import { ProtoParseContext } from "@cosmology/ast"; -import { ProtoRoot } from "@cosmology/types"; import { expectCode, getTestProtoStore } from "../../../test-utils"; +const p = convert('cosmwasm/wasm/v1/types.proto'); const store = getTestProtoStore({ + prototypes: { + patch: { + 'cosmwasm/wasm/v1/types.proto': [ + { op: 'replace', path: p + '/AccessType/valuesOptions/ACCESS_TYPE_NOBODY/(gogoproto.enumvalue_customname)', value: 'NobodyAccess' }, + { op: 'replace', path: p + '/AccessType/valuesOptions/ACCESS_TYPE_EVERYBODY/(gogoproto.enumvalue_customname)', value: 'EverybodyAccess' }, + ] + } + }, enums: { useCustomNames: true } diff --git a/packages/parser/package.json b/packages/parser/package.json index aa37be9dd..c827a3276 100644 --- a/packages/parser/package.json +++ b/packages/parser/package.json @@ -77,8 +77,9 @@ "@cosmology/types": "^1.7.0", "@cosmology/utils": "^1.7.0", "dotty": "0.1.2", + "fast-json-patch": "3.1.1", "glob": "8.0.3", "minimatch": "5.1.0", "mkdirp": "3.0.0" } -} \ No newline at end of file +} diff --git a/packages/parser/src/store.ts b/packages/parser/src/store.ts index d7f594ed0..3a4967856 100644 --- a/packages/parser/src/store.ts +++ b/packages/parser/src/store.ts @@ -17,6 +17,7 @@ import google_field_mask from './native/field_mask'; import google_struct from './native/struct'; import google_wrappers from './native/wrappers'; import { ProtoResolver } from './resolver'; +import { applyOperation, applyPatch } from 'fast-json-patch'; const GOOGLE_PROTOS = [ ['google/protobuf/any.proto', google_any], @@ -62,8 +63,8 @@ export class ProtoStore implements IProtoStore { _symbols: TraversalSymbol[] = []; _enumValueMapping: Record; + syntax: string; + valueSet: Set; }> = {}; constructor(protoDirs: string[] = [], options: TelescopeOptions = defaultTelescopeOptions) { @@ -102,11 +103,16 @@ export class ProtoStore implements IProtoStore { processProtos(contents: { absolute: string, filename: string, content: string }[]) { return contents.map(({ absolute, filename, content }) => { try { - const proto = parseProto(content, this.options.prototypes.parser); + let protoJson = parseProto(content, this.options.prototypes.parser); + if (this.options.prototypes.patch && this.options.prototypes.patch[filename]) { + const ops = this.options.prototypes.patch[filename]; + const result = applyPatch(protoJson, ops); + protoJson = result.newDocument; + } return { absolute, filename, - proto, + proto: protoJson, }; } catch (e) { console.error(`${filename} has a proto syntax error`) @@ -221,11 +227,11 @@ export class ProtoStore implements IProtoStore { this.protos = this.getProtos().map((ref: ProtoRef) => { const isHardExcluded = this.options?.prototypes?.excluded?.hardProtos && isRefExcluded(ref, { - protos: this.options?.prototypes?.excluded?.hardProtos + protos: this.options?.prototypes?.excluded?.hardProtos }) - if(isHardExcluded){ - return null; + if (isHardExcluded) { + return null; } if (!actualFiles.has(ref.filename)) { @@ -339,7 +345,7 @@ export class ProtoStore implements IProtoStore { return packages; } - setEnumValues(pkg: string, name: string, protoSyntex:string, values: number[]) { + setEnumValues(pkg: string, name: string, protoSyntex: string, values: number[]) { this._enumValueMapping[`${pkg}.${name}`] = { syntax: protoSyntex, valueSet: new Set(values) @@ -350,13 +356,13 @@ export class ProtoStore implements IProtoStore { const enumObj = this._enumValueMapping[`${pkg}.${name}`]; if (enumObj?.syntax === 'proto2') { - if(enumObj?.valueSet?.has(ENUM_PROTO2_DEFAULT)){ - return ENUM_PROTO2_DEFAULT; - } + if (enumObj?.valueSet?.has(ENUM_PROTO2_DEFAULT)) { + return ENUM_PROTO2_DEFAULT; + } } else { - if(enumObj?.valueSet?.has(ENUM_PROTO3_DEFAULT)){ - return ENUM_PROTO3_DEFAULT; - } + if (enumObj?.valueSet?.has(ENUM_PROTO3_DEFAULT)) { + return ENUM_PROTO3_DEFAULT; + } } return Math.min(...Array.from(enumObj?.valueSet ?? [])); diff --git a/packages/types/package.json b/packages/types/package.json index c6ece27cc..39f1f201a 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -57,6 +57,7 @@ "typescript": "^5.0.4" }, "dependencies": { - "case": "1.6.3" + "case": "1.6.3", + "fast-json-patch": "3.1.1" } } diff --git a/packages/types/src/telescope.ts b/packages/types/src/telescope.ts index 1552b0d2b..831a42841 100644 --- a/packages/types/src/telescope.ts +++ b/packages/types/src/telescope.ts @@ -1,6 +1,7 @@ import { TSBuilderInput } from '@cosmwasm/ts-codegen'; import { AminoExceptions, DEFAULT_AMINO_EXCEPTIONS } from "./aminos"; import { snake, camel } from 'case'; +import { Operation } from 'fast-json-patch'; export enum TelescopeLogLevel { None = 0, @@ -105,6 +106,10 @@ interface TelescopeOpts { useTelescopeGeneratedType?: boolean }; + + patch?: { + [key: string]: Operation[] + } }; enums?: { diff --git a/packages/types/types/telescope.d.ts b/packages/types/types/telescope.d.ts index 700a3cb54..c7148862c 100644 --- a/packages/types/types/telescope.d.ts +++ b/packages/types/types/telescope.d.ts @@ -1,5 +1,6 @@ import { TSBuilderInput } from '@cosmwasm/ts-codegen'; import { AminoExceptions } from "./aminos"; +import { Operation } from 'fast-json-patch'; export declare enum TelescopeLogLevel { None = 0, Info = 1, @@ -86,6 +87,9 @@ interface TelescopeOpts { updatedDuration?: boolean; useTelescopeGeneratedType?: boolean; }; + patch?: { + [key: string]: Operation[]; + }; }; enums?: { useCustomNames?: boolean; diff --git a/packages/utils/src/utils.ts b/packages/utils/src/utils.ts index 1894a8c5f..9e4f47121 100644 --- a/packages/utils/src/utils.ts +++ b/packages/utils/src/utils.ts @@ -225,4 +225,47 @@ function excludePackageFromTraversal(pkg: string, traversal: string[]) { const connectedPkg = pkg.split('.').join('_') + '_'; return traversal.join('_').replace(connectedPkg, ''); -} \ No newline at end of file +} + + +/** + * Converts a protobuf package path into a deeply nested JSON path format suitable for + * hierarchical representations. This function is ideal for adapting proto package paths + * to structured configurations or mappings in JSON. + * + * @param protoPath The protobuf package path as a string, typically formatted as + * a directory-like structure such as "cosmwasm/wasm/v1/types.proto". + * The "types.proto" or any file name with '.proto' is excluded from + * the conversion. + * + * Example input: + * - "cosmwasm/wasm/v1/types.proto" + * + * @returns A string representing the deeply nested JSON path. The function constructs this + * by starting with "/root/nested" and appending "/nested/{part}" for each segment + * of the original path. + * + * Example output for "cosmwasm/wasm/v1/types.proto": + * - "/root/nested/cosmwasm/nested/wasm/nested/v1" + * + * Usage: + * const jsonNestedPath = convertProtoPathToNestedJSONPath("cosmwasm/wasm/v1/types.proto"); + * console.log(jsonNestedPath); // Outputs: "/root/nested/cosmwasm/nested/wasm/nested/v1/nested" + */ +export function convertProtoPathToNestedJSONPath(protoPath: string): string { + // Split the path to isolate components, ignoring any '.proto' file specification + const parts = protoPath.split('/'); + if (parts[parts.length - 1].includes('.proto')) { + parts.pop(); + } + + // Start with the base path + let jsonPath = '/root/nested'; + + // Append '/nested/{part}' for each path segment + parts.forEach(part => { + jsonPath += `/${part}/nested`; + }); + + return jsonPath; +} diff --git a/packages/utils/types/utils.d.ts b/packages/utils/types/utils.d.ts index a5b10d711..17d27f7b5 100644 --- a/packages/utils/types/utils.d.ts +++ b/packages/utils/types/utils.d.ts @@ -60,3 +60,28 @@ export declare const getEnumValues: (proto: ProtoEnum) => EnumValue[]; * @returns */ export declare const getTypeNameByEnumObj: (field: any, pkg: string, traversal: string[], isNested: boolean) => any; +/** + * Converts a protobuf package path into a deeply nested JSON path format suitable for + * hierarchical representations. This function is ideal for adapting proto package paths + * to structured configurations or mappings in JSON. + * + * @param protoPath The protobuf package path as a string, typically formatted as + * a directory-like structure such as "cosmwasm/wasm/v1/types.proto". + * The "types.proto" or any file name with '.proto' is excluded from + * the conversion. + * + * Example input: + * - "cosmwasm/wasm/v1/types.proto" + * + * @returns A string representing the deeply nested JSON path. The function constructs this + * by starting with "/root/nested" and appending "/nested/{part}" for each segment + * of the original path. + * + * Example output for "cosmwasm/wasm/v1/types.proto": + * - "/root/nested/cosmwasm/nested/wasm/nested/v1" + * + * Usage: + * const jsonNestedPath = convertProtoPathToNestedJSONPath("cosmwasm/wasm/v1/types.proto"); + * console.log(jsonNestedPath); // Outputs: "/root/nested/cosmwasm/nested/wasm/nested/v1/nested" + */ +export declare function convertProtoPathToNestedJSONPath(protoPath: string): string; diff --git a/yarn.lock b/yarn.lock index d7e793252..5032535c4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4638,6 +4638,11 @@ fast-glob@^3.2.9: merge2 "^1.3.0" micromatch "^4.0.4" +fast-json-patch@3.1.1: + version "3.1.1" + resolved "https://registry.npmjs.org/fast-json-patch/-/fast-json-patch-3.1.1.tgz#85064ea1b1ebf97a3f7ad01e23f9337e72c66947" + integrity sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ== + fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0, fast-json-stable-stringify@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz" From df44cc1da4601584738f0851ee06b9a0eb3ed07f Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Wed, 12 Jun 2024 20:17:01 -0700 Subject: [PATCH 2/3] patch enums, tests --- .../proto.enum.custom.spec.ts.snap | 8 +- .../proto.json.patch.spec.ts.snap | 63 ++++++++++++++ .../encoding/proto/proto.enum.custom.spec.ts | 13 +-- .../encoding/proto/proto.json.patch.spec.ts | 82 +++++++++++++++++++ packages/parser/src/store.ts | 24 ++++-- packages/utils/src/utils.ts | 34 ++++++++ packages/utils/types/utils.d.ts | 24 ++++++ 7 files changed, 229 insertions(+), 19 deletions(-) create mode 100644 packages/ast/src/encoding/proto/__snapshots__/proto.json.patch.spec.ts.snap create mode 100644 packages/ast/src/encoding/proto/proto.json.patch.spec.ts diff --git a/packages/ast/src/encoding/proto/__snapshots__/proto.enum.custom.spec.ts.snap b/packages/ast/src/encoding/proto/__snapshots__/proto.enum.custom.spec.ts.snap index f1af1d841..7d0860a70 100644 --- a/packages/ast/src/encoding/proto/__snapshots__/proto.enum.custom.spec.ts.snap +++ b/packages/ast/src/encoding/proto/__snapshots__/proto.enum.custom.spec.ts.snap @@ -22,13 +22,13 @@ exports[`createProtoEnumFromJSON 1`] = ` case "AccessTypeUnspecified": return AccessType.ACCESS_TYPE_UNSPECIFIED; case 1: - case "NobodyAccess": + case "AccessTypeNobody": return AccessType.ACCESS_TYPE_NOBODY; case 2: case "AccessTypeOnlyAddress": return AccessType.ACCESS_TYPE_ONLY_ADDRESS; case 3: - case "EverybodyAccess": + case "AccessTypeEverybody": return AccessType.ACCESS_TYPE_EVERYBODY; case -1: case "UNRECOGNIZED": @@ -44,11 +44,11 @@ exports[`createProtoEnumToJSON 1`] = ` case AccessType.ACCESS_TYPE_UNSPECIFIED: return "AccessTypeUnspecified"; case AccessType.ACCESS_TYPE_NOBODY: - return "NobodyAccess"; + return "AccessTypeNobody"; case AccessType.ACCESS_TYPE_ONLY_ADDRESS: return "AccessTypeOnlyAddress"; case AccessType.ACCESS_TYPE_EVERYBODY: - return "EverybodyAccess"; + return "AccessTypeEverybody"; case AccessType.UNRECOGNIZED: default: return "UNRECOGNIZED"; diff --git a/packages/ast/src/encoding/proto/__snapshots__/proto.json.patch.spec.ts.snap b/packages/ast/src/encoding/proto/__snapshots__/proto.json.patch.spec.ts.snap new file mode 100644 index 000000000..dbf8148ee --- /dev/null +++ b/packages/ast/src/encoding/proto/__snapshots__/proto.json.patch.spec.ts.snap @@ -0,0 +1,63 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`createProtoEnum 1`] = ` +"/** AccessType permission types */ +export enum NullValue { + /** ACCESS_TYPE_UNSPECIFIED - AccessTypeUnspecified placeholder for empty value */ + ACCESS_TYPE_UNSPECIFIED = 0, + /** ACCESS_TYPE_NOBODY - AccessTypeNobody forbidden */ + ACCESS_TYPE_NOBODY = 1, + /** ACCESS_TYPE_ONLY_ADDRESS - AccessTypeOnlyAddress restricted to an address */ + ACCESS_TYPE_ONLY_ADDRESS = 2, + /** ACCESS_TYPE_EVERYBODY - AccessTypeEverybody unrestricted */ + ACCESS_TYPE_EVERYBODY = 3, + ACCESS_TYPE_SUPER_FUN = 4, + UNRECOGNIZED = -1, +}" +`; + +exports[`createProtoEnumFromJSON 1`] = ` +"export function accessTypeFromJSON(object: any): AccessType { + switch (object) { + case 0: + case "UnspecifiedAccess": + return AccessType.ACCESS_TYPE_UNSPECIFIED; + case 1: + case "NobodyAccess": + return AccessType.ACCESS_TYPE_NOBODY; + case 2: + case "OnlyAddressAccess": + return AccessType.ACCESS_TYPE_ONLY_ADDRESS; + case 3: + case "EverybodyAccess": + return AccessType.ACCESS_TYPE_EVERYBODY; + case 4: + case "SuperFunAccessType": + return AccessType.ACCESS_TYPE_SUPER_FUN; + case -1: + case "UNRECOGNIZED": + default: + return AccessType.UNRECOGNIZED; + } +}" +`; + +exports[`createProtoEnumToJSON 1`] = ` +"export function accessTypeToJSON(object: AccessType): string { + switch (object) { + case AccessType.ACCESS_TYPE_UNSPECIFIED: + return "UnspecifiedAccess"; + case AccessType.ACCESS_TYPE_NOBODY: + return "NobodyAccess"; + case AccessType.ACCESS_TYPE_ONLY_ADDRESS: + return "OnlyAddressAccess"; + case AccessType.ACCESS_TYPE_EVERYBODY: + return "EverybodyAccess"; + case AccessType.ACCESS_TYPE_SUPER_FUN: + return "SuperFunAccessType"; + case AccessType.UNRECOGNIZED: + default: + return "UNRECOGNIZED"; + } +}" +`; diff --git a/packages/ast/src/encoding/proto/proto.enum.custom.spec.ts b/packages/ast/src/encoding/proto/proto.enum.custom.spec.ts index fb0db86d9..74cecd412 100644 --- a/packages/ast/src/encoding/proto/proto.enum.custom.spec.ts +++ b/packages/ast/src/encoding/proto/proto.enum.custom.spec.ts @@ -3,23 +3,16 @@ import { createProtoEnumFromJSON, createProtoEnumToJSON, } from "./enums"; -import { getNestedProto, convertProtoPathToNestedJSONPath as convert } from "@cosmology/utils"; +import { + getNestedProto +} from "@cosmology/utils"; import { ProtoParseContext } from "@cosmology/ast"; import { expectCode, getTestProtoStore } from "../../../test-utils"; -const p = convert('cosmwasm/wasm/v1/types.proto'); const store = getTestProtoStore({ - prototypes: { - patch: { - 'cosmwasm/wasm/v1/types.proto': [ - { op: 'replace', path: p + '/AccessType/valuesOptions/ACCESS_TYPE_NOBODY/(gogoproto.enumvalue_customname)', value: 'NobodyAccess' }, - { op: 'replace', path: p + '/AccessType/valuesOptions/ACCESS_TYPE_EVERYBODY/(gogoproto.enumvalue_customname)', value: 'EverybodyAccess' }, - ] - } - }, enums: { useCustomNames: true } diff --git a/packages/ast/src/encoding/proto/proto.json.patch.spec.ts b/packages/ast/src/encoding/proto/proto.json.patch.spec.ts new file mode 100644 index 000000000..de96f2a14 --- /dev/null +++ b/packages/ast/src/encoding/proto/proto.json.patch.spec.ts @@ -0,0 +1,82 @@ +import { + createProtoEnum, + createProtoEnumFromJSON, + createProtoEnumToJSON, +} from "./enums"; +import { + getNestedProto, + convertProtoPathToNestedJSONPath as convert, + convertPackageNameToNestedJSONPath as convertPackage +} from "@cosmology/utils"; +import { ProtoParseContext } from "@cosmology/ast"; +import { + expectCode, + getTestProtoStore +} from "../../../test-utils"; + +const p = convert('cosmwasm/wasm/v1/types.proto'); +const cp = convertPackage('cosmwasm.wasm.v1'); +const store = getTestProtoStore({ + prototypes: { + patch: { + 'cosmwasm/wasm/v1/types.proto': [ + { op: 'replace', path: '~/AccessType/valuesOptions/ACCESS_TYPE_NOBODY/(gogoproto.enumvalue_customname)', value: 'NobodyAccess' }, + { op: 'replace', path: '@/AccessType/valuesOptions/ACCESS_TYPE_UNSPECIFIED/(gogoproto.enumvalue_customname)', value: 'UnspecifiedAccess' }, + { op: 'replace', path: p + '/AccessType/valuesOptions/ACCESS_TYPE_EVERYBODY/(gogoproto.enumvalue_customname)', value: 'EverybodyAccess' }, + { op: 'replace', path: cp + '/AccessType/valuesOptions/ACCESS_TYPE_ONLY_ADDRESS/(gogoproto.enumvalue_customname)', value: 'OnlyAddressAccess' }, + { + op: "add", + path: "/root/nested/cosmwasm/nested/wasm/nested/v1/nested/AccessType/values/ACCESS_TYPE_SUPER_FUN", + value: 4 + }, + { + op: "add", + path: "/root/nested/cosmwasm/nested/wasm/nested/v1/nested/AccessType/valuesOptions/ACCESS_TYPE_SUPER_FUN", + value: { + "(gogoproto.enumvalue_customname)": "SuperFunAccessType" + } + } + ] + } + }, + enums: { + useCustomNames: true + } +}); +store.traverseAll(); + +it("createProtoEnum", async () => { + const ref = store.findProto('cosmwasm/wasm/v1/types.proto'); + const context = new ProtoParseContext(ref, store, store.options); + expectCode( + createProtoEnum( + context, + "NullValue", + getNestedProto(ref.traversed).AccessType + ) + ); +}); + +it("createProtoEnumFromJSON", async () => { + const ref = store.findProto('cosmwasm/wasm/v1/types.proto'); + const context = new ProtoParseContext(ref, store, store.options); + expectCode( + createProtoEnumFromJSON( + context, + "AccessType", + getNestedProto(ref.traversed).AccessType + ) + ); +}); + +it("createProtoEnumToJSON", async () => { + const ref = store.findProto('cosmwasm/wasm/v1/types.proto'); + const context = new ProtoParseContext(ref, store, store.options); + expectCode( + createProtoEnumToJSON( + context, + "AccessType", + getNestedProto(ref.traversed).AccessType + ) + ); +}); diff --git a/packages/parser/src/store.ts b/packages/parser/src/store.ts index 3a4967856..7782a406b 100644 --- a/packages/parser/src/store.ts +++ b/packages/parser/src/store.ts @@ -2,7 +2,7 @@ import { sync as glob } from 'glob'; import { parse } from '@cosmology/protobufjs'; import { readFileSync } from 'fs'; import { join, resolve as pathResolve } from 'path'; -import { ALLOWED_RPC_SERVICES, ProtoDep, ProtoField, ProtoRef, ProtoServiceMethod, ProtoType, TelescopeOptions, ENUM_PROTO2_DEFAULT, ENUM_PROTO3_DEFAULT } from '@cosmology/types'; +import { ProtoDep, ProtoRef, ProtoServiceMethod, TelescopeOptions, ENUM_PROTO2_DEFAULT, ENUM_PROTO3_DEFAULT } from '@cosmology/types'; import { createTypeUrlTypeMap, getNestedProto, getPackageAndNestedFromStr, isRefIncluded, isRefExcluded } from './'; import { parseFullyTraversedProtoImports, symbolsToImportNames, traverse } from './traverse'; import { lookupAny, lookupAnyFromImports } from './lookup'; @@ -17,7 +17,8 @@ import google_field_mask from './native/field_mask'; import google_struct from './native/struct'; import google_wrappers from './native/wrappers'; import { ProtoResolver } from './resolver'; -import { applyOperation, applyPatch } from 'fast-json-patch'; +import { applyPatch } from 'fast-json-patch'; +import { convertProtoPathToNestedJSONPath, convertPackageNameToNestedJSONPath } from '@cosmology/utils'; const GOOGLE_PROTOS = [ ['google/protobuf/any.proto', google_any], @@ -105,9 +106,22 @@ export class ProtoStore implements IProtoStore { try { let protoJson = parseProto(content, this.options.prototypes.parser); if (this.options.prototypes.patch && this.options.prototypes.patch[filename]) { - const ops = this.options.prototypes.patch[filename]; - const result = applyPatch(protoJson, ops); - protoJson = result.newDocument; + const ops = this.options.prototypes.patch[filename] ?? []; + try { + const result = applyPatch(protoJson, ops.map(op => { + if (op.path.startsWith('~')) { + op.path = convertProtoPathToNestedJSONPath(filename) + op.path.substring(1); + } + if (op.path.startsWith('@')) { + op.path = convertPackageNameToNestedJSONPath(protoJson.package) + op.path.substring(1); + } + return op; + })); + protoJson = result.newDocument; + } catch (e2) { + console.error('JSON Patch error on proto: ' + filename); + } + } return { absolute, diff --git a/packages/utils/src/utils.ts b/packages/utils/src/utils.ts index 9e4f47121..e2eae6108 100644 --- a/packages/utils/src/utils.ts +++ b/packages/utils/src/utils.ts @@ -269,3 +269,37 @@ export function convertProtoPathToNestedJSONPath(protoPath: string): string { return jsonPath; } + +/** + * Converts a dot-separated package name into a deeply nested JSON path format suitable for + * hierarchical representations. This function is ideal for adapting package names like those + * used in Protobuf or similar systems to structured configurations or mappings in JSON. + * + * @param packageName The dot-separated package name as a string, typically formatted as + * a namespace structure such as "cosmwasm.wasm.v1". + * + * Example input: + * - "cosmwasm.wasm.v1" + * + * @returns A string representing the deeply nested JSON path. The function constructs this + * by starting with "/root/nested" and appending "/nested/{part}" for each segment + * of the package name. This mimics a directory path structure in a JSON tree, where + * each package level is further nested within its parent. + * + * Example output for "cosmwasm.wasm.v1": + * - "/root/nested/cosmwasm/nested/wasm/nested/v1" + * + * Usage: + * const jsonNestedPath = convertPackageNameToNestedJSONPath("cosmwasm.wasm.v1"); + * console.log(jsonNestedPath); // Outputs: "/root/nested/cosmwasm/nested/wasm/nested/v1/nested" + */ +export function convertPackageNameToNestedJSONPath(packageName: string): string { + const parts = packageName.split('.'); + + let jsonPath = '/root/nested'; + parts.forEach(part => { + jsonPath += `/${part}/nested`; + }); + + return jsonPath; +} diff --git a/packages/utils/types/utils.d.ts b/packages/utils/types/utils.d.ts index 17d27f7b5..407d8d3d8 100644 --- a/packages/utils/types/utils.d.ts +++ b/packages/utils/types/utils.d.ts @@ -85,3 +85,27 @@ export declare const getTypeNameByEnumObj: (field: any, pkg: string, traversal: * console.log(jsonNestedPath); // Outputs: "/root/nested/cosmwasm/nested/wasm/nested/v1/nested" */ export declare function convertProtoPathToNestedJSONPath(protoPath: string): string; +/** + * Converts a dot-separated package name into a deeply nested JSON path format suitable for + * hierarchical representations. This function is ideal for adapting package names like those + * used in Protobuf or similar systems to structured configurations or mappings in JSON. + * + * @param packageName The dot-separated package name as a string, typically formatted as + * a namespace structure such as "cosmwasm.wasm.v1". + * + * Example input: + * - "cosmwasm.wasm.v1" + * + * @returns A string representing the deeply nested JSON path. The function constructs this + * by starting with "/root/nested" and appending "/nested/{part}" for each segment + * of the package name. This mimics a directory path structure in a JSON tree, where + * each package level is further nested within its parent. + * + * Example output for "cosmwasm.wasm.v1": + * - "/root/nested/cosmwasm/nested/wasm/nested/v1" + * + * Usage: + * const jsonNestedPath = convertPackageNameToNestedJSONPath("cosmwasm.wasm.v1"); + * console.log(jsonNestedPath); // Outputs: "/root/nested/cosmwasm/nested/wasm/nested/v1/nested" + */ +export declare function convertPackageNameToNestedJSONPath(packageName: string): string; From f7619218b537238835c501fb7bb7ac5d2e3cb672 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Wed, 12 Jun 2024 20:33:54 -0700 Subject: [PATCH 3/3] cleanup JSON patch --- README.md | 55 +++++++++++++++++++ .../encoding/proto/proto.json.patch.spec.ts | 6 +- packages/parser/src/store.ts | 5 +- packages/telescope/README.md | 55 +++++++++++++++++++ packages/utils/src/utils.ts | 43 --------------- 5 files changed, 113 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 7a4c8dccd..b372d9326 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ The following blockchain libraries (generated by Telescope) are available via np - [Amino Encoding](#amino-encoding) - [Prototypes Options](#prototypes-options) - [Prototypes Methods](#prototypes-methods) + - [Enums Methods](#enums-options) - [LCD Client Options](#lcd-client-options) - [RPC Client Options](#rpc-client-options) - [Stargate Client Options](#stargate-client-options) @@ -79,6 +80,7 @@ The following blockchain libraries (generated by Telescope) are available via np - [RPC Client Classes](#rpc-client-classes) - [Instant RPC Methods](#instant-rpc-methods) - [Manually registering types](#manually-registering-types) + - [JSON Patch Protos](#json-patch-protos) - [CosmWasm](#cosmwasm) - [Dependencies](#dependencies) - [Troubleshooting](#troubleshooting) @@ -314,6 +316,7 @@ telescope({ | `prototypes.addTypeUrlToObjects` | add typeUrl field to generated Decoders | `true` | | `prototypes.enableRegistryLoader` | generate Registry loader to *.registry.ts files | `true` | | `prototypes.enableMessageComposer` | generate MessageComposer to *.registry.ts files | `true` | +| `prototypes.patch` | An object mapping filenames to an array of `Operation` to be applied as patches to proto files during generation. See [JSON Patch Protos](#json-patch-protos) | `undefined`| ### Prototypes Methods @@ -327,6 +330,13 @@ telescope({ | `prototypes.methods.fromSDK` | boolean to enable `fromSDK` method on proto objects | `false` | | `prototypes.methods.toSDK` | boolean to enable `toSDK` method on proto objects | `false` | +### Enums Options + +| option | description | defaults | +| ------------------------------------- | --------------------------------------------------------------- | ---------- | +| `enums.useCustomNames` | Enables the usage of custom names for enums if specified through proto options or annotations, allowing for more descriptive or project-specific naming conventions. | `false` | + + ### LCD Client Options | option | description | defaults | @@ -986,6 +996,51 @@ export const getCustomSigningClient = async ({ rpcEndpoint, signer }: { rpcEndpo }; ``` +## JSON Patch Protos + +The `prototypes.patch` configuration within the options object allows for dynamic modifications to protobuf definitions during code generation. This feature is designed to apply specific changes to proto files without altering the original source. By using JSON Patch operations such as `replace` and `add`, developers can customize the generated output to better fit project requirements when upstream SDK PRs are lagging or not in production. + +Patches are specified as arrays of `Operation`s, where each operation is defined by: +- `op`: The operation type (`add` or `replace`). +- `path`: The JSON path to the target field, optionally prefixed with `@` to denote paths derived automatically from the package name, simplifying navigation within the proto file's structure. +- `value`: The new value to be set at the target location specified by the path. + +Here is how these patches can be defined within the prototypes configuration: + +```json +{ + "prototypes": { + "patch": { + "cosmwasm/wasm/v1/types.proto": [ + { + "op": "replace", + "path": "@/AccessType/valuesOptions/ACCESS_TYPE_UNSPECIFIED/(gogoproto.enumvalue_customname)", + "value": "UnspecifiedAccess" + }, + { + "op": "replace", + "path": "@/AccessType/valuesOptions/ACCESS_TYPE_NOBODY/(gogoproto.enumvalue_customname)", + "value": "NobodyAccess" + }, + { + "op": "add", + "path": "@/AccessType/values/ACCESS_TYPE_SUPER_FUN", + "value": 4 + }, + { + "op": "add", + "path": "@/AccessType/valuesOptions/ACCESS_TYPE_SUPER_FUN", + "value": { + "(gogoproto.enumvalue_customname)": "SuperFunAccessType" + } + } + ] + } + } +} +``` + + ## CosmWasm Generate TypeScript SDKs for your CosmWasm smart contracts by using the `cosmwasm` option on `TelescopeOptions`. The `cosmwasm` option is actually a direct reference to the `TSBuilderInput` object, for the most up-to-date documentation, visit [@cosmwasm/ts-codegen](https://github.com/CosmWasm/ts-codegen). diff --git a/packages/ast/src/encoding/proto/proto.json.patch.spec.ts b/packages/ast/src/encoding/proto/proto.json.patch.spec.ts index de96f2a14..05ce9f639 100644 --- a/packages/ast/src/encoding/proto/proto.json.patch.spec.ts +++ b/packages/ast/src/encoding/proto/proto.json.patch.spec.ts @@ -5,7 +5,6 @@ import { } from "./enums"; import { getNestedProto, - convertProtoPathToNestedJSONPath as convert, convertPackageNameToNestedJSONPath as convertPackage } from "@cosmology/utils"; import { ProtoParseContext } from "@cosmology/ast"; @@ -14,15 +13,14 @@ import { getTestProtoStore } from "../../../test-utils"; -const p = convert('cosmwasm/wasm/v1/types.proto'); const cp = convertPackage('cosmwasm.wasm.v1'); const store = getTestProtoStore({ prototypes: { patch: { 'cosmwasm/wasm/v1/types.proto': [ - { op: 'replace', path: '~/AccessType/valuesOptions/ACCESS_TYPE_NOBODY/(gogoproto.enumvalue_customname)', value: 'NobodyAccess' }, + { op: 'replace', path: '@/AccessType/valuesOptions/ACCESS_TYPE_NOBODY/(gogoproto.enumvalue_customname)', value: 'NobodyAccess' }, { op: 'replace', path: '@/AccessType/valuesOptions/ACCESS_TYPE_UNSPECIFIED/(gogoproto.enumvalue_customname)', value: 'UnspecifiedAccess' }, - { op: 'replace', path: p + '/AccessType/valuesOptions/ACCESS_TYPE_EVERYBODY/(gogoproto.enumvalue_customname)', value: 'EverybodyAccess' }, + { op: 'replace', path: cp + '/AccessType/valuesOptions/ACCESS_TYPE_EVERYBODY/(gogoproto.enumvalue_customname)', value: 'EverybodyAccess' }, { op: 'replace', path: cp + '/AccessType/valuesOptions/ACCESS_TYPE_ONLY_ADDRESS/(gogoproto.enumvalue_customname)', value: 'OnlyAddressAccess' }, { op: "add", diff --git a/packages/parser/src/store.ts b/packages/parser/src/store.ts index 7782a406b..ff8e3388c 100644 --- a/packages/parser/src/store.ts +++ b/packages/parser/src/store.ts @@ -18,7 +18,7 @@ import google_struct from './native/struct'; import google_wrappers from './native/wrappers'; import { ProtoResolver } from './resolver'; import { applyPatch } from 'fast-json-patch'; -import { convertProtoPathToNestedJSONPath, convertPackageNameToNestedJSONPath } from '@cosmology/utils'; +import { convertPackageNameToNestedJSONPath } from '@cosmology/utils'; const GOOGLE_PROTOS = [ ['google/protobuf/any.proto', google_any], @@ -109,9 +109,6 @@ export class ProtoStore implements IProtoStore { const ops = this.options.prototypes.patch[filename] ?? []; try { const result = applyPatch(protoJson, ops.map(op => { - if (op.path.startsWith('~')) { - op.path = convertProtoPathToNestedJSONPath(filename) + op.path.substring(1); - } if (op.path.startsWith('@')) { op.path = convertPackageNameToNestedJSONPath(protoJson.package) + op.path.substring(1); } diff --git a/packages/telescope/README.md b/packages/telescope/README.md index 7a4c8dccd..b372d9326 100644 --- a/packages/telescope/README.md +++ b/packages/telescope/README.md @@ -47,6 +47,7 @@ The following blockchain libraries (generated by Telescope) are available via np - [Amino Encoding](#amino-encoding) - [Prototypes Options](#prototypes-options) - [Prototypes Methods](#prototypes-methods) + - [Enums Methods](#enums-options) - [LCD Client Options](#lcd-client-options) - [RPC Client Options](#rpc-client-options) - [Stargate Client Options](#stargate-client-options) @@ -79,6 +80,7 @@ The following blockchain libraries (generated by Telescope) are available via np - [RPC Client Classes](#rpc-client-classes) - [Instant RPC Methods](#instant-rpc-methods) - [Manually registering types](#manually-registering-types) + - [JSON Patch Protos](#json-patch-protos) - [CosmWasm](#cosmwasm) - [Dependencies](#dependencies) - [Troubleshooting](#troubleshooting) @@ -314,6 +316,7 @@ telescope({ | `prototypes.addTypeUrlToObjects` | add typeUrl field to generated Decoders | `true` | | `prototypes.enableRegistryLoader` | generate Registry loader to *.registry.ts files | `true` | | `prototypes.enableMessageComposer` | generate MessageComposer to *.registry.ts files | `true` | +| `prototypes.patch` | An object mapping filenames to an array of `Operation` to be applied as patches to proto files during generation. See [JSON Patch Protos](#json-patch-protos) | `undefined`| ### Prototypes Methods @@ -327,6 +330,13 @@ telescope({ | `prototypes.methods.fromSDK` | boolean to enable `fromSDK` method on proto objects | `false` | | `prototypes.methods.toSDK` | boolean to enable `toSDK` method on proto objects | `false` | +### Enums Options + +| option | description | defaults | +| ------------------------------------- | --------------------------------------------------------------- | ---------- | +| `enums.useCustomNames` | Enables the usage of custom names for enums if specified through proto options or annotations, allowing for more descriptive or project-specific naming conventions. | `false` | + + ### LCD Client Options | option | description | defaults | @@ -986,6 +996,51 @@ export const getCustomSigningClient = async ({ rpcEndpoint, signer }: { rpcEndpo }; ``` +## JSON Patch Protos + +The `prototypes.patch` configuration within the options object allows for dynamic modifications to protobuf definitions during code generation. This feature is designed to apply specific changes to proto files without altering the original source. By using JSON Patch operations such as `replace` and `add`, developers can customize the generated output to better fit project requirements when upstream SDK PRs are lagging or not in production. + +Patches are specified as arrays of `Operation`s, where each operation is defined by: +- `op`: The operation type (`add` or `replace`). +- `path`: The JSON path to the target field, optionally prefixed with `@` to denote paths derived automatically from the package name, simplifying navigation within the proto file's structure. +- `value`: The new value to be set at the target location specified by the path. + +Here is how these patches can be defined within the prototypes configuration: + +```json +{ + "prototypes": { + "patch": { + "cosmwasm/wasm/v1/types.proto": [ + { + "op": "replace", + "path": "@/AccessType/valuesOptions/ACCESS_TYPE_UNSPECIFIED/(gogoproto.enumvalue_customname)", + "value": "UnspecifiedAccess" + }, + { + "op": "replace", + "path": "@/AccessType/valuesOptions/ACCESS_TYPE_NOBODY/(gogoproto.enumvalue_customname)", + "value": "NobodyAccess" + }, + { + "op": "add", + "path": "@/AccessType/values/ACCESS_TYPE_SUPER_FUN", + "value": 4 + }, + { + "op": "add", + "path": "@/AccessType/valuesOptions/ACCESS_TYPE_SUPER_FUN", + "value": { + "(gogoproto.enumvalue_customname)": "SuperFunAccessType" + } + } + ] + } + } +} +``` + + ## CosmWasm Generate TypeScript SDKs for your CosmWasm smart contracts by using the `cosmwasm` option on `TelescopeOptions`. The `cosmwasm` option is actually a direct reference to the `TSBuilderInput` object, for the most up-to-date documentation, visit [@cosmwasm/ts-codegen](https://github.com/CosmWasm/ts-codegen). diff --git a/packages/utils/src/utils.ts b/packages/utils/src/utils.ts index e2eae6108..e5948271a 100644 --- a/packages/utils/src/utils.ts +++ b/packages/utils/src/utils.ts @@ -227,49 +227,6 @@ function excludePackageFromTraversal(pkg: string, traversal: string[]) { return traversal.join('_').replace(connectedPkg, ''); } - -/** - * Converts a protobuf package path into a deeply nested JSON path format suitable for - * hierarchical representations. This function is ideal for adapting proto package paths - * to structured configurations or mappings in JSON. - * - * @param protoPath The protobuf package path as a string, typically formatted as - * a directory-like structure such as "cosmwasm/wasm/v1/types.proto". - * The "types.proto" or any file name with '.proto' is excluded from - * the conversion. - * - * Example input: - * - "cosmwasm/wasm/v1/types.proto" - * - * @returns A string representing the deeply nested JSON path. The function constructs this - * by starting with "/root/nested" and appending "/nested/{part}" for each segment - * of the original path. - * - * Example output for "cosmwasm/wasm/v1/types.proto": - * - "/root/nested/cosmwasm/nested/wasm/nested/v1" - * - * Usage: - * const jsonNestedPath = convertProtoPathToNestedJSONPath("cosmwasm/wasm/v1/types.proto"); - * console.log(jsonNestedPath); // Outputs: "/root/nested/cosmwasm/nested/wasm/nested/v1/nested" - */ -export function convertProtoPathToNestedJSONPath(protoPath: string): string { - // Split the path to isolate components, ignoring any '.proto' file specification - const parts = protoPath.split('/'); - if (parts[parts.length - 1].includes('.proto')) { - parts.pop(); - } - - // Start with the base path - let jsonPath = '/root/nested'; - - // Append '/nested/{part}' for each path segment - parts.forEach(part => { - jsonPath += `/${part}/nested`; - }); - - return jsonPath; -} - /** * Converts a dot-separated package name into a deeply nested JSON path format suitable for * hierarchical representations. This function is ideal for adapting package names like those