Merge pull request #616 from cosmology-tech/proto-download

multi repos when download proto

packages/telescope/.gitignore

@@ -45,4 +45,6 @@ package-lock.json
 yarn.lock
 
 # others
-.DS_Store
+.DS_Store
+
+git-modules/**

packages/telescope/src/commands/download.ts

@@ -1,6 +1,6 @@
 import * as shell from "shelljs";
 import { readFileSync } from "fs";
-import { clone, extractProto } from "../protod/recursive";
+import { clone, cloneAll, extractProto } from "../protod/recursive";
 import { removeFolder } from "../protod/utils";
 import { DownloadOptions } from "src/protod/types";
 import deepmerge from "deepmerge";
@@ -32,27 +32,28 @@ export default async (argv: { [key: string]: string | string[] }) => {
     configJson = {};
   }
 
-  const gitRepo = Array.isArray(argv["git-repo"])
-    ? argv["git-repo"][0]?.split("/")
-    : argv["git-repo"]?.split?.("/");
+  const gitRepos = Array.isArray(argv["git-repo"])
+    ? argv["git-repo"]
+    : [argv["git-repo"]];
 
-  if (gitRepo) {
-    // shell.echo("missing `git-repo` argument.\n");
-    // return shell.exit(1);
-    const [owner, repo] = gitRepo;
+  if (argv["git-repo"] && gitRepos && gitRepos.length > 0) {
+    configJson.repos = gitRepos.map((gitRepo) => {
+      const [owner, repo, branch] = gitRepo.split("/");
 
-    if (!owner || !repo) {
-      shell.echo("wrong `git-repo` format (i.e. <owner>/<repository>).\n");
-      return shell.exit(1);
-    }
+      if (!owner || !repo) {
+        shell.echo(
+          "wrong `git-repo` format (i.e. <owner>/<repository> or <owner>/<repository>/<branch>).\n"
+        );
+        return shell.exit(1);
+      }
 
-    configJson.owner = owner;
-    configJson.repo = repo;
+      return { owner, repo, branch };
+    });
   }
 
-  if (!configJson.owner || !configJson.repo) {
+  if (!configJson.repos || configJson.repos.length === 0) {
     shell.echo(
-      "missing `git-repo` argument, you can set through `--git-repo` argument or set `owner` and `repo` in config file.\n"
+      "missing `git-repo` argument, you can set through `--git-repo` argument or set repos field in config file.\n"
     );
     return shell.exit(1);
   }
@@ -90,23 +91,16 @@ export default async (argv: { [key: string]: string | string[] }) => {
     configJson.outDir = "proto";
   }
 
-  let branch = Array.isArray(argv["branch"])
-    ? argv["branch"][0]
-    : argv["branch"];
-
-  if (branch) {
-    configJson.branch = branch;
-  }
-
   let sshOpt = argv["ssh"];
 
   configJson.ssh = configJson.ssh || !!sshOpt;
 
-  removeFolder("git-modules");
-  const result = await clone({
-    owner: configJson.owner,
-    repo: configJson.repo,
-    branch: configJson.branch,
+  if (configJson.deleteTempRepoDir) {
+    removeFolder(configJson.tempRepoDir);
+  }
+
+  const result = await cloneAll({
+    repos: configJson.repos,
     gitModulesDir: "./git-modules",
     ssh: configJson.ssh,
     protoDirMapping: configJson.protoDirMapping,

packages/telescope/src/protod/git-repo.ts

@@ -1,4 +1,12 @@
-import { exec, getMainBranchName, makeDir, removeFolder } from "./utils";
+import {
+  exec,
+  getMainBranchName,
+  isPathExist,
+  makeDir,
+  removeFolder,
+} from "./utils";
+
+const branchCache: Record<string, string> = {};
 
 export class GitRepo {
   constructor(
@@ -19,21 +27,41 @@ export class GitRepo {
     return `git@github.com:${this.fullName}.git`;
   }
 
-  get mainBranchName() {
-    return getMainBranchName(this.ssh ? this.sshUrl : this.httpsUrl);
+  async getMainBranchName() {
+    const url = this.ssh ? this.sshUrl : this.httpsUrl;
+
+    if (branchCache[url]) {
+      return branchCache[url];
+    }
+
+    branchCache[url] = await getMainBranchName(url);
+
+    return branchCache[url];
   }
 
-  clone(branch: string, depth: number = 1, outDir: string = "./git-modules") {
+  clone(
+    branch: string,
+    depth: number = 1,
+    outDir: string = "./git-modules",
+    isOverride?: boolean
+  ) {
     const dir = `${outDir}/${this.fullName}/${branch}`;
     try {
-      console.log(`Cloning to ${dir}`);
+      if (!isOverride && isPathExist(dir)) {
+        console.log(`Folder ${dir} already exists, skip cloning`);
+        return dir;
+      } else {
+        console.log(`Cloning to ${dir}`);
+      }
+
       removeFolder(dir);
       makeDir(dir);
       exec(
         `git clone ${
           this.ssh ? this.sshUrl : this.httpsUrl
         } --depth ${depth} --branch ${branch} --single-branch ${dir}`
       );
+      console.log(`Cloned ${this.fullName}/${branch} to ${dir}`);
       return dir;
     } catch (error) {
       if ((error as Error).message.startsWith("Cloning into")) {

packages/telescope/src/protod/recursive.spec.ts

@@ -2,20 +2,25 @@ import { TelescopeInput } from "..";
 import { TelescopeBuilder } from "../builder";
 import { TelescopeOptions } from "@cosmology/types";
 
-import { clone, extractProto } from "./recursive";
-import { findAllProtoFiles, removeFolder } from "./utils";
+import { clone, cloneAll, extractProto } from "./recursive";
+import { removeFolder } from "./utils";
 
 import { join } from "path";
 
-import { sync as globSync } from "glob";
-import { file } from "@babel/types";
-
 const GIT_MOUDLES = "git-modules";
 let gitModules = join(
   __dirname,
   "../../../../__fixtures__/misc/proto-download",
   GIT_MOUDLES
 );
+
+const GIT_MOUDLES_ALL = "git-modules-all";
+let gitModulesAll = join(
+  __dirname,
+  "../../../../__fixtures__/misc/proto-download",
+  GIT_MOUDLES_ALL
+);
+
 const PROTO_FOLDER = "proto";
 let protoFolder = join(
   __dirname,
@@ -98,8 +103,31 @@ const input: TelescopeInput = {
   options,
 };
 
+const testSwitch = false;
+
+describe("Test `cloneAll`", () => {
+  (testSwitch ? it : it.skip)("should download `injective`, `ibc-go` to `./git-modules`", async () => {
+    removeFolder(gitModulesAll);
+    const result = await cloneAll({
+      repos: [
+        { owner: "cosmos", repo: "ibc-go" },
+        { owner: "injectivelabs", repo: "sdk-go" },
+      ],
+      gitModulesDir: gitModulesAll,
+      protoDirMapping: {
+        "gogo/protobuf/master": ".",
+        "googleapis/googleapis/master": ".",
+        "protocolbuffers/protobuf/main": "src",
+      },
+      ssh: true,
+    });
+
+    console.log(result);
+  });
+});
+
 describe("Test `clone`", () => {
-  it("should download `cosmos-sdk`, `googleapis`, `protobuf` to `./git-modules` and extract `cosmos/bank/v1beta1/tx.proto` to `./proto`", async () => {
+  (testSwitch ? it : it.skip)("should download `cosmos-sdk`, `googleapis`, `protobuf` to `./git-modules` and extract `cosmos/bank/v1beta1/tx.proto` to `./proto`", async () => {
     removeFolder(gitModules);
     const result = await clone({
       owner: "cosmos",
@@ -128,7 +156,7 @@ describe("Test `clone`", () => {
     }
   });
 
-  it("should successfully generate code in `./codegen`", async () => {
+  (testSwitch ? it : it.skip)("should successfully generate code in `./codegen`", async () => {
     removeFolder(codegenFolder);
     const telescope = new TelescopeBuilder(input);
     await telescope.build();

packages/telescope/src/protod/recursive.ts

@@ -1,36 +1,64 @@
 import { getAllBufDeps } from "./bufbuild";
 import { GitRepo } from "./git-repo";
-import { CloneOptions, GitInfo, ProtoCopyOptions } from "./types";
-import { join, dirname, resolve } from "path";
 import {
-  findAllProtoFiles,
-  getCorrespondingGit,
-  getMainBranchName,
-  isPathExist,
-  makeDir,
-  parseProtoFile,
-} from "./utils";
+  CloneOptions,
+  GitInfo,
+  ProtoCopyOptions,
+  CloneAllOptions,
+} from "./types";
+import { join, dirname, resolve } from "path";
+import { getCorrespondingGit, makeDir, parseProtoFile } from "./utils";
 import fs from "fs";
 import { sync as globSync } from "glob";
 
+export async function cloneAll({
+  repos,
+  gitModulesDir,
+  protoDirMapping,
+  ssh,
+}: CloneAllOptions) {
+  let clonedResult: Record<string, GitInfo> = {};
+
+  for (const { owner, repo, branch } of repos) {
+    const cloned = await clone({
+      owner,
+      repo,
+      branch,
+      gitModulesDir,
+      protoDirMapping,
+      ssh,
+      cloned: clonedResult,
+    });
+
+    for (const [key, value] of Object.entries(cloned)) {
+      clonedResult[key] = value;
+    }
+  }
+
+  return clonedResult;
+}
+
 export async function clone({
   owner,
   repo,
   branch,
   gitModulesDir: outDir,
   protoDirMapping,
   ssh,
+  cloned,
 }: CloneOptions) {
-  let clonedResult: Record<string, GitInfo> = {};
+  let clonedResult: Record<string, GitInfo> = cloned ?? {};
+
   const gitRepo = new GitRepo(owner, repo, ssh);
-  const gitBranch = branch ?? (await gitRepo.mainBranchName);
-  const outPath = `${outDir}/${owner}/${repo}`;
-  if (isPathExist(outPath)) {
-    console.warn(`Folder ${outPath} already exists, skip cloning`);
-    return;
+  const gitBranch = branch ?? (await gitRepo.getMainBranchName());
+  const resultKey = `${owner}/${repo}/${gitBranch}`;
+
+  if (clonedResult[resultKey]) {
+    console.log(`Skip cloning ${resultKey}`);
+    return clonedResult;
   }
+
   const gitDir = gitRepo.clone(gitBranch, 1, outDir);
-  console.log(`Cloned ${owner}/${repo}/${gitBranch} to ${gitDir}`);
   const protoDir =
     protoDirMapping?.[`${owner}/${repo}/${gitBranch}`] ?? "proto";
   clonedResult[`${owner}/${repo}/${gitBranch}`] = {
@@ -47,18 +75,18 @@ export async function clone({
       await Promise.all(
         gitRepos.map(async (gitRepo) => {
           const gitRepoObj = new GitRepo(gitRepo.owner, gitRepo.repo, ssh);
-          const branch = await gitRepoObj.mainBranchName;
+          const branch = await gitRepoObj.getMainBranchName();
           const depsClonedResult = await clone({
             ...gitRepo,
             gitModulesDir: outDir,
             branch,
             protoDirMapping,
             ssh,
+            cloned: clonedResult,
           });
-          clonedResult = {
-            ...clonedResult,
-            ...depsClonedResult,
-          };
+          for (const [key, value] of Object.entries(depsClonedResult)) {
+            clonedResult[key] = value;
+          }
         })
       );
     })

packages/telescope/src/protod/test-data/.protod.config.json

@@ -1,17 +1,42 @@
 {
-      "owner": "cosmos",
-      "repo": "cosmos-sdk",
-      "branch": "release/v0.50.x",
+      "repos": [
+        { "owner": "cosmos", "repo": "cosmos-sdk", "branch": "release/v0.50.x" },
+        { "owner": "cosmos", "repo": "ibc-go" },
+        { "owner": "injectivelabs", "repo": "sdk-go" }
+      ],
       "protoDirMapping": {
         "gogo/protobuf/master": ".",
         "googleapis/googleapis/master": ".",
         "protocolbuffers/protobuf/main": "src"
       },
-      "outDir": "testproto",
+      "outDir": "../../__fixtures__/misc/proto-download/testproto",
       "ssh": true,
+      "tempRepoDir": "git-modules",
       "targets": [
+        "cosmos/auth/v1beta1/auth.proto",
+        "cosmos/vesting/v1beta1/vesting.proto",
+        "cosmos/crypto/secp256k1/keys.proto",
+        "cosmos/crypto/secp256r1/keys.proto",
+        "cosmos/crypto/ed25519/keys.proto",
+        "cosmos/authz/v1beta1/tx.proto",
         "cosmos/bank/v1beta1/tx.proto",
-        "cosmos/gov/**/*.proto",
-        "cosmos/authz/**/*.proto"
+        "cosmos/distribution/v1beta1/tx.proto",
+        "cosmos/feegrant/v1beta1/tx.proto",
+        "cosmos/gov/v1beta1/tx.proto",
+        "cosmos/gov/v1/tx.proto",
+        "cosmos/group/v1/tx.proto",
+        "cosmos/staking/v1beta1/tx.proto",
+        "cosmos/vesting/v1beta1/tx.proto",
+        "cosmos/auth/v1beta1/query.proto",
+        "cosmos/bank/v1beta1/query.proto",
+        "cosmos/staking/v1beta1/query.proto",
+        "cosmos/gov/v1beta1/query.proto",
+        "cosmwasm/wasm/v1/tx.proto",
+        "cosmwasm/wasm/v1/query.proto",
+        "ibc/applications/transfer/v1/tx.proto",
+        "ibc/core/channel/v1/tx.proto",
+        "ibc/core/client/v1/tx.proto",
+        "ibc/core/connection/v1/tx.proto",
+        "injective/types/v1beta1/account.proto"
       ]
 }