diff --git a/.github/workflows/automated-release.yml b/.github/workflows/automated-release.yml index c804e5653..4736b3c70 100644 --- a/.github/workflows/automated-release.yml +++ b/.github/workflows/automated-release.yml @@ -42,7 +42,7 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ap-southeast-1 - name: upload core - run: bash -e ./hack/release/upload_assets.sh ${{ secrets.GITHUB_TOKEN }} ${{ env.TAG }} ${{ env.GOOS }} ${{ env.GOARCH }} + run: bash -e ./hack/release/upload_dtm_core.sh ${{ secrets.GITHUB_TOKEN }} ${{ env.TAG }} ${{ env.GOOS }} ${{ env.GOARCH }} - name: upload plugin run: aws s3 cp ~/.devstream/plugins/ s3://download.devstream.io/${{ env.TAG }}/ --recursive --acl public-read @@ -77,8 +77,10 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ap-southeast-1 - - name: upload - run: bash -e ./hack/release/upload_assets.sh ${{ secrets.GITHUB_TOKEN }} ${{ env.TAG }} ${{ env.GOOS }} ${{ env.GOARCH }} - - name: upload plugin + - name: update dtm version on s3 + run: ./hack/release/update_dtm_version_on_s3.sh ${{ env.TAG }} + - name: upload dtm core to github release and s3 + run: bash -e ./hack/release/upload_dtm_core.sh ${{ secrets.GITHUB_TOKEN }} ${{ env.TAG }} ${{ env.GOOS }} ${{ env.GOARCH }} + - name: upload plugins to s3 run: aws s3 cp ~/.devstream/plugins/ s3://download.devstream.io/${{ env.TAG }}/ --recursive --acl public-read diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 4950a5730..485ea5331 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -84,7 +84,7 @@ jobs: - name: test 1 - clean run: ./dtm delete -f e2e-tools.yaml -y - name: test 2 - apply (apps) - run: ./dtm apply -f e2e-apps.yaml -y + run: ./dtm apply --debug -f e2e-apps.yaml -y - name: test 2 - apply (apps) again run: ./dtm apply -f e2e-apps.yaml -y - name: test 2 - check if pod is ready diff --git a/.github/workflows/update-download-script.yml b/.github/workflows/update-download-script.yml new file mode 100644 index 000000000..7e57f3e13 --- /dev/null +++ b/.github/workflows/update-download-script.yml @@ -0,0 +1,23 @@ +name: update download.sh + +on: + push: + branches: [ main ] + paths: + - hack/install.sh + workflow_dispatch: + +jobs: + update-download-script: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ap-southeast-1 + - name: Update download.sh + run: aws s3 cp hack/install/download.sh s3://download.devstream.io/download.sh --acl public-read diff --git a/README.md b/README.md index ed4215bfa..d20283043 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ I am happy to tell you that we have, and we are constantly adding more possible so you are more than welcome to tell us what combinations you expect. - [GitOps Toolchain](https://docs.devstream.io/en/latest/best-practices/gitops/) +- [GitLab, Jenkins and Harbor On Premise Toolchain (Chinese only for now)](https://docs.devstream.io/en/latest/best-practices/gitlab-jenkins-harbor.zh/) ## Supported DevOps Tools diff --git a/README_zh.md b/README_zh.md index dfeebe53d..b5a416925 100644 --- a/README_zh.md +++ b/README_zh.md @@ -92,6 +92,7 @@ DevStream支持许多工具的管理。你可以灵活地结合一些工具来 我们非常欢迎你告诉我们你期望的组合。 - [GitOps工具链](https://docs.devstream.io/en/latest/best-practices/gitops.zh/) +- [用 DevStream 搭建 GitLab + Jenkins + Harbor 工具链，管理 Java Spring Boot 项目开发生命周期全流程](https://docs.devstream.io/en/latest/best-practices/gitlab-jenkins-harbor.zh/) ## 支持的DevOps工具 diff --git a/docs/best-practices/air-gapped-deployment.zh.md b/docs/best-practices/air-gapped-deployment.zh.md index e3ff1d890..95c8913e3 100644 --- a/docs/best-practices/air-gapped-deployment.zh.md +++ b/docs/best-practices/air-gapped-deployment.zh.md @@ -1,12 +1,10 @@ # 离线部署 -// TODO(daniel-hutao): to update according helm-installer plugin added. - 本文将和你介绍如何在离线环境中使用 DevStream。 !!! info "提醒" - 当前并不是所有 DevStream 插件都支持离线部署，具体支持情况请查看具体的插件文档确认。 + 当前并不是所有 DevStream 插件都支持离线部署，比如 `githubactions-golang` 插件执行过程中需要与 GitHub 交互，就不可能支持离线使用。 本文以 Jenkins + Harbor 两个插件为例演示离线部署流程。 ## 1、下载 dtm 和 DevStream Plugins @@ -15,7 +13,13 @@ ### 1.1、下载 dtm -你可以在 [Release](https://github.com/devstream-io/devstream/releases/) 页面找到当前最新版本 `dtm`，然后点击下载。 +你可以直接运行如下命令完成 dtm 的下载： + +```shell +sh -c "$(curl -fsSL https://download.devstream.io/download.sh)" +``` + +如果你希望有更多的掌控感，你可以选择手动下载，直接访问 DevStream 的 [Release](https://github.com/devstream-io/devstream/releases/) 页面找到当前最新版本 `dtm`，然后点击下载。 需要注意的是当前 `dtm` 提供了多个版本，分别是： 1. Darwin/arm64 @@ -33,7 +37,7 @@ chmod +x dtm ```shell $ dtm version -0.10.0 +0.10.2 ``` ### 1.2、下载 plugins @@ -46,7 +50,7 @@ dtm init --download-only --plugins="jenkins, harbor" -d=plugins ```shell $ ls -harbor-linux-amd64_0.9.1.md5 harbor-linux-amd64_0.9.1.so jenkins-linux-amd64_0.9.1.md5 jenkins-linux-amd64_0.9.1.so +harbor-linux-amd64_0.10.2.md5 harbor-linux-amd64_0.10.2.so jenkins-linux-amd64_0.10.2.md5 jenkins-linux-amd64_0.10.2.so ``` ## 2、下载镜像 @@ -122,230 +126,26 @@ harbor-1.10.0.tgz jenkins-4.2.5.tgz 这时候部署 Jenkins 和 Harbor 所需要的"物料"就准备好了。你可以准备配置文件，开始部署 Jenkins 和 Harbor。 -### 4.1、编写插件配置（plugin config） - -你可以这样编写两个插件的配置： - -=== "jenkins 插件" - - jenkins 插件的参考配置： - - ```yaml title="Plugin Config with jenkins" - --- - # plugin config - tools: - - name: jenkins - instanceID: default - dependsOn: [ ] - options: - chart: - chartPath: "./jenkins-4.2.5.tgz" - valuesYaml: | - serviceAccount: - create: true - name: jenkins - persistence: - storageClass: "" - controller: - image: [[ imageRepo ]]/devstreamdev/jenkins - tag: 2.361.1-jdk11-dtm-0.1 - imagePullPolicy: "IfNotPresent" - sidecars: - configAutoReload: - image: [[ imageRepo ]]/kiwigrid/k8s-sidecar:1.15.0 - adminUser: "admin" - adminPassword: "changeme" - ingress: - enabled: true - hostName: [[ jenkinsURL ]] - # Enable HTML parsing using OWASP Markup Formatter Plugin (antisamy-markup-formatter), useful with ghprb plugin. - enableRawHtmlMarkupFormatter: true - # Jenkins Configuraction as Code, refer to https://plugins.jenkins.io/configuration-as-code/ for more details - # notice: All configuration files that are discovered MUST be supplementary. They cannot overwrite each other's configuration values. This creates a conflict and raises a ConfiguratorException. - JCasC: - defaultConfig: true - agent: - image: [[ imageRepo ]]/jenkins/inbound-agent - tag: 4.11.2-4 - backup: - image: - repository: [[ imageRepo ]]/maorfr/kube-tasks - tag: 0.2.0 - ``` - - 关于 jenkins 插件的详细文档可以看[ jenkins 插件文档](../plugins/helm-installer/helm-installer.zh.md)。 - -=== "harbor 插件" - - harbor 插件的参考配置： - - ```yaml title="Plugin Config with harbor" - --- - # plugin config - tools: - - name: harbor - instanceID: default - dependsOn: [ ] - options: - chart: - chartPath: "./harbor-1.10.0.tgz" - valuesYaml: | - externalURL: http://[[ harborURL ]] - expose: - type: ingress - tls: - enabled: false - ingress: - hosts: - core: [[ harborURL ]] - nginx: - image: - repository: [[ imageRepo ]]/goharbor/nginx-photon - tag: v2.5.3 - portal: - image: - repository: [[ imageRepo ]]/goharbor/harbor-portal - tag: v2.5.3 - core: - image: - repository: [[ imageRepo ]]/goharbor/harbor-core - tag: v2.5.3 - jobservice: - image: - repository: [[ imageRepo ]]/goharbor/harbor-jobservice - tag: v2.5.3 - registry: - registry: - image: - repository: [[ imageRepo ]]/goharbor/registry-photon - tag: v2.5.3 - controller: - image: - repository: [[ imageRepo ]]/goharbor/harbor-registryctl - tag: v2.5.3 - chartmuseum: - enabled: false - image: - repository: [[ imageRepo ]]/goharbor/chartmuseum-photon - tag: v2.5.3 - trivy: - enabled: false - image: - repository: [[ imageRepo ]]/goharbor/trivy-adapter-photon - tag: v2.5.3 - notary: - enabled: false - server: - image: - repository: [[ imageRepo ]]/goharbor/notary-server-photon - tag: v2.5.3 - signer: - image: - repository: [[ imageRepo ]]/goharbor/notary-signer-photon - tag: v2.5.3 - database: - internal: - image: - repository: [[ imageRepo ]]/goharbor/harbor-db - tag: v2.5.3 - redis: - internal: - image: - repository: [[ imageRepo ]]/goharbor/redis-photon - tag: v2.5.3 - exporter: - image: - repository: [[ imageRepo ]]/goharbor/harbor-exporter - tag: v2.5.3 - persistence: - persistentVolumeClaim: - registry: - storageClass: "" - accessMode: ReadWriteOnce - size: 5Gi - jobservice: - storageClass: "" - accessMode: ReadWriteOnce - size: 1Gi - database: - storageClass: "" - accessMode: ReadWriteOnce - size: 1Gi - redis: - storageClass: "" - accessMode: ReadWriteOnce - size: 1Gi - ``` - 关于 harbor 插件的详细文档可以看[ harbor 插件文档](../plugins/helm-installer/helm-installer.zh.md)。 - -### 4.2、编写主配置和变量配置 - -除了插件配置外，我们还需要准备"主配置"（core config）和"变量配置"（variable config）： - -最简单的 core config 是以 **local** 作为 Backend，也就是将状态保存到本地文件； -企业 On premise 环境部署可以选择使用 **k8s** Backend 将状态通过 `kube-apiserver` 存入 etcd，两种方式配置分别如下： - -=== "Core Config with 'local' Backend" - - ```yaml title="local Backend" - varFile: "" # If not empty, use the specified external variables config file - toolFile: "" # If not empty, use the specified external tools config file - state: # state config, backend can be local, s3 or k8s - backend: local - options: - stateFile: devstream.state - ``` - -=== "Core Config with 'k8s' Backend" - - ```yaml title="k8s Backend" - varFile: "" # If not empty, use the specified external variables config file - toolFile: "" # If not empty, use the specified external tools config file - state: # state config, backend can be local, s3 or k8s - backend: k8s - options: - namespace: devstream # optional, default is "devstream", will be created if not exists - configmap: state # optional, default is "state", will be created if not exists - ``` - -同时前面插件配置里还引用了一个变量，你可以这样编写对应的变量配置： - -```yaml ---- -# variable config -imageRepo: harbor.devstream.io -jenkinsURL: jenkins.example.com -harborURL: harbor.example.com -``` - -### 4.3、组装完整的配置文件 - -这时候"主配置"、"变量配置"和"插件配置"都有了，你可以选择将其分别保存到三个不同的文件中，在"主配置"里通过`varFile`和`toolFile`去引用另外两个配置文件， -也可以选择直接将三个配置写入一个文件，就像这样： -```yaml title="config.yaml" ---- -# core config -varFile: "" # If not empty, use the specified external variables config file -toolFile: "" # If not empty, use the specified external tools config file -state: # state config, backend can be local, s3 or k8s - backend: k8s - options: - namespace: devstream # optional, default is "devstream", will be created if not exists - configmap: state # optional, default is "state", will be created if not exists +```yaml title="DevStream Config" +config: + state: + backend: local + options: + stateFile: devstream.state ---- -# variable config -imageRepo: harbor.devstream.io -jenkinsURL: jenkins.example.com -harborURL: harbor.example.com +vars: + imageRepo: harbor.devstream.io + jenkinsHostname: jenkins.example.com + harborHostname: harbor.example.com + harborURL: http://harbor.example.com + jenkinsAdminUser: admin + jenkinsAdminPassword: changeme ---- -# plugin config tools: - name: jenkins instanceID: default - dependsOn: [ ] + dependsOn: [] options: chart: chartPath: "./jenkins-4.2.5.tgz" @@ -362,15 +162,12 @@ tools: sidecars: configAutoReload: image: [[ imageRepo ]]/kiwigrid/k8s-sidecar:1.15.0 - adminUser: "admin" - adminPassword: "changeme" + adminUser: [[ jenkinsAdminUser ]] + adminPassword: [[ jenkinsAdminPassword ]] ingress: enabled: true - hostName: [[ jenkinsURL ]] - # Enable HTML parsing using OWASP Markup Formatter Plugin (antisamy-markup-formatter), useful with ghprb plugin. + hostName: [[ jenkinsHostname ]] enableRawHtmlMarkupFormatter: true - # Jenkins Configuraction as Code, refer to https://plugins.jenkins.io/configuration-as-code/ for more details - # notice: All configuration files that are discovered MUST be supplementary. They cannot overwrite each other's configuration values. This creates a conflict and raises a ConfiguratorException. JCasC: defaultConfig: true agent: @@ -382,19 +179,19 @@ tools: tag: 0.2.0 - name: harbor instanceID: default - dependsOn: [ ] + dependsOn: [] options: chart: chartPath: "./harbor-1.10.0.tgz" valuesYaml: | - externalURL: http://[[ harborURL ]] + externalURL: [[ harborURL ]] expose: type: ingress tls: enabled: false ingress: hosts: - core: [[ harborURL ]] + core: [[ harborHostname ]] nginx: image: repository: [[ imageRepo ]]/goharbor/nginx-photon @@ -474,6 +271,10 @@ tools: size: 1Gi ``` +关于 jenkins 插件的详细文档可以查看[ jenkins 插件文档](../plugins/helm-installer/helm-installer.zh.md)。 + +关于 harbor 插件的详细文档可以查看[ harbor 插件文档](../plugins/helm-installer/helm-installer.zh.md)。 + ## 5、开始部署 现在你可以通过如下命令开始部署 Jenkins 和 Harbor 了： @@ -489,41 +290,41 @@ dtm apply -f config.yaml -y 如果成功执行，你可以看到类似如下日志： ```text -2022-09-27 09:08:05 ℹ [INFO] Apply started. -2022-09-27 09:08:05 ℹ [INFO] Using dir <./plugins> to store plugins. -2022-09-27 09:08:05 ℹ [INFO] Using configmap backend. Namespace: devstream, ConfigMap name: state. -2022-09-27 09:08:05 ℹ [INFO] Tool (jenkins/default) found in config but doesn't exist in the state, will be created. -2022-09-27 09:08:05 ℹ [INFO] Tool (harbor/default) found in config but doesn't exist in the state, will be created. -2022-09-27 09:08:05 ℹ [INFO] Start executing the plan. -2022-09-27 09:08:05 ℹ [INFO] Changes count: 2. -2022-09-27 09:08:05 ℹ [INFO] -------------------- [ Processing progress: 1/2. ] -------------------- -2022-09-27 09:08:05 ℹ [INFO] Processing: (jenkins/default) -> Create ... -2022-09-27 09:08:05 ℹ [INFO] The chart package already exists in the cache directory. -2022-09-27 09:08:05 ℹ [INFO] Creating or updating helm chart ... -2022/09/27 09:08:06 creating 13 resource(s) -2022/09/27 09:08:06 beginning wait for 13 resources with timeout of 5m0s -2022/09/27 09:08:06 StatefulSet is not ready: jenkins/jenkins. 0 out of 1 expected pods are ready +2022-11-27 09:08:05 ℹ [INFO] Apply started. +2022-11-27 09:08:05 ℹ [INFO] Using dir <./plugins> to store plugins. +2022-11-27 09:08:05 ℹ [INFO] Using configmap backend. Namespace: devstream, ConfigMap name: state. +2022-11-27 09:08:05 ℹ [INFO] Tool (jenkins/default) found in config but doesn't exist in the state, will be created. +2022-11-27 09:08:05 ℹ [INFO] Tool (harbor/default) found in config but doesn't exist in the state, will be created. +2022-11-27 09:08:05 ℹ [INFO] Start executing the plan. +2022-11-27 09:08:05 ℹ [INFO] Changes count: 2. +2022-11-27 09:08:05 ℹ [INFO] -------------------- [ Processing progress: 1/2. ] -------------------- +2022-11-27 09:08:05 ℹ [INFO] Processing: (jenkins/default) -> Create ... +2022-11-27 09:08:05 ℹ [INFO] The chart package already exists in the cache directory. +2022-11-27 09:08:05 ℹ [INFO] Creating or updating helm chart ... +2022/11/27 09:08:06 creating 13 resource(s) +2022/11/27 09:08:06 beginning wait for 13 resources with timeout of 5m0s +2022/11/27 09:08:06 StatefulSet is not ready: jenkins/jenkins. 0 out of 1 expected pods are ready ... -2022/09/27 09:08:46 StatefulSet is not ready: jenkins/jenkins. 0 out of 1 expected pods are ready -2022/09/27 09:08:48 release installed successfully: jenkins/jenkins-4.2.5 -2022-09-27 09:08:48 ✔ [SUCCESS] Tool (jenkins/default) Create done. -2022-09-27 09:08:48 ℹ [INFO] -------------------- [ Processing progress: 2/2. ] -------------------- -2022-09-27 09:08:48 ℹ [INFO] Processing: (harbor/default) -> Create ... -2022-09-27 09:08:48 ℹ [INFO] The chart package already exists in the cache directory. -2022-09-27 09:08:48 ℹ [INFO] Creating or updating helm chart ... -2022/09/27 09:08:49 creating 28 resource(s) -2022/09/27 09:08:49 beginning wait for 28 resources with timeout of 10m0s -2022/09/27 09:08:49 Deployment is not ready: harbor/harbor-core. 0 out of 1 expected pods are ready +2022/11/27 09:08:46 StatefulSet is not ready: jenkins/jenkins. 0 out of 1 expected pods are ready +2022/11/27 09:08:48 release installed successfully: jenkins/jenkins-4.2.5 +2022-11-27 09:08:48 ✔ [SUCCESS] Tool (jenkins/default) Create done. +2022-11-27 09:08:48 ℹ [INFO] -------------------- [ Processing progress: 2/2. ] -------------------- +2022-11-27 09:08:48 ℹ [INFO] Processing: (harbor/default) -> Create ... +2022-11-27 09:08:48 ℹ [INFO] The chart package already exists in the cache directory. +2022-11-27 09:08:48 ℹ [INFO] Creating or updating helm chart ... +2022/11/27 09:08:49 creating 28 resource(s) +2022/11/27 09:08:49 beginning wait for 28 resources with timeout of 10m0s +2022/11/27 09:08:49 Deployment is not ready: harbor/harbor-core. 0 out of 1 expected pods are ready ... -2022/09/27 09:09:17 Deployment is not ready: harbor/harbor-core. 0 out of 1 expected pods are ready -2022/09/27 09:09:19 Deployment is not ready: harbor/harbor-jobservice. 0 out of 1 expected pods are ready +2022/11/27 09:09:17 Deployment is not ready: harbor/harbor-core. 0 out of 1 expected pods are ready +2022/11/27 09:09:19 Deployment is not ready: harbor/harbor-jobservice. 0 out of 1 expected pods are ready ... -2022/09/27 09:09:37 Deployment is not ready: harbor/harbor-jobservice. 0 out of 1 expected pods are ready -2022/09/27 09:09:39 release installed successfully: harbor/harbor-1.10.0 -2022-09-27 09:09:39 ✔ [SUCCESS] Tool (harbor/default) Create done. -2022-09-27 09:09:39 ℹ [INFO] -------------------- [ Processing done. ] -------------------- -2022-09-27 09:09:39 ✔ [SUCCESS] All plugins applied successfully. -2022-09-27 09:09:39 ✔ [SUCCESS] Apply finished. +2022/11/27 09:09:37 Deployment is not ready: harbor/harbor-jobservice. 0 out of 1 expected pods are ready +2022/11/27 09:09:39 release installed successfully: harbor/harbor-1.10.0 +2022-11-27 09:09:39 ✔ [SUCCESS] Tool (harbor/default) Create done. +2022-11-27 09:09:39 ℹ [INFO] -------------------- [ Processing done. ] -------------------- +2022-11-27 09:09:39 ✔ [SUCCESS] All plugins applied successfully. +2022-11-27 09:09:39 ✔ [SUCCESS] Apply finished. ``` ## 6、验证部署结果 @@ -543,7 +344,7 @@ jenkins jenkins nginx jenkins.example.com 10.18.7.29 80 !!! Tip "说明" 演示环境是一台云主机，其内部 IP 是 10.18.7.29，公网 IP 是 11.22.33.44， - 所以下面很多地方也可以成内部 IP，但是为了方便，下文一律以公网 IP 为例。 + 所以下面很多地方也可以配置成内部 IP，但是为了更好理解，下文一律以公网 IP 为例。 前面 Jenkins 和 Harbor 三个工具的配置文件里我们都设置了域名，你可以直接将这些域名与 IP 的映射关系配置到 DNS 服务器里。 diff --git a/docs/best-practices/gitlab-jenkins-harbor.zh.md b/docs/best-practices/gitlab-jenkins-harbor.zh.md index c391b26ac..4c7a2d938 100644 --- a/docs/best-practices/gitlab-jenkins-harbor.zh.md +++ b/docs/best-practices/gitlab-jenkins-harbor.zh.md @@ -1,4 +1,4 @@ -# 用 DevStream 搭建 GitLab + Jenkins + Harbor 工具链，管理 Java Sprint Boot 项目开发生命周期全流程 +# 用 DevStream 搭建 GitLab + Jenkins + Harbor 工具链，管理 Java Spring Boot 项目开发生命周期全流程 ## 1、概述 @@ -359,7 +359,7 @@ Harbor 的 admin 用户初始登录密码是 `Harbor12345`，你可以尝试用 ## 3、开始应用 apps -本节你将继续使用 DevStream apps 管理能力实现一个 Java Sprint Boot 项目的脚手架创建和 CI 流程配置等过程。 +本节你将继续使用 DevStream apps 管理能力实现一个 Java Spring Boot 项目的脚手架创建和 CI 流程配置等过程。 ### 3.1、准备 apps 配置文件（config-apps.yaml） diff --git a/docs/best-practices/gitops.md b/docs/best-practices/gitops.md index d0e6f1e6a..2872d06aa 100644 --- a/docs/best-practices/gitops.md +++ b/docs/best-practices/gitops.md @@ -1,179 +1,362 @@ -# GitOps Toolchain +# GitOps -// TODO(daniel-hutao): to update according helm-installer plugin added. +## 0 Goal -If you are interested in watching a video demo, see the youtube video below: +In this tutorial, we will try to use DevStream to achieve the following: + +1. create a new repository for a Python web application written with [Flask](https://flask.palletsprojects.com/en/2.2.x/); +2. setup basic CI pipelines for the repo we created with GitHub Actions; +3. install [Argo CD](https://argo-cd.readthedocs.io/en/stable/) for GitOps in _an existing Kubernetes cluster_; +4. create an Argo CD application that deploys the web application generated in step 1. + +> Note: +> +> in step 3, Argo CD is installed in an existing Kubernetes cluster. Setting up infrastructure like a Kubernetes cluster is something DevStream chooses not to do. +> +> If you want to follow this tutorial and give it a try yourself but don't know how to get a Kubernetes cluster up and running locally, maybe the following blogs (also from DevStream) would help: +> +> - [Creating a Local Kubernetes Cluster from the Ground Up - a Tutorial of "Kind"](https://blog.devstream.io/posts/creating-a-local-k8s-cluster-with-kind/) +> - [Getting Started with minikube](https://blog.devstream.io/posts/getting-started-with-minikube/) + +--- + +## 1 TL;DR: See It in Action + +If you prefer to see this GitOps doc in action, check out the video demo below: -[https://www.youtube.com/watch?v=q7TK3vFr1kg](https://www.youtube.com/watch?v=q7TK3vFr1kg) +It's from an older version of DevStream with slightly different configuration files, but you get the gist. We will update the video with the latest version of DevStream soon; bear with us for a little while. -For Chinese readers, watch this one instead: +For Chinese readers, watch this one: +However, if you are like us, who prefer to do things hands-on and get their hands dirty, read on, follow the steps, and have a go yourself! +--- -[https://www.bilibili.com/video/BV1W3411P7oW/](https://www.bilibili.com/video/BV1W3411P7oW/) +## 2 Overview -## Plugins needed +DevStream will use the following plugins to achieve the goal described in [Section 0](#0-goal): 1. [repo-scaffolding](../plugins/repo-scaffolding.md) -2. [jira-github](../plugins/jira-github-integ.md) -3. [githubactions-golang](../plugins/githubactions-golang.md) -4. [argocd](../plugins/helm-installer/helm-installer.md) -5. [argocdapp](../plugins/argocdapp.md) +2. [githubactions-golang](../plugins/githubactions-golang.md) +3. [argocd](../plugins/helm-installer/helm-installer.md) (TODO: this doc needs to be updated) +4. [argocdapp](../plugins/argocdapp.md) -The dependencies of these plugins are(`a -> b` means for `a depends on b`): +However, you do not have to worry about these plugins because DevStream will manage them automatically for you. -- `jira-github` -> `repo-scaffolding` -- `githubactions-golang` -> `repo-scaffolding` -- `argocdapp` -> `argocd`, `githubactions-golang` and `repo-scaffolding` +--- -Note: These dependencies are optional; you can use dependency to make sure a certain tool is installed before another. We should use dependency according to the actual usage situation. +## 3 Getting Started: Download DevStream (`dtm`) -## 1 Download DevStream (`dtm`) +Create a temporary working directory for this tutorial: -In your working directory, run: +```bash +mkdir test +cd test/ +``` + +Then, under the newly created directory, execute the following command: ```shell -sh -c "$(curl -fsSL https://raw.githubusercontent.com/devstream-io/devstream/main/hack/install/download.sh)" +sh -c "$(curl -fsSL https://download.devstream.io/download.sh) ``` -This will download the corresponding `dtm` binary to your working directory according to your OS and chip architecture, and grant the binary execution permission. +This script checks your system and downloads the corresponding `dtm` binary. Then the binary will be granted execution permission. -> Optional: you can then move `dtm` to a place which is in your PATH. For example: `mv dtm /usr/local/bin/`. +If you do an `ls`, you can see the binary `dtm` has been downloaded: -_For more details on how to install, see [install dtm](../install.md)._ +```bash +tiexin@mbp ~/work/devstream-io/test $ ls +dtm +``` -## 2 Prepare the Config File +And as a test, try to execute it, and you will get similar output to the following: -Run the following command to generate a template configuration file for gitops `gitops.yaml`. +```bash +tiexin@mbp ~/work/devstream-io/test $ ./dtm +DevStream is an open-source DevOps toolchain manager + +###### ##### +# # ###### # # # # ##### ##### ###### ## # # +# # # # # # # # # # # # ## ## +# # ##### # # ##### # # # ##### # # # ## # +# # # # # # # ##### # ###### # # +# # # # # # # # # # # # # # # +###### ###### ## ##### # # # ###### # # # # + +Usage: + dtm [command] + +Available Commands: + apply Create or update DevOps tools according to DevStream configuration file + completion Generate the autocompletion script for dtm for the specified shell + delete Delete DevOps tools according to DevStream configuration file + destroy Destroy DevOps tools deployment according to DevStream configuration file & state file + develop Develop is used for develop a new plugin + help Help about any command + init Download needed plugins according to the config file + list This command only supports listing plugins now + show Show is used to print plugins' configuration templates or status + upgrade Upgrade dtm to the latest release version + verify Verify DevOps tools according to DevStream config file and state + version Print the version number of DevStream + +Flags: + --debug debug level log + -h, --help help for dtm + +Use "dtm [command] --help" for more information about a command. +``` -```shell -./dtm show config -t gitops > gitops.yaml +> Optional: you can move `dtm` to a directory which is in your $PATH. For example: `mv dtm /usr/local/bin/`. This will allow you to run `dtm` directly without having to prefix it with the dot and slash (`./dtm`). +> +> For more methods on how to install DevStream, see [install dtm](../install.md). + +--- + +## 4 Config File + +Create a file named `config.yaml` and paste the following content into it: + +```yaml +config: + state: + backend: local + options: + stateFile: devstream.state +vars: + githubUser: IronCore864 + dockerUser: ironcore864 + app: helloworld + +tools: +- name: repo-scaffolding + instanceID: myapp + options: + destinationRepo: + owner: [[ githubUser ]] + repo: [[ app ]] + branch: main + repoType: github + sourceRepo: + org: devstream-io + repo: dtm-scaffolding-python + repoType: github + vars: + imageRepo: [[ dockerUser ]]/[[ app ]] +- name: githubactions-python + instanceID: default + dependsOn: [ repo-scaffolding.myapp ] + options: + owner: [[ githubUser ]] + repo: [[ app ]] + language: + name: python + branch: main + docker: + registry: + type: dockerhub + username: [[ dockerUser ]] + repository: [[ app ]] +- name: helm-installer + instanceID: argocd +- name: argocdapp + instanceID: default + dependsOn: [ "helm-installer.argocd", "githubactions-python.default" ] + options: + app: + name: [[ app ]] + namespace: argocd + destination: + server: https://kubernetes.default.svc + namespace: default + source: + valuefile: values.yaml + path: helm/[[ app ]] + repoURL: ${{repo-scaffolding.myapp.outputs.repoURL}} ``` -Then modify the `gitops.yaml` file accordingly. +Then modify the `vars` section in the `config.yaml` file accordingly. Please update the values for `githubUser` and `dockerUser` to your real users. -For me I can set these variables like: +In the example above, I set these vars like the following: -| Variable | Example | Note | -| ------------------------------ | ----------------- | ------------------------------------------------------------ | -| defaultBranch | main | The branch name you want to use | -| githubUsername | daniel-hutao | It should be case-sensitive here; strictly use your GitHub username | -| repoName | go-webapp | As long as it doesn't exist in your GitHub account and the name is legal | -| dockerhubUsername | exploitht | It should be case-sensitive here; strictly use your DockerHub username | -| jiraID | merico | This is a domain name prefix like merico in https://merico.atlassian.net | -| jiraProjectKey | DT | A descriptive prefix for your project’s issue keys to recognize work from this project | -| jiraUserEmail | tao.hu@merico.dev | The email you use to log in to Jira | -| argocdNameSpace | argocd | The namespace used by ArgoCD | -| argocdDeployTimeout | 10m | How long does ArgoCD deployment timeout | +| Variable | Example | Note | +|------------|-------------|-----------------------------------------------------------| +| githubUser | IronCore864 | case-sensitive, use your GitHub username strictly here | +| dockerUser | ironcore864 | case-sensitive, use your DockerHub username strictly here | +## 5 Environment Variables -These plugins require some environment variables to work, so let's set them: +The following environment variables are required for this to work: ```bash export GITHUB_TOKEN="YOUR_GITHUB_TOKEN_HERE" -export JIRA_API_TOKEN="YOUR_JIRA_API_TOKEN_HERE" export DOCKERHUB_TOKEN="YOUR_DOCKERHUB_TOKEN_HERE" ``` -If you don't know how to create these three tokens, check out: +> Note: +> +> if you don't know how to create these two tokens, check out: +> +> - GITHUB_TOKEN: [Manage API tokens for your Atlassian account](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) +> - DOCKERHUB_TOKEN: [Manage access tokens](https://docs.docker.com/docker-hub/access-tokens/) -- GITHUB_TOKEN: [Manage API tokens for your Atlassian account](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) -- JIRA_API_TOKEN: [Creating a personal access token](https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/) -- DOCKERHUB_TOKEN: [Manage access tokens](https://docs.docker.com/docker-hub/access-tokens/) +--- -## 3 Initialize +## 6 Init Run: ```bash -dtm init -f gitops.yaml +./dtm init +``` + +This downloads the required plugins, according to the config file, automatically. + +You'll get some outputs similar to the following: + +```bash +2022-12-05 17:46:01 ℹ [INFO] Using dir to store plugins. +2022-12-05 17:46:01 ℹ [INFO] -------------------- [ repo-scaffolding-darwin-arm64_0.10.1 ] -------------------- +... (omitted) +... (omitted) +2022-12-05 17:46:51 ✔ [SUCCESS] Initialize finished. ``` -## 4 Apply +--- + +## 7 Apply Run: ```bash -dtm apply -f gitops.yaml +./dtm apply -y ``` -and confirm to continue, then you should see similar output to: +You will see similar outputs as the following: ``` -... -2022-03-11 13:36:11 ✔ [SUCCESS] All plugins applied successfully. -2022-03-11 13:36:11 ✔ [SUCCESS] Apply finished. +2022-12-05 17:49:49 ℹ [INFO] Apply started. +2022-12-05 17:49:49 ℹ [INFO] Using local backend. State file: devstream.state. +2022-12-05 17:49:49 ℹ [INFO] Tool (repo-scaffolding/myapp) found in config but doesn't exist in the state, will be created. +2022-12-05 17:49:49 ℹ [INFO] Tool (helm-installer/argocd) found in config but doesn't exist in the state, will be created. +2022-12-05 17:49:49 ℹ [INFO] Tool (githubactions-python/default) found in config but doesn't exist in the state, will be created. +2022-12-05 17:49:49 ℹ [INFO] Tool (argocdapp/default) found in config but doesn't exist in the state, will be created. +2022-12-05 17:49:49 ℹ [INFO] Start executing the plan. +2022-12-05 17:49:49 ℹ [INFO] Changes count: 4. +... (omitted) +... (omitted) +2022-12-05 17:51:51 ℹ [INFO] -------------------- [ Processing progress: 4/4. ] -------------------- +2022-12-05 17:51:51 ℹ [INFO] Processing: (argocdapp/default) -> Create ... +2022-12-05 17:51:52 ℹ [INFO] application.argoproj.io/helloworld created +2022-12-05 17:51:52 ✔ [SUCCESS] Tool (argocdapp/default) Create done. +2022-12-05 17:51:52 ℹ [INFO] -------------------- [ Processing done. ] -------------------- +2022-12-05 17:51:52 ✔ [SUCCESS] All plugins applied successfully. +2022-12-05 17:51:52 ✔ [SUCCESS] Apply finished. ``` -## 5 Check the Results +--- + +## 8 Check the Results Let's continue to look at the results of the `apply` command. -### 5.1 Repository Scaffolding +### 8.1 Repository -- The repository scaffolding we got looks like this: +The repository is created automatically by DevStream with scaffolding code: ![](gitops/a.png) -### 5.2 Jira-Github Integration +### 8.2 CI Pipelines with GitHub Actions -- How do Jira and Github integrate? Let's create a new issue: +GitHub Actions pipelines are created and executed: ![](gitops/b.png) -- The issue will be renamed automatically like this: +### 8.3 Argo CD Installation -![](gitops/c.png) +Argo CD is installed in your Kubernetes cluster: -- We can find this auto-synced `Story` in Jira: - -![](gitops/d.png) - -- If we continue to leave a comment on this issue: - -![](gitops/d1.png) - -- The comment will also be automatically synced to Jira: - -![](gitops/e.png) - -### 5.3 GitHub Actions CI for Golang - -- What does CI do here? - -![](gitops/f.png) - -- The CI processes also build an image, and this image is automatically pushed to DockerHub: +```bash +tiexin@mbp ~/work/devstream-io/test $ kubectl get namespaces +NAME STATUS AGE +argocd Active 5m42s +default Active 6m28s +kube-node-lease Active 6m29s +kube-public Active 6m29s +kube-system Active 6m29s +local-path-storage Active 6m25s +tiexin@mbp ~/work/devstream-io/test $ kubectl get pods -n argocd +NAME READY STATUS RESTARTS AGE +argocd-application-controller-0 1/1 Running 0 5m43s +argocd-applicationset-controller-66687659f-dsrtd 1/1 Running 0 5m43s +argocd-dex-server-6944757486-clshl 1/1 Running 0 5m43s +argocd-notifications-controller-7944945879-b9878 1/1 Running 0 5m43s +argocd-redis-7887bbdbbb-xzppj 1/1 Running 0 5m43s +argocd-repo-server-d4f5cc7cb-8gj24 1/1 Running 0 5m43s +argocd-server-5bb75c4bd9-g948r 1/1 Running 0 5m43s +``` -![](gitops/g.png) +### 8.4 Continuous Deployment with Argo CD -### 5.4 ArgoCD Deployment +The CI pipelines build a Docker image and push it into Dockerhub, and an Argo CD application created by DevStream deploys the app already: -- Of course, the ArgoCD must have been installed as expected. +```bash +tiexin@mbp ~/work/devstream-io/test $ kubectl get deployment -n default +NAME READY UP-TO-DATE AVAILABLE AGE +helloworld 1/1 1 1 5m16s +tiexin@mbp ~/work/devstream-io/test $ kubectl get pods -n default +NAME READY STATUS RESTARTS AGE +helloworld-69b5586b94-wjwd9 1/1 Running 0 5m18s +tiexin@mbp ~/work/devstream-io/test $ kubectl get services -n default +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +helloworld ClusterIP 10.96.73.97 80/TCP 5m27s +kubernetes ClusterIP 10.96.0.1 443/TCP 8m2s +``` -![](gitops/h.png) +If you do a port-forwarding: -### 5.5 ArgoCD Application Deployment +```bash +kubectl port-forward -n default svc/helloworld 8080:80 +``` -- Our code has just been built into an image, at this time the image is automatically deployed to our k8s as a Pod: +And accesses `localhost:8080` in your browser, you can see the deployed app return a "Hello, World!" to you. Hooray! -![](gitops/i.png) +--- -## 6 Clean Up +## 9 Clean Up Run: ```bash -dtm destroy -f gitops.yaml +./dtm delete -y ``` -and you should see similar output: +And you will get similar outputs to the following: +```bash +2022-12-05 17:59:25 ℹ [INFO] Delete started. +2022-12-05 17:59:26 ℹ [INFO] Using local backend. State file: devstream.state. +2022-12-05 17:59:26 ℹ [INFO] Tool (argocdapp/default) will be deleted. +2022-12-05 17:59:26 ℹ [INFO] Tool (githubactions-python/default) will be deleted. +2022-12-05 17:59:26 ℹ [INFO] Tool (repo-scaffolding/myapp) will be deleted. +2022-12-05 17:59:26 ℹ [INFO] Tool (helm-installer/argocd) will be deleted. +2022-12-05 17:59:26 ℹ [INFO] Start executing the plan. +2022-12-05 17:59:26 ℹ [INFO] Changes count: 4. +... (omitted) +... (omitted) +2022-12-05 17:59:35 ℹ [INFO] -------------------- [ Processing done. ] -------------------- +2022-12-05 17:59:35 ✔ [SUCCESS] All plugins deleted successfully. +2022-12-05 17:59:35 ✔ [SUCCESS] Delete finished. ``` -2022-03-11 13:39:11 ✔ [SUCCESS] All plugins destroyed successfully. -2022-03-11 13:39:11 ✔ [SUCCESS] Destroy finished. + +Then you can delete what we created: + +```bash +cd ../ +rm -rf test/ +rm -rf ~/.devstream/ ``` diff --git a/docs/best-practices/gitops.zh.md b/docs/best-practices/gitops.zh.md index 3973c1c55..34ab94399 100644 --- a/docs/best-practices/gitops.zh.md +++ b/docs/best-practices/gitops.zh.md @@ -1,171 +1,359 @@ -# GitOps 工具链 +# GitOps -// TODO(daniel-hutao): to update according helm-installer plugin added. + +## 0 目标 +在本教程中，我们将尝试通过 DevStream 来实现以下目标： + +1. 创建一个 Python Web 应用程序仓库，基于 [Flask](https://flask.palletsprojects.com/en/2.2.x/) 框架； +2. 使用 GitHub Actions 为我们创建的仓库设置基本的 CI 流水线； +3. 在 _一个已有的 Kubernetes 集群_ 中安装 [Argo CD](https://argo-cd.readthedocs.io/en/stable/) 以实现 GitOps； +4. 创建一个 Argo CD 应用程序，用于部署第 1 步中生成的 Web 应用程序。 + +> 注意： +> +> 在第 3 步中，Argo CD 安装在一个已有的 Kubernetes 集群中。DevStream 不配置基础设施，例如 Kubernetes 集群。 +> +> 如果你想跟着本教程自己尝试一下，但不知道如何在本地启动和运行 Kubernetes 集群，下面的博客（也来自 DevStream）可能会有所帮助： +> +> - [用 Kind 从零开始快速搭建本地 Kubernetes 测试环节](https://blog.devstream.io/posts/%E7%94%A8kind%E9%83%A8%E7%BD%B2k8s%E7%8E%AF%E5%A2%83/) +> - [minikube结合阿里云镜像搭建本地开发测试环境](https://blog.devstream.io/posts/%E4%BD%BF%E7%94%A8minikube%E5%92%8C%E9%98%BF%E9%87%8C%E4%BA%91%E9%95%9C%E5%83%8F%E5%AE%89%E8%A3%85k8s/) + +--- + +## 1 太长不看版：Demo 演示 + +如果你想看看 GitOps 的实际运行效果，可以看看下面的视频演示： -请参考视频demo来快速熟悉用DevStream来实施GitOps工具链的部署和整合： -- YouTube -- 哔哩哔哩 - -## 所需插件 +这个演示录制于 DevStream 的旧版本，配置文件略有不同，但你还是能从中领略 DevStream GitOps 流程的魅力与要点。我们会尽快更新 DevStream 的最新版本的视频演示，敬请期待~ -1. [repo-scaffolding](../plugins/repo-scaffolding.zh.md) -2. [jira-github](../plugins/jira-github-integ.zh.md) -3. [githubactions-golang](../plugins/githubactions-golang.zh.md) -4. [argocd](../plugins/helm-installer/helm-installer.zh.md) -5. [argocdapp](../plugins/argocdapp.zh.md) +对于中文读者，可以看看这个： -这些插件的依赖关系如下（`a -> b`意味着`a依赖b`）： + -- `jira-github` -> `repo-scaffolding` -- `githubactions-golang` -> `repo-scaffolding` -- `argocdapp` -> `argocd`, `githubactions-golang` 和 `repo-scaffolding` +光看完全不尽兴吧？跟着后面的步骤一起试一试吧！ -**注意**：依赖并不是必须指定的，我们可以用依赖确保某个工具可以先于另外一个工具安装。我们应该根据实际的使用场景来使用`dependsOn`。 +--- -## 1 下载 DevStream（`dtm`） +## 2 概览 -进入你的工作目录，运行： +DevStream 将使用下面的插件来实现[第 0 节](#0-目标)中描述的目标： -```shell -sh -c "$(curl -fsSL https://raw.githubusercontent.com/devstream-io/devstream/main/hack/install/download.sh)" -``` +1. [repo-scaffolding](../plugins/repo-scaffolding.md) +2. [githubactions-golang](../plugins/githubactions-golang.md) +3. [argocd](../plugins/helm-installer/helm-installer.md) (TODO: 此文档待更新) +4. [argocdapp](../plugins/argocdapp.md) -这个命令会根据你的操作系统和芯片架构下载对应的 `dtm` 二进制文件到你的工作目录中，并赋予二进制文件执行权限。 +不过，你不需要担心这些插件，因为 DevStream 会帮你自动管理它们。 -> 可选：建议你将 dtm 移动到包含于 PATH 的目录下，比如 `mv dtm /usr/local/bin/`。 +--- -_更多安装方式详见[安装dtm](../install.zh.md)。_ +## 3 启程：下载 DevStream (`dtm`) -## 2 准备配置文件 +为本教程创建一个临时工作目录： -运行以下命令来生成 gitops 的模板配置文件 `gitops.yaml` 。 +```bash +mkdir test +cd test/ +``` + +接着，在新创建的目录下，运行下面的命令： ```shell -./dtm show config -t gitops > gitops.yaml +sh -c "$(curl -fsSL https://download.devstream.io/download.sh) ``` -然后对 `gitops.yaml` 文件做相应的修改。 +这个脚本会根据你的操作系统来下载对应的 `dtm` 二进制文件。然后，赋予其可执行权限。 -配置文件中用到的变量的解释和示例值如下： +如果你执行 `ls` 命令，你会看到 `dtm` 二进制文件已经被下载下来了： -| Variable | Example | Note | -| ------------------------------ | ----------------- |-------------------------------------------------| -| defaultBranch | main | 你想使用的分支名称 | -| githubUsername | daniel-hutao | 你的GitHub用户名（区分大小写） | -| repoName | go-webapp | 只要它不在你的GitHub账户下，这个名称就是合法的 | -| dockerhubUsername | exploitht | 你的DockerHub用户名（区分大小写） | -| jiraID | merico | 这是一个域名前缀就像https://merico.atlassian.net 中的merico | -| jiraProjectKey | DT | 项目issue key的描述性前缀，用于识别来自该项目的工作 | -| jiraUserEmail | tao.hu@merico.dev | 登录Jira的邮箱 | -| argocdNameSpace | argocd | ArgoCD用的namespace | -| argocdDeployTimeout | 10m | ArgoCD部署的timeout时长 | +```bash +tiexin@mbp ~/work/devstream-io/test $ ls +dtm +``` -这些插件需要设定一下环境变量： +然后，出于测试目的，我们可以尝试运行它，你会看到类似下面的输出： ```bash -export GITHUB_TOKEN="YOUR_GITHUB_TOKEN_HERE" -export JIRA_API_TOKEN="YOUR_JIRA_API_TOKEN_HERE" -export DOCKERHUB_TOKEN="YOUR_DOCKERHUB_TOKEN_HERE" +tiexin@mbp ~/work/devstream-io/test $ ./dtm +DevStream is an open-source DevOps toolchain manager + +###### ##### +# # ###### # # # # ##### ##### ###### ## # # +# # # # # # # # # # # # ## ## +# # ##### # # ##### # # # ##### # # # ## # +# # # # # # # ##### # ###### # # +# # # # # # # # # # # # # # # +###### ###### ## ##### # # # ###### # # # # + +Usage: + dtm [command] + +Available Commands: + apply Create or update DevOps tools according to DevStream configuration file + completion Generate the autocompletion script for dtm for the specified shell + delete Delete DevOps tools according to DevStream configuration file + destroy Destroy DevOps tools deployment according to DevStream configuration file & state file + develop Develop is used for develop a new plugin + help Help about any command + init Download needed plugins according to the config file + list This command only supports listing plugins now + show Show is used to print plugins' configuration templates or status + upgrade Upgrade dtm to the latest release version + verify Verify DevOps tools according to DevStream config file and state + version Print the version number of DevStream + +Flags: + --debug debug level log + -h, --help help for dtm + +Use "dtm [command] --help" for more information about a command. ``` -如果你不知道如何获取以上环境变量的值，请参考以下链接（英文）： +> 可选：你可以把 `dtm` 移动到 $PATH 环境变量中的某个目录下。例如：`mv dtm /usr/local/bin/`。这样，你就可以直接运行 `dtm` 而不需要再加上 `./` 前缀了。 +> +> 更多安装方式详见[安装 dtm](../install.zh.md)。 + +--- + +## 4 配置文件 + +创建一个名为 `config.yaml` 的文件，并把下面的内容粘贴进去： + +```yaml +config: + state: + backend: local + options: + stateFile: devstream.state +vars: + githubUser: IronCore864 + dockerUser: ironcore864 + app: helloworld + +tools: +- name: repo-scaffolding + instanceID: myapp + options: + destinationRepo: + owner: [[ githubUser ]] + repo: [[ app ]] + branch: main + repoType: github + sourceRepo: + org: devstream-io + repo: dtm-scaffolding-python + repoType: github + vars: + imageRepo: [[ dockerUser ]]/[[ app ]] +- name: githubactions-python + instanceID: default + dependsOn: [ repo-scaffolding.myapp ] + options: + owner: [[ githubUser ]] + repo: [[ app ]] + language: + name: python + branch: main + docker: + registry: + type: dockerhub + username: [[ dockerUser ]] + repository: [[ app ]] +- name: helm-installer + instanceID: argocd +- name: argocdapp + instanceID: default + dependsOn: [ "helm-installer.argocd", "githubactions-python.default" ] + options: + app: + name: [[ app ]] + namespace: argocd + destination: + server: https://kubernetes.default.svc + namespace: default + source: + valuefile: values.yaml + path: helm/[[ app ]] + repoURL: ${{repo-scaffolding.myapp.outputs.repoURL}} +``` + +按需修改 `config.yaml` 文件中的 `vars` 部分。记得修改 `githubUser` 和 `dockerUser` 的值为你自己的用户名。 -- GITHUB_TOKEN: [Manage API tokens for your Atlassian account](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) -- JIRA_API_TOKEN: [Creating a personal access token](https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/) -- DOCKERHUB_TOKEN: [Manage access tokens](https://docs.docker.com/docker-hub/access-tokens/) +在上面的例子中，我把这些变量设置成了下面的值： +| 变量 | 例子 | 说明 | +|------------|-------------|------------------------------------| +| githubUser | IronCore864 | 大小写敏感，请改成你的 GitHub 用户名 | +| dockerUser | ironcore864 | 大小写敏感，请改成你的 DockerHub 用户名 | -## 3 初始化 +## 5 环境变量 -执行： +我们还需要设置以下环境变量： ```bash -dtm init -f gitops.yaml +export GITHUB_TOKEN="YOUR_GITHUB_TOKEN_HERE" +export DOCKERHUB_TOKEN="YOUR_DOCKERHUB_TOKEN_HERE" ``` -## 4 Apply +> 提示： +> 如果你不知道如何创建这两个 token，可以参考： +> +> - GITHUB_TOKEN：[Manage API tokens for your Atlassian account](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) +> - DOCKERHUB_TOKEN：[Manage access tokens](https://docs.docker.com/docker-hub/access-tokens/) + +--- -执行： +## 6 初始化（Init） + +运行以下命令，以根据配置文件自动下载所需插件： ```bash -dtm apply -f gitops.yaml +./dtm init ``` -并且确认继续，然后你就能看到类似如下的输出： +你会看到类似下面的输出： -``` -... -2022-03-11 13:36:11 ✔ [SUCCESS] All plugins applied successfully. -2022-03-11 13:36:11 ✔ [SUCCESS] Apply finished. +```bash +2022-12-05 17:46:01 ℹ [INFO] Using dir to store plugins. +2022-12-05 17:46:01 ℹ [INFO] -------------------- [ repo-scaffolding-darwin-arm64_0.10.1 ] -------------------- +... (略) +... (略) +2022-12-05 17:46:51 ✔ [SUCCESS] Initialize finished. ``` -## 5 检查结果 +--- -我们接着来看`apply`命令的结果。 +## 7 应用（Apply） -### 5.1 仓库结构 +运行： -- 我们得到的仓库结构看起来如下： +```bash +./dtm apply -y +``` -![](gitops/a.png) +你会看到类似下面的输出： -### 5.2 Jira-Github 集成 +``` +2022-12-05 17:49:49 ℹ [INFO] Apply started. +2022-12-05 17:49:49 ℹ [INFO] Using local backend. State file: devstream.state. +2022-12-05 17:49:49 ℹ [INFO] Tool (repo-scaffolding/myapp) found in config but doesn't exist in the state, will be created. +2022-12-05 17:49:49 ℹ [INFO] Tool (helm-installer/argocd) found in config but doesn't exist in the state, will be created. +2022-12-05 17:49:49 ℹ [INFO] Tool (githubactions-python/default) found in config but doesn't exist in the state, will be created. +2022-12-05 17:49:49 ℹ [INFO] Tool (argocdapp/default) found in config but doesn't exist in the state, will be created. +2022-12-05 17:49:49 ℹ [INFO] Start executing the plan. +2022-12-05 17:49:49 ℹ [INFO] Changes count: 4. +... (略) +... (略) +2022-12-05 17:51:51 ℹ [INFO] -------------------- [ Processing progress: 4/4. ] -------------------- +2022-12-05 17:51:51 ℹ [INFO] Processing: (argocdapp/default) -> Create ... +2022-12-05 17:51:52 ℹ [INFO] application.argoproj.io/helloworld created +2022-12-05 17:51:52 ✔ [SUCCESS] Tool (argocdapp/default) Create done. +2022-12-05 17:51:52 ℹ [INFO] -------------------- [ Processing done. ] -------------------- +2022-12-05 17:51:52 ✔ [SUCCESS] All plugins applied successfully. +2022-12-05 17:51:52 ✔ [SUCCESS] Apply finished. +``` -- 测试Jira和Github之间的整合，让我们新创建一个issue： +--- -![](gitops/b.png) +## 8 查看结果 -- 上述issue将会自动地被重命名： +让我们来看看 `apply` 命令的结果。 -![](gitops/c.png) +### 8.1 GitHub 仓库 -- 我们可以在Jira中找到这个自动同步的`Story`： +DevStream 已经通过 `repo-scaffolding` 插件自动创建了一个仓库： -![](gitops/d.png) +![](gitops/a.png) -- 如果我们在这个issue留下评论： +### 8.2 基于 GitHub Actions 的 CI 流水线 -![](gitops/d1.png) +GitHub Actions 流水线已经被创建并运行： -- 这个评论也会被自动同步到Jira中： +![](gitops/b.png) -![](gitops/e.png) +### 8.3 Argo CD 的安装 -### 5.3 为Golang设置Github Actions CI +Argo CD 已经被安装到了 Kubernetes 集群中： -- CI在这里可以做以下事情： +```bash +tiexin@mbp ~/work/devstream-io/test $ kubectl get namespaces +NAME STATUS AGE +argocd Active 5m42s +default Active 6m28s +kube-node-lease Active 6m29s +kube-public Active 6m29s +kube-system Active 6m29s +local-path-storage Active 6m25s +tiexin@mbp ~/work/devstream-io/test $ kubectl get pods -n argocd +NAME READY STATUS RESTARTS AGE +argocd-application-controller-0 1/1 Running 0 5m43s +argocd-applicationset-controller-66687659f-dsrtd 1/1 Running 0 5m43s +argocd-dex-server-6944757486-clshl 1/1 Running 0 5m43s +argocd-notifications-controller-7944945879-b9878 1/1 Running 0 5m43s +argocd-redis-7887bbdbbb-xzppj 1/1 Running 0 5m43s +argocd-repo-server-d4f5cc7cb-8gj24 1/1 Running 0 5m43s +argocd-server-5bb75c4bd9-g948r 1/1 Running 0 5m43s +``` -![](gitops/f.png) +### 8.4 使用 Argo CD 持续部署 -- CI流程同时也会构建Docker镜像，之后该镜像会被自动推送到DockerHub中： +CI 流水线已经构建了一个 Docker 镜像并推送到了 Dockerhub，而 DevStream 创建的 Argo CD 应用也部署了这个应用： -![](gitops/g.png) +```bash +tiexin@mbp ~/work/devstream-io/test $ kubectl get deployment -n default +NAME READY UP-TO-DATE AVAILABLE AGE +helloworld 1/1 1 1 5m16s +tiexin@mbp ~/work/devstream-io/test $ kubectl get pods -n default +NAME READY STATUS RESTARTS AGE +helloworld-69b5586b94-wjwd9 1/1 Running 0 5m18s +tiexin@mbp ~/work/devstream-io/test $ kubectl get services -n default +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +helloworld ClusterIP 10.96.73.97 80/TCP 5m27s +kubernetes ClusterIP 10.96.0.1 443/TCP 8m2s +``` -### 5.4 ArgoCD部署 +我们可以通过端口转发来访问这个应用： -- ArgoCD已经被装好了： +```bash +kubectl port-forward -n default svc/helloworld 8080:80 +``` -![](gitops/h.png) +在浏览器中访问 `localhost:8080`，你可以看到应用返回了一个 "Hello, World!"。大功告成！ -### 5.5 ArgoCD应用部署 +--- -- 我们的代码刚刚被构建到镜像中，此时这个镜像会作为一个Pod自动部署到我们的k8s中： +## 9 清理 -![](gitops/i.png) +运行： -## 6 清理 +```bash +./dtm delete -y +``` -执行： +你会看到如下的输出： ```bash -dtm destroy -f gitops.yaml +2022-12-05 17:59:25 ℹ [INFO] Delete started. +2022-12-05 17:59:26 ℹ [INFO] Using local backend. State file: devstream.state. +2022-12-05 17:59:26 ℹ [INFO] Tool (argocdapp/default) will be deleted. +2022-12-05 17:59:26 ℹ [INFO] Tool (githubactions-python/default) will be deleted. +2022-12-05 17:59:26 ℹ [INFO] Tool (repo-scaffolding/myapp) will be deleted. +2022-12-05 17:59:26 ℹ [INFO] Tool (helm-installer/argocd) will be deleted. +2022-12-05 17:59:26 ℹ [INFO] Start executing the plan. +2022-12-05 17:59:26 ℹ [INFO] Changes count: 4. +... (略) +... (略) +2022-12-05 17:59:35 ℹ [INFO] -------------------- [ Processing done. ] -------------------- +2022-12-05 17:59:35 ✔ [SUCCESS] All plugins deleted successfully. +2022-12-05 17:59:35 ✔ [SUCCESS] Delete finished. ``` -然后你应该能看到类似的输出结果： +后面我们就能删除创建的所有文件了： -``` -2022-03-11 13:39:11 ✔ [SUCCESS] All plugins destroyed successfully. -2022-03-11 13:39:11 ✔ [SUCCESS] Destroy finished. +```bash +cd ../ +rm -rf test/ +rm -rf ~/.devstream/ ``` diff --git a/docs/best-practices/gitops/a.png b/docs/best-practices/gitops/a.png index 9454bd3b3..e3cdabeca 100644 Binary files a/docs/best-practices/gitops/a.png and b/docs/best-practices/gitops/a.png differ diff --git a/docs/best-practices/gitops/b.png b/docs/best-practices/gitops/b.png index d153c4161..c17318970 100644 Binary files a/docs/best-practices/gitops/b.png and b/docs/best-practices/gitops/b.png differ diff --git a/docs/best-practices/gitops/c.png b/docs/best-practices/gitops/c.png deleted file mode 100644 index 0b44caf4e..000000000 Binary files a/docs/best-practices/gitops/c.png and /dev/null differ diff --git a/docs/best-practices/gitops/d.png b/docs/best-practices/gitops/d.png deleted file mode 100644 index a89cdcfa3..000000000 Binary files a/docs/best-practices/gitops/d.png and /dev/null differ diff --git a/docs/best-practices/gitops/d1.png b/docs/best-practices/gitops/d1.png deleted file mode 100644 index ff4b1c849..000000000 Binary files a/docs/best-practices/gitops/d1.png and /dev/null differ diff --git a/docs/best-practices/gitops/e.png b/docs/best-practices/gitops/e.png deleted file mode 100644 index 876f5f71f..000000000 Binary files a/docs/best-practices/gitops/e.png and /dev/null differ diff --git a/docs/best-practices/gitops/f.png b/docs/best-practices/gitops/f.png deleted file mode 100644 index 91fffb91b..000000000 Binary files a/docs/best-practices/gitops/f.png and /dev/null differ diff --git a/docs/best-practices/gitops/g.png b/docs/best-practices/gitops/g.png deleted file mode 100644 index 638bc0445..000000000 Binary files a/docs/best-practices/gitops/g.png and /dev/null differ diff --git a/docs/best-practices/gitops/h.png b/docs/best-practices/gitops/h.png deleted file mode 100644 index 7b9cba87b..000000000 Binary files a/docs/best-practices/gitops/h.png and /dev/null differ diff --git a/docs/best-practices/gitops/i.png b/docs/best-practices/gitops/i.png deleted file mode 100644 index aae94aad1..000000000 Binary files a/docs/best-practices/gitops/i.png and /dev/null differ diff --git a/docs/core-concepts/apps.zh.md b/docs/core-concepts/apps.zh.md new file mode 100644 index 000000000..1a8b3163a --- /dev/null +++ b/docs/core-concepts/apps.zh.md @@ -0,0 +1,180 @@ +# 应用 + +## 概念 + +应用在 devstream 中表示对一个服务的生命周期配置，包括对服务的项目脚手架，CI 流程以及 CD 流程的配置，在 devstream 中使用应用可以使用少数几行配置就构建出服务的整条 CI/CD 流水线配置。 + +## 应用配置 + +应用的示例配置如下： + +```yaml +apps: +- name: testApp #应用名称 + spec: # 该配置项用于配置应用特定的信息 + language: java #应用所使用的编程语言 + framework: springboot #应用所使用的编程框架 + repo: # 该配置项用于应用的代码仓库信息 + url: https://github.com/testUser/testApp.git + branch: main + repoTemplate: # 该配置用于创建应用的脚手架 + url: https://github.com/devstream-io/dtm-repo-scaffolding-java-springboot.git + vars: + imageRepoOwner: repoOwner # 用于渲染脚手架模版的变量 + ci: # 配置应用的 ci 流程，如下即为使用 github-actions 运行应用 ci 流程 + - type: github-actions +- name: testApp2 + spec: + language: go + framework: gin + repo: # 该配置项用于应用的代码仓库信息 + owner: test_user + type: github + branch: main + repoTemplate: # 该配置用于创建应用的脚手架 + org: devstream-io + name: dtm-repo-scaffolding-java-springboot + type: github + ci: # 配置应用的 ci 流程，如下即为使用 github-actions 运行应用 ci 流程 + - type: github-actions + options: + imageRepo: + owner: repoOwner + cd: # 配置应用的 cd，如果为使用 argocdapp 运行应用的 cd 流程 + - type: argocdapp +``` + +使用该配置就会在 `gitlab` 仓库中创建两个应用，项目的脚手架均为 devstream 官方提供的 [springboot](https://github.com/devstream-io/dtm-repo-scaffolding-java-springboot.git) 项目。应用 `testApp` 会在每次代码提交后使用 `github-actions` 运行测试。应用 `testApp2` 会在每次代码提交后使用 `github-actions` 运行测试并构建镜像推送到 `repoOwner` 的镜像仓库中，最后使用 argocd 将应用部署到集群中。 + +### repo/repoTemplate 配置 + +应用配置中的 `repo` 和 `repoTemplate` 均表示一个代码仓库，支持以下两种配置代码仓库的方式： + +#### 使用 url 来配置代码仓库： + +```yaml + repo: + url: git@gitlab.example.com:root/myapps.git # url 表示仓库的地址，支持 git 地址和 http 地址 + apiURL: https://gitlab.example.com # 非必填，如果使用 gitlab 而且 url 使用的是 git 地址，则需要配置该字段用于表示 gitlab 的 api 请求地址 + branch: "" #非必填，github 默认为 main 分支，gitlab 默认为 master 分支 +``` + +该配置表示使用的仓库为 `gitlab`, 要使用 `git@gitlab.example.com:root/myapps.git` 来克隆代码，devstream 会使用使用 `https://gitlab.example.com` 和 gitlab 进行交互，仓库的主分支为 `master` 分支。 + +#### 使用仓库的详细字段来配置仓库： + +```yaml + repo: + org: "" # 非必填，仓库的拥有组织名称，如果使用 github 的组织则需要填写此字段 + owner："test_user" # 如果仓库是非组织的，则需要填写该字段表示仓库拥有者 + name: "" # 非必填，默认为应用名 + baseURL: https://gitlab.example.com # 非必填，如果使用 gitlab 则需要填写该字段表示 gitlab 的域名 + branch: master #非必填，github 默认为 main 分支，gitlab 默认为 master 分支 + type: gitlab #必填，表示该仓库的类型，目前支持 gitlab/github +``` + +该配置表示代码仓库使用的是 `gitlab` 且其地址为 `https://gitlab.example.com`，仓库名称为应用名，仓库的所有者为 `test_user`，主分支为 `master` 分支。 + +### ci 配置 + +应用配置中的 `ci` 目前支持 `github-actions`/`gitlab-ci`/`jenkins-pipeline`/`template` 4 种类型，前 3 种类型分别对应了 `github` 中的 actions 流水线，`gitlab` 中 ci 流水线和 `jenkins` 中的 pipeline，它们的具体配置如下： + +```yaml + ci: + - type: jenkins-pipieline # 表明当前 ci 的类型 + options: # ci 的具体配置项，如果该配置项为空，则 ci 只会运行单元测试然后结束 + jenkins: # 该配置项之用于 jenkins，表示 jenkins 的一些配置信息 + url: jenkins.exmaple.com # jenkins 的地址 + user: admin # jenkins 用户 + imageRepo: # 需要推送的镜像仓库信息，如果设置了该字段，则 ci 流程会在测试成功后构建镜像推送到该镜像仓库 + url: http://harbor.example.com # 镜像仓库地址，若为空则默认为 dockerhub + owner: admin # 镜像仓库拥有者名称 + dingTalk: # 钉钉通知的配置信息，如果设置了该字段，则 ci 流程中会将最后的构建结构通过钉钉发送通知 + name: dingTalk + webhook: https://oapi.dingtalk.com/robot/send?access_token=changemeByConfig # 钉钉的回调地址 + securityType: SECRET # 使用 secret 模式来加密钉钉的信息 + securityValue: SECRETDATA # 钉钉的 secret 加密字符串 + sonarqube: # sonarqube 的配置信息，如果设置了该字段，则 ci 流程会和测试并行执行 sonarqube 的代码扫描 + url: http://sonar.example.com # sonarqube 的地址 + token: YOUR_SONAR_TOKEN # soanrqube 的认证 token + name: sonar_test +``` + +上述的配置即会在应用更新推送到代码仓库后先并行执行单元测试和代码扫描，然后构建镜像推送到代码仓库，流程都成功后发送通知消息到指定的钉钉群中。如果所有应用都要配置一遍该类型 `ci`, 那配置就会变得比较繁琐，所以 devstream 还提供了 `template` 用于在多个应用间共享 `ci` 的流程配置，具体如下所示： + +```yaml +apps: +- name: javaProject1 + spec: + language: java + framework: springboot + repo: + owner: testUser + type: github + repoTemplate: + url: https://github.com/devstream-io/dtm-repo-scaffolding-java-springboot.git + ci: + - type: template # 表示该 ci 流程使用模版 + templateName: ci-pipeline # ci 使用的模版名称 + vars: + dingdingAccessToken: tokenForProject1 #用于渲染 ci 模版的变量 + dingdingSecretValue: secretValProject1 +- name: javaProject2 + spec: + language: java + framework: springboot + repo: + owner: testUser + type: github + repoTemplate: + url: https://github.com/devstream-io/dtm-repo-scaffolding-java-springboot.git + ci: + - type: template # 表示该 ci 流程使用模版 + templateName: ci-pipeline # ci 使用的模版名称 + vars: + dingdingAccessToken: tokenForProject2 #用于渲染 ci 模版的变量 + dingdingSecretValue: secretValProject2 + +pipelineTemplates: # 即配置的 ci/cd 模版 +- name: ci-pipeline # 模版名称 + type: jenkins-pipeline #模版类型，支持 jenkins-pipeline，github-actions 和 gitlab-ci + options: # options 和 ci 的 options 完全一致 + jenkins: + url: jenkins.exmaple.com + user: admin + imageRepo: + url: http://harbor.example.com + owner: admin + dingTalk: + name: dingTalk + webhook: https://oapi.dingtalk.com/robot/send?access_token=[[ dingdingAccessToken ]] # 用于被 app 渲染的模版，这样就可以实现不同应用使用同一个模版发送通知到不同的钉钉群 + securityType: SECRET + securityValue: [[ dingdingSecretValue ]] + sonarqube: + url: http://sonar.example.com + token: sonar_token + name: sonar_test + +``` + +使用以上的配置，就会创建两个和上述流程一致的 `jenkins` 流水线，不同之处只在于两个应用通知的钉钉群不一样。 + + +### cd 配置 + +应用的 cd 配置目前只支持 `argocdapp`，可以使用 `argocd` 来将应用部署在集群中，具体配置如下： + +```yaml +cd: +- type: argocdapp + options: + app: + name: hello # argocd 引用名称 + namespace: argocd # argocd 的命名空间 + destination: + server: https://kubernetes.default.svc # 部署的 kubernetes 服务地址 + namespace: default # 应用要部署的命名空间 + source: + valuefile: values.yaml # 项目中的 helm 变量文件名 + path: charts/go-hello-http # 项目中的 helm 配置路径 +``` diff --git a/docs/core-concepts/config.zh.md b/docs/core-concepts/config.zh.md index bfdb19a7b..76b419c74 100644 --- a/docs/core-concepts/config.zh.md +++ b/docs/core-concepts/config.zh.md @@ -10,9 +10,6 @@ DevStream使用YAML文件来描述你的DevOps工具链配置。 主配置包含三个部分： -- `varFile`: 指向定义变量的文件路径 -- `toolFile`: 指向定义插件的文件路径 -- `pluginDir`: 指向插件目录的路径，默认为 `~/.devstream/plugins`, 或使用 `-d` 参数指定一个目录 - `state`: 指定DevStream状态存储位置 ### 主配置文件示例 diff --git a/docs/core-concepts/output.md b/docs/core-concepts/output.md index d078c14e8..1b9aaf190 100644 --- a/docs/core-concepts/output.md +++ b/docs/core-concepts/output.md @@ -57,7 +57,7 @@ tools: branch: main repoType: github vars: - ImageRepo: "ironcore864/golang-demo" + imageRepo: "ironcore864/golang-demo" sourceRepo: org: devstream-io repo: dtm-scaffolding-golang diff --git a/docs/development/architecture.zh.md b/docs/development/architecture.zh.md index 773c573af..6be738ce0 100644 --- a/docs/development/architecture.zh.md +++ b/docs/development/architecture.zh.md @@ -24,7 +24,7 @@ _注意：为了简单起见，CLI被命名为`dtm`（DevOps Toolchain Manager) `pluginengine`首先调用`configmanager`，将本地YAML配置文件读取到一个结构体中，然后调用`pluginmanager`来下载所需的插件。 -之后，`pluginengine`调用`statemanager`来计算congfig、状态和实际DevOps工具的状态之间的"差异"。最后，`pluginengine`根据这变更执行对应的操作，并更新状态。在执行过程中，`pluginengine`加载每个插件（`*.so`文件）并根据每个变更调用相应的接口。 +之后，`pluginengine`调用`statemanager`来计算config、状态和实际DevOps工具的状态之间的"差异"。最后，`pluginengine`根据这变更执行对应的操作，并更新状态。在执行过程中，`pluginengine`加载每个插件（`*.so`文件）并根据每个变更调用相应的接口。 ## 2 插件引擎 diff --git a/docs/development/dev/creating-a-plugin.zh.md b/docs/development/dev/creating-a-plugin.zh.md index 43baa819b..4d5250cb2 100644 --- a/docs/development/dev/creating-a-plugin.zh.md +++ b/docs/development/dev/creating-a-plugin.zh.md @@ -41,11 +41,9 @@ ### 2.2 返回值 -`create`、`read`和`update`方法返回两个值`(statemanager.ResourceStatus, error)`；第一个是 "状态"。 +`Create`, `Read` 和 `Update` 方法返回两个值 `(statemanager.ResourceStatus, error)`。第一个是 "状态"，第二个是 "错误"。 -`delete'接口返回两个值`(bool, error)`。如果没有错误，它返回`(true, nil)`；否则将返回`(false, error)`。 - -如果没有发生错误，返回值将是`(true, nil)`。否则，结果将是`(false, error)`。 +`Delete` 接口返回两个值 `(bool, error)`。如果没有错误，它应当返回 `(true, nil)`；否则应返回 `(false, error)`。 ## 3 插件是如何工作的？ @@ -53,4 +51,4 @@ DevStream是使用[go plugin](https://pkg.go.dev/plugin)来实现自定义插件 当你执行一个调用任何接口(`Create`, `Read`, `Update`, `Delete`)的命令时，DevStream的`pluginengine`会调用[`plugin.Lookup("DevStreamPlugin")`函数](https://github.com/devstream-io/devstream/blob/38307894bbc08f691b2c5015366d9e45cc87970c/internal/pkg/pluginengine/plugin_helper.go#L28)来加载插件，获得实现`DevStreamPlugin`接口的变量`DevStreamPlugin`，然后你就可以调用相应的插件接口。所以我们不建议你直接修改`/cmd/plugin/YOUR-PLUGIN-NAME/main.go`文件，因为该文件是根据接口定义自动生成好的。 -注意：`/cmd/plugin/YOUR-PLUGIN-NAME/main.go`文件中的`main()`不会被执行，它只是用来避免golangci-lint错误。 +注意：`/cmd/plugin/YOUR-PLUGIN-NAME/main.go`文件中的`main()`不会被执行，它只是用来避免 `golangci-lint` 错误。 diff --git a/docs/install.md b/docs/install.md index 323f63dc0..0502b479b 100644 --- a/docs/install.md +++ b/docs/install.md @@ -11,7 +11,7 @@ In your working directory, run: ```shell -sh -c "$(curl -fsSL https://raw.githubusercontent.com/devstream-io/devstream/main/hack/install/download.sh)" +sh -c "$(curl -fsSL https://download.devstream.io/download.sh) ``` This will download the corresponding `dtm` binary to your working directory according to your OS and chip architecture, and grant the binary execution permission. diff --git a/docs/install.zh.md b/docs/install.zh.md index 0ac0dc677..a8320a06a 100644 --- a/docs/install.zh.md +++ b/docs/install.zh.md @@ -11,7 +11,7 @@ 进入你的工作目录，运行： ```shell -sh -c "$(curl -fsSL https://raw.githubusercontent.com/devstream-io/devstream/main/hack/install/download.sh)" +sh -c "$(curl -fsSL https://download.devstream.io/download.sh) ``` 这个命令会根据你的操作系统和芯片架构下载对应的 `dtm` 二进制文件到你的工作目录中，并赋予二进制文件执行权限。 diff --git a/docs/plugins/argocdapp.md b/docs/plugins/argocdapp.md index df56db1c5..d89f3f4d4 100644 --- a/docs/plugins/argocdapp.md +++ b/docs/plugins/argocdapp.md @@ -1,14 +1,14 @@ # argocdapp Plugin -This plugin creates an [ArgoCD Application](https://argo-cd.readthedocs.io/en/stable/core_concepts/) custom resource. +This plugin creates an [Argo CD Application](https://argo-cd.readthedocs.io/en/stable/core_concepts/) custom resource. **Notes:** -- ArgoCD itself must have been already installed before the usage of this plugin. - To install ArgoCD, use the [helm-installer plugin](./helm-installer/argocd.md). +- Argo CD itself must have been already installed before the usage of this plugin. + To install Argo CD, use the [helm-installer plugin](./helm-installer/argocd.md). Or you can use both plugins(argocd+argocdapp) at the same time. See [GitOps Toolchain](../best-practices/gitops.md) for more info. -- Currently, only the Helm chart is supported when creating the ArgoCD application. +- Currently, only the Helm chart is supported when creating the Argo CD application. - Modify the file accordingly. Especially remember to modify `ARGOCD_TOOL_NAME`. ## Usage @@ -45,7 +45,7 @@ tools: branch: [[ defaultBranch ]] repoType: github vars: - ImageRepo: "[[ dockerhubUsername ]]/[[ repoName ]]" + imageRepo: "[[ dockerhubUsername ]]/[[ repoName ]]" sourceRepo: org: devstream-io repo: dtm-scaffolding-golang diff --git a/docs/plugins/helm-installer/helm-installer.zh.md b/docs/plugins/helm-installer/helm-installer.zh.md index accb2d5ef..e5d02af9c 100644 --- a/docs/plugins/helm-installer/helm-installer.zh.md +++ b/docs/plugins/helm-installer/helm-installer.zh.md @@ -7,15 +7,11 @@ 只需要一个最小化配置，你就可以快速使用默认配置部署一个 Helm Chart。你可以将如下配置内容保存到本地 config.yaml 文件中： ```yaml ---- -varFile: "" -toolFile: "" -state: - backend: local - options: - stateFile: devstream.state - ---- +config: + state: + backend: local + options: + stateFile: devstream.state tools: - name: helm-installer instanceID: argocd-001 diff --git a/docs/plugins/helm-installer/jenkins/jenkins-images.txt b/docs/plugins/helm-installer/jenkins/jenkins-images.txt index af2abd143..bd0945efc 100644 --- a/docs/plugins/helm-installer/jenkins/jenkins-images.txt +++ b/docs/plugins/helm-installer/jenkins/jenkins-images.txt @@ -1,5 +1,5 @@ ##jenkins-images -devstreamdev/jenkins:2.361.1-jdk11-dtm-0.1 +devstreamdev/jenkins:2.361.1-jdk11-dtm-0.2 kiwigrid/k8s-sidecar:1.15.0 jenkins/inbound-agent:4.11.2-4 maorfr/kube-tasks:0.2.0 diff --git a/docs/plugins/jenkins-pipeline.zh.md b/docs/plugins/jenkins-pipeline.zh.md index 9eb8f8361..a3e931781 100644 --- a/docs/plugins/jenkins-pipeline.zh.md +++ b/docs/plugins/jenkins-pipeline.zh.md @@ -4,7 +4,7 @@ 本文将演示： -1. 通过 [`repo-scaffolding`](./repo-scaffolding.zh.md) 插件在 GitLab 上创建一个 Java Sprint Boot 项目脚手架； +1. 通过 [`repo-scaffolding`](./repo-scaffolding.zh.md) 插件在 GitLab 上创建一个 Java Spring Boot 项目脚手架； 2. 通过 `jenkins-pipeline` 插件在 Jenkins 上创建一条 Java Spring Boot 的 CI 流水线； 3. 通过 `jenkins-pipeline` 插件实现在 GitLab 和 Jenkins 上分别配置相应参数，实现当 GitLab 上的代码库有 push 或者 merge 事件时，自动触发 Jenkins 上的流水线运行，同时流水线的执行结果自动回写到 GitLab 上。 @@ -45,7 +45,7 @@ export GITLAB_SSHKEY=YOUR_REPO_PRIVATE_KEY !!! tip "提示" 如果是 GitHub，则这里的环境变量改成： - + ```shell export IMAGE_REPO_PASSWORD=YOUR_IMAGE_REPO_PASSWORD export GITHUB_TOKEN=YOUR_GITHUB_TOKEN @@ -196,3 +196,82 @@ tools: ``` yaml --8<-- "jenkins-pipeline.yaml" ``` + + +## 离线配置 （下文待进一步整理） + +目前的 `jenkins-pipeline` 插件配置过程中会连接外部网络来安装插件和配置 `pipeline` 的共享库，为了支持在离线环境中使用 `jenkins-pipeline` 来创建和管理 `jenkins` 流水线，我们在 `jenkins-pipeline` 插件中提供了一个选项 `offline` 用于表示需要在离线环境下配置 `jenkins-pipeline`。 + +### Jenkins 插件 + +流水线正常运行需要依赖 `jenkins` 的几个插件。在离线环境中因为无法下载外部环境的插件，需要在 `jenkins` 预先安装好插件或者使用已经有插件的 `jenkins` 镜像，devstream 官方也提供了已预先安装好所有依赖的镜像 `devstreamdev/jenkins:2.361.1-jdk11-dtm-0.2`。 + +依赖的插件如下： +| 插件名 | 作用 | 备注 | +|--------------|----------------|----------------------------| +| dingding-notifications | 用于发送钉钉通知 | 如果需要使用钉钉通知，则需要安装此插件 | +| github-branch-source | jenkins github 插件 | 如果代码仓库使用的是 github，则必须安装此插件 | +| gitlab-plugin | jenkins gitlab 插件 | 如果代码仓库使用的是 gitlab，则必须安装此插件 | +| sonar | sonar sanner 代码扫描插件 | 如果需要使用 sonarqube，则需要安装此插件 | +| kubernetes | jenkins kubernetes 插件 | 用于 jenkins runner， 必须安装此插件 | +| git | jenkins git 插件 | 用于代码的克隆和权限认证， 必须安装此插件 | +| configuration-as-code | jenkins 配置即代码插件 | 用于 devstream 配置 jenkins 的全局配置， 必须安装此插件| + +### 共享库 + +在 `jenkins` 中 devstream 默认使用共享库来管理 jenkins 流水线的共用代码，在离线环境中因为无法连接共享库，所以 devstream 提供了单独的 `Jenkinfile` 配置。 + + +### app 示例配置 + +```yaml +apps: +- name: test + spec: + language: java + framework: springboot + repo: + url: gitlab.com/root/test.git + branch: main + repoTemplate: + url: https://github.com/devstream-io/dtm-repo-scaffolding-java-springboot.git + ci: + - type: template + templateName: ci-pipeline + +pipelineTemplates: +- name: ci-pipeline + type: jenkins-pipeline + options: + branch: main + jenkins: + url: jenkins.com + user: admin + offline: true # 在此处设置 offline 为 true， 即开启该 jenkins-pipeline 的离线模式 + imageRepo: + user: repoUser +``` + +使用该配置可得到以下输出： + +```text +2022-12-02 19:51:52 ℹ [INFO] Apply started. +2022-12-02 19:51:52 ℹ [INFO] Using local backend. State file: devstream-app.state. +2022-12-02 19:51:52 ℹ [INFO] Tool (repo-scaffolding/test) found in config but doesn't exist in the state, will be created. +2022-12-02 19:51:52 ℹ [INFO] Tool (jenkins-pipeline/test) found in config but doesn't exist in the state, will be created. +2022-12-02 19:51:52 ℹ [INFO] Start executing the plan. +2022-12-02 19:51:52 ℹ [INFO] Changes count: 2. +2022-12-02 19:51:52 ℹ [INFO] -------------------- [ Processing progress: 1/2. ] -------------------- +2022-12-02 19:51:52 ℹ [INFO] Processing: (repo-scaffolding/test) -> Create ... +2022-12-02 19:51:52 ℹ [INFO] github start to download repoTemplate... +2022-12-02 19:51:57 ✔ [SUCCESS] Tool (repo-scaffolding/test) Create done. +2022-12-02 19:51:57 ℹ [INFO] -------------------- [ Processing progress: 2/2. ] -------------------- +2022-12-02 19:51:57 ℹ [INFO] Processing: (jenkins-pipeline/test) -> Create ... +2022-12-02 19:51:58 ℹ [INFO] jenkins plugin imageRepo start config... +2022-12-02 19:51:58 ⚠ [WARN] jenkins gitlab ssh key not config, private repo can't be clone +2022-12-02 19:52:00 ℹ [INFO] jenkins start config casc... +2022-12-02 19:52:07 ✔ [SUCCESS] Tool (jenkins-pipeline/test) Create done. +2022-12-02 19:52:07 ℹ [INFO] -------------------- [ Processing done. ] -------------------- +2022-12-02 19:52:07 ✔ [SUCCESS] All plugins applied successfully. +2022-12-02 19:52:07 ✔ [SUCCESS] Apply finished. +``` \ No newline at end of file diff --git a/docs/plugins/plugins-list.md b/docs/plugins/plugins-list.md index eb2ed0db3..c0ce49a7c 100644 --- a/docs/plugins/plugins-list.md +++ b/docs/plugins/plugins-list.md @@ -16,7 +16,7 @@ | CI | gitlabci-java | GitLab CI for Java | [doc](gitlabci-java.md) | | CI | gitlabci-generic | Generic GitLab CI | [doc](gitlabci-generic.md) | | CI | ci-generic | Generic CI plugin | [doc](ci-generic.md) | -| CD/GitOps | argocdapp | ArgoCD Application creation | [doc](argocdapp.md) | +| CD/GitOps | argocdapp | Argo CD Application creation | [doc](argocdapp.md) | | Image Repository | harbor-docker | Harbor Docker compose install | [doc](harbor-docker.md) | | Deployment | helm-installer | Helm chart install | [doc](helm-installer/helm-installer.md) | diff --git a/docs/plugins/repo-scaffolding.md b/docs/plugins/repo-scaffolding.md index 748aa6855..48280c385 100644 --- a/docs/plugins/repo-scaffolding.md +++ b/docs/plugins/repo-scaffolding.md @@ -116,7 +116,7 @@ tools: repo: dtm-scaffolding-golang repoType: github vars: - ImageRepo: dtm-test/golang-repo + imageRepo: dtm-test/golang-repo ``` This config will create `dtm-test-golang` repo for user test_owner in GitHub, and the variable ImageRepo will be used for template render. diff --git a/docs/plugins/zentao.md b/docs/plugins/zentao.md index c5ad911ff..c30a05b50 100644 --- a/docs/plugins/zentao.md +++ b/docs/plugins/zentao.md @@ -16,17 +16,12 @@ The following content is an example of the "tool file". For more information on the main config, the tool file and the var file of DevStream, see [Core Concepts Overview](../core-concepts/overview.md) and [DevStream Configuration](../core-concepts/config.md). ```yaml ---- -# core config -varFile: "" -toolFile: "" -state: # state config, backend can be local or s3 - backend: local - options: - stateFile: devstream.state - ---- -# plugins config +config: + state: + backend: local + options: + stateFile: devstream.state + tools: # name of the tool - name: zentao diff --git a/docs/plugins/zentao.zh.md b/docs/plugins/zentao.zh.md index 4275606fb..e2791d277 100644 --- a/docs/plugins/zentao.zh.md +++ b/docs/plugins/zentao.zh.md @@ -15,17 +15,12 @@ 关于更多关于DevStream的主配置、tool file、var file的信息，请阅读[核心概念概览](../core-concepts/overview.zh.md)和[DevStream配置](../core-concepts/config.zh.md). ```yaml ---- -# core config -varFile: "" -toolFile: "" -state: # state config, backend can be local or s3 - backend: local - options: - stateFile: devstream.state - ---- -# plugins config +config: + state: + backend: local + options: + stateFile: devstream.state + tools: # name of the tool - name: zentao diff --git a/docs/quickstart.md b/docs/quickstart.md index e41a15815..abfe3d9b0 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,27 +1,34 @@ # Quick Start -[Watch the demo](./index.md) first if you prefer to see DevStream in action. - -> Note: currently, we only have Linux and macOS versions of DevStream. Windows support will come later. - In this quickstart, we will do the following automatically with DevStream: -- create a GitHub repository with Golang web app scaffolding; -- set up GitHub Actions workflow for the Golang app we created, which contains test and build stages for our Go web app. +- create a GitHub repository with automatically generated code for a web application written in Golang with the [gin](https://github.com/gin-gonic/gin) framework; +- set up GitHub Actions workflow for the app created in the previous step. + +--- ## 1 Download In your working directory, run: ```shell -sh -c "$(curl -fsSL https://raw.githubusercontent.com/devstream-io/devstream/main/hack/install/download.sh)" +sh -c "$(curl -fsSL https://download.devstream.io/download.sh) ``` -This will download the corresponding `dtm` binary to your working directory according to your OS and chip architecture, and grant the binary execution permission. +!!! note "Note" + the command above does the following: + + - find out your OS and chip architecture + - find the latest version of the `dtm` binary + - download the correct `dtm` according to OS/architecture + - grant the binary execution permission. -> Optional: you can then move `dtm` to a place which is in your PATH. For example: `mv dtm /usr/local/bin/`. +!!! quote "Optional" + You can then move `dtm` to a place which is in your PATH. For example: `mv dtm /usr/local/bin/`. + + For more ways to install `dtm`, see [install dtm](./install.md). -_For more details on how to install, see [install dtm](./install.md)._ +--- ## 2 Configuration @@ -31,13 +38,7 @@ Run the following command to generate the template configuration file `config.ya ./dtm show config -t quickstart > config.yaml ``` -As aforementioned, we will handle GitHub repo scaffolding and CI workflows in GitHub Actions, so, we will need the following environment variables (env vars) to be set: - -- GITHUB_USER -- GITHUB_TOKEN -- DOCKERHUB_USERNAME - -Run the following commands to set values for these env vars (replace values within the double quotes): +Then set the following environment variables by running (replace values within the double quotes): ```shell export GITHUB_USER="" @@ -45,115 +46,98 @@ export GITHUB_TOKEN="" export DOCKERHUB_USERNAME="" ``` -> Tip: Go to [Personal Access Token](https://github.com/settings/tokens/new) to generate a new `GITHUB_TOKEN` for `dtm`. -> -> For "Quick Start", we only need `repo`,`workflow`,`delete_repo` scopes. However, we recommend that you check all and future plugins may require more scopes. +!!! tip "Tip" + Go to [Personal Access Token](https://github.com/settings/tokens/new) to generate a new `GITHUB_TOKEN` for `dtm`. + + For "Quick Start", we only need `repo`,`workflow`,`delete_repo` permissions. Then we run the following commands to update our config file with those env vars: -For **macOS** or **FreeBSD** based systems: +=== "**macOS** or **FreeBSD** based systems" -```shell -sed -i.bak "s@YOUR_GITHUB_USERNAME_CASE_SENSITIVE@${GITHUB_USER}@g" quickstart.yaml -sed -i.bak "s@YOUR_DOCKER_USERNAME@${DOCKERHUB_USERNAME}@g" quickstart.yaml -``` + ```shell + sed -i.bak "s@YOUR_GITHUB_USERNAME_CASE_SENSITIVE@${GITHUB_USER}@g" config.yaml + sed -i.bak "s@YOUR_DOCKER_USERNAME@${DOCKERHUB_USERNAME}@g" config.yaml + ``` -For **GNU** Linux users: +=== "**GNU** Linux users" + + ```shell + sed -i "s@YOUR_GITHUB_USERNAME_CASE_SENSITIVE@${GITHUB_USER}@g" config.yaml + sed -i "s@YOUR_DOCKER_USERNAME@${DOCKERHUB_USERNAME}@g" config.yaml + ``` -```shell -sed -i "s@YOUR_GITHUB_USERNAME_CASE_SENSITIVE@${GITHUB_USER}@g" quickstart.yaml -sed -i "s@YOUR_DOCKER_USERNAME@${DOCKERHUB_USERNAME}@g" quickstart.yaml -``` +--- ## 3 Init Run: ```shell -./dtm init -f quickstart.yaml +./dtm init ``` -and you should see similar output to the following: - -``` -2022-06-30 11:21:48 ℹ [INFO] Got Backend from config: local -2022-06-30 11:21:48 ℹ [INFO] Using dir <.devstream> to store plugins. -2022-06-30 11:21:48 ℹ [INFO] Downloading: [repo-scaffolding-darwin-arm64_0.7.0.so] ... - 15.05 MiB / 15.05 MiB [================================] 100.00% 21.17 MiB/s 0s -2022-06-30 11:21:49 ✔ [SUCCESS] [repo-scaffolding-darwin-arm64_0.7.0.so] download succeeded. -2022-06-30 11:21:49 ℹ [INFO] Downloading: [repo-scaffolding-darwin-arm64_0.7.0.md5] ... - 33 B / 33 B [==========================================] 100.00% 35.29 KiB/s 0s -2022-06-30 11:21:49 ✔ [SUCCESS] [repo-scaffolding-darwin-arm64_0.7.0.md5] download succeeded. -2022-06-30 11:21:49 ℹ [INFO] Plugin: repo-scaffolding-darwin-arm64_0.7.0.so doesn't match with .md5 and will be downloaded. -2022-06-30 11:21:49 ℹ [INFO] Downloading: [repo-scaffolding-darwin-arm64_0.7.0.so] ... - 15.05 MiB / 15.05 MiB [================================] 100.00% 31.25 MiB/s 0s -2022-06-30 11:21:50 ✔ [SUCCESS] [repo-scaffolding-darwin-arm64_0.7.0.so] download succeeded. -2022-06-30 11:21:50 ℹ [INFO] Downloading: [repo-scaffolding-darwin-arm64_0.7.0.md5] ... - 33 B / 33 B [==========================================] 100.00% 43.43 KiB/s 0s -2022-06-30 11:21:50 ✔ [SUCCESS] [repo-scaffolding-darwin-arm64_0.7.0.md5] download succeeded. -2022-06-30 11:21:50 ℹ [INFO] Downloading: [githubactions-golang-darwin-arm64_0.7.0.so] ... - 17.49 MiB / 17.49 MiB [================================] 100.00% 31.18 MiB/s 0s -2022-06-30 11:21:51 ✔ [SUCCESS] [githubactions-golang-darwin-arm64_0.7.0.so] download succeeded. -2022-06-30 11:21:51 ℹ [INFO] Downloading: [githubactions-golang-darwin-arm64_0.7.0.md5] ... - 33 B / 33 B [=========================================] 100.00% 160.70 KiB/s 0s -2022-06-30 11:21:51 ✔ [SUCCESS] [githubactions-golang-darwin-arm64_0.7.0.md5] download succeeded. -2022-06-30 11:21:51 ℹ [INFO] Plugin: githubactions-golang-darwin-arm64_0.7.0.so doesn't match with .md5 and will be downloaded. -2022-06-30 11:21:51 ℹ [INFO] Downloading: [githubactions-golang-darwin-arm64_0.7.0.so] ... - 17.49 MiB / 17.49 MiB [================================] 100.00% 31.78 MiB/s 0s -2022-06-30 11:21:52 ✔ [SUCCESS] [githubactions-golang-darwin-arm64_0.7.0.so] download succeeded. -2022-06-30 11:21:52 ℹ [INFO] Downloading: [githubactions-golang-darwin-arm64_0.7.0.md5] ... - 33 B / 33 B [==========================================] 100.00% 87.12 KiB/s 0s -2022-06-30 11:21:52 ✔ [SUCCESS] [githubactions-golang-darwin-arm64_0.7.0.md5] download succeeded. -2022-06-30 11:21:52 ✔ [SUCCESS] Initialize finished. -``` +!!! success "You should see some output similar to the following" + ```text title="" + 2022-12-02 16:11:55 ℹ [INFO] Using dir to store plugins. + 2022-12-02 16:11:55 ℹ [INFO] -------------------- [ repo-scaffolding-darwin-arm64_0.10.1 ] -------------------- + 2022-12-02 16:11:57 ℹ [INFO] Downloading: [repo-scaffolding-darwin-arm64_0.10.1.so] ... + 87.82 MiB / 87.82 MiB [================================] 100.00% 12.30 MiB/s 7s + 2022-12-02 16:12:04 ✔ [SUCCESS] [repo-scaffolding-darwin-arm64_0.10.1.so] download succeeded. + 2022-12-02 16:12:04 ℹ [INFO] Downloading: [repo-scaffolding-darwin-arm64_0.10.1.md5] ... + 33 B / 33 B [==========================================] 100.00% 50.98 KiB/s 0s + 2022-12-02 16:12:04 ✔ [SUCCESS] [repo-scaffolding-darwin-arm64_0.10.1.md5] download succeeded. + 2022-12-02 16:12:04 ℹ [INFO] Initialize [repo-scaffolding-darwin-arm64_0.10.1] finished. + 2022-12-02 16:12:04 ℹ [INFO] -------------------- [ repo-scaffolding-darwin-arm64_0.10.1 ] -------------------- + 2022-12-02 16:12:04 ℹ [INFO] -------------------- [ githubactions-golang-darwin-arm64_0.10.1 ] -------------------- + 2022-12-02 16:12:05 ℹ [INFO] Downloading: [githubactions-golang-darwin-arm64_0.10.1.so] ... + 86.44 MiB / 86.44 MiB [================================] 100.00% 15.12 MiB/s 5s + 2022-12-02 16:12:10 ✔ [SUCCESS] [githubactions-golang-darwin-arm64_0.10.1.so] download succeeded. + 2022-12-02 16:12:10 ℹ [INFO] Downloading: [githubactions-golang-darwin-arm64_0.10.1.md5] ... + 33 B / 33 B [==========================================] 100.00% 71.24 KiB/s 0s + 2022-12-02 16:12:10 ✔ [SUCCESS] [githubactions-golang-darwin-arm64_0.10.1.md5] download succeeded. + 2022-12-02 16:12:11 ℹ [INFO] Initialize [githubactions-golang-darwin-arm64_0.10.1] finished. + 2022-12-02 16:12:11 ℹ [INFO] -------------------- [ githubactions-golang-darwin-arm64_0.10.1 ] -------------------- + 2022-12-02 16:12:11 ✔ [SUCCESS] Initialize finished. + ``` + +--- ## 4 Apply Run: ```shell -./dtm apply -f quickstart.yaml +./dtm apply -y ``` -When it prompts: - -```shell -...(omitted) -Continue? [y/n] -Enter a value (Default is n): -``` - -input `y` and hit the enter key. - -You should see similar output to the following - -``` -2022-06-30 11:25:47 ℹ [INFO] Apply started. -2022-06-30 11:25:47 ℹ [INFO] Got Backend from config: local -2022-06-30 11:25:47 ℹ [INFO] Using dir <.devstream> to store plugins. -2022-06-30 11:25:47 ℹ [INFO] Using local backend. State file: devstream.state. -2022-06-30 11:25:47 ℹ [INFO] Tool (repo-scaffolding/default) found in config but doesn't exist in the state, will be created. -2022-06-30 11:25:47 ℹ [INFO] Tool (githubactions-golang/default) found in config but doesn't exist in the state, will be created. -Continue? [y/n] -Enter a value (Default is n): y - -2022-06-30 11:26:20 ℹ [INFO] Start executing the plan. -2022-06-30 11:26:20 ℹ [INFO] Changes count: 2. -2022-06-30 11:26:20 ℹ [INFO] -------------------- [ Processing progress: 1/2. ] -------------------- -2022-06-30 11:26:20 ℹ [INFO] Processing: (repo-scaffolding/default) -> Create ... -2022-06-30 11:26:24 ℹ [INFO] The repo go-webapp-devstream-demo has been created. -2022-06-30 11:26:37 ✔ [SUCCESS] Tool (repo-scaffolding/default) Create done. -2022-06-30 11:26:37 ℹ [INFO] -------------------- [ Processing progress: 2/2. ] -------------------- -2022-06-30 11:26:37 ℹ [INFO] Processing: (githubactions-golang/default) -> Create ... -2022-06-30 11:26:38 ℹ [INFO] Creating GitHub Actions workflow pr-builder.yml ... -2022-06-30 11:26:38 ✔ [SUCCESS] Github Actions workflow pr-builder.yml created. -2022-06-30 11:26:38 ℹ [INFO] Creating GitHub Actions workflow main-builder.yml ... -2022-06-30 11:26:39 ✔ [SUCCESS] Github Actions workflow main-builder.yml created. -2022-06-30 11:26:39 ✔ [SUCCESS] Tool (githubactions-golang/default) Create done. -2022-06-30 11:26:39 ℹ [INFO] -------------------- [ Processing done. ] -------------------- -2022-06-30 11:26:39 ✔ [SUCCESS] All plugins applied successfully. -2022-06-30 11:26:39 ✔ [SUCCESS] Apply finished. -``` +!!! success "You should see similar output to the following" + + ```text title="" + 2022-12-02 16:18:00 ℹ [INFO] Apply started. + 2022-12-02 16:18:00 ℹ [INFO] Using local backend. State file: devstream.state. + 2022-12-02 16:18:00 ℹ [INFO] Tool (repo-scaffolding/golang-github) found in config but doesn't exist in the state, will be created. + 2022-12-02 16:18:00 ℹ [INFO] Tool (githubactions-golang/default) found in config but doesn't exist in the state, will be created. + 2022-12-02 16:18:00 ℹ [INFO] Start executing the plan. + 2022-12-02 16:18:00 ℹ [INFO] Changes count: 2. + 2022-12-02 16:18:00 ℹ [INFO] -------------------- [ Processing progress: 1/2. ] -------------------- + 2022-12-02 16:18:00 ℹ [INFO] Processing: (repo-scaffolding/golang-github) -> Create ... + 2022-12-02 16:18:00 ℹ [INFO] github start to download repoTemplate... + 2022-12-02 16:18:04 ✔ [SUCCESS] The repo go-webapp-devstream-demo has been created. + 2022-12-02 16:18:12 ✔ [SUCCESS] Tool (repo-scaffolding/golang-github) Create done. + 2022-12-02 16:18:12 ℹ [INFO] -------------------- [ Processing progress: 2/2. ] -------------------- + 2022-12-02 16:18:12 ℹ [INFO] Processing: (githubactions-golang/default) -> Create ... + 2022-12-02 16:18:13 ℹ [INFO] Creating GitHub Actions workflow pr-builder.yml ... + 2022-12-02 16:18:14 ✔ [SUCCESS] Github Actions workflow pr-builder.yml created. + 2022-12-02 16:18:14 ℹ [INFO] Creating GitHub Actions workflow main-builder.yml ... + 2022-12-02 16:18:15 ✔ [SUCCESS] Github Actions workflow main-builder.yml created. + 2022-12-02 16:18:15 ✔ [SUCCESS] Tool (githubactions-golang/default) Create done. + 2022-12-02 16:18:15 ℹ [INFO] -------------------- [ Processing done. ] -------------------- + 2022-12-02 16:18:15 ✔ [SUCCESS] All plugins applied successfully. + 2022-12-02 16:18:15 ✔ [SUCCESS] Apply finished. + ``` + +--- ## 5 Check the Results @@ -165,46 +149,48 @@ The commits (made by DevStream when scaffolding the repo and creating workflows) ![](./images/repo-scaffolding.png) +--- + ## 6 Clean Up Run: ```shell -./dtm delete -f quickstart.yaml +./dtm delete ``` -input `y` the same just like you did in the previous steps, and you should see similar output: - -``` -2022-06-30 11:31:01 ℹ [INFO] Delete started. -2022-06-30 11:31:01 ℹ [INFO] Got Backend from config: local -2022-06-30 11:31:01 ℹ [INFO] Using dir <.devstream> to store plugins. -2022-06-30 11:31:01 ℹ [INFO] Using local backend. State file: devstream.state. -2022-06-30 11:31:01 ℹ [INFO] Tool (githubactions-golang/default) will be deleted. -2022-06-30 11:31:01 ℹ [INFO] Tool (repo-scaffolding/default) will be deleted. -Continue? [y/n] -Enter a value (Default is n): y - -2022-06-30 11:31:03 ℹ [INFO] Start executing the plan. -2022-06-30 11:31:03 ℹ [INFO] Changes count: 2. -2022-06-30 11:31:03 ℹ [INFO] -------------------- [ Processing progress: 1/2. ] -------------------- -2022-06-30 11:31:03 ℹ [INFO] Processing: (githubactions-golang/default) -> Delete ... -2022-06-30 11:31:04 ℹ [INFO] Deleting GitHub Actions workflow pr-builder.yml ... -2022-06-30 11:31:05 ✔ [SUCCESS] GitHub Actions workflow pr-builder.yml removed. -2022-06-30 11:31:05 ℹ [INFO] Deleting GitHub Actions workflow main-builder.yml ... -2022-06-30 11:31:06 ✔ [SUCCESS] GitHub Actions workflow main-builder.yml removed. -2022-06-30 11:31:06 ℹ [INFO] Prepare to delete 'githubactions-golang_default' from States. -2022-06-30 11:31:06 ✔ [SUCCESS] Tool (githubactions-golang/default) delete done. -2022-06-30 11:31:06 ℹ [INFO] -------------------- [ Processing progress: 2/2. ] -------------------- -2022-06-30 11:31:06 ℹ [INFO] Processing: (repo-scaffolding/default) -> Delete ... -2022-06-30 11:31:06 ✔ [SUCCESS] GitHub repo go-webapp-devstream-demo removed. -2022-06-30 11:31:06 ℹ [INFO] Prepare to delete 'repo-scaffolding_default' from States. -2022-06-30 11:31:06 ✔ [SUCCESS] Tool (repo-scaffolding/default) delete done. -2022-06-30 11:31:06 ℹ [INFO] -------------------- [ Processing done. ] -------------------- -2022-06-30 11:31:06 ✔ [SUCCESS] All plugins deleted successfully. -2022-06-30 11:31:06 ✔ [SUCCESS] Delete finished. -``` +Input `y` then press enter to continue, and you should see similar output: + +!!! success "Output" + + ```text title="" + 2022-12-02 16:19:07 ℹ [INFO] Delete started. + 2022-12-02 16:19:07 ℹ [INFO] Using local backend. State file: devstream.state. + 2022-12-02 16:19:07 ℹ [INFO] Tool (githubactions-golang/default) will be deleted. + 2022-12-02 16:19:07 ℹ [INFO] Tool (repo-scaffolding/golang-github) will be deleted. + Continue? [y/n] + Enter a value (Default is n): y + + 2022-12-02 16:19:08 ℹ [INFO] Start executing the plan. + 2022-12-02 16:19:08 ℹ [INFO] Changes count: 2. + 2022-12-02 16:19:08 ℹ [INFO] -------------------- [ Processing progress: 1/2. ] -------------------- + 2022-12-02 16:19:08 ℹ [INFO] Processing: (githubactions-golang/default) -> Delete ... + 2022-12-02 16:19:09 ℹ [INFO] Deleting GitHub Actions workflow pr-builder.yml ... + 2022-12-02 16:19:09 ✔ [SUCCESS] GitHub Actions workflow pr-builder.yml removed. + 2022-12-02 16:19:10 ℹ [INFO] Deleting GitHub Actions workflow main-builder.yml ... + 2022-12-02 16:19:10 ✔ [SUCCESS] GitHub Actions workflow main-builder.yml removed. + 2022-12-02 16:19:10 ℹ [INFO] Prepare to delete 'githubactions-golang_default' from States. + 2022-12-02 16:19:10 ✔ [SUCCESS] Tool (githubactions-golang/default) delete done. + 2022-12-02 16:19:10 ℹ [INFO] -------------------- [ Processing progress: 2/2. ] -------------------- + 2022-12-02 16:19:10 ℹ [INFO] Processing: (repo-scaffolding/golang-github) -> Delete ... + 2022-12-02 16:19:11 ✔ [SUCCESS] GitHub repo go-webapp-devstream-demo removed. + 2022-12-02 16:19:11 ℹ [INFO] Prepare to delete 'repo-scaffolding_golang-github' from States. + 2022-12-02 16:19:11 ✔ [SUCCESS] Tool (repo-scaffolding/golang-github) delete done. + 2022-12-02 16:19:11 ℹ [INFO] -------------------- [ Processing done. ] -------------------- + 2022-12-02 16:19:11 ✔ [SUCCESS] All plugins deleted successfully. + 2022-12-02 16:19:11 ✔ [SUCCESS] Delete finished. + ``` Now if you check your GitHub repo list again, everything has been nuked by DevStream. Hooray! -> Optional: you can also remove the DevStream state file (which should be empty now) by running: `rm devstream.state`. +You can also remove the DevStream state file (which should be empty now) by running: `rm devstream.state`. diff --git a/docs/quickstart.zh.md b/docs/quickstart.zh.md index 7ad4ca749..8ba65c372 100644 --- a/docs/quickstart.zh.md +++ b/docs/quickstart.zh.md @@ -1,27 +1,34 @@ # 快速开始 -如果你更喜欢看 DevStream 的实际操作，请先观看[演示视频](./index.zh.md)。 +我们将在本文使用 DevStream 自动完成以下操作： -> 注意：DevStream 目前只有 Linux 和 macOS 版本，Windows 将在以后支持。 +- 创建一个包含了 web 应用程序的 GitHub 仓库，代码基于 [gin](https://github.com/gin-gonic/gin) 框架（用Go语言编写）自动生成； +- 为前面创建的仓库设置 GitHub Actions 工作流。 -在这个快速开始的示例中，我们将使用 DevStream 做以下自动化工作： - -1. 使用 Golang 的 web 应用程序脚手架在 GitHub 创建仓库。 -2. 为我们创建的 Golang 应用程序设置 GitHub Actions，包含 Go web 应用程序的构建和测试阶段。 +--- ## 1 下载 进入你的工作目录，运行： ```shell -sh -c "$(curl -fsSL https://raw.githubusercontent.com/devstream-io/devstream/main/hack/install/download.sh)" +sh -c "$(curl -fsSL https://download.devstream.io/download.sh) ``` -这个命令会根据你的操作系统和芯片架构下载对应的 `dtm` 二进制文件到你的工作目录中，并赋予二进制文件执行权限。 +!!! note "提示" + 上面的命令会做以下事情： + + - 检测你的操作系统和芯片架构 + - 找到最新版本的 `dtm` 二进制文件 + - 根据操作系统和架构下载正确的 `dtm` 二进制文件 + - 授予二进制文件执行权限 -> 可选：建议你将 dtm 移动到包含于 PATH 的目录下，比如 `mv dtm /usr/local/bin/`。 +!!! quote "可选" + 你可以将 `dtm` 移到 PATH 中。例如：`mv dtm /usr/local/bin/`。 -_更多安装方式详见[安装dtm](./install.zh.md)。_ + 更多安装方式详见[安装dtm](./install.zh.md)。 + +--- ## 2 配置 @@ -31,12 +38,6 @@ _更多安装方式详见[安装dtm](./install.zh.md)。_ ./dtm show config -t quickstart > config.yaml ``` -正如前文所述，我们将在 GitHub Actions 中操作 GitHub 仓库的脚手架和 CI 工作流。所以，我们需要设置以下环境变量： - -- GITHUB_USER -- GITHUB_TOKEN -- DOCKERHUB_USERNAME - 运行以下命令以设置这些环境变量（记得替换双引号内的值）： ```shell @@ -45,124 +46,102 @@ export GITHUB_TOKEN="" export DOCKERHUB_USERNAME="" ``` -> 小贴士：前往 [Personal Access Token](https://github.com/settings/tokens/new) 为 `dtm` 生成新的 `GITHUB_TOKEN`。 -> -> 对于“快速开始”，我们只需要勾选 `repo`、`workflow`、`delete_repo` 权限，但我们更建议你全部勾选，未来的插件可能需要更多权限。 +!!! tip "提示" + 前往 [Personal Access Token](https://github.com/settings/tokens/new) 为 `dtm` 生成新的 `GITHUB_TOKEN`。 + + 对于“快速开始”，我们只需要勾选 `repo`、`workflow`、`delete_repo` 权限。 接着，让我们运行以下命令，以使用环境变量来修改配置文件： -对于 **macOS** 或 基于 **FreeBSD** 的操作系统： +=== "**macOS** 或 基于 **FreeBSD** 的操作系统" + ```shell title="" + sed -i.bak "s@YOUR_GITHUB_USERNAME_CASE_SENSITIVE@${GITHUB_USER}@g" config.yaml + sed -i.bak "s@YOUR_DOCKER_USERNAME@${DOCKERHUB_USERNAME}@g" config.yaml + ``` -```shell -sed -i.bak "s@YOUR_GITHUB_USERNAME_CASE_SENSITIVE@${GITHUB_USER}@g" quickstart.yaml -sed -i.bak "s@YOUR_DOCKER_USERNAME@${DOCKERHUB_USERNAME}@g" quickstart.yaml -``` - -对于 **GNU** Linux 用户: - -```shell -sed -i "s@YOUR_GITHUB_USERNAME_CASE_SENSITIVE@${GITHUB_USER}@g" quickstart.yaml -sed -i "s@YOUR_DOCKER_USERNAME@${DOCKERHUB_USERNAME}@g" quickstart.yaml -``` +=== "**GNU** Linux 用户" + ```shell title="" + sed -i "s@YOUR_GITHUB_USERNAME_CASE_SENSITIVE@${GITHUB_USER}@g" config.yaml + sed -i "s@YOUR_DOCKER_USERNAME@${DOCKERHUB_USERNAME}@g" config.yaml + ``` +--- ## 3 初始化 运行： ```shell -./dtm init -f quickstart.yaml +./dtm init ``` -你会看到类似这样的日志输出： - -``` -2022-06-30 11:21:48 ℹ [INFO] Got Backend from config: local -2022-06-30 11:21:48 ℹ [INFO] Using dir <.devstream> to store plugins. -2022-06-30 11:21:48 ℹ [INFO] Downloading: [github-repo-scaffolding-golang-darwin-arm64_0.7.0.so] ... - 15.05 MiB / 15.05 MiB [================================] 100.00% 21.17 MiB/s 0s -2022-06-30 11:21:49 ✔ [SUCCESS] [github-repo-scaffolding-golang-darwin-arm64_0.7.0.so] download succeeded. -2022-06-30 11:21:49 ℹ [INFO] Downloading: [github-repo-scaffolding-golang-darwin-arm64_0.7.0.md5] ... - 33 B / 33 B [==========================================] 100.00% 35.29 KiB/s 0s -2022-06-30 11:21:49 ✔ [SUCCESS] [github-repo-scaffolding-golang-darwin-arm64_0.7.0.md5] download succeeded. -2022-06-30 11:21:49 ℹ [INFO] Plugin: github-repo-scaffolding-golang-darwin-arm64_0.7.0.so doesn't match with .md5 and will be downloaded. -2022-06-30 11:21:49 ℹ [INFO] Downloading: [github-repo-scaffolding-golang-darwin-arm64_0.7.0.so] ... - 15.05 MiB / 15.05 MiB [================================] 100.00% 31.25 MiB/s 0s -2022-06-30 11:21:50 ✔ [SUCCESS] [github-repo-scaffolding-golang-darwin-arm64_0.7.0.so] download succeeded. -2022-06-30 11:21:50 ℹ [INFO] Downloading: [github-repo-scaffolding-golang-darwin-arm64_0.7.0.md5] ... - 33 B / 33 B [==========================================] 100.00% 43.43 KiB/s 0s -2022-06-30 11:21:50 ✔ [SUCCESS] [github-repo-scaffolding-golang-darwin-arm64_0.7.0.md5] download succeeded. -2022-06-30 11:21:50 ℹ [INFO] Downloading: [githubactions-golang-darwin-arm64_0.7.0.so] ... - 17.49 MiB / 17.49 MiB [================================] 100.00% 31.18 MiB/s 0s -2022-06-30 11:21:51 ✔ [SUCCESS] [githubactions-golang-darwin-arm64_0.7.0.so] download succeeded. -2022-06-30 11:21:51 ℹ [INFO] Downloading: [githubactions-golang-darwin-arm64_0.7.0.md5] ... - 33 B / 33 B [=========================================] 100.00% 160.70 KiB/s 0s -2022-06-30 11:21:51 ✔ [SUCCESS] [githubactions-golang-darwin-arm64_0.7.0.md5] download succeeded. -2022-06-30 11:21:51 ℹ [INFO] Plugin: githubactions-golang-darwin-arm64_0.7.0.so doesn't match with .md5 and will be downloaded. -2022-06-30 11:21:51 ℹ [INFO] Downloading: [githubactions-golang-darwin-arm64_0.7.0.so] ... - 17.49 MiB / 17.49 MiB [================================] 100.00% 31.78 MiB/s 0s -2022-06-30 11:21:52 ✔ [SUCCESS] [githubactions-golang-darwin-arm64_0.7.0.so] download succeeded. -2022-06-30 11:21:52 ℹ [INFO] Downloading: [githubactions-golang-darwin-arm64_0.7.0.md5] ... - 33 B / 33 B [==========================================] 100.00% 87.12 KiB/s 0s -2022-06-30 11:21:52 ✔ [SUCCESS] [githubactions-golang-darwin-arm64_0.7.0.md5] download succeeded. -2022-06-30 11:21:52 ✔ [SUCCESS] Initialize finished. -``` +!!! success "你会看到类似下面的输出" + ``` title="" + 2022-12-02 16:11:55 ℹ [INFO] Using dir to store plugins. + 2022-12-02 16:11:55 ℹ [INFO] -------------------- [ repo-scaffolding-darwin-arm64_0.10.1 ] -------------------- + 2022-12-02 16:11:57 ℹ [INFO] Downloading: [repo-scaffolding-darwin-arm64_0.10.1.so] ... + 87.82 MiB / 87.82 MiB [================================] 100.00% 12.30 MiB/s 7s + 2022-12-02 16:12:04 ✔ [SUCCESS] [repo-scaffolding-darwin-arm64_0.10.1.so] download succeeded. + 2022-12-02 16:12:04 ℹ [INFO] Downloading: [repo-scaffolding-darwin-arm64_0.10.1.md5] ... + 33 B / 33 B [==========================================] 100.00% 50.98 KiB/s 0s + 2022-12-02 16:12:04 ✔ [SUCCESS] [repo-scaffolding-darwin-arm64_0.10.1.md5] download succeeded. + 2022-12-02 16:12:04 ℹ [INFO] Initialize [repo-scaffolding-darwin-arm64_0.10.1] finished. + 2022-12-02 16:12:04 ℹ [INFO] -------------------- [ repo-scaffolding-darwin-arm64_0.10.1 ] -------------------- + 2022-12-02 16:12:04 ℹ [INFO] -------------------- [ githubactions-golang-darwin-arm64_0.10.1 ] -------------------- + 2022-12-02 16:12:05 ℹ [INFO] Downloading: [githubactions-golang-darwin-arm64_0.10.1.so] ... + 86.44 MiB / 86.44 MiB [================================] 100.00% 15.12 MiB/s 5s + 2022-12-02 16:12:10 ✔ [SUCCESS] [githubactions-golang-darwin-arm64_0.10.1.so] download succeeded. + 2022-12-02 16:12:10 ℹ [INFO] Downloading: [githubactions-golang-darwin-arm64_0.10.1.md5] ... + 33 B / 33 B [==========================================] 100.00% 71.24 KiB/s 0s + 2022-12-02 16:12:10 ✔ [SUCCESS] [githubactions-golang-darwin-arm64_0.10.1.md5] download succeeded. + 2022-12-02 16:12:11 ℹ [INFO] Initialize [githubactions-golang-darwin-arm64_0.10.1] finished. + 2022-12-02 16:12:11 ℹ [INFO] -------------------- [ githubactions-golang-darwin-arm64_0.10.1 ] -------------------- + 2022-12-02 16:12:11 ✔ [SUCCESS] Initialize finished. + ``` + +--- ## 4 应用（Apply） 运行： ```shell -./dtm apply -f quickstart.yaml +./dtm apply -y ``` -当它提示： - -```shell -...(以上省略) -Continue? [y/n] -Enter a value (Default is n): -``` - -请输入 `y` 并按回车键。 - -你会看到类似下面的输出： - -``` -2022-06-30 11:25:47 ℹ [INFO] Apply started. -2022-06-30 11:25:47 ℹ [INFO] Got Backend from config: local -2022-06-30 11:25:47 ℹ [INFO] Using dir <.devstream> to store plugins. -2022-06-30 11:25:47 ℹ [INFO] Using local backend. State file: devstream.state. -2022-06-30 11:25:47 ℹ [INFO] Tool (github-repo-scaffolding-golang/default) found in config but doesn't exist in the state, will be created. -2022-06-30 11:25:47 ℹ [INFO] Tool (githubactions-golang/default) found in config but doesn't exist in the state, will be created. -Continue? [y/n] -Enter a value (Default is n): y - -2022-06-30 11:26:20 ℹ [INFO] Start executing the plan. -2022-06-30 11:26:20 ℹ [INFO] Changes count: 2. -2022-06-30 11:26:20 ℹ [INFO] -------------------- [ Processing progress: 1/2. ] -------------------- -2022-06-30 11:26:20 ℹ [INFO] Processing: (github-repo-scaffolding-golang/default) -> Create ... -2022-06-30 11:26:24 ℹ [INFO] The repo go-webapp-devstream-demo has been created. -2022-06-30 11:26:37 ✔ [SUCCESS] Tool (github-repo-scaffolding-golang/default) Create done. -2022-06-30 11:26:37 ℹ [INFO] -------------------- [ Processing progress: 2/2. ] -------------------- -2022-06-30 11:26:37 ℹ [INFO] Processing: (githubactions-golang/default) -> Create ... -2022-06-30 11:26:38 ℹ [INFO] Creating GitHub Actions workflow pr-builder.yml ... -2022-06-30 11:26:38 ✔ [SUCCESS] Github Actions workflow pr-builder.yml created. -2022-06-30 11:26:38 ℹ [INFO] Creating GitHub Actions workflow main-builder.yml ... -2022-06-30 11:26:39 ✔ [SUCCESS] Github Actions workflow main-builder.yml created. -2022-06-30 11:26:39 ✔ [SUCCESS] Tool (githubactions-golang/default) Create done. -2022-06-30 11:26:39 ℹ [INFO] -------------------- [ Processing done. ] -------------------- -2022-06-30 11:26:39 ✔ [SUCCESS] All plugins applied successfully. -2022-06-30 11:26:39 ✔ [SUCCESS] Apply finished. -``` - - +!!! success "你会看到类似下面的输出" + + ```text title="" + 2022-12-02 16:18:00 ℹ [INFO] Apply started. + 2022-12-02 16:18:00 ℹ [INFO] Using local backend. State file: devstream.state. + 2022-12-02 16:18:00 ℹ [INFO] Tool (repo-scaffolding/golang-github) found in config but doesn't exist in the state, will be created. + 2022-12-02 16:18:00 ℹ [INFO] Tool (githubactions-golang/default) found in config but doesn't exist in the state, will be created. + 2022-12-02 16:18:00 ℹ [INFO] Start executing the plan. + 2022-12-02 16:18:00 ℹ [INFO] Changes count: 2. + 2022-12-02 16:18:00 ℹ [INFO] -------------------- [ Processing progress: 1/2. ] -------------------- + 2022-12-02 16:18:00 ℹ [INFO] Processing: (repo-scaffolding/golang-github) -> Create ... + 2022-12-02 16:18:00 ℹ [INFO] github start to download repoTemplate... + 2022-12-02 16:18:04 ✔ [SUCCESS] The repo go-webapp-devstream-demo has been created. + 2022-12-02 16:18:12 ✔ [SUCCESS] Tool (repo-scaffolding/golang-github) Create done. + 2022-12-02 16:18:12 ℹ [INFO] -------------------- [ Processing progress: 2/2. ] -------------------- + 2022-12-02 16:18:12 ℹ [INFO] Processing: (githubactions-golang/default) -> Create ... + 2022-12-02 16:18:13 ℹ [INFO] Creating GitHub Actions workflow pr-builder.yml ... + 2022-12-02 16:18:14 ✔ [SUCCESS] Github Actions workflow pr-builder.yml created. + 2022-12-02 16:18:14 ℹ [INFO] Creating GitHub Actions workflow main-builder.yml ... + 2022-12-02 16:18:15 ✔ [SUCCESS] Github Actions workflow main-builder.yml created. + 2022-12-02 16:18:15 ✔ [SUCCESS] Tool (githubactions-golang/default) Create done. + 2022-12-02 16:18:15 ℹ [INFO] -------------------- [ Processing done. ] -------------------- + 2022-12-02 16:18:15 ✔ [SUCCESS] All plugins applied successfully. + 2022-12-02 16:18:15 ✔ [SUCCESS] Apply finished. + ``` + +--- ## 5 检查结果 -前往你的 GitHub 仓库列表，你可以看到一个新的仓库 `go-webapp-devstream-demo` 已经被创建了。 +前往你的 GitHub 仓库列表，可以看到一个新的仓库 `go-webapp-devstream-demo` 已经被创建了。 包含了 Golang web 应用程序的脚手架代码，并正确设置了 GitHub Actions CI 工作流。 @@ -170,46 +149,47 @@ DevStream 在生成仓库脚手架和创建工作流时的代码提交，已经 ![](./images/repo-scaffolding.png) +--- + ## 6 清理 运行： ```shell -./dtm delete -f quickstart.yaml +./dtm delete ``` -像之前那样，在收到提示时输入 `y`，你将会看到类似输出： - -``` -2022-06-30 11:31:01 ℹ [INFO] Delete started. -2022-06-30 11:31:01 ℹ [INFO] Got Backend from config: local -2022-06-30 11:31:01 ℹ [INFO] Using dir <.devstream> to store plugins. -2022-06-30 11:31:01 ℹ [INFO] Using local backend. State file: devstream.state. -2022-06-30 11:31:01 ℹ [INFO] Tool (githubactions-golang/default) will be deleted. -2022-06-30 11:31:01 ℹ [INFO] Tool (github-repo-scaffolding-golang/default) will be deleted. -Continue? [y/n] -Enter a value (Default is n): y - -2022-06-30 11:31:03 ℹ [INFO] Start executing the plan. -2022-06-30 11:31:03 ℹ [INFO] Changes count: 2. -2022-06-30 11:31:03 ℹ [INFO] -------------------- [ Processing progress: 1/2. ] -------------------- -2022-06-30 11:31:03 ℹ [INFO] Processing: (githubactions-golang/default) -> Delete ... -2022-06-30 11:31:04 ℹ [INFO] Deleting GitHub Actions workflow pr-builder.yml ... -2022-06-30 11:31:05 ✔ [SUCCESS] GitHub Actions workflow pr-builder.yml removed. -2022-06-30 11:31:05 ℹ [INFO] Deleting GitHub Actions workflow main-builder.yml ... -2022-06-30 11:31:06 ✔ [SUCCESS] GitHub Actions workflow main-builder.yml removed. -2022-06-30 11:31:06 ℹ [INFO] Prepare to delete 'githubactions-golang_default' from States. -2022-06-30 11:31:06 ✔ [SUCCESS] Tool (githubactions-golang/default) delete done. -2022-06-30 11:31:06 ℹ [INFO] -------------------- [ Processing progress: 2/2. ] -------------------- -2022-06-30 11:31:06 ℹ [INFO] Processing: (github-repo-scaffolding-golang/default) -> Delete ... -2022-06-30 11:31:06 ✔ [SUCCESS] GitHub repo go-webapp-devstream-demo removed. -2022-06-30 11:31:06 ℹ [INFO] Prepare to delete 'github-repo-scaffolding-golang_default' from States. -2022-06-30 11:31:06 ✔ [SUCCESS] Tool (github-repo-scaffolding-golang/default) delete done. -2022-06-30 11:31:06 ℹ [INFO] -------------------- [ Processing done. ] -------------------- -2022-06-30 11:31:06 ✔ [SUCCESS] All plugins deleted successfully. -2022-06-30 11:31:06 ✔ [SUCCESS] Delete finished. -``` +输入 `y` 然后回车，你会看到类似下面的输出： + +!!! success "输出" + ```title="" + 2022-12-02 16:19:07 ℹ [INFO] Delete started. + 2022-12-02 16:19:07 ℹ [INFO] Using local backend. State file: devstream.state. + 2022-12-02 16:19:07 ℹ [INFO] Tool (githubactions-golang/default) will be deleted. + 2022-12-02 16:19:07 ℹ [INFO] Tool (repo-scaffolding/golang-github) will be deleted. + Continue? [y/n] + Enter a value (Default is n): y + + 2022-12-02 16:19:08 ℹ [INFO] Start executing the plan. + 2022-12-02 16:19:08 ℹ [INFO] Changes count: 2. + 2022-12-02 16:19:08 ℹ [INFO] -------------------- [ Processing progress: 1/2. ] -------------------- + 2022-12-02 16:19:08 ℹ [INFO] Processing: (githubactions-golang/default) -> Delete ... + 2022-12-02 16:19:09 ℹ [INFO] Deleting GitHub Actions workflow pr-builder.yml ... + 2022-12-02 16:19:09 ✔ [SUCCESS] GitHub Actions workflow pr-builder.yml removed. + 2022-12-02 16:19:10 ℹ [INFO] Deleting GitHub Actions workflow main-builder.yml ... + 2022-12-02 16:19:10 ✔ [SUCCESS] GitHub Actions workflow main-builder.yml removed. + 2022-12-02 16:19:10 ℹ [INFO] Prepare to delete 'githubactions-golang_default' from States. + 2022-12-02 16:19:10 ✔ [SUCCESS] Tool (githubactions-golang/default) delete done. + 2022-12-02 16:19:10 ℹ [INFO] -------------------- [ Processing progress: 2/2. ] -------------------- + 2022-12-02 16:19:10 ℹ [INFO] Processing: (repo-scaffolding/golang-github) -> Delete ... + 2022-12-02 16:19:11 ✔ [SUCCESS] GitHub repo go-webapp-devstream-demo removed. + 2022-12-02 16:19:11 ℹ [INFO] Prepare to delete 'repo-scaffolding_golang-github' from States. + 2022-12-02 16:19:11 ✔ [SUCCESS] Tool (repo-scaffolding/golang-github) delete done. + 2022-12-02 16:19:11 ℹ [INFO] -------------------- [ Processing done. ] -------------------- + 2022-12-02 16:19:11 ✔ [SUCCESS] All plugins deleted successfully. + 2022-12-02 16:19:11 ✔ [SUCCESS] Delete finished. + ``` 现在，如果你看看 GitHub 仓库列表，所有东西都被 DevStream 消灭了。妙哉！ -> 可选：你也可以通过运行：`rm devstream.state` 来删除 DevStream 状态文件（现在应该是个空文件）。 +你也可以通过运行 `rm devstream.state` 来删除 DevStream 状态文件（现在应该是个空文件）。 diff --git a/examples/gitops.yaml b/examples/gitops.yaml index c9b539a10..3c00c7596 100644 --- a/examples/gitops.yaml +++ b/examples/gitops.yaml @@ -30,7 +30,7 @@ tools: repo: dtm-scaffolding-golang repoType: github vars: - ImageRepo: "[[ dockerhubUsername ]]/[[ repoName ]]" + imageRepo: "[[ dockerhubUsername ]]/[[ repoName ]]" - name: jira-github-integ instanceID: default dependsOn: [ "repo-scaffolding.golang-github" ] diff --git a/examples/quickstart.yaml b/examples/quickstart.yaml index 9aa76e307..e7ce1f65f 100644 --- a/examples/quickstart.yaml +++ b/examples/quickstart.yaml @@ -19,7 +19,7 @@ tools: repo: dtm-scaffolding-golang repoType: github vars: - ImageRepo: YOUR_DOCKER_USERNAME/go-webapp-devstream-demo + imageRepo: YOUR_DOCKER_USERNAME/go-webapp-devstream-demo - name: githubactions-golang instanceID: default dependsOn: ["repo-scaffolding.golang-github"] diff --git a/hack/install/download.sh b/hack/install/download.sh old mode 100644 new mode 100755 index b5716a500..af650c61e --- a/hack/install/download.sh +++ b/hack/install/download.sh @@ -1,6 +1,7 @@ #!/bin/bash function init() { + # todo customize dtm version, default is latest if [ "$(uname)" == "Darwin" ];then HOST_OS="darwin" elif [ "$(uname)" == "Linux" ];then @@ -23,23 +24,24 @@ function init() { } function getLatestReleaseVersion() { - if [ -n "${GITHUB_TOKEN}" ]; then - AUTH_HEADER="-H Authorization: token ${GITHUB_TOKEN}" - fi + # get latest release version from aws s3 + STORAGE_BASE_URL=https://download.devstream.io + LATEST_VERSION_FILE="latest_version" + + LATEST_VERSION=$(curl -fsSL ${STORAGE_BASE_URL}/${LATEST_VERSION_FILE}) - # like "v1.2.3" - latestVersion=$(curl ${AUTH_HEADER} -s https://api.github.com/repos/devstream-io/devstream/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') - if [ -z "$latestVersion" ]; then + if [ -z "${LATEST_VERSION}" ];then echo "Failed to get latest release version" exit 1 fi - echo "Latest dtm release version: ${latestVersion}" + + echo "Got latest release version: ${LATEST_VERSION}" } function downloadDtm() { # 1. download the release and rename it to "dtm" # 2. count the download count of the release - fullReleaseUrl="https://devstream.gateway.scarf.sh/releases/$latestVersion/dtm-$HOST_OS-$HOST_ARCH" + fullReleaseUrl="${STORAGE_BASE_URL}/${LATEST_VERSION}/dtm-${HOST_OS}-${HOST_ARCH}" echo "Downloading dtm from: $fullReleaseUrl" # use -L to follow redirects curl -L -o dtm $fullReleaseUrl diff --git a/hack/release/auto-release-darwin-arm64.sh b/hack/release/auto-release-darwin-arm64.sh index 2981e00e7..3ed6ddf0d 100755 --- a/hack/release/auto-release-darwin-arm64.sh +++ b/hack/release/auto-release-darwin-arm64.sh @@ -32,6 +32,9 @@ plugin_dir=~/.devstream/plugins GOOS=$(go env GOOS) GOARCH=$(go env GOARCH) +DTM_CORE_BINARY=dtm-${GOOS}-${GOARCH} +STORAGE_BASE_URL=s3://download.devstream.io +STORAGE_URL_WITH_TAG=${STORAGE_BASE_URL}/${tag} if [ ! $tag ] || [ ! $user ] || [ ! $repo ] || [ ! $github_token ] || [ ! $plugin_dir ]; then echo "The following variables cannot be empty!" @@ -56,11 +59,15 @@ make build -j8 go install github.com/github-release/github-release@latest # upload dtm -echo 'Uploading 'dtm-${GOOS}-${GOARCH}' ...' -github-release upload --security-token $github_token --user $user --repo $repo --tag $tag --file dtm --name dtm-${GOOS}-${GOARCH} -echo dtm-${GOOS}-${GOARCH}' uploaded.' +echo "Uploading ${DTM_CORE_BINARY} ..." +# upload dtm to github release +github-release upload --security-token $github_token --user $user --repo $repo --tag $tag --file dtm --name ${DTM_CORE_BINARY} +# upload dtm to aws s3 +aws s3 cp dtm ${STORAGE_URL_WITH_TAG}/${DTM_CORE_BINARY} --acl public-read +echo "${DTM_CORE_BINARY} uploaded." # upload plugins and .md5 files -# In order to upload plug-ins to s3, you need to download awscli. After downloading awscli, you need to configure aws credentials. +# In order to upload plug-ins to s3, you need to download aws cli. +# After downloading aws cli, you need to configure aws credentials. pip3 install awscli -aws s3 cp $plugin_dir s3://download.devstream.io/${tag} --recursive --acl public-read +aws s3 cp $plugin_dir $STORAGE_URL_WITH_TAG --recursive --acl public-read diff --git a/hack/release/update_download_sh.sh b/hack/release/update_download_sh.sh new file mode 100644 index 000000000..f1867b853 --- /dev/null +++ b/hack/release/update_download_sh.sh @@ -0,0 +1,9 @@ +#! /bin/bash -e + +DOWNLOAD_SCRIPT_PATH="hack/install/download.sh" +STORAGE_BASE_URL=s3://download.devstream.io +DOWNLOAD_SCRIPT_S3_URL=${STORAGE_BASE_URL}/${DOWNLOAD_SCRIPT_PATH} + +# upload download.sh to aws s3 +echo "Uploading ${DOWNLOAD_SCRIPT_PATH} to ${DOWNLOAD_SCRIPT_S3_URL} ..." +aws s3 cp ${DOWNLOAD_SCRIPT_PATH} ${DOWNLOAD_SCRIPT_S3_URL} --acl public-read diff --git a/hack/release/update_dtm_version_on_s3.sh b/hack/release/update_dtm_version_on_s3.sh new file mode 100755 index 000000000..6b2d6b4b5 --- /dev/null +++ b/hack/release/update_dtm_version_on_s3.sh @@ -0,0 +1,14 @@ +#! /bin/bash -e + +version=$1 + +LATEST_VERSION_FILE="latest_version" +STORAGE_BASE_URL=s3://download.devstream.io + +# create latest_version file +echo "Saving latest version(${version}) to ${LATEST_VERSION_FILE} ..." +echo $version > ${LATEST_VERSION_FILE} + +# create or update latest_version.txt on s3 +aws s3 cp ${LATEST_VERSION_FILE} ${STORAGE_BASE_URL}/${LATEST_VERSION_FILE} --acl public-read +echo "${LATEST_VERSION_FILE} uploaded." diff --git a/hack/release/upload_assets.sh b/hack/release/upload_assets.sh deleted file mode 100755 index 27b3f6725..000000000 --- a/hack/release/upload_assets.sh +++ /dev/null @@ -1,15 +0,0 @@ -#! /bin/bash -e - -github_token=$1 -tag=$2 -GOOS=$3 -GOARCH=$4 -user=devstream-io -repo=devstream -plugin_dir=~/.devstream/plugins - -# upload dtm -echo 'Uploading 'dtm-${GOOS}-${GOARCH}' ...' -github-release upload --security-token $github_token --user $user --repo $repo --tag $tag --file dtm --name dtm-${GOOS}-${GOARCH} -echo dtm-${GOOS}-${GOARCH}' uploaded.' - diff --git a/hack/release/upload_dtm_core.sh b/hack/release/upload_dtm_core.sh new file mode 100755 index 000000000..633d34397 --- /dev/null +++ b/hack/release/upload_dtm_core.sh @@ -0,0 +1,23 @@ +#! /bin/bash -e + +github_token=$1 +tag=$2 +GOOS=$3 +GOARCH=$4 + +DTM_CORE_BINARY=dtm-${GOOS}-${GOARCH} +STORAGE_BASE_URL=s3://download.devstream.io +STORAGE_URL_WITH_TAG=${STORAGE_BASE_URL}/${tag} + +user=devstream-io +repo=devstream +plugin_dir=~/.devstream/plugins + +# upload dtm core +echo "Uploading ${DTM_CORE_BINARY} ..." +# upload dtm to github release +github-release upload --security-token $github_token --user $user --repo $repo --tag $tag --file dtm --name ${DTM_CORE_BINARY} +# upload dtm to aws s3 +aws s3 cp dtm ${STORAGE_URL_WITH_TAG}/${DTM_CORE_BINARY} --acl public-read +echo "${DTM_CORE_BINARY} uploaded." + diff --git a/internal/pkg/configmanager/configmanager_test.go b/internal/pkg/configmanager/configmanager_test.go index 596c17163..3f1fd60cc 100644 --- a/internal/pkg/configmanager/configmanager_test.go +++ b/internal/pkg/configmanager/configmanager_test.go @@ -144,7 +144,7 @@ var _ = Describe("LoadConfig", func() { }, }, "branch": "main", - "configLocation": "git@github.com:devstream-io/ci-template.git//github-actions", + "configLocation": "https://raw.githubusercontent.com/devstream-io/ci-template/main/github-actions/workflows/main.yml", }, "scm": RawOptions{ "apiURL": "gitlab.com/some/path/to/your/api", diff --git a/internal/pkg/configmanager/pipelineDefault.go b/internal/pkg/configmanager/pipelineDefault.go index c2826caff..51bc19536 100644 --- a/internal/pkg/configmanager/pipelineDefault.go +++ b/internal/pkg/configmanager/pipelineDefault.go @@ -24,7 +24,7 @@ type pipelineOption struct { var ( // github actions pipeline options githubGeneral = pipelineOption{ - defaultConfigLocation: "git@github.com:devstream-io/ci-template.git//github-actions", + defaultConfigLocation: "https://raw.githubusercontent.com/devstream-io/ci-template/main/github-actions/workflows/main.yml", optionGeneratorFunc: pipelineGeneralGenerator, } gitlabGeneral = pipelineOption{ diff --git a/internal/pkg/configmanager/pipelinetemplate_test.go b/internal/pkg/configmanager/pipelinetemplate_test.go index 1e35abe81..30c36b6a9 100644 --- a/internal/pkg/configmanager/pipelinetemplate_test.go +++ b/internal/pkg/configmanager/pipelinetemplate_test.go @@ -231,7 +231,7 @@ var _ = Describe("PipelineTemplate struct", func() { Options: RawOptions{ "pipeline": RawOptions{ "test": "testV", - "configLocation": "git@github.com:devstream-io/ci-template.git//github-actions", + "configLocation": "https://raw.githubusercontent.com/devstream-io/ci-template/main/github-actions/workflows/main.yml", }, "scm": RawOptions{ "url": cloneURL, diff --git a/internal/pkg/plugin/argocdapp/argocdapp.go b/internal/pkg/plugin/argocdapp/argocdapp.go index b6c939681..48342f61c 100644 --- a/internal/pkg/plugin/argocdapp/argocdapp.go +++ b/internal/pkg/plugin/argocdapp/argocdapp.go @@ -2,7 +2,53 @@ package argocdapp import ( _ "embed" + "fmt" + + "github.com/devstream-io/devstream/internal/pkg/configmanager" + "github.com/devstream-io/devstream/pkg/util/downloader" + "github.com/devstream-io/devstream/pkg/util/scm" + "github.com/devstream-io/devstream/pkg/util/scm/git" ) -//go:embed tpl/argocd.tpl.yaml -var templateFileLoc string +//go:embed tpl/helm.tpl.yaml +var helmApplicationConfig string + +func pushArgocdConfigFiles(rawOptions configmanager.RawOptions) error { + const createCommitMsg = "create argocdapp config files" + const commitBranch = "feat-argocdapp-configs" + opts, err := newOptions(rawOptions) + if err != nil { + return err + } + + // 1. init scm client for check argocdapp config exists and push argocdapp files + scmInfo := scm.SCMInfo{CloneURL: opts.Source.RepoURL} + scmClient, err := scmInfo.NewClientWithAuthFromScm() + if err != nil { + return err + } + + // 2. check argocdapp config existed in project repo + configFileExist, err := opts.Source.checkPathExist(scmClient) + if err != nil { + return fmt.Errorf("argocdapp scm client get config path info failed: %w", err) + } + if configFileExist { + return nil + } + + // 3. get argocd configFiles from remote + const configLocation = downloader.ResourceLocation("https://github.com/devstream-io/ci-template.git//argocdapp/helm") + gitFiles, err := opts.getArgocdDefaultConfigFiles(configLocation) + if err != nil { + return err + } + + // 4. push git files to ProjectRepo + _, err = scmClient.PushFiles(&git.CommitInfo{ + CommitMsg: createCommitMsg, + GitFileMap: gitFiles, + CommitBranch: commitBranch, + }, true) + return err +} diff --git a/internal/pkg/plugin/argocdapp/argocdapp_suite_test.go b/internal/pkg/plugin/argocdapp/argocdapp_suite_test.go new file mode 100644 index 000000000..974ce657a --- /dev/null +++ b/internal/pkg/plugin/argocdapp/argocdapp_suite_test.go @@ -0,0 +1,13 @@ +package argocdapp_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestCommon(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Plugin Argocdapp Suite") +} diff --git a/internal/pkg/plugin/argocdapp/create.go b/internal/pkg/plugin/argocdapp/create.go index 18e49e59f..6abb07a26 100644 --- a/internal/pkg/plugin/argocdapp/create.go +++ b/internal/pkg/plugin/argocdapp/create.go @@ -17,7 +17,8 @@ func Create(options configmanager.RawOptions) (statemanager.ResourceStatus, erro validate, }, ExecuteOperations: installer.ExecuteOperations{ - kubectl.ProcessByContent(kubectlUtil.Create, templateFileLoc), + pushArgocdConfigFiles, + kubectl.ProcessByContent(kubectlUtil.Create, helmApplicationConfig), }, GetStatusOperation: getStaticStatus, } diff --git a/internal/pkg/plugin/argocdapp/delete.go b/internal/pkg/plugin/argocdapp/delete.go index fc4ebb114..1f4a20f1c 100644 --- a/internal/pkg/plugin/argocdapp/delete.go +++ b/internal/pkg/plugin/argocdapp/delete.go @@ -14,7 +14,7 @@ func Delete(options configmanager.RawOptions) (bool, error) { validate, }, ExecuteOperations: installer.ExecuteOperations{ - kubectl.ProcessByContent(kubectlUtil.Delete, templateFileLoc), + kubectl.ProcessByContent(kubectlUtil.Delete, helmApplicationConfig), }, } diff --git a/internal/pkg/plugin/argocdapp/options.go b/internal/pkg/plugin/argocdapp/options.go index f964d576e..01264ddd5 100644 --- a/internal/pkg/plugin/argocdapp/options.go +++ b/internal/pkg/plugin/argocdapp/options.go @@ -1,42 +1,90 @@ package argocdapp import ( + "path/filepath" + "github.com/mitchellh/mapstructure" "github.com/devstream-io/devstream/internal/pkg/configmanager" + "github.com/devstream-io/devstream/pkg/util/downloader" + "github.com/devstream-io/devstream/pkg/util/file" + "github.com/devstream-io/devstream/pkg/util/log" + "github.com/devstream-io/devstream/pkg/util/scm" + "github.com/devstream-io/devstream/pkg/util/scm/git" + "github.com/devstream-io/devstream/pkg/util/template" ) -// Param is the struct for parameters used by the argocdapp package. -type Options struct { - App App - Destination Destination - Source Source +// options is the struct for parameters used by the argocdapp package. +type options struct { + App *app `mapstructure:"app" validate:"required"` + Destination *destination `mapstructure:"destination"` + Source *source `mapstructure:"source" validate:"required"` + ImageRepo imageRepo `mapstructure:"imageRepo"` +} + +type imageRepo struct { + Address string `mapstructure:"address"` + Tag string `mapstructure:"tag"` } -// App is the struct for an ArgoCD app. -type App struct { - Name string `validate:"required,dns1123subdomain"` - Namespace string +// app is the struct for an ArgoCD app. +type app struct { + Name string `mapstructure:"name" validate:"dns1123subdomain"` + Namespace string `mapstructure:"namespace"` } -// Destination is the struct for the destination of an ArgoCD app. -type Destination struct { - Server string - Namespace string +// destination is the struct for the destination of an ArgoCD app. +type destination struct { + Server string `mapstructure:"server"` + Namespace string `mapstructure:"namespace"` } -// Source is the struct for the source of an ArgoCD app. -type Source struct { - Valuefile string - Path string `validate:"required"` - RepoURL string `validate:"required"` +// source is the struct for the source of an ArgoCD app. +type source struct { + Valuefile string `mapstructure:"valuefile"` + Path string `mapstructure:"path" validate:"required"` + RepoURL string `mapstructure:"repoURL" validate:"required"` } -// / NewOptions create options by raw options -func NewOptions(options configmanager.RawOptions) (Options, error) { - var opts Options - if err := mapstructure.Decode(options, &opts); err != nil { +// / newOptions create options by raw options +func newOptions(rawOptions configmanager.RawOptions) (options, error) { + var opts options + if err := mapstructure.Decode(rawOptions, &opts); err != nil { return opts, err } return opts, nil } + +// checkPathExist will check argocdapp config already exist +func (s *source) checkPathExist(scmClient scm.ClientOperation) (bool, error) { + existFiles, err := scmClient.GetPathInfo(s.Path) + if err != nil { + return false, err + } + if len(existFiles) > 0 { + log.Debugf("argocdapp check config path %s already exist, pass...", s.Path) + return true, nil + } + return false, nil +} + +func (o *options) getArgocdDefaultConfigFiles(configLocation downloader.ResourceLocation) (git.GitFileContentMap, error) { + // 1. get configs from remote url + configFiles, err := configLocation.Download() + if err != nil { + return nil, err + } + // 2. get file content + fContentFunc := func(filePath string) ([]byte, error) { + renderContent, err := template.New().FromLocalFile(filePath).SetDefaultRender("argocd config", o).Render() + if err != nil { + return nil, err + } + return []byte(renderContent), nil + } + fNameFunc := func(filePath, srcPath string) string { + relativePath, _ := filepath.Rel(srcPath, filePath) + return filepath.Join(o.Source.Path, relativePath) + } + return file.GetFileMap(configFiles, file.DirFileFilterDefaultFunc, fNameFunc, fContentFunc) +} diff --git a/internal/pkg/plugin/argocdapp/options_test.go b/internal/pkg/plugin/argocdapp/options_test.go new file mode 100644 index 000000000..adc190325 --- /dev/null +++ b/internal/pkg/plugin/argocdapp/options_test.go @@ -0,0 +1,131 @@ +package argocdapp + +import ( + "errors" + "fmt" + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/ghttp" + + "github.com/devstream-io/devstream/pkg/util/downloader" + "github.com/devstream-io/devstream/pkg/util/scm" + "github.com/devstream-io/devstream/pkg/util/scm/git" +) + +var _ = Describe("source struct", func() { + var ( + scmClient scm.ClientOperation + errMsg string + s *source + ) + BeforeEach(func() { + s = &source{ + Path: "test_path", + } + }) + Context("checkPathExist method", func() { + When("scm client error", func() { + BeforeEach(func() { + errMsg = "scm get path error" + scmClient = &scm.MockScmClient{ + GetPathInfoError: errors.New(errMsg), + } + }) + It("should return error", func() { + _, err := s.checkPathExist(scmClient) + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).Should(Equal(errMsg)) + }) + }) + When("path list is empty", func() { + BeforeEach(func() { + scmClient = &scm.MockScmClient{ + GetPathInfoReturnValue: []*git.RepoFileStatus{}, + } + }) + It("should return false", func() { + exist, err := s.checkPathExist(scmClient) + Expect(err).ShouldNot(HaveOccurred()) + Expect(exist).Should(BeFalse()) + }) + }) + When("path list is not empty", func() { + BeforeEach(func() { + scmClient = &scm.MockScmClient{ + GetPathInfoReturnValue: []*git.RepoFileStatus{ + {Path: "gg"}, + }, + } + }) + It("should return true", func() { + exist, err := s.checkPathExist(scmClient) + Expect(err).ShouldNot(HaveOccurred()) + Expect(exist).Should(BeTrue()) + }) + }) + }) +}) + +var _ = Describe("options struct", func() { + var ( + configLocation downloader.ResourceLocation + o *options + s *ghttp.Server + pathName, reqPath string + ) + BeforeEach(func() { + s = ghttp.NewServer() + o = &options{ + App: &app{ + Name: "test", + }, + Source: &source{ + Path: "test_path", + RepoURL: "https://test.com/owner/repo", + }, + } + pathName = "/configLoc" + reqPath = fmt.Sprintf("%s%s", s.URL(), pathName) + s.RouteToHandler("HEAD", pathName, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "ok") + }) + }) + Context("getArgocdDefaultConfigFiles method", func() { + When("can't get location", func() { + BeforeEach(func() { + s.RouteToHandler("GET", pathName, ghttp.CombineHandlers( + ghttp.VerifyRequest("GET", pathName), + ghttp.RespondWith(http.StatusNotFound, ""), + )) + configLocation = downloader.ResourceLocation(reqPath) + }) + It("should return error", func() { + _, err := o.getArgocdDefaultConfigFiles(configLocation) + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).Should(ContainSubstring("bad response code: 404")) + }) + }) + When("all valid", func() { + BeforeEach(func() { + configFileContent := ` +app: + name: [[ .App.Name ]]` + s.RouteToHandler("GET", pathName, ghttp.CombineHandlers( + ghttp.VerifyRequest("GET", pathName), + ghttp.RespondWith(http.StatusOK, configFileContent), + )) + configLocation = downloader.ResourceLocation(reqPath) + }) + It("should return valid data", func() { + data, err := o.getArgocdDefaultConfigFiles(configLocation) + Expect(err).ShouldNot(HaveOccurred()) + Expect(data).ShouldNot(BeEmpty()) + }) + }) + }) + AfterEach(func() { + s.Close() + }) +}) diff --git a/internal/pkg/plugin/argocdapp/state.go b/internal/pkg/plugin/argocdapp/state.go index a4cc812eb..db6dbbcef 100644 --- a/internal/pkg/plugin/argocdapp/state.go +++ b/internal/pkg/plugin/argocdapp/state.go @@ -11,7 +11,7 @@ import ( ) func getStaticStatus(options configmanager.RawOptions) (statemanager.ResourceStatus, error) { - opts, err := NewOptions(options) + opts, err := newOptions(options) if err != nil { return nil, err } @@ -37,7 +37,7 @@ func getStaticStatus(options configmanager.RawOptions) (statemanager.ResourceSta } func getDynamicStatus(options configmanager.RawOptions) (statemanager.ResourceStatus, error) { - opts, err := NewOptions(options) + opts, err := newOptions(options) if err != nil { return nil, err } diff --git a/internal/pkg/plugin/argocdapp/tpl/argocd.tpl.yaml b/internal/pkg/plugin/argocdapp/tpl/helm.tpl.yaml similarity index 100% rename from internal/pkg/plugin/argocdapp/tpl/argocd.tpl.yaml rename to internal/pkg/plugin/argocdapp/tpl/helm.tpl.yaml diff --git a/internal/pkg/plugin/argocdapp/validate.go b/internal/pkg/plugin/argocdapp/validate.go index 3690af6eb..ab5da4df4 100644 --- a/internal/pkg/plugin/argocdapp/validate.go +++ b/internal/pkg/plugin/argocdapp/validate.go @@ -1,25 +1,18 @@ package argocdapp import ( - "fmt" - "github.com/devstream-io/devstream/internal/pkg/configmanager" - "github.com/devstream-io/devstream/pkg/util/log" "github.com/devstream-io/devstream/pkg/util/validator" ) // validate validates the options provided by the core. func validate(options configmanager.RawOptions) (configmanager.RawOptions, error) { - opts, err := NewOptions(options) + opts, err := newOptions(options) if err != nil { return nil, err } - errs := validator.Struct(opts) - if len(errs) > 0 { - for _, e := range errs { - log.Errorf("Options error: %s.", e) - } - return nil, fmt.Errorf("opts are illegal") + if err = validator.StructAllError(opts); err != nil { + return nil, err } return options, nil } diff --git a/internal/pkg/plugin/installer/ci/ci.go b/internal/pkg/plugin/installer/ci/ci.go index 832a90fdc..3be5e4504 100644 --- a/internal/pkg/plugin/installer/ci/ci.go +++ b/internal/pkg/plugin/installer/ci/ci.go @@ -49,14 +49,13 @@ func (p *PipelineConfig) BuildCIFileConfig(ciType server.CIServerType, repoInfo ConfigLocation: downloader.ResourceLocation(p.ConfigLocation), } // update ci render variables by plugins - rawConfigVars := p.generateCIFileVars(repoInfo) - rawConfigVars.Set("AppName", repoInfo.Repo) + rawConfigVars := p.GenerateCIFileVars(repoInfo) CIFileConfig.Vars = rawConfigVars log.Debugf("gitlab-ci pipeline get render vars: %+v", CIFileConfig) return CIFileConfig } -func (p *PipelineConfig) generateCIFileVars(repoInfo *git.RepoInfo) cifile.CIFileVarsMap { +func (p *PipelineConfig) GenerateCIFileVars(repoInfo *git.RepoInfo) cifile.CIFileVarsMap { // set default command for language p.setDefault() varMap, _ := mapz.DecodeStructToMap(p) @@ -67,6 +66,7 @@ func (p *PipelineConfig) generateCIFileVars(repoInfo *git.RepoInfo) cifile.CIFil if err != nil { log.Warnf("cifile merge CIFileVarsMap failed: %+v", err) } + varMap["AppName"] = repoInfo.Repo return varMap } diff --git a/internal/pkg/plugin/installer/ci/ci_test.go b/internal/pkg/plugin/installer/ci/ci_test.go index 7e9d6818d..1aba144e2 100644 --- a/internal/pkg/plugin/installer/ci/ci_test.go +++ b/internal/pkg/plugin/installer/ci/ci_test.go @@ -80,7 +80,7 @@ var _ = Describe("PipelineConfig struct", func() { }) }) It("should return file Vars", func() { - varMap := a.generateCIFileVars(r) + varMap := a.GenerateCIFileVars(r) var emptyDingtalk *step.DingtalkStepConfig var emptySonar *step.SonarQubeStepConfig var emptyBool *bool @@ -90,6 +90,7 @@ var _ = Describe("PipelineConfig struct", func() { "DingTalkSecretToken": "DINGTALK_SECURITY_TOKEN", "ImageRepoSecret": "IMAGE_REPO_SECRET", "ImageRepoDockerSecret": "image-repo-auth", + "AppName": "test_repo", "StepGlobalVars": "", "RepoType": "gitlab", "imageRepo": map[string]interface{}{ diff --git a/internal/pkg/plugin/installer/ci/validate.go b/internal/pkg/plugin/installer/ci/validate.go index 04b400929..05353772b 100644 --- a/internal/pkg/plugin/installer/ci/validate.go +++ b/internal/pkg/plugin/installer/ci/validate.go @@ -19,7 +19,7 @@ func Validate(options configmanager.RawOptions) (configmanager.RawOptions, error } if err := opts.ProjectRepo.CheckValid(); err != nil { - log.Debugf("github action validate repo invalid: %+v", err) + log.Debugf("ci config validate repo invalid: %+v", err) return nil, err } return options, nil diff --git a/internal/pkg/plugin/installer/reposcaffolding/installer.go b/internal/pkg/plugin/installer/reposcaffolding/installer.go index 7d6719b9b..55a9f2fdd 100644 --- a/internal/pkg/plugin/installer/reposcaffolding/installer.go +++ b/internal/pkg/plugin/installer/reposcaffolding/installer.go @@ -35,8 +35,7 @@ func InstallRepo(options configmanager.RawOptions) error { processRepoFileFunc(appName, opts.renderTplConfig()), ) if err != nil { - log.Warnf("repoScaffolding render repoTemplate failed=> %+v", err) - return fmt.Errorf("render RepoTemplate files failed") + return fmt.Errorf("render RepoTemplate files failed with error: %w", err) } // 3. push repo to DestinationRepo diff --git a/internal/pkg/plugin/jenkinspipeline/option.go b/internal/pkg/plugin/jenkinspipeline/option.go index 1ed0bf464..a5ae0c783 100644 --- a/internal/pkg/plugin/jenkinspipeline/option.go +++ b/internal/pkg/plugin/jenkinspipeline/option.go @@ -99,6 +99,14 @@ func (j *jobOptions) extractPlugins() []step.StepConfigAPI { return stepConfigs } +// check config need offline config +func (j *jobOptions) needOfflineConfig() bool { + // since we use github as default config location + // we use this to check whether this pipeline need default offline Jenkinsfile + const githubContentHost = "raw.githubusercontent.com" + return j.Jenkins.Offline && strings.Contains(string(j.Pipeline.ConfigLocation), githubContentHost) +} + // jenkins jobName, can be like folder/jobName or jobName type jenkinsJobName string diff --git a/internal/pkg/plugin/jenkinspipeline/tpl/Jenkinsfile_offline.tpl b/internal/pkg/plugin/jenkinspipeline/tpl/Jenkinsfile_offline.tpl new file mode 100644 index 000000000..fa498808c --- /dev/null +++ b/internal/pkg/plugin/jenkinspipeline/tpl/Jenkinsfile_offline.tpl @@ -0,0 +1,51 @@ +podTemplate(containers: [ + containerTemplate(name: 'maven', image: 'maven:3.8.6-openjdk-18', command: 'sleep', args: '99d'), + containerTemplate(name: 'buildkit', image: 'moby/buildkit:master', ttyEnabled: true, privileged: true), + ], volumes: [ + secretVolume(secretName: '[[ .ImageRepoDockerSecret ]]', mountPath: '/root/.docker') + ]) { + node(POD_LABEL) { + stage("Get Project") { + checkout scm + } + stage('Run Maven test') { + gitlabCommitStatus("test") { + container('maven') { + stage('run mvn test') { + sh 'mvn -B test' + } + } + } + } + stage("Build Docker image") { + gitlabCommitStatus("build image") { + container('buildkit') { + stage('build a Maven project') { + String opts = "" + String imageRepo = "[[ .imageRepo.user ]]/[[ .AppName ]]" + String imageURL = "[[ .imageRepo.url ]]" + if (imageURL) { + imageRepo = "${imageURL}/${imageRepo}" + } + if (imageRepo.contains("http://")) { + opts = ",registry.insecure=true" + imageRepo = imageRepo.replace("http://", "") + } + String version + if (env.GIT_COMMIT) { + version = env.GIT_COMMIT.substring(0, 8) + } else { + sh "git config --global --add safe.directory '*'" + String gitCommitLang = sh (script: "git log -n 1 --pretty=format:'%H'", returnStdout: true) + version = gitCommitLang.substring(0, 8) + } + sh """ + buildctl build --frontend dockerfile.v0 --local context=. --local dockerfile=. --output type=image,name=${imageRepo}:latest,push=true${opts} + buildctl build --frontend dockerfile.v0 --local context=. --local dockerfile=. --output type=image,name=${imageRepo}:${version},push=true${opts} + """ + } + } + } + } + } +} diff --git a/internal/pkg/plugin/jenkinspipeline/validate.go b/internal/pkg/plugin/jenkinspipeline/validate.go index 685fa30e7..a273892f0 100644 --- a/internal/pkg/plugin/jenkinspipeline/validate.go +++ b/internal/pkg/plugin/jenkinspipeline/validate.go @@ -1,12 +1,18 @@ package jenkinspipeline import ( + _ "embed" + "github.com/devstream-io/devstream/internal/pkg/configmanager" + "github.com/devstream-io/devstream/internal/pkg/plugin/installer/ci/cifile" "github.com/devstream-io/devstream/pkg/util/log" "github.com/devstream-io/devstream/pkg/util/types" "github.com/devstream-io/devstream/pkg/util/validator" ) +//go:embed tpl/Jenkinsfile_offline.tpl +var offlineJenkinsScript string + // setJenkinsDefault config default fields for usage func setJenkinsDefault(options configmanager.RawOptions) (configmanager.RawOptions, error) { opts, err := newJobOptions(options) @@ -19,7 +25,18 @@ func setJenkinsDefault(options configmanager.RawOptions) (configmanager.RawOptio return nil, err } opts.ProjectRepo = projectRepo - opts.CIFileConfig = opts.Pipeline.BuildCIFileConfig(ciType, projectRepo) + // if jenkins is offline, just use offline Jenkinsfile + if opts.needOfflineConfig() { + opts.CIFileConfig = &cifile.CIFileConfig{ + Type: ciType, + ConfigContentMap: map[string]string{ + "Jenkinsfile": offlineJenkinsScript, + }, + Vars: opts.Pipeline.GenerateCIFileVars(projectRepo), + } + } else { + opts.CIFileConfig = opts.Pipeline.BuildCIFileConfig(ciType, projectRepo) + } // set field value if empty if opts.Jenkins.Namespace == "" { opts.Jenkins.Namespace = "jenkins" diff --git a/internal/pkg/show/config/plugins/argocdapp.yaml b/internal/pkg/show/config/plugins/argocdapp.yaml index ec6535b80..978b9f99c 100644 --- a/internal/pkg/show/config/plugins/argocdapp.yaml +++ b/internal/pkg/show/config/plugins/argocdapp.yaml @@ -27,3 +27,9 @@ tools: path: charts/go-hello-http # Helm chart repo URL, this is only an example, do not use this repoURL: YOUR_CHART_REPO_URL + # if repo doesn't contain path, use imageRepo to create a helm config + imageRepo: + # imageRepo address + address: IMAGE_REPO_ADDRESS + # image tag + tag: IMAGE_REPO_TAG diff --git a/internal/pkg/show/config/plugins/jenkins-pipeline.yaml b/internal/pkg/show/config/plugins/jenkins-pipeline.yaml index 0ef937b61..59de251e4 100644 --- a/internal/pkg/show/config/plugins/jenkins-pipeline.yaml +++ b/internal/pkg/show/config/plugins/jenkins-pipeline.yaml @@ -15,7 +15,10 @@ tools: # jenkins namespace in k8s cluster namespace: jenkins # restart jenkins if true for plugin install - enableRestart: true + enableRestart: false + # if offline is true, jenkins-pipeline will not install jenkins plugins and share library + # it will use a local Jenkinsfile for jenkins-pipeline + offline: false scm: # url is the project repo location, this can be http address or git address url: git@gitlab.com/root/test-exmaple.git diff --git a/internal/pkg/show/config/plugins/repo-scaffolding.yaml b/internal/pkg/show/config/plugins/repo-scaffolding.yaml index 392ddbbc6..cb7d80415 100644 --- a/internal/pkg/show/config/plugins/repo-scaffolding.yaml +++ b/internal/pkg/show/config/plugins/repo-scaffolding.yaml @@ -23,4 +23,4 @@ tools: repoType: YOUR_TEMPLATE_REPO_TYPE # this is used for template render vars: - ImageRepo: YOUR_DOCKERHUB_REPOSITORY + imageRepo: YOUR_DOCKERHUB_REPOSITORY diff --git a/internal/pkg/show/config/templates/gitops.yaml b/internal/pkg/show/config/templates/gitops.yaml index c9b539a10..3c00c7596 100644 --- a/internal/pkg/show/config/templates/gitops.yaml +++ b/internal/pkg/show/config/templates/gitops.yaml @@ -30,7 +30,7 @@ tools: repo: dtm-scaffolding-golang repoType: github vars: - ImageRepo: "[[ dockerhubUsername ]]/[[ repoName ]]" + imageRepo: "[[ dockerhubUsername ]]/[[ repoName ]]" - name: jira-github-integ instanceID: default dependsOn: [ "repo-scaffolding.golang-github" ] diff --git a/internal/pkg/show/config/templates/quickstart.yaml b/internal/pkg/show/config/templates/quickstart.yaml index 9aa76e307..e7ce1f65f 100644 --- a/internal/pkg/show/config/templates/quickstart.yaml +++ b/internal/pkg/show/config/templates/quickstart.yaml @@ -19,7 +19,7 @@ tools: repo: dtm-scaffolding-golang repoType: github vars: - ImageRepo: YOUR_DOCKER_USERNAME/go-webapp-devstream-demo + imageRepo: YOUR_DOCKER_USERNAME/go-webapp-devstream-demo - name: githubactions-golang instanceID: default dependsOn: ["repo-scaffolding.golang-github"] diff --git a/pkg/util/file/dir.go b/pkg/util/file/dir.go index 9bed3f5d0..728510917 100644 --- a/pkg/util/file/dir.go +++ b/pkg/util/file/dir.go @@ -44,6 +44,7 @@ func GetFileMapByWalkDir( contentMap[dstFileName] = content return nil }); err != nil { + log.Debugf("Walk Dir %s failed: %+v", srcPath, err) return nil, err } return contentMap, nil @@ -85,3 +86,9 @@ func CreateTempDir(dirPattern string) (string, error) { } return tempDir, err } + +// DirFileFilterDefaultFunc is used for GetFileMap +// it will return false if isDir is true +func DirFileFilterDefaultFunc(filePath string, isDir bool) bool { + return !isDir +} diff --git a/pkg/util/scm/scm.go b/pkg/util/scm/scm.go index d260507d8..b43b120e0 100644 --- a/pkg/util/scm/scm.go +++ b/pkg/util/scm/scm.go @@ -113,6 +113,14 @@ func (s *SCMInfo) Encode() map[string]any { return m } +func (s *SCMInfo) NewClientWithAuthFromScm() (ClientOperation, error) { + repo, err := s.BuildRepoInfo() + if err != nil { + return nil, err + } + return NewClientWithAuth(repo) +} + func formatGithubCloneURL(cloneURL string) string { if !strings.Contains(cloneURL, "git@") && !strings.HasPrefix(cloneURL, "http") { return fmt.Sprintf("https://%s", cloneURL) diff --git a/staging/dtm-jenkins-pipeline-example/springboot/Jenkinsfile b/staging/dtm-jenkins-pipeline-example/springboot/Jenkinsfile index 4976eb98f..a286041ed 100644 --- a/staging/dtm-jenkins-pipeline-example/springboot/Jenkinsfile +++ b/staging/dtm-jenkins-pipeline-example/springboot/Jenkinsfile @@ -1,8 +1,3 @@ -version="1.0.${env.BUILD_NUMBER}" -repository="[[ .ImageRepoAddress ]]/[[ .ImageName ]]" -tag="latest" -image="${repository}:${version}" - podTemplate(containers: [ containerTemplate(name: 'maven', image: 'maven:3.8.6-openjdk-18', command: 'sleep', args: '99d'), containerTemplate(name: 'buildkit', image: 'moby/buildkit:master', ttyEnabled: true, privileged: true), @@ -26,9 +21,20 @@ podTemplate(containers: [ gitlabCommitStatus("build image") { container('buildkit') { stage('build a Maven project') { + String imageRepo = "[[ .imageRepo.user ]]/[[ .AppName ]]" + String imageURL = "[[ .imageRepo.url ]]" + if (imageURL) { + imageRepo = "${imageURL}/${imageRepo}" + } + if (imageRepo.contains("http://")) { + opts = ",registry.insecure=true" + imageRepo = imageRepo.replace("http://", "") + } + gitUtil = new Git() + version = gitUtil.getCommitIDHead() sh """ - buildctl build --frontend dockerfile.v0 --local context=. --local dockerfile=. --output type=image,name=${image},push=true,registry.insecure=true - buildctl build --frontend dockerfile.v0 --local context=. --local dockerfile=. --output type=image,name=${repository}:${tag},push=true,registry.insecure=true + buildctl build --frontend dockerfile.v0 --local context=. --local dockerfile=. --output type=image,name=${imageRepo}:${defaultTag},push=true${opts} + buildctl build --frontend dockerfile.v0 --local context=. --local dockerfile=. --output type=image,name=${imageRepo}:${version},push=true${opts} """ } } diff --git a/test/e2e/yaml/e2e-apps.yaml b/test/e2e/yaml/e2e-apps.yaml index c08762e28..a9d9490a6 100644 --- a/test/e2e/yaml/e2e-apps.yaml +++ b/test/e2e/yaml/e2e-apps.yaml @@ -8,7 +8,6 @@ vars: defaultBranch: main repoOwner: devstream-io repoName: dtm-e2e-go - dockerRegistryUserName: dtme2etest imageRepoOwner: dtme2etest tools: diff --git a/test/e2e/yaml/e2e-test-local.yaml b/test/e2e/yaml/e2e-test-local.yaml index ae5a7b734..7f93f9b95 100644 --- a/test/e2e/yaml/e2e-test-local.yaml +++ b/test/e2e/yaml/e2e-test-local.yaml @@ -23,7 +23,7 @@ tools: branch: [[ defaultBranch ]] repoType: github vars: - ImageRepo: "[[ dockerhubUsername ]]/[[ repoName ]]" + imageRepo: "[[ dockerhubUsername ]]/[[ repoName ]]" sourceRepo: org: devstream-io repo: dtm-scaffolding-golang