CSS-4546 - Backport ci/cd and supporting files

.github/workflows/charm-build.yaml

@@ -0,0 +1,21 @@
+name: Charm Build
+on:
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  build-charm:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        charm-type: ["jimm","jimm-k8s"]
+    steps:
+      - uses: actions/checkout@v3
+      - run: git fetch --prune --unshallow
+      - run: sudo snap install charmcraft --channel=2.x/stable --classic
+      - run: sudo charmcraft pack --project-dir ./charms/${{ matrix.charm-type }} --destructive-mode --verbosity=trace
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.charm-type }}-charm
+          path: ./*.charm
+          if-no-files-found: error

.github/workflows/charm-release.yaml

@@ -0,0 +1,68 @@
+name: Release to latest/edge
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - 'v*'
+
+# Note this workflow requires a Github secret to provide auth against charmhub.
+# This can be generated with the following command
+# charmcraft login --export=secrets.auth --charm=<charm-name> --permission=package-manage --permission=package-view --channel=latest/edge --ttl=999999999
+
+jobs:
+  ci-tests:
+    uses: ./.github/workflows/ci.yaml
+  charm-tests:
+    uses: ./.github/workflows/charm-test.yaml
+  snap-build:
+    uses: ./.github/workflows/snap.yaml
+
+  release-k8s-charm:
+    name: Release k8s charm
+    needs:
+      - ci-tests
+      - charm-tests
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Build local images
+        run: make jimm-image
+      - name: Upload charm to charmhub
+        uses: kian99/charming-actions/upload-charm@add-local-image-option
+        with:
+          credentials: "${{ secrets.CHARMHUB_TOKEN }}"
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+          channel: "v1/edge"
+          charm-path: "./charms/jimm-k8s"
+          local-image: "true"
+
+  release-machine-charm:
+    name: Release machine charm
+    needs:
+      - ci-tests
+      - charm-tests
+      - snap-build
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: actions/download-artifact@master
+        with:
+          name: jimm-snap
+      - name: Install charmcraft
+        run: sudo snap install charmcraft --channel=2.x/stable
+      - name: Publish Charm Resource
+        run: charmcraft upload-resource juju-jimm jimm-snap --filepath ./jimm-snap
+        env:
+          CHARMCRAFT_AUTH: "${{ secrets.CHARMHUB_TOKEN }}"
+      - name: Upload charm to charmhub
+        uses: canonical/charming-actions/upload-charm@2.3.0
+        with:
+          credentials: "${{ secrets.CHARMHUB_TOKEN }}"
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+          channel: "v1/edge"
+          charm-path: "./charms/jimm"

.github/workflows/charm-test.yaml

@@ -0,0 +1,79 @@
+name: Charm Test
+on:
+  workflow_call:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - charms/**
+
+jobs:
+  charm-build:
+    uses: ./.github/workflows/charm-build.yaml
+
+  lint:
+    # Delete this if statement once charms are updated and include tox.
+    if: ${{ 1 }} == ${{ 2 }} 
+    name: Lint
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        charm-type: ["jimm","jimm-k8s"]
+    defaults:
+      run:
+        working-directory: ./charms/${{ matrix.charm-type }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install dependencies
+        run: python3 -m pip install tox
+      - name: Run linters
+        run: tox -e lint
+  unit-tests:
+    # Delete this if statement once charms are updated and include tox.
+    if: ${{ 1 }} == ${{ 2 }} 
+    name: Unit tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        charm-type: ["jimm","jimm-k8s"]
+    defaults:
+      run:
+        working-directory: ./charms/${{ matrix.charm-type }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install dependencies
+        run: python -m pip install tox
+      - name: Run tests
+        run: tox -e unit
+
+  # TODO(Kian): Fix this
+  # integration-tests:
+  #   name: Integration tests
+  #   needs:
+  #     - charm-build
+  #   runs-on: ubuntu-latest
+  #   env:
+  #     charm-type: "jimm-k8s"
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v3
+  #     - name: Setup operator environment
+  #       uses: charmed-kubernetes/actions-operator@main
+  #       with:
+  #         juju-channel: 2.9/stable
+  #         provider: microk8s
+  #         microk8s-addons: "ingress storage dns rbac registry"
+  #         channel: 1.27/stable
+  #     # Download the charm from the build to speed up integration tests.
+  #     - uses: actions/download-artifact@master
+  #       with:
+  #         name: jimm-k8s-charm
+  #         path: ./charms/${{ env.charm-type }}
+  #     - name: Create OCI Image
+  #       run: make push-microk8s
+  #     - name: Install tox
+  #       run: python -m pip install tox
+  #     - name: Integration tests
+  #       run: tox -e integration -- --localCharm
+  #       working-directory: ./charms/${{ env.charm-type }}

.github/workflows/ci.yaml

@@ -1,63 +1,56 @@
 name: CI
 on:
   pull_request:
+  workflow_call:
   workflow_dispatch:
-env:
-  GH_AUTH: ${{ secrets.GH_AUTH }}
-  GH_USER: ${{ secrets.GH_USER }}
 
 jobs:
-  lint:
-    runs-on: ubuntu-20.04
-    continue-on-error: true
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/workflows/setupgo118amd64
-        with:
-          user: ${{ secrets.GH_USER }}
-          pat: ${{ secrets.GH_AUTH }}
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
-        with:
-          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-          version: latest
-          skip-pkg-cache: true
-          skip-build-cache: true
+  # lint:
+  #   runs-on: ubuntu-20.04
+  #   continue-on-error: true
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #     - uses: ./.github/workflows/setupgo118amd64
+  #     - name: golangci-lint
+  #       uses: golangci/golangci-lint-action@v3
+  #       with:
+  #         # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
+  #         version: latest
+  #         skip-pkg-cache: true
+  #         skip-build-cache: true
 
   build_test:
     name: Build and Test
-    #    needs:
-    #      - lint
     runs-on: ubuntu-20.04
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_PASSWORD: password
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
     steps:
       - uses: actions/checkout@v3
+      - uses: ./.github/workflows/setupgo118amd64
       - name: Install dependencies
-        run: sudo apt-get update -y && sudo apt-get install -y gcc git-core gnupg
+        run: sudo apt-get update -y && sudo apt-get install -y gcc git-core gnupg build-essential
       - name: Remove installed mongodb
         run: sudo apt purge mongodb-org && sudo apt autoremove
       - run: sudo snap install juju-db --channel 4.4/stable
-      - name: Install vault
-        run: sudo snap install vault --classic
-      - uses: ./.github/workflows/setupgo118amd64
+      - name: Pull candid repo for test environment
+        run: |
+          git clone https://github.com/canonical/candid.git ./tmp/candid
+          cd ./tmp/candid
+          DOCKER_BUILDKIT=1 \
+          docker build \
+          --cache-from candid:latest \
+          . -f ./Dockerfile -t candid
+          docker image ls candid
+      - name: Add volume files
+        run: |
+          touch ./local/vault/approle.json
+          touch ./local/vault/roleid.txt
+      - name: Start test environment
+        run: docker compose up -d
       - name: Build and Test
         run: go test -mod readonly ./...
         env:
-          JIMM_DSN: postgresql://postgres:password@localhost:5432/jimm
+          JIMM_DSN: postgresql://jimm:jimm@localhost:5432/jimm
           PGHOST: localhost
-          PGPASSWORD: password
+          PGPASSWORD: jimm
           PGSSLMODE: disable
-          PGUSER: postgres
+          PGUSER: jimm
           PGPORT: 5432

.github/workflows/oci-image.yaml

@@ -1,9 +1,6 @@
 name: BuildOCIImage
 on:
   workflow_dispatch:
-env:
-  GH_USER: ${{ secrets.GH_USER }}
-  GH_AUTH: ${{ secrets.GH_AUTH }}
 
 jobs:
   candid-oci-image:
@@ -12,9 +9,6 @@ jobs:
       - uses: actions/checkout@v3
       - run: git fetch --prune --unshallow
       - uses: ./.github/workflows/setupgo118amd64
-        with:
-          user: ${{ secrets.GH_USER }}
-          pat: ${{ secrets.GH_AUTH }}
       - uses: docker/setup-qemu-action@v2
       - uses: docker/setup-buildx-action@v2
       - name: Setup version and commit
@@ -29,12 +23,8 @@ jobs:
           target: deploy-env
           tags: candid:latest
           build-args: |
-            AUTH_TYPE=pat
             GIT_COMMIT=${{ env.GIT_COMMIT }}
             VERSION=${{ env.VERSION }}
-          secrets: |
-            "ghuser=${{ env.GH_USER }}"
-            "ghpat=${{ env.GH_AUTH }}"
           outputs: |
             type=docker,dest=candid-image.tar
       - uses: actions/upload-artifact@v3

.github/workflows/snap.yaml

@@ -1,5 +1,6 @@
 name: BuildSnap
 on:
+  workflow_call:
   workflow_dispatch:
 env:
   GH_AUTH: ${{ secrets.GH_AUTH }}

.gitignore

@@ -11,4 +11,12 @@
 /version/commit.txt
 /version/version.txt
 /tmp
-/local/vault/approle.yaml
+/local/vault/approle.json
+local/vault/approle.json
+local/vault/roleid.txt
+
+*.crt
+*.key
+*.csr
+jimmctl
+qa-controller

Dockerfile

@@ -11,26 +11,12 @@ RUN bash < <(curl -SL -v https://raw.githubusercontent.com/moovweb/gvm/${GVM_VER
     gvm use go$(cat go.mod | sed -n "/^go/p" | cut -d ' ' -f 2)  --default
 
 
-FROM build as build-with-github-auth
-ARG AUTH_TYPE
+FROM build as build-env
 ARG GIT_COMMIT
 ARG VERSION
 WORKDIR /usr/src/jimm
 SHELL ["/bin/bash", "-c"]
 COPY . .
-RUN --mount=type=secret,id=ghuser \
-    --mount=type=secret,id=ghpat \ 
-    --mount=type=ssh \ 
-    if [ "$AUTH_TYPE" = "pat" ]; then \
-    echo "machine github.com login $(cat /run/secrets/ghuser) password $(cat /run/secrets/ghpat)" > $HOME/.netrc && \
-    echo "PAT auth selected"; \
-    elif [ "$AUTH_TYPE" = "ssh" ]; then \
-    git config --global user.name $(cat /run/secrets/ghuser) && \
-    mkdir -p -m 0600 ~/.ssh && \
-    echo $(ssh-keyscan github.com) >> ~/.ssh/known_hosts && \
-    git config --global --add url."git@github.com:".insteadOf "https://github.com/" && \
-    echo "SSH auth selected"; \
-    fi
 RUN echo "${GIT_COMMIT}" | tee ./version/commit.txt
 RUN echo "${VERSION}" | tee ./version/version.txt
 RUN --mount=type=ssh source /root/.gvm/scripts/gvm && go mod vendor
@@ -40,8 +26,8 @@ RUN --mount=type=ssh source /root/.gvm/scripts/gvm && go build -o jimmsrv -race
 FROM ${DOCKER_REGISTRY}ubuntu:20.04 AS deploy-env
 RUN apt-get -qq update && apt-get -qq install -y ca-certificates postgresql-client
 WORKDIR /root/
-COPY --from=build-with-github-auth /usr/src/jimm/jimmsrv .
-COPY --from=build-with-github-auth /usr/src/jimm/internal/dbmodel/sql ./sql/
+COPY --from=build-env /usr/src/jimm/jimmsrv .
+COPY --from=build-env /usr/src/jimm/internal/dbmodel/sql ./sql/
 ENTRYPOINT [ "./jimmsrv" ]
 CMD ["./config.yaml"]