feat: Add support for Azure Blob Storage artifacts Fixes argoproj#1540 (

argoproj#9026) Signed-off-by: Brian Loss <brianloss@gmail.com> Signed-off-by: Reddy <Rajshekar.Reddy@lowes.com>
reddymh · Jan 2, 2023 · d179ae5 · d179ae5
1 parent fbb2fd1
commit d179ae5
Show file tree

Hide file tree

Showing 71 changed files with 9,717 additions and 764 deletions.
diff --git a/.devcontainer/startup.sh b/.devcontainer/startup.sh
@@ -5,7 +5,7 @@ sudo apt update
 sudo chown $USER:docker /var/run/docker.sock
 sudo chown -fR $USER:golang $GOPATH
 
-echo '127.0.0.1 dex\n127.0.0.1 minio\n127.0.0.1 postgres\n127.0.0.1 mysql' | sudo tee -a /etc/hosts
+echo '127.0.0.1 dex\n127.0.0.1 minio\n127.0.0.1 postgres\n127.0.0.1 mysql\n127.0.0.1 azurite' | sudo tee -a /etc/hosts
 
 if k3d cluster list | grep k3s-default;
 then

diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml
@@ -144,14 +144,15 @@ jobs:
           echo '127.0.0.1 minio'    | sudo tee -a /etc/hosts
           echo '127.0.0.1 postgres' | sudo tee -a /etc/hosts
           echo '127.0.0.1 mysql'    | sudo tee -a /etc/hosts
+          echo '127.0.0.1 azurite'  | sudo tee -a /etc/hosts
       - run: make install PROFILE=${{matrix.profile}} STATIC_FILES=false
       - run: make controller $(go env GOPATH)/bin/goreman STATIC_FILES=false
       - run: make cli STATIC_FILES=false
         if: ${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}}
-      - run: make start PROFILE=${{matrix.profile}} AUTH_MODE=client STATIC_FILES=false LOG_LEVEL=info API=${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}} UI=false > /tmp/argo.log 2>&1 &
+      - run: make start PROFILE=${{matrix.profile}} AUTH_MODE=client STATIC_FILES=false LOG_LEVEL=info API=${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}} UI=false AZURE=true > /tmp/argo.log 2>&1 &
       - run: make wait
         timeout-minutes: 4
-      - run: make ${{matrix.test}} E2E_SUITE_TIMEOUT=20m STATIC_FILES=false
+      - run: make ${{matrix.test}} E2E_SUITE_TIMEOUT=20m STATIC_FILES=false AZURE=true
       - if: ${{ failure() }}
         run: |
           [ -e /tmp/argo.log ] && cat /tmp/argo.log

diff --git a/Makefile b/Makefile
@@ -70,6 +70,8 @@ AUTH_MODE             := hybrid
 ifeq ($(PROFILE),sso)
 AUTH_MODE             := sso
 endif
+# whether or not to start the Azurite test service for Azure Blob Storage
+AZURE                 := false
 
 # Which mode to run in:
 # * `local` run the workflow–controller and argo-server as single replicas on the local machine (default)
@@ -92,7 +94,7 @@ ALWAYS_OFFLOAD_NODE_STATUS := false
 
 $(info GIT_COMMIT=$(GIT_COMMIT) GIT_BRANCH=$(GIT_BRANCH) GIT_TAG=$(GIT_TAG) GIT_TREE_STATE=$(GIT_TREE_STATE) RELEASE_TAG=$(RELEASE_TAG) DEV_BRANCH=$(DEV_BRANCH) VERSION=$(VERSION))
 $(info KUBECTX=$(KUBECTX) DOCKER_DESKTOP=$(DOCKER_DESKTOP) K3D=$(K3D) DOCKER_PUSH=$(DOCKER_PUSH))
-$(info RUN_MODE=$(RUN_MODE) PROFILE=$(PROFILE) AUTH_MODE=$(AUTH_MODE) SECURE=$(SECURE) STATIC_FILES=$(STATIC_FILES) ALWAYS_OFFLOAD_NODE_STATUS=$(ALWAYS_OFFLOAD_NODE_STATUS) UPPERIO_DB_DEBUG=$(UPPERIO_DB_DEBUG) LOG_LEVEL=$(LOG_LEVEL) NAMESPACED=$(NAMESPACED))
+$(info RUN_MODE=$(RUN_MODE) PROFILE=$(PROFILE) AUTH_MODE=$(AUTH_MODE) SECURE=$(SECURE) STATIC_FILES=$(STATIC_FILES) ALWAYS_OFFLOAD_NODE_STATUS=$(ALWAYS_OFFLOAD_NODE_STATUS) UPPERIO_DB_DEBUG=$(UPPERIO_DB_DEBUG) LOG_LEVEL=$(LOG_LEVEL) NAMESPACED=$(NAMESPACED) AZURE=$(AZURE))
 
 override LDFLAGS += \
   -X github.com/argoproj/argo-workflows/v3.version=$(VERSION) \
@@ -429,6 +431,9 @@ ifeq ($(RUN_MODE),kubernetes)
 	kubectl -n $(KUBE_NAMESPACE) scale deploy/workflow-controller --replicas 1
 	kubectl -n $(KUBE_NAMESPACE) scale deploy/argo-server --replicas 1
 endif
+ifeq ($(AZURE),true)
+	kubectl -n $(KUBE_NAMESPACE) apply -f test/e2e/azure/deploy-azurite.yaml
+endif
 
 .PHONY: argosay
 argosay:
@@ -457,7 +462,7 @@ endif
 else
 start: install
 endif
-	@echo "starting STATIC_FILES=$(STATIC_FILES) (DEV_BRANCH=$(DEV_BRANCH), GIT_BRANCH=$(GIT_BRANCH)), AUTH_MODE=$(AUTH_MODE), RUN_MODE=$(RUN_MODE), MANAGED_NAMESPACE=$(MANAGED_NAMESPACE)"
+	@echo "starting STATIC_FILES=$(STATIC_FILES) (DEV_BRANCH=$(DEV_BRANCH), GIT_BRANCH=$(GIT_BRANCH)), AUTH_MODE=$(AUTH_MODE), RUN_MODE=$(RUN_MODE), MANAGED_NAMESPACE=$(MANAGED_NAMESPACE), AZURE=$(AZURE)"
 ifneq ($(CTRL),true)
 	@echo "⚠️️  not starting controller. If you want to test the controller, use 'make start CTRL=true' to start it"
 endif
@@ -476,6 +481,9 @@ endif
 	# Check dex, minio, postgres and mysql are in hosts file
 ifeq ($(AUTH_MODE),sso)
 	grep '127.0.0.1.*dex' /etc/hosts
+endif
+ifeq ($(AZURE),true)
+	grep '127.0.0.1.*azurite' /etc/hosts
 endif
 	grep '127.0.0.1.*minio' /etc/hosts
 	grep '127.0.0.1.*postgres' /etc/hosts

diff --git a/README.md b/README.md
@@ -84,7 +84,7 @@ kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo-workflo
 ## Features
 
 * UI to visualize and manage Workflows
-* Artifact support (S3, Artifactory, Alibaba Cloud OSS, HTTP, Git, GCS, raw)
+* Artifact support (S3, Artifactory, Alibaba Cloud OSS, Azure Blob Storage, HTTP, Git, GCS, raw)
 * Workflow templating to store commonly used Workflows in the cluster
 * Archiving Workflows after executing for later access
 * Scheduled workflows using cron

diff --git a/api/jsonschema/schema.json b/api/jsonschema/schema.json
@@ -3343,6 +3343,10 @@
           "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ArtifactoryArtifact",
           "description": "Artifactory contains artifactory artifact location details"
         },
+        "azure": {
+          "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.AzureArtifact",
+          "description": "Azure contains Azure Storage artifact location details"
+        },
         "deleted": {
           "description": "Has this been deleted?",
           "type": "boolean"
@@ -3438,6 +3442,10 @@
           "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ArtifactoryArtifact",
           "description": "Artifactory contains artifactory artifact location details"
         },
+        "azure": {
+          "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.AzureArtifact",
+          "description": "Azure contains Azure Storage artifact location details"
+        },
         "gcs": {
           "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.GCSArtifact",
           "description": "GCS contains GCS artifact location details"
@@ -3488,6 +3496,10 @@
           "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ArtifactoryArtifact",
           "description": "Artifactory contains artifactory artifact location details"
         },
+        "azure": {
+          "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.AzureArtifact",
+          "description": "Azure contains Azure Storage artifact location details"
+        },
         "deleted": {
           "description": "Has this been deleted?",
           "type": "boolean"
@@ -3573,6 +3585,10 @@
           "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ArtifactoryArtifactRepository",
           "description": "Artifactory stores artifacts to JFrog Artifactory"
         },
+        "azure": {
+          "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.AzureArtifactRepository",
+          "description": "Azure stores artifact in an Azure Storage account"
+        },
         "gcs": {
           "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.GCSArtifactRepository",
           "description": "GCS stores artifact in a GCS object store"
@@ -3669,6 +3685,67 @@
       },
       "type": "object"
     },
+    "io.argoproj.workflow.v1alpha1.AzureArtifact": {
+      "description": "AzureArtifact is the location of a an Azure Storage artifact",
+      "properties": {
+        "accountKeySecret": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector",
+          "description": "AccountKeySecret is the secret selector to the Azure Blob Storage account access key"
+        },
+        "blob": {
+          "description": "Blob is the blob name (i.e., path) in the container where the artifact resides",
+          "type": "string"
+        },
+        "container": {
+          "description": "Container is the container where resources will be stored",
+          "type": "string"
+        },
+        "endpoint": {
+          "description": "Endpoint is the service url associated with an account. It is most likely \"https://\u003cACCOUNT_NAME\u003e.blob.core.windows.net\"",
+          "type": "string"
+        },
+        "useSDKCreds": {
+          "description": "UseSDKCreds tells the driver to figure out credentials based on sdk defaults.",
+          "type": "boolean"
+        }
+      },
+      "required": [
+        "endpoint",
+        "container",
+        "blob"
+      ],
+      "type": "object"
+    },
+    "io.argoproj.workflow.v1alpha1.AzureArtifactRepository": {
+      "description": "AzureArtifactRepository defines the controller configuration for an Azure Blob Storage artifact repository",
+      "properties": {
+        "accountKeySecret": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector",
+          "description": "AccountKeySecret is the secret selector to the Azure Blob Storage account access key"
+        },
+        "blobNameFormat": {
+          "description": "BlobNameFormat is defines the format of how to store blob names. Can reference workflow variables",
+          "type": "string"
+        },
+        "container": {
+          "description": "Container is the container where resources will be stored",
+          "type": "string"
+        },
+        "endpoint": {
+          "description": "Endpoint is the service url associated with an account. It is most likely \"https://\u003cACCOUNT_NAME\u003e.blob.core.windows.net\"",
+          "type": "string"
+        },
+        "useSDKCreds": {
+          "description": "UseSDKCreds tells the driver to figure out credentials based on sdk defaults.",
+          "type": "boolean"
+        }
+      },
+      "required": [
+        "endpoint",
+        "container"
+      ],
+      "type": "object"
+    },
     "io.argoproj.workflow.v1alpha1.Backoff": {
       "description": "Backoff is a backoff strategy to use within retryStrategy",
       "properties": {

diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
@@ -7263,6 +7263,10 @@
           "description": "Artifactory contains artifactory artifact location details",
           "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ArtifactoryArtifact"
         },
+        "azure": {
+          "description": "Azure contains Azure Storage artifact location details",
+          "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.AzureArtifact"
+        },
         "deleted": {
           "description": "Has this been deleted?",
           "type": "boolean"
@@ -7355,6 +7359,10 @@
           "description": "Artifactory contains artifactory artifact location details",
           "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ArtifactoryArtifact"
         },
+        "azure": {
+          "description": "Azure contains Azure Storage artifact location details",
+          "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.AzureArtifact"
+        },
         "gcs": {
           "description": "GCS contains GCS artifact location details",
           "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.GCSArtifact"
@@ -7408,6 +7416,10 @@
           "description": "Artifactory contains artifactory artifact location details",
           "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ArtifactoryArtifact"
         },
+        "azure": {
+          "description": "Azure contains Azure Storage artifact location details",
+          "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.AzureArtifact"
+        },
         "deleted": {
           "description": "Has this been deleted?",
           "type": "boolean"
@@ -7490,6 +7502,10 @@
           "description": "Artifactory stores artifacts to JFrog Artifactory",
           "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ArtifactoryArtifactRepository"
         },
+        "azure": {
+          "description": "Azure stores artifact in an Azure Storage account",
+          "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.AzureArtifactRepository"
+        },
         "gcs": {
           "description": "GCS stores artifact in a GCS object store",
           "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.GCSArtifactRepository"
@@ -7585,6 +7601,67 @@
         }
       }
     },
+    "io.argoproj.workflow.v1alpha1.AzureArtifact": {
+      "description": "AzureArtifact is the location of a an Azure Storage artifact",
+      "type": "object",
+      "required": [
+        "endpoint",
+        "container",
+        "blob"
+      ],
+      "properties": {
+        "accountKeySecret": {
+          "description": "AccountKeySecret is the secret selector to the Azure Blob Storage account access key",
+          "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector"
+        },
+        "blob": {
+          "description": "Blob is the blob name (i.e., path) in the container where the artifact resides",
+          "type": "string"
+        },
+        "container": {
+          "description": "Container is the container where resources will be stored",
+          "type": "string"
+        },
+        "endpoint": {
+          "description": "Endpoint is the service url associated with an account. It is most likely \"https://\u003cACCOUNT_NAME\u003e.blob.core.windows.net\"",
+          "type": "string"
+        },
+        "useSDKCreds": {
+          "description": "UseSDKCreds tells the driver to figure out credentials based on sdk defaults.",
+          "type": "boolean"
+        }
+      }
+    },
+    "io.argoproj.workflow.v1alpha1.AzureArtifactRepository": {
+      "description": "AzureArtifactRepository defines the controller configuration for an Azure Blob Storage artifact repository",
+      "type": "object",
+      "required": [
+        "endpoint",
+        "container"
+      ],
+      "properties": {
+        "accountKeySecret": {
+          "description": "AccountKeySecret is the secret selector to the Azure Blob Storage account access key",
+          "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector"
+        },
+        "blobNameFormat": {
+          "description": "BlobNameFormat is defines the format of how to store blob names. Can reference workflow variables",
+          "type": "string"
+        },
+        "container": {
+          "description": "Container is the container where resources will be stored",
+          "type": "string"
+        },
+        "endpoint": {
+          "description": "Endpoint is the service url associated with an account. It is most likely \"https://\u003cACCOUNT_NAME\u003e.blob.core.windows.net\"",
+          "type": "string"
+        },
+        "useSDKCreds": {
+          "description": "UseSDKCreds tells the driver to figure out credentials based on sdk defaults.",
+          "type": "boolean"
+        }
+      }
+    },
     "io.argoproj.workflow.v1alpha1.Backoff": {
       "description": "Backoff is a backoff strategy to use within retryStrategy",
       "type": "object",

diff --git a/cmd/argo/commands/common/get.go b/cmd/argo/commands/common/get.go
@@ -118,6 +118,8 @@ func PrintWorkflowHelper(wf *wfv1.Workflow, getArgs GetFlags) string {
 					out += fmt.Sprintf(fmtStr, "  "+art.Name+":", art.S3.String())
 				} else if art.Artifactory != nil {
 					out += fmt.Sprintf(fmtStr, "  "+art.Name+":", art.Artifactory.String())
+				} else if art.Azure != nil {
+					out += fmt.Sprintf(fmtStr, "  "+art.Name+":", art.Azure.String())
 				}
 			}
 		}

diff --git a/docs/configure-artifact-repository.md b/docs/configure-artifact-repository.md
@@ -8,6 +8,7 @@ Subsequent sections will show how to use it.
 | Name | Inputs | Outputs | Usage (Feb 2020) |
 |---|---|---|---|
 | Artifactory | Yes | Yes | 11% |
+| Azure Blob | Yes | Yes | - |
 | GCS | Yes | Yes | - |
 | Git | Yes | No | - |
 | HDFS | Yes | Yes | 3% |
@@ -205,6 +206,61 @@ artifacts:
 
 You can also set `createBucketIfNotPresent` to `true` to tell the artifact driver to automatically create the OSS bucket if it doesn't exist yet when saving artifacts. Note that you'll need to set additional permission for your OSS account to create new buckets.
 
+## Configuring Azure Blob Storage
+
+Create an Azure Storage account and a container within that account. There are a number of
+ways to accomplish this, including the [Azure Portal](https://portal.azure.com) or the
+[CLI](https://docs.microsoft.com/en-us/cli/azure/).
+
+1. Retrieve the blob service endpoint for the storage account. For example:
+
+   ```bash
+   az storage account show -n mystorageaccountname --query 'primaryEndpoints.blob' -otsv
+   ```
+
+2. Retrieve the access key for the storage account. For example:
+
+   ```bash
+   az storage account keys list -n mystorageaccountname --query '[0].value' -otsv
+   ```
+
+3. Create a kubernetes secret to hold the storage account key. For example:
+
+   ```bash
+   kubectl create secret generic my-azure-storage-credentials \
+     --from-literal "account-access-key=$(az storage account keys list -n mystorageaccountname --query '[0].value' -otsv)"
+   ```
+
+4. Configure `azure` artifact as following in the yaml.
+
+```yaml
+artifacts:
+  - name: message
+    path: /tmp/message
+    azure:
+      endpoint: https://mystorageaccountname.blob.core.windows.net
+      container: my-container-name
+      blob: path/in/container
+      # accountKeySecret is a secret selector.
+      # It references the k8s secret named 'my-azure-storage-credentials'.
+      # This secret is expected to have have the key 'account-access-key',
+      # containing the base64 encoded credentials to the storage account.
+      #
+      # If a managed identity has been assigned to the machines running the
+      # workflow (e.g., https://docs.microsoft.com/en-us/azure/aks/use-managed-identity)
+      # then accountKeySecret is not needed, and useSDKCreds should be
+      # set to true instead:
+      # useSDKCreds: true
+      accountKeySecret:
+        name: my-azure-storage-credentials
+        key: account-access-key     
+```
+
+If `useSDKCreds` is set to `true`, then the `accountKeySecret` value is not
+used and authentication with Azure will be attempted using a
+[`DefaultAzureCredential`](https://docs.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication)
+instead.
+
 ## Configure the Default Artifact Repository
 
 In order for Argo to use your artifact repository, you can configure it as the