Use Azure Storage Tables for collection and container configuration

CHANGELOG.md

@@ -9,11 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
-- Update titiler-pgstac version to `0.1.0.a4` in `pctiler` (https://github.com/microsoft/planetary-computer-apis/pull/46)
+- Update titiler-pgstac version to `0.1.0.a4` in `pctiler` [#46](https://github.com/microsoft/planetary-computer-apis/pull/46)
+- Move render config, queryables and mosaic info into Azure Storage Tables [#48](https://github.com/microsoft/planetary-computer-apis/pull/48)
 
 ### Added
 
 - Added support for /queryables endpoint [#44](https://github.com/microsoft/planetary-computer-apis/pull/44)
+- Addd `/mosaic/info` endpoint [#48](https://github.com/microsoft/planetary-computer-apis/pull/48)
 
 ### Fixed
 

deployment/README.md

@@ -41,3 +41,11 @@ Container Registry repo where you published your local images:
 - `IMAGE_TAG`
 
 __Note:__ Remember to bring down your resources after testing with `terraform destroy`!
+
+## Loading configuration data
+
+Configuration data is stored in Azure Storage Tables. Use the `pcapis` command line interface that is installed with the `pccommon` package to load data. For example:
+
+```
+>  pcapis load -t collection --sas "${SAS_TOKEN}" --account pctapissatyasa  --table collectionconfig --file pccommon/tests/data-files/collection_config.json
+```

deployment/helm/deploy-values.template.yaml

@@ -12,6 +12,12 @@ stac:
     maxWorkers: "1"
     poolSize: "1"
 
+  storage:
+    account_name: {{ tf.storage_account_name }}
+    account_key: {{ tf.storage_account_key }}
+    collection_config_table_name: {{ tf.collection_config_table_name }}
+    container_config_table_name: {{ tf.container_config_table_name }}
+
   deploy:
     replicaCount: {{ tf.stac_replica_count }}
     podAnnotations:
@@ -39,6 +45,12 @@ tiler:
     workersPerCore: "1"
     webConcurrency: "1"
 
+  storage:
+    account_name: {{ tf.storage_account_name }}
+    account_key: {{ tf.storage_account_key }}
+    collection_config_table_name: {{ tf.collection_config_table_name }}
+    container_config_table_name: {{ tf.container_config_table_name }}
+
   deploy:
     replicaCount: {{ tf.tiler_replica_count }}
     podAnnotations:

deployment/helm/pc-apis-ingress/templates/ingress.yaml

@@ -1,7 +1,7 @@
 {{- if .Values.pcingress.ingress.enabled -}}
 {{- $fullName := include "pcingress.fullname" . -}}
 {{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
-apiVersion: networking.k8s.io/v1beta1
+apiVersion: networking.k8s.io/v1
 {{- else -}}
 apiVersion: extensions/v1beta1
 {{- end }}
@@ -26,15 +26,21 @@ spec:
         paths:
           {{ if $.Values.stac.enabled }}
           - path: {{ $.Values.pcingress.services.stac.path }}
+            pathType: Exact
             backend:
-              serviceName: {{ $.Values.pcingress.services.stac.name }}
-              servicePort: {{ $.Values.pcingress.services.stac.port }}
+              service:
+                name: {{ $.Values.pcingress.services.stac.name }}
+                port:
+                  number: {{ $.Values.pcingress.services.stac.port }}
           {{- end}}
           {{ if $.Values.tiler.enabled }}
           - path: {{ $.Values.pcingress.services.tiler.path }}
+            pathType: Exact
             backend:
-              serviceName: {{ $.Values.pcingress.services.tiler.name }}
-              servicePort: {{ $.Values.pcingress.services.tiler.port }}
+              service:
+                name: {{ $.Values.pcingress.services.tiler.name }}
+                port:
+                  number: {{ $.Values.pcingress.services.tiler.port }}
           {{- end}}
   {{- end }}
 {{- end }}

deployment/helm/published/planetary-computer-stac/templates/deployment.yaml

@@ -79,8 +79,20 @@ spec:
               value: "{{ .Values.environment }}"
             - name: "PGSSLMODE"
               value: "require"
-            - name: "DEBUG"
+            - name: "PCAPIS_DEBUG"
               value: "{{ .Values.stac.debug }}"
+            - name: "PCAPIS_COLLECTION_CONFIG__ACCOUNT_NAME"
+              value: "{{ .Values.stac.storage.account_name }}"
+            - name: "PCAPIS_COLLECTION_CONFIG__ACCOUNT_KEY"
+              value: "{{ .Values.stac.storage.account_key }}"
+            - name: "PCAPIS_COLLECTION_CONFIG__TABLE_NAME"
+              value: "{{ .Values.stac.storage.collection_config_table_name }}"
+            - name: "PCAPIS_CONTAINER_CONFIG__ACCOUNT_NAME"
+              value: "{{ .Values.stac.storage.account_name }}"
+            - name: "PCAPIS_CONTAINER_CONFIG__ACCOUNT_KEY"
+              value: "{{ .Values.stac.storage.account_key }}"
+            - name: "PCAPIS_CONTAINER_CONFIG__TABLE_NAME"
+              value: "{{ .Values.stac.storage.container_config_table_name }}"
             - name: APP_INSIGHTS_INSTRUMENTATION_KEY
               value: "{{ .Values.metrics.instrumentationKey }}"
 

deployment/helm/published/planetary-computer-stac/values.yaml

@@ -17,6 +17,12 @@ stac:
     port: 80
     annotations: {}
 
+  storage:
+    account_name: ""
+    account_key: ""
+    collection_config_table_name: ""
+    container_config_table_name: ""
+
   deploy:
     replicaCount: 1
     podAnnotations: {}

deployment/helm/published/planetary-computer-tiler/templates/deployment.yaml

@@ -79,6 +79,18 @@ spec:
               value: "{{ .Values.tiler.stac_api_href}}"
             - name: PC_SDK_SAS_URL
               value: "{{ .Values.tiler.pc_sdk_sas_url}}"
+            - name: "PCAPIS_COLLECTION_CONFIG__ACCOUNT_NAME"
+              value: "{{ .Values.tiler.storage.account_name }}"
+            - name: "PCAPIS_COLLECTION_CONFIG__ACCOUNT_KEY"
+              value: "{{ .Values.tiler.storage.account_key }}"
+            - name: "PCAPIS_COLLECTION_CONFIG__TABLE_NAME"
+              value: "{{ .Values.tiler.storage.collection_config_table_name }}"
+            - name: "PCAPIS_CONTAINER_CONFIG__ACCOUNT_NAME"
+              value: "{{ .Values.tiler.storage.account_name }}"
+            - name: "PCAPIS_CONTAINER_CONFIG__ACCOUNT_KEY"
+              value: "{{ .Values.tiler.storage.account_key }}"
+            - name: "PCAPIS_CONTAINER_CONFIG__TABLE_NAME"
+              value: "{{ .Values.tiler.storage.container_config_table_name }}"
             - name: APP_INSIGHTS_INSTRUMENTATION_KEY
               value: "{{ .Values.metrics.instrumentationKey }}"
 

deployment/helm/published/planetary-computer-tiler/values.yaml

@@ -20,6 +20,12 @@ tiler:
     port: 80
     annotations: {}
 
+  storage:
+    account_name: ""
+    account_key: ""
+    collection_config_table_name: ""
+    container_config_table_name: ""
+
   deploy:
     replicaCount: 10
     podAnnotations: {}

deployment/terraform/dev/main.tf

@@ -2,22 +2,14 @@ variable "username" {
   type = string
 }
 
-variable "cluster_cert_issuer" {
-  type    = string
-  default = "letsencrypt-staging"
-}
-
-variable "cluster_cert_server" {
-  type    = string
-  default = "https://acme-staging-v02.api.letsencrypt.org/directory"
-}
-
 module "resources" {
   source = "../resources"
 
   environment = var.username
   region      = "West Europe"
 
+  k8s_version = "1.22.4"
+
   cluster_cert_issuer = "letsencrypt"
   cluster_cert_server = "https://acme-v02.api.letsencrypt.org/directory"
 

deployment/terraform/resources/acr.tf

@@ -0,0 +1,11 @@
+data "azurerm_container_registry" "pc" {
+  name                = var.pc_test_resources_acr
+  resource_group_name = var.pc_test_resources_rg
+}
+
+# add the role to the identity the kubernetes cluster was assigned
+resource "azurerm_role_assignment" "attach_acr" {
+  scope                = data.azurerm_container_registry.pc.id
+  role_definition_name = "AcrPull"
+  principal_id         = azurerm_kubernetes_cluster.pc.kubelet_identity[0].object_id
+}

deployment/terraform/resources/aks.tf

@@ -3,7 +3,7 @@ resource "azurerm_kubernetes_cluster" "pc" {
   location            = azurerm_resource_group.pc.location
   resource_group_name = azurerm_resource_group.pc.name
   dns_prefix          = "${local.prefix}-cluster"
-  kubernetes_version  = "1.20.7"
+  kubernetes_version  = var.k8s_version
 
   addon_profile {
     kube_dashboard {

deployment/terraform/resources/output.tf

@@ -1,9 +1,9 @@
 output "environment" {
-    value = var.environment
+  value = var.environment
 }
 
 output "location" {
-    value = local.location
+  value = local.location
 }
 
 output "cluster_name" {
@@ -33,8 +33,8 @@ output "pg_database" {
 }
 
 output "pg_password" {
-    value = data.azurerm_key_vault_secret.db_admin_password.value
-    sensitive = true
+  value     = data.azurerm_key_vault_secret.db_admin_password.value
+  sensitive = true
 }
 
 # Helm pass-through vars
@@ -73,3 +73,21 @@ output "tiler_replica_count" {
 output "instrumentation_key" {
   value = azurerm_application_insights.pc_application_insights.instrumentation_key
 }
+
+## Storage
+
+output "storage_account_name" {
+  value = azurerm_storage_account.pc.name
+}
+
+output "storage_account_key" {
+  value = azurerm_storage_account.pc.primary_access_key
+}
+
+output "collection_config_table_name" {
+  value = azurerm_storage_table.collectionconfig.name
+}
+
+output "container_config_table_name" {
+  value = azurerm_storage_table.containerconfig.name
+}

deployment/terraform/resources/storage_account.tf

@@ -0,0 +1,19 @@
+resource "azurerm_storage_account" "pc" {
+  name                     = "${local.nodash_prefix}sa"
+  resource_group_name      = azurerm_resource_group.pc.name
+  location                 = azurerm_resource_group.pc.location
+  account_tier             = "Standard"
+  account_replication_type = "LRS"
+}
+
+# Tables
+
+resource "azurerm_storage_table" "collectionconfig" {
+  name                 = "collectionconfig"
+  storage_account_name = azurerm_storage_account.pc.name
+}
+
+resource "azurerm_storage_table" "containerconfig" {
+  name                 = "containerconfig"
+  storage_account_name = azurerm_storage_account.pc.name
+}

deployment/terraform/resources/variables.tf

@@ -16,6 +16,11 @@ variable "pc_test_resources_kv" {
   default = "pc-test-deploy-secrets"
 }
 
+variable "pc_test_resources_acr" {
+  type    = string
+  default = "pccomponentstest"
+}
+
 variable "aks_node_count" {
   type = number
 }
@@ -41,6 +46,10 @@ variable "tiler_replica_count" {
   type = number
 }
 
+variable "k8s_version" {
+  type = string
+}
+
 # -- Postgres
 
 variable "pg_host" {
@@ -76,4 +85,5 @@ locals {
   stack_id              = "pct-apis"
   location              = lower(replace(var.region, " ", ""))
   prefix                = "${local.stack_id}-${local.location}-${var.environment}"
+  nodash_prefix         = replace("${local.stack_id}${var.environment}", "-", "")
 }

deployment/terraform/staging/main.tf

@@ -4,6 +4,8 @@ module "resources" {
   environment = "staging"
   region = "West Europe"
 
+  k8s_version = "1.20.7"
+
   cluster_cert_issuer = "letsencrypt"
   cluster_cert_server = "https://acme-v02.api.letsencrypt.org/directory"
 

docker-compose.dev.yml

@@ -13,6 +13,17 @@ services:
       - POSTGRES_HOST_READER=database
       - POSTGRES_HOST_WRITER=database
       - POSTGRES_PORT=5432
+
+      # Azure Storage
+      - PCAPIS_COLLECTION_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1
+      - PCAPIS_COLLECTION_CONFIG__ACCOUNT_NAME=devstoreaccount1
+      - PCAPIS_COLLECTION_CONFIG__ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+      - PCAPIS_COLLECTION_CONFIG__TABLE_NAME=collectionconfig
+
+      - PCAPIS_CONTAINER_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1
+      - PCAPIS_CONTAINER_CONFIG__ACCOUNT_NAME=devstoreaccount1
+      - PCAPIS_CONTAINER_CONFIG__ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+      - PCAPIS_CONTAINER_CONFIG__TABLE_NAME=containerconfig
     volumes:
       - .:/opt/src
     command: >
@@ -43,6 +54,17 @@ services:
       - POSTGRES_PORT=5432
       - WEB_CONCURRENCY=1
       - WORKERS_PER_CORE=1
+
+      # Azure Storage
+      - PCAPIS_COLLECTION_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1
+      - PCAPIS_COLLECTION_CONFIG__ACCOUNT_NAME=devstoreaccount1
+      - PCAPIS_COLLECTION_CONFIG__ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+      - PCAPIS_COLLECTION_CONFIG__TABLE_NAME=collectionconfig
+
+      - PCAPIS_CONTAINER_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1
+      - PCAPIS_CONTAINER_CONFIG__ACCOUNT_NAME=devstoreaccount1
+      - PCAPIS_CONTAINER_CONFIG__ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+      - PCAPIS_CONTAINER_CONFIG__TABLE_NAME=containerconfig
     volumes:
       - .:/opt/src
     command: >