Setup (Cloud-Backed) SQLite backend for reading GeoPackages

.github/workflows/build-and-publish-image.yml

@@ -1,3 +1,4 @@
+---
 name: build
 env:
   image: pdok/gokoala

.github/workflows/build-and-publish-webcomponent-on-github-pages.yml

@@ -1,6 +1,5 @@
-
+---
 name: deploy web components
-
 on:
   push:
     branches: ["master"]

.github/workflows/lint-go.yml

@@ -1,3 +1,4 @@
+---
 name: lint (go)
 on:
   push:
@@ -11,11 +12,16 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
+      - name: setup cgo dependencies
+        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
+
       - uses: actions/setup-go@v4
         with:
           go-version: '1.20'
           cache: false
+
       - uses: actions/checkout@v3
+
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:

.github/workflows/lint-ts.yml

@@ -1,5 +1,5 @@
+---
 name: lint (web/typescript)
-
 on:
   push:
     branches:
@@ -11,7 +11,7 @@ defaults:
     working-directory: ./webcomponents/vectortile-view-component
 
 jobs:
-  build:
+  lint-ts:
     runs-on: ubuntu-latest
 
     strategy:

.github/workflows/test-example.yml

@@ -0,0 +1,12 @@
+---
+name: test-examples
+on:
+  pull_request:
+jobs:
+  test-examples:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: test OGC API Features with Azure GeoPackage
+        run: docker-compose -f ./examples/docker-compose-features-azure.yaml up --exit-code-from smoketest

.github/workflows/test.yml

@@ -1,13 +1,16 @@
+---
 name: test
-
 on:
   pull_request:
 jobs:
-  build:
+  test:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
 
+      - name: setup cgo dependencies
+        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
+
       - name: Set up Go
         uses: actions/setup-go@v3
         with:

.github/workflows/tidy.yml

@@ -1,3 +1,4 @@
+---
 name: tidy
 on:
   push:

Dockerfile

@@ -1,35 +1,49 @@
 ARG REGISTRY="docker.io"
+
+####### Node.js build
 FROM ${REGISTRY}/node:lts-alpine3.17 AS build-component
 RUN mkdir -p /usr/src/app
 COPY ./webcomponents/vectortile-view-component /usr/src/app
 WORKDIR /usr/src/app
 RUN npm install
 RUN npm run build
 
-FROM ${REGISTRY}/golang:1.20 AS build-env
-
+####### Go build
+FROM ${REGISTRY}/golang:1.20-bookworm AS build-env
 WORKDIR /go/src/service
 ADD . /go/src/service
 
-# disable crosscompiling
-ENV CGO_ENABLED=0
-# compile linux only
+# enable cgo in order to interface with sqlite
+ENV CGO_ENABLED=1
 ENV GOOS=linux
 
+# install cloud-backed sqlite compile-time dependencies
+RUN set -eux && \
+    apt-get update && \
+    apt-get install -y libcurl4-openssl-dev libssl-dev && \
+    rm -rf /var/lib/apt/lists/*
+
 RUN go mod download all
 
-# build the binary with debug information removed.
-# also run tests, the short flag skips integration tests since we can't run Testcontainers in multistage Docker :-(
-RUN go test -short && go build -v -ldflags '-w -s' -a -installsuffix cgo -o /gokoala github.com/PDOK/gokoala
+# build & test the binary with debug information removed.
+RUN go test -short && \
+    go build -v -ldflags '-w -s' -a -installsuffix cgo -o /gokoala github.com/PDOK/gokoala
 
 # delete all go files (and testdata dirs) so only assets/templates/etc remain, since in a later
 # stage we need to copy these remaining files including their subdirectories to the final docker image.
 RUN find . -type f -name "*.go" -delete && find . -type d -name "testdata" -prune -exec rm -rf {} \;
 
-##########################################################
-FROM scratch
+####### Final image (use debian tag since we rely on C-libs)
+FROM ${REGISTRY}/debian:bookworm-slim
+
+# install cloud-backed sqlite runtime dependencies
+RUN set -eux && \
+    apt-get update && \
+    apt-get install -y libcurl4 openssl && \
+    rm -rf /var/lib/apt/lists/*
+
 EXPOSE 8080
-# use the WORKDIR to create a /tmp folder, mkdir is not available
+# use the WORKDIR to create a /tmp folder
 WORKDIR /tmp
 WORKDIR /
 ENV PATH=/

README.md

@@ -7,7 +7,8 @@
 _Cloud Native OGC APIs server, written in Go._ 
 
 [![Build](https://github.com/PDOK/gokoala/actions/workflows/build-and-publish-image.yml/badge.svg)](https://github.com/PDOK/gokoala/actions/workflows/build-and-publish-image.yml)
-[![Lint](https://github.com/PDOK/gokoala/actions/workflows/lint.yml/badge.svg)](https://github.com/PDOK/gokoala/actions/workflows/lint.yml)
+[![Lint (go)](https://github.com/PDOK/gokoala/blob/master/.github/workflows/lint-go.yml/badge.svg)](https://github.com/PDOK/gokoala/actions/workflows/lint-go.yml)
+[![Lint (ts)](https://github.com/PDOK/gokoala/blob/master/.github/workflows/lint-ts.yml/badge.svg)](https://github.com/PDOK/gokoala/actions/workflows/lint-ts.yml)
 [![Go Report Card](https://goreportcard.com/badge/github.com/PDOK/gokoala)](https://goreportcard.com/report/github.com/PDOK/gokoala)
 [![GitHub
 license](https://img.shields.io/github/license/PDOK/gokoala)](https://github.com/PDOK/gokoala/blob/master/LICENSE)
@@ -77,7 +78,7 @@ Example (config-file is mandatory):
 docker run -v `pwd`/examples:/examples -p 8080:8080 -it pdok/gokoala --config-file /examples/config_vectortiles.yaml
 ```
 
-Now open <http://localhost:8080>
+Now open <http://localhost:8080>. See [examples](examples) for more details.
 
 ### Configuration file
 
@@ -96,7 +97,7 @@ ogcApi:
 ### OpenAPI spec
 
 GoKoala ships with OGC OpenAPI support out of the box, see [OpenAPI
-specs](assets/openapi-specs/README.md) for details. You can overwrite or extend
+specs](engine/templates/openapi) for details. You can overwrite or extend
 the defaults by providing your own spec using the `openapi-file` CLI flag.
 
 ### Observability
@@ -128,6 +129,7 @@ A similar flow can be used to profile memory issues.
 Design principles:
 
 - Performance and scalability are key!
+- Be opinionated when you can, only make stuff configurable when you must.
 - The `ogc` [package](ogc/README.md) contains logic per specific OGC API
   building block.
 - The `engine` package should contain general logic. `ogc` may reference
@@ -139,6 +141,7 @@ Design principles:
   of during request handling.
 - Assets/templates/etc should be explicitly included in the Docker image, see COPY
   commands in [Dockerfile](Dockerfile).
+- Document your changes to [OGC OpenAPI example specs](engine/templates/openapi/README.md).
 
 ### Linting
 

assets/i18n/active.en.toml

@@ -3,7 +3,6 @@ ViewAs = "View as"
 AvailableIn = "The page is available in both HTML and JSON."
 LastUpdated = "Last updated"
 UpdatedBy = "Updated by"
-On = "on"
 Keywords = "Keywords"
 License = "License"
 Support = "Support"
@@ -13,6 +12,8 @@ LinkToExternalWebsite = "link to external website"
 Description = "Description"
 None = "None"
 Example = "Example"
+On = "on"
+As = "as"
 
 # Layout frame
 ToMain = "To main content"
@@ -89,6 +90,8 @@ ViewCollectionAs = "View collection as"
 Extent = "Geographic extent"
 GoTo = "Go to the"
 ViewIn = "View in the"
+Browse = "Browse through the"
+FeaturesExplanation = "TODO Explain here GeoJSON vs JSON-FG"
 
 # Features page
 Geometry = "geometry"

assets/i18n/active.nl.toml

@@ -3,7 +3,6 @@ ViewAs = "Bekijk als"
 AvailableIn = "De pagina is zowel in HTML als JSON beschikbaar."
 LastUpdated = "Bijgewerkt tot"
 UpdatedBy = "Bijgewerkt door"
-On = "op"
 Keywords = "Trefwoorden"
 License = "Licentie"
 Support = "Support"
@@ -13,6 +12,8 @@ LinkToExternalWebsite = "link naar externe website"
 Description = "Omschrijving"
 None = "Geen"
 Example = "Voorbeeld"
+On = "op"
+As = "als"
 
 # Layout frame
 ToMain = "Naar hoofdinhoud"
@@ -94,6 +95,8 @@ ViewCollectionAs = "Bekijk collectie als"
 Extent = "Geografische begrenzing"
 GoTo = "Ga naar de"
 ViewIn = "Bekijk in de"
+Browse = "Blader door de"
+FeaturesExplanation = "Uitleg over welke JSON, wanneer kies je voor GeoJSON en wanneer voor JSON-FG. Verschil tussen projecties, etc."
 
 # Features page
 Geometry = "geometrie"

engine/config.go

@@ -7,14 +7,16 @@ import (
 	"os"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/go-playground/validator/v10"
 	"golang.org/x/text/language"
 	"gopkg.in/yaml.v3"
 )
 
 const (
-	cookieMaxAge = 60 * 60 * 24
+	cookieMaxAge        = 60 * 60 * 24
+	defaultQueryTimeout = 10 * time.Second
 )
 
 func readConfigFile(configFile string) *Config {
@@ -252,21 +254,62 @@ type OgcAPIProcesses struct {
 type Datasource struct {
 	GeoPackage *GeoPackage `yaml:"geopackage" validate:"required_without_all=FakeDB"`
 	FakeDB     bool        `yaml:"fakedb" validate:"required_without_all=GeoPackage"`
-	// Add more datasources here such as PostGIS, Mongo, etc
+	// Add more datasources here such as PostGIS, Mongo, Elastic, etc
 }
 
 type GeoPackage struct {
-	File  GeoPackageFile  `yaml:"file"`
-	Azure GeoPackageAzure `yaml:"azure"`
+	Local *GeoPackageLocal `yaml:"local" validate:"required_without_all=Cloud"`
+	Cloud *GeoPackageCloud `yaml:"cloud" validate:"required_without_all=Local"`
 }
 
-type GeoPackageFile struct {
-	Filepath string  `yaml:"filepath" validate:"filepath"`
-	Fid      *string `yaml:"fid"`
+// GeoPackageCommon shared config between local and cloud GeoPackage
+type GeoPackageCommon struct {
+	// feature id column name
+	Fid string `yaml:"fid" validate:"required"`
+
+	// optional timeout after which queries are canceled (default is 10s, see constant)
+	QueryTimeout *time.Duration `yaml:"queryTimeout"`
+}
+
+func (gc *GeoPackageCommon) GetQueryTimeout() time.Duration {
+	if gc.QueryTimeout != nil {
+		return *gc.QueryTimeout
+	}
+	return defaultQueryTimeout
+}
+
+// GeoPackageLocal settings to read a GeoPackage from local disk
+type GeoPackageLocal struct {
+	GeoPackageCommon `yaml:",inline"`
+
+	// location of GeoPackage on disk
+	File string `yaml:"file" validate:"file"`
 }
 
-type GeoPackageAzure struct {
-	// TODO: settings for Azure Cloud Backed Sqlite
+// GeoPackageCloud settings to read a GeoPackage as a Cloud-Backed SQLite database
+type GeoPackageCloud struct {
+	GeoPackageCommon `yaml:",inline"`
+
+	// reference to the cloud storage (either azure or google at the moment), e.g:
+	// - azure?emulator=127.0.0.1:10000&sas=0
+	// - google
+	Connection string `yaml:"connection" validate:"required"`
+
+	// username of the storage account, e.g: devstoreaccount1 when using Azurite
+	User string `yaml:"user" validate:"required"`
+
+	// some kind of credential like a password or key to authenticate with the storage backend, e.g:
+	// 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==' when using Azurite
+	Auth string `yaml:"auth" validate:"required"`
+
+	// container/bucket on the storage account
+	Container string `yaml:"container" validate:"required"`
+
+	// filename of the GeoPackage
+	File string `yaml:"file" validate:"required"`
+
+	// local cache of fetched blocks from cloud storage
+	Cache *string `yaml:"cache" validate:"omitempty,dir"`
 }
 
 type SupportedSrs struct {

engine/openapi.go

@@ -74,7 +74,7 @@ func newOpenAPI(config *Config, openAPIFile string) *OpenAPI {
 
 	for _, server := range resultSpec.Servers {
 		server.URL = normalizeBaseURL(server.URL)
-		log.Printf("URL used for OpenAPI validation: %v", server.URL)
+		log.Printf("url used for OpenAPI validation: %v", server.URL)
 	}
 
 	return &OpenAPI{

engine/slices.go

@@ -19,3 +19,13 @@ func Index[E comparable](s []E, v E) int {
 	}
 	return -1
 }
+
+// Keys returns the keys of the map m.
+// The keys will be an indeterminate order.
+func Keys[M ~map[K]V, K comparable, V any](m M) []K {
+	r := make([]K, 0, len(m))
+	for k := range m {
+		r = append(r, k)
+	}
+	return r
+}