Setup (Cloud-Backed) SQLite backend for reading GeoPackages

PDOK · Oct 9, 2023 · e1b6dcf · e1b6dcf
1 parent af2a67e
commit e1b6dcf
Show file tree

Hide file tree

Showing 13 changed files with 350 additions and 51 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,33 +1,52 @@
 ARG REGISTRY="docker.io"
+
+####### Node.js build
 FROM ${REGISTRY}/node:lts-alpine3.17 AS build-component
 RUN mkdir -p /usr/src/app
 COPY ./webcomponents/vectortile-view-component /usr/src/app
 WORKDIR /usr/src/app
 RUN npm install
 RUN npm run build
 
-FROM ${REGISTRY}/golang:1.20 AS build-env
-
+####### Go build
+FROM ${REGISTRY}/golang:1.20-bookworm AS build-env
 WORKDIR /go/src/service
 ADD . /go/src/service
 
-# disable crosscompiling
-ENV CGO_ENABLED=0
-# compile linux only
+# enable cgo in order to interface with sqlite
+ENV CGO_ENABLED=1
 ENV GOOS=linux
 
+# install cloud-backed sqlite compile-time dependencies
+RUN set -eux && \
+	apt-get update && \
+	apt-get install -y \
+		libcurl4-openssl-dev \
+        libssl-dev && \
+	rm -rf /var/lib/apt/lists/*
+
 RUN go mod download all
 
 # build the binary with debug information removed.
 # also run tests, the short flag skips integration tests since we can't run Testcontainers in multistage Docker :-(
-RUN go test -short && go build -v -ldflags '-w -s' -a -installsuffix cgo -o /gokoala github.com/PDOK/gokoala
+RUN go test -short && \
+    go build -v -ldflags '-w -s' -a -installsuffix cgo -o /gokoala github.com/PDOK/gokoala
 
 # delete all go files (and testdata dirs) so only assets/templates/etc remain, since in a later
 # stage we need to copy these remaining files including their subdirectories to the final docker image.
 RUN find . -type f -name "*.go" -delete && find . -type d -name "testdata" -prune -exec rm -rf {} \;
 
-##########################################################
-FROM scratch
+####### Final image
+FROM ${REGISTRY}/debian:bookworm-slim
+
+# install cloud-backed sqlite runtime dependencies
+RUN set -eux && \
+    apt-get update && \
+	apt-get install -y \
+		libcurl4 \
+        openssl && \
+	rm -rf /var/lib/apt/lists/*
+
 EXPOSE 8080
 # use the WORKDIR to create a /tmp folder, mkdir is not available
 WORKDIR /tmp

diff --git a/engine/config.go b/engine/config.go
@@ -252,21 +252,41 @@ type OgcAPIProcesses struct {
 type Datasource struct {
 	GeoPackage *GeoPackage `yaml:"geopackage" validate:"required_without_all=FakeDB"`
 	FakeDB     bool        `yaml:"fakedb" validate:"required_without_all=GeoPackage"`
-	// Add more datasources here such as PostGIS, Mongo, etc
+	// Add more datasources here such as PostGIS, Mongo, Elastic, etc
 }
 
 type GeoPackage struct {
-	File  GeoPackageFile  `yaml:"file"`
-	Azure GeoPackageAzure `yaml:"azure"`
+	Local *GeoPackageLocal `yaml:"local" validate:"required_without_all=Cloud"`
+	Cloud *GeoPackageCloud `yaml:"cloud" validate:"required_without_all=Local"`
 }
 
-type GeoPackageFile struct {
-	Filepath string  `yaml:"filepath" validate:"filepath"`
-	Fid      *string `yaml:"fid"`
+type GeoPackageLocal struct {
+	File string  `yaml:"file" validate:"filepath"`
+	Fid  *string `yaml:"fid"`
 }
 
-type GeoPackageAzure struct {
-	// TODO: settings for Azure Cloud Backed Sqlite
+// GeoPackageCloud settings to read a GeoPackage as a Cloud-Backed SQLite database
+type GeoPackageCloud struct {
+	// reference to the cloud storage (either azure or google at the moment), e.g:
+	// - azure?emulator=127.0.0.1:10000&sas=0
+	// - google
+	Connection string `yaml:"connection" validate:"required"`
+
+	// username of the storage account, e.g: devstoreaccount1 when using Azurite
+	User string `yaml:"user" validate:"required"`
+
+	// some kind of credential like a password or key to authenticate with the storage backend, e.g:
+	// 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==' when using Azurite
+	Auth string `yaml:"auth" validate:"required"`
+
+	// container/bucket on the storage account
+	Container string `yaml:"container" validate:"required"`
+
+	// filename of the GeoPackage
+	File string `yaml:"file" validate:"required"`
+
+	// local cache of fetched blocks from cloud storage
+	Cache *string `yaml:"cache" validate:"omitempty,dir"`
 }
 
 type SupportedSrs struct {

diff --git a/examples/chinook.db b/examples/chinook.db
diff --git a/examples/config_features_azure.yaml b/examples/config_features_azure.yaml
@@ -0,0 +1,83 @@
+---
+collectionMetadata: &exampleMetadata
+  description: >-
+    This is a description about lorum ipsum
+  keywords:
+    - Keyword1
+    - Keyword2
+  thumbnail: old.png
+  lastUpdated: "2030-01-02T12:00:00Z"
+  extent:
+    srs: EPSG:3857
+    bbox: ["-13512504.4191", "-19500889.3517", "31493617.8353", "19869683.6812"]
+
+version: 1.0.0
+title: Basisregistratie Grootschalige Topografie (BGT)
+# shortened title, used in breadcrumb path
+serviceIdentifier: BGT
+abstract: >-
+  This is a description about the dataset in Markdown. See [PDOK BGT dataset](https://www.pdok.nl/introductie/-/article/basisregistratie-grootschalige-topografie-bgt-) for details.
+# just a dummy picture, but you can put an actual thumbnail here
+thumbnail: bgt.png
+resources:
+  directory: ./examples/resources
+keywords:
+  - keyword1
+  - keyword2
+lastUpdated: "2023-09-08T12:00:00Z"
+license:
+  name: CC0 1.0
+  url: https://creativecommons.org/publicdomain/zero/1.0/deed.nl
+support:
+  name: Example Support
+  email: support@example.com
+  url: https://support.example.com
+# further details of the dataset to be shown on the landing page; supports markdown
+datasetDetails:
+  - name: Dataset-aanbieder
+    value: Kadaster (LV-BGT)
+  - name: Kosten afname
+    value: Geen
+datasetMetadata:
+  source: Example Register
+  api: https://example.com/my-api/metadata
+  dataset: https://example.com/my-dataset/metadata
+datasetCatalogUrl: https://www.pdok.nl/datasets
+baseUrl: http://localhost:8080
+availableLanguages:
+  - nl
+  - en
+ogcApi:
+  # which OGC apis to enable. Possible values: tiles, styles, features, maps
+  features:
+    datasource:
+      geopackage:
+        cloud:
+          # connect to Azurite docker container (docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite azurite-blob --blobHost 0.0.0.0)
+          connection: azure?emulator=172.17.0.1:10000
+          user: devstoreaccount1
+          auth: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+          container: example
+          file: chinook.db
+    collections:
+      - id: wegdelen
+        metadata:
+          # inherit common metadata (to tidy example)
+          <<: *exampleMetadata
+          title: Wegdelen
+      - id: panden
+        metadata:
+          <<: *exampleMetadata
+          title: Panden
+      - id: tunneldelen
+        metadata:
+          <<: *exampleMetadata
+          title: Tunneldelen
+      - id: waterdelen
+        metadata:
+          <<: *exampleMetadata
+          title: Waterdelen
+      - id: spoordelen
+        metadata:
+          <<: *exampleMetadata
+          title: Spoordelen
diff --git a/go.mod b/go.mod
@@ -4,6 +4,7 @@ go 1.20
 
 require (
 	github.com/BurntSushi/toml v1.3.2
+	github.com/PDOK/go-cloud-sqlite-vfs v0.2.3
 	github.com/brianvoe/gofakeit/v6 v6.23.2
 	github.com/elnormous/contenttype v1.0.4
 	github.com/getkin/kin-openapi v0.116.0
@@ -12,32 +13,51 @@ require (
 	github.com/go-spatial/geom v0.0.0-20220918193402-3cd2f5a9a082
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572
 	github.com/gomarkdown/markdown v0.0.0-20230322041520-c84983bdbf2a
+	github.com/jmoiron/sqlx v1.3.5
+	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/nicksnyder/go-i18n/v2 v2.2.1
 	github.com/sqids/sqids-go v0.4.1
 	github.com/stretchr/testify v1.8.2
 	github.com/urfave/cli/v2 v2.25.3
 	github.com/writeas/go-strip-markdown/v2 v2.1.1
 	golang.org/x/text v0.9.0
 	gopkg.in/yaml.v3 v3.0.1
+	modernc.org/sqlite v1.26.0
 )
 
 require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/invopop/yaml v0.2.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/leodido/go-urn v1.2.3 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/perimeterx/marshmallow v1.1.4 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	golang.org/x/crypto v0.9.0 // indirect
+	golang.org/x/mod v0.8.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/tools v0.6.0 // indirect
+	lukechampine.com/uint128 v1.2.0 // indirect
+	modernc.org/cc/v3 v3.40.0 // indirect
+	modernc.org/ccgo/v3 v3.16.13 // indirect
+	modernc.org/libc v1.24.1 // indirect
+	modernc.org/mathutil v1.5.0 // indirect
+	modernc.org/memory v1.6.0 // indirect
+	modernc.org/opt v0.1.3 // indirect
+	modernc.org/strutil v1.1.3 // indirect
+	modernc.org/token v1.0.1 // indirect
 )