Merge branch 'master' into map-component-for-features

.github/workflows/lint-go.yml

@@ -13,11 +13,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: setup cgo dependencies
-        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
+        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev libsqlite3-mod-spatialite
 
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21'
           cache: false
 
       - uses: actions/checkout@v3
@@ -26,7 +26,7 @@ jobs:
         uses: golangci/golangci-lint-action@v3
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-          version: v1.52
+          version: latest
 
           # Optional: working directory, useful for monorepos
           # working-directory: somedir

.github/workflows/test.yml

@@ -1,6 +1,9 @@
 ---
 name: test
 on:
+  push:
+    branches:
+      - master
   pull_request:
 jobs:
   test:
@@ -9,7 +12,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: setup cgo dependencies
-        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
+        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev libsqlite3-mod-spatialite
 
       - name: Set up Go
         uses: actions/setup-go@v3

.github/workflows/tidy.yml

@@ -14,6 +14,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - uses: actions/checkout@v3
       - uses: katexochen/go-tidy-check@v2

Dockerfile

@@ -9,18 +9,18 @@ RUN npm install
 RUN npm run build
 
 ####### Go build
-FROM ${REGISTRY}/golang:1.20-bookworm AS build-env
+FROM ${REGISTRY}/golang:1.21-bookworm AS build-env
 WORKDIR /go/src/service
 ADD . /go/src/service
 
 # enable cgo in order to interface with sqlite
 ENV CGO_ENABLED=1
 ENV GOOS=linux
 
-# install cloud-backed sqlite compile-time dependencies
+# install sqlite-related compile-time dependencies
 RUN set -eux && \
     apt-get update && \
-    apt-get install -y libcurl4-openssl-dev libssl-dev && \
+    apt-get install -y libcurl4-openssl-dev libssl-dev libsqlite3-mod-spatialite && \
     rm -rf /var/lib/apt/lists/*
 
 RUN go mod download all
@@ -36,10 +36,10 @@ RUN find . -type f -name "*.go" -delete && find . -type d -name "testdata" -prun
 ####### Final image (use debian tag since we rely on C-libs)
 FROM ${REGISTRY}/debian:bookworm-slim
 
-# install cloud-backed sqlite runtime dependencies
+# install sqlite-related runtime dependencies
 RUN set -eux && \
     apt-get update && \
-    apt-get install -y libcurl4 openssl && \
+    apt-get install -y libcurl4 openssl libsqlite3-mod-spatialite && \
     rm -rf /var/lib/apt/lists/*
 
 EXPOSE 8080
@@ -63,9 +63,6 @@ COPY --from=build-component /usr/src/app/dist/vectortile-view-component/3rdparty
 COPY --from=build-env /go/src/service/engine/ /engine/
 COPY --from=build-env /go/src/service/ogc/ /ogc/
 
-COPY --from=build-env /usr/share/zoneinfo /usr/share/zoneinfo
-COPY --from=build-env /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-
 # run as non-root
 USER 1001
 ENTRYPOINT ["gokoala"]

README.md

@@ -124,6 +124,10 @@ through a tunnel/port-forward. The debug server exposes `/debug` for use by
 
 A similar flow can be used to profile memory issues.
 
+#### SQL query logging
+
+Set `LOG_SQL=true` to enable query logging stdout for debug purposes. Only applies to OGC API Features.
+
 ## Develop
 
 Design principles:

engine/config.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/creasty/defaults"
 	"github.com/go-playground/validator/v10"
 	"golang.org/x/text/language"
 	"gopkg.in/yaml.v3"
@@ -40,11 +41,15 @@ func readConfigFile(configFile string) *Config {
 }
 
 func setDefaults(config *Config) {
+	// process 'default' tags
+	if err := defaults.Set(config); err != nil {
+		log.Fatalf("failed to set default configuration: %v", err)
+	}
+
 	config.CookieMaxAge = cookieMaxAge
 
 	if len(config.AvailableLanguages) == 0 {
-		// default to Dutch only
-		config.AvailableLanguages = append(config.AvailableLanguages, language.Dutch)
+		config.AvailableLanguages = append(config.AvailableLanguages, language.Dutch) // default to Dutch only
 	}
 }
 
@@ -258,6 +263,7 @@ type OgcAPIStyles struct {
 }
 
 type OgcAPIFeatures struct {
+	Limit       Limit                 `yaml:"limit"`
 	Collections GeoSpatialCollections `yaml:"collections" validate:"required"`
 	Datasource  Datasource            `yaml:"datasource" validate:"required"`
 }
@@ -272,10 +278,19 @@ type OgcAPIProcesses struct {
 	ProcessesServer  YAMLURL `yaml:"processesServer" validate:"url"`
 }
 
+type Limit struct {
+	Default int `yaml:"default" validate:"gt=1" default:"10"`
+	Max     int `yaml:"max" validate:"gt=1" default:"1000"`
+}
+
 type Datasource struct {
-	GeoPackage *GeoPackage `yaml:"geopackage" validate:"required_without_all=FakeDB"`
-	FakeDB     bool        `yaml:"fakedb" validate:"required_without_all=GeoPackage"`
-	// Add more datasources here such as PostGIS, Mongo, Elastic, etc
+	GeoPackage *GeoPackage `yaml:"geopackage" validate:"required_without_all=PostGIS"`
+	PostGIS    *PostGIS    `yaml:"postgis" validate:"required_without_all=GeoPackage"`
+	// Add more datasources here such as Mongo, Elastic, etc
+}
+
+type PostGIS struct {
+	// placeholder
 }
 
 type GeoPackage struct {

engine/contentnegotiation.go

@@ -9,7 +9,7 @@ import (
 )
 
 const (
-	formatParam   = "f"
+	FormatParam   = "f"
 	languageParam = "lang"
 
 	MediaTypeJSON          = "application/json"
@@ -127,11 +127,11 @@ func (cn *ContentNegotiation) formatToMediaType(format string) string {
 func (cn *ContentNegotiation) getFormatFromQueryParam(req *http.Request) string {
 	var requestedFormat = ""
 	queryParams := req.URL.Query()
-	if queryParams.Get(formatParam) != "" {
-		requestedFormat = queryParams.Get(formatParam)
+	if queryParams.Get(FormatParam) != "" {
+		requestedFormat = queryParams.Get(FormatParam)
 
 		// remove ?f= parameter, to prepare for rewrite
-		queryParams.Del(formatParam)
+		queryParams.Del(FormatParam)
 		req.URL.RawQuery = queryParams.Encode()
 	}
 	return requestedFormat

engine/openapi.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 	texttemplate "text/template"
 
+	"github.com/PDOK/gokoala/engine/util"
 	"github.com/getkin/kin-openapi/openapi3"
 	"github.com/getkin/kin-openapi/openapi3filter"
 	"github.com/getkin/kin-openapi/routers"
@@ -80,7 +81,7 @@ func newOpenAPI(config *Config, openAPIFile string) *OpenAPI {
 	return &OpenAPI{
 		config:   config,
 		spec:     resultSpec,
-		SpecJSON: prettyPrintJSON(resultSpecJSON, ""),
+		SpecJSON: util.PrettyPrintJSON(resultSpecJSON, ""),
 		router:   newOpenAPIRouter(resultSpec),
 	}
 }
@@ -140,10 +141,10 @@ func mergeSpecs(ctx context.Context, config *Config, files []string) (*openapi3.
 			mergedJSON = specJSON
 		} else {
 			var err error
-			mergedJSON, err = mergeJSON(resultSpecJSON, specJSON)
+			mergedJSON, err = util.MergeJSON(resultSpecJSON, specJSON)
 			if err != nil {
 				log.Print(string(mergedJSON))
-				log.Fatalf("failed to merge openapi specs: %v", err)
+				log.Fatalf("failed to merge OpenAPI specs: %v", err)
 			}
 		}
 		resultSpecJSON = mergedJSON
@@ -156,7 +157,7 @@ func loadSpec(loader *openapi3.Loader, mergedJSON []byte, fileName ...string) *o
 	resultSpec, err := loader.LoadFromData(mergedJSON)
 	if err != nil {
 		log.Print(string(mergedJSON))
-		log.Fatalf("failed to load merged openapi spec %s, due to %v", fileName, err)
+		log.Fatalf("failed to load merged OpenAPI spec %s, due to %v", fileName, err)
 	}
 	return resultSpec
 }
@@ -166,14 +167,14 @@ func validateSpec(ctx context.Context, finalSpec *openapi3.T, finalSpecRaw []byt
 	err := finalSpec.Validate(ctx, openapi3.DisableExamplesValidation())
 	if err != nil {
 		log.Print(string(finalSpecRaw))
-		log.Fatalf("invalid openapi spec: %v", err)
+		log.Fatalf("invalid OpenAPI spec: %v", err)
 	}
 }
 
 func newOpenAPIRouter(doc *openapi3.T) routers.Router {
 	openAPIRouter, err := gorillamux.NewRouter(doc)
 	if err != nil {
-		log.Fatalf("failed to setup openapi router: %v", err)
+		log.Fatalf("failed to setup OpenAPI router: %v", err)
 	}
 	return openAPIRouter
 }
@@ -194,7 +195,7 @@ func (o *OpenAPI) validateRequest(r *http.Request) error {
 	if requestValidationInput != nil {
 		err := openapi3filter.ValidateRequest(context.Background(), requestValidationInput)
 		if err != nil {
-			return fmt.Errorf("invalid request, doesn't conform to openapi spec %w", err)
+			return fmt.Errorf("request doesn't conform to OpenAPI spec: %w", err)
 		}
 	}
 	return nil
@@ -214,7 +215,7 @@ func (o *OpenAPI) validateResponse(contentType string, body []byte, r *http.Requ
 		responseValidationInput.SetBodyBytes(body)
 		err := openapi3filter.ValidateResponse(context.Background(), responseValidationInput)
 		if err != nil {
-			return fmt.Errorf("response doesn't conform to openapi spec: %w", err)
+			return fmt.Errorf("response doesn't conform to OpenAPI spec: %w", err)
 		}
 	}
 	return nil
@@ -223,7 +224,7 @@ func (o *OpenAPI) validateResponse(contentType string, body []byte, r *http.Requ
 func (o *OpenAPI) getRequestValidationInput(r *http.Request) (*openapi3filter.RequestValidationInput, error) {
 	route, pathParams, err := o.router.FindRoute(r)
 	if err != nil {
-		log.Printf("route not found in openapi spec for url %s (host: %s), "+
+		log.Printf("route not found in OpenAPI spec for url %s (host: %s), "+
 			"skipping OpenAPI validation", r.URL, r.Host)
 		return nil, err
 	}

engine/template.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 	texttemplate "text/template"
 
+	"github.com/PDOK/gokoala/engine/util"
 	sprig "github.com/go-task/slim-sprig"
 	gomarkdown "github.com/gomarkdown/markdown"
 	gomarkdownhtml "github.com/gomarkdown/markdown/html"
@@ -212,7 +213,7 @@ func (t *Templates) renderNonHTMLTemplate(parsed *texttemplate.Template, params
 	var result = rendered.Bytes()
 	if strings.Contains(key.Format, FormatJSON) {
 		// pretty print all JSON (or derivatives like TileJSON)
-		result = prettyPrintJSON(result, key.Name)
+		result = util.PrettyPrintJSON(result, key.Name)
 	}
 	return result
 }

engine/templates/openapi/features.go.json

@@ -1,4 +1,5 @@
 {{- /*gotype: github.com/PDOK/gokoala/engine.TemplateData*/ -}}
+{{ $cfg := .Config }}
 {
   "openapi": "3.0.2",
   "info": {
@@ -28,10 +29,10 @@
             "style": "form",
             "explode": false,
             "schema": {
-              "maximum": 1000,
+              "maximum": {{ $cfg.OgcAPI.Features.Limit.Max }},
               "minimum": 1,
               "type": "integer",
-              "default": 10
+              "default": {{ $cfg.OgcAPI.Features.Limit.Default }}
             }
           },
           {
@@ -1177,10 +1178,10 @@
         "style": "form",
         "explode": false,
         "schema": {
-          "maximum": 1000,
+          "maximum": {{ $cfg.OgcAPI.Features.Limit.Max }},
           "minimum": 1,
           "type": "integer",
-          "default": 10
+          "default": {{ $cfg.OgcAPI.Features.Limit.Default }}
         }
       },
       "cursor": {

engine/json.go → engine/util/json.go

@@ -1,12 +1,12 @@
-package engine
+package util
 
 import (
 	"bytes"
 	"encoding/json"
 	"log"
 )
 
-func prettyPrintJSON(content []byte, name string) []byte {
+func PrettyPrintJSON(content []byte, name string) []byte {
 	var pretty bytes.Buffer
 	if err := json.Indent(&pretty, content, "", " "); err != nil {
 		log.Print(string(content))
@@ -15,14 +15,14 @@ func prettyPrintJSON(content []byte, name string) []byte {
 	return pretty.Bytes()
 }
 
-// mergeJSON merges the two JSON byte slices containing x1 and x2,
+// MergeJSON merges the two JSON byte slices containing x1 and x2,
 // preferring x1 over x2 except where x1 and x2 are
 // JSON objects, in which case the keys from both objects
 // are included and their values merged recursively.
 //
 // It returns an error if x1 or x2 cannot be JSON-unmarshalled,
 // or the merged JSON is invalid.
-func mergeJSON(x1, x2 []byte) ([]byte, error) {
+func MergeJSON(x1, x2 []byte) ([]byte, error) {
 	var j1 interface{}
 	err := json.Unmarshal(x1, &j1)
 	if err != nil {