Merge pull request #75 from PDOK/bbox

Bbox

.github/workflows/lint-go.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: setup cgo dependencies
-        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
+        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev libsqlite3-mod-spatialite
 
       - uses: actions/setup-go@v4
         with:

.github/workflows/test.yml

@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: setup cgo dependencies
-        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev
+        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev libsqlite3-mod-spatialite
 
       - name: Set up Go
         uses: actions/setup-go@v3

Dockerfile

@@ -17,10 +17,10 @@ ADD . /go/src/service
 ENV CGO_ENABLED=1
 ENV GOOS=linux
 
-# install cloud-backed sqlite compile-time dependencies
+# install sqlite-related compile-time dependencies
 RUN set -eux && \
     apt-get update && \
-    apt-get install -y libcurl4-openssl-dev libssl-dev && \
+    apt-get install -y libcurl4-openssl-dev libssl-dev libsqlite3-mod-spatialite && \
     rm -rf /var/lib/apt/lists/*
 
 RUN go mod download all
@@ -36,10 +36,10 @@ RUN find . -type f -name "*.go" -delete && find . -type d -name "testdata" -prun
 ####### Final image (use debian tag since we rely on C-libs)
 FROM ${REGISTRY}/debian:bookworm-slim
 
-# install cloud-backed sqlite runtime dependencies
+# install sqlite-related runtime dependencies
 RUN set -eux && \
     apt-get update && \
-    apt-get install -y libcurl4 openssl && \
+    apt-get install -y libcurl4 openssl libsqlite3-mod-spatialite && \
     rm -rf /var/lib/apt/lists/*
 
 EXPOSE 8080
@@ -63,9 +63,6 @@ COPY --from=build-component /usr/src/app/dist/vectortile-view-component/3rdparty
 COPY --from=build-env /go/src/service/engine/ /engine/
 COPY --from=build-env /go/src/service/ogc/ /ogc/
 
-COPY --from=build-env /usr/share/zoneinfo /usr/share/zoneinfo
-COPY --from=build-env /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-
 # run as non-root
 USER 1001
 ENTRYPOINT ["gokoala"]

README.md

@@ -124,6 +124,10 @@ through a tunnel/port-forward. The debug server exposes `/debug` for use by
 
 A similar flow can be used to profile memory issues.
 
+#### SQL query logging
+
+Set `LOG_SQL=true` to enable query logging stdout for debug purposes. Only applies to OGC API Features.
+
 ## Develop
 
 Design principles:

engine/config.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/creasty/defaults"
 	"github.com/go-playground/validator/v10"
 	"golang.org/x/text/language"
 	"gopkg.in/yaml.v3"
@@ -40,11 +41,15 @@ func readConfigFile(configFile string) *Config {
 }
 
 func setDefaults(config *Config) {
+	// process 'default' tags
+	if err := defaults.Set(config); err != nil {
+		log.Fatalf("failed to set default configuration: %v", err)
+	}
+
 	config.CookieMaxAge = cookieMaxAge
 
 	if len(config.AvailableLanguages) == 0 {
-		// default to Dutch only
-		config.AvailableLanguages = append(config.AvailableLanguages, language.Dutch)
+		config.AvailableLanguages = append(config.AvailableLanguages, language.Dutch) // default to Dutch only
 	}
 }
 
@@ -258,6 +263,7 @@ type OgcAPIStyles struct {
 }
 
 type OgcAPIFeatures struct {
+	Limit       Limit                 `yaml:"limit"`
 	Collections GeoSpatialCollections `yaml:"collections" validate:"required"`
 	Datasource  Datasource            `yaml:"datasource" validate:"required"`
 }
@@ -272,6 +278,11 @@ type OgcAPIProcesses struct {
 	ProcessesServer  YAMLURL `yaml:"processesServer" validate:"url"`
 }
 
+type Limit struct {
+	Default int `yaml:"default" validate:"gt=1" default:"10"`
+	Max     int `yaml:"max" validate:"gt=1" default:"1000"`
+}
+
 type Datasource struct {
 	GeoPackage *GeoPackage `yaml:"geopackage" validate:"required_without_all=PostGIS"`
 	PostGIS    *PostGIS    `yaml:"postgis" validate:"required_without_all=GeoPackage"`

engine/openapi.go

@@ -144,7 +144,7 @@ func mergeSpecs(ctx context.Context, config *Config, files []string) (*openapi3.
 			mergedJSON, err = util.MergeJSON(resultSpecJSON, specJSON)
 			if err != nil {
 				log.Print(string(mergedJSON))
-				log.Fatalf("failed to merge openapi specs: %v", err)
+				log.Fatalf("failed to merge OpenAPI specs: %v", err)
 			}
 		}
 		resultSpecJSON = mergedJSON
@@ -157,7 +157,7 @@ func loadSpec(loader *openapi3.Loader, mergedJSON []byte, fileName ...string) *o
 	resultSpec, err := loader.LoadFromData(mergedJSON)
 	if err != nil {
 		log.Print(string(mergedJSON))
-		log.Fatalf("failed to load merged openapi spec %s, due to %v", fileName, err)
+		log.Fatalf("failed to load merged OpenAPI spec %s, due to %v", fileName, err)
 	}
 	return resultSpec
 }
@@ -167,14 +167,14 @@ func validateSpec(ctx context.Context, finalSpec *openapi3.T, finalSpecRaw []byt
 	err := finalSpec.Validate(ctx, openapi3.DisableExamplesValidation())
 	if err != nil {
 		log.Print(string(finalSpecRaw))
-		log.Fatalf("invalid openapi spec: %v", err)
+		log.Fatalf("invalid OpenAPI spec: %v", err)
 	}
 }
 
 func newOpenAPIRouter(doc *openapi3.T) routers.Router {
 	openAPIRouter, err := gorillamux.NewRouter(doc)
 	if err != nil {
-		log.Fatalf("failed to setup openapi router: %v", err)
+		log.Fatalf("failed to setup OpenAPI router: %v", err)
 	}
 	return openAPIRouter
 }
@@ -195,7 +195,7 @@ func (o *OpenAPI) validateRequest(r *http.Request) error {
 	if requestValidationInput != nil {
 		err := openapi3filter.ValidateRequest(context.Background(), requestValidationInput)
 		if err != nil {
-			return fmt.Errorf("invalid request, doesn't conform to openapi spec %w", err)
+			return fmt.Errorf("request doesn't conform to OpenAPI spec: %w", err)
 		}
 	}
 	return nil
@@ -215,7 +215,7 @@ func (o *OpenAPI) validateResponse(contentType string, body []byte, r *http.Requ
 		responseValidationInput.SetBodyBytes(body)
 		err := openapi3filter.ValidateResponse(context.Background(), responseValidationInput)
 		if err != nil {
-			return fmt.Errorf("response doesn't conform to openapi spec: %w", err)
+			return fmt.Errorf("response doesn't conform to OpenAPI spec: %w", err)
 		}
 	}
 	return nil
@@ -224,7 +224,7 @@ func (o *OpenAPI) validateResponse(contentType string, body []byte, r *http.Requ
 func (o *OpenAPI) getRequestValidationInput(r *http.Request) (*openapi3filter.RequestValidationInput, error) {
 	route, pathParams, err := o.router.FindRoute(r)
 	if err != nil {
-		log.Printf("route not found in openapi spec for url %s (host: %s), "+
+		log.Printf("route not found in OpenAPI spec for url %s (host: %s), "+
 			"skipping OpenAPI validation", r.URL, r.Host)
 		return nil, err
 	}

engine/templates/openapi/features.go.json

@@ -1,4 +1,5 @@
 {{- /*gotype: github.com/PDOK/gokoala/engine.TemplateData*/ -}}
+{{ $cfg := .Config }}
 {
   "openapi": "3.0.2",
   "info": {
@@ -28,10 +29,10 @@
             "style": "form",
             "explode": false,
             "schema": {
-              "maximum": 1000,
+              "maximum": {{ $cfg.OgcAPI.Features.Limit.Max }},
               "minimum": 1,
               "type": "integer",
-              "default": 10
+              "default": {{ $cfg.OgcAPI.Features.Limit.Default }}
             }
           },
           {
@@ -1177,10 +1178,10 @@
         "style": "form",
         "explode": false,
         "schema": {
-          "maximum": 1000,
+          "maximum": {{ $cfg.OgcAPI.Features.Limit.Max }},
           "minimum": 1,
           "type": "integer",
-          "default": 10
+          "default": {{ $cfg.OgcAPI.Features.Limit.Default }}
         }
       },
       "cursor": {

examples/config_features_azure.yaml

@@ -60,5 +60,5 @@ ogcApi:
           thumbnail: old.png
           lastUpdated: "2030-01-02T12:00:00Z"
           extent:
-            srs: EPSG:3857
-            bbox: [ "-13512504.4191", "-19500889.3517", "31493617.8353", "19869683.6812" ]
+            srs: EPSG:4326
+            bbox: [ "50.2129", "2.52713", "55.7212", "7.37403" ]

examples/config_features_local.yaml

@@ -56,5 +56,5 @@ ogcApi:
           thumbnail: old.png
           lastUpdated: "2030-01-02T12:00:00Z"
           extent:
-            srs: EPSG:3857
-            bbox: [ "-13512504.4191", "-19500889.3517", "31493617.8353", "19869683.6812" ]
+            srs: EPSG:4326
+            bbox: [ "50.2129", "2.52713", "55.7212", "7.37403" ]

examples/resources/addresses-cloudbacked-gpkg/.gitattributes

@@ -0,0 +1 @@
+*.bcv binary

go.mod

@@ -5,6 +5,7 @@ go 1.21
 require (
 	github.com/BurntSushi/toml v1.3.2
 	github.com/PDOK/go-cloud-sqlite-vfs v0.2.4
+	github.com/creasty/defaults v1.7.0
 	github.com/elnormous/contenttype v1.0.4
 	github.com/getkin/kin-openapi v0.116.0
 	github.com/go-chi/chi/v5 v5.0.8
@@ -13,8 +14,9 @@ require (
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572
 	github.com/gomarkdown/markdown v0.0.0-20230322041520-c84983bdbf2a
 	github.com/jmoiron/sqlx v1.3.5
-	github.com/mattn/go-sqlite3 v1.14.17
+	github.com/mattn/go-sqlite3 v1.14.18
 	github.com/nicksnyder/go-i18n/v2 v2.2.1
+	github.com/qustavo/sqlhooks/v2 v2.1.0
 	github.com/stretchr/testify v1.8.2
 	github.com/urfave/cli/v2 v2.25.3
 	github.com/writeas/go-strip-markdown/v2 v2.1.1

go.sum

@@ -7,6 +7,8 @@ github.com/arolek/p v0.0.0-20191103215535-df3c295ed582/go.mod h1:JPNItmi3yb44Q5Q
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA=
+github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -35,6 +37,7 @@ github.com/go-playground/validator/v10 v10.13.0/go.mod h1:dwu7+CG8/CtBiJFZDz4e+5
 github.com/go-spatial/geom v0.0.0-20220918193402-3cd2f5a9a082 h1:3+Swhq7I1stScm1DE4PUWKjGJRbi+VPvisj6tFz0prs=
 github.com/go-spatial/geom v0.0.0-20220918193402-3cd2f5a9a082/go.mod h1:YU06tBGGstQCUX7vMuyF44RRneTdjGxOrp8OJHu4t9Q=
 github.com/go-spatial/proj v0.2.0/go.mod h1:ePHHp7ITVc4eIVW5sgG/0Eu9RMAGOQUgM/D1ZkccY+0=
+github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
@@ -69,20 +72,24 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
-github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI=
+github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/mattn/goveralls v0.0.3-0.20180319021929-1c14a4061c1c/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/nicksnyder/go-i18n/v2 v2.2.1 h1:aOzRCdwsJuoExfZhoiXHy4bjruwCMdt5otbYojM/PaA=
 github.com/nicksnyder/go-i18n/v2 v2.2.1/go.mod h1:fF2++lPHlo+/kPaj3nB0uxtPwzlPm+BlgwGX7MkeGj0=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/perimeterx/marshmallow v1.1.4 h1:pZLDH9RjlLGGorbXhcaQLhfuV0pFMNfPO55FuFkxqLw=
 github.com/perimeterx/marshmallow v1.1.4/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/qustavo/sqlhooks/v2 v2.1.0 h1:54yBemHnGHp/7xgT+pxwmIlMSDNYKx5JW5dfRAiCZi0=
+github.com/qustavo/sqlhooks/v2 v2.1.0/go.mod h1:aMREyKo7fOKTwiLuWPsaHRXEmtqG4yREztO0idF83AU=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

main.go

@@ -15,7 +15,6 @@ import (
 	"github.com/PDOK/gokoala/ogc/processes"
 	"github.com/PDOK/gokoala/ogc/styles"
 	"github.com/PDOK/gokoala/ogc/tiles"
-
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/urfave/cli/v2"

ogc/common/geospatial/main.go

@@ -90,5 +90,5 @@ type CollectionSupport interface {
 	// - OGC API Maps could have: /collections/{collectionId}/maps
 	// - OGC API 3d GeoVolumes could have: /collections/{collectionId}/3dtiles
 	// etc.
-	CollectionContent() http.HandlerFunc
+	CollectionContent(args ...any) http.HandlerFunc
 }

ogc/common/geospatial/templates/collection.go.json

@@ -19,7 +19,9 @@
       "crs" : "http://www.opengis.net/def/crs/EPSG/0/{{ trimPrefix "EPSG:" .Params.Metadata.Extent.Srs }}"
     }
   },
-  {{/* "crs" : [], */}}
+  "crs" : [
+    "http://www.opengis.net/def/crs/EPSG/0/{{ trimPrefix "EPSG:" .Params.Metadata.Extent.Srs }}"
+  ],
   {{/* "storageCrs" : "", */}}
   {{ end }}
   "links" : [

ogc/features/datasources/datasource.go

@@ -31,7 +31,7 @@ type FeatureOptions struct {
 
 	// filtering by bounding box
 	Bbox    *geom.Extent
-	BboxCrs string
+	BboxCrs int
 
 	// filtering by CQL
 	Filter    string