diff --git a/atlas/atlas.go b/atlas/atlas.go index 71e2d23c1..5ea55e0a5 100644 --- a/atlas/atlas.go +++ b/atlas/atlas.go @@ -121,6 +121,10 @@ func (a *Atlas) SeedMapTile(ctx context.Context, m Map, z, x, y uint) error { return defaultAtlas.SeedMapTile(ctx, m, z, x, y) } + if len(m.Params) > 0 { + return nil + } + ctx = context.WithValue(ctx, observability.ObserveVarMapName, m.Name) // confirm we have a cache backend if a.cacher == nil { @@ -130,7 +134,7 @@ func (a *Atlas) SeedMapTile(ctx context.Context, m Map, z, x, y uint) error { tile := slippy.NewTile(z, x, y) // encode the tile - b, err := m.Encode(ctx, tile) + b, err := m.Encode(ctx, tile, nil) if err != nil { return err } @@ -154,6 +158,10 @@ func (a *Atlas) PurgeMapTile(m Map, tile *tegola.Tile) error { return defaultAtlas.PurgeMapTile(m, tile) } + if len(m.Params) > 0 { + return nil + } + if a.cacher == nil { return ErrMissingCache } diff --git a/atlas/map.go b/atlas/map.go index 2b635e5b4..aab8e79de 100644 --- a/atlas/map.go +++ b/atlas/map.go @@ -27,7 +27,7 @@ import ( "github.com/go-spatial/tegola/provider/debug" ) -// NewMap creates a new map with the necessary default values +// NewWebMercatorMap creates a new map with the necessary default values func NewWebMercatorMap(name string) Map { return Map{ Name: name, @@ -40,6 +40,7 @@ func NewWebMercatorMap(name string) Map { } } +// Map defines a Web Mercator map type Map struct { Name string // Contains an attribution to be displayed when the map is shown to a user. @@ -53,6 +54,8 @@ type Map struct { // WGS:84 values), the third value is the zoom level. Center [3]float64 Layers []Layer + // Params holds configured query parameters + Params []provider.QueryParameter SRID uint64 // MVT output values @@ -116,7 +119,7 @@ func (m Map) AddDebugLayers() Map { m.Layers = layers // setup a debug provider - debugProvider, _ := debug.NewTileProvider(dict.Dict{}) + debugProvider, _ := debug.NewTileProvider(dict.Dict{}, nil) m.Layers = append(layers, []Layer{ { @@ -179,7 +182,7 @@ func (m Map) FilterLayersByName(names ...string) Map { return m } -func (m Map) encodeMVTProviderTile(ctx context.Context, tile *slippy.Tile) ([]byte, error) { +func (m Map) encodeMVTProviderTile(ctx context.Context, tile *slippy.Tile, params provider.Params) ([]byte, error) { // get the list of our layers ptile := provider.NewTile(tile.Z, tile.X, tile.Y, uint(m.TileBuffer), uint(m.SRID)) @@ -190,13 +193,13 @@ func (m Map) encodeMVTProviderTile(ctx context.Context, tile *slippy.Tile) ([]by MVTName: m.Layers[i].MVTName(), } } - return m.mvtProvider.MVTForLayers(ctx, ptile, layers) + return m.mvtProvider.MVTForLayers(ctx, ptile, params, layers) } // encodeMVTTile will encode the given tile into mvt format // TODO (arolek): support for max zoom -func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile) ([]byte, error) { +func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile, params provider.Params) ([]byte, error) { // tile container var mvtTile mvt.Tile @@ -225,7 +228,7 @@ func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile) ([]byte, erro uint(m.TileBuffer), uint(m.SRID)) // fetch layer from data provider - err := l.Provider.TileFeatures(ctx, l.ProviderLayerName, ptile, func(f *provider.Feature) error { + err := l.Provider.TileFeatures(ctx, l.ProviderLayerName, ptile, params, func(f *provider.Feature) error { // skip row if geometry collection empty. g, ok := f.Geometry.(geom.Collection) if ok && len(g.Geometries()) == 0 { @@ -360,7 +363,10 @@ func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile) ([]byte, erro } // add layers to our tile - mvtTile.AddLayers(mvtLayers...) + err := mvtTile.AddLayers(mvtLayers...) + if err != nil { + return nil, err + } // generate the MVT tile vtile, err := mvtTile.VTile(ctx) @@ -373,15 +379,15 @@ func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile) ([]byte, erro } // Encode will encode the given tile into mvt format -func (m Map) Encode(ctx context.Context, tile *slippy.Tile) ([]byte, error) { +func (m Map) Encode(ctx context.Context, tile *slippy.Tile, params provider.Params) ([]byte, error) { var ( tileBytes []byte err error ) if m.HasMVTProvider() { - tileBytes, err = m.encodeMVTProviderTile(ctx, tile) + tileBytes, err = m.encodeMVTProviderTile(ctx, tile, params) } else { - tileBytes, err = m.encodeMVTTile(ctx, tile) + tileBytes, err = m.encodeMVTTile(ctx, tile, params) } if err != nil { return nil, err diff --git a/atlas/map_test.go b/atlas/map_test.go index 0642a95f0..699df275a 100644 --- a/atlas/map_test.go +++ b/atlas/map_test.go @@ -202,7 +202,7 @@ func TestEncode(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { - out, err := tc.grid.Encode(context.Background(), tc.tile) + out, err := tc.grid.Encode(context.Background(), tc.tile, nil) if err != nil { t.Errorf("err: %v", err) return diff --git a/cache/cache.go b/cache/cache.go index d5d7609b6..a32e1b1cc 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -38,6 +38,7 @@ func ParseKey(str string) (*Key, error) { // remove the base-path and the first slash, then split the parts keyParts := strings.Split(strings.TrimLeft(str, "/"), "/") + // we're expecting a z/x/y scheme if len(keyParts) < 3 || len(keyParts) > 5 { err = ErrInvalidFileKeyParts{ diff --git a/cmd/internal/register/maps.go b/cmd/internal/register/maps.go index ed043500a..bb4e62c9f 100644 --- a/cmd/internal/register/maps.go +++ b/cmd/internal/register/maps.go @@ -11,9 +11,10 @@ import ( "github.com/go-spatial/tegola/provider" ) -func webMercatorMapFromConfigMap(cfg config.Map) (newMap atlas.Map) { +func webMercatorMapFromConfigMap(cfg provider.Map) (newMap atlas.Map) { newMap = atlas.NewWebMercatorMap(string(cfg.Name)) newMap.Attribution = SanitizeAttribution(string(cfg.Attribution)) + newMap.Params = cfg.Parameters // convert from env package for i, v := range cfg.Center { @@ -46,7 +47,7 @@ func layerInfosFindByName(infos []provider.LayerInfo, name string) provider.Laye return nil } -func atlasLayerFromConfigLayer(cfg *config.MapLayer, mapName string, layerProvider provider.Layerer) (layer atlas.Layer, err error) { +func atlasLayerFromConfigLayer(cfg *provider.MapLayer, mapName string, layerProvider provider.Layerer) (layer atlas.Layer, err error) { var ( // providerLayer is primary used for error reporting. providerLayer = string(cfg.ProviderLayer) @@ -123,7 +124,7 @@ func selectProvider(name string, mapName string, newMap *atlas.Map, providers ma } // Maps registers maps with with atlas -func Maps(a *atlas.Atlas, maps []config.Map, providers map[string]provider.TilerUnion) error { +func Maps(a *atlas.Atlas, maps []provider.Map, providers map[string]provider.TilerUnion) error { var ( layerer provider.Layerer @@ -155,6 +156,7 @@ func Maps(a *atlas.Atlas, maps []config.Map, providers map[string]provider.Tiler } newMap.Layers = append(newMap.Layers, layer) } + a.AddMap(newMap) } return nil diff --git a/cmd/internal/register/maps_test.go b/cmd/internal/register/maps_test.go index abe05dcc1..5573bb8c4 100644 --- a/cmd/internal/register/maps_test.go +++ b/cmd/internal/register/maps_test.go @@ -6,15 +6,15 @@ import ( "github.com/go-spatial/tegola/atlas" "github.com/go-spatial/tegola/cmd/internal/register" - "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/dict" "github.com/go-spatial/tegola/internal/env" + "github.com/go-spatial/tegola/provider" ) func TestMaps(t *testing.T) { type tcase struct { atlas atlas.Atlas - maps []config.Map + maps []provider.Map providers []dict.Dict expectedErr error } @@ -29,7 +29,7 @@ func TestMaps(t *testing.T) { provArr[i] = tc.providers[i] } - providers, err := register.Providers(provArr) + providers, err := register.Providers(provArr, tc.maps) if err != nil { t.Errorf("unexpected err: %v", err) return @@ -45,10 +45,10 @@ func TestMaps(t *testing.T) { tests := map[string]tcase{ "provider layer invalid": { - maps: []config.Map{ + maps: []provider.Map{ { Name: "foo", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "bar", }, @@ -67,10 +67,10 @@ func TestMaps(t *testing.T) { }, }, "provider not found": { - maps: []config.Map{ + maps: []provider.Map{ { Name: "foo", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "bar.baz", }, @@ -82,10 +82,10 @@ func TestMaps(t *testing.T) { }, }, "provider layer not registered with provider": { - maps: []config.Map{ + maps: []provider.Map{ { Name: "foo", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "test.bar", }, @@ -105,10 +105,10 @@ func TestMaps(t *testing.T) { }, }, "default tags": { - maps: []config.Map{ + maps: []provider.Map{ { Name: "foo", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "test.debug-tile-outline", DefaultTags: env.Dict{ @@ -126,7 +126,7 @@ func TestMaps(t *testing.T) { }, }, "success": { - maps: []config.Map{}, + maps: []provider.Map{}, providers: []dict.Dict{ { "name": "test", diff --git a/cmd/internal/register/providers.go b/cmd/internal/register/providers.go index 1eb16faa9..4553b9fc1 100644 --- a/cmd/internal/register/providers.go +++ b/cmd/internal/register/providers.go @@ -33,7 +33,7 @@ func (e ErrProviderTypeInvalid) Error() string { } // Providers registers data provider backends -func Providers(providers []dict.Dicter) (map[string]provider.TilerUnion, error) { +func Providers(providers []dict.Dicter, maps []provider.Map) (map[string]provider.TilerUnion, error) { // holder for registered providers registeredProviders := map[string]provider.TilerUnion{} @@ -72,7 +72,7 @@ func Providers(providers []dict.Dicter) (map[string]provider.TilerUnion, error) } // register the provider - prov, err := provider.For(ptype, p) + prov, err := provider.For(ptype, p, maps) if err != nil { return registeredProviders, err } diff --git a/cmd/internal/register/providers_test.go b/cmd/internal/register/providers_test.go index c1f62a45b..9f8d44410 100644 --- a/cmd/internal/register/providers_test.go +++ b/cmd/internal/register/providers_test.go @@ -23,7 +23,7 @@ func TestProviders(t *testing.T) { provArr[i] = tc.config[i] } - _, err = register.Providers(provArr) + _, err = register.Providers(provArr, nil) if tc.expectedErr != nil { if err.Error() != tc.expectedErr.Error() { t.Errorf("invalid error. expected: %v, got %v", tc.expectedErr, err.Error()) diff --git a/cmd/tegola/cmd/cache/cache.go b/cmd/tegola/cmd/cache/cache.go index be2c00ca6..8c2d998c9 100644 --- a/cmd/tegola/cmd/cache/cache.go +++ b/cmd/tegola/cmd/cache/cache.go @@ -159,11 +159,20 @@ func doWork(ctx context.Context, tileChannel *TileChannel, maps []atlas.Map, con }(i) } + nonParamMaps := make([]atlas.Map, 0) + + for _, m := range maps { + // we don't support caching for maps with custom parameters + if m.Params != nil || len(m.Params) > 0 { + log.Warnf("caching is disabled for map %s as it has custom parameters configures", m.Name) + } + nonParamMaps = append(nonParamMaps, m) + } + // run through the incoming tiles, and generate the mapTiles as needed. TileChannelLoop: for tile := range tileChannel.Channel() { - for m := range maps { - + for _, m := range nonParamMaps { if ctx.Err() != nil { cleanup = true break @@ -180,7 +189,7 @@ TileChannelLoop: } mapTile := MapTile{ - MapName: maps[m].Name, + MapName: m.Name, Tile: tile, } diff --git a/cmd/tegola/cmd/root.go b/cmd/tegola/cmd/root.go index 5c3dbd8ea..177638ae2 100644 --- a/cmd/tegola/cmd/root.go +++ b/cmd/tegola/cmd/root.go @@ -77,7 +77,7 @@ var RootCmd = &cobra.Command{ Use: "tegola", Short: "tegola is a vector tile server", Long: fmt.Sprintf(`tegola is a vector tile server - Version: %v`, build.Version), +Version: %v`, build.Version), PersistentPreRunE: rootCmdValidatePersistent, } @@ -121,7 +121,7 @@ func initConfig(configFile string, cacheRequired bool, logLevel string, logger s provArr[i] = conf.Providers[i] } - providers, err := register.Providers(provArr) + providers, err := register.Providers(provArr, conf.Maps) if err != nil { return fmt.Errorf("could not register providers: %v", err) } diff --git a/cmd/tegola_lambda/main.go b/cmd/tegola_lambda/main.go index 4d1e19f4a..cf96b11ab 100644 --- a/cmd/tegola_lambda/main.go +++ b/cmd/tegola_lambda/main.go @@ -66,7 +66,7 @@ func init() { } // register the providers - providers, err := register.Providers(provArr) + providers, err := register.Providers(provArr, nil) if err != nil { log.Fatal(err) } diff --git a/config/config.go b/config/config.go index 79d3d2549..aef1a70fa 100644 --- a/config/config.go +++ b/config/config.go @@ -16,6 +16,35 @@ import ( "github.com/go-spatial/tegola/provider" ) +const ( + BboxToken = "!BBOX!" + ZoomToken = "!ZOOM!" + XToken = "!X!" + YToken = "!Y!" + ZToken = "!Z!" + ScaleDenominatorToken = "!SCALE_DENOMINATOR!" + PixelWidthToken = "!PIXEL_WIDTH!" + PixelHeightToken = "!PIXEL_HEIGHT!" + IdFieldToken = "!ID_FIELD!" + GeomFieldToken = "!GEOM_FIELD!" + GeomTypeToken = "!GEOM_TYPE!" +) + +// ReservedTokens for query injection +var ReservedTokens = map[string]struct{}{ + BboxToken: {}, + ZoomToken: {}, + XToken: {}, + YToken: {}, + ZToken: {}, + ScaleDenominatorToken: {}, + PixelWidthToken: {}, + PixelHeightToken: {}, + IdFieldToken: {}, + GeomFieldToken: {}, + GeomTypeToken: {}, +} + var blacklistHeaders = []string{"content-encoding", "content-length", "content-type"} // Config represents a tegola config file. @@ -36,8 +65,8 @@ type Config struct { // 2. type -- this is the name the provider modules register // themselves under. (e.g. postgis, gpkg, mvt_postgis ) // Note: Use the type to figure out if the provider is a mvt or std provider - Providers []env.Dict `toml:"providers"` - Maps []Map `toml:"maps"` + Providers []env.Dict `toml:"providers"` + Maps []provider.Map `toml:"maps"` } // Webserver represents the config options for the webserver part of Tegola @@ -50,55 +79,79 @@ type Webserver struct { SSLKey env.String `toml:"ssl_key"` } -// A Map represents a map in the Tegola Config file. -type Map struct { - Name env.String `toml:"name"` - Attribution env.String `toml:"attribution"` - Bounds []env.Float `toml:"bounds"` - Center [3]env.Float `toml:"center"` - Layers []MapLayer `toml:"layers"` - TileBuffer *env.Int `toml:"tile_buffer"` -} +// ValidateAndRegisterParams ensures configured params don't conflict with existing +// query tokens or have overlapping names +func ValidateAndRegisterParams(mapName string, params []provider.QueryParameter) error { + if len(params) == 0 { + return nil + } -// MapLayer represents a the config for a layer in a map -type MapLayer struct { - // Name is optional. If it's not defined the name of the ProviderLayer will be used. - // Name can also be used to group multiple ProviderLayers under the same namespace. - Name env.String `toml:"name"` - ProviderLayer env.String `toml:"provider_layer"` - MinZoom *env.Uint `toml:"min_zoom"` - MaxZoom *env.Uint `toml:"max_zoom"` - DefaultTags env.Dict `toml:"default_tags"` - // DontSimplify indicates whether feature simplification should be applied. - // We use a negative in the name so the default is to simplify - DontSimplify env.Bool `toml:"dont_simplify"` - // DontClip indicates whether feature clipping should be applied. - // We use a negative in the name so the default is to clipping - DontClip env.Bool `toml:"dont_clip"` - // DontClip indicates whether feature cleaning (e.g. make valid) should be applied. - // We use a negative in the name so the default is to clean - DontClean env.Bool `toml:"dont_clean"` -} + usedNames := make(map[string]struct{}) + usedTokens := make(map[string]struct{}) + + for _, param := range params { + if _, ok := provider.ParamTypeDecoders[param.Type]; !ok { + return ErrParamUnknownType{ + MapName: string(mapName), + Parameter: param, + } + } + + if len(param.DefaultSQL) > 0 && len(param.DefaultValue) > 0 { + return ErrParamTwoDefaults{ + MapName: string(mapName), + Parameter: param, + } + } + + if len(param.DefaultValue) > 0 { + decoderFn := provider.ParamTypeDecoders[param.Type] + if _, err := decoderFn(param.DefaultValue); err != nil { + return ErrParamInvalidDefault{ + MapName: string(mapName), + Parameter: param, + } + } + } + + if _, ok := ReservedTokens[param.Token]; ok { + return ErrParamTokenReserved{ + MapName: string(mapName), + Parameter: param, + } + } + + if !provider.ParameterTokenRegexp.MatchString(param.Token) { + return ErrParamBadTokenName{ + MapName: string(mapName), + Parameter: param, + } + } + + if _, ok := usedNames[param.Name]; ok { + return ErrParamDuplicateName{ + MapName: string(mapName), + Parameter: param, + } + } -// ProviderLayerName returns the names of the layer and provider or an error -func (ml MapLayer) ProviderLayerName() (provider, layer string, err error) { - // split the provider layer (syntax is provider.layer) - plParts := strings.Split(string(ml.ProviderLayer), ".") - if len(plParts) != 2 { - return "", "", ErrInvalidProviderLayerName{ProviderLayerName: string(ml.ProviderLayer)} + if _, ok := usedTokens[param.Token]; ok { + return ErrParamDuplicateToken{ + MapName: string(mapName), + Parameter: param, + } + } + + usedNames[param.Name] = struct{}{} + usedTokens[param.Token] = struct{}{} } - return plParts[0], plParts[1], nil -} -// GetName will return the user-defined Layer name from the config, -// or if the name is empty, return the name of the layer associated with -// the provider -func (ml MapLayer) GetName() (string, error) { - if ml.Name != "" { - return string(ml.Name), nil + // Mark all used tokens as reserved + for token := range usedTokens { + ReservedTokens[token] = struct{}{} } - _, name, err := ml.ProviderLayerName() - return name, err + + return nil } // Validate checks the config for issues @@ -142,17 +195,29 @@ func (c *Config) Validate() error { } // check for map layer name / zoom collisions // map of layers to providers - mapLayers := map[string]map[string]MapLayer{} + mapLayers := map[string]map[string]provider.MapLayer{} + // maps with configured parameters for logging + mapsWithCustomParams := []string{} for mapKey, m := range c.Maps { + + // validate any declared query parameters + if err := ValidateAndRegisterParams(string(m.Name), m.Parameters); err != nil { + return err + } + + if len(m.Parameters) > 0 { + mapsWithCustomParams = append(mapsWithCustomParams, string(m.Name)) + } + if _, ok := mapLayers[string(m.Name)]; !ok { - mapLayers[string(m.Name)] = map[string]MapLayer{} + mapLayers[string(m.Name)] = map[string]provider.MapLayer{} } // Set current provider to empty, for MVT providers // we can only have the same provider for all layers. // This allow us to track what the first found provider // is. - provider := "" + currentProvider := "" isMVTProvider := false for layerKey, l := range m.Layers { pname, _, err := l.ProviderLayerName() @@ -160,11 +225,11 @@ func (c *Config) Validate() error { return err } - if provider == "" { + if currentProvider == "" { // This is the first provider we found. // For MVTProviders all others need to be the same, so store it // so we can check later - provider = pname + currentProvider = pname } isMvt, doesExists := mvtproviders[pname] @@ -181,13 +246,13 @@ func (c *Config) Validate() error { isMVTProvider = isMVTProvider || isMvt // only need to do this check if we are dealing with MVTProviders - if isMVTProvider && pname != provider { + if isMVTProvider && pname != currentProvider { // for mvt_providers we can only have the same provider // for all layers // check to see if mvtproviders[pname] || isMVTProvider { return ErrMVTDifferentProviders{ - Original: provider, + Original: currentProvider, Current: pname, } } @@ -241,6 +306,13 @@ func (c *Config) Validate() error { } } + if len(mapsWithCustomParams) > 0 { + log.Infof( + "Caching is disabled for these maps, since they have configured custom parameters: %s", + strings.Join(mapsWithCustomParams, ", "), + ) + } + // check for blacklisted headers for k := range c.Webserver.Headers { for _, v := range blacklistHeaders { @@ -286,11 +358,18 @@ func (c *Config) ConfigureTileBuffers() { // Parse will parse the Tegola config file provided by the io.Reader. func Parse(reader io.Reader, location string) (conf Config, err error) { // decode conf file, don't care about the meta data. - _, err = toml.DecodeReader(reader, &conf) + _, err = toml.NewDecoder(reader).Decode(&conf) if err != nil { return conf, err } + for _, m := range conf.Maps { + for k, p := range m.Parameters { + p.Normalize() + m.Parameters[k] = p + } + } + conf.LocationName = location conf.ConfigureTileBuffers() diff --git a/config/config_test.go b/config/config_test.go index d41572e9b..5bf34ced6 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -9,6 +9,7 @@ import ( "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/internal/env" + "github.com/go-spatial/tegola/provider" _ "github.com/go-spatial/tegola/provider/debug" _ "github.com/go-spatial/tegola/provider/postgis" _ "github.com/go-spatial/tegola/provider/test" @@ -156,7 +157,26 @@ func TestParse(t *testing.T) { max_zoom = 20 dont_simplify = true dont_clip = true - dont_clean = true`, + dont_clean = true + + [[maps.params]] + name = "param1" + token = "!param1!" + type = "string" + + [[maps.params]] + name = "param2" + token = "!PARAM2!" + type = "int" + sql = "AND answer = ?" + default_value = "42" + + [[maps.params]] + name = "param3" + token = "!PARAM3!" + type = "float" + default_sql = "AND pi = 3.1415926" + `, expected: config.Config{ TileBuffer: env.IntPtr(env.Int(12)), LocationName: "", @@ -191,14 +211,14 @@ func TestParse(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, TileBuffer: env.IntPtr(env.Int(12)), - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(10), @@ -208,6 +228,28 @@ func TestParse(t *testing.T) { DontClean: true, }, }, + Parameters: []provider.QueryParameter{ + { + Name: "param1", + Token: "!PARAM1!", + SQL: "?", + Type: "string", + }, + { + Name: "param2", + Token: "!PARAM2!", + Type: "int", + SQL: "AND answer = ?", + DefaultValue: "42", + }, + { + Name: "param3", + Token: "!PARAM3!", + Type: "float", + SQL: "?", + DefaultSQL: "AND pi = 3.1415926", + }, + }, }, }, }, @@ -319,14 +361,14 @@ func TestParse(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{ENV_TEST_CENTER_X, ENV_TEST_CENTER_Y, ENV_TEST_CENTER_Z}, TileBuffer: env.IntPtr(env.Int(64)), - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { Name: "water", ProviderLayer: ENV_TEST_PROVIDER_LAYER, @@ -347,7 +389,7 @@ func TestParse(t *testing.T) { Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, TileBuffer: env.IntPtr(env.Int(64)), - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { Name: "water", ProviderLayer: "provider1.water_0_5", @@ -501,13 +543,13 @@ func TestValidateMutateZoom(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: nil, @@ -545,13 +587,13 @@ func TestValidateMutateZoom(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(0), @@ -631,13 +673,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(10), @@ -695,13 +737,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { Name: "water", ProviderLayer: "provider1.water_0_5", @@ -765,13 +807,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(10), @@ -789,7 +831,7 @@ func TestValidate(t *testing.T) { Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(10), @@ -848,13 +890,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", }, @@ -865,7 +907,7 @@ func TestValidate(t *testing.T) { Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider2.water", }, @@ -917,13 +959,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1035,11 +1077,11 @@ func TestValidate(t *testing.T) { "type": "mvt_test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "happy", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1056,11 +1098,11 @@ func TestValidate(t *testing.T) { "type": "mvt_test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "happy", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1084,11 +1126,11 @@ func TestValidate(t *testing.T) { "type": "test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "happy", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1112,11 +1154,11 @@ func TestValidate(t *testing.T) { "type": "mvt_test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "happy", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "bad.water_default_z", }, @@ -1141,11 +1183,11 @@ func TestValidate(t *testing.T) { "type": "test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "comingle", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1173,11 +1215,11 @@ func TestValidate(t *testing.T) { "type": "mvt_test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "comingle", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "stdprovider1.water_default_z", }, @@ -1189,6 +1231,191 @@ func TestValidate(t *testing.T) { }, }, }, + "13 reserved token name": { + config: config.Config{ + Maps: []provider.Map{ + { + Name: "bad_param", + Parameters: []provider.QueryParameter{ + { + Name: "param", + Token: "!BBOX!", + Type: "int", + }, + }, + }, + }, + }, + expectedErr: config.ErrParamTokenReserved{ + MapName: "bad_param", + Parameter: provider.QueryParameter{ + Name: "param", + Token: "!BBOX!", + Type: "int", + }, + }, + }, + "13 duplicate parameter name": { + config: config.Config{ + Maps: []provider.Map{ + { + Name: "dupe_param_name", + Parameters: []provider.QueryParameter{ + { + Name: "param", + Token: "!PARAM!", + Type: "int", + }, + { + Name: "param", + Token: "!PARAM2!", + Type: "int", + }, + }, + }, + }, + }, + expectedErr: config.ErrParamDuplicateName{ + MapName: "dupe_param_name", + Parameter: provider.QueryParameter{ + Name: "param", + Token: "!PARAM2!", + Type: "int", + }, + }, + }, + "13 duplicate token name": { + config: config.Config{ + Maps: []provider.Map{ + { + Name: "dupe_param_token", + Parameters: []provider.QueryParameter{ + { + Name: "param", + Token: "!PARAM!", + Type: "int", + }, + { + Name: "param2", + Token: "!PARAM!", + Type: "int", + }, + }, + }, + }, + }, + expectedErr: config.ErrParamDuplicateToken{ + MapName: "dupe_param_token", + Parameter: provider.QueryParameter{ + Name: "param2", + Token: "!PARAM!", + Type: "int", + }, + }, + }, + "13 parameter unknown type": { + config: config.Config{ + Maps: []provider.Map{ + { + Name: "unknown_param_type", + Parameters: []provider.QueryParameter{ + { + Name: "param", + Token: "!BBOX!", + Type: "foo", + }, + }, + }, + }, + }, + expectedErr: config.ErrParamUnknownType{ + MapName: "unknown_param_type", + Parameter: provider.QueryParameter{ + Name: "param", + Token: "!BBOX!", + Type: "foo", + }, + }, + }, + "13 parameter two defaults": { + config: config.Config{ + Maps: []provider.Map{ + { + Name: "unknown_two_defaults", + Parameters: []provider.QueryParameter{ + { + Name: "param", + Token: "!BBOX!", + Type: "string", + DefaultSQL: "foo", + DefaultValue: "bar", + }, + }, + }, + }, + }, + expectedErr: config.ErrParamTwoDefaults{ + MapName: "unknown_two_defaults", + Parameter: provider.QueryParameter{ + Name: "param", + Token: "!BBOX!", + Type: "string", + DefaultSQL: "foo", + DefaultValue: "bar", + }, + }, + }, + "13 parameter invalid default": { + config: config.Config{ + Maps: []provider.Map{ + { + Name: "parameter_invalid_default", + + Parameters: []provider.QueryParameter{ + { + Name: "param", + Token: "!BBOX!", + Type: "int", + DefaultValue: "foo", + }, + }, + }, + }, + }, + expectedErr: config.ErrParamInvalidDefault{ + MapName: "parameter_invalid_default", + Parameter: provider.QueryParameter{ + Name: "param", + Token: "!BBOX!", + Type: "int", + DefaultValue: "foo", + }, + }, + }, + "13 invalid token name": { + config: config.Config{ + Maps: []provider.Map{ + { + Name: "parameter_invalid_token", + Parameters: []provider.QueryParameter{ + { + Name: "param", + Token: "!Token with spaces!", + Type: "int", + }, + }, + }, + }, + }, + expectedErr: config.ErrParamBadTokenName{ + MapName: "parameter_invalid_token", + Parameter: provider.QueryParameter{ + Name: "param", + Token: "!Token with spaces!", + Type: "int", + }, + }, + }, } for name, tc := range tests { @@ -1218,14 +1445,14 @@ func TestConfigureTileBuffers(t *testing.T) { tests := map[string]tcase{ "1 tilebuffer is not set": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", }, }, }, expected: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(64)), @@ -1236,7 +1463,7 @@ func TestConfigureTileBuffers(t *testing.T) { "2 tilebuffer is set in global section": { config: config.Config{ TileBuffer: env.IntPtr(env.Int(32)), - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", }, @@ -1247,7 +1474,7 @@ func TestConfigureTileBuffers(t *testing.T) { }, expected: config.Config{ TileBuffer: env.IntPtr(env.Int(32)), - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(32)), @@ -1261,7 +1488,7 @@ func TestConfigureTileBuffers(t *testing.T) { }, "3 tilebuffer is set in map section": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(16)), @@ -1273,7 +1500,7 @@ func TestConfigureTileBuffers(t *testing.T) { }, }, expected: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(16)), @@ -1288,7 +1515,7 @@ func TestConfigureTileBuffers(t *testing.T) { "4 tilebuffer is set in global and map sections": { config: config.Config{ TileBuffer: env.IntPtr(env.Int(32)), - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(16)), @@ -1297,7 +1524,7 @@ func TestConfigureTileBuffers(t *testing.T) { }, expected: config.Config{ TileBuffer: env.IntPtr(env.Int(32)), - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(16)), diff --git a/config/errors.go b/config/errors.go index 3bc6754e5..ded68ddfb 100644 --- a/config/errors.go +++ b/config/errors.go @@ -3,6 +3,8 @@ package config import ( "fmt" "strings" + + "github.com/go-spatial/tegola/provider" ) type ErrMapNotFound struct { @@ -13,6 +15,81 @@ func (e ErrMapNotFound) Error() string { return fmt.Sprintf("config: map (%v) not found", e.MapName) } +type ErrParamTokenReserved struct { + MapName string + Parameter provider.QueryParameter +} + +func (e ErrParamTokenReserved) Error() string { + return fmt.Sprintf("config: map %s parameter %s uses a reserved token name %s", + e.MapName, e.Parameter.Name, e.Parameter.Token) +} + +type ErrParamDuplicateName struct { + MapName string + Parameter provider.QueryParameter +} + +func (e ErrParamDuplicateName) Error() string { + return fmt.Sprintf("config: map %s parameter %s has a name already used by another parameter", + e.MapName, e.Parameter.Name) +} + +type ErrParamDuplicateToken struct { + MapName string + Parameter provider.QueryParameter +} + +func (e ErrParamDuplicateToken) Error() string { + return fmt.Sprintf("config: map %s parameter %s has a token name %s already used by another parameter", + e.MapName, e.Parameter.Token, e.Parameter.Name) +} + +type ErrParamUnknownType struct { + MapName string + Parameter provider.QueryParameter +} + +func (e ErrParamUnknownType) Error() string { + validTypes := make([]string, 0, len(provider.ParamTypeDecoders)) + for k := range provider.ParamTypeDecoders { + validTypes = append(validTypes, k) + } + + return fmt.Sprintf("config: map %s parameter %s has an unknown type %s, must be one of: %s", + e.MapName, e.Parameter.Name, e.Parameter.Type, strings.Join(validTypes, ",")) +} + +type ErrParamTwoDefaults struct { + MapName string + Parameter provider.QueryParameter +} + +func (e ErrParamTwoDefaults) Error() string { + return fmt.Sprintf("config: map %s parameter %s has both default_value and default_sql", + e.MapName, e.Parameter.Name) +} + +type ErrParamInvalidDefault struct { + MapName string + Parameter provider.QueryParameter +} + +func (e ErrParamInvalidDefault) Error() string { + return fmt.Sprintf("config: map %s parameter %s has a default value that is invalid for type %s", + e.MapName, e.Parameter.Name, e.Parameter.Type) +} + +type ErrParamBadTokenName struct { + MapName string + Parameter provider.QueryParameter +} + +func (e ErrParamBadTokenName) Error() string { + return fmt.Sprintf("config: map %s parameter %s has an invalid token name %s", + e.MapName, e.Parameter.Name, e.Parameter.Token) +} + type ErrInvalidProviderForMap struct { MapName string ProviderName string diff --git a/provider/debug/debug.go b/provider/debug/debug.go index 639e80ba6..ba96f41f1 100644 --- a/provider/debug/debug.go +++ b/provider/debug/debug.go @@ -6,6 +6,7 @@ package debug import ( "context" "fmt" + "strings" "github.com/go-spatial/geom" "github.com/go-spatial/tegola" @@ -25,18 +26,25 @@ func init() { } // NewProvider Setups a debug provider. there are not currently any config params supported -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { return &Provider{}, nil } // Provider provides the debug provider type Provider struct{} -func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, fn func(f *provider.Feature) error) error { +func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams provider.Params, fn func(f *provider.Feature) error) error { // get tile bounding box ext, srid := tile.Extent() + params := make([]string, 0, len(queryParams)) + for _, param := range queryParams { + params = append(params, fmt.Sprintf("%s=%s", param.RawParam, param.RawValue)) + } + + paramsStr := strings.Join(params, " ") + switch layer { case "debug-tile-outline": debugTileOutline := provider.Feature{ @@ -44,7 +52,8 @@ func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider Geometry: ext.AsPolygon(), SRID: srid, Tags: map[string]interface{}{ - "type": "debug_buffer_outline", + "type": "debug_buffer_outline", + "params": paramsStr, }, } @@ -67,8 +76,9 @@ func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider }, SRID: srid, Tags: map[string]interface{}{ - "type": "debug_text", - "zxy": fmt.Sprintf("Z:%v, X:%v, Y:%v", z, x, y), + "type": "debug_text", + "params": paramsStr, + "zxy": fmt.Sprintf("Z:%v, X:%v, Y:%v", z, x, y), }, } diff --git a/provider/gpkg/cgo_test.go b/provider/gpkg/cgo_test.go index 8a64fb9b1..9d7f7902e 100644 --- a/provider/gpkg/cgo_test.go +++ b/provider/gpkg/cgo_test.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package gpkg @@ -11,7 +12,7 @@ import ( // This is a test to just see that the init function is doing something. func TestNewProviderStartup(t *testing.T) { - _, err := NewTileProvider(dict.Dict{}) + _, err := NewTileProvider(dict.Dict{}, nil) if err == provider.ErrUnsupported { t.Fatalf("supported, expected any but unsupported got %v", err) } diff --git a/provider/gpkg/gpkg.go b/provider/gpkg/gpkg.go index ddb402084..65649ed07 100644 --- a/provider/gpkg/gpkg.go +++ b/provider/gpkg/gpkg.go @@ -76,7 +76,7 @@ func (p *Provider) Layers() ([]provider.LayerInfo, error) { return ls, nil } -func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, fn func(f *provider.Feature) error) error { +func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams provider.Params, fn func(f *provider.Feature) error) error { log.Debugf("fetching layer %v", layer) pLayer := p.layers[layer] @@ -101,6 +101,7 @@ func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider } var qtext string + args := make([]interface{}, 0) if pLayer.tablename != "" { // If layer was specified via "tablename" in config, construct query. @@ -121,11 +122,12 @@ func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider // If layer was specified via "sql" in config, collect it z, _, _ := tile.ZXY() qtext = replaceTokens(pLayer.sql, z, tileBBox) + qtext = queryParams.ReplaceParams(qtext, &args) } log.Debugf("qtext: %v", qtext) - rows, err := p.db.Query(qtext) + rows, err := p.db.Query(qtext, args...) if err != nil { log.Errorf("err during query: %v - %v", qtext, err) return err diff --git a/provider/gpkg/gpkg_register.go b/provider/gpkg/gpkg_register.go index 46235e28d..791550fe9 100644 --- a/provider/gpkg/gpkg_register.go +++ b/provider/gpkg/gpkg_register.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package gpkg @@ -12,6 +13,7 @@ import ( "sort" "strings" + conf "github.com/go-spatial/tegola/config" _ "github.com/mattn/go-sqlite3" "github.com/go-spatial/geom" @@ -219,7 +221,7 @@ func featureTableMetaData(gpkg *sql.DB) (map[string]featureTableDetails, error) return geomTableDetails, nil } -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { log.Debugf("config: %v", config) @@ -319,7 +321,6 @@ func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { return nil, fmt.Errorf("table %q does not exist", tablename) } - layer.tablename = tablename layer.tagFieldnames = tagFieldnames layer.geomFieldname = d.geomFieldname @@ -341,22 +342,22 @@ func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { // solution without using an SQL parser on custom SQL statements allZoomsSQL := "IN (0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24)" tokenReplacer := strings.NewReplacer( - ">= "+zoomToken, allZoomsSQL, - ">="+zoomToken, allZoomsSQL, - "=> "+zoomToken, allZoomsSQL, - "=>"+zoomToken, allZoomsSQL, - "=< "+zoomToken, allZoomsSQL, - "=<"+zoomToken, allZoomsSQL, - "<= "+zoomToken, allZoomsSQL, - "<="+zoomToken, allZoomsSQL, - "!= "+zoomToken, allZoomsSQL, - "!="+zoomToken, allZoomsSQL, - "= "+zoomToken, allZoomsSQL, - "="+zoomToken, allZoomsSQL, - "> "+zoomToken, allZoomsSQL, - ">"+zoomToken, allZoomsSQL, - "< "+zoomToken, allZoomsSQL, - "<"+zoomToken, allZoomsSQL, + ">= "+conf.ZoomToken, allZoomsSQL, + ">="+conf.ZoomToken, allZoomsSQL, + "=> "+conf.ZoomToken, allZoomsSQL, + "=>"+conf.ZoomToken, allZoomsSQL, + "=< "+conf.ZoomToken, allZoomsSQL, + "=<"+conf.ZoomToken, allZoomsSQL, + "<= "+conf.ZoomToken, allZoomsSQL, + "<="+conf.ZoomToken, allZoomsSQL, + "!= "+conf.ZoomToken, allZoomsSQL, + "!="+conf.ZoomToken, allZoomsSQL, + "= "+conf.ZoomToken, allZoomsSQL, + "="+conf.ZoomToken, allZoomsSQL, + "> "+conf.ZoomToken, allZoomsSQL, + ">"+conf.ZoomToken, allZoomsSQL, + "< "+conf.ZoomToken, allZoomsSQL, + "<"+conf.ZoomToken, allZoomsSQL, ) customSQL = tokenReplacer.Replace(customSQL) diff --git a/provider/gpkg/gpkg_register_internal_test.go b/provider/gpkg/gpkg_register_internal_test.go index c99b4b563..a0ba0b808 100644 --- a/provider/gpkg/gpkg_register_internal_test.go +++ b/provider/gpkg/gpkg_register_internal_test.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package gpkg @@ -1095,7 +1096,7 @@ func TestCleanup(t *testing.T) { fn := func(tc tcase) func(*testing.T) { return func(t *testing.T) { - _, err := NewTileProvider(tc.config) + _, err := NewTileProvider(tc.config, nil) if err != nil { t.Fatalf("err creating NewTileProvider: %v", err) return diff --git a/provider/gpkg/gpkg_test.go b/provider/gpkg/gpkg_test.go index 4b461e059..b3142dd13 100644 --- a/provider/gpkg/gpkg_test.go +++ b/provider/gpkg/gpkg_test.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package gpkg_test @@ -183,7 +184,7 @@ func TestNewTileProvider(t *testing.T) { return func(t *testing.T) { t.Parallel() - p, err := gpkg.NewTileProvider(tc.config) + p, err := gpkg.NewTileProvider(tc.config, nil) if err != nil { if err.Error() != tc.expectedErr.Error() { t.Errorf("expectedErr %v got %v", tc.expectedErr, err) @@ -262,14 +263,14 @@ func TestTileFeatures(t *testing.T) { return func(t *testing.T) { t.Parallel() - p, err := gpkg.NewTileProvider(tc.config) + p, err := gpkg.NewTileProvider(tc.config, nil) if err != nil { t.Fatalf("new tile, expected nil got %v", err) return } var featureCount int - err = p.TileFeatures(context.TODO(), tc.layerName, &tc.tile, func(f *provider.Feature) error { + err = p.TileFeatures(context.TODO(), tc.layerName, &tc.tile, nil, func(f *provider.Feature) error { featureCount++ return nil }) @@ -410,13 +411,13 @@ func TestConfigs(t *testing.T) { fn := func(tc tcase) func(*testing.T) { return func(t *testing.T) { - p, err := gpkg.NewTileProvider(tc.config) + p, err := gpkg.NewTileProvider(tc.config, nil) if err != nil { t.Fatalf("err creating NewTileProvider: %v", err) return } - err = p.TileFeatures(context.TODO(), tc.layerName, &tc.tile, func(f *provider.Feature) error { + err = p.TileFeatures(context.TODO(), tc.layerName, &tc.tile, nil, func(f *provider.Feature) error { // check if the feature is part of the test if _, ok := tc.expectedTags[f.ID]; !ok { return nil @@ -592,7 +593,7 @@ func TestOpenNonExistantFile(t *testing.T) { os.Remove(NONEXISTANTFILE) fn := func(tc tcase) func(*testing.T) { return func(t *testing.T) { - _, err := gpkg.NewTileProvider(tc.config) + _, err := gpkg.NewTileProvider(tc.config, nil) if reflect.TypeOf(err) != reflect.TypeOf(tc.err) { t.Errorf("expected error, expected %v got %v", tc.err, err) } @@ -600,13 +601,13 @@ func TestOpenNonExistantFile(t *testing.T) { } tests := map[string]tcase{ - "empty": tcase{ + "empty": { config: dict.Dict{ gpkg.ConfigKeyFilePath: "", }, err: gpkg.ErrInvalidFilePath{FilePath: ""}, }, - "nonexistance": tcase{ + "nonexistance": { // should not exists config: dict.Dict{ gpkg.ConfigKeyFilePath: NONEXISTANTFILE, diff --git a/provider/gpkg/util.go b/provider/gpkg/util.go index 38292a262..dba8f2f1d 100644 --- a/provider/gpkg/util.go +++ b/provider/gpkg/util.go @@ -6,11 +6,7 @@ import ( "strings" "github.com/go-spatial/geom" -) - -const ( - bboxToken = "!BBOX!" - zoomToken = "!ZOOM!" + "github.com/go-spatial/tegola/config" ) func replaceTokens(qtext string, zoom uint, extent *geom.Extent) string { @@ -34,8 +30,8 @@ func replaceTokens(qtext string, zoom uint, extent *geom.Extent) string { tokenReplacer := strings.NewReplacer( // The BBOX token requires parameters ordered as [maxx, minx, maxy, miny] and checks for overlap. // Until support for named parameters, we'll only support one BBOX token per query. - bboxToken, fmt.Sprintf("minx <= %v AND maxx >= %v AND miny <= %v AND maxy >= %v", extent.MaxX(), extent.MinX(), extent.MaxY(), extent.MinY()), - zoomToken, strconv.FormatUint(uint64(zoom), 10), + config.BboxToken, fmt.Sprintf("minx <= %v AND maxx >= %v AND miny <= %v AND maxy >= %v", extent.MaxX(), extent.MinX(), extent.MaxY(), extent.MinY()), + config.ZoomToken, strconv.FormatUint(uint64(zoom), 10), ) return tokenReplacer.Replace(qtext) diff --git a/provider/map.go b/provider/map.go new file mode 100644 index 000000000..d59ccd0b9 --- /dev/null +++ b/provider/map.go @@ -0,0 +1,14 @@ +package provider + +import "github.com/go-spatial/tegola/internal/env" + +// A Map represents a map in the Tegola Config file. +type Map struct { + Name env.String `toml:"name"` + Attribution env.String `toml:"attribution"` + Bounds []env.Float `toml:"bounds"` + Center [3]env.Float `toml:"center"` + Layers []MapLayer `toml:"layers"` + Parameters []QueryParameter `toml:"params"` + TileBuffer *env.Int `toml:"tile_buffer"` +} diff --git a/provider/map_layer.go b/provider/map_layer.go new file mode 100644 index 000000000..4a29d61a4 --- /dev/null +++ b/provider/map_layer.go @@ -0,0 +1,51 @@ +package provider + +import ( + "fmt" + "strings" + + "github.com/go-spatial/tegola/internal/env" +) + +// MapLayer represents a the config for a layer in a map +type MapLayer struct { + // Name is optional. If it's not defined the name of the ProviderLayer will be used. + // Name can also be used to group multiple ProviderLayers under the same namespace. + Name env.String `toml:"name"` + ProviderLayer env.String `toml:"provider_layer"` + MinZoom *env.Uint `toml:"min_zoom"` + MaxZoom *env.Uint `toml:"max_zoom"` + DefaultTags env.Dict `toml:"default_tags"` + // DontSimplify indicates whether feature simplification should be applied. + // We use a negative in the name so the default is to simplify + DontSimplify env.Bool `toml:"dont_simplify"` + // DontClip indicates whether feature clipping should be applied. + // We use a negative in the name so the default is to clipping + DontClip env.Bool `toml:"dont_clip"` + // DontClip indicates whether feature cleaning (e.g. make valid) should be applied. + // We use a negative in the name so the default is to clean + DontClean env.Bool `toml:"dont_clean"` +} + +// ProviderLayerName returns the names of the layer and provider or an error +func (ml MapLayer) ProviderLayerName() (provider, layer string, err error) { + // split the provider layer (syntax is provider.layer) + plParts := strings.Split(string(ml.ProviderLayer), ".") + if len(plParts) != 2 { + // TODO (beymak): Properly handle the error + return "", "", fmt.Errorf("config: invalid provider layer name (%v)", ml.ProviderLayer) + // return "", "", ErrInvalidProviderLayerName{ProviderLayerName: string(ml.ProviderLayer)} + } + return plParts[0], plParts[1], nil +} + +// GetName will return the user-defined Layer name from the config, +// or if the name is empty, return the name of the layer associated with +// the provider +func (ml MapLayer) GetName() (string, error) { + if ml.Name != "" { + return string(ml.Name), nil + } + _, name, err := ml.ProviderLayerName() + return name, err +} diff --git a/provider/mvt_provider.go b/provider/mvt_provider.go index b0a41207b..4f192fec8 100644 --- a/provider/mvt_provider.go +++ b/provider/mvt_provider.go @@ -10,8 +10,8 @@ type MVTTiler interface { Layerer // MVTForLayers will return a MVT byte array or an error for the given layer names. - MVTForLayers(ctx context.Context, tile Tile, layers []Layer) ([]byte, error) + MVTForLayers(ctx context.Context, tile Tile, params Params, layers []Layer) ([]byte, error) } // MVTInitFunc initialize a provider given a config map. The init function should validate the config map, and report any errors. This is called by the For function. -type MVTInitFunc func(dicter dict.Dicter) (MVTTiler, error) +type MVTInitFunc func(dicter dict.Dicter, maps []Map) (MVTTiler, error) diff --git a/provider/paramater_decoders.go b/provider/paramater_decoders.go new file mode 100644 index 000000000..d1a2f7b91 --- /dev/null +++ b/provider/paramater_decoders.go @@ -0,0 +1,19 @@ +package provider + +import "strconv" + +// ParamTypeDecoders is a collection of parsers for different types of user-defined parameters +var ParamTypeDecoders = map[string]func(string) (interface{}, error){ + "int": func(s string) (interface{}, error) { + return strconv.Atoi(s) + }, + "float": func(s string) (interface{}, error) { + return strconv.ParseFloat(s, 32) + }, + "string": func(s string) (interface{}, error) { + return s, nil + }, + "bool": func(s string) (interface{}, error) { + return strconv.ParseBool(s) + }, +} diff --git a/provider/postgis/postgis.go b/provider/postgis/postgis.go index 13e5a0678..8c7be5014 100644 --- a/provider/postgis/postgis.go +++ b/provider/postgis/postgis.go @@ -17,6 +17,7 @@ import ( "github.com/go-spatial/geom" "github.com/go-spatial/geom/encoding/wkb" "github.com/go-spatial/tegola" + conf "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/dict" "github.com/go-spatial/tegola/internal/log" "github.com/go-spatial/tegola/observability" @@ -150,7 +151,7 @@ func (p *Provider) Collectors(prefix string, cfgFn func(configKey string) map[st const ( // We quote the field and table names to prevent colliding with postgres keywords. - stdSQL = `SELECT %[1]v FROM %[2]v WHERE "%[3]v" && ` + bboxToken + stdSQL = `SELECT %[1]v FROM %[2]v WHERE "%[3]v" && ` + conf.BboxToken mvtSQL = `SELECT %[1]v FROM %[2]v` // SQL to get the column names, without hitting the information_schema. Though it might be better to hit the information_schema. @@ -170,6 +171,7 @@ const ( ) const ( + ConfigKeyName = "name" ConfigKeyURI = "uri" ConfigKeyHost = "host" ConfigKeyPort = "port" @@ -195,7 +197,7 @@ const ( ) // isSelectQuery is a regexp to check if a query starts with `SELECT`, -// case-insensitive and ignoring any preceeding whitespace and SQL comments. +// case-insensitive and ignoring any preceding whitespace and SQL comments. var isSelectQuery = regexp.MustCompile(`(?i)^((\s*)(--.*\n)?)*select`) type hstoreOID struct { @@ -405,27 +407,24 @@ func BuildDBConfig(uri string) (*pgxpool.Config, error) { // trying to create a driver. This Provider supports the following fields // in the provided map[string]interface{} map: // -// host (string): [Required] postgis database host -// port (int): [Required] postgis database port (required) -// database (string): [Required] postgis database name -// user (string): [Required] postgis database user -// password (string): [Required] postgis database password -// srid (int): [Optional] The default SRID for the provider. Defaults to WebMercator (3857) but also supports WGS84 (4326) -// max_connections : [Optional] The max connections to maintain in the connection pool. Default is 100. 0 means no max. -// layers (map[string]struct{}) — This is map of layers keyed by the layer name. supports the following properties +// name (string): [Required] name of the provider +// host (string): [Required] postgis database host +// port (int): [Required] postgis database port (required) +// database (string): [Required] postgis database name +// user (string): [Required] postgis database user +// password (string): [Required] postgis database password +// srid (int): [Optional] The default SRID for the provider. Defaults to WebMercator (3857) but also supports WGS84 (4326) +// max_connections : [Optional] The max connections to maintain in the connection pool. Default is 100. 0 means no max. +// layers (map[string]struct{}) — This is map of layers keyed by the layer name. supports the following properties // -// name (string): [Required] the name of the layer. This is used to reference this layer from map layers. -// tablename (string): [*Required] the name of the database table to query against. Required if sql is not defined. -// geometry_fieldname (string): [Optional] the name of the filed which contains the geometry for the feature. defaults to geom -// id_fieldname (string): [Optional] the name of the feature id field. defaults to gid -// fields ([]string): [Optional] a list of fields to include alongside the feature. Can be used if sql is not defined. -// srid (int): [Optional] the SRID of the layer. Supports 3857 (WebMercator) or 4326 (WGS84). -// sql (string): [*Required] custom SQL to use use. Required if tablename is not defined. Supports the following tokens: -// -// !BBOX! - [Required] will be replaced with the bounding box of the tile before the query is sent to the database. -// !ZOOM! - [Optional] will be replaced with the "Z" (zoom) value of the requested tile. -// -func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) { +// name (string): [Required] the name of the layer. This is used to reference this layer from map layers. +// tablename (string): [*Required] the name of the database table to query against. Required if sql is not defined. +// geometry_fieldname (string): [Optional] the name of the filed which contains the geometry for the feature. defaults to geom +// id_fieldname (string): [Optional] the name of the feature id field. defaults to gid +// fields ([]string): [Optional] a list of fields to include alongside the feature. Can be used if sql is not defined. +// srid (int): [Optional] the SRID of the layer. Supports 3857 (WebMercator) or 4326 (WGS84). +// sql (string): [*Required] custom SQL to use use. Required if tablename is not defined. Supports the following tokens: +func CreateProvider(config dict.Dicter, maps []provider.Map, providerType string) (*Provider, error) { uri, params, err := BuildURI(config) if err != nil { return nil, err @@ -453,7 +452,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) dbconfig, err := BuildDBConfig(uri.String()) if err != nil { - return nil, fmt.Errorf("Failed while building db config: %w", err) + return nil, fmt.Errorf("failed while building db config: %w", err) } srid := DefaultSRID @@ -472,7 +471,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) pool, err := pgxpool.ConnectConfig(context.Background(), &p.config) if err != nil { - return nil, fmt.Errorf("Failed while creating connection pool: %w", err) + return nil, fmt.Errorf("failed while creating connection pool: %w", err) } p.pool = &connectionPoolCollector{Pool: pool} @@ -488,7 +487,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) lName, err := layer.String(ConfigKeyLayerName, nil) if err != nil { - return nil, fmt.Errorf("For layer (%v) we got the following error trying to get the layer's name field: %w", i, err) + return nil, fmt.Errorf("for layer (%v) we got the following error trying to get the layer's name field: %w", i, err) } if j, ok := lyrsSeen[lName]; ok { @@ -563,10 +562,10 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) if sql != "" { // convert !BOX! (MapServer) and !bbox! (Mapnik) to !BBOX! for compatibility - sql := strings.Replace(strings.Replace(sql, "!BOX!", "!BBOX!", -1), "!bbox!", "!BBOX!", -1) + sql := strings.Replace(strings.Replace(sql, "!BOX!", conf.BboxToken, -1), "!bbox!", conf.BboxToken, -1) // make sure that the sql has a !BBOX! token - if !strings.Contains(sql, bboxToken) { - return nil, fmt.Errorf("SQL for layer (%v) %v is missing required token: %v", i, lName, bboxToken) + if !strings.Contains(sql, conf.BboxToken) { + return nil, fmt.Errorf("SQL for layer (%v) %v is missing required token: %v", i, lName, conf.BboxToken) } if !strings.Contains(sql, "*") { if !strings.Contains(sql, geomfld) { @@ -577,6 +576,13 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) } } + // check all tokens are valid + for _, token := range provider.ParameterTokenRegexp.FindAllString(sql, -1) { + if _, ok := conf.ReservedTokens[token]; !ok { + return nil, fmt.Errorf("SQL for layer (%v) %v references an unknown token %s: %v", i, lName, token, sql) + } + } + l.sql = sql } else { // Tablename and Fields will be used to build the query. @@ -598,8 +604,13 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) return nil, fmt.Errorf("error fetching geometry type for layer (%v): %w", l.name, err) } } else { - if err = p.inspectLayerGeomType(&l); err != nil { - return nil, fmt.Errorf("error fetching geometry type for layer (%v): %w", l.name, err) + pname, err := config.String(ConfigKeyName, nil) + if err != nil { + return nil, err + } + + if err = p.inspectLayerGeomType(pname, &l, maps); err != nil { + return nil, fmt.Errorf("error fetching geometry type for layer (%v): %w\nif custom parameters are used, remember to set %s for the provider", l.name, err, ConfigKeyGeomType) } } @@ -613,7 +624,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) return &p, nil } -// derived from github.com/jackc/pgx configTLS (https://github.com/jackc/pgx/blob/master/conn.go) +// ConfigTLS is derived from github.com/jackc/pgx configTLS (https://github.com/jackc/pgx/blob/master/conn.go) func ConfigTLS(sslMode string, sslKey string, sslCert string, sslRootCert string, cc *pgxpool.Config) error { switch sslMode { @@ -691,7 +702,7 @@ func (p Provider) setLayerGeomType(l *Layer, geomType string) error { // inspectLayerGeomType sets the geomType field on the layer by running the SQL // and reading the geom type in the result set -func (p Provider) inspectLayerGeomType(l *Layer) error { +func (p Provider) inspectLayerGeomType(pname string, l *Layer, maps []provider.Map) error { var err error // we want to know the geom type instead of returning the geom data so we modify the SQL @@ -725,7 +736,20 @@ func (p Provider) inspectLayerGeomType(l *Layer) error { return err } - rows, err := p.pool.Query(context.Background(), sql) + // substitute default values to parameter + params := extractQueryParamValues(pname, maps, l) + + args := make([]interface{}, 0) + sql = params.ReplaceParams(sql, &args) + + if provider.ParameterTokenRegexp.MatchString(sql) { + // remove all parameter tokens for inspection + // crossing our fingers that the query is still valid 🤞 + // if not, the user will have to specify `geometry_type` in the config + sql = provider.ParameterTokenRegexp.ReplaceAllString(sql, "") + } + + rows, err := p.pool.Query(context.Background(), sql, args...) if err != nil { return err } @@ -792,7 +816,7 @@ func (p Provider) Layers() ([]provider.LayerInfo, error) { } // TileFeatures adheres to the provider.Tiler interface -func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, fn func(f *provider.Feature) error) error { +func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, params provider.Params, fn func(f *provider.Feature) error) error { var mapName string { @@ -813,8 +837,15 @@ func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider. return fmt.Errorf("error replacing layer tokens for layer (%v) SQL (%v): %w", layer, sql, err) } + // replace configured query parameters if any + args := make([]interface{}, 0) + sql = params.ReplaceParams(sql, &args) + if err != nil { + return err + } + if debugExecuteSQL { - log.Debugf("TEGOLA_SQL_DEBUG:EXECUTE_SQL for layer (%v): %v", layer, sql) + log.Debugf("TEGOLA_SQL_DEBUG:EXECUTE_SQL for layer (%v): %v with args %v", layer, sql, args) } // context check @@ -823,7 +854,7 @@ func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider. } now := time.Now() - rows, err := p.pool.Query(ctx, sql) + rows, err := p.pool.Query(ctx, sql, args...) if p.queryHistogramSeconds != nil { z, _, _ := tile.ZXY() lbls := prometheus.Labels{ @@ -840,7 +871,7 @@ func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider. // trying to clean itself up. defer rows.Close() if err := ctxErr(ctx, err); err != nil { - return fmt.Errorf("error running layer (%v) SQL (%v): %w", layer, sql, err) + return fmt.Errorf("error running layer (%v) SQL (%v) with args %v: %w", layer, sql, args, err) } // fieldDescriptions @@ -921,7 +952,7 @@ func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider. return rows.Err() } -func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, layers []provider.Layer) ([]byte, error) { +func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, params provider.Params, layers []provider.Layer) ([]byte, error) { var ( err error sqls = make([]string, 0, len(layers)) @@ -936,6 +967,8 @@ func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, layers [ } } + args := make([]interface{}, 0) + for i := range layers { if debug { log.Debugf("looking for layer: %v", layers[i]) @@ -947,13 +980,16 @@ func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, layers [ log.Warnf("provider layer not found %v", layers[i].Name) } if debugLayerSQL { - log.Debugf("SQL for Layer(%v):\n%v\n", l.Name(), l.sql) + log.Debugf("SQL for Layer(%v):\n%v\nargs:%v\n", l.Name(), l.sql, args) } sql, err := replaceTokens(l.sql, &l, tile, false) if err := ctxErr(ctx, err); err != nil { return nil, err } + // replace configured query parameters if any + sql = params.ReplaceParams(sql, &args) + // ref: https://postgis.net/docs/ST_AsMVT.html // bytea ST_AsMVT(any_element row, text name, integer extent, text geom_name, text feature_id_name) @@ -982,7 +1018,7 @@ func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, layers [ } { now := time.Now() - err = p.pool.QueryRow(ctx, fsql).Scan(&data) + err = p.pool.QueryRow(ctx, fsql, args...).Scan(&data) if p.mvtProviderQueryHistogramSeconds != nil { z, _, _ := tile.ZXY() lbls := prometheus.Labels{ diff --git a/provider/postgis/postgis_internal_test.go b/provider/postgis/postgis_internal_test.go index ab0108faf..bec0787b1 100644 --- a/provider/postgis/postgis_internal_test.go +++ b/provider/postgis/postgis_internal_test.go @@ -17,7 +17,19 @@ import ( // TESTENV is the environment variable that must be set to "yes" to run postgis tests. const TESTENV = "RUN_POSTGIS_TESTS" -var defaultEnvConfig map[string]interface{} +var DefaultEnvConfig map[string]interface{} + +var DefaultConfig map[string]interface{} = map[string]interface{}{ + ConfigKeyHost: "localhost", + ConfigKeyPort: 5432, + ConfigKeyDB: "tegola", + ConfigKeyUser: "postgres", + ConfigKeyPassword: "postgres", + ConfigKeySSLMode: "disable", + ConfigKeySSLKey: "", + ConfigKeySSLCert: "", + ConfigKeySSLRootCert: "", +} func getConfigFromEnv() map[string]interface{} { port, err := strconv.Atoi(ttools.GetEnvDefault("PGPORT", "5432")) @@ -40,7 +52,7 @@ func getConfigFromEnv() map[string]interface{} { } func init() { - defaultEnvConfig = getConfigFromEnv() + DefaultEnvConfig = getConfigFromEnv() } type TCConfig struct { @@ -49,9 +61,8 @@ type TCConfig struct { LayerConfig []map[string]interface{} } -func (cfg TCConfig) Config() dict.Dict { +func (cfg TCConfig) Config(mConfig map[string]interface{}) dict.Dict { var config map[string]interface{} - mConfig := defaultEnvConfig if cfg.BaseConfig != nil { mConfig = cfg.BaseConfig } @@ -86,8 +97,9 @@ func TestMVTProviders(t *testing.T) { } fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { - config := tc.Config() - prvd, err := NewMVTTileProvider(config) + config := tc.Config(DefaultEnvConfig) + config[ConfigKeyName] = "provider_name" + prvd, err := NewMVTTileProvider(config, nil) // for now we will just check the length of the bytes. if tc.err != "" { if err == nil || !strings.Contains(err.Error(), tc.err) { @@ -108,7 +120,7 @@ func TestMVTProviders(t *testing.T) { MVTName: tc.layerNames[i], } } - mvtTile, err := prvd.MVTForLayers(context.Background(), tc.tile, layers) + mvtTile, err := prvd.MVTForLayers(context.Background(), tc.tile, nil, layers) if err != nil { t.Errorf("NewProvider unexpected error: %v", err) return @@ -156,8 +168,9 @@ func TestLayerGeomType(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { - config := tc.Config() - provider, err := NewTileProvider(config) + config := tc.Config(DefaultEnvConfig) + config[ConfigKeyName] = "provider_name" + provider, err := NewTileProvider(config, nil) if tc.err != "" { if err == nil || !strings.Contains(err.Error(), tc.err) { t.Logf("error %#v", err) @@ -265,11 +278,11 @@ func TestLayerGeomType(t *testing.T) { TCConfig: TCConfig{ ConfigOverride: map[string]interface{}{ ConfigKeyURI: fmt.Sprintf("postgres://%v:%v@%v:%v/%v", - defaultEnvConfig["user"], - defaultEnvConfig["password"], - defaultEnvConfig["host"], - defaultEnvConfig["port"], - defaultEnvConfig["database"], + DefaultEnvConfig["user"], + DefaultEnvConfig["password"], + DefaultEnvConfig["host"], + DefaultEnvConfig["port"], + DefaultEnvConfig["database"], ), ConfigKeyHost: "", ConfigKeyPort: "", @@ -443,7 +456,7 @@ func TestBuildUri(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { - config := tc.Config() + config := tc.Config(DefaultConfig) uri, _, err := BuildURI(config) if tc.err != "" { diff --git a/provider/postgis/postgis_test.go b/provider/postgis/postgis_test.go index 17cd03b2f..22f4f3ffc 100644 --- a/provider/postgis/postgis_test.go +++ b/provider/postgis/postgis_test.go @@ -164,8 +164,9 @@ func TestNewTileProvider(t *testing.T) { fn := func(tc postgis.TCConfig) func(t *testing.T) { return func(t *testing.T) { - config := tc.Config() - _, err := postgis.NewTileProvider(config) + config := tc.Config(postgis.DefaultEnvConfig) + config[postgis.ConfigKeyName] = "provider_name" + _, err := postgis.NewTileProvider(config, nil) if err != nil { t.Errorf("unable to create a new provider. err: %v", err) return @@ -202,8 +203,9 @@ func TestTileFeatures(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { - config := tc.Config() - p, err := postgis.NewTileProvider(config) + config := tc.Config(postgis.DefaultEnvConfig) + config[postgis.ConfigKeyName] = "provider_name" + p, err := postgis.NewTileProvider(config, nil) if err != nil { t.Errorf("unexpected error; unable to create a new provider, expected: nil Got %v", err) return @@ -212,7 +214,7 @@ func TestTileFeatures(t *testing.T) { layerName := tc.LayerConfig[0][postgis.ConfigKeyLayerName].(string) var featureCount int - err = p.TileFeatures(context.Background(), layerName, tc.tile, func(f *provider.Feature) error { + err = p.TileFeatures(context.Background(), layerName, tc.tile, nil, func(f *provider.Feature) error { // only verify tags on first feature if featureCount == 0 { for _, tag := range tc.expectedTags { diff --git a/provider/postgis/register.go b/provider/postgis/register.go index ab0939be5..4c3a271b2 100644 --- a/provider/postgis/register.go +++ b/provider/postgis/register.go @@ -40,9 +40,9 @@ const ( // !BBOX! - [Required] will be replaced with the bounding box of the tile before the query is sent to the database. // !ZOOM! - [Optional] will be replaced with the "Z" (zoom) value of the requested tile. // -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { - return CreateProvider(config, ProviderType) +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { + return CreateProvider(config, maps, ProviderType) } -func NewMVTTileProvider(config dict.Dicter) (provider.MVTTiler, error) { - return CreateProvider(config, MVTProviderType) +func NewMVTTileProvider(config dict.Dicter, maps []provider.Map) (provider.MVTTiler, error) { + return CreateProvider(config, maps, MVTProviderType) } diff --git a/provider/postgis/util.go b/provider/postgis/util.go index 43796e0c3..7dcb5df07 100644 --- a/provider/postgis/util.go +++ b/provider/postgis/util.go @@ -3,13 +3,14 @@ package postgis import ( "context" "fmt" - "regexp" "strconv" "strings" "github.com/go-spatial/geom" "github.com/go-spatial/tegola" "github.com/go-spatial/tegola/basic" + "github.com/go-spatial/tegola/config" + "github.com/go-spatial/tegola/internal/env" "github.com/go-spatial/tegola/internal/log" "github.com/go-spatial/tegola/provider" "github.com/jackc/pgproto3/v2" @@ -97,20 +98,6 @@ func genSQL(l *Layer, pool *connectionPoolCollector, tblname string, flds []stri return fmt.Sprintf(sqlTmpl, selectClause, tblname, l.geomField), nil } -const ( - bboxToken = "!BBOX!" - zoomToken = "!ZOOM!" - xToken = "!X!" - yToken = "!Y!" - zToken = "!Z!" - scaleDenominatorToken = "!SCALE_DENOMINATOR!" - pixelWidthToken = "!PIXEL_WIDTH!" - pixelHeightToken = "!PIXEL_HEIGHT!" - idFieldToken = "!ID_FIELD!" - geomFieldToken = "!GEOM_FIELD!" - geomTypeToken = "!GEOM_TYPE!" -) - // replaceTokens replaces tokens in the provided SQL string // // !BBOX! - the bounding box of the tile @@ -169,17 +156,17 @@ func replaceTokens(sql string, lyr *Layer, tile provider.Tile, withBuffer bool) // replace query string tokens z, x, y := tile.ZXY() tokenReplacer := strings.NewReplacer( - bboxToken, bbox, - zoomToken, strconv.FormatUint(uint64(z), 10), - zToken, strconv.FormatUint(uint64(z), 10), - xToken, strconv.FormatUint(uint64(x), 10), - yToken, strconv.FormatUint(uint64(y), 10), - idFieldToken, lyr.IDFieldName(), - geomFieldToken, lyr.GeomFieldName(), - geomTypeToken, geoType, - scaleDenominatorToken, strconv.FormatFloat(scaleDenominator, 'f', -1, 64), - pixelWidthToken, strconv.FormatFloat(pixelWidth, 'f', -1, 64), - pixelHeightToken, strconv.FormatFloat(pixelHeight, 'f', -1, 64), + config.BboxToken, bbox, + config.ZoomToken, strconv.FormatUint(uint64(z), 10), + config.ZToken, strconv.FormatUint(uint64(z), 10), + config.XToken, strconv.FormatUint(uint64(x), 10), + config.YToken, strconv.FormatUint(uint64(y), 10), + config.ScaleDenominatorToken, strconv.FormatFloat(scaleDenominator, 'f', -1, 64), + config.PixelWidthToken, strconv.FormatFloat(pixelWidth, 'f', -1, 64), + config.PixelHeightToken, strconv.FormatFloat(pixelHeight, 'f', -1, 64), + config.IdFieldToken, lyr.IDFieldName(), + config.GeomFieldToken, lyr.GeomFieldName(), + config.GeomTypeToken, geoType, ) uppercaseTokenSQL := uppercaseTokens(sql) @@ -187,12 +174,31 @@ func replaceTokens(sql string, lyr *Layer, tile provider.Tile, withBuffer bool) return tokenReplacer.Replace(uppercaseTokenSQL), nil } -var tokenRe = regexp.MustCompile("![a-zA-Z0-9_-]+!") +// extractQueryParamValues finds default values for SQL tokens and constructs query parameter values out of them +func extractQueryParamValues(pname string, maps []provider.Map, layer *Layer) provider.Params { + result := make(provider.Params, 0) + + expectedMapName := fmt.Sprintf("%s.%s", pname, layer.name) + for _, m := range maps { + for _, l := range m.Layers { + if l.ProviderLayer == env.String(expectedMapName) { + for _, p := range m.Parameters { + pv, err := p.ToDefaultValue() + if err == nil { + result[p.Token] = pv + } + } + } + } + } + + return result +} -// uppercaseTokens converts all !tokens! to uppercase !TOKENS!. Tokens can -// contain alphanumerics, dash and underline chars. +// uppercaseTokens converts all !tokens! to uppercase !TOKENS!. Tokens can +// contain alphanumerics, dash and underline chars. func uppercaseTokens(str string) string { - return tokenRe.ReplaceAllStringFunc(str, strings.ToUpper) + return provider.ParameterTokenRegexp.ReplaceAllStringFunc(str, strings.ToUpper) } func transformVal(valType pgtype.OID, val interface{}) (interface{}, error) { diff --git a/provider/provider.go b/provider/provider.go index e203de9a4..6813892cb 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -3,6 +3,7 @@ package provider import ( "context" "fmt" + "regexp" "github.com/go-spatial/geom" "github.com/go-spatial/geom/slippy" @@ -104,13 +105,16 @@ type Tile interface { BufferedExtent() (extent *geom.Extent, srid uint64) } +// ParameterTokenRegexp to validate QueryParameters +var ParameterTokenRegexp = regexp.MustCompile("![a-zA-Z0-9_-]+!") + // Tiler is a Layers that allows one to encode features in that layer type Tiler interface { Layerer // TileFeature will stream decoded features to the callback function fn // if fn returns ErrCanceled, the TileFeatures method should stop processing - TileFeatures(ctx context.Context, layer string, t Tile, fn func(f *Feature) error) error + TileFeatures(ctx context.Context, layer string, t Tile, params Params, fn func(f *Feature) error) error } // TilerUnion represents either a Std Tiler or and MVTTiler; only one should be not nil. @@ -132,7 +136,7 @@ func (tu TilerUnion) Layers() ([]LayerInfo, error) { } // InitFunc initialize a provider given a config map. The init function should validate the config map, and report any errors. This is called by the For function. -type InitFunc func(dicter dict.Dicter) (Tiler, error) +type InitFunc func(dicter dict.Dicter, maps []Map) (Tiler, error) // CleanupFunc is called to when the system is shutting down, this allows the provider to cleanup. type CleanupFunc func() @@ -229,7 +233,7 @@ func Drivers(types ...providerType) (l []string) { // For function returns a configure provider of the given type; The provider may be a mvt provider or // a std provider. The correct entry in TilerUnion will not be nil. If there is an error both entries // will be nil. -func For(name string, config dict.Dicter) (val TilerUnion, err error) { +func For(name string, config dict.Dicter, maps []Map) (val TilerUnion, err error) { var ( driversList = Drivers() ) @@ -241,11 +245,11 @@ func For(name string, config dict.Dicter) (val TilerUnion, err error) { return val, ErrUnknownProvider{KnownProviders: driversList, Name: name} } if p.init != nil { - val.Std, err = p.init(config) + val.Std, err = p.init(config, maps) return val, err } if p.mvtInit != nil { - val.Mvt, err = p.mvtInit(config) + val.Mvt, err = p.mvtInit(config, maps) return val, err } return val, ErrInvalidRegisteredProvider{Name: name} diff --git a/provider/provider_test.go b/provider/provider_test.go index 220ef94a4..d67a98a93 100644 --- a/provider/provider_test.go +++ b/provider/provider_test.go @@ -12,7 +12,7 @@ func TestProviderInterface(t *testing.T) { stdName = provider.TypeStd.Prefix() + test.Name mvtName = provider.TypeMvt.Prefix() + test.Name ) - if _, err := provider.For(stdName, nil); err != nil { + if _, err := provider.For(stdName, nil, nil); err != nil { t.Errorf("retrieve provider err , expected nil got %v", err) return } @@ -23,7 +23,7 @@ func TestProviderInterface(t *testing.T) { if test.Count != 0 { t.Errorf(" expected count , expected 0 got %v", test.Count) } - if _, err := provider.For(mvtName, nil); err != nil { + if _, err := provider.For(mvtName, nil, nil); err != nil { t.Errorf("retrieve provider err , expected nil got %v", err) return } diff --git a/provider/query_parameter.go b/provider/query_parameter.go new file mode 100644 index 000000000..181169bcb --- /dev/null +++ b/provider/query_parameter.go @@ -0,0 +1,65 @@ +package provider + +import ( + "fmt" + "strings" +) + +// QueryParameter represents an HTTP query parameter specified for use with +// a given map instance. +type QueryParameter struct { + Name string `toml:"name"` + Token string `toml:"token"` + Type string `toml:"type"` + SQL string `toml:"sql"` + // DefaultSQL replaces SQL if param wasn't passed. Either default_sql or + // default_value can be specified + DefaultSQL string `toml:"default_sql"` + DefaultValue string `toml:"default_value"` +} + +// Normalize normalizes param and sets default values +func (param *QueryParameter) Normalize() { + param.Token = strings.ToUpper(param.Token) + + if len(param.SQL) == 0 { + param.SQL = "?" + } +} + +func (param *QueryParameter) ToValue(rawValue string) (QueryParameterValue, error) { + val, err := ParamTypeDecoders[param.Type](rawValue) + if err != nil { + return QueryParameterValue{}, err + } + return QueryParameterValue{ + Token: param.Token, + SQL: param.SQL, + Value: val, + RawParam: param.Name, + RawValue: rawValue, + }, nil +} + +func (param *QueryParameter) ToDefaultValue() (QueryParameterValue, error) { + if len(param.DefaultValue) > 0 { + val, err := ParamTypeDecoders[param.Type](param.DefaultValue) + return QueryParameterValue{ + Token: param.Token, + SQL: param.SQL, + Value: val, + RawParam: param.Name, + RawValue: "", + }, err + } + if len(param.DefaultSQL) > 0 { + return QueryParameterValue{ + Token: param.Token, + SQL: param.DefaultSQL, + Value: nil, + RawParam: param.Name, + RawValue: "", + }, nil + } + return QueryParameterValue{}, fmt.Errorf("the required parameter %s is not specified", param.Name) +} diff --git a/provider/query_parameter_value.go b/provider/query_parameter_value.go new file mode 100644 index 000000000..10fec3170 --- /dev/null +++ b/provider/query_parameter_value.go @@ -0,0 +1,76 @@ +package provider + +import ( + "fmt" + "strings" +) + +// Query parameter holds normalized parameter data ready to be inserted in the +// final query +type QueryParameterValue struct { + // Token to replace e.g., !TOKEN! + Token string + // SQL expression to be inserted. Contains "?" that will be replaced with an + // ordinal argument e.g., "$1" + SQL string + // Value that will be passed to the final query in arguments list + Value interface{} + // Raw parameter and value for debugging and monitoring + RawParam string + // RawValue will be "" if the param wasn't passed and defaults were used + RawValue string +} + +type Params map[string]QueryParameterValue + +// ReplaceParams substitutes configured query parameter tokens for their values +// within the provided SQL string +func (params Params) ReplaceParams(sql string, args *[]interface{}) string { + if params == nil { + return sql + } + + var ( + cache = make(map[string]string) + sb strings.Builder + ) + + for _, token := range ParameterTokenRegexp.FindAllString(sql, -1) { + resultSQL, ok := cache[token] + if ok { + // Already have it cached, replace the token and move on. + sql = strings.ReplaceAll(sql, token, resultSQL) + continue + } + + param, ok := params[token] + if !ok { + // Unknown token, ignoring + continue + } + + sb.Reset() + sb.Grow(len(param.SQL)) + argFound := false + + // Replace every `?` in the param's SQL with a positional argument + for _, c := range param.SQL { + if c != '?' { + sb.WriteRune(c) + continue + } + + if !argFound { + *args = append(*args, param.Value) + argFound = true + } + sb.WriteString(fmt.Sprintf("$%d", len(*args))) + } + + resultSQL = sb.String() + cache[token] = resultSQL + sql = strings.ReplaceAll(sql, token, resultSQL) + } + + return sql +} diff --git a/provider/query_parameter_value_test.go b/provider/query_parameter_value_test.go new file mode 100644 index 000000000..204bca97f --- /dev/null +++ b/provider/query_parameter_value_test.go @@ -0,0 +1,137 @@ +package provider + +import "testing" + +func TestReplaceParams(t *testing.T) { + type tcase struct { + params Params + sql string + expectedSql string + expectedArgs []interface{} + } + + fn := func(tc tcase) func(t *testing.T) { + return func(t *testing.T) { + args := make([]interface{}, 0) + out := tc.params.ReplaceParams(tc.sql, &args) + + if out != tc.expectedSql { + t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedSql, out) + return + } + + if len(tc.expectedArgs) != len(args) { + t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedArgs, args) + return + } + for i, arg := range tc.expectedArgs { + if arg != args[i] { + t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedArgs, args) + return + } + } + } + } + + tests := map[string]tcase{ + "nil params": { + params: nil, + sql: "SELECT * FROM table", + expectedSql: "SELECT * FROM table", + expectedArgs: []interface{}{}, + }, + "int replacement": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: 1, + }, + }, + sql: "SELECT * FROM table WHERE param = !PARAM!", + expectedSql: "SELECT * FROM table WHERE param = $1", + expectedArgs: []interface{}{1}, + }, + "string replacement": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: "test", + }, + }, + sql: "SELECT * FROM table WHERE param = !PARAM!", + expectedSql: "SELECT * FROM table WHERE param = $1", + expectedArgs: []interface{}{"test"}, + }, + "null replacement": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: nil, + }, + }, + sql: "SELECT * FROM table WHERE param = !PARAM!", + expectedSql: "SELECT * FROM table WHERE param = $1", + expectedArgs: []interface{}{nil}, + }, + "complex sql replacement": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "WHERE param=?", + Value: 1, + }, + }, + sql: "SELECT * FROM table !PARAM!", + expectedSql: "SELECT * FROM table WHERE param=$1", + expectedArgs: []interface{}{1}, + }, + "subquery removal": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "", + Value: nil, + }, + }, + sql: "SELECT * FROM table !PARAM!", + expectedSql: "SELECT * FROM table ", + expectedArgs: []interface{}{}, + }, + "multiple params": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "? ? ?", + Value: 1, + }, + "!PARAM2!": { + Token: "!PARAM2!", + SQL: "???", + Value: 2, + }, + }, + sql: "!PARAM! !PARAM2! !PARAM!", + expectedSql: "$1 $1 $1 $2$2$2 $1 $1 $1", + expectedArgs: []interface{}{1, 2}, + }, + "unknown token": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: 1, + }, + }, + sql: "!NOT_PARAM! !PARAM! !NOT_PARAM!", + expectedSql: "!NOT_PARAM! $1 !NOT_PARAM!", + expectedArgs: []interface{}{1}, + }, + } + + for name, tc := range tests { + t.Run(name, fn(tc)) + } +} diff --git a/provider/test/emptycollection/provider.go b/provider/test/emptycollection/provider.go index 5512099f1..eef8d9ac0 100644 --- a/provider/test/emptycollection/provider.go +++ b/provider/test/emptycollection/provider.go @@ -19,7 +19,7 @@ func init() { } // NewProvider setups a test provider. there are not currently any config params supported -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { Count++ return &TileProvider{}, nil } @@ -40,7 +40,7 @@ func (tp *TileProvider) Layers() ([]provider.LayerInfo, error) { } // TilFeatures always returns a feature with a polygon outlining the tile's Extent (not Buffered Extent) -func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, fn func(f *provider.Feature) error) error { +func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, queryParams provider.Params, fn func(f *provider.Feature) error) error { // get tile bounding box _, srid := t.Extent() diff --git a/provider/test/provider.go b/provider/test/provider.go index b5288b45c..011242afb 100644 --- a/provider/test/provider.go +++ b/provider/test/provider.go @@ -28,7 +28,7 @@ func init() { } // NewTileProvider setups a test provider. there are not currently any config params supported -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { lock.Lock() Count++ lock.Unlock() @@ -37,7 +37,7 @@ func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { // NewMVTTileProvider setups a test provider for mvt tiles providers. The only supported parameter is // "test_file", which should point to a mvt tile file to return for MVTForLayers -func NewMVTTileProvider(config dict.Dicter) (provider.MVTTiler, error) { +func NewMVTTileProvider(config dict.Dicter, maps []provider.Map) (provider.MVTTiler, error) { lock.Lock() MVTCount++ lock.Unlock() @@ -86,7 +86,7 @@ func (tp *TileProvider) Layers() ([]provider.LayerInfo, error) { } // TileFeatures always returns a feature with a polygon outlining the tile's Extent (not Buffered Extent) -func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, fn func(f *provider.Feature) error) error { +func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, queryParams provider.Params, fn func(f *provider.Feature) error) error { // get tile bounding box ext, srid := t.Extent() @@ -103,7 +103,7 @@ func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provid } // MVTForLayers mocks out MVTForLayers by just returning the MVTTile bytes, this will never error -func (tp *TileProvider) MVTForLayers(ctx context.Context, _ provider.Tile, _ []provider.Layer) ([]byte, error) { +func (tp *TileProvider) MVTForLayers(ctx context.Context, _ provider.Tile, _ provider.Params, _ []provider.Layer) ([]byte, error) { // TODO(gdey): fill this out. if tp == nil { return nil, nil diff --git a/server/handle_map_layer_zxy.go b/server/handle_map_layer_zxy.go index ddc923cc4..39345a3b0 100644 --- a/server/handle_map_layer_zxy.go +++ b/server/handle_map_layer_zxy.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/go-spatial/tegola/observability" + "github.com/go-spatial/tegola/provider" "github.com/dimfeld/httptreemux" "github.com/go-spatial/geom/encoding/mvt" @@ -95,13 +96,16 @@ func (req *HandleMapLayerZXY) parseURI(r *http.Request) error { return nil } -// URI scheme: /maps/:map_name/:layer_name/:z/:x/:y +// URI scheme: /maps/:map_name/:layer_name/:z/:x/:y?param=value // map_name - map name in the config file // layer_name - name of the single map layer to render // z, x, y - tile coordinates as described in the Slippy Map Tilenames specification -// z - zoom level -// x - row -// y - column +// +// z - zoom level +// x - row +// y - column +// +// param - configurable query parameters and their values func (req HandleMapLayerZXY) ServeHTTP(w http.ResponseWriter, r *http.Request) { // parse our URI if err := req.parseURI(r); err != nil { @@ -158,8 +162,16 @@ func (req HandleMapLayerZXY) ServeHTTP(w http.ResponseWriter, r *http.Request) { m = m.AddDebugLayers() } + // check for query parameters and populate param map with their values + params, err := extractParameters(m, r) + if err != nil { + log.Error(err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + encodeCtx := context.WithValue(r.Context(), observability.ObserveVarMapName, m.Name) - pbyte, err := m.Encode(encodeCtx, tile) + pbyte, err := m.Encode(encodeCtx, tile, params) if err != nil { switch { @@ -183,10 +195,42 @@ func (req HandleMapLayerZXY) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Add("Content-Type", mvt.MimeType) w.Header().Add("Content-Length", fmt.Sprintf("%d", len(pbyte))) w.WriteHeader(http.StatusOK) - w.Write(pbyte) + + _, err = w.Write(pbyte) + if err != nil { + log.Errorf("error writing tile z:%v, x:%v, y:%v - %v", req.z, req.x, req.y, err) + } // check for tile size warnings if len(pbyte) > MaxTileSize { log.Infof("tile z:%v, x:%v, y:%v is rather large - %vKb", req.z, req.x, req.y, len(pbyte)/1024) } } + +func extractParameters(m atlas.Map, r *http.Request) (provider.Params, error) { + var params provider.Params + if m.Params != nil && len(m.Params) > 0 { + params = make(provider.Params) + err := r.ParseForm() + if err != nil { + return nil, err + } + + for _, param := range m.Params { + if r.Form.Has(param.Name) { + val, err := param.ToValue(r.Form.Get(param.Name)) + if err != nil { + return nil, err + } + params[param.Token] = val + } else { + p, err := param.ToDefaultValue() + if err != nil { + return nil, err + } + params[param.Token] = p + } + } + } + return params, nil +} diff --git a/server/middleware_tile_cache.go b/server/middleware_tile_cache.go index a187da91d..243f3d03e 100644 --- a/server/middleware_tile_cache.go +++ b/server/middleware_tile_cache.go @@ -28,6 +28,12 @@ func TileCacheHandler(a *atlas.Atlas, next http.Handler) http.Handler { return } + // ignore requests with query parameters + if r.URL.RawQuery != "" { + next.ServeHTTP(w, r) + return + } + // parse our URI into a cache key structure (remove any configured URIPrefix + "maps/" ) key, err := cache.ParseKey(strings.TrimPrefix(r.URL.Path, path.Join(URIPrefix, "maps"))) if err != nil { diff --git a/server/middleware_tile_cache_test.go b/server/middleware_tile_cache_test.go index c69b2ced7..7563839fe 100644 --- a/server/middleware_tile_cache_test.go +++ b/server/middleware_tile_cache_test.go @@ -79,3 +79,63 @@ func TestMiddlewareTileCacheHandler(t *testing.T) { t.Run(name, fn(tc)) } } + +func TestMiddlewareTileCacheHandlerIgnoreParams(t *testing.T) { + type tcase struct { + uri string + uriPrefix string + } + + fn := func(tc tcase) func(t *testing.T) { + return func(t *testing.T) { + var err error + + if tc.uriPrefix != "" { + server.URIPrefix = tc.uriPrefix + } else { + server.URIPrefix = "/" + } + + a := newTestMapWithLayers(testLayer1, testLayer2, testLayer3) + cacher, _ := memory.New(nil) + a.SetCache(cacher) + + w, router, err := doRequest(a, "GET", tc.uri, nil) + if err != nil { + t.Errorf("error making request, expected nil got %v", err) + return + } + + // we expect the cache to not being used + if w.Header().Get("Tegola-Cache") != "" { + t.Errorf("no header Tegola-Cache is expected, got %v", w.Header().Get("Tegola-Cache")) + return + } + + // play the request again + r, err := http.NewRequest("GET", tc.uri, nil) + if err != nil { + t.Errorf("error making request, expected nil got %v", err) + return + } + + w = httptest.NewRecorder() + router.ServeHTTP(w, r) + + if w.Header().Get("Tegola-Cache") != "" { + t.Errorf("no header Tegola-Cache is expected, got %v", w.Header().Get("Tegola-Cache")) + return + } + } + } + + tests := map[string]tcase{ + "map params": { + uri: "/maps/test-map/10/2/3.pbf?param=value", + }, + } + + for name, tc := range tests { + t.Run(name, fn(tc)) + } +} diff --git a/tile.go b/tile.go index 609238f6f..38ecd7bc5 100644 --- a/tile.go +++ b/tile.go @@ -17,7 +17,7 @@ const ( var UnknownConversionError = fmt.Errorf("do not know how to convert value to requested value") -//Tile slippy map tilenames +// Tile slippy map tilenames // http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames type Tile struct { Z uint @@ -198,7 +198,7 @@ func (t *Tile) ZLevel() uint { return t.Z } -//ZRes takes a web mercator zoom level and returns the pixel resolution for that +// ZRes takes a web mercator zoom level and returns the pixel resolution for that // scale, assuming t.Extent x t.Extent pixel tiles. Non-integer zoom levels are accepted. // ported from: https://raw.githubusercontent.com/mapbox/postgis-vt-util/master/postgis-vt-util.sql // 40075016.6855785 is the equator in meters for WGS84 at z=0