From 52756c760ab1530c151b79734f02cb82834b57f4 Mon Sep 17 00:00:00 2001 From: Andrew DeChristopher Date: Thu, 22 Apr 2021 14:03:04 -0400 Subject: [PATCH 1/8] feat: add Query Parameters to Map config fix: small typos and doc cleanup in provider/postgis feat: implemented Query Parameters into postgres provider feat: implemented default parameter values feat: revamp param specification in config --- atlas/atlas.go | 35 +++++ atlas/map.go | 8 +- cmd/internal/register/maps.go | 5 + config/config.go | 191 +++++++++++++++++++++++++-- config/config_test.go | 231 ++++++++++++++++++++++++++++++++- config/errors.go | 67 ++++++++++ provider/gpkg/gpkg_register.go | 34 ++--- provider/gpkg/util.go | 10 +- provider/postgis/postgis.go | 32 +++-- provider/postgis/util.go | 80 ++++++++---- server/handle_map_layer_zxy.go | 34 ++++- 11 files changed, 655 insertions(+), 72 deletions(-) diff --git a/atlas/atlas.go b/atlas/atlas.go index 71e2d23c1..5a181add2 100644 --- a/atlas/atlas.go +++ b/atlas/atlas.go @@ -11,6 +11,7 @@ import ( "github.com/go-spatial/geom/slippy" "github.com/go-spatial/tegola" "github.com/go-spatial/tegola/cache" + "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/internal/log" "github.com/go-spatial/tegola/internal/observer" "github.com/go-spatial/tegola/observability" @@ -78,6 +79,10 @@ type Atlas struct { // holds a reference to the observer backend observer observability.Interface + // holds a reference to configured query parameters for maps + // key = map name + params map[string][]config.QueryParameter + // publishBuildInfo indicates if we should publish the build info on change of observer // this is set by calling PublishBuildInfo, which will publish // the build info on the observer and insure changes to observer @@ -213,6 +218,36 @@ func (a *Atlas) AddMap(m Map) { a.maps[m.Name] = m } +// AddParams adds the given query parameters to the atlas params map +// keyed by the map name, with upper-cased tokens +func (a *Atlas) AddParams(name string, params []config.QueryParameter) { + if a == nil { + defaultAtlas.AddParams(name, params) + return + } + if a.params == nil { + a.params = make(map[string][]config.QueryParameter) + } + a.params[name] = params +} + +// GetParams returns any configured query parameters for the given +// map by name +func (a *Atlas) GetParams(name string) []config.QueryParameter { + if a == nil { + return defaultAtlas.GetParams(name) + } + return a.params[name] +} + +// HasParams returns true if the given map by name has configured query parameters +func (a *Atlas) HasParams(name string) bool { + if a == nil { + return defaultAtlas.HasParams(name) + } + return len(a.params[name]) > 0 +} + // GetCache returns the registered cache if one is registered, otherwise nil func (a *Atlas) GetCache() cache.Interface { if a == nil { diff --git a/atlas/map.go b/atlas/map.go index 2b635e5b4..d468d49b3 100644 --- a/atlas/map.go +++ b/atlas/map.go @@ -27,7 +27,7 @@ import ( "github.com/go-spatial/tegola/provider/debug" ) -// NewMap creates a new map with the necessary default values +// NewWebMercatorMap creates a new map with the necessary default values func NewWebMercatorMap(name string) Map { return Map{ Name: name, @@ -40,6 +40,7 @@ func NewWebMercatorMap(name string) Map { } } +// Map defines a Web Mercator map type Map struct { Name string // Contains an attribution to be displayed when the map is shown to a user. @@ -360,7 +361,10 @@ func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile) ([]byte, erro } // add layers to our tile - mvtTile.AddLayers(mvtLayers...) + err := mvtTile.AddLayers(mvtLayers...) + if err != nil { + return nil, err + } // generate the MVT tile vtile, err := mvtTile.VTile(ctx) diff --git a/cmd/internal/register/maps.go b/cmd/internal/register/maps.go index ed043500a..1289c340c 100644 --- a/cmd/internal/register/maps.go +++ b/cmd/internal/register/maps.go @@ -155,6 +155,11 @@ func Maps(a *atlas.Atlas, maps []config.Map, providers map[string]provider.Tiler } newMap.Layers = append(newMap.Layers, layer) } + + if len(m.Parameters) > 0 { + a.AddParams(string(m.Name), m.Parameters) + } + a.AddMap(newMap) } return nil diff --git a/config/config.go b/config/config.go index 79d3d2549..de13e2992 100644 --- a/config/config.go +++ b/config/config.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "os" + "strconv" "strings" "time" @@ -16,6 +17,61 @@ import ( "github.com/go-spatial/tegola/provider" ) +const ( + BboxToken = "!BBOX!" + ZoomToken = "!ZOOM!" + XToken = "!X!" + YToken = "!Y!" + ZToken = "!Z!" + ScaleDenominatorToken = "!SCALE_DENOMINATOR!" + PixelWidthToken = "!PIXEL_WIDTH!" + PixelHeightToken = "!PIXEL_HEIGHT!" + IdFieldToken = "!ID_FIELD!" + GeomFieldToken = "!GEOM_FIELD!" + GeomTypeToken = "!GEOM_TYPE!" +) + +// ReservedTokens for query injection +var ReservedTokens = []string{ + BboxToken, + ZoomToken, + XToken, + YToken, + ZToken, + ScaleDenominatorToken, + PixelWidthToken, + PixelHeightToken, + IdFieldToken, + GeomFieldToken, + GeomTypeToken, +} + +// IsReservedToken returns true if the specified token is reserved +func IsReservedToken(token string) bool { + for _, t := range ReservedTokens { + if token == t { + return true + } + } + return false +} + +// ParamTypeDecoders is a collection of parsers for different types of user-defined parameters +var ParamTypeDecoders = map[string]func(string) (interface{}, error){ + "int": func(s string) (interface{}, error) { + return strconv.Atoi(s) + }, + "float": func(s string) (interface{}, error) { + return strconv.ParseFloat(s, 32) + }, + "string": func(s string) (interface{}, error) { + return s, nil + }, + "bool": func(s string) (interface{}, error) { + return strconv.ParseBool(s) + }, +} + var blacklistHeaders = []string{"content-encoding", "content-length", "content-type"} // Config represents a tegola config file. @@ -52,12 +108,79 @@ type Webserver struct { // A Map represents a map in the Tegola Config file. type Map struct { - Name env.String `toml:"name"` - Attribution env.String `toml:"attribution"` - Bounds []env.Float `toml:"bounds"` - Center [3]env.Float `toml:"center"` - Layers []MapLayer `toml:"layers"` - TileBuffer *env.Int `toml:"tile_buffer"` + Name env.String `toml:"name"` + Attribution env.String `toml:"attribution"` + Bounds []env.Float `toml:"bounds"` + Center [3]env.Float `toml:"center"` + Layers []MapLayer `toml:"layers"` + Parameters []QueryParameter `toml:"params"` + TileBuffer *env.Int `toml:"tile_buffer"` +} + +// ValidateParams ensures configured params don't conflict with existing +// query tokens or have overlapping names +func (m Map) ValidateParams() error { + if len(m.Parameters) == 0 { + return nil + } + + var usedNames, usedTokens []string + + for _, param := range m.Parameters { + if _, ok := ParamTypeDecoders[param.Type]; !ok { + return ErrParamUnknownType{ + MapName: string(m.Name), + Parameter: param, + } + } + + if len(param.DefaultSQL) > 0 && len(param.DefaultValue) > 0 { + return ErrParamTwoDefaults{ + MapName: string(m.Name), + Parameter: param, + } + } + + if len(param.DefaultValue) > 0 { + decoderFn := ParamTypeDecoders[param.Type] + if _, err := decoderFn(param.DefaultValue); err != nil { + return ErrParamInvalidDefault{ + MapName: string(m.Name), + Parameter: param, + } + } + } + + if IsReservedToken(param.Token) { + return ErrParamTokenReserved{ + MapName: string(m.Name), + Parameter: param, + } + } + + for _, name := range usedNames { + if name == param.Name { + return ErrParamNameDuplicate{ + MapName: string(m.Name), + Parameter: param, + } + } + } + + for _, token := range usedTokens { + if token == param.Token { + return ErrParamTokenDuplicate{ + MapName: string(m.Name), + Parameter: param, + } + } + } + + usedNames = append(usedNames, param.Name) + usedTokens = append(usedTokens, param.Token) + } + + return nil } // MapLayer represents a the config for a layer in a map @@ -101,6 +224,37 @@ func (ml MapLayer) GetName() (string, error) { return name, err } +// QueryParameter represents an HTTP query parameter specified for use with +// a given map instance. +type QueryParameter struct { + Name string `toml:"name"` + Token string `toml:"token"` + Type string `toml:"type"` + SQL string `toml:"sql"` + // DefaultSQL replaces SQL if param wasn't passed. Either default_sql or + // default_value can be specified + DefaultSQL string `toml:"default_sql"` + DefaultValue string `toml:"default_value"` + IsRequired bool +} + +// Normalize will normalize param and set the default values +func (param *QueryParameter) Normalize() { + param.Token = strings.ToUpper(param.Token) + + sql := "?" + if len(param.SQL) > 0 { + sql = param.SQL + } + param.SQL = sql + + isRequired := true + if len(param.DefaultSQL) > 0 || len(param.DefaultValue) > 0 { + isRequired = false + } + param.IsRequired = isRequired +} + // Validate checks the config for issues func (c *Config) Validate() error { @@ -144,6 +298,12 @@ func (c *Config) Validate() error { // map of layers to providers mapLayers := map[string]map[string]MapLayer{} for mapKey, m := range c.Maps { + + // validate any declared query parameters + if err := m.ValidateParams(); err != nil { + return err + } + if _, ok := mapLayers[string(m.Name)]; !ok { mapLayers[string(m.Name)] = map[string]MapLayer{} } @@ -152,7 +312,7 @@ func (c *Config) Validate() error { // we can only have the same provider for all layers. // This allow us to track what the first found provider // is. - provider := "" + currentProvider := "" isMVTProvider := false for layerKey, l := range m.Layers { pname, _, err := l.ProviderLayerName() @@ -160,11 +320,11 @@ func (c *Config) Validate() error { return err } - if provider == "" { + if currentProvider == "" { // This is the first provider we found. // For MVTProviders all others need to be the same, so store it // so we can check later - provider = pname + currentProvider = pname } isMvt, doesExists := mvtproviders[pname] @@ -181,13 +341,13 @@ func (c *Config) Validate() error { isMVTProvider = isMVTProvider || isMvt // only need to do this check if we are dealing with MVTProviders - if isMVTProvider && pname != provider { + if isMVTProvider && pname != currentProvider { // for mvt_providers we can only have the same provider // for all layers // check to see if mvtproviders[pname] || isMVTProvider { return ErrMVTDifferentProviders{ - Original: provider, + Original: currentProvider, Current: pname, } } @@ -286,11 +446,18 @@ func (c *Config) ConfigureTileBuffers() { // Parse will parse the Tegola config file provided by the io.Reader. func Parse(reader io.Reader, location string) (conf Config, err error) { // decode conf file, don't care about the meta data. - _, err = toml.DecodeReader(reader, &conf) + _, err = toml.NewDecoder(reader).Decode(&conf) if err != nil { return conf, err } + for _, m := range conf.Maps { + for k, p := range m.Parameters { + p.Normalize() + m.Parameters[k] = p + } + } + conf.LocationName = location conf.ConfigureTileBuffers() diff --git a/config/config_test.go b/config/config_test.go index d41572e9b..e6f860954 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -156,7 +156,26 @@ func TestParse(t *testing.T) { max_zoom = 20 dont_simplify = true dont_clip = true - dont_clean = true`, + dont_clean = true + + [[maps.params]] + name = "param1" + token = "!param1!" + type = "string" + + [[maps.params]] + name = "param2" + token = "!PARAM2!" + type = "int" + sql = "AND ANSWER = ?" + default_value = "42" + + [[maps.params]] + name = "param3" + token = "!PARAM3!" + type = "float" + default_sql = "AND PI = 3.1415926" + `, expected: config.Config{ TileBuffer: env.IntPtr(env.Int(12)), LocationName: "", @@ -208,6 +227,31 @@ func TestParse(t *testing.T) { DontClean: true, }, }, + Parameters: []config.QueryParameter{ + { + Name: "param1", + Token: "!PARAM1!", + SQL: "?", + Type: "string", + IsRequired: true, + }, + { + Name: "param2", + Token: "!PARAM2!", + Type: "int", + SQL: "AND ANSWER = ?", + DefaultValue: "42", + IsRequired: false, + }, + { + Name: "param3", + Token: "!PARAM3!", + Type: "float", + SQL: "?", + DefaultSQL: "AND PI = 3.1415926", + IsRequired: false, + }, + }, }, }, }, @@ -1189,6 +1233,191 @@ func TestValidate(t *testing.T) { }, }, }, + "13 reserved parameter token": { + config: config.Config{ + Maps: []config.Map{ + { + Name: "bad_param", + Parameters: []config.QueryParameter{ + { + Name: "param", + Token: "!BBOX!", + Type: "int", + }, + }, + }, + }, + }, + expectedErr: config.ErrParamTokenReserved{ + MapName: "bad_param", + Parameter: config.QueryParameter{ + Name: "param", + Token: "!BBOX!", + Type: "int", + }, + }, + }, + "13 duplicate parameter name": { + config: config.Config{ + Maps: []config.Map{ + { + Name: "dupe_param_name", + Parameters: []config.QueryParameter{ + { + Name: "param", + Token: "!PARAM!", + Type: "int", + }, + { + Name: "param", + Token: "!PARAM2!", + Type: "int", + }, + }, + }, + }, + }, + expectedErr: config.ErrParamNameDuplicate{ + MapName: "dupe_param_name", + Parameter: config.QueryParameter{ + Name: "param", + Token: "!PARAM2!", + Type: "int", + }, + }, + }, + "13 duplicate parameter token": { + config: config.Config{ + Maps: []config.Map{ + { + Name: "dupe_param_token", + Parameters: []config.QueryParameter{ + { + Name: "param", + Token: "!PARAM!", + Type: "int", + }, + { + Name: "param2", + Token: "!PARAM!", + Type: "int", + }, + }, + }, + }, + }, + expectedErr: config.ErrParamTokenDuplicate{ + MapName: "dupe_param_token", + Parameter: config.QueryParameter{ + Name: "param2", + Token: "!PARAM!", + Type: "int", + }, + }, + }, + "13 parameter unknown type": { + config: config.Config{ + Maps: []config.Map{ + { + Name: "unknown_param_type", + Parameters: []config.QueryParameter{ + { + Name: "param", + Token: "!BBOX!", + Type: "foo", + }, + }, + }, + }, + }, + expectedErr: config.ErrParamUnknownType{ + MapName: "unknown_param_type", + Parameter: config.QueryParameter{ + Name: "param", + Token: "!BBOX!", + Type: "foo", + }, + }, + }, + "13 parameter two defaults": { + config: config.Config{ + Maps: []config.Map{ + { + Name: "unknown_two_defaults", + Parameters: []config.QueryParameter{ + { + Name: "param", + Token: "!BBOX!", + Type: "string", + DefaultSQL: "foo", + DefaultValue: "bar", + }, + }, + }, + }, + }, + expectedErr: config.ErrParamTwoDefaults{ + MapName: "unknown_two_defaults", + Parameter: config.QueryParameter{ + Name: "param", + Token: "!BBOX!", + Type: "string", + DefaultSQL: "foo", + DefaultValue: "bar", + }, + }, + }, + "13 parameter invalid default": { + config: config.Config{ + Maps: []config.Map{ + { + Name: "parameter_invalid_default", + + Parameters: []config.QueryParameter{ + { + Name: "param", + Token: "!BBOX!", + Type: "int", + DefaultValue: "foo", + }, + }, + }, + }, + }, + expectedErr: config.ErrParamInvalidDefault{ + MapName: "parameter_invalid_default", + Parameter: config.QueryParameter{ + Name: "param", + Token: "!BBOX!", + Type: "int", + DefaultValue: "foo", + }, + }, + }, + "13 invalid token name": { + config: config.Config{ + Maps: []config.Map{ + { + Name: "parameter_invalid_token", + Parameters: []config.QueryParameter{ + { + Name: "param", + Token: "!Token with spaces!", + Type: "int", + }, + }, + }, + }, + }, + expectedErr: config.ErrParamBadTokenName{ + MapName: "parameter_invalid_token", + Parameter: config.QueryParameter{ + Name: "param", + Token: "!Token with spaces!", + Type: "int", + }, + }, + }, } for name, tc := range tests { diff --git a/config/errors.go b/config/errors.go index 3bc6754e5..f62ee744e 100644 --- a/config/errors.go +++ b/config/errors.go @@ -13,6 +13,73 @@ func (e ErrMapNotFound) Error() string { return fmt.Sprintf("config: map (%v) not found", e.MapName) } +type ErrParamTokenReserved struct { + MapName string + Parameter QueryParameter +} + +func (e ErrParamTokenReserved) Error() string { + return fmt.Sprintf("config: map %s has parameter %s referencing reserved token %s", + e.MapName, e.Parameter.Name, e.Parameter.Token) +} + +type ErrParamNameDuplicate struct { + MapName string + Parameter QueryParameter +} + +func (e ErrParamNameDuplicate) Error() string { + return fmt.Sprintf("config: map %s redeclares duplicate parameter with name %s", + e.MapName, e.Parameter.Name) +} + +type ErrParamTokenDuplicate struct { + MapName string + Parameter QueryParameter +} + +func (e ErrParamTokenDuplicate) Error() string { + return fmt.Sprintf("config: map %s redeclares existing parameter token %s in param %s", + e.MapName, e.Parameter.Token, e.Parameter.Name) +} + +type ErrParamUnknownType struct { + MapName string + Parameter QueryParameter +} + +func (e ErrParamUnknownType) Error() string { + validTypes := make([]string, len(ParamTypeDecoders)) + i := 0 + for k := range ParamTypeDecoders { + validTypes[i] = k + i++ + } + + return fmt.Sprintf("config: map %s has type %s in param %s, which is not one of the known types: %s", + e.MapName, e.Parameter.Type, e.Parameter.Name, strings.Join(validTypes, ",")) +} + +type ErrParamTwoDefaults struct { + MapName string + Parameter QueryParameter +} + +func (e ErrParamTwoDefaults) Error() string { + return fmt.Sprintf("config: map %s has both default_value and default_sql defined in param %s", + e.MapName, e.Parameter.Name) +} + +type ErrParamInvalidDefault struct { + MapName string + Parameter QueryParameter +} + +func (e ErrParamInvalidDefault) Error() string { + return fmt.Sprintf("config: map %s has default value in param %s that doesn't match the parameter's type %s", + e.MapName, e.Parameter.Name, e.Parameter.Type) +} + type ErrInvalidProviderForMap struct { MapName string ProviderName string diff --git a/provider/gpkg/gpkg_register.go b/provider/gpkg/gpkg_register.go index 46235e28d..c1b4c52f3 100644 --- a/provider/gpkg/gpkg_register.go +++ b/provider/gpkg/gpkg_register.go @@ -12,6 +12,7 @@ import ( "sort" "strings" + conf "github.com/go-spatial/tegola/config" _ "github.com/mattn/go-sqlite3" "github.com/go-spatial/geom" @@ -159,6 +160,7 @@ func extractColDefsFromSQL(sql string) []string { func featureTableMetaData(gpkg *sql.DB) (map[string]featureTableDetails, error) { // this query is used to read the metadata from the gpkg_contents, gpkg_geometry_columns, and // sqlite_master tables for tables that store geographic features. + //goland:noinspection SqlResolve qtext := ` SELECT c.table_name, c.min_x, c.min_y, c.max_x, c.max_y, c.srs_id, gc.column_name, gc.geometry_type_name, sm.sql @@ -341,22 +343,22 @@ func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { // solution without using an SQL parser on custom SQL statements allZoomsSQL := "IN (0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24)" tokenReplacer := strings.NewReplacer( - ">= "+zoomToken, allZoomsSQL, - ">="+zoomToken, allZoomsSQL, - "=> "+zoomToken, allZoomsSQL, - "=>"+zoomToken, allZoomsSQL, - "=< "+zoomToken, allZoomsSQL, - "=<"+zoomToken, allZoomsSQL, - "<= "+zoomToken, allZoomsSQL, - "<="+zoomToken, allZoomsSQL, - "!= "+zoomToken, allZoomsSQL, - "!="+zoomToken, allZoomsSQL, - "= "+zoomToken, allZoomsSQL, - "="+zoomToken, allZoomsSQL, - "> "+zoomToken, allZoomsSQL, - ">"+zoomToken, allZoomsSQL, - "< "+zoomToken, allZoomsSQL, - "<"+zoomToken, allZoomsSQL, + ">= "+conf.ZoomToken, allZoomsSQL, + ">="+conf.ZoomToken, allZoomsSQL, + "=> "+conf.ZoomToken, allZoomsSQL, + "=>"+conf.ZoomToken, allZoomsSQL, + "=< "+conf.ZoomToken, allZoomsSQL, + "=<"+conf.ZoomToken, allZoomsSQL, + "<= "+conf.ZoomToken, allZoomsSQL, + "<="+conf.ZoomToken, allZoomsSQL, + "!= "+conf.ZoomToken, allZoomsSQL, + "!="+conf.ZoomToken, allZoomsSQL, + "= "+conf.ZoomToken, allZoomsSQL, + "="+conf.ZoomToken, allZoomsSQL, + "> "+conf.ZoomToken, allZoomsSQL, + ">"+conf.ZoomToken, allZoomsSQL, + "< "+conf.ZoomToken, allZoomsSQL, + "<"+conf.ZoomToken, allZoomsSQL, ) customSQL = tokenReplacer.Replace(customSQL) diff --git a/provider/gpkg/util.go b/provider/gpkg/util.go index 38292a262..dba8f2f1d 100644 --- a/provider/gpkg/util.go +++ b/provider/gpkg/util.go @@ -6,11 +6,7 @@ import ( "strings" "github.com/go-spatial/geom" -) - -const ( - bboxToken = "!BBOX!" - zoomToken = "!ZOOM!" + "github.com/go-spatial/tegola/config" ) func replaceTokens(qtext string, zoom uint, extent *geom.Extent) string { @@ -34,8 +30,8 @@ func replaceTokens(qtext string, zoom uint, extent *geom.Extent) string { tokenReplacer := strings.NewReplacer( // The BBOX token requires parameters ordered as [maxx, minx, maxy, miny] and checks for overlap. // Until support for named parameters, we'll only support one BBOX token per query. - bboxToken, fmt.Sprintf("minx <= %v AND maxx >= %v AND miny <= %v AND maxy >= %v", extent.MaxX(), extent.MinX(), extent.MaxY(), extent.MinY()), - zoomToken, strconv.FormatUint(uint64(zoom), 10), + config.BboxToken, fmt.Sprintf("minx <= %v AND maxx >= %v AND miny <= %v AND maxy >= %v", extent.MaxX(), extent.MinX(), extent.MaxY(), extent.MinY()), + config.ZoomToken, strconv.FormatUint(uint64(zoom), 10), ) return tokenReplacer.Replace(qtext) diff --git a/provider/postgis/postgis.go b/provider/postgis/postgis.go index 13e5a0678..4a3d2e0d4 100644 --- a/provider/postgis/postgis.go +++ b/provider/postgis/postgis.go @@ -17,6 +17,7 @@ import ( "github.com/go-spatial/geom" "github.com/go-spatial/geom/encoding/wkb" "github.com/go-spatial/tegola" + conf "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/dict" "github.com/go-spatial/tegola/internal/log" "github.com/go-spatial/tegola/observability" @@ -150,7 +151,7 @@ func (p *Provider) Collectors(prefix string, cfgFn func(configKey string) map[st const ( // We quote the field and table names to prevent colliding with postgres keywords. - stdSQL = `SELECT %[1]v FROM %[2]v WHERE "%[3]v" && ` + bboxToken + stdSQL = `SELECT %[1]v FROM %[2]v WHERE "%[3]v" && ` + conf.BboxToken mvtSQL = `SELECT %[1]v FROM %[2]v` // SQL to get the column names, without hitting the information_schema. Though it might be better to hit the information_schema. @@ -195,7 +196,7 @@ const ( ) // isSelectQuery is a regexp to check if a query starts with `SELECT`, -// case-insensitive and ignoring any preceeding whitespace and SQL comments. +// case-insensitive and ignoring any preceding whitespace and SQL comments. var isSelectQuery = regexp.MustCompile(`(?i)^((\s*)(--.*\n)?)*select`) type hstoreOID struct { @@ -453,7 +454,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) dbconfig, err := BuildDBConfig(uri.String()) if err != nil { - return nil, fmt.Errorf("Failed while building db config: %w", err) + return nil, fmt.Errorf("failed while building db config: %v", err) } srid := DefaultSRID @@ -472,7 +473,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) pool, err := pgxpool.ConnectConfig(context.Background(), &p.config) if err != nil { - return nil, fmt.Errorf("Failed while creating connection pool: %w", err) + return nil, fmt.Errorf("failed while creating connection pool: %v", err) } p.pool = &connectionPoolCollector{Pool: pool} @@ -488,7 +489,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) lName, err := layer.String(ConfigKeyLayerName, nil) if err != nil { - return nil, fmt.Errorf("For layer (%v) we got the following error trying to get the layer's name field: %w", i, err) + return nil, fmt.Errorf("for layer (%v) we got the following error trying to get the layer's name field: %v", i, err) } if j, ok := lyrsSeen[lName]; ok { @@ -565,8 +566,8 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) // convert !BOX! (MapServer) and !bbox! (Mapnik) to !BBOX! for compatibility sql := strings.Replace(strings.Replace(sql, "!BOX!", "!BBOX!", -1), "!bbox!", "!BBOX!", -1) // make sure that the sql has a !BBOX! token - if !strings.Contains(sql, bboxToken) { - return nil, fmt.Errorf("SQL for layer (%v) %v is missing required token: %v", i, lName, bboxToken) + if !strings.Contains(sql, conf.BboxToken) { + return nil, fmt.Errorf("SQL for layer (%v) %v is missing required token: %v", i, lName, conf.BboxToken) } if !strings.Contains(sql, "*") { if !strings.Contains(sql, geomfld) { @@ -613,7 +614,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) return &p, nil } -// derived from github.com/jackc/pgx configTLS (https://github.com/jackc/pgx/blob/master/conn.go) +// ConfigTLS is derived from github.com/jackc/pgx configTLS (https://github.com/jackc/pgx/blob/master/conn.go) func ConfigTLS(sslMode string, sslKey string, sslCert string, sslRootCert string, cc *pgxpool.Config) error { switch sslMode { @@ -725,6 +726,9 @@ func (p Provider) inspectLayerGeomType(l *Layer) error { return err } + // remove all parameter tokens for inspection + sql = stripParams(sql) + rows, err := p.pool.Query(context.Background(), sql) if err != nil { return err @@ -813,6 +817,12 @@ func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider. return fmt.Errorf("error replacing layer tokens for layer (%v) SQL (%v): %w", layer, sql, err) } + // replace configured query parameters if any + sql, err = replaceParams(ctx, sql) + if err != nil { + return err + } + if debugExecuteSQL { log.Debugf("TEGOLA_SQL_DEBUG:EXECUTE_SQL for layer (%v): %v", layer, sql) } @@ -954,6 +964,12 @@ func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, layers [ return nil, err } + // replace configured query parameters if any + sql, err = replaceParams(ctx, sql) + if err != nil { + return nil, err + } + // ref: https://postgis.net/docs/ST_AsMVT.html // bytea ST_AsMVT(any_element row, text name, integer extent, text geom_name, text feature_id_name) diff --git a/provider/postgis/util.go b/provider/postgis/util.go index 43796e0c3..16d467188 100644 --- a/provider/postgis/util.go +++ b/provider/postgis/util.go @@ -10,6 +10,7 @@ import ( "github.com/go-spatial/geom" "github.com/go-spatial/tegola" "github.com/go-spatial/tegola/basic" + "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/internal/log" "github.com/go-spatial/tegola/provider" "github.com/jackc/pgproto3/v2" @@ -97,20 +98,6 @@ func genSQL(l *Layer, pool *connectionPoolCollector, tblname string, flds []stri return fmt.Sprintf(sqlTmpl, selectClause, tblname, l.geomField), nil } -const ( - bboxToken = "!BBOX!" - zoomToken = "!ZOOM!" - xToken = "!X!" - yToken = "!Y!" - zToken = "!Z!" - scaleDenominatorToken = "!SCALE_DENOMINATOR!" - pixelWidthToken = "!PIXEL_WIDTH!" - pixelHeightToken = "!PIXEL_HEIGHT!" - idFieldToken = "!ID_FIELD!" - geomFieldToken = "!GEOM_FIELD!" - geomTypeToken = "!GEOM_TYPE!" -) - // replaceTokens replaces tokens in the provided SQL string // // !BBOX! - the bounding box of the tile @@ -169,17 +156,17 @@ func replaceTokens(sql string, lyr *Layer, tile provider.Tile, withBuffer bool) // replace query string tokens z, x, y := tile.ZXY() tokenReplacer := strings.NewReplacer( - bboxToken, bbox, - zoomToken, strconv.FormatUint(uint64(z), 10), - zToken, strconv.FormatUint(uint64(z), 10), - xToken, strconv.FormatUint(uint64(x), 10), - yToken, strconv.FormatUint(uint64(y), 10), - idFieldToken, lyr.IDFieldName(), - geomFieldToken, lyr.GeomFieldName(), - geomTypeToken, geoType, - scaleDenominatorToken, strconv.FormatFloat(scaleDenominator, 'f', -1, 64), - pixelWidthToken, strconv.FormatFloat(pixelWidth, 'f', -1, 64), - pixelHeightToken, strconv.FormatFloat(pixelHeight, 'f', -1, 64), + config.BboxToken, bbox, + config.ZoomToken, strconv.FormatUint(uint64(z), 10), + config.ZToken, strconv.FormatUint(uint64(z), 10), + config.XToken, strconv.FormatUint(uint64(x), 10), + config.YToken, strconv.FormatUint(uint64(y), 10), + config.ScaleDenominatorToken, strconv.FormatFloat(scaleDenominator, 'f', -1, 64), + config.PixelWidthToken, strconv.FormatFloat(pixelWidth, 'f', -1, 64), + config.PixelHeightToken, strconv.FormatFloat(pixelHeight, 'f', -1, 64), + config.IdFieldToken, lyr.IDFieldName(), + config.GeomFieldToken, lyr.GeomFieldName(), + config.GeomTypeToken, geoType, ) uppercaseTokenSQL := uppercaseTokens(sql) @@ -187,6 +174,49 @@ func replaceTokens(sql string, lyr *Layer, tile provider.Tile, withBuffer bool) return tokenReplacer.Replace(uppercaseTokenSQL), nil } +// replaceParams substitutes configured query parameter tokens for their values +// within the provided SQL string +func replaceParams(ctx context.Context, sql string) (string, error) { + if ctx.Value("params") == nil { + return sql, nil + } + + params, ok := ctx.Value("params").(map[string]string) + if !ok { + return "", fmt.Errorf("param value is not a map: %v ", + ctx.Value("params")) + } + + replacerVals := make([]string, 0) + + for token, val := range params { + if val != "" && !paramValRe.Match([]byte(val)) { + return "", fmt.Errorf("param value for token %s contains illegal characters: %s ", + token, val) + } + + replacerVals = append(replacerVals, token, val) + } + + // generate replacer from token/value pairs + replacer := strings.NewReplacer(replacerVals...) + + uppercaseTokenSQL := uppercaseTokens(sql) + + return replacer.Replace(uppercaseTokenSQL), nil +} + +// stripParams will remove all parameter tokens from the query +func stripParams(sql string) string { + return tokenRe.ReplaceAllStringFunc(sql, func(s string) string { + return "" + }) +} + +// paramValRe restricts parameter values to a limited character set to prevent +// SQL injection attacks. TODO This is very primitive and needs more thought. +var paramValRe = regexp.MustCompile("[a-zA-Z0-9,._-]+") + var tokenRe = regexp.MustCompile("![a-zA-Z0-9_-]+!") // uppercaseTokens converts all !tokens! to uppercase !TOKENS!. Tokens can diff --git a/server/handle_map_layer_zxy.go b/server/handle_map_layer_zxy.go index ddc923cc4..45ed3a49b 100644 --- a/server/handle_map_layer_zxy.go +++ b/server/handle_map_layer_zxy.go @@ -159,6 +159,34 @@ func (req HandleMapLayerZXY) ServeHTTP(w http.ResponseWriter, r *http.Request) { } encodeCtx := context.WithValue(r.Context(), observability.ObserveVarMapName, m.Name) + var params map[string]string + + // check for query parameters and populate param map with their values + if req.Atlas.HasParams(req.mapName) { + params = make(map[string]string) + err = r.ParseForm() + if err == nil { + for _, param := range req.Atlas.GetParams(req.mapName) { + // Fetch parameter values + val := r.Form.Get(param.Name) + // Empty values are injected here to replace tokens with + // an empty string during sql query parameter replacement + // otherwise, the default value is used if configured. + if val == "" && param.DefaultValue != "" { + val = param.DefaultValue + } + + // inject parameter value or default into params map + params[param.Token] = val + } + + // update context passed to encoding if any params were parsed + if len(params) > 0 { + encodeCtx = context.WithValue(encodeCtx, "params", params) + } + } + } + pbyte, err := m.Encode(encodeCtx, tile) if err != nil { @@ -183,7 +211,11 @@ func (req HandleMapLayerZXY) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Add("Content-Type", mvt.MimeType) w.Header().Add("Content-Length", fmt.Sprintf("%d", len(pbyte))) w.WriteHeader(http.StatusOK) - w.Write(pbyte) + + _, err = w.Write(pbyte) + if err != nil { + log.Errorf("error writing tile z:%v, x:%v, y:%v - %v", req.z, req.x, req.y, err) + } // check for tile size warnings if len(pbyte) > MaxTileSize { From d0b2e4ce8fec762b095cee2ef116689c122e408e Mon Sep 17 00:00:00 2001 From: Sergei Gureev Date: Tue, 7 Jun 2022 20:00:36 +0300 Subject: [PATCH 2/8] feat: replaceParams with tests --- config/config.go | 11 +++ config/errors.go | 10 +++ provider/postgis/postgis.go | 57 +++++++------- provider/postgis/util.go | 51 +++++------- provider/postgis/util_internal_test.go | 105 +++++++++++++++++++++++++ 5 files changed, 175 insertions(+), 59 deletions(-) diff --git a/config/config.go b/config/config.go index de13e2992..08f0d0e3d 100644 --- a/config/config.go +++ b/config/config.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "os" + "regexp" "strconv" "strings" "time" @@ -56,6 +57,9 @@ func IsReservedToken(token string) bool { return false } +// ParameterTokenRegexp to validate QueryParameters +var ParameterTokenRegexp = regexp.MustCompile("![a-zA-Z0-9_-]+!") + // ParamTypeDecoders is a collection of parsers for different types of user-defined parameters var ParamTypeDecoders = map[string]func(string) (interface{}, error){ "int": func(s string) (interface{}, error) { @@ -158,6 +162,13 @@ func (m Map) ValidateParams() error { } } + if !ParameterTokenRegexp.MatchString(param.Token) { + return ErrParamBadTokenName{ + MapName: string(m.Name), + Parameter: param, + } + } + for _, name := range usedNames { if name == param.Name { return ErrParamNameDuplicate{ diff --git a/config/errors.go b/config/errors.go index f62ee744e..be6e5a59f 100644 --- a/config/errors.go +++ b/config/errors.go @@ -80,6 +80,16 @@ func (e ErrParamInvalidDefault) Error() string { e.MapName, e.Parameter.Name, e.Parameter.Type) } +type ErrParamBadTokenName struct { + MapName string + Parameter QueryParameter +} + +func (e ErrParamBadTokenName) Error() string { + return fmt.Sprintf("config: map %s has parameter %s referencing token with an invalid name %s", + e.MapName, e.Parameter.Name, e.Parameter.Token) +} + type ErrInvalidProviderForMap struct { MapName string ProviderName string diff --git a/provider/postgis/postgis.go b/provider/postgis/postgis.go index 4a3d2e0d4..9710cb6a9 100644 --- a/provider/postgis/postgis.go +++ b/provider/postgis/postgis.go @@ -406,26 +406,25 @@ func BuildDBConfig(uri string) (*pgxpool.Config, error) { // trying to create a driver. This Provider supports the following fields // in the provided map[string]interface{} map: // -// host (string): [Required] postgis database host -// port (int): [Required] postgis database port (required) -// database (string): [Required] postgis database name -// user (string): [Required] postgis database user -// password (string): [Required] postgis database password -// srid (int): [Optional] The default SRID for the provider. Defaults to WebMercator (3857) but also supports WGS84 (4326) -// max_connections : [Optional] The max connections to maintain in the connection pool. Default is 100. 0 means no max. -// layers (map[string]struct{}) — This is map of layers keyed by the layer name. supports the following properties +// host (string): [Required] postgis database host +// port (int): [Required] postgis database port (required) +// database (string): [Required] postgis database name +// user (string): [Required] postgis database user +// password (string): [Required] postgis database password +// srid (int): [Optional] The default SRID for the provider. Defaults to WebMercator (3857) but also supports WGS84 (4326) +// max_connections : [Optional] The max connections to maintain in the connection pool. Default is 100. 0 means no max. +// layers (map[string]struct{}) — This is map of layers keyed by the layer name. supports the following properties // -// name (string): [Required] the name of the layer. This is used to reference this layer from map layers. -// tablename (string): [*Required] the name of the database table to query against. Required if sql is not defined. -// geometry_fieldname (string): [Optional] the name of the filed which contains the geometry for the feature. defaults to geom -// id_fieldname (string): [Optional] the name of the feature id field. defaults to gid -// fields ([]string): [Optional] a list of fields to include alongside the feature. Can be used if sql is not defined. -// srid (int): [Optional] the SRID of the layer. Supports 3857 (WebMercator) or 4326 (WGS84). -// sql (string): [*Required] custom SQL to use use. Required if tablename is not defined. Supports the following tokens: -// -// !BBOX! - [Required] will be replaced with the bounding box of the tile before the query is sent to the database. -// !ZOOM! - [Optional] will be replaced with the "Z" (zoom) value of the requested tile. +// name (string): [Required] the name of the layer. This is used to reference this layer from map layers. +// tablename (string): [*Required] the name of the database table to query against. Required if sql is not defined. +// geometry_fieldname (string): [Optional] the name of the filed which contains the geometry for the feature. defaults to geom +// id_fieldname (string): [Optional] the name of the feature id field. defaults to gid +// fields ([]string): [Optional] a list of fields to include alongside the feature. Can be used if sql is not defined. +// srid (int): [Optional] the SRID of the layer. Supports 3857 (WebMercator) or 4326 (WGS84). +// sql (string): [*Required] custom SQL to use use. Required if tablename is not defined. Supports the following tokens: // +// !BBOX! - [Required] will be replaced with the bounding box of the tile before the query is sent to the database. +// !ZOOM! - [Optional] will be replaced with the "Z" (zoom) value of the requested tile. func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) { uri, params, err := BuildURI(config) if err != nil { @@ -726,6 +725,7 @@ func (p Provider) inspectLayerGeomType(l *Layer) error { return err } + // TODO (bemyak): Figure out what is this // remove all parameter tokens for inspection sql = stripParams(sql) @@ -818,13 +818,14 @@ func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider. } // replace configured query parameters if any - sql, err = replaceParams(ctx, sql) + args := make([]interface{}, 0) + sql = replaceParams(queryParams, sql, &args) if err != nil { return err } if debugExecuteSQL { - log.Debugf("TEGOLA_SQL_DEBUG:EXECUTE_SQL for layer (%v): %v", layer, sql) + log.Debugf("TEGOLA_SQL_DEBUG:EXECUTE_SQL for layer (%v): %v with args %v", layer, sql, args) } // context check @@ -833,7 +834,7 @@ func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider. } now := time.Now() - rows, err := p.pool.Query(ctx, sql) + rows, err := p.pool.Query(ctx, sql, args...) if p.queryHistogramSeconds != nil { z, _, _ := tile.ZXY() lbls := prometheus.Labels{ @@ -850,7 +851,7 @@ func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider. // trying to clean itself up. defer rows.Close() if err := ctxErr(ctx, err); err != nil { - return fmt.Errorf("error running layer (%v) SQL (%v): %w", layer, sql, err) + return fmt.Errorf("error running layer (%v) SQL (%v) with args %v: %w", layer, sql, args, err) } // fieldDescriptions @@ -946,6 +947,8 @@ func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, layers [ } } + args := make([]interface{}, 0) + for i := range layers { if debug { log.Debugf("looking for layer: %v", layers[i]) @@ -957,7 +960,7 @@ func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, layers [ log.Warnf("provider layer not found %v", layers[i].Name) } if debugLayerSQL { - log.Debugf("SQL for Layer(%v):\n%v\n", l.Name(), l.sql) + log.Debugf("SQL for Layer(%v):\n%v\nargs:%v\n", l.Name(), l.sql, args) } sql, err := replaceTokens(l.sql, &l, tile, false) if err := ctxErr(ctx, err); err != nil { @@ -965,10 +968,7 @@ func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, layers [ } // replace configured query parameters if any - sql, err = replaceParams(ctx, sql) - if err != nil { - return nil, err - } + sql = replaceParams(params, sql, &args) // ref: https://postgis.net/docs/ST_AsMVT.html // bytea ST_AsMVT(any_element row, text name, integer extent, text geom_name, text feature_id_name) @@ -998,9 +998,10 @@ func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, layers [ } { now := time.Now() - err = p.pool.QueryRow(ctx, fsql).Scan(&data) + err = p.pool.QueryRow(ctx, fsql, args...).Scan(&data) if p.mvtProviderQueryHistogramSeconds != nil { z, _, _ := tile.ZXY() + // TODO (bemyak): add params as labels to prometheus? lbls := prometheus.Labels{ "z": strconv.FormatUint(uint64(z), 10), "map_name": mapName, diff --git a/provider/postgis/util.go b/provider/postgis/util.go index 16d467188..56f45a5b4 100644 --- a/provider/postgis/util.go +++ b/provider/postgis/util.go @@ -3,7 +3,6 @@ package postgis import ( "context" "fmt" - "regexp" "strconv" "strings" @@ -176,53 +175,43 @@ func replaceTokens(sql string, lyr *Layer, tile provider.Tile, withBuffer bool) // replaceParams substitutes configured query parameter tokens for their values // within the provided SQL string -func replaceParams(ctx context.Context, sql string) (string, error) { - if ctx.Value("params") == nil { - return sql, nil +func replaceParams(params map[string]provider.QueryParameter, sql string, args *[]interface{}) string { + if params == nil { + return sql } - params, ok := ctx.Value("params").(map[string]string) - if !ok { - return "", fmt.Errorf("param value is not a map: %v ", - ctx.Value("params")) - } - - replacerVals := make([]string, 0) - - for token, val := range params { - if val != "" && !paramValRe.Match([]byte(val)) { - return "", fmt.Errorf("param value for token %s contains illegal characters: %s ", - token, val) + for _, token := range config.ParameterTokenRegexp.FindAllString(sql, -1) { + param := params[token] + + // Replace every ? in the param's SQL with a positional argument + paramSQL := "" + for _, c := range param.SQL { + if c == '?' { + *args = append(*args, param.Value) + paramSQL = paramSQL + "$" + fmt.Sprint(len(*args)) + } else { + paramSQL += string(c) + } } - replacerVals = append(replacerVals, token, val) + // Finally, replace current token with the prepared SQL + sql = strings.Replace(sql, token, paramSQL, 1) } - // generate replacer from token/value pairs - replacer := strings.NewReplacer(replacerVals...) - - uppercaseTokenSQL := uppercaseTokens(sql) - - return replacer.Replace(uppercaseTokenSQL), nil + return sql } // stripParams will remove all parameter tokens from the query func stripParams(sql string) string { - return tokenRe.ReplaceAllStringFunc(sql, func(s string) string { + return config.ParameterTokenRegexp.ReplaceAllStringFunc(sql, func(s string) string { return "" }) } -// paramValRe restricts parameter values to a limited character set to prevent -// SQL injection attacks. TODO This is very primitive and needs more thought. -var paramValRe = regexp.MustCompile("[a-zA-Z0-9,._-]+") - -var tokenRe = regexp.MustCompile("![a-zA-Z0-9_-]+!") - // uppercaseTokens converts all !tokens! to uppercase !TOKENS!. Tokens can // contain alphanumerics, dash and underline chars. func uppercaseTokens(str string) string { - return tokenRe.ReplaceAllStringFunc(str, strings.ToUpper) + return config.ParameterTokenRegexp.ReplaceAllStringFunc(str, strings.ToUpper) } func transformVal(valType pgtype.OID, val interface{}) (interface{}, error) { diff --git a/provider/postgis/util_internal_test.go b/provider/postgis/util_internal_test.go index cf67d37b9..5a43eb2bb 100644 --- a/provider/postgis/util_internal_test.go +++ b/provider/postgis/util_internal_test.go @@ -74,6 +74,111 @@ func TestReplaceTokens(t *testing.T) { } } +func TestReplaceParams(t *testing.T) { + type tcase struct { + params map[string]provider.QueryParameter + sql string + expectedSql string + expectedArgs []interface{} + } + + fn := func(tc tcase) func(t *testing.T) { + return func(t *testing.T) { + args := make([]interface{}, 0) + out := replaceParams(tc.params, tc.sql, &args) + + if out != tc.expectedSql { + t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedSql, out) + return + } + + if len(tc.expectedArgs) != len(args) { + t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedArgs, args) + return + } + for i, arg := range tc.expectedArgs { + if arg != args[i] { + t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedArgs, args) + return + } + } + } + } + + tests := map[string]tcase{ + "nil params": { + params: nil, + sql: "SELECT * FROM table", + expectedSql: "SELECT * FROM table", + expectedArgs: []interface{}{}, + }, + "int replacement": { + params: map[string]provider.QueryParameter{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: 1, + }, + }, + sql: "SELECT * FROM table WHERE PARAM = !PARAM!", + expectedSql: "SELECT * FROM table WHERE PARAM = $1", + expectedArgs: []interface{}{1}, + }, + "string replacement": { + params: map[string]provider.QueryParameter{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: "test", + }, + }, + sql: "SELECT * FROM table WHERE PARAM = !PARAM!", + expectedSql: "SELECT * FROM table WHERE PARAM = $1", + expectedArgs: []interface{}{"test"}, + }, + "null replacement": { + params: map[string]provider.QueryParameter{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: nil, + }, + }, + sql: "SELECT * FROM table WHERE PARAM = !PARAM!", + expectedSql: "SELECT * FROM table WHERE PARAM = $1", + expectedArgs: []interface{}{nil}, + }, + "complex sql replacement": { + params: map[string]provider.QueryParameter{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "WHERE PARAM=?", + Value: 1, + }, + }, + sql: "SELECT * FROM table !PARAM!", + expectedSql: "SELECT * FROM table WHERE PARAM=$1", + expectedArgs: []interface{}{1}, + }, + "subquery removal": { + params: map[string]provider.QueryParameter{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "", + Value: nil, + }, + }, + sql: "SELECT * FROM table !PARAM!", + expectedSql: "SELECT * FROM table ", + expectedArgs: []interface{}{}, + }, + } + + for name, tc := range tests { + t.Run(name, fn(tc)) + } +} + func TestUppercaseTokens(t *testing.T) { type tcase struct { str string From f01a322ca0ae8297a97fbef59e5b1d9b882fe291 Mon Sep 17 00:00:00 2001 From: Sergei Gureev Date: Tue, 7 Jun 2022 10:49:07 +0300 Subject: [PATCH 3/8] refactor: thread query parameters all the way to provider --- atlas/atlas.go | 3 +- atlas/map.go | 14 ++-- atlas/map_test.go | 2 +- config/config.go | 7 -- config/config_test.go | 11 ++- provider/debug/debug.go | 22 ++++-- provider/gpkg/gpkg.go | 6 +- provider/gpkg/gpkg_test.go | 5 +- provider/mvt_provider.go | 2 +- provider/postgis/postgis.go | 6 +- provider/postgis/postgis_internal_test.go | 4 +- provider/postgis/postgis_test.go | 2 +- provider/provider.go | 12 +++- provider/test/emptycollection/provider.go | 2 +- provider/test/provider.go | 4 +- server/handle_map_layer_zxy.go | 83 +++++++++++++++-------- 16 files changed, 116 insertions(+), 69 deletions(-) diff --git a/atlas/atlas.go b/atlas/atlas.go index 5a181add2..2130e4358 100644 --- a/atlas/atlas.go +++ b/atlas/atlas.go @@ -135,7 +135,8 @@ func (a *Atlas) SeedMapTile(ctx context.Context, m Map, z, x, y uint) error { tile := slippy.NewTile(z, x, y) // encode the tile - b, err := m.Encode(ctx, tile) + // TODO (bemyak): Make query parameters work with cache + b, err := m.Encode(ctx, tile, nil) if err != nil { return err } diff --git a/atlas/map.go b/atlas/map.go index d468d49b3..279f36500 100644 --- a/atlas/map.go +++ b/atlas/map.go @@ -180,7 +180,7 @@ func (m Map) FilterLayersByName(names ...string) Map { return m } -func (m Map) encodeMVTProviderTile(ctx context.Context, tile *slippy.Tile) ([]byte, error) { +func (m Map) encodeMVTProviderTile(ctx context.Context, tile *slippy.Tile, params map[string]provider.QueryParameter) ([]byte, error) { // get the list of our layers ptile := provider.NewTile(tile.Z, tile.X, tile.Y, uint(m.TileBuffer), uint(m.SRID)) @@ -191,13 +191,13 @@ func (m Map) encodeMVTProviderTile(ctx context.Context, tile *slippy.Tile) ([]by MVTName: m.Layers[i].MVTName(), } } - return m.mvtProvider.MVTForLayers(ctx, ptile, layers) + return m.mvtProvider.MVTForLayers(ctx, ptile, params, layers) } // encodeMVTTile will encode the given tile into mvt format // TODO (arolek): support for max zoom -func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile) ([]byte, error) { +func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile, params map[string]provider.QueryParameter) ([]byte, error) { // tile container var mvtTile mvt.Tile @@ -226,7 +226,7 @@ func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile) ([]byte, erro uint(m.TileBuffer), uint(m.SRID)) // fetch layer from data provider - err := l.Provider.TileFeatures(ctx, l.ProviderLayerName, ptile, func(f *provider.Feature) error { + err := l.Provider.TileFeatures(ctx, l.ProviderLayerName, ptile, params, func(f *provider.Feature) error { // skip row if geometry collection empty. g, ok := f.Geometry.(geom.Collection) if ok && len(g.Geometries()) == 0 { @@ -377,15 +377,15 @@ func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile) ([]byte, erro } // Encode will encode the given tile into mvt format -func (m Map) Encode(ctx context.Context, tile *slippy.Tile) ([]byte, error) { +func (m Map) Encode(ctx context.Context, tile *slippy.Tile, params map[string]provider.QueryParameter) ([]byte, error) { var ( tileBytes []byte err error ) if m.HasMVTProvider() { - tileBytes, err = m.encodeMVTProviderTile(ctx, tile) + tileBytes, err = m.encodeMVTProviderTile(ctx, tile, params) } else { - tileBytes, err = m.encodeMVTTile(ctx, tile) + tileBytes, err = m.encodeMVTTile(ctx, tile, params) } if err != nil { return nil, err diff --git a/atlas/map_test.go b/atlas/map_test.go index 0642a95f0..699df275a 100644 --- a/atlas/map_test.go +++ b/atlas/map_test.go @@ -202,7 +202,7 @@ func TestEncode(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { - out, err := tc.grid.Encode(context.Background(), tc.tile) + out, err := tc.grid.Encode(context.Background(), tc.tile, nil) if err != nil { t.Errorf("err: %v", err) return diff --git a/config/config.go b/config/config.go index 08f0d0e3d..2bc2ac7ec 100644 --- a/config/config.go +++ b/config/config.go @@ -246,7 +246,6 @@ type QueryParameter struct { // default_value can be specified DefaultSQL string `toml:"default_sql"` DefaultValue string `toml:"default_value"` - IsRequired bool } // Normalize will normalize param and set the default values @@ -258,12 +257,6 @@ func (param *QueryParameter) Normalize() { sql = param.SQL } param.SQL = sql - - isRequired := true - if len(param.DefaultSQL) > 0 || len(param.DefaultValue) > 0 { - isRequired = false - } - param.IsRequired = isRequired } // Validate checks the config for issues diff --git a/config/config_test.go b/config/config_test.go index e6f860954..8ae0f59e9 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -229,11 +229,10 @@ func TestParse(t *testing.T) { }, Parameters: []config.QueryParameter{ { - Name: "param1", - Token: "!PARAM1!", - SQL: "?", - Type: "string", - IsRequired: true, + Name: "param1", + Token: "!PARAM1!", + SQL: "?", + Type: "string", }, { Name: "param2", @@ -241,7 +240,6 @@ func TestParse(t *testing.T) { Type: "int", SQL: "AND ANSWER = ?", DefaultValue: "42", - IsRequired: false, }, { Name: "param3", @@ -249,7 +247,6 @@ func TestParse(t *testing.T) { Type: "float", SQL: "?", DefaultSQL: "AND PI = 3.1415926", - IsRequired: false, }, }, }, diff --git a/provider/debug/debug.go b/provider/debug/debug.go index 639e80ba6..212eab06e 100644 --- a/provider/debug/debug.go +++ b/provider/debug/debug.go @@ -6,6 +6,7 @@ package debug import ( "context" "fmt" + "strings" "github.com/go-spatial/geom" "github.com/go-spatial/tegola" @@ -32,11 +33,22 @@ func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { // Provider provides the debug provider type Provider struct{} -func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, fn func(f *provider.Feature) error) error { +func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { // get tile bounding box ext, srid := tile.Extent() + params := make([]string, len(queryParams)) + i := 0 + for _, param := range queryParams { + for k, v := range param.RawValues { + params[i] = fmt.Sprintf("%s=%s", k, v) + i++ + } + } + + paramsStr := strings.Join(params, " ") + switch layer { case "debug-tile-outline": debugTileOutline := provider.Feature{ @@ -44,7 +56,8 @@ func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider Geometry: ext.AsPolygon(), SRID: srid, Tags: map[string]interface{}{ - "type": "debug_buffer_outline", + "type": "debug_buffer_outline", + "params": paramsStr, }, } @@ -67,8 +80,9 @@ func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider }, SRID: srid, Tags: map[string]interface{}{ - "type": "debug_text", - "zxy": fmt.Sprintf("Z:%v, X:%v, Y:%v", z, x, y), + "type": "debug_text", + "params": paramsStr, + "zxy": fmt.Sprintf("Z:%v, X:%v, Y:%v", z, x, y), }, } diff --git a/provider/gpkg/gpkg.go b/provider/gpkg/gpkg.go index ddb402084..fe369219f 100644 --- a/provider/gpkg/gpkg.go +++ b/provider/gpkg/gpkg.go @@ -76,7 +76,7 @@ func (p *Provider) Layers() ([]provider.LayerInfo, error) { return ls, nil } -func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, fn func(f *provider.Feature) error) error { +func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { log.Debugf("fetching layer %v", layer) pLayer := p.layers[layer] @@ -101,6 +101,7 @@ func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider } var qtext string + args := make([]interface{}, 0) if pLayer.tablename != "" { // If layer was specified via "tablename" in config, construct query. @@ -121,11 +122,12 @@ func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider // If layer was specified via "sql" in config, collect it z, _, _ := tile.ZXY() qtext = replaceTokens(pLayer.sql, z, tileBBox) + qtext = provider.ReplaceParams(queryParams, qtext, &args) } log.Debugf("qtext: %v", qtext) - rows, err := p.db.Query(qtext) + rows, err := p.db.Query(qtext, args...) if err != nil { log.Errorf("err during query: %v - %v", qtext, err) return err diff --git a/provider/gpkg/gpkg_test.go b/provider/gpkg/gpkg_test.go index 4b461e059..8941e74af 100644 --- a/provider/gpkg/gpkg_test.go +++ b/provider/gpkg/gpkg_test.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package gpkg_test @@ -269,7 +270,7 @@ func TestTileFeatures(t *testing.T) { } var featureCount int - err = p.TileFeatures(context.TODO(), tc.layerName, &tc.tile, func(f *provider.Feature) error { + err = p.TileFeatures(context.TODO(), tc.layerName, &tc.tile, nil, func(f *provider.Feature) error { featureCount++ return nil }) @@ -416,7 +417,7 @@ func TestConfigs(t *testing.T) { return } - err = p.TileFeatures(context.TODO(), tc.layerName, &tc.tile, func(f *provider.Feature) error { + err = p.TileFeatures(context.TODO(), tc.layerName, &tc.tile, nil, func(f *provider.Feature) error { // check if the feature is part of the test if _, ok := tc.expectedTags[f.ID]; !ok { return nil diff --git a/provider/mvt_provider.go b/provider/mvt_provider.go index b0a41207b..558682a9f 100644 --- a/provider/mvt_provider.go +++ b/provider/mvt_provider.go @@ -10,7 +10,7 @@ type MVTTiler interface { Layerer // MVTForLayers will return a MVT byte array or an error for the given layer names. - MVTForLayers(ctx context.Context, tile Tile, layers []Layer) ([]byte, error) + MVTForLayers(ctx context.Context, tile Tile, params map[string]QueryParameter, layers []Layer) ([]byte, error) } // MVTInitFunc initialize a provider given a config map. The init function should validate the config map, and report any errors. This is called by the For function. diff --git a/provider/postgis/postgis.go b/provider/postgis/postgis.go index 9710cb6a9..16741c862 100644 --- a/provider/postgis/postgis.go +++ b/provider/postgis/postgis.go @@ -796,7 +796,8 @@ func (p Provider) Layers() ([]provider.LayerInfo, error) { } // TileFeatures adheres to the provider.Tiler interface -func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, fn func(f *provider.Feature) error) error { +// TODO (bemyak): Make an actual use of QueryParams +func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { var mapName string { @@ -932,7 +933,8 @@ func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider. return rows.Err() } -func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, layers []provider.Layer) ([]byte, error) { +// TODO (bemyak): Make an actual use of QueryParams +func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, params map[string]provider.QueryParameter, layers []provider.Layer) ([]byte, error) { var ( err error sqls = make([]string, 0, len(layers)) diff --git a/provider/postgis/postgis_internal_test.go b/provider/postgis/postgis_internal_test.go index ab0108faf..ae5f012ee 100644 --- a/provider/postgis/postgis_internal_test.go +++ b/provider/postgis/postgis_internal_test.go @@ -108,7 +108,7 @@ func TestMVTProviders(t *testing.T) { MVTName: tc.layerNames[i], } } - mvtTile, err := prvd.MVTForLayers(context.Background(), tc.tile, layers) + mvtTile, err := prvd.MVTForLayers(context.Background(), tc.tile, nil, layers) if err != nil { t.Errorf("NewProvider unexpected error: %v", err) return @@ -133,7 +133,7 @@ func TestMVTProviders(t *testing.T) { }, }, layerNames: []string{"land"}, - mvtTile: make([]byte, 174689), + mvtTile: make([]byte, 208993), tile: provider.NewTile(0, 0, 0, 16, 4326), }, } diff --git a/provider/postgis/postgis_test.go b/provider/postgis/postgis_test.go index 17cd03b2f..dc46241a2 100644 --- a/provider/postgis/postgis_test.go +++ b/provider/postgis/postgis_test.go @@ -212,7 +212,7 @@ func TestTileFeatures(t *testing.T) { layerName := tc.LayerConfig[0][postgis.ConfigKeyLayerName].(string) var featureCount int - err = p.TileFeatures(context.Background(), layerName, tc.tile, func(f *provider.Feature) error { + err = p.TileFeatures(context.Background(), layerName, tc.tile, nil, func(f *provider.Feature) error { // only verify tags on first feature if featureCount == 0 { for _, tag := range tc.expectedTags { diff --git a/provider/provider.go b/provider/provider.go index e203de9a4..7a52efe97 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -104,13 +104,23 @@ type Tile interface { BufferedExtent() (extent *geom.Extent, srid uint64) } +// Query parameter holds normalized parameter data ready to be inserted in the final query +type QueryParameter struct { + // Token to replace e.g., !TOKEN! + Token string + // SQL expression to be inserted. Contains "?" that will be replaced with an ordinal argument e.g., "$1" + SQL string + // Value that will be passed to the final query + Value interface{} +} + // Tiler is a Layers that allows one to encode features in that layer type Tiler interface { Layerer // TileFeature will stream decoded features to the callback function fn // if fn returns ErrCanceled, the TileFeatures method should stop processing - TileFeatures(ctx context.Context, layer string, t Tile, fn func(f *Feature) error) error + TileFeatures(ctx context.Context, layer string, t Tile, queryParams map[string]QueryParameter, fn func(f *Feature) error) error } // TilerUnion represents either a Std Tiler or and MVTTiler; only one should be not nil. diff --git a/provider/test/emptycollection/provider.go b/provider/test/emptycollection/provider.go index 5512099f1..458a4e540 100644 --- a/provider/test/emptycollection/provider.go +++ b/provider/test/emptycollection/provider.go @@ -40,7 +40,7 @@ func (tp *TileProvider) Layers() ([]provider.LayerInfo, error) { } // TilFeatures always returns a feature with a polygon outlining the tile's Extent (not Buffered Extent) -func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, fn func(f *provider.Feature) error) error { +func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { // get tile bounding box _, srid := t.Extent() diff --git a/provider/test/provider.go b/provider/test/provider.go index b5288b45c..ca0942693 100644 --- a/provider/test/provider.go +++ b/provider/test/provider.go @@ -86,7 +86,7 @@ func (tp *TileProvider) Layers() ([]provider.LayerInfo, error) { } // TileFeatures always returns a feature with a polygon outlining the tile's Extent (not Buffered Extent) -func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, fn func(f *provider.Feature) error) error { +func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { // get tile bounding box ext, srid := t.Extent() @@ -103,7 +103,7 @@ func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provid } // MVTForLayers mocks out MVTForLayers by just returning the MVTTile bytes, this will never error -func (tp *TileProvider) MVTForLayers(ctx context.Context, _ provider.Tile, _ []provider.Layer) ([]byte, error) { +func (tp *TileProvider) MVTForLayers(ctx context.Context, _ provider.Tile, _ map[string]provider.QueryParameter, _ []provider.Layer) ([]byte, error) { // TODO(gdey): fill this out. if tp == nil { return nil, nil diff --git a/server/handle_map_layer_zxy.go b/server/handle_map_layer_zxy.go index 45ed3a49b..136696097 100644 --- a/server/handle_map_layer_zxy.go +++ b/server/handle_map_layer_zxy.go @@ -8,7 +8,9 @@ import ( "strconv" "strings" + "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/observability" + "github.com/go-spatial/tegola/provider" "github.com/dimfeld/httptreemux" "github.com/go-spatial/geom/encoding/mvt" @@ -95,13 +97,14 @@ func (req *HandleMapLayerZXY) parseURI(r *http.Request) error { return nil } -// URI scheme: /maps/:map_name/:layer_name/:z/:x/:y +// URI scheme: /maps/:map_name/:layer_name/:z/:x/:y?param=value // map_name - map name in the config file // layer_name - name of the single map layer to render // z, x, y - tile coordinates as described in the Slippy Map Tilenames specification // z - zoom level // x - row // y - column +// param - configurable query parameters and their values func (req HandleMapLayerZXY) ServeHTTP(w http.ResponseWriter, r *http.Request) { // parse our URI if err := req.parseURI(r); err != nil { @@ -158,36 +161,16 @@ func (req HandleMapLayerZXY) ServeHTTP(w http.ResponseWriter, r *http.Request) { m = m.AddDebugLayers() } - encodeCtx := context.WithValue(r.Context(), observability.ObserveVarMapName, m.Name) - var params map[string]string - // check for query parameters and populate param map with their values - if req.Atlas.HasParams(req.mapName) { - params = make(map[string]string) - err = r.ParseForm() - if err == nil { - for _, param := range req.Atlas.GetParams(req.mapName) { - // Fetch parameter values - val := r.Form.Get(param.Name) - // Empty values are injected here to replace tokens with - // an empty string during sql query parameter replacement - // otherwise, the default value is used if configured. - if val == "" && param.DefaultValue != "" { - val = param.DefaultValue - } - - // inject parameter value or default into params map - params[param.Token] = val - } - - // update context passed to encoding if any params were parsed - if len(params) > 0 { - encodeCtx = context.WithValue(encodeCtx, "params", params) - } - } + params, err := req.extractParameters(r) + if err != nil { + log.Error(err) + http.Error(w, err.Error(), http.StatusBadRequest) + return } - pbyte, err := m.Encode(encodeCtx, tile) + encodeCtx := context.WithValue(r.Context(), observability.ObserveVarMapName, m.Name) + pbyte, err := m.Encode(encodeCtx, tile, params) if err != nil { switch { @@ -222,3 +205,47 @@ func (req HandleMapLayerZXY) ServeHTTP(w http.ResponseWriter, r *http.Request) { log.Infof("tile z:%v, x:%v, y:%v is rather large - %vKb", req.z, req.x, req.y, len(pbyte)/1024) } } + +func (req *HandleMapLayerZXY) extractParameters(r *http.Request) (map[string]provider.QueryParameter, error) { + var params map[string]provider.QueryParameter + if req.Atlas.HasParams(req.mapName) { + params = make(map[string]provider.QueryParameter) + err := r.ParseForm() + if err != nil { + return nil, err + } + + for _, param := range req.Atlas.GetParams(req.mapName) { + if r.Form.Has(param.Name) { + val, err := config.ParamTypeDecoders[param.Type](r.Form.Get(param.Name)) + if err != nil { + return nil, err + } + params[param.Token] = provider.QueryParameter{ + Token: param.Type, + SQL: param.SQL, + Value: val, + } + } else if len(param.DefaultValue) > 0 { + val, err := config.ParamTypeDecoders[param.Type](param.DefaultValue) + if err != nil { + return nil, err + } + params[param.Token] = provider.QueryParameter{ + Token: param.Type, + SQL: "?", + Value: val, + } + } else if len(param.DefaultSQL) > 0 { + params[param.Token] = provider.QueryParameter{ + Token: param.Type, + SQL: param.DefaultSQL, + Value: nil, + } + } else { + return nil, fmt.Errorf("the required parameter %s is not specified", param.Name) + } + } + } + return params, nil +} From e86b27904cf5ee0b7c3e5c677be57a8b82195864 Mon Sep 17 00:00:00 2001 From: Sergei Gureev Date: Tue, 18 Oct 2022 14:48:12 +0300 Subject: [PATCH 4/8] refactor: move replaceParams fn to provider --- config/config.go | 6 +--- provider/postgis/postgis.go | 11 +++---- provider/postgis/util.go | 38 +++------------------- provider/postgis/util_internal_test.go | 2 +- provider/provider.go | 45 ++++++++++++++++++++++++-- 5 files changed, 54 insertions(+), 48 deletions(-) diff --git a/config/config.go b/config/config.go index 2bc2ac7ec..dd3e5a5ae 100644 --- a/config/config.go +++ b/config/config.go @@ -6,7 +6,6 @@ import ( "io" "net/http" "os" - "regexp" "strconv" "strings" "time" @@ -57,9 +56,6 @@ func IsReservedToken(token string) bool { return false } -// ParameterTokenRegexp to validate QueryParameters -var ParameterTokenRegexp = regexp.MustCompile("![a-zA-Z0-9_-]+!") - // ParamTypeDecoders is a collection of parsers for different types of user-defined parameters var ParamTypeDecoders = map[string]func(string) (interface{}, error){ "int": func(s string) (interface{}, error) { @@ -162,7 +158,7 @@ func (m Map) ValidateParams() error { } } - if !ParameterTokenRegexp.MatchString(param.Token) { + if !provider.ParameterTokenRegexp.MatchString(param.Token) { return ErrParamBadTokenName{ MapName: string(m.Name), Parameter: param, diff --git a/provider/postgis/postgis.go b/provider/postgis/postgis.go index 16741c862..2d5d0cde6 100644 --- a/provider/postgis/postgis.go +++ b/provider/postgis/postgis.go @@ -599,7 +599,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) } } else { if err = p.inspectLayerGeomType(&l); err != nil { - return nil, fmt.Errorf("error fetching geometry type for layer (%v): %w", l.name, err) + return nil, fmt.Errorf("error fetching geometry type for layer (%v): %w\nif custom parameters are used, remember to set %s for the provider", l.name, err, ConfigKeyGeomType) } } @@ -725,8 +725,9 @@ func (p Provider) inspectLayerGeomType(l *Layer) error { return err } - // TODO (bemyak): Figure out what is this // remove all parameter tokens for inspection + // crossing our fingers that the query is still valid 🤞 + // if not, the user will have to specify `geometry_type` in the config sql = stripParams(sql) rows, err := p.pool.Query(context.Background(), sql) @@ -820,7 +821,7 @@ func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider. // replace configured query parameters if any args := make([]interface{}, 0) - sql = replaceParams(queryParams, sql, &args) + sql = provider.ReplaceParams(queryParams, sql, &args) if err != nil { return err } @@ -933,7 +934,6 @@ func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider. return rows.Err() } -// TODO (bemyak): Make an actual use of QueryParams func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, params map[string]provider.QueryParameter, layers []provider.Layer) ([]byte, error) { var ( err error @@ -970,7 +970,7 @@ func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, params m } // replace configured query parameters if any - sql = replaceParams(params, sql, &args) + sql = provider.ReplaceParams(params, sql, &args) // ref: https://postgis.net/docs/ST_AsMVT.html // bytea ST_AsMVT(any_element row, text name, integer extent, text geom_name, text feature_id_name) @@ -1003,7 +1003,6 @@ func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, params m err = p.pool.QueryRow(ctx, fsql, args...).Scan(&data) if p.mvtProviderQueryHistogramSeconds != nil { z, _, _ := tile.ZXY() - // TODO (bemyak): add params as labels to prometheus? lbls := prometheus.Labels{ "z": strconv.FormatUint(uint64(z), 10), "map_name": mapName, diff --git a/provider/postgis/util.go b/provider/postgis/util.go index 56f45a5b4..1d68c6b07 100644 --- a/provider/postgis/util.go +++ b/provider/postgis/util.go @@ -173,45 +173,15 @@ func replaceTokens(sql string, lyr *Layer, tile provider.Tile, withBuffer bool) return tokenReplacer.Replace(uppercaseTokenSQL), nil } -// replaceParams substitutes configured query parameter tokens for their values -// within the provided SQL string -func replaceParams(params map[string]provider.QueryParameter, sql string, args *[]interface{}) string { - if params == nil { - return sql - } - - for _, token := range config.ParameterTokenRegexp.FindAllString(sql, -1) { - param := params[token] - - // Replace every ? in the param's SQL with a positional argument - paramSQL := "" - for _, c := range param.SQL { - if c == '?' { - *args = append(*args, param.Value) - paramSQL = paramSQL + "$" + fmt.Sprint(len(*args)) - } else { - paramSQL += string(c) - } - } - - // Finally, replace current token with the prepared SQL - sql = strings.Replace(sql, token, paramSQL, 1) - } - - return sql -} - // stripParams will remove all parameter tokens from the query func stripParams(sql string) string { - return config.ParameterTokenRegexp.ReplaceAllStringFunc(sql, func(s string) string { - return "" - }) + return provider.ParameterTokenRegexp.ReplaceAllString(sql, "") } -// uppercaseTokens converts all !tokens! to uppercase !TOKENS!. Tokens can -// contain alphanumerics, dash and underline chars. +// uppercaseTokens converts all !tokens! to uppercase !TOKENS!. Tokens can +// contain alphanumerics, dash and underline chars. func uppercaseTokens(str string) string { - return config.ParameterTokenRegexp.ReplaceAllStringFunc(str, strings.ToUpper) + return provider.ParameterTokenRegexp.ReplaceAllStringFunc(str, strings.ToUpper) } func transformVal(valType pgtype.OID, val interface{}) (interface{}, error) { diff --git a/provider/postgis/util_internal_test.go b/provider/postgis/util_internal_test.go index 5a43eb2bb..2cc5360ee 100644 --- a/provider/postgis/util_internal_test.go +++ b/provider/postgis/util_internal_test.go @@ -85,7 +85,7 @@ func TestReplaceParams(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { args := make([]interface{}, 0) - out := replaceParams(tc.params, tc.sql, &args) + out := provider.ReplaceParams(tc.params, tc.sql, &args) if out != tc.expectedSql { t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedSql, out) diff --git a/provider/provider.go b/provider/provider.go index 7a52efe97..20f5ff884 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -3,6 +3,8 @@ package provider import ( "context" "fmt" + "regexp" + "strings" "github.com/go-spatial/geom" "github.com/go-spatial/geom/slippy" @@ -104,14 +106,53 @@ type Tile interface { BufferedExtent() (extent *geom.Extent, srid uint64) } -// Query parameter holds normalized parameter data ready to be inserted in the final query +// ParameterTokenRegexp to validate QueryParameters +var ParameterTokenRegexp = regexp.MustCompile("![a-zA-Z0-9_-]+!") + +// Query parameter holds normalized parameter data ready to be inserted in the +// final query type QueryParameter struct { // Token to replace e.g., !TOKEN! Token string - // SQL expression to be inserted. Contains "?" that will be replaced with an ordinal argument e.g., "$1" + // SQL expression to be inserted. Contains "?" that will be replaced with an + // ordinal argument e.g., "$1" SQL string // Value that will be passed to the final query Value interface{} + // Raw parameter values for debugging and monitoring + RawValues map[string]string +} + +// ReplaceParams substitutes configured query parameter tokens for their values +// within the provided SQL string +func ReplaceParams(params map[string]QueryParameter, sql string, args *[]interface{}) string { + if params == nil { + return sql + } + + for _, token := range ParameterTokenRegexp.FindAllString(sql, -1) { + param := params[token] + + // Replace every ? in the param's SQL with a positional argument + paramSQL := "" + argFound := false + for _, c := range param.SQL { + if c == '?' { + if !argFound { + *args = append(*args, param.Value) + argFound = true + } + paramSQL += fmt.Sprintf("$%d", len(*args)) + } else { + paramSQL += string(c) + } + } + + // Finally, replace current token with the prepared SQL + sql = strings.Replace(sql, token, paramSQL, 1) + } + + return sql } // Tiler is a Layers that allows one to encode features in that layer From e3fedc5faeae7e773d567905ef8c66660c12227e Mon Sep 17 00:00:00 2001 From: Sergei Gureev Date: Thu, 9 Jun 2022 08:37:40 +0300 Subject: [PATCH 5/8] feat: check all sql tokens exist - Fixed test in case no env vars are defined - QueryParams inserted only once per token - Minor fixes --- cmd/tegola/cmd/root.go | 2 +- config/config.go | 76 ++++++++++------------- provider/postgis/postgis.go | 9 ++- provider/postgis/postgis_internal_test.go | 27 +++++--- provider/postgis/postgis_test.go | 4 +- provider/postgis/util_internal_test.go | 17 +++++ provider/provider.go | 2 +- server/handle_map_layer_zxy.go | 2 +- 8 files changed, 82 insertions(+), 57 deletions(-) diff --git a/cmd/tegola/cmd/root.go b/cmd/tegola/cmd/root.go index 5c3dbd8ea..3e41e8c34 100644 --- a/cmd/tegola/cmd/root.go +++ b/cmd/tegola/cmd/root.go @@ -77,7 +77,7 @@ var RootCmd = &cobra.Command{ Use: "tegola", Short: "tegola is a vector tile server", Long: fmt.Sprintf(`tegola is a vector tile server - Version: %v`, build.Version), +Version: %v`, build.Version), PersistentPreRunE: rootCmdValidatePersistent, } diff --git a/config/config.go b/config/config.go index dd3e5a5ae..09685c755 100644 --- a/config/config.go +++ b/config/config.go @@ -32,28 +32,18 @@ const ( ) // ReservedTokens for query injection -var ReservedTokens = []string{ - BboxToken, - ZoomToken, - XToken, - YToken, - ZToken, - ScaleDenominatorToken, - PixelWidthToken, - PixelHeightToken, - IdFieldToken, - GeomFieldToken, - GeomTypeToken, -} - -// IsReservedToken returns true if the specified token is reserved -func IsReservedToken(token string) bool { - for _, t := range ReservedTokens { - if token == t { - return true - } - } - return false +var ReservedTokens = map[string]struct{}{ + BboxToken: {}, + ZoomToken: {}, + XToken: {}, + YToken: {}, + ZToken: {}, + ScaleDenominatorToken: {}, + PixelWidthToken: {}, + PixelHeightToken: {}, + IdFieldToken: {}, + GeomFieldToken: {}, + GeomTypeToken: {}, } // ParamTypeDecoders is a collection of parsers for different types of user-defined parameters @@ -124,7 +114,8 @@ func (m Map) ValidateParams() error { return nil } - var usedNames, usedTokens []string + usedNames := make(map[string]struct{}) + usedTokens := make(map[string]struct{}) for _, param := range m.Parameters { if _, ok := ParamTypeDecoders[param.Type]; !ok { @@ -151,7 +142,7 @@ func (m Map) ValidateParams() error { } } - if IsReservedToken(param.Token) { + if _, ok := ReservedTokens[param.Token]; ok { return ErrParamTokenReserved{ MapName: string(m.Name), Parameter: param, @@ -165,26 +156,27 @@ func (m Map) ValidateParams() error { } } - for _, name := range usedNames { - if name == param.Name { - return ErrParamNameDuplicate{ - MapName: string(m.Name), - Parameter: param, - } + if _, ok := usedNames[param.Name]; ok { + return ErrParamNameDuplicate{ + MapName: string(m.Name), + Parameter: param, } } - for _, token := range usedTokens { - if token == param.Token { - return ErrParamTokenDuplicate{ - MapName: string(m.Name), - Parameter: param, - } + if _, ok := usedTokens[param.Token]; ok { + return ErrParamTokenDuplicate{ + MapName: string(m.Name), + Parameter: param, } } - usedNames = append(usedNames, param.Name) - usedTokens = append(usedTokens, param.Token) + usedNames[param.Name] = struct{}{} + usedTokens[param.Token] = struct{}{} + } + + // Mark all used tokens as reserved + for token := range usedTokens { + ReservedTokens[token] = struct{}{} } return nil @@ -244,15 +236,13 @@ type QueryParameter struct { DefaultValue string `toml:"default_value"` } -// Normalize will normalize param and set the default values +// Normalize normalizes param and sets default values func (param *QueryParameter) Normalize() { param.Token = strings.ToUpper(param.Token) - sql := "?" - if len(param.SQL) > 0 { - sql = param.SQL + if len(param.SQL) == 0 { + param.SQL = "?" } - param.SQL = sql } // Validate checks the config for issues diff --git a/provider/postgis/postgis.go b/provider/postgis/postgis.go index 2d5d0cde6..23a907099 100644 --- a/provider/postgis/postgis.go +++ b/provider/postgis/postgis.go @@ -563,7 +563,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) if sql != "" { // convert !BOX! (MapServer) and !bbox! (Mapnik) to !BBOX! for compatibility - sql := strings.Replace(strings.Replace(sql, "!BOX!", "!BBOX!", -1), "!bbox!", "!BBOX!", -1) + sql := strings.Replace(strings.Replace(sql, "!BOX!", conf.BboxToken, -1), "!bbox!", conf.BboxToken, -1) // make sure that the sql has a !BBOX! token if !strings.Contains(sql, conf.BboxToken) { return nil, fmt.Errorf("SQL for layer (%v) %v is missing required token: %v", i, lName, conf.BboxToken) @@ -577,6 +577,13 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) } } + // check all tokens are valid + for _, token := range provider.ParameterTokenRegexp.FindAllString(sql, -1) { + if _, ok := conf.ReservedTokens[token]; !ok { + return nil, fmt.Errorf("SQL for layer (%v) %v references an unknown token %s: %v", i, lName, token, sql) + } + } + l.sql = sql } else { // Tablename and Fields will be used to build the query. diff --git a/provider/postgis/postgis_internal_test.go b/provider/postgis/postgis_internal_test.go index ae5f012ee..bd1346bd0 100644 --- a/provider/postgis/postgis_internal_test.go +++ b/provider/postgis/postgis_internal_test.go @@ -17,7 +17,19 @@ import ( // TESTENV is the environment variable that must be set to "yes" to run postgis tests. const TESTENV = "RUN_POSTGIS_TESTS" -var defaultEnvConfig map[string]interface{} +var DefaultEnvConfig map[string]interface{} + +var DefaultConfig map[string]interface{} = map[string]interface{}{ + ConfigKeyHost: "localhost", + ConfigKeyPort: 5432, + ConfigKeyDB: "tegola", + ConfigKeyUser: "postgres", + ConfigKeyPassword: "postgres", + ConfigKeySSLMode: "disable", + ConfigKeySSLKey: "", + ConfigKeySSLCert: "", + ConfigKeySSLRootCert: "", +} func getConfigFromEnv() map[string]interface{} { port, err := strconv.Atoi(ttools.GetEnvDefault("PGPORT", "5432")) @@ -40,7 +52,7 @@ func getConfigFromEnv() map[string]interface{} { } func init() { - defaultEnvConfig = getConfigFromEnv() + DefaultEnvConfig = getConfigFromEnv() } type TCConfig struct { @@ -49,9 +61,8 @@ type TCConfig struct { LayerConfig []map[string]interface{} } -func (cfg TCConfig) Config() dict.Dict { +func (cfg TCConfig) Config(mConfig map[string]interface{}) dict.Dict { var config map[string]interface{} - mConfig := defaultEnvConfig if cfg.BaseConfig != nil { mConfig = cfg.BaseConfig } @@ -86,7 +97,7 @@ func TestMVTProviders(t *testing.T) { } fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { - config := tc.Config() + config := tc.Config(DefaultEnvConfig) prvd, err := NewMVTTileProvider(config) // for now we will just check the length of the bytes. if tc.err != "" { @@ -133,7 +144,7 @@ func TestMVTProviders(t *testing.T) { }, }, layerNames: []string{"land"}, - mvtTile: make([]byte, 208993), + mvtTile: make([]byte, 174689), tile: provider.NewTile(0, 0, 0, 16, 4326), }, } @@ -156,7 +167,7 @@ func TestLayerGeomType(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { - config := tc.Config() + config := tc.Config(DefaultEnvConfig) provider, err := NewTileProvider(config) if tc.err != "" { if err == nil || !strings.Contains(err.Error(), tc.err) { @@ -443,7 +454,7 @@ func TestBuildUri(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { - config := tc.Config() + config := tc.Config(DefaultConfig) uri, _, err := BuildURI(config) if tc.err != "" { diff --git a/provider/postgis/postgis_test.go b/provider/postgis/postgis_test.go index dc46241a2..3907e1d90 100644 --- a/provider/postgis/postgis_test.go +++ b/provider/postgis/postgis_test.go @@ -164,7 +164,7 @@ func TestNewTileProvider(t *testing.T) { fn := func(tc postgis.TCConfig) func(t *testing.T) { return func(t *testing.T) { - config := tc.Config() + config := tc.Config(postgis.DefaultEnvConfig) _, err := postgis.NewTileProvider(config) if err != nil { t.Errorf("unable to create a new provider. err: %v", err) @@ -202,7 +202,7 @@ func TestTileFeatures(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { - config := tc.Config() + config := tc.Config(postgis.DefaultEnvConfig) p, err := postgis.NewTileProvider(config) if err != nil { t.Errorf("unexpected error; unable to create a new provider, expected: nil Got %v", err) diff --git a/provider/postgis/util_internal_test.go b/provider/postgis/util_internal_test.go index 2cc5360ee..a662a047b 100644 --- a/provider/postgis/util_internal_test.go +++ b/provider/postgis/util_internal_test.go @@ -172,6 +172,23 @@ func TestReplaceParams(t *testing.T) { expectedSql: "SELECT * FROM table ", expectedArgs: []interface{}{}, }, + "multiple params": { + params: map[string]provider.QueryParameter{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "???", + Value: 1, + }, + "!PARAM2!": { + Token: "!PARAM2!", + SQL: "???", + Value: 2, + }, + }, + sql: "!PARAM!!PARAM2!", + expectedSql: "$1$1$1$2$2$2", + expectedArgs: []interface{}{1, 2}, + }, } for name, tc := range tests { diff --git a/provider/provider.go b/provider/provider.go index 20f5ff884..6fc2fe2c2 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -117,7 +117,7 @@ type QueryParameter struct { // SQL expression to be inserted. Contains "?" that will be replaced with an // ordinal argument e.g., "$1" SQL string - // Value that will be passed to the final query + // Value that will be passed to the final query in arguments list Value interface{} // Raw parameter values for debugging and monitoring RawValues map[string]string diff --git a/server/handle_map_layer_zxy.go b/server/handle_map_layer_zxy.go index 136696097..9cfbfa65e 100644 --- a/server/handle_map_layer_zxy.go +++ b/server/handle_map_layer_zxy.go @@ -233,7 +233,7 @@ func (req *HandleMapLayerZXY) extractParameters(r *http.Request) (map[string]pro } params[param.Token] = provider.QueryParameter{ Token: param.Type, - SQL: "?", + SQL: param.SQL, Value: val, } } else if len(param.DefaultSQL) > 0 { From 8329ac4353a2ac664ad7cb2df2799179bed64aac Mon Sep 17 00:00:00 2001 From: Sergei Gureev Date: Tue, 18 Oct 2022 14:07:26 +0300 Subject: [PATCH 6/8] feat: disable caching for maps with params Warn about disabled caching --- atlas/atlas.go | 36 ----------------------------------- atlas/map.go | 3 +++ cmd/internal/register/maps.go | 5 +---- cmd/tegola/cmd/cache/cache.go | 15 ++++++++++++--- 4 files changed, 16 insertions(+), 43 deletions(-) diff --git a/atlas/atlas.go b/atlas/atlas.go index 2130e4358..06677c651 100644 --- a/atlas/atlas.go +++ b/atlas/atlas.go @@ -11,7 +11,6 @@ import ( "github.com/go-spatial/geom/slippy" "github.com/go-spatial/tegola" "github.com/go-spatial/tegola/cache" - "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/internal/log" "github.com/go-spatial/tegola/internal/observer" "github.com/go-spatial/tegola/observability" @@ -79,10 +78,6 @@ type Atlas struct { // holds a reference to the observer backend observer observability.Interface - // holds a reference to configured query parameters for maps - // key = map name - params map[string][]config.QueryParameter - // publishBuildInfo indicates if we should publish the build info on change of observer // this is set by calling PublishBuildInfo, which will publish // the build info on the observer and insure changes to observer @@ -135,7 +130,6 @@ func (a *Atlas) SeedMapTile(ctx context.Context, m Map, z, x, y uint) error { tile := slippy.NewTile(z, x, y) // encode the tile - // TODO (bemyak): Make query parameters work with cache b, err := m.Encode(ctx, tile, nil) if err != nil { return err @@ -219,36 +213,6 @@ func (a *Atlas) AddMap(m Map) { a.maps[m.Name] = m } -// AddParams adds the given query parameters to the atlas params map -// keyed by the map name, with upper-cased tokens -func (a *Atlas) AddParams(name string, params []config.QueryParameter) { - if a == nil { - defaultAtlas.AddParams(name, params) - return - } - if a.params == nil { - a.params = make(map[string][]config.QueryParameter) - } - a.params[name] = params -} - -// GetParams returns any configured query parameters for the given -// map by name -func (a *Atlas) GetParams(name string) []config.QueryParameter { - if a == nil { - return defaultAtlas.GetParams(name) - } - return a.params[name] -} - -// HasParams returns true if the given map by name has configured query parameters -func (a *Atlas) HasParams(name string) bool { - if a == nil { - return defaultAtlas.HasParams(name) - } - return len(a.params[name]) > 0 -} - // GetCache returns the registered cache if one is registered, otherwise nil func (a *Atlas) GetCache() cache.Interface { if a == nil { diff --git a/atlas/map.go b/atlas/map.go index 279f36500..5f1385c8b 100644 --- a/atlas/map.go +++ b/atlas/map.go @@ -9,6 +9,7 @@ import ( "strings" "sync" + "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/observability" "github.com/golang/protobuf/proto" @@ -54,6 +55,8 @@ type Map struct { // WGS:84 values), the third value is the zoom level. Center [3]float64 Layers []Layer + // Params holds configured query parameters + Params []config.QueryParameter SRID uint64 // MVT output values diff --git a/cmd/internal/register/maps.go b/cmd/internal/register/maps.go index 1289c340c..69dec943e 100644 --- a/cmd/internal/register/maps.go +++ b/cmd/internal/register/maps.go @@ -14,6 +14,7 @@ import ( func webMercatorMapFromConfigMap(cfg config.Map) (newMap atlas.Map) { newMap = atlas.NewWebMercatorMap(string(cfg.Name)) newMap.Attribution = SanitizeAttribution(string(cfg.Attribution)) + newMap.Params = cfg.Parameters // convert from env package for i, v := range cfg.Center { @@ -156,10 +157,6 @@ func Maps(a *atlas.Atlas, maps []config.Map, providers map[string]provider.Tiler newMap.Layers = append(newMap.Layers, layer) } - if len(m.Parameters) > 0 { - a.AddParams(string(m.Name), m.Parameters) - } - a.AddMap(newMap) } return nil diff --git a/cmd/tegola/cmd/cache/cache.go b/cmd/tegola/cmd/cache/cache.go index be2c00ca6..8c2d998c9 100644 --- a/cmd/tegola/cmd/cache/cache.go +++ b/cmd/tegola/cmd/cache/cache.go @@ -159,11 +159,20 @@ func doWork(ctx context.Context, tileChannel *TileChannel, maps []atlas.Map, con }(i) } + nonParamMaps := make([]atlas.Map, 0) + + for _, m := range maps { + // we don't support caching for maps with custom parameters + if m.Params != nil || len(m.Params) > 0 { + log.Warnf("caching is disabled for map %s as it has custom parameters configures", m.Name) + } + nonParamMaps = append(nonParamMaps, m) + } + // run through the incoming tiles, and generate the mapTiles as needed. TileChannelLoop: for tile := range tileChannel.Channel() { - for m := range maps { - + for _, m := range nonParamMaps { if ctx.Err() != nil { cleanup = true break @@ -180,7 +189,7 @@ TileChannelLoop: } mapTile := MapTile{ - MapName: maps[m].Name, + MapName: m.Name, Tile: tile, } From 780b651e23fd0485f459b6d075d7aba65cec236b Mon Sep 17 00:00:00 2001 From: Sergei Gureev Date: Tue, 18 Oct 2022 15:00:28 +0300 Subject: [PATCH 7/8] refactor: move Map and related types to provider pkg This is needed to access configured map parameters from the provider's initialization code to inspect geom type. Unfortunately, config pkg depends on the provider pkg, so simply referencing these types creates a dependency loop. --- atlas/map.go | 11 +- cmd/internal/register/maps.go | 6 +- cmd/internal/register/maps_test.go | 24 ++-- cmd/internal/register/providers.go | 4 +- cmd/internal/register/providers_test.go | 2 +- cmd/tegola/cmd/root.go | 2 +- cmd/tegola_lambda/main.go | 2 +- config/config.go | 131 +++-------------- config/config_test.go | 143 ++++++++++--------- config/errors.go | 48 +++---- provider/debug/debug.go | 12 +- provider/gpkg/cgo_test.go | 3 +- provider/gpkg/gpkg.go | 4 +- provider/gpkg/gpkg_register.go | 5 +- provider/gpkg/gpkg_register_internal_test.go | 3 +- provider/gpkg/gpkg_test.go | 12 +- provider/map.go | 14 ++ provider/map_layer.go | 51 +++++++ provider/mvt_provider.go | 4 +- provider/paramater_decoders.go | 19 +++ provider/postgis/postgis.go | 79 +++++----- provider/postgis/postgis_internal_test.go | 16 ++- provider/postgis/postgis_test.go | 6 +- provider/postgis/register.go | 8 +- provider/postgis/util.go | 23 ++- provider/postgis/util_internal_test.go | 122 ---------------- provider/provider.go | 57 +------- provider/provider_test.go | 4 +- provider/query_parameter.go | 65 +++++++++ provider/query_parameter_value.go | 76 ++++++++++ provider/query_parameter_value_test.go | 137 ++++++++++++++++++ provider/test/emptycollection/provider.go | 4 +- provider/test/provider.go | 8 +- server/handle_map_layer_zxy.go | 47 +++--- tile.go | 4 +- 35 files changed, 636 insertions(+), 520 deletions(-) create mode 100644 provider/map.go create mode 100644 provider/map_layer.go create mode 100644 provider/paramater_decoders.go create mode 100644 provider/query_parameter.go create mode 100644 provider/query_parameter_value.go create mode 100644 provider/query_parameter_value_test.go diff --git a/atlas/map.go b/atlas/map.go index 5f1385c8b..aab8e79de 100644 --- a/atlas/map.go +++ b/atlas/map.go @@ -9,7 +9,6 @@ import ( "strings" "sync" - "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/observability" "github.com/golang/protobuf/proto" @@ -56,7 +55,7 @@ type Map struct { Center [3]float64 Layers []Layer // Params holds configured query parameters - Params []config.QueryParameter + Params []provider.QueryParameter SRID uint64 // MVT output values @@ -120,7 +119,7 @@ func (m Map) AddDebugLayers() Map { m.Layers = layers // setup a debug provider - debugProvider, _ := debug.NewTileProvider(dict.Dict{}) + debugProvider, _ := debug.NewTileProvider(dict.Dict{}, nil) m.Layers = append(layers, []Layer{ { @@ -183,7 +182,7 @@ func (m Map) FilterLayersByName(names ...string) Map { return m } -func (m Map) encodeMVTProviderTile(ctx context.Context, tile *slippy.Tile, params map[string]provider.QueryParameter) ([]byte, error) { +func (m Map) encodeMVTProviderTile(ctx context.Context, tile *slippy.Tile, params provider.Params) ([]byte, error) { // get the list of our layers ptile := provider.NewTile(tile.Z, tile.X, tile.Y, uint(m.TileBuffer), uint(m.SRID)) @@ -200,7 +199,7 @@ func (m Map) encodeMVTProviderTile(ctx context.Context, tile *slippy.Tile, param // encodeMVTTile will encode the given tile into mvt format // TODO (arolek): support for max zoom -func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile, params map[string]provider.QueryParameter) ([]byte, error) { +func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile, params provider.Params) ([]byte, error) { // tile container var mvtTile mvt.Tile @@ -380,7 +379,7 @@ func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile, params map[st } // Encode will encode the given tile into mvt format -func (m Map) Encode(ctx context.Context, tile *slippy.Tile, params map[string]provider.QueryParameter) ([]byte, error) { +func (m Map) Encode(ctx context.Context, tile *slippy.Tile, params provider.Params) ([]byte, error) { var ( tileBytes []byte err error diff --git a/cmd/internal/register/maps.go b/cmd/internal/register/maps.go index 69dec943e..bb4e62c9f 100644 --- a/cmd/internal/register/maps.go +++ b/cmd/internal/register/maps.go @@ -11,7 +11,7 @@ import ( "github.com/go-spatial/tegola/provider" ) -func webMercatorMapFromConfigMap(cfg config.Map) (newMap atlas.Map) { +func webMercatorMapFromConfigMap(cfg provider.Map) (newMap atlas.Map) { newMap = atlas.NewWebMercatorMap(string(cfg.Name)) newMap.Attribution = SanitizeAttribution(string(cfg.Attribution)) newMap.Params = cfg.Parameters @@ -47,7 +47,7 @@ func layerInfosFindByName(infos []provider.LayerInfo, name string) provider.Laye return nil } -func atlasLayerFromConfigLayer(cfg *config.MapLayer, mapName string, layerProvider provider.Layerer) (layer atlas.Layer, err error) { +func atlasLayerFromConfigLayer(cfg *provider.MapLayer, mapName string, layerProvider provider.Layerer) (layer atlas.Layer, err error) { var ( // providerLayer is primary used for error reporting. providerLayer = string(cfg.ProviderLayer) @@ -124,7 +124,7 @@ func selectProvider(name string, mapName string, newMap *atlas.Map, providers ma } // Maps registers maps with with atlas -func Maps(a *atlas.Atlas, maps []config.Map, providers map[string]provider.TilerUnion) error { +func Maps(a *atlas.Atlas, maps []provider.Map, providers map[string]provider.TilerUnion) error { var ( layerer provider.Layerer diff --git a/cmd/internal/register/maps_test.go b/cmd/internal/register/maps_test.go index abe05dcc1..5573bb8c4 100644 --- a/cmd/internal/register/maps_test.go +++ b/cmd/internal/register/maps_test.go @@ -6,15 +6,15 @@ import ( "github.com/go-spatial/tegola/atlas" "github.com/go-spatial/tegola/cmd/internal/register" - "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/dict" "github.com/go-spatial/tegola/internal/env" + "github.com/go-spatial/tegola/provider" ) func TestMaps(t *testing.T) { type tcase struct { atlas atlas.Atlas - maps []config.Map + maps []provider.Map providers []dict.Dict expectedErr error } @@ -29,7 +29,7 @@ func TestMaps(t *testing.T) { provArr[i] = tc.providers[i] } - providers, err := register.Providers(provArr) + providers, err := register.Providers(provArr, tc.maps) if err != nil { t.Errorf("unexpected err: %v", err) return @@ -45,10 +45,10 @@ func TestMaps(t *testing.T) { tests := map[string]tcase{ "provider layer invalid": { - maps: []config.Map{ + maps: []provider.Map{ { Name: "foo", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "bar", }, @@ -67,10 +67,10 @@ func TestMaps(t *testing.T) { }, }, "provider not found": { - maps: []config.Map{ + maps: []provider.Map{ { Name: "foo", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "bar.baz", }, @@ -82,10 +82,10 @@ func TestMaps(t *testing.T) { }, }, "provider layer not registered with provider": { - maps: []config.Map{ + maps: []provider.Map{ { Name: "foo", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "test.bar", }, @@ -105,10 +105,10 @@ func TestMaps(t *testing.T) { }, }, "default tags": { - maps: []config.Map{ + maps: []provider.Map{ { Name: "foo", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "test.debug-tile-outline", DefaultTags: env.Dict{ @@ -126,7 +126,7 @@ func TestMaps(t *testing.T) { }, }, "success": { - maps: []config.Map{}, + maps: []provider.Map{}, providers: []dict.Dict{ { "name": "test", diff --git a/cmd/internal/register/providers.go b/cmd/internal/register/providers.go index 1eb16faa9..4553b9fc1 100644 --- a/cmd/internal/register/providers.go +++ b/cmd/internal/register/providers.go @@ -33,7 +33,7 @@ func (e ErrProviderTypeInvalid) Error() string { } // Providers registers data provider backends -func Providers(providers []dict.Dicter) (map[string]provider.TilerUnion, error) { +func Providers(providers []dict.Dicter, maps []provider.Map) (map[string]provider.TilerUnion, error) { // holder for registered providers registeredProviders := map[string]provider.TilerUnion{} @@ -72,7 +72,7 @@ func Providers(providers []dict.Dicter) (map[string]provider.TilerUnion, error) } // register the provider - prov, err := provider.For(ptype, p) + prov, err := provider.For(ptype, p, maps) if err != nil { return registeredProviders, err } diff --git a/cmd/internal/register/providers_test.go b/cmd/internal/register/providers_test.go index c1f62a45b..9f8d44410 100644 --- a/cmd/internal/register/providers_test.go +++ b/cmd/internal/register/providers_test.go @@ -23,7 +23,7 @@ func TestProviders(t *testing.T) { provArr[i] = tc.config[i] } - _, err = register.Providers(provArr) + _, err = register.Providers(provArr, nil) if tc.expectedErr != nil { if err.Error() != tc.expectedErr.Error() { t.Errorf("invalid error. expected: %v, got %v", tc.expectedErr, err.Error()) diff --git a/cmd/tegola/cmd/root.go b/cmd/tegola/cmd/root.go index 3e41e8c34..177638ae2 100644 --- a/cmd/tegola/cmd/root.go +++ b/cmd/tegola/cmd/root.go @@ -121,7 +121,7 @@ func initConfig(configFile string, cacheRequired bool, logLevel string, logger s provArr[i] = conf.Providers[i] } - providers, err := register.Providers(provArr) + providers, err := register.Providers(provArr, conf.Maps) if err != nil { return fmt.Errorf("could not register providers: %v", err) } diff --git a/cmd/tegola_lambda/main.go b/cmd/tegola_lambda/main.go index 4d1e19f4a..cf96b11ab 100644 --- a/cmd/tegola_lambda/main.go +++ b/cmd/tegola_lambda/main.go @@ -66,7 +66,7 @@ func init() { } // register the providers - providers, err := register.Providers(provArr) + providers, err := register.Providers(provArr, nil) if err != nil { log.Fatal(err) } diff --git a/config/config.go b/config/config.go index 09685c755..e4dd0ca48 100644 --- a/config/config.go +++ b/config/config.go @@ -6,7 +6,6 @@ import ( "io" "net/http" "os" - "strconv" "strings" "time" @@ -46,22 +45,6 @@ var ReservedTokens = map[string]struct{}{ GeomTypeToken: {}, } -// ParamTypeDecoders is a collection of parsers for different types of user-defined parameters -var ParamTypeDecoders = map[string]func(string) (interface{}, error){ - "int": func(s string) (interface{}, error) { - return strconv.Atoi(s) - }, - "float": func(s string) (interface{}, error) { - return strconv.ParseFloat(s, 32) - }, - "string": func(s string) (interface{}, error) { - return s, nil - }, - "bool": func(s string) (interface{}, error) { - return strconv.ParseBool(s) - }, -} - var blacklistHeaders = []string{"content-encoding", "content-length", "content-type"} // Config represents a tegola config file. @@ -82,8 +65,8 @@ type Config struct { // 2. type -- this is the name the provider modules register // themselves under. (e.g. postgis, gpkg, mvt_postgis ) // Note: Use the type to figure out if the provider is a mvt or std provider - Providers []env.Dict `toml:"providers"` - Maps []Map `toml:"maps"` + Providers []env.Dict `toml:"providers"` + Maps []provider.Map `toml:"maps"` } // Webserver represents the config options for the webserver part of Tegola @@ -96,47 +79,36 @@ type Webserver struct { SSLKey env.String `toml:"ssl_key"` } -// A Map represents a map in the Tegola Config file. -type Map struct { - Name env.String `toml:"name"` - Attribution env.String `toml:"attribution"` - Bounds []env.Float `toml:"bounds"` - Center [3]env.Float `toml:"center"` - Layers []MapLayer `toml:"layers"` - Parameters []QueryParameter `toml:"params"` - TileBuffer *env.Int `toml:"tile_buffer"` -} - -// ValidateParams ensures configured params don't conflict with existing +// ValidateAndRegisterParams ensures configured params don't conflict with existing // query tokens or have overlapping names -func (m Map) ValidateParams() error { - if len(m.Parameters) == 0 { +func ValidateAndRegisterParams(mapName string, params []provider.QueryParameter) error { + if len(params) == 0 { return nil } usedNames := make(map[string]struct{}) usedTokens := make(map[string]struct{}) - for _, param := range m.Parameters { - if _, ok := ParamTypeDecoders[param.Type]; !ok { + for _, param := range params { + if _, ok := provider.ParamTypeDecoders[param.Type]; !ok { return ErrParamUnknownType{ - MapName: string(m.Name), + MapName: string(mapName), Parameter: param, } } if len(param.DefaultSQL) > 0 && len(param.DefaultValue) > 0 { return ErrParamTwoDefaults{ - MapName: string(m.Name), + MapName: string(mapName), Parameter: param, } } if len(param.DefaultValue) > 0 { - decoderFn := ParamTypeDecoders[param.Type] + decoderFn := provider.ParamTypeDecoders[param.Type] if _, err := decoderFn(param.DefaultValue); err != nil { return ErrParamInvalidDefault{ - MapName: string(m.Name), + MapName: string(mapName), Parameter: param, } } @@ -144,28 +116,28 @@ func (m Map) ValidateParams() error { if _, ok := ReservedTokens[param.Token]; ok { return ErrParamTokenReserved{ - MapName: string(m.Name), + MapName: string(mapName), Parameter: param, } } if !provider.ParameterTokenRegexp.MatchString(param.Token) { return ErrParamBadTokenName{ - MapName: string(m.Name), + MapName: string(mapName), Parameter: param, } } if _, ok := usedNames[param.Name]; ok { - return ErrParamNameDuplicate{ - MapName: string(m.Name), + return ErrParamDuplicateName{ + MapName: string(mapName), Parameter: param, } } if _, ok := usedTokens[param.Token]; ok { - return ErrParamTokenDuplicate{ - MapName: string(m.Name), + return ErrParamDuplicateToken{ + MapName: string(mapName), Parameter: param, } } @@ -182,69 +154,6 @@ func (m Map) ValidateParams() error { return nil } -// MapLayer represents a the config for a layer in a map -type MapLayer struct { - // Name is optional. If it's not defined the name of the ProviderLayer will be used. - // Name can also be used to group multiple ProviderLayers under the same namespace. - Name env.String `toml:"name"` - ProviderLayer env.String `toml:"provider_layer"` - MinZoom *env.Uint `toml:"min_zoom"` - MaxZoom *env.Uint `toml:"max_zoom"` - DefaultTags env.Dict `toml:"default_tags"` - // DontSimplify indicates whether feature simplification should be applied. - // We use a negative in the name so the default is to simplify - DontSimplify env.Bool `toml:"dont_simplify"` - // DontClip indicates whether feature clipping should be applied. - // We use a negative in the name so the default is to clipping - DontClip env.Bool `toml:"dont_clip"` - // DontClip indicates whether feature cleaning (e.g. make valid) should be applied. - // We use a negative in the name so the default is to clean - DontClean env.Bool `toml:"dont_clean"` -} - -// ProviderLayerName returns the names of the layer and provider or an error -func (ml MapLayer) ProviderLayerName() (provider, layer string, err error) { - // split the provider layer (syntax is provider.layer) - plParts := strings.Split(string(ml.ProviderLayer), ".") - if len(plParts) != 2 { - return "", "", ErrInvalidProviderLayerName{ProviderLayerName: string(ml.ProviderLayer)} - } - return plParts[0], plParts[1], nil -} - -// GetName will return the user-defined Layer name from the config, -// or if the name is empty, return the name of the layer associated with -// the provider -func (ml MapLayer) GetName() (string, error) { - if ml.Name != "" { - return string(ml.Name), nil - } - _, name, err := ml.ProviderLayerName() - return name, err -} - -// QueryParameter represents an HTTP query parameter specified for use with -// a given map instance. -type QueryParameter struct { - Name string `toml:"name"` - Token string `toml:"token"` - Type string `toml:"type"` - SQL string `toml:"sql"` - // DefaultSQL replaces SQL if param wasn't passed. Either default_sql or - // default_value can be specified - DefaultSQL string `toml:"default_sql"` - DefaultValue string `toml:"default_value"` -} - -// Normalize normalizes param and sets default values -func (param *QueryParameter) Normalize() { - param.Token = strings.ToUpper(param.Token) - - if len(param.SQL) == 0 { - param.SQL = "?" - } -} - // Validate checks the config for issues func (c *Config) Validate() error { @@ -286,16 +195,16 @@ func (c *Config) Validate() error { } // check for map layer name / zoom collisions // map of layers to providers - mapLayers := map[string]map[string]MapLayer{} + mapLayers := map[string]map[string]provider.MapLayer{} for mapKey, m := range c.Maps { // validate any declared query parameters - if err := m.ValidateParams(); err != nil { + if err := ValidateAndRegisterParams(string(m.Name), m.Parameters); err != nil { return err } if _, ok := mapLayers[string(m.Name)]; !ok { - mapLayers[string(m.Name)] = map[string]MapLayer{} + mapLayers[string(m.Name)] = map[string]provider.MapLayer{} } // Set current provider to empty, for MVT providers diff --git a/config/config_test.go b/config/config_test.go index 8ae0f59e9..5bf34ced6 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -9,6 +9,7 @@ import ( "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/internal/env" + "github.com/go-spatial/tegola/provider" _ "github.com/go-spatial/tegola/provider/debug" _ "github.com/go-spatial/tegola/provider/postgis" _ "github.com/go-spatial/tegola/provider/test" @@ -167,14 +168,14 @@ func TestParse(t *testing.T) { name = "param2" token = "!PARAM2!" type = "int" - sql = "AND ANSWER = ?" + sql = "AND answer = ?" default_value = "42" [[maps.params]] name = "param3" token = "!PARAM3!" type = "float" - default_sql = "AND PI = 3.1415926" + default_sql = "AND pi = 3.1415926" `, expected: config.Config{ TileBuffer: env.IntPtr(env.Int(12)), @@ -210,14 +211,14 @@ func TestParse(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, TileBuffer: env.IntPtr(env.Int(12)), - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(10), @@ -227,7 +228,7 @@ func TestParse(t *testing.T) { DontClean: true, }, }, - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param1", Token: "!PARAM1!", @@ -238,7 +239,7 @@ func TestParse(t *testing.T) { Name: "param2", Token: "!PARAM2!", Type: "int", - SQL: "AND ANSWER = ?", + SQL: "AND answer = ?", DefaultValue: "42", }, { @@ -246,7 +247,7 @@ func TestParse(t *testing.T) { Token: "!PARAM3!", Type: "float", SQL: "?", - DefaultSQL: "AND PI = 3.1415926", + DefaultSQL: "AND pi = 3.1415926", }, }, }, @@ -360,14 +361,14 @@ func TestParse(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{ENV_TEST_CENTER_X, ENV_TEST_CENTER_Y, ENV_TEST_CENTER_Z}, TileBuffer: env.IntPtr(env.Int(64)), - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { Name: "water", ProviderLayer: ENV_TEST_PROVIDER_LAYER, @@ -388,7 +389,7 @@ func TestParse(t *testing.T) { Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, TileBuffer: env.IntPtr(env.Int(64)), - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { Name: "water", ProviderLayer: "provider1.water_0_5", @@ -542,13 +543,13 @@ func TestValidateMutateZoom(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: nil, @@ -586,13 +587,13 @@ func TestValidateMutateZoom(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(0), @@ -672,13 +673,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(10), @@ -736,13 +737,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { Name: "water", ProviderLayer: "provider1.water_0_5", @@ -806,13 +807,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(10), @@ -830,7 +831,7 @@ func TestValidate(t *testing.T) { Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(10), @@ -889,13 +890,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", }, @@ -906,7 +907,7 @@ func TestValidate(t *testing.T) { Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider2.water", }, @@ -958,13 +959,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1076,11 +1077,11 @@ func TestValidate(t *testing.T) { "type": "mvt_test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "happy", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1097,11 +1098,11 @@ func TestValidate(t *testing.T) { "type": "mvt_test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "happy", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1125,11 +1126,11 @@ func TestValidate(t *testing.T) { "type": "test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "happy", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1153,11 +1154,11 @@ func TestValidate(t *testing.T) { "type": "mvt_test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "happy", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "bad.water_default_z", }, @@ -1182,11 +1183,11 @@ func TestValidate(t *testing.T) { "type": "test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "comingle", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1214,11 +1215,11 @@ func TestValidate(t *testing.T) { "type": "mvt_test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "comingle", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "stdprovider1.water_default_z", }, @@ -1230,12 +1231,12 @@ func TestValidate(t *testing.T) { }, }, }, - "13 reserved parameter token": { + "13 reserved token name": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "bad_param", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!BBOX!", @@ -1247,7 +1248,7 @@ func TestValidate(t *testing.T) { }, expectedErr: config.ErrParamTokenReserved{ MapName: "bad_param", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param", Token: "!BBOX!", Type: "int", @@ -1256,10 +1257,10 @@ func TestValidate(t *testing.T) { }, "13 duplicate parameter name": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "dupe_param_name", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!PARAM!", @@ -1274,21 +1275,21 @@ func TestValidate(t *testing.T) { }, }, }, - expectedErr: config.ErrParamNameDuplicate{ + expectedErr: config.ErrParamDuplicateName{ MapName: "dupe_param_name", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param", Token: "!PARAM2!", Type: "int", }, }, }, - "13 duplicate parameter token": { + "13 duplicate token name": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "dupe_param_token", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!PARAM!", @@ -1303,9 +1304,9 @@ func TestValidate(t *testing.T) { }, }, }, - expectedErr: config.ErrParamTokenDuplicate{ + expectedErr: config.ErrParamDuplicateToken{ MapName: "dupe_param_token", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param2", Token: "!PARAM!", Type: "int", @@ -1314,10 +1315,10 @@ func TestValidate(t *testing.T) { }, "13 parameter unknown type": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "unknown_param_type", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!BBOX!", @@ -1329,7 +1330,7 @@ func TestValidate(t *testing.T) { }, expectedErr: config.ErrParamUnknownType{ MapName: "unknown_param_type", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param", Token: "!BBOX!", Type: "foo", @@ -1338,10 +1339,10 @@ func TestValidate(t *testing.T) { }, "13 parameter two defaults": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "unknown_two_defaults", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!BBOX!", @@ -1355,7 +1356,7 @@ func TestValidate(t *testing.T) { }, expectedErr: config.ErrParamTwoDefaults{ MapName: "unknown_two_defaults", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param", Token: "!BBOX!", Type: "string", @@ -1366,11 +1367,11 @@ func TestValidate(t *testing.T) { }, "13 parameter invalid default": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "parameter_invalid_default", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!BBOX!", @@ -1383,7 +1384,7 @@ func TestValidate(t *testing.T) { }, expectedErr: config.ErrParamInvalidDefault{ MapName: "parameter_invalid_default", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param", Token: "!BBOX!", Type: "int", @@ -1393,10 +1394,10 @@ func TestValidate(t *testing.T) { }, "13 invalid token name": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "parameter_invalid_token", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!Token with spaces!", @@ -1408,7 +1409,7 @@ func TestValidate(t *testing.T) { }, expectedErr: config.ErrParamBadTokenName{ MapName: "parameter_invalid_token", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param", Token: "!Token with spaces!", Type: "int", @@ -1444,14 +1445,14 @@ func TestConfigureTileBuffers(t *testing.T) { tests := map[string]tcase{ "1 tilebuffer is not set": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", }, }, }, expected: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(64)), @@ -1462,7 +1463,7 @@ func TestConfigureTileBuffers(t *testing.T) { "2 tilebuffer is set in global section": { config: config.Config{ TileBuffer: env.IntPtr(env.Int(32)), - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", }, @@ -1473,7 +1474,7 @@ func TestConfigureTileBuffers(t *testing.T) { }, expected: config.Config{ TileBuffer: env.IntPtr(env.Int(32)), - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(32)), @@ -1487,7 +1488,7 @@ func TestConfigureTileBuffers(t *testing.T) { }, "3 tilebuffer is set in map section": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(16)), @@ -1499,7 +1500,7 @@ func TestConfigureTileBuffers(t *testing.T) { }, }, expected: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(16)), @@ -1514,7 +1515,7 @@ func TestConfigureTileBuffers(t *testing.T) { "4 tilebuffer is set in global and map sections": { config: config.Config{ TileBuffer: env.IntPtr(env.Int(32)), - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(16)), @@ -1523,7 +1524,7 @@ func TestConfigureTileBuffers(t *testing.T) { }, expected: config.Config{ TileBuffer: env.IntPtr(env.Int(32)), - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(16)), diff --git a/config/errors.go b/config/errors.go index be6e5a59f..ded68ddfb 100644 --- a/config/errors.go +++ b/config/errors.go @@ -3,6 +3,8 @@ package config import ( "fmt" "strings" + + "github.com/go-spatial/tegola/provider" ) type ErrMapNotFound struct { @@ -15,78 +17,76 @@ func (e ErrMapNotFound) Error() string { type ErrParamTokenReserved struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } func (e ErrParamTokenReserved) Error() string { - return fmt.Sprintf("config: map %s has parameter %s referencing reserved token %s", + return fmt.Sprintf("config: map %s parameter %s uses a reserved token name %s", e.MapName, e.Parameter.Name, e.Parameter.Token) } -type ErrParamNameDuplicate struct { +type ErrParamDuplicateName struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } -func (e ErrParamNameDuplicate) Error() string { - return fmt.Sprintf("config: map %s redeclares duplicate parameter with name %s", +func (e ErrParamDuplicateName) Error() string { + return fmt.Sprintf("config: map %s parameter %s has a name already used by another parameter", e.MapName, e.Parameter.Name) } -type ErrParamTokenDuplicate struct { +type ErrParamDuplicateToken struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } -func (e ErrParamTokenDuplicate) Error() string { - return fmt.Sprintf("config: map %s redeclares existing parameter token %s in param %s", +func (e ErrParamDuplicateToken) Error() string { + return fmt.Sprintf("config: map %s parameter %s has a token name %s already used by another parameter", e.MapName, e.Parameter.Token, e.Parameter.Name) } type ErrParamUnknownType struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } func (e ErrParamUnknownType) Error() string { - validTypes := make([]string, len(ParamTypeDecoders)) - i := 0 - for k := range ParamTypeDecoders { - validTypes[i] = k - i++ + validTypes := make([]string, 0, len(provider.ParamTypeDecoders)) + for k := range provider.ParamTypeDecoders { + validTypes = append(validTypes, k) } - return fmt.Sprintf("config: map %s has type %s in param %s, which is not one of the known types: %s", - e.MapName, e.Parameter.Type, e.Parameter.Name, strings.Join(validTypes, ",")) + return fmt.Sprintf("config: map %s parameter %s has an unknown type %s, must be one of: %s", + e.MapName, e.Parameter.Name, e.Parameter.Type, strings.Join(validTypes, ",")) } type ErrParamTwoDefaults struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } func (e ErrParamTwoDefaults) Error() string { - return fmt.Sprintf("config: map %s has both default_value and default_sql defined in param %s", + return fmt.Sprintf("config: map %s parameter %s has both default_value and default_sql", e.MapName, e.Parameter.Name) } type ErrParamInvalidDefault struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } func (e ErrParamInvalidDefault) Error() string { - return fmt.Sprintf("config: map %s has default value in param %s that doesn't match the parameter's type %s", + return fmt.Sprintf("config: map %s parameter %s has a default value that is invalid for type %s", e.MapName, e.Parameter.Name, e.Parameter.Type) } type ErrParamBadTokenName struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } func (e ErrParamBadTokenName) Error() string { - return fmt.Sprintf("config: map %s has parameter %s referencing token with an invalid name %s", + return fmt.Sprintf("config: map %s parameter %s has an invalid token name %s", e.MapName, e.Parameter.Name, e.Parameter.Token) } diff --git a/provider/debug/debug.go b/provider/debug/debug.go index 212eab06e..ba96f41f1 100644 --- a/provider/debug/debug.go +++ b/provider/debug/debug.go @@ -26,25 +26,21 @@ func init() { } // NewProvider Setups a debug provider. there are not currently any config params supported -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { return &Provider{}, nil } // Provider provides the debug provider type Provider struct{} -func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { +func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams provider.Params, fn func(f *provider.Feature) error) error { // get tile bounding box ext, srid := tile.Extent() - params := make([]string, len(queryParams)) - i := 0 + params := make([]string, 0, len(queryParams)) for _, param := range queryParams { - for k, v := range param.RawValues { - params[i] = fmt.Sprintf("%s=%s", k, v) - i++ - } + params = append(params, fmt.Sprintf("%s=%s", param.RawParam, param.RawValue)) } paramsStr := strings.Join(params, " ") diff --git a/provider/gpkg/cgo_test.go b/provider/gpkg/cgo_test.go index 8a64fb9b1..9d7f7902e 100644 --- a/provider/gpkg/cgo_test.go +++ b/provider/gpkg/cgo_test.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package gpkg @@ -11,7 +12,7 @@ import ( // This is a test to just see that the init function is doing something. func TestNewProviderStartup(t *testing.T) { - _, err := NewTileProvider(dict.Dict{}) + _, err := NewTileProvider(dict.Dict{}, nil) if err == provider.ErrUnsupported { t.Fatalf("supported, expected any but unsupported got %v", err) } diff --git a/provider/gpkg/gpkg.go b/provider/gpkg/gpkg.go index fe369219f..65649ed07 100644 --- a/provider/gpkg/gpkg.go +++ b/provider/gpkg/gpkg.go @@ -76,7 +76,7 @@ func (p *Provider) Layers() ([]provider.LayerInfo, error) { return ls, nil } -func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { +func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams provider.Params, fn func(f *provider.Feature) error) error { log.Debugf("fetching layer %v", layer) pLayer := p.layers[layer] @@ -122,7 +122,7 @@ func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider // If layer was specified via "sql" in config, collect it z, _, _ := tile.ZXY() qtext = replaceTokens(pLayer.sql, z, tileBBox) - qtext = provider.ReplaceParams(queryParams, qtext, &args) + qtext = queryParams.ReplaceParams(qtext, &args) } log.Debugf("qtext: %v", qtext) diff --git a/provider/gpkg/gpkg_register.go b/provider/gpkg/gpkg_register.go index c1b4c52f3..791550fe9 100644 --- a/provider/gpkg/gpkg_register.go +++ b/provider/gpkg/gpkg_register.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package gpkg @@ -160,7 +161,6 @@ func extractColDefsFromSQL(sql string) []string { func featureTableMetaData(gpkg *sql.DB) (map[string]featureTableDetails, error) { // this query is used to read the metadata from the gpkg_contents, gpkg_geometry_columns, and // sqlite_master tables for tables that store geographic features. - //goland:noinspection SqlResolve qtext := ` SELECT c.table_name, c.min_x, c.min_y, c.max_x, c.max_y, c.srs_id, gc.column_name, gc.geometry_type_name, sm.sql @@ -221,7 +221,7 @@ func featureTableMetaData(gpkg *sql.DB) (map[string]featureTableDetails, error) return geomTableDetails, nil } -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { log.Debugf("config: %v", config) @@ -321,7 +321,6 @@ func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { return nil, fmt.Errorf("table %q does not exist", tablename) } - layer.tablename = tablename layer.tagFieldnames = tagFieldnames layer.geomFieldname = d.geomFieldname diff --git a/provider/gpkg/gpkg_register_internal_test.go b/provider/gpkg/gpkg_register_internal_test.go index c99b4b563..a0ba0b808 100644 --- a/provider/gpkg/gpkg_register_internal_test.go +++ b/provider/gpkg/gpkg_register_internal_test.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package gpkg @@ -1095,7 +1096,7 @@ func TestCleanup(t *testing.T) { fn := func(tc tcase) func(*testing.T) { return func(t *testing.T) { - _, err := NewTileProvider(tc.config) + _, err := NewTileProvider(tc.config, nil) if err != nil { t.Fatalf("err creating NewTileProvider: %v", err) return diff --git a/provider/gpkg/gpkg_test.go b/provider/gpkg/gpkg_test.go index 8941e74af..b3142dd13 100644 --- a/provider/gpkg/gpkg_test.go +++ b/provider/gpkg/gpkg_test.go @@ -184,7 +184,7 @@ func TestNewTileProvider(t *testing.T) { return func(t *testing.T) { t.Parallel() - p, err := gpkg.NewTileProvider(tc.config) + p, err := gpkg.NewTileProvider(tc.config, nil) if err != nil { if err.Error() != tc.expectedErr.Error() { t.Errorf("expectedErr %v got %v", tc.expectedErr, err) @@ -263,7 +263,7 @@ func TestTileFeatures(t *testing.T) { return func(t *testing.T) { t.Parallel() - p, err := gpkg.NewTileProvider(tc.config) + p, err := gpkg.NewTileProvider(tc.config, nil) if err != nil { t.Fatalf("new tile, expected nil got %v", err) return @@ -411,7 +411,7 @@ func TestConfigs(t *testing.T) { fn := func(tc tcase) func(*testing.T) { return func(t *testing.T) { - p, err := gpkg.NewTileProvider(tc.config) + p, err := gpkg.NewTileProvider(tc.config, nil) if err != nil { t.Fatalf("err creating NewTileProvider: %v", err) return @@ -593,7 +593,7 @@ func TestOpenNonExistantFile(t *testing.T) { os.Remove(NONEXISTANTFILE) fn := func(tc tcase) func(*testing.T) { return func(t *testing.T) { - _, err := gpkg.NewTileProvider(tc.config) + _, err := gpkg.NewTileProvider(tc.config, nil) if reflect.TypeOf(err) != reflect.TypeOf(tc.err) { t.Errorf("expected error, expected %v got %v", tc.err, err) } @@ -601,13 +601,13 @@ func TestOpenNonExistantFile(t *testing.T) { } tests := map[string]tcase{ - "empty": tcase{ + "empty": { config: dict.Dict{ gpkg.ConfigKeyFilePath: "", }, err: gpkg.ErrInvalidFilePath{FilePath: ""}, }, - "nonexistance": tcase{ + "nonexistance": { // should not exists config: dict.Dict{ gpkg.ConfigKeyFilePath: NONEXISTANTFILE, diff --git a/provider/map.go b/provider/map.go new file mode 100644 index 000000000..d59ccd0b9 --- /dev/null +++ b/provider/map.go @@ -0,0 +1,14 @@ +package provider + +import "github.com/go-spatial/tegola/internal/env" + +// A Map represents a map in the Tegola Config file. +type Map struct { + Name env.String `toml:"name"` + Attribution env.String `toml:"attribution"` + Bounds []env.Float `toml:"bounds"` + Center [3]env.Float `toml:"center"` + Layers []MapLayer `toml:"layers"` + Parameters []QueryParameter `toml:"params"` + TileBuffer *env.Int `toml:"tile_buffer"` +} diff --git a/provider/map_layer.go b/provider/map_layer.go new file mode 100644 index 000000000..4a29d61a4 --- /dev/null +++ b/provider/map_layer.go @@ -0,0 +1,51 @@ +package provider + +import ( + "fmt" + "strings" + + "github.com/go-spatial/tegola/internal/env" +) + +// MapLayer represents a the config for a layer in a map +type MapLayer struct { + // Name is optional. If it's not defined the name of the ProviderLayer will be used. + // Name can also be used to group multiple ProviderLayers under the same namespace. + Name env.String `toml:"name"` + ProviderLayer env.String `toml:"provider_layer"` + MinZoom *env.Uint `toml:"min_zoom"` + MaxZoom *env.Uint `toml:"max_zoom"` + DefaultTags env.Dict `toml:"default_tags"` + // DontSimplify indicates whether feature simplification should be applied. + // We use a negative in the name so the default is to simplify + DontSimplify env.Bool `toml:"dont_simplify"` + // DontClip indicates whether feature clipping should be applied. + // We use a negative in the name so the default is to clipping + DontClip env.Bool `toml:"dont_clip"` + // DontClip indicates whether feature cleaning (e.g. make valid) should be applied. + // We use a negative in the name so the default is to clean + DontClean env.Bool `toml:"dont_clean"` +} + +// ProviderLayerName returns the names of the layer and provider or an error +func (ml MapLayer) ProviderLayerName() (provider, layer string, err error) { + // split the provider layer (syntax is provider.layer) + plParts := strings.Split(string(ml.ProviderLayer), ".") + if len(plParts) != 2 { + // TODO (beymak): Properly handle the error + return "", "", fmt.Errorf("config: invalid provider layer name (%v)", ml.ProviderLayer) + // return "", "", ErrInvalidProviderLayerName{ProviderLayerName: string(ml.ProviderLayer)} + } + return plParts[0], plParts[1], nil +} + +// GetName will return the user-defined Layer name from the config, +// or if the name is empty, return the name of the layer associated with +// the provider +func (ml MapLayer) GetName() (string, error) { + if ml.Name != "" { + return string(ml.Name), nil + } + _, name, err := ml.ProviderLayerName() + return name, err +} diff --git a/provider/mvt_provider.go b/provider/mvt_provider.go index 558682a9f..4f192fec8 100644 --- a/provider/mvt_provider.go +++ b/provider/mvt_provider.go @@ -10,8 +10,8 @@ type MVTTiler interface { Layerer // MVTForLayers will return a MVT byte array or an error for the given layer names. - MVTForLayers(ctx context.Context, tile Tile, params map[string]QueryParameter, layers []Layer) ([]byte, error) + MVTForLayers(ctx context.Context, tile Tile, params Params, layers []Layer) ([]byte, error) } // MVTInitFunc initialize a provider given a config map. The init function should validate the config map, and report any errors. This is called by the For function. -type MVTInitFunc func(dicter dict.Dicter) (MVTTiler, error) +type MVTInitFunc func(dicter dict.Dicter, maps []Map) (MVTTiler, error) diff --git a/provider/paramater_decoders.go b/provider/paramater_decoders.go new file mode 100644 index 000000000..d1a2f7b91 --- /dev/null +++ b/provider/paramater_decoders.go @@ -0,0 +1,19 @@ +package provider + +import "strconv" + +// ParamTypeDecoders is a collection of parsers for different types of user-defined parameters +var ParamTypeDecoders = map[string]func(string) (interface{}, error){ + "int": func(s string) (interface{}, error) { + return strconv.Atoi(s) + }, + "float": func(s string) (interface{}, error) { + return strconv.ParseFloat(s, 32) + }, + "string": func(s string) (interface{}, error) { + return s, nil + }, + "bool": func(s string) (interface{}, error) { + return strconv.ParseBool(s) + }, +} diff --git a/provider/postgis/postgis.go b/provider/postgis/postgis.go index 23a907099..8c7be5014 100644 --- a/provider/postgis/postgis.go +++ b/provider/postgis/postgis.go @@ -171,6 +171,7 @@ const ( ) const ( + ConfigKeyName = "name" ConfigKeyURI = "uri" ConfigKeyHost = "host" ConfigKeyPort = "port" @@ -406,26 +407,24 @@ func BuildDBConfig(uri string) (*pgxpool.Config, error) { // trying to create a driver. This Provider supports the following fields // in the provided map[string]interface{} map: // -// host (string): [Required] postgis database host -// port (int): [Required] postgis database port (required) -// database (string): [Required] postgis database name -// user (string): [Required] postgis database user -// password (string): [Required] postgis database password -// srid (int): [Optional] The default SRID for the provider. Defaults to WebMercator (3857) but also supports WGS84 (4326) -// max_connections : [Optional] The max connections to maintain in the connection pool. Default is 100. 0 means no max. -// layers (map[string]struct{}) — This is map of layers keyed by the layer name. supports the following properties +// name (string): [Required] name of the provider +// host (string): [Required] postgis database host +// port (int): [Required] postgis database port (required) +// database (string): [Required] postgis database name +// user (string): [Required] postgis database user +// password (string): [Required] postgis database password +// srid (int): [Optional] The default SRID for the provider. Defaults to WebMercator (3857) but also supports WGS84 (4326) +// max_connections : [Optional] The max connections to maintain in the connection pool. Default is 100. 0 means no max. +// layers (map[string]struct{}) — This is map of layers keyed by the layer name. supports the following properties // -// name (string): [Required] the name of the layer. This is used to reference this layer from map layers. -// tablename (string): [*Required] the name of the database table to query against. Required if sql is not defined. -// geometry_fieldname (string): [Optional] the name of the filed which contains the geometry for the feature. defaults to geom -// id_fieldname (string): [Optional] the name of the feature id field. defaults to gid -// fields ([]string): [Optional] a list of fields to include alongside the feature. Can be used if sql is not defined. -// srid (int): [Optional] the SRID of the layer. Supports 3857 (WebMercator) or 4326 (WGS84). -// sql (string): [*Required] custom SQL to use use. Required if tablename is not defined. Supports the following tokens: -// -// !BBOX! - [Required] will be replaced with the bounding box of the tile before the query is sent to the database. -// !ZOOM! - [Optional] will be replaced with the "Z" (zoom) value of the requested tile. -func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) { +// name (string): [Required] the name of the layer. This is used to reference this layer from map layers. +// tablename (string): [*Required] the name of the database table to query against. Required if sql is not defined. +// geometry_fieldname (string): [Optional] the name of the filed which contains the geometry for the feature. defaults to geom +// id_fieldname (string): [Optional] the name of the feature id field. defaults to gid +// fields ([]string): [Optional] a list of fields to include alongside the feature. Can be used if sql is not defined. +// srid (int): [Optional] the SRID of the layer. Supports 3857 (WebMercator) or 4326 (WGS84). +// sql (string): [*Required] custom SQL to use use. Required if tablename is not defined. Supports the following tokens: +func CreateProvider(config dict.Dicter, maps []provider.Map, providerType string) (*Provider, error) { uri, params, err := BuildURI(config) if err != nil { return nil, err @@ -453,7 +452,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) dbconfig, err := BuildDBConfig(uri.String()) if err != nil { - return nil, fmt.Errorf("failed while building db config: %v", err) + return nil, fmt.Errorf("failed while building db config: %w", err) } srid := DefaultSRID @@ -472,7 +471,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) pool, err := pgxpool.ConnectConfig(context.Background(), &p.config) if err != nil { - return nil, fmt.Errorf("failed while creating connection pool: %v", err) + return nil, fmt.Errorf("failed while creating connection pool: %w", err) } p.pool = &connectionPoolCollector{Pool: pool} @@ -488,7 +487,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) lName, err := layer.String(ConfigKeyLayerName, nil) if err != nil { - return nil, fmt.Errorf("for layer (%v) we got the following error trying to get the layer's name field: %v", i, err) + return nil, fmt.Errorf("for layer (%v) we got the following error trying to get the layer's name field: %w", i, err) } if j, ok := lyrsSeen[lName]; ok { @@ -605,7 +604,12 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) return nil, fmt.Errorf("error fetching geometry type for layer (%v): %w", l.name, err) } } else { - if err = p.inspectLayerGeomType(&l); err != nil { + pname, err := config.String(ConfigKeyName, nil) + if err != nil { + return nil, err + } + + if err = p.inspectLayerGeomType(pname, &l, maps); err != nil { return nil, fmt.Errorf("error fetching geometry type for layer (%v): %w\nif custom parameters are used, remember to set %s for the provider", l.name, err, ConfigKeyGeomType) } } @@ -698,7 +702,7 @@ func (p Provider) setLayerGeomType(l *Layer, geomType string) error { // inspectLayerGeomType sets the geomType field on the layer by running the SQL // and reading the geom type in the result set -func (p Provider) inspectLayerGeomType(l *Layer) error { +func (p Provider) inspectLayerGeomType(pname string, l *Layer, maps []provider.Map) error { var err error // we want to know the geom type instead of returning the geom data so we modify the SQL @@ -732,12 +736,20 @@ func (p Provider) inspectLayerGeomType(l *Layer) error { return err } - // remove all parameter tokens for inspection - // crossing our fingers that the query is still valid 🤞 - // if not, the user will have to specify `geometry_type` in the config - sql = stripParams(sql) + // substitute default values to parameter + params := extractQueryParamValues(pname, maps, l) + + args := make([]interface{}, 0) + sql = params.ReplaceParams(sql, &args) + + if provider.ParameterTokenRegexp.MatchString(sql) { + // remove all parameter tokens for inspection + // crossing our fingers that the query is still valid 🤞 + // if not, the user will have to specify `geometry_type` in the config + sql = provider.ParameterTokenRegexp.ReplaceAllString(sql, "") + } - rows, err := p.pool.Query(context.Background(), sql) + rows, err := p.pool.Query(context.Background(), sql, args...) if err != nil { return err } @@ -804,8 +816,7 @@ func (p Provider) Layers() ([]provider.LayerInfo, error) { } // TileFeatures adheres to the provider.Tiler interface -// TODO (bemyak): Make an actual use of QueryParams -func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { +func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, params provider.Params, fn func(f *provider.Feature) error) error { var mapName string { @@ -828,7 +839,7 @@ func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider. // replace configured query parameters if any args := make([]interface{}, 0) - sql = provider.ReplaceParams(queryParams, sql, &args) + sql = params.ReplaceParams(sql, &args) if err != nil { return err } @@ -941,7 +952,7 @@ func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider. return rows.Err() } -func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, params map[string]provider.QueryParameter, layers []provider.Layer) ([]byte, error) { +func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, params provider.Params, layers []provider.Layer) ([]byte, error) { var ( err error sqls = make([]string, 0, len(layers)) @@ -977,7 +988,7 @@ func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, params m } // replace configured query parameters if any - sql = provider.ReplaceParams(params, sql, &args) + sql = params.ReplaceParams(sql, &args) // ref: https://postgis.net/docs/ST_AsMVT.html // bytea ST_AsMVT(any_element row, text name, integer extent, text geom_name, text feature_id_name) diff --git a/provider/postgis/postgis_internal_test.go b/provider/postgis/postgis_internal_test.go index bd1346bd0..bec0787b1 100644 --- a/provider/postgis/postgis_internal_test.go +++ b/provider/postgis/postgis_internal_test.go @@ -98,7 +98,8 @@ func TestMVTProviders(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { config := tc.Config(DefaultEnvConfig) - prvd, err := NewMVTTileProvider(config) + config[ConfigKeyName] = "provider_name" + prvd, err := NewMVTTileProvider(config, nil) // for now we will just check the length of the bytes. if tc.err != "" { if err == nil || !strings.Contains(err.Error(), tc.err) { @@ -168,7 +169,8 @@ func TestLayerGeomType(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { config := tc.Config(DefaultEnvConfig) - provider, err := NewTileProvider(config) + config[ConfigKeyName] = "provider_name" + provider, err := NewTileProvider(config, nil) if tc.err != "" { if err == nil || !strings.Contains(err.Error(), tc.err) { t.Logf("error %#v", err) @@ -276,11 +278,11 @@ func TestLayerGeomType(t *testing.T) { TCConfig: TCConfig{ ConfigOverride: map[string]interface{}{ ConfigKeyURI: fmt.Sprintf("postgres://%v:%v@%v:%v/%v", - defaultEnvConfig["user"], - defaultEnvConfig["password"], - defaultEnvConfig["host"], - defaultEnvConfig["port"], - defaultEnvConfig["database"], + DefaultEnvConfig["user"], + DefaultEnvConfig["password"], + DefaultEnvConfig["host"], + DefaultEnvConfig["port"], + DefaultEnvConfig["database"], ), ConfigKeyHost: "", ConfigKeyPort: "", diff --git a/provider/postgis/postgis_test.go b/provider/postgis/postgis_test.go index 3907e1d90..22f4f3ffc 100644 --- a/provider/postgis/postgis_test.go +++ b/provider/postgis/postgis_test.go @@ -165,7 +165,8 @@ func TestNewTileProvider(t *testing.T) { fn := func(tc postgis.TCConfig) func(t *testing.T) { return func(t *testing.T) { config := tc.Config(postgis.DefaultEnvConfig) - _, err := postgis.NewTileProvider(config) + config[postgis.ConfigKeyName] = "provider_name" + _, err := postgis.NewTileProvider(config, nil) if err != nil { t.Errorf("unable to create a new provider. err: %v", err) return @@ -203,7 +204,8 @@ func TestTileFeatures(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { config := tc.Config(postgis.DefaultEnvConfig) - p, err := postgis.NewTileProvider(config) + config[postgis.ConfigKeyName] = "provider_name" + p, err := postgis.NewTileProvider(config, nil) if err != nil { t.Errorf("unexpected error; unable to create a new provider, expected: nil Got %v", err) return diff --git a/provider/postgis/register.go b/provider/postgis/register.go index ab0939be5..4c3a271b2 100644 --- a/provider/postgis/register.go +++ b/provider/postgis/register.go @@ -40,9 +40,9 @@ const ( // !BBOX! - [Required] will be replaced with the bounding box of the tile before the query is sent to the database. // !ZOOM! - [Optional] will be replaced with the "Z" (zoom) value of the requested tile. // -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { - return CreateProvider(config, ProviderType) +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { + return CreateProvider(config, maps, ProviderType) } -func NewMVTTileProvider(config dict.Dicter) (provider.MVTTiler, error) { - return CreateProvider(config, MVTProviderType) +func NewMVTTileProvider(config dict.Dicter, maps []provider.Map) (provider.MVTTiler, error) { + return CreateProvider(config, maps, MVTProviderType) } diff --git a/provider/postgis/util.go b/provider/postgis/util.go index 1d68c6b07..7dcb5df07 100644 --- a/provider/postgis/util.go +++ b/provider/postgis/util.go @@ -10,6 +10,7 @@ import ( "github.com/go-spatial/tegola" "github.com/go-spatial/tegola/basic" "github.com/go-spatial/tegola/config" + "github.com/go-spatial/tegola/internal/env" "github.com/go-spatial/tegola/internal/log" "github.com/go-spatial/tegola/provider" "github.com/jackc/pgproto3/v2" @@ -173,9 +174,25 @@ func replaceTokens(sql string, lyr *Layer, tile provider.Tile, withBuffer bool) return tokenReplacer.Replace(uppercaseTokenSQL), nil } -// stripParams will remove all parameter tokens from the query -func stripParams(sql string) string { - return provider.ParameterTokenRegexp.ReplaceAllString(sql, "") +// extractQueryParamValues finds default values for SQL tokens and constructs query parameter values out of them +func extractQueryParamValues(pname string, maps []provider.Map, layer *Layer) provider.Params { + result := make(provider.Params, 0) + + expectedMapName := fmt.Sprintf("%s.%s", pname, layer.name) + for _, m := range maps { + for _, l := range m.Layers { + if l.ProviderLayer == env.String(expectedMapName) { + for _, p := range m.Parameters { + pv, err := p.ToDefaultValue() + if err == nil { + result[p.Token] = pv + } + } + } + } + } + + return result } // uppercaseTokens converts all !tokens! to uppercase !TOKENS!. Tokens can diff --git a/provider/postgis/util_internal_test.go b/provider/postgis/util_internal_test.go index a662a047b..cf67d37b9 100644 --- a/provider/postgis/util_internal_test.go +++ b/provider/postgis/util_internal_test.go @@ -74,128 +74,6 @@ func TestReplaceTokens(t *testing.T) { } } -func TestReplaceParams(t *testing.T) { - type tcase struct { - params map[string]provider.QueryParameter - sql string - expectedSql string - expectedArgs []interface{} - } - - fn := func(tc tcase) func(t *testing.T) { - return func(t *testing.T) { - args := make([]interface{}, 0) - out := provider.ReplaceParams(tc.params, tc.sql, &args) - - if out != tc.expectedSql { - t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedSql, out) - return - } - - if len(tc.expectedArgs) != len(args) { - t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedArgs, args) - return - } - for i, arg := range tc.expectedArgs { - if arg != args[i] { - t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedArgs, args) - return - } - } - } - } - - tests := map[string]tcase{ - "nil params": { - params: nil, - sql: "SELECT * FROM table", - expectedSql: "SELECT * FROM table", - expectedArgs: []interface{}{}, - }, - "int replacement": { - params: map[string]provider.QueryParameter{ - "!PARAM!": { - Token: "!PARAM!", - SQL: "?", - Value: 1, - }, - }, - sql: "SELECT * FROM table WHERE PARAM = !PARAM!", - expectedSql: "SELECT * FROM table WHERE PARAM = $1", - expectedArgs: []interface{}{1}, - }, - "string replacement": { - params: map[string]provider.QueryParameter{ - "!PARAM!": { - Token: "!PARAM!", - SQL: "?", - Value: "test", - }, - }, - sql: "SELECT * FROM table WHERE PARAM = !PARAM!", - expectedSql: "SELECT * FROM table WHERE PARAM = $1", - expectedArgs: []interface{}{"test"}, - }, - "null replacement": { - params: map[string]provider.QueryParameter{ - "!PARAM!": { - Token: "!PARAM!", - SQL: "?", - Value: nil, - }, - }, - sql: "SELECT * FROM table WHERE PARAM = !PARAM!", - expectedSql: "SELECT * FROM table WHERE PARAM = $1", - expectedArgs: []interface{}{nil}, - }, - "complex sql replacement": { - params: map[string]provider.QueryParameter{ - "!PARAM!": { - Token: "!PARAM!", - SQL: "WHERE PARAM=?", - Value: 1, - }, - }, - sql: "SELECT * FROM table !PARAM!", - expectedSql: "SELECT * FROM table WHERE PARAM=$1", - expectedArgs: []interface{}{1}, - }, - "subquery removal": { - params: map[string]provider.QueryParameter{ - "!PARAM!": { - Token: "!PARAM!", - SQL: "", - Value: nil, - }, - }, - sql: "SELECT * FROM table !PARAM!", - expectedSql: "SELECT * FROM table ", - expectedArgs: []interface{}{}, - }, - "multiple params": { - params: map[string]provider.QueryParameter{ - "!PARAM!": { - Token: "!PARAM!", - SQL: "???", - Value: 1, - }, - "!PARAM2!": { - Token: "!PARAM2!", - SQL: "???", - Value: 2, - }, - }, - sql: "!PARAM!!PARAM2!", - expectedSql: "$1$1$1$2$2$2", - expectedArgs: []interface{}{1, 2}, - }, - } - - for name, tc := range tests { - t.Run(name, fn(tc)) - } -} - func TestUppercaseTokens(t *testing.T) { type tcase struct { str string diff --git a/provider/provider.go b/provider/provider.go index 6fc2fe2c2..6813892cb 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "regexp" - "strings" "github.com/go-spatial/geom" "github.com/go-spatial/geom/slippy" @@ -109,59 +108,13 @@ type Tile interface { // ParameterTokenRegexp to validate QueryParameters var ParameterTokenRegexp = regexp.MustCompile("![a-zA-Z0-9_-]+!") -// Query parameter holds normalized parameter data ready to be inserted in the -// final query -type QueryParameter struct { - // Token to replace e.g., !TOKEN! - Token string - // SQL expression to be inserted. Contains "?" that will be replaced with an - // ordinal argument e.g., "$1" - SQL string - // Value that will be passed to the final query in arguments list - Value interface{} - // Raw parameter values for debugging and monitoring - RawValues map[string]string -} - -// ReplaceParams substitutes configured query parameter tokens for their values -// within the provided SQL string -func ReplaceParams(params map[string]QueryParameter, sql string, args *[]interface{}) string { - if params == nil { - return sql - } - - for _, token := range ParameterTokenRegexp.FindAllString(sql, -1) { - param := params[token] - - // Replace every ? in the param's SQL with a positional argument - paramSQL := "" - argFound := false - for _, c := range param.SQL { - if c == '?' { - if !argFound { - *args = append(*args, param.Value) - argFound = true - } - paramSQL += fmt.Sprintf("$%d", len(*args)) - } else { - paramSQL += string(c) - } - } - - // Finally, replace current token with the prepared SQL - sql = strings.Replace(sql, token, paramSQL, 1) - } - - return sql -} - // Tiler is a Layers that allows one to encode features in that layer type Tiler interface { Layerer // TileFeature will stream decoded features to the callback function fn // if fn returns ErrCanceled, the TileFeatures method should stop processing - TileFeatures(ctx context.Context, layer string, t Tile, queryParams map[string]QueryParameter, fn func(f *Feature) error) error + TileFeatures(ctx context.Context, layer string, t Tile, params Params, fn func(f *Feature) error) error } // TilerUnion represents either a Std Tiler or and MVTTiler; only one should be not nil. @@ -183,7 +136,7 @@ func (tu TilerUnion) Layers() ([]LayerInfo, error) { } // InitFunc initialize a provider given a config map. The init function should validate the config map, and report any errors. This is called by the For function. -type InitFunc func(dicter dict.Dicter) (Tiler, error) +type InitFunc func(dicter dict.Dicter, maps []Map) (Tiler, error) // CleanupFunc is called to when the system is shutting down, this allows the provider to cleanup. type CleanupFunc func() @@ -280,7 +233,7 @@ func Drivers(types ...providerType) (l []string) { // For function returns a configure provider of the given type; The provider may be a mvt provider or // a std provider. The correct entry in TilerUnion will not be nil. If there is an error both entries // will be nil. -func For(name string, config dict.Dicter) (val TilerUnion, err error) { +func For(name string, config dict.Dicter, maps []Map) (val TilerUnion, err error) { var ( driversList = Drivers() ) @@ -292,11 +245,11 @@ func For(name string, config dict.Dicter) (val TilerUnion, err error) { return val, ErrUnknownProvider{KnownProviders: driversList, Name: name} } if p.init != nil { - val.Std, err = p.init(config) + val.Std, err = p.init(config, maps) return val, err } if p.mvtInit != nil { - val.Mvt, err = p.mvtInit(config) + val.Mvt, err = p.mvtInit(config, maps) return val, err } return val, ErrInvalidRegisteredProvider{Name: name} diff --git a/provider/provider_test.go b/provider/provider_test.go index 220ef94a4..d67a98a93 100644 --- a/provider/provider_test.go +++ b/provider/provider_test.go @@ -12,7 +12,7 @@ func TestProviderInterface(t *testing.T) { stdName = provider.TypeStd.Prefix() + test.Name mvtName = provider.TypeMvt.Prefix() + test.Name ) - if _, err := provider.For(stdName, nil); err != nil { + if _, err := provider.For(stdName, nil, nil); err != nil { t.Errorf("retrieve provider err , expected nil got %v", err) return } @@ -23,7 +23,7 @@ func TestProviderInterface(t *testing.T) { if test.Count != 0 { t.Errorf(" expected count , expected 0 got %v", test.Count) } - if _, err := provider.For(mvtName, nil); err != nil { + if _, err := provider.For(mvtName, nil, nil); err != nil { t.Errorf("retrieve provider err , expected nil got %v", err) return } diff --git a/provider/query_parameter.go b/provider/query_parameter.go new file mode 100644 index 000000000..181169bcb --- /dev/null +++ b/provider/query_parameter.go @@ -0,0 +1,65 @@ +package provider + +import ( + "fmt" + "strings" +) + +// QueryParameter represents an HTTP query parameter specified for use with +// a given map instance. +type QueryParameter struct { + Name string `toml:"name"` + Token string `toml:"token"` + Type string `toml:"type"` + SQL string `toml:"sql"` + // DefaultSQL replaces SQL if param wasn't passed. Either default_sql or + // default_value can be specified + DefaultSQL string `toml:"default_sql"` + DefaultValue string `toml:"default_value"` +} + +// Normalize normalizes param and sets default values +func (param *QueryParameter) Normalize() { + param.Token = strings.ToUpper(param.Token) + + if len(param.SQL) == 0 { + param.SQL = "?" + } +} + +func (param *QueryParameter) ToValue(rawValue string) (QueryParameterValue, error) { + val, err := ParamTypeDecoders[param.Type](rawValue) + if err != nil { + return QueryParameterValue{}, err + } + return QueryParameterValue{ + Token: param.Token, + SQL: param.SQL, + Value: val, + RawParam: param.Name, + RawValue: rawValue, + }, nil +} + +func (param *QueryParameter) ToDefaultValue() (QueryParameterValue, error) { + if len(param.DefaultValue) > 0 { + val, err := ParamTypeDecoders[param.Type](param.DefaultValue) + return QueryParameterValue{ + Token: param.Token, + SQL: param.SQL, + Value: val, + RawParam: param.Name, + RawValue: "", + }, err + } + if len(param.DefaultSQL) > 0 { + return QueryParameterValue{ + Token: param.Token, + SQL: param.DefaultSQL, + Value: nil, + RawParam: param.Name, + RawValue: "", + }, nil + } + return QueryParameterValue{}, fmt.Errorf("the required parameter %s is not specified", param.Name) +} diff --git a/provider/query_parameter_value.go b/provider/query_parameter_value.go new file mode 100644 index 000000000..10fec3170 --- /dev/null +++ b/provider/query_parameter_value.go @@ -0,0 +1,76 @@ +package provider + +import ( + "fmt" + "strings" +) + +// Query parameter holds normalized parameter data ready to be inserted in the +// final query +type QueryParameterValue struct { + // Token to replace e.g., !TOKEN! + Token string + // SQL expression to be inserted. Contains "?" that will be replaced with an + // ordinal argument e.g., "$1" + SQL string + // Value that will be passed to the final query in arguments list + Value interface{} + // Raw parameter and value for debugging and monitoring + RawParam string + // RawValue will be "" if the param wasn't passed and defaults were used + RawValue string +} + +type Params map[string]QueryParameterValue + +// ReplaceParams substitutes configured query parameter tokens for their values +// within the provided SQL string +func (params Params) ReplaceParams(sql string, args *[]interface{}) string { + if params == nil { + return sql + } + + var ( + cache = make(map[string]string) + sb strings.Builder + ) + + for _, token := range ParameterTokenRegexp.FindAllString(sql, -1) { + resultSQL, ok := cache[token] + if ok { + // Already have it cached, replace the token and move on. + sql = strings.ReplaceAll(sql, token, resultSQL) + continue + } + + param, ok := params[token] + if !ok { + // Unknown token, ignoring + continue + } + + sb.Reset() + sb.Grow(len(param.SQL)) + argFound := false + + // Replace every `?` in the param's SQL with a positional argument + for _, c := range param.SQL { + if c != '?' { + sb.WriteRune(c) + continue + } + + if !argFound { + *args = append(*args, param.Value) + argFound = true + } + sb.WriteString(fmt.Sprintf("$%d", len(*args))) + } + + resultSQL = sb.String() + cache[token] = resultSQL + sql = strings.ReplaceAll(sql, token, resultSQL) + } + + return sql +} diff --git a/provider/query_parameter_value_test.go b/provider/query_parameter_value_test.go new file mode 100644 index 000000000..204bca97f --- /dev/null +++ b/provider/query_parameter_value_test.go @@ -0,0 +1,137 @@ +package provider + +import "testing" + +func TestReplaceParams(t *testing.T) { + type tcase struct { + params Params + sql string + expectedSql string + expectedArgs []interface{} + } + + fn := func(tc tcase) func(t *testing.T) { + return func(t *testing.T) { + args := make([]interface{}, 0) + out := tc.params.ReplaceParams(tc.sql, &args) + + if out != tc.expectedSql { + t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedSql, out) + return + } + + if len(tc.expectedArgs) != len(args) { + t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedArgs, args) + return + } + for i, arg := range tc.expectedArgs { + if arg != args[i] { + t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedArgs, args) + return + } + } + } + } + + tests := map[string]tcase{ + "nil params": { + params: nil, + sql: "SELECT * FROM table", + expectedSql: "SELECT * FROM table", + expectedArgs: []interface{}{}, + }, + "int replacement": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: 1, + }, + }, + sql: "SELECT * FROM table WHERE param = !PARAM!", + expectedSql: "SELECT * FROM table WHERE param = $1", + expectedArgs: []interface{}{1}, + }, + "string replacement": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: "test", + }, + }, + sql: "SELECT * FROM table WHERE param = !PARAM!", + expectedSql: "SELECT * FROM table WHERE param = $1", + expectedArgs: []interface{}{"test"}, + }, + "null replacement": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: nil, + }, + }, + sql: "SELECT * FROM table WHERE param = !PARAM!", + expectedSql: "SELECT * FROM table WHERE param = $1", + expectedArgs: []interface{}{nil}, + }, + "complex sql replacement": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "WHERE param=?", + Value: 1, + }, + }, + sql: "SELECT * FROM table !PARAM!", + expectedSql: "SELECT * FROM table WHERE param=$1", + expectedArgs: []interface{}{1}, + }, + "subquery removal": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "", + Value: nil, + }, + }, + sql: "SELECT * FROM table !PARAM!", + expectedSql: "SELECT * FROM table ", + expectedArgs: []interface{}{}, + }, + "multiple params": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "? ? ?", + Value: 1, + }, + "!PARAM2!": { + Token: "!PARAM2!", + SQL: "???", + Value: 2, + }, + }, + sql: "!PARAM! !PARAM2! !PARAM!", + expectedSql: "$1 $1 $1 $2$2$2 $1 $1 $1", + expectedArgs: []interface{}{1, 2}, + }, + "unknown token": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: 1, + }, + }, + sql: "!NOT_PARAM! !PARAM! !NOT_PARAM!", + expectedSql: "!NOT_PARAM! $1 !NOT_PARAM!", + expectedArgs: []interface{}{1}, + }, + } + + for name, tc := range tests { + t.Run(name, fn(tc)) + } +} diff --git a/provider/test/emptycollection/provider.go b/provider/test/emptycollection/provider.go index 458a4e540..eef8d9ac0 100644 --- a/provider/test/emptycollection/provider.go +++ b/provider/test/emptycollection/provider.go @@ -19,7 +19,7 @@ func init() { } // NewProvider setups a test provider. there are not currently any config params supported -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { Count++ return &TileProvider{}, nil } @@ -40,7 +40,7 @@ func (tp *TileProvider) Layers() ([]provider.LayerInfo, error) { } // TilFeatures always returns a feature with a polygon outlining the tile's Extent (not Buffered Extent) -func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { +func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, queryParams provider.Params, fn func(f *provider.Feature) error) error { // get tile bounding box _, srid := t.Extent() diff --git a/provider/test/provider.go b/provider/test/provider.go index ca0942693..011242afb 100644 --- a/provider/test/provider.go +++ b/provider/test/provider.go @@ -28,7 +28,7 @@ func init() { } // NewTileProvider setups a test provider. there are not currently any config params supported -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { lock.Lock() Count++ lock.Unlock() @@ -37,7 +37,7 @@ func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { // NewMVTTileProvider setups a test provider for mvt tiles providers. The only supported parameter is // "test_file", which should point to a mvt tile file to return for MVTForLayers -func NewMVTTileProvider(config dict.Dicter) (provider.MVTTiler, error) { +func NewMVTTileProvider(config dict.Dicter, maps []provider.Map) (provider.MVTTiler, error) { lock.Lock() MVTCount++ lock.Unlock() @@ -86,7 +86,7 @@ func (tp *TileProvider) Layers() ([]provider.LayerInfo, error) { } // TileFeatures always returns a feature with a polygon outlining the tile's Extent (not Buffered Extent) -func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { +func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, queryParams provider.Params, fn func(f *provider.Feature) error) error { // get tile bounding box ext, srid := t.Extent() @@ -103,7 +103,7 @@ func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provid } // MVTForLayers mocks out MVTForLayers by just returning the MVTTile bytes, this will never error -func (tp *TileProvider) MVTForLayers(ctx context.Context, _ provider.Tile, _ map[string]provider.QueryParameter, _ []provider.Layer) ([]byte, error) { +func (tp *TileProvider) MVTForLayers(ctx context.Context, _ provider.Tile, _ provider.Params, _ []provider.Layer) ([]byte, error) { // TODO(gdey): fill this out. if tp == nil { return nil, nil diff --git a/server/handle_map_layer_zxy.go b/server/handle_map_layer_zxy.go index 9cfbfa65e..39345a3b0 100644 --- a/server/handle_map_layer_zxy.go +++ b/server/handle_map_layer_zxy.go @@ -8,7 +8,6 @@ import ( "strconv" "strings" - "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/observability" "github.com/go-spatial/tegola/provider" @@ -101,9 +100,11 @@ func (req *HandleMapLayerZXY) parseURI(r *http.Request) error { // map_name - map name in the config file // layer_name - name of the single map layer to render // z, x, y - tile coordinates as described in the Slippy Map Tilenames specification -// z - zoom level -// x - row -// y - column +// +// z - zoom level +// x - row +// y - column +// // param - configurable query parameters and their values func (req HandleMapLayerZXY) ServeHTTP(w http.ResponseWriter, r *http.Request) { // parse our URI @@ -162,7 +163,7 @@ func (req HandleMapLayerZXY) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // check for query parameters and populate param map with their values - params, err := req.extractParameters(r) + params, err := extractParameters(m, r) if err != nil { log.Error(err) http.Error(w, err.Error(), http.StatusBadRequest) @@ -206,44 +207,28 @@ func (req HandleMapLayerZXY) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } -func (req *HandleMapLayerZXY) extractParameters(r *http.Request) (map[string]provider.QueryParameter, error) { - var params map[string]provider.QueryParameter - if req.Atlas.HasParams(req.mapName) { - params = make(map[string]provider.QueryParameter) +func extractParameters(m atlas.Map, r *http.Request) (provider.Params, error) { + var params provider.Params + if m.Params != nil && len(m.Params) > 0 { + params = make(provider.Params) err := r.ParseForm() if err != nil { return nil, err } - for _, param := range req.Atlas.GetParams(req.mapName) { + for _, param := range m.Params { if r.Form.Has(param.Name) { - val, err := config.ParamTypeDecoders[param.Type](r.Form.Get(param.Name)) + val, err := param.ToValue(r.Form.Get(param.Name)) if err != nil { return nil, err } - params[param.Token] = provider.QueryParameter{ - Token: param.Type, - SQL: param.SQL, - Value: val, - } - } else if len(param.DefaultValue) > 0 { - val, err := config.ParamTypeDecoders[param.Type](param.DefaultValue) + params[param.Token] = val + } else { + p, err := param.ToDefaultValue() if err != nil { return nil, err } - params[param.Token] = provider.QueryParameter{ - Token: param.Type, - SQL: param.SQL, - Value: val, - } - } else if len(param.DefaultSQL) > 0 { - params[param.Token] = provider.QueryParameter{ - Token: param.Type, - SQL: param.DefaultSQL, - Value: nil, - } - } else { - return nil, fmt.Errorf("the required parameter %s is not specified", param.Name) + params[param.Token] = p } } } diff --git a/tile.go b/tile.go index 609238f6f..38ecd7bc5 100644 --- a/tile.go +++ b/tile.go @@ -17,7 +17,7 @@ const ( var UnknownConversionError = fmt.Errorf("do not know how to convert value to requested value") -//Tile slippy map tilenames +// Tile slippy map tilenames // http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames type Tile struct { Z uint @@ -198,7 +198,7 @@ func (t *Tile) ZLevel() uint { return t.Z } -//ZRes takes a web mercator zoom level and returns the pixel resolution for that +// ZRes takes a web mercator zoom level and returns the pixel resolution for that // scale, assuming t.Extent x t.Extent pixel tiles. Non-integer zoom levels are accepted. // ported from: https://raw.githubusercontent.com/mapbox/postgis-vt-util/master/postgis-vt-util.sql // 40075016.6855785 is the equator in meters for WGS84 at z=0 From 1201df2c52ac8cf572d5d6150dafaab6ae8c1889 Mon Sep 17 00:00:00 2001 From: Sergei Gureev Date: Tue, 15 Nov 2022 15:59:04 +0200 Subject: [PATCH 8/8] feat: disable caching for maps with custom params --- atlas/atlas.go | 8 ++++ cache/cache.go | 1 + config/config.go | 13 ++++++ server/middleware_tile_cache.go | 6 +++ server/middleware_tile_cache_test.go | 60 ++++++++++++++++++++++++++++ 5 files changed, 88 insertions(+) diff --git a/atlas/atlas.go b/atlas/atlas.go index 06677c651..5ea55e0a5 100644 --- a/atlas/atlas.go +++ b/atlas/atlas.go @@ -121,6 +121,10 @@ func (a *Atlas) SeedMapTile(ctx context.Context, m Map, z, x, y uint) error { return defaultAtlas.SeedMapTile(ctx, m, z, x, y) } + if len(m.Params) > 0 { + return nil + } + ctx = context.WithValue(ctx, observability.ObserveVarMapName, m.Name) // confirm we have a cache backend if a.cacher == nil { @@ -154,6 +158,10 @@ func (a *Atlas) PurgeMapTile(m Map, tile *tegola.Tile) error { return defaultAtlas.PurgeMapTile(m, tile) } + if len(m.Params) > 0 { + return nil + } + if a.cacher == nil { return ErrMissingCache } diff --git a/cache/cache.go b/cache/cache.go index d5d7609b6..a32e1b1cc 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -38,6 +38,7 @@ func ParseKey(str string) (*Key, error) { // remove the base-path and the first slash, then split the parts keyParts := strings.Split(strings.TrimLeft(str, "/"), "/") + // we're expecting a z/x/y scheme if len(keyParts) < 3 || len(keyParts) > 5 { err = ErrInvalidFileKeyParts{ diff --git a/config/config.go b/config/config.go index e4dd0ca48..aef1a70fa 100644 --- a/config/config.go +++ b/config/config.go @@ -196,6 +196,8 @@ func (c *Config) Validate() error { // check for map layer name / zoom collisions // map of layers to providers mapLayers := map[string]map[string]provider.MapLayer{} + // maps with configured parameters for logging + mapsWithCustomParams := []string{} for mapKey, m := range c.Maps { // validate any declared query parameters @@ -203,6 +205,10 @@ func (c *Config) Validate() error { return err } + if len(m.Parameters) > 0 { + mapsWithCustomParams = append(mapsWithCustomParams, string(m.Name)) + } + if _, ok := mapLayers[string(m.Name)]; !ok { mapLayers[string(m.Name)] = map[string]provider.MapLayer{} } @@ -300,6 +306,13 @@ func (c *Config) Validate() error { } } + if len(mapsWithCustomParams) > 0 { + log.Infof( + "Caching is disabled for these maps, since they have configured custom parameters: %s", + strings.Join(mapsWithCustomParams, ", "), + ) + } + // check for blacklisted headers for k := range c.Webserver.Headers { for _, v := range blacklistHeaders { diff --git a/server/middleware_tile_cache.go b/server/middleware_tile_cache.go index a187da91d..243f3d03e 100644 --- a/server/middleware_tile_cache.go +++ b/server/middleware_tile_cache.go @@ -28,6 +28,12 @@ func TileCacheHandler(a *atlas.Atlas, next http.Handler) http.Handler { return } + // ignore requests with query parameters + if r.URL.RawQuery != "" { + next.ServeHTTP(w, r) + return + } + // parse our URI into a cache key structure (remove any configured URIPrefix + "maps/" ) key, err := cache.ParseKey(strings.TrimPrefix(r.URL.Path, path.Join(URIPrefix, "maps"))) if err != nil { diff --git a/server/middleware_tile_cache_test.go b/server/middleware_tile_cache_test.go index c69b2ced7..7563839fe 100644 --- a/server/middleware_tile_cache_test.go +++ b/server/middleware_tile_cache_test.go @@ -79,3 +79,63 @@ func TestMiddlewareTileCacheHandler(t *testing.T) { t.Run(name, fn(tc)) } } + +func TestMiddlewareTileCacheHandlerIgnoreParams(t *testing.T) { + type tcase struct { + uri string + uriPrefix string + } + + fn := func(tc tcase) func(t *testing.T) { + return func(t *testing.T) { + var err error + + if tc.uriPrefix != "" { + server.URIPrefix = tc.uriPrefix + } else { + server.URIPrefix = "/" + } + + a := newTestMapWithLayers(testLayer1, testLayer2, testLayer3) + cacher, _ := memory.New(nil) + a.SetCache(cacher) + + w, router, err := doRequest(a, "GET", tc.uri, nil) + if err != nil { + t.Errorf("error making request, expected nil got %v", err) + return + } + + // we expect the cache to not being used + if w.Header().Get("Tegola-Cache") != "" { + t.Errorf("no header Tegola-Cache is expected, got %v", w.Header().Get("Tegola-Cache")) + return + } + + // play the request again + r, err := http.NewRequest("GET", tc.uri, nil) + if err != nil { + t.Errorf("error making request, expected nil got %v", err) + return + } + + w = httptest.NewRecorder() + router.ServeHTTP(w, r) + + if w.Header().Get("Tegola-Cache") != "" { + t.Errorf("no header Tegola-Cache is expected, got %v", w.Header().Get("Tegola-Cache")) + return + } + } + } + + tests := map[string]tcase{ + "map params": { + uri: "/maps/test-map/10/2/3.pbf?param=value", + }, + } + + for name, tc := range tests { + t.Run(name, fn(tc)) + } +}