forked from smithoss/gonymizer
-
Notifications
You must be signed in to change notification settings - Fork 0
/
mapper.go
342 lines (286 loc) · 9.18 KB
/
mapper.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
package gonymizer
import (
"database/sql"
"encoding/json"
"errors"
"fmt"
"os"
"strings"
log "github.com/sirupsen/logrus"
)
// ProcessorDefinition is the processor data structure used to map database columns to their specified column processor.
type ProcessorDefinition struct {
Name string
// optional helpers
Max float64
Min float64
Variance float64
Comment string
}
// ColumnMapper is the data structure that contains all gonymizer required information for the specified column.
type ColumnMapper struct {
Comment string
TableSchema string
TableName string
ColumnName string
DataType string
ParentSchema string
ParentTable string
ParentColumn string
OrdinalPosition int
IsNullable bool
Processors []ProcessorDefinition
}
// DBMapper is the main structure for the map file JSON object and is used to map all database columns that will be
// anonymized.
type DBMapper struct {
DBName string
SchemaPrefix string
Seed int64
ColumnMaps []ColumnMapper
}
// ColumnMapper returns the address of the ColumnMapper object if it matches the given parameters otherwise it returns
// nil. Special cases exist for sharded schemas using the schema-prefix. See documentation for details.
func (dbMap DBMapper) ColumnMapper(schemaName, tableName, columnName string) *ColumnMapper {
// Some names may contain quotes if the name is a reserved word. For example tableName public.order would be a
// conflict with ORDER BY so PSQL will add quotes to the name. I.E. public."order". Remove the quotes so we can match
// whatever is in the map file.
if strings.Contains(schemaName, "\"") {
schemaName = strings.Replace(schemaName, "\"", "", -1)
}
if strings.Contains(tableName, "\"") {
tableName = strings.Replace(tableName, "\"", "", -1)
}
if strings.Contains(columnName, "\"") {
columnName = strings.Replace(columnName, "\"", "", -1)
}
for _, cmap := range dbMap.ColumnMaps {
//log.Infoln("dbMap.SchemaPrefix-> ", dbMap.SchemaPrefix)
//log.Infoln("schemaName-> ", schemaName)
if len(dbMap.SchemaPrefix) > 0 && strings.HasPrefix(schemaName, dbMap.SchemaPrefix) && cmap.TableName == tableName &&
cmap.ColumnName == columnName {
return &cmap
} else if cmap.TableSchema == schemaName && cmap.TableName == tableName && cmap.ColumnName == columnName {
return &cmap
}
}
return nil
}
// Validate is used to verify that a database map is complete and correct.
func (dbMap *DBMapper) Validate() error {
if len(dbMap.DBName) == 0 {
return errors.New("Expected non-empty DBName")
}
return nil
}
// GenerateConfigSkeleton will generate a column-map based on the supplied PGConfig and previously configured map file.
func GenerateConfigSkeleton(conf PGConfig, schemaPrefix string, schemas, excludeTables []string) (*DBMapper, error) {
var (
dbmap *DBMapper
columnMap []ColumnMapper
)
db, err := OpenDB(conf)
if err != nil {
log.Error(err)
return nil, err
}
dbmap = new(DBMapper)
dbmap.DBName = conf.DefaultDBName
dbmap.SchemaPrefix = schemaPrefix
columnMap = []ColumnMapper{}
if len(schemas) < 1 {
schemas = append(schemas, "public")
}
log.Info("Schemas to map: ", schemas)
for _, schema := range schemas {
log.Info("Mapping columns for schema: ", schema)
columnMap, err = mapColumns(db, columnMap, schemaPrefix, schema, excludeTables)
if err != nil {
return nil, err
}
}
dbmap.ColumnMaps = columnMap
return dbmap, nil
}
// WriteConfigSkeleton will save the supplied DBMap to filepath.
func WriteConfigSkeleton(dbmap *DBMapper, filepath string) error {
f, err := os.Create(filepath)
if err != nil {
log.Error("Failure to open file: ", err)
log.Error("filepath: ", filepath)
return err
}
defer f.Close()
jsonEncoder := json.NewEncoder(f)
jsonEncoder.SetIndent("", " ")
err = jsonEncoder.Encode(dbmap)
if err != nil {
log.Error(err)
log.Error("filepath", filepath)
return err
}
return nil
}
// LoadConfigSkeleton will load the column-map into memory for use in dumping, processing, and loading of SQL files.
func LoadConfigSkeleton(givenPathToFile string) (*DBMapper, error) {
pathToFile := givenPathToFile
f, err := os.Open(pathToFile)
if err != nil {
log.Error("Failure to open file: ", err)
log.Error("givenPathToFile: ", givenPathToFile)
log.Error("pathToFile: ", pathToFile)
return nil, err
}
defer f.Close()
jsonDecoder := json.NewDecoder(f)
dbmap := new(DBMapper)
err = jsonDecoder.Decode(dbmap)
if err != nil {
log.Error(err)
log.Error("givenPathToFile: ", givenPathToFile)
log.Error("pathToFile: ", pathToFile)
log.Error("f: ", f)
return nil, err
}
err = dbmap.Validate()
if err != nil {
log.Error(err)
log.Error("dbmap: ", dbmap)
return nil, err
}
return dbmap, nil
}
// findColumn searches the in-memory loaded column map using the specified parameters.
func findColumn(columns []ColumnMapper, columnName, tableName, schemaPrefix, schema,
dataType string) (col ColumnMapper) {
for _, col = range columns {
// Regular Column
if col.ColumnName == columnName && col.TableName == tableName && col.TableSchema == schema &&
col.DataType == dataType {
return col
// Sharded Column
} else if col.ColumnName == columnName && col.TableName == tableName && col.TableSchema == schemaPrefix+"*" &&
col.DataType == dataType {
return col
}
}
return ColumnMapper{}
}
// addColumn creates a ColumnMapper structure based on the input parameters.
func addColumn(columnName, tableName, schema, dataType string, ordinalPosition int,
isNullable bool) ColumnMapper {
col := ColumnMapper{}
col.Processors = []ProcessorDefinition{
{
Name: "Identity",
},
}
col.TableName = tableName
col.ColumnName = columnName
col.DataType = dataType
col.OrdinalPosition = ordinalPosition
col.IsNullable = isNullable
col.TableSchema = schema
return col
}
// mapColumns
func mapColumns(db *sql.DB, columns []ColumnMapper, schemaPrefix, schema string,
excludeTables []string) ([]ColumnMapper, error) {
var (
err error
rows *sql.Rows
prefixPresent bool
)
// Below is a high level state diagram based on the schema prefix and schema being supplied.
// empty = empty string or ""
// group_ = example schema prefix
// public = example schema name
// =====================================================
// Shard | Schema | Outcome
// -----------------------------------------------------
// empty | empty | Map All Schemas
// -----------------------------------------------------
// empty | public | Map only provided schema name
// -----------------------------------------------------
// group_ | empty | Invalid
// -----------------------------------------------------
// group_ | group | build single map for schema prefix
// -----------------------------------------------------
// group_ | public | Map only provided schema
// -----------------------------------------------------
prefixPresent = false
if len(schemaPrefix) == 0 && len(schema) == 0 {
log.Debug("Mapping all schemas")
rows, err = GetAllSchemaColumns(db)
} else if len(schemaPrefix) == 0 && len(schema) > 0 {
log.Debug("Mapping a single schema")
rows, err = GetSchemaColumnEquals(db, schema)
} else if schemaPrefix != "" && schema == "" {
// Invalid
return nil, errors.New("You cannot use SchemaPrefix option without a schema to map it to")
} else if strings.HasPrefix(schemaPrefix, schema) {
log.Debug("Mapping a schema with SchemaPrefix present")
prefixPresent = true
rows, err = GetSchemaColumnsLike(db, schemaPrefix)
} else {
log.Debug("Mapping a single schema")
rows, err = GetSchemaColumnEquals(db, schema)
}
defer rows.Close()
log.Debug("Iterating through rows and creating skeleton map")
for {
var (
tableCatalog string
tableSchema string
tableName string
columnName string
dataType string
ordinalPosition int
isNullable bool
exclude bool
col ColumnMapper
)
// Iterate through each row and add the columns
for rows.Next() {
err = rows.Scan(
&tableCatalog,
&tableSchema,
&tableName,
&columnName,
&dataType,
&ordinalPosition,
&isNullable,
)
// If we are working on a schema prefix, make sure to use the schema prefix + * as a name, otherwise empty
if prefixPresent {
tableSchema = schemaPrefix + "*"
} else {
schemaPrefix = ""
}
// check to see if table is in the list of skipped tables or data for the table (leave them out of map)
exclude = false
for _, item := range excludeTables {
schemaTableName := fmt.Sprintf("%s.%s", tableSchema, tableName)
if schemaTableName == item {
exclude = true
break
}
}
if exclude {
continue
}
// Search for columnName in columns, if the column exists in the dbmap leave as-is otherwise create a new one and
// add to the column map
col = findColumn(columns, columnName, tableName, schemaPrefix, schema, dataType)
if col.TableSchema == "" && col.ColumnName == "" {
col = addColumn(columnName, tableName, schema, dataType, ordinalPosition, isNullable)
// Continuously append into the column map (old and new together)
columns = append(columns, col)
}
}
if !rows.NextResultSet() {
break
}
}
return columns, err
}