This repository has been archived by the owner on Jul 8, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 49
/
anonymize.json
executable file
·100 lines (100 loc) · 4.48 KB
/
anonymize.json
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
{
"setup":{
"license":"Licensed under the Apache License, Version 2.0",
"copyright":"Copyright 2020 Google LLC"
},
"tasks":[
{ "dataset":{
"auth":"service",
"dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to read from.", "prefix":"Test_Anonymize_From_", "default":"Manual"}},
"clear":true
}},
{ "dataset":{
"auth":"service",
"dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to write to.", "prefix":"Test_Anonymize_To_", "default":"Manual" }},
"clear":true
}},
{ "bigquery": {
"auth":"service",
"from":{
"values":[
["2018-02-27", "dog", 7, 67],
["2018-03-01", "cat", 5, 1.5],
["2018-03-02", "bird", 12, 0.44],
["2018-03-03", "lizard", 22, 1],
["2018-03-04", "dinosaur", 1600, 273.97]
]
},
"to":{
"dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to read from.", "prefix":"Test_Anonymize_From_", "default":"Manual"}},
"table":"Anonymize_Table"
},
"schema":[
{ "name":"Date_Column", "type":"DATE", "mode": "NULLABLE" },
{ "name":"String_Column", "type":"STRING", "mode": "NULLABLE" },
{ "name":"Integer_Column", "type":"INTEGER", "mode": "NULLABLE" },
{ "name":"Float_Column", "type":"FLOAT", "mode": "NULLABLE" }
]
}},
{ "bigquery": {
"auth":"service",
"from":{
"dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to read from.", "prefix":"Test_Anonymize_From_", "default":"Manual"}},
"query":"SELECT * FROM {dataset_from}.Anonymize_Table;",
"parameters": {
"dataset_from": {"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to read from.", "prefix":"Test_Anonymize_From_", "default":"Manual"}}
}
},
"to":{
"dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to read from.", "prefix":"Test_Anonymize_From_", "default":"Manual"}},
"view":"Anonymize_View"
}
}},
{ "include":{
"script":"scripts/anonymize.json",
"parameters":{
"auth":"service",
"from_project":{"field":{"name":"project", "kind":"string", "description":"Project where BigQuery dataset is read from."}},
"from_dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to read from.", "prefix":"Test_Anonymize_From_", "default":"Manual"}},
"to_project":{"field":{ "name":"project", "kind":"string", "description":"Project where BigQuery dataset is created."}},
"to_dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to write to.", "prefix":"Test_Anonymize_To_", "default":"Manual" }}
}
}},
{ "test": {
"auth":"service",
"bigquery":{
"dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to write to.", "prefix":"Test_Anonymize_To_", "default":"Manual" }},
"table":"Anonymize_Table",
"schema":[
{ "name":"Date_Column", "type":"DATE", "mode": "NULLABLE" },
{ "name":"String_Column", "type":"STRING", "mode": "NULLABLE" },
{ "name":"Integer_Column", "type":"INTEGER", "mode": "NULLABLE" },
{ "name":"Float_Column", "type":"FLOAT", "mode": "NULLABLE" }
]
}
}},
{ "test": {
"auth":"service",
"bigquery":{
"dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to read from.", "prefix":"Test_Anonymize_From_", "default":"Manual"}},
"table":"Anonymize_View",
"schema":[
{ "name":"Date_Column", "type":"DATE", "mode": "NULLABLE" },
{ "name":"String_Column", "type":"STRING", "mode": "NULLABLE" },
{ "name":"Integer_Column", "type":"INTEGER", "mode": "NULLABLE" },
{ "name":"Float_Column", "type":"FLOAT", "mode": "NULLABLE" }
]
}
}},
{ "dataset":{
"auth":"service",
"dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to read from.", "prefix":"Test_Anonymize_From_", "default":"Manual"}},
"delete":true
}},
{ "dataset":{
"auth":"service",
"dataset":{"field":{ "name":"test_run_id", "kind":"string", "description":"The dataset to read from.", "prefix":"Test_Anonymize_To_", "default":"Manual"}},
"delete":true
}}
]
}