forked from GoogleCloudPlatform/cloudml-samples
-
Notifications
You must be signed in to change notification settings - Fork 0
/
task.py
389 lines (341 loc) · 12.2 KB
/
task.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
#!/usr/bin/env python
# Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import argparse
from datetime import datetime
import tensorflow as tf
import metadata
import input
import model
# ******************************************************************************
# YOU MAY MODIFY THIS FUNCTION TO ADD/REMOVE PARAMS OR CHANGE THE DEFAULT VALUES
# ******************************************************************************
def initialise_hyper_params(args_parser):
"""
Define the arguments with the default values,
parses the arguments passed to the task,
and set the HYPER_PARAMS global variable
Args:
args_parser
"""
# Data files arguments
args_parser.add_argument(
'--train-files',
help='GCS or local paths to training data',
nargs='+',
required=True
)
args_parser.add_argument(
'--eval-files',
help='GCS or local paths to evaluation data',
nargs='+',
required=True
)
args_parser.add_argument(
'--feature-stats-file',
help='GCS or local paths to feature statistics json file',
nargs='+',
default=None
)
###########################################
# Experiment arguments - training
args_parser.add_argument(
'--train-steps',
help="""
Steps to run the training job for. If --num-epochs and --train-size are not specified,
this must be. Otherwise the training job will run indefinitely.
if --num-epochs and --train-size are specified, then --train-steps will be:
(train-size/train-batch-size) * num-epochs\
""",
default=1000,
type=int
)
args_parser.add_argument(
'--train-batch-size',
help='Batch size for each training step',
type=int,
default=200
)
args_parser.add_argument(
'--train-size',
help='Size of training set (instance count)',
type=int,
default=None
)
args_parser.add_argument(
'--num-epochs',
help="""\
Maximum number of training data epochs on which to train.
If both --train-size and --num-epochs are specified,
--train-steps will be: (train-size/train-batch-size) * num-epochs.\
""",
default=10,
type=int,
)
###########################################
# Experiment arguments - evaluation
args_parser.add_argument(
'--eval-every-secs',
help='How long to wait before running the next evaluation',
default=120,
type=int
)
args_parser.add_argument(
'--eval-steps',
help="""\
Number of steps to run evaluation for at each checkpoint',
Set to None to evaluate on the whole evaluation data
""",
default=None,
type=int
)
args_parser.add_argument(
'--eval-batch-size',
help='Batch size for evaluation steps',
type=int,
default=200
)
###########################################
# Features processing arguments
args_parser.add_argument(
'--num-buckets',
help='Number of buckets into which to discretize numeric columns',
default=10,
type=int
)
args_parser.add_argument(
'--embedding-size',
help='Number of embedding dimensions for categorical columns. value of 0 means no embedding',
default=4,
type=int
)
###########################################
# Estimator arguments
args_parser.add_argument(
'--learning-rate',
help="Learning rate value for the optimizers",
default=0.1,
type=float
)
args_parser.add_argument(
'--learning-rate-decay-factor',
help="""\
**VALID FOR CUSTOM MODELS**
The factor by which the learning rate should decay by the end of the training.
decayed_learning_rate = learning_rate * decay_rate ^ (global_step / decay_steps)
If set to 1.0 (default), then no decay will occur
If set to 0.5, then the learning rate should reach 0.5 of its original value at the end of the training.
Note that, decay_steps is set to train_steps\
""",
default=1.0,
type=float
)
args_parser.add_argument(
'--hidden-units',
help="""\
Hidden layer sizes to use for DNN feature columns, provided in comma-separated layers.
If --scale-factor > 0, then only the size of the first layer will be used to compute
the sizes of subsequent layers \
""",
default='30,30,30'
)
args_parser.add_argument(
'--layer-sizes-scale-factor',
help="""\
Determine how the size of the layers in the DNN decays.
If value = 0 then the provided --hidden-units will be taken as is\
""",
default=0.7,
type=float
)
args_parser.add_argument(
'--num-layers',
help='Number of layers in the DNN. If --scale-factor > 0, then this parameter is ignored',
default=4,
type=int
)
args_parser.add_argument(
'--dropout-prob',
help="The probability we will drop out a given coordinate",
default=None
)
args_parser.add_argument(
'--encode-one-hot',
help="""\
If set to True, the categorical columns will be encoded as One-Hot indicators in the deep part of the DNN model.
Otherwise, the categorical columns will only be used in the wide part of the DNN model
""",
action='store_true',
default=True,
)
args_parser.add_argument(
'--as-wide-columns',
help="""\
If set to True, the categorical columns will be used in the wide part of the DNN model
""",
action='store_true',
default=True,
)
###########################################
# Saved model arguments
args_parser.add_argument(
'--job-dir',
help='GCS location to write checkpoints and export models',
required=True
)
args_parser.add_argument(
'--reuse-job-dir',
action='store_true',
default=False,
help="""\
Flag to decide if the model checkpoint should
be re-used from the job-dir. If False then the
job-dir will be deleted"""
)
args_parser.add_argument(
'--export-format',
help='The input format of the exported SavedModel binary',
choices=['JSON', 'CSV', 'EXAMPLE'],
default='JSON'
)
###########################################
# Argument to turn on all logging
args_parser.add_argument(
'--verbosity',
choices=[
'DEBUG',
'ERROR',
'FATAL',
'INFO',
'WARN'
],
default='INFO',
)
return args_parser.parse_args()
# ******************************************************************************
# YOU NEED NOT TO CHANGE THE FUNCTION TO RUN THE EXPERIMENT
# ******************************************************************************
def run_experiment(run_config):
"""Train, evaluate, and export the model using tf.estimator.train_and_evaluate API"""
train_input_fn = input.generate_input_fn(
file_names_pattern=HYPER_PARAMS.train_files,
mode=tf.estimator.ModeKeys.TRAIN,
num_epochs=HYPER_PARAMS.num_epochs,
batch_size=HYPER_PARAMS.train_batch_size
)
eval_input_fn = input.generate_input_fn(
file_names_pattern=HYPER_PARAMS.eval_files,
mode=tf.estimator.ModeKeys.EVAL,
batch_size=HYPER_PARAMS.eval_batch_size
)
exporter = tf.estimator.FinalExporter(
'estimator',
input.SERVING_FUNCTIONS[HYPER_PARAMS.export_format],
as_text=False # change to true if you want to export the model as readable text
)
# compute the number of training steps based on num_epoch, train_size, and train_batch_size
if HYPER_PARAMS.train_size is not None and HYPER_PARAMS.num_epochs is not None:
train_steps = (HYPER_PARAMS.train_size / HYPER_PARAMS.train_batch_size) * \
HYPER_PARAMS.num_epochs
else:
train_steps = HYPER_PARAMS.train_steps
train_spec = tf.estimator.TrainSpec(
train_input_fn,
max_steps=int(train_steps)
)
eval_spec = tf.estimator.EvalSpec(
eval_input_fn,
steps=HYPER_PARAMS.eval_steps,
exporters=[exporter],
name='estimator-eval',
throttle_secs=HYPER_PARAMS.eval_every_secs,
)
print("* experiment configurations")
print("===========================")
print("Train size: {}".format(HYPER_PARAMS.train_size))
print("Epoch count: {}".format(HYPER_PARAMS.num_epochs))
print("Train batch size: {}".format(HYPER_PARAMS.train_batch_size))
print("Training steps: {} ({})".format(int(train_steps),
"supplied" if HYPER_PARAMS.train_size is None else "computed"))
print("Evaluate every {} seconds".format(HYPER_PARAMS.eval_every_secs))
print("===========================")
if metadata.TASK_TYPE == "classification":
estimator = model.create_classifier(
config=run_config
)
elif metadata.TASK_TYPE == "regression":
estimator = model.create_regressor(
config=run_config
)
else:
estimator = model.create_estimator(
config=run_config
)
# train and evaluate
tf.estimator.train_and_evaluate(
estimator,
train_spec,
eval_spec
)
# ******************************************************************************
# THIS IS ENTRY POINT FOR THE TRAINER TASK
# ******************************************************************************
def main():
print('')
print('Hyper-parameters:')
print(HYPER_PARAMS)
print('')
# Set python level verbosity
tf.logging.set_verbosity(HYPER_PARAMS.verbosity)
# Set C++ Graph Execution level verbosity
os.environ['TF_CPP_MIN_LOG_LEVEL'] = str(tf.logging.__dict__[HYPER_PARAMS.verbosity] / 10)
# Directory to store output model and checkpoints
model_dir = HYPER_PARAMS.job_dir
# If job_dir_reuse is False then remove the job_dir if it exists
print("Resume training:", HYPER_PARAMS.reuse_job_dir)
if not HYPER_PARAMS.reuse_job_dir:
if tf.gfile.Exists(HYPER_PARAMS.job_dir):
tf.gfile.DeleteRecursively(HYPER_PARAMS.job_dir)
print("Deleted job_dir {} to avoid re-use".format(HYPER_PARAMS.job_dir))
else:
print("No job_dir available to delete")
else:
print("Reusing job_dir {} if it exists".format(HYPER_PARAMS.job_dir))
run_config = tf.estimator.RunConfig(
tf_random_seed=19830610,
log_step_count_steps=1000,
save_checkpoints_secs=120, # change if you want to change frequency of saving checkpoints
keep_checkpoint_max=3,
model_dir=model_dir
)
run_config = run_config.replace(model_dir=model_dir)
print("Model Directory:", run_config.model_dir)
# Run the train and evaluate experiment
time_start = datetime.utcnow()
print("")
print("Experiment started at {}".format(time_start.strftime("%H:%M:%S")))
print(".......................................")
run_experiment(run_config)
time_end = datetime.utcnow()
print(".......................................")
print("Experiment finished at {}".format(time_end.strftime("%H:%M:%S")))
print("")
time_elapsed = time_end - time_start
print("Experiment elapsed time: {} seconds".format(time_elapsed.total_seconds()))
print("")
args_parser = argparse.ArgumentParser()
HYPER_PARAMS = initialise_hyper_params(args_parser)
if __name__ == '__main__':
main()