From 913e13d2133acb34a7f5dc44063ebf4050b9a474 Mon Sep 17 00:00:00 2001 From: Nathaniel Kofalt Date: Wed, 14 Mar 2018 15:53:34 -0500 Subject: [PATCH] Add context input type --- gears/generator.py | 45 ++++++++++++++++++++++++++++------- gears/invocation.example.json | 5 ++++ gears/manifest.example.json | 3 +++ spec/manifest.schema.json | 2 +- spec/readme.md | 28 +++++++++++++++++++--- validate.py | 18 ++++++++++---- 6 files changed, 85 insertions(+), 16 deletions(-) diff --git a/gears/generator.py b/gears/generator.py index d625d0f..4730c4e 100644 --- a/gears/generator.py +++ b/gears/generator.py @@ -94,18 +94,46 @@ def derive_invocation_schema(manifest): for kind in ['config', 'inputs']: for key in manifest[kind]: # Copy constraints, removing 'base' and 'description' keywords which are not constraints - val = copy.deepcopy(manifest[kind][key]) - val.pop('base', None) - val.pop('description', None) - optional = val.pop('optional', False) + value = copy.deepcopy(manifest[kind][key]) + value.pop('base', None) + value.pop('description', None) + optional = value.pop('optional', False) # The config map holds scalars, while the inputs map holds objects. if kind == 'config': - result['properties'][kind]['properties'][key] = val + result['properties'][kind]['properties'][key] = value else: - result['properties'][kind]['properties'][key] = {} - result['properties'][kind]['properties'][key]['properties'] = val - result['properties'][kind]['properties'][key]['type'] = 'object' + keyType = manifest[kind][key]['base'] + spec = {} + + if keyType == 'file' or keyType == 'api-key': + # Object with any particular properties (could be refined further) + spec = { + 'type': 'object', + 'properties': value, # copy over schema snippet from manifest + } + + elif keyType == 'context': + # Object with information about a lookup value + spec = { + 'type': 'object', + 'properties': { + 'base': { + 'type': 'string', + }, + 'found': { + 'type': 'boolean', + }, + 'value': { }, # Context inputs can have any type, or none at all + }, + 'required': [ 'base', 'found', 'value' ] + } + else: + # Whitelist input types + raise Exception("Unknown input type " + str(keyType)) + + # Save into result + result['properties'][kind]['properties'][key] = spec # Require the key be present unless optional flag is set. if not optional: @@ -165,3 +193,4 @@ def validate_invocation(manifest, invocation): inv_schema = derive_invocation_schema(manifest) jsonschema.validate(invocation, inv_schema) + return inv_schema diff --git a/gears/invocation.example.json b/gears/invocation.example.json index 91abe83..4bc619a 100644 --- a/gears/invocation.example.json +++ b/gears/invocation.example.json @@ -5,6 +5,11 @@ "inputs": { "dicom": { "type": "dicom" + }, + "matlab_license_code": { + "base": "context", + "found": true, + "value": "ABC" } } } diff --git a/gears/manifest.example.json b/gears/manifest.example.json index bd3aa7d..e805bee 100644 --- a/gears/manifest.example.json +++ b/gears/manifest.example.json @@ -19,6 +19,9 @@ "base": "file", "type": { "enum": [ "dicom" ] } + }, + "matlab_license_code": { + "base": "context" } }, "capabilities": [ diff --git a/spec/manifest.schema.json b/spec/manifest.schema.json index 62fd330..ed579f7 100644 --- a/spec/manifest.schema.json +++ b/spec/manifest.schema.json @@ -71,7 +71,7 @@ "properties": { "base": { "type": "string", - "enum": [ "file", "api-key" ], + "enum": [ "file", "api-key", "context" ], "description": "The type of gear input." }, "description": { diff --git a/spec/readme.md b/spec/readme.md index a2bc7e3..56aef8f 100644 --- a/spec/readme.md +++ b/spec/readme.md @@ -100,10 +100,10 @@ Note, the `// comments` shown below are not JSON syntax and cannot be included i }, "description": "A set of 3D coordinates." - } + }, }, - // Inputs (files) that the gear consumes + // Inputs that the gear consumes "inputs": { // A label - describes one of the inputs. Used by the user interface and by the run script. @@ -116,7 +116,13 @@ Note, the `// comments` shown below are not JSON syntax and cannot be included i "type": { "enum": [ "dicom" ] }, "description": "Any dicom file." - } + }, + + // A contextual key-value, provided by the environment. Used for values that are generally the same for an entire project. + // Not guaranteed to be found - the gear should decide if it can continue to run, or exit with an error. + "matlab_license_code": { + "base": "context", + }, }, // Capabilities the gear requires. Not necessary unless you need a specific feature. @@ -144,6 +150,16 @@ Like the inputs, you can add JSON schema constraints as desired. Please specify The example has named one config option, called "speed", which must be an integer between zero and three, and another called "coordinates", which must be a set of three floats. +### Contextual values + +Context inputs are values that are generally provided by the environment, rather than the human or process running the gear. These are generally values that are incidentally required rather than directly a part of the algorithm - for example, a license key. + +It is up to the gear executor to decide how (and if) context is provided. In the Flywheel system, the values can be provided by setting a `context.key-name` value on a container's metadata. For example, you could set `context.matlab_license_code: "AEX"` on the project, and then any gear running in that project with a context input called `matlab_license_code` would receive the value. + +Unlike a gear's config values, contexts are not guaranteed to exist _or_ have a specific type or format. It is up to the gear to decide if it can continue, or exit with an error, when a context value does not match what the gear expects. In the example config file below, note that the `found` key can be checked to determine if a value was provided by the environment. + +Because context values are not namespaced, it is suggested that you use a specific and descriptive name. The `matlab_license_code` example is a good, self-explanatory key that many gears could likely reuse. + ### The input folder When a gear is executed, an `input` folder will be created relative to the base folder. If a gear has anything previously existing in the `input` folder it will be removed at launch time. @@ -181,6 +197,12 @@ Inside the `/flywheel/v0` folder a `config.json` file will be provided with any "modality" : null, "size" : 2913379 } + }, + + "matlab_license_code": { + "base": "context", + "found": true, + "value": "ABC" } } } diff --git a/validate.py b/validate.py index c215db9..f7c452d 100644 --- a/validate.py +++ b/validate.py @@ -1,7 +1,17 @@ import gears -x = gears.load_json_from_file("gears/manifest.example.json") -gears.validate_manifest(x) +manifest = gears.load_json_from_file("gears/manifest.example.json") +invocation = gears.load_json_from_file("gears/invocation.example.json") -y = gears.load_json_from_file("gears/invocation.example.json") -gears.validate_invocation(x, y) +print "Validating manifest:" +print gears.format_json(manifest) +print +gears.validate_manifest(manifest) + +print "Validating invocation:" +print gears.format_json(invocation) +print +schema = gears.validate_invocation(manifest, invocation) + +print "Resulted in invocation schema:" +print gears.format_json(schema)