From acddb57833f701e836b31b4aeabc2bcff2e94caa Mon Sep 17 00:00:00 2001 From: jieguangzhou Date: Wed, 14 Sep 2022 15:15:14 +0800 Subject: [PATCH] add sagemaker yaml example --- .../docs/source/tasks/sagemaker.rst | 12 ++++++++ .../examples/yaml_define/Sagemaker.yaml | 28 +++++++++++++++++++ .../yaml_define/example_sagemaker_params.json | 18 ++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Sagemaker.yaml create mode 100644 dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/example_sagemaker_params.json diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/sagemaker.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/sagemaker.rst index 96e7c6d4a098..36880d91d2fb 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/sagemaker.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/sagemaker.rst @@ -32,3 +32,15 @@ Dive Into --------- .. automodule:: pydolphinscheduler.tasks.sagemaker + +YAML file example +----------------- + +.. literalinclude:: ../../../examples/yaml_define/Sagemaker.yaml + :start-after: # under the License. + :language: yaml + +example_sagemaker_params.json: + +.. literalinclude:: ../../../examples/yaml_define/example_sagemaker_params.json + :language: json diff --git a/dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Sagemaker.yaml b/dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Sagemaker.yaml new file mode 100644 index 000000000000..9f77a3caa8ea --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Sagemaker.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Define the workflow +workflow: + name: "Sagemaker" + release_state: "offline" + +# Define the tasks under the process +tasks: + - name: sagemaker + task_type: Sagemaker + sagemaker_request_json: $FILE{"example_sagemaker_params.json"} + diff --git a/dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/example_sagemaker_params.json b/dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/example_sagemaker_params.json new file mode 100644 index 000000000000..94033203558f --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/example_sagemaker_params.json @@ -0,0 +1,18 @@ +{ + "ParallelismConfiguration":{ + "MaxParallelExecutionSteps":1 + }, + "PipelineExecutionDescription":"run pipeline using ds", + "PipelineExecutionDisplayName":"ds-sagemaker-pipeline", + "PipelineName":"DsSagemakerPipeline", + "PipelineParameters":[ + { + "Name":"InputData", + "Value": "s3://sagemaker/dataset/dataset.csv" + }, + { + "Name":"InferenceData", + "Value": "s3://sagemaker/dataset/inference.csv" + } + ] +}