diff --git a/samples/contrib/kubeflow-katib/README.md b/samples/contrib/kubeflow-katib/README.md deleted file mode 100644 index 791875e9d8..0000000000 --- a/samples/contrib/kubeflow-katib/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Kubeflow Katib Component Samples - -These samples demonstrate how to create a Kubeflow Pipeline using -[Katib](https://github.com/kubeflow/katib). -The source code for the Katib Pipeline component can be found -[here](../../../components/kubeflow/katib-launcher). - -Check the following examples: - -- Run Pipeline from Jupyter Notebook using Katib Experiment with - [random search algorithm and early stopping](early-stopping.ipynb). - -- Compile compressed YAML definition of the Pipeline using Katib Experiment with - [Kubeflow MPIJob and Horovod training container](mpi-job-horovod.py). diff --git a/samples/contrib/kubeflow-katib/mpi-job-horovod.py b/samples/contrib/kubeflow-katib/mpi-job-horovod.py deleted file mode 100644 index 5476677651..0000000000 --- a/samples/contrib/kubeflow-katib/mpi-job-horovod.py +++ /dev/null @@ -1,234 +0,0 @@ -# Kubeflow Pipeline with Katib component. - -# In this example you will create Katib Experiment using Bayesian optimization algorithm. -# As a Trial template you will use Kubeflow MPIJob with Horovod mnist training container. -# After that, you will compile a Kubeflow Pipeline with your Katib Experiment. -# Use Kubeflow Pipelines UI to upload the Pipeline and create the Run. - -# This Experiment is similar to this: https://github.com/kubeflow/katib/blob/master/examples/v1beta1/mpijob-horovod.yaml -# Check the training container source code here: https://github.com/kubeflow/mpi-operator/tree/master/examples/horovod. - -# Note: To run this example, your Kubernetes cluster should run MPIJob operator. -# Follow this guide to install MPIJob on your cluster: https://www.kubeflow.org/docs/components/training/mpi/ - -# Note: You have to install kfp>=1.1.1 SDK and kubeflow-katib>=0.10.1 SDK to run this example. - -import kfp -import kfp.dsl as dsl -from kfp import components - -from kubeflow.katib import ApiClient -from kubeflow.katib import V1beta1ExperimentSpec -from kubeflow.katib import V1beta1AlgorithmSpec -from kubeflow.katib import V1beta1AlgorithmSetting -from kubeflow.katib import V1beta1ObjectiveSpec -from kubeflow.katib import V1beta1ParameterSpec -from kubeflow.katib import V1beta1FeasibleSpace -from kubeflow.katib import V1beta1TrialTemplate -from kubeflow.katib import V1beta1TrialParameterSpec - - -@dsl.pipeline( - name="Launch Katib MPIJob Experiment", - description="An example to launch Katib Experiment with MPIJob" -) -def horovod_mnist_hpo(): - # Experiment name and namespace. - experiment_name = "mpi-horovod-mnist" - experiment_namespace = "anonymous" - - # Trial count specification. - max_trial_count = 6 - max_failed_trial_count = 3 - parallel_trial_count = 2 - - # Objective specification. - objective = V1beta1ObjectiveSpec( - type="minimize", - goal=0.01, - objective_metric_name="loss", - ) - - # Algorithm specification. - algorithm = V1beta1AlgorithmSpec( - algorithm_name="bayesianoptimization", - algorithm_settings=[ - V1beta1AlgorithmSetting( - name="random_state", - value="10" - ) - ] - ) - - # Experiment search space. - # In this example we tune learning rate and number of training steps. - parameters = [ - V1beta1ParameterSpec( - name="lr", - parameter_type="double", - feasible_space=V1beta1FeasibleSpace( - min="0.001", - max="0.003" - ), - ), - V1beta1ParameterSpec( - name="num-steps", - parameter_type="int", - feasible_space=V1beta1FeasibleSpace( - min="50", - max="150", - step="10" - ), - ), - ] - - # JSON template specification for the Trial's Worker Kubeflow MPIJob. - trial_spec = { - "apiVersion": "kubeflow.org/v1", - "kind": "MPIJob", - "spec": { - "slotsPerWorker": 1, - "cleanPodPolicy": "Running", - "mpiReplicaSpecs": { - "Launcher": { - "replicas": 1, - "template": { - "metadata": { - "annotations": { - "sidecar.istio.io/inject": "false" - } - }, - "spec": { - "containers": [ - { - "image": "docker.io/kubeflow/mpi-horovod-mnist", - "name": "mpi-launcher", - "command": [ - "mpirun" - ], - "args": [ - "-np", - "2", - "--allow-run-as-root", - "-bind-to", - "none", - "-map-by", - "slot", - "-x", - "LD_LIBRARY_PATH", - "-x", - "PATH", - "-mca", - "pml", - "ob1", - "-mca", - "btl", - "^openib", - "python", - "/examples/tensorflow_mnist.py", - "--lr", - "${trialParameters.learningRate}", - "--num-steps", - "${trialParameters.numberSteps}" - ], - "resources": { - "limits": { - "cpu": "500m", - "memory": "2Gi" - } - } - } - ] - } - } - }, - "Worker": { - "replicas": 2, - "template": { - "metadata": { - "annotations": { - "sidecar.istio.io/inject": "false" - } - }, - "spec": { - "containers": [ - { - "image": "docker.io/kubeflow/mpi-horovod-mnist", - "name": "mpi-worker", - "resources": { - "limits": { - "cpu": "500m", - "memory": "4Gi" - } - } - } - ] - } - } - } - } - } - } - - # Configure parameters for the Trial template. - trial_template = V1beta1TrialTemplate( - primary_pod_labels={ - "mpi-job-role": "launcher" - }, - primary_container_name="mpi-launcher", - success_condition='status.conditions.#(type=="Succeeded")#|#(status=="True")#', - failure_condition='status.conditions.#(type=="Failed")#|#(status=="True")#', - trial_parameters=[ - V1beta1TrialParameterSpec( - name="learningRate", - description="Learning rate for the training model", - reference="lr" - ), - V1beta1TrialParameterSpec( - name="numberSteps", - description="Number of training steps", - reference="num-steps" - ), - ], - trial_spec=trial_spec - ) - - # Create Experiment specification. - experiment_spec = V1beta1ExperimentSpec( - max_trial_count=max_trial_count, - max_failed_trial_count=max_failed_trial_count, - parallel_trial_count=parallel_trial_count, - objective=objective, - algorithm=algorithm, - parameters=parameters, - trial_template=trial_template - ) - - # Get the Katib launcher. - # Load component from the URL or from the file. - katib_experiment_launcher_op = components.load_component_from_url( - "https://raw.githubusercontent.com/kubeflow/pipelines/master/components/kubeflow/katib-launcher/component.yaml") - # katib_experiment_launcher_op = components.load_component_from_file( - # "../../../components/kubeflow/katib-launcher/component.yaml" - # ) - - # Katib launcher component. - # Experiment Spec should be serialized to a valid Kubernetes object. - # The Experiment is deleted after the Pipeline is finished. - op = katib_experiment_launcher_op( - experiment_name=experiment_name, - experiment_namespace=experiment_namespace, - experiment_spec=ApiClient().sanitize_for_serialization(experiment_spec), - experiment_timeout_minutes=60) - - # Output container to print the results. - dsl.ContainerOp( - name="best-hp", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo Best HyperParameters: %s" % op.output], - ) - - -if __name__ == "__main__": - kfp.compiler.Compiler().compile(horovod_mnist_hpo, __file__ + ".tar.gz") diff --git a/samples/katib/README.md b/samples/katib/README.md index 675e27727d..3804062322 100644 --- a/samples/katib/README.md +++ b/samples/katib/README.md @@ -1,23 +1,15 @@ -# Hyperparameter tuning using Katib +# Kubeflow Katib Component Samples -[Katib](https://github.com/kubeflow/katib) is a Kubernetes-based system for Hyperparameter Tuning and Neural Architecture Search. This pipeline demonstrates how to use the Katib component to find the hyperparameter tuning results. +The sample demonstrate how to create a Kubeflow Pipeline using +[Katib](https://github.com/kubeflow/katib). +The source code for the Katib Pipeline component can be found +[here](https://github.com/kubeflow/pipelines/blob/master/components/kubeflow/katib-launcher/src/launch_experiment.py). -This pipeline uses the [MNIST dataset](http://yann.lecun.com/exdb/mnist/) to train the model and try to find the best hyperparameters using random search. Learning rate, number of convolutional layers, and the optimizer are the training parameters we want to search. Once the pipeline is completed, the ending step will print out the best hyperparameters in this pipeline experiment and clean up the workspace. +Check the following examples: -## Prerequisites -- Install [KFP Tekton prerequisites](/samples/README.md) +- Run Pipeline from Jupyter Notebook using Katib Experiment with + [random search algorithm and early stopping](early-stopping.ipynb). -## Instructions +Please note that katib currently does not support k8s version 1.19.x, +it is currently being worked on at https://github.com/kubeflow/katib/pull/1450. -1. First, go to the Kubeflow dashboard and create a user namespace. The Kubeflow dashboard is the endpoint to your istio-ingressgateway. We will be using the namespace `anonymous` for this example. - -2. Compile the Katib pipeline. The kfp-tekton SDK will produce a Tekton pipeline yaml definition in the same directory called `katib.yaml`. -```shell -python katib.py -``` - -3. Next, upload the `katib.yaml` file to the Kubeflow pipeline dashboard with Tekton Backend to run this pipeline. This pipeline will run for 10 to 15 minutes. - -## Acknowledgements - -Thanks [Hougang Liu](https://github.com/hougangliu) for creating the original katib example. diff --git a/samples/contrib/kubeflow-katib/early-stopping.ipynb b/samples/katib/early-stopping.ipynb similarity index 66% rename from samples/contrib/kubeflow-katib/early-stopping.ipynb rename to samples/katib/early-stopping.ipynb index 02ce4cd935..5c1950bf5c 100644 --- a/samples/contrib/kubeflow-katib/early-stopping.ipynb +++ b/samples/katib/early-stopping.ipynb @@ -38,162 +38,194 @@ "text": [ "Defaulting to user installation because normal site-packages is not writeable\n", "Collecting pip\n", - " Downloading pip-20.3-py2.py3-none-any.whl (1.5 MB)\n", - "\u001b[K |████████████████████████████████| 1.5 MB 30.4 MB/s eta 0:00:01\n", + " Downloading pip-21.0.1-py3-none-any.whl (1.5 MB)\n", + "\u001b[K |████████████████████████████████| 1.5 MB 42 kB/s eta 0:00:012\n", "\u001b[?25hInstalling collected packages: pip\n", "\u001b[33m WARNING: The scripts pip, pip3 and pip3.6 are installed in '/home/jovyan/.local/bin' which is not on PATH.\n", " Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n", - "Successfully installed pip-20.3\n", - "\u001b[33mWARNING: You are using pip version 20.0.2; however, version 20.3 is available.\n", + "Successfully installed pip-21.0.1\n", + "\u001b[33mWARNING: You are using pip version 20.0.2; however, version 21.0.1 is available.\n", "You should consider upgrading via the '/usr/local/bin/python -m pip install --upgrade pip' command.\u001b[0m\n", "Defaulting to user installation because normal site-packages is not writeable\n", "Collecting kfp==1.1.1\n", " Downloading kfp-1.1.1.tar.gz (162 kB)\n", - "\u001b[K |████████████████████████████████| 162 kB 16.2 MB/s eta 0:00:01\n", + "\u001b[K |████████████████████████████████| 162 kB 90 kB/s eta 0:00:011\n", "\u001b[?25hRequirement already satisfied: PyYAML in /usr/local/lib/python3.6/dist-packages (from kfp==1.1.1) (5.3)\n", "Requirement already satisfied: google-cloud-storage>=1.13.0 in /usr/local/lib/python3.6/dist-packages (from kfp==1.1.1) (1.25.0)\n", "Requirement already satisfied: kubernetes<12.0.0,>=8.0.0 in /usr/local/lib/python3.6/dist-packages (from kfp==1.1.1) (10.0.1)\n", "Requirement already satisfied: google-auth>=1.6.1 in /usr/local/lib/python3.6/dist-packages (from kfp==1.1.1) (1.11.0)\n", - "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.6/dist-packages (from kfp==1.1.1) (1.2.2)\n", - "Requirement already satisfied: jsonschema>=3.0.1 in /usr/local/lib/python3.6/dist-packages (from kfp==1.1.1) (3.2.0)\n", + "\u001b[33mWARNING: Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ReadTimeoutError(\"HTTPSConnectionPool(host='pypi.org', port=443): Read timed out. (read timeout=15)\",)': /simple/requests-toolbelt/\u001b[0m\n", + "Collecting requests_toolbelt>=0.8.0\n", + " Downloading requests_toolbelt-0.9.1-py2.py3-none-any.whl (54 kB)\n", + "\u001b[K |████████████████████████████████| 54 kB 202 kB/s eta 0:00:011\n", + "\u001b[?25hRequirement already satisfied: cloudpickle in /usr/local/lib/python3.6/dist-packages (from kfp==1.1.1) (1.2.2)\n", + "\u001b[33mWARNING: Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProtocolError('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))': /simple/kfp-server-api/\u001b[0m\n", + "Collecting kfp-server-api<2.0.0,>=0.2.5\n", + " Downloading kfp-server-api-1.4.1.tar.gz (50 kB)\n", + "\u001b[K |████████████████████████████████| 50 kB 211 kB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: jsonschema>=3.0.1 in /usr/local/lib/python3.6/dist-packages (from kfp==1.1.1) (3.2.0)\n", + "Collecting tabulate\n", + " Downloading tabulate-0.8.9-py3-none-any.whl (25 kB)\n", "Collecting click\n", " Downloading click-7.1.2-py2.py3-none-any.whl (82 kB)\n", - "\u001b[K |████████████████████████████████| 82 kB 1.2 MB/s eta 0:00:01\n", + "\u001b[K |████████████████████████████████| 82 kB 139 kB/s eta 0:00:01\n", "\u001b[?25hCollecting Deprecated\n", - " Downloading Deprecated-1.2.10-py2.py3-none-any.whl (8.7 kB)\n", - "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.6/dist-packages (from Deprecated->kfp==1.1.1) (1.11.2)\n", + " Downloading Deprecated-1.2.11-py2.py3-none-any.whl (9.1 kB)\n", + "Collecting strip-hints\n", + " Downloading strip-hints-0.1.9.tar.gz (30 kB)\n", "Collecting docstring-parser>=0.7.3\n", " Downloading docstring_parser-0.7.3.tar.gz (13 kB)\n", " Installing build dependencies ... \u001b[?25ldone\n", "\u001b[?25h Getting requirements to build wheel ... \u001b[?25ldone\n", "\u001b[?25h Preparing wheel metadata ... \u001b[?25ldone\n", - "\u001b[?25hRequirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (0.2.8)\n", + "\u001b[?25hCollecting kfp-pipeline-spec<0.2.0,>=0.1.0\n", + " Downloading kfp_pipeline_spec-0.1.6-py3-none-any.whl (26 kB)\n", + "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (4.0.0)\n", + "Requirement already satisfied: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (4.0)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (0.2.8)\n", "Requirement already satisfied: six>=1.9.0 in /usr/lib/python3/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (1.11.0)\n", "Requirement already satisfied: setuptools>=40.3.0 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (45.1.0)\n", - "Requirement already satisfied: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (4.0)\n", - "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (4.0.0)\n", - "Requirement already satisfied: google-auth>=1.6.1 in /usr/local/lib/python3.6/dist-packages (from kfp==1.1.1) (1.11.0)\n", "Requirement already satisfied: google-resumable-media<0.6dev,>=0.5.0 in /usr/local/lib/python3.6/dist-packages (from google-cloud-storage>=1.13.0->kfp==1.1.1) (0.5.0)\n", "Requirement already satisfied: google-cloud-core<2.0dev,>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from google-cloud-storage>=1.13.0->kfp==1.1.1) (1.3.0)\n", "Requirement already satisfied: google-api-core<2.0.0dev,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.1.1) (1.16.0)\n", "Requirement already satisfied: pytz in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.1.1) (2019.3)\n", - "Requirement already satisfied: six>=1.9.0 in /usr/lib/python3/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (1.11.0)\n", - "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.1.1) (1.51.0)\n", - "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.1.1) (2.22.0)\n", - "Requirement already satisfied: setuptools>=40.3.0 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (45.1.0)\n", "Requirement already satisfied: protobuf>=3.4.0 in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.1.1) (3.11.2)\n", - "Requirement already satisfied: google-auth>=1.6.1 in /usr/local/lib/python3.6/dist-packages (from kfp==1.1.1) (1.11.0)\n", - "Requirement already satisfied: six>=1.9.0 in /usr/lib/python3/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (1.11.0)\n", - "Requirement already satisfied: protobuf>=3.4.0 in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.1.1) (3.11.2)\n", - "Requirement already satisfied: six>=1.9.0 in /usr/lib/python3/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (1.11.0)\n", + "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.1.1) (2.22.0)\n", + "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.1.1) (1.51.0)\n", "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.6/dist-packages (from jsonschema>=3.0.1->kfp==1.1.1) (19.3.0)\n", - "Requirement already satisfied: setuptools>=40.3.0 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (45.1.0)\n", - "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.6/dist-packages (from jsonschema>=3.0.1->kfp==1.1.1) (1.4.0)\n", "Requirement already satisfied: pyrsistent>=0.14.0 in /usr/local/lib/python3.6/dist-packages (from jsonschema>=3.0.1->kfp==1.1.1) (0.15.7)\n", - "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata->jsonschema>=3.0.1->kfp==1.1.1) (2.1.0)\n", - "Collecting kfp-pipeline-spec<0.2.0,>=0.1.0\n", - " Downloading kfp_pipeline_spec-0.1.2-py3-none-any.whl (21 kB)\n", - "Collecting kfp-server-api<2.0.0,>=0.2.5\n", - " Downloading kfp-server-api-1.0.4.tar.gz (51 kB)\n", - "\u001b[K |████████████████████████████████| 51 kB 945 kB/s eta 0:00:01\n", + "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.6/dist-packages (from jsonschema>=3.0.1->kfp==1.1.1) (1.4.0)\n", + "Collecting protobuf>=3.4.0\n", + " Downloading protobuf-3.15.3-cp36-cp36m-manylinux1_x86_64.whl (1.0 MB)\n", + "\u001b[K |████████████████████████████████| 1.0 MB 150 kB/s eta 0:00:01\n", "\u001b[?25hRequirement already satisfied: urllib3>=1.15 in /usr/local/lib/python3.6/dist-packages (from kfp-server-api<2.0.0,>=0.2.5->kfp==1.1.1) (1.25.8)\n", - "Requirement already satisfied: six>=1.9.0 in /usr/lib/python3/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (1.11.0)\n", "Requirement already satisfied: certifi in /usr/local/lib/python3.6/dist-packages (from kfp-server-api<2.0.0,>=0.2.5->kfp==1.1.1) (2019.11.28)\n", "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.6/dist-packages (from kfp-server-api<2.0.0,>=0.2.5->kfp==1.1.1) (2.8.1)\n", - "Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.6/dist-packages (from kubernetes<12.0.0,>=8.0.0->kfp==1.1.1) (1.3.0)\n", - "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.1.1) (2.22.0)\n", - "Requirement already satisfied: certifi in /usr/local/lib/python3.6/dist-packages (from kfp-server-api<2.0.0,>=0.2.5->kfp==1.1.1) (2019.11.28)\n", - "Requirement already satisfied: PyYAML in /usr/local/lib/python3.6/dist-packages (from kfp==1.1.1) (5.3)\n", - "Requirement already satisfied: urllib3>=1.15 in /usr/local/lib/python3.6/dist-packages (from kfp-server-api<2.0.0,>=0.2.5->kfp==1.1.1) (1.25.8)\n", - "Requirement already satisfied: setuptools>=40.3.0 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (45.1.0)\n", - "Requirement already satisfied: google-auth>=1.6.1 in /usr/local/lib/python3.6/dist-packages (from kfp==1.1.1) (1.11.0)\n", "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /usr/local/lib/python3.6/dist-packages (from kubernetes<12.0.0,>=8.0.0->kfp==1.1.1) (0.57.0)\n", - "Requirement already satisfied: six>=1.9.0 in /usr/lib/python3/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (1.11.0)\n", - "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.6/dist-packages (from kfp-server-api<2.0.0,>=0.2.5->kfp==1.1.1) (2.8.1)\n", - "Requirement already satisfied: six>=1.9.0 in /usr/lib/python3/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (1.11.0)\n", - "Requirement already satisfied: setuptools>=40.3.0 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (45.1.0)\n", + "Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.6/dist-packages (from kubernetes<12.0.0,>=8.0.0->kfp==1.1.1) (1.3.0)\n", "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.6/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.6.1->kfp==1.1.1) (0.4.8)\n", - "Requirement already satisfied: six>=1.9.0 in /usr/lib/python3/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (1.11.0)\n", - "Requirement already satisfied: six>=1.9.0 in /usr/lib/python3/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (1.11.0)\n", "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.1.1) (3.0.4)\n", - "Requirement already satisfied: certifi in /usr/local/lib/python3.6/dist-packages (from kfp-server-api<2.0.0,>=0.2.5->kfp==1.1.1) (2019.11.28)\n", - "Requirement already satisfied: urllib3>=1.15 in /usr/local/lib/python3.6/dist-packages (from kfp-server-api<2.0.0,>=0.2.5->kfp==1.1.1) (1.25.8)\n", "Requirement already satisfied: idna<2.9,>=2.5 in /usr/lib/python3/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.1.1) (2.6)\n", - "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.1.1) (2.22.0)\n", + "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.6/dist-packages (from Deprecated->kfp==1.1.1) (1.11.2)\n", + "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata->jsonschema>=3.0.1->kfp==1.1.1) (2.1.0)\n", "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib->kubernetes<12.0.0,>=8.0.0->kfp==1.1.1) (3.1.0)\n", - "Collecting requests_toolbelt>=0.8.0\n", - " Downloading requests_toolbelt-0.9.1-py2.py3-none-any.whl (54 kB)\n", - "\u001b[K |████████████████████████████████| 54 kB 3.7 MB/s eta 0:00:01\n", - "\u001b[?25hRequirement already satisfied: requests<3.0.0dev,>=2.18.0 in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.1.1) (2.22.0)\n", - "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.6/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.6.1->kfp==1.1.1) (0.4.8)\n", - "Collecting strip-hints\n", - " Downloading strip-hints-0.1.9.tar.gz (30 kB)\n", "Requirement already satisfied: wheel in /usr/lib/python3/dist-packages (from strip-hints->kfp==1.1.1) (0.30.0)\n", - "Collecting tabulate\n", - " Downloading tabulate-0.8.7-py3-none-any.whl (24 kB)\n", - "Requirement already satisfied: six>=1.9.0 in /usr/lib/python3/dist-packages (from google-auth>=1.6.1->kfp==1.1.1) (1.11.0)\n", "Building wheels for collected packages: kfp, docstring-parser, kfp-server-api, strip-hints\n", " Building wheel for kfp (setup.py) ... \u001b[?25ldone\n", - "\u001b[?25h Created wheel for kfp: filename=kfp-1.1.1-py3-none-any.whl size=222518 sha256=a3a3530489f3ca42e25ad6a221909142725d94df20b45106775801e271807ecb\n", + "\u001b[?25h Created wheel for kfp: filename=kfp-1.1.1-py3-none-any.whl size=222518 sha256=c1527b2ca28aa9186e1812e399db6afe5d6432f70dc2f229eb5b7b4126363f56\n", " Stored in directory: /home/jovyan/.cache/pip/wheels/88/9f/b1/149db749eeb26a01ddc590b68d2b9c27a9c7eedb799f47d9f1\n", " Building wheel for docstring-parser (PEP 517) ... \u001b[?25ldone\n", - "\u001b[?25h Created wheel for docstring-parser: filename=docstring_parser-0.7.3-py3-none-any.whl size=19230 sha256=df43b4eb88d8f80a3c47f9aa2ecd07ddbfcbead920ed2697911659cd9f185a64\n", + "\u001b[?25h Created wheel for docstring-parser: filename=docstring_parser-0.7.3-py3-none-any.whl size=19230 sha256=5ab87eaca9414153748185c3c6fc5e40e08328dea033cbf632da102e3ea95bf3\n", " Stored in directory: /home/jovyan/.cache/pip/wheels/32/63/02/bb6eebc5261f10a6de3dcf26336a7b2b8b8dc8cacb6c00f75f\n", " Building wheel for kfp-server-api (setup.py) ... \u001b[?25ldone\n", - "\u001b[?25h Created wheel for kfp-server-api: filename=kfp_server_api-1.0.4-py3-none-any.whl size=105034 sha256=74c422f07d14ea87395dc7d3287076bf79f9391bcbc076171ea7afeb43ed77bc\n", - " Stored in directory: /home/jovyan/.cache/pip/wheels/0f/50/2d/28c9ae498b2e5ff5bf5a9765bca3dcd08ab4a79333670de6cb\n", + "\u001b[?25h Created wheel for kfp-server-api: filename=kfp_server_api-1.4.1-py3-none-any.whl size=93183 sha256=7aeb9a0410ee528a3a715e266f7f8098396703f4bbead428ca09db83d31d0aaf\n", + " Stored in directory: /home/jovyan/.cache/pip/wheels/9f/f2/77/c453e44646d30d61924641a0c247ffbefe2878f2f63f1ad2ca\n", " Building wheel for strip-hints (setup.py) ... \u001b[?25ldone\n", - "\u001b[?25h Created wheel for strip-hints: filename=strip_hints-0.1.9-py2.py3-none-any.whl size=24671 sha256=33ada8d9d7a77196257d7b1205856c86ae7390df64b65dba4911c42895b568e4\n", + "\u001b[?25h Created wheel for strip-hints: filename=strip_hints-0.1.9-py2.py3-none-any.whl size=24671 sha256=e9ebec364c0f51c11e9377b9c3624a5ce778899d500be72f551e3e355c72901f\n", " Stored in directory: /home/jovyan/.cache/pip/wheels/21/6d/fa/7ed7c0560e1ef39ebabd5cc0241e7fca711660bae1ad752e2b\n", "Successfully built kfp docstring-parser kfp-server-api strip-hints\n", - "Installing collected packages: tabulate, strip-hints, requests-toolbelt, kfp-server-api, kfp-pipeline-spec, docstring-parser, Deprecated, click, kfp\n", + "Installing collected packages: protobuf, tabulate, strip-hints, requests-toolbelt, kfp-server-api, kfp-pipeline-spec, docstring-parser, Deprecated, click, kfp\n", "\u001b[33m WARNING: The script tabulate is installed in '/home/jovyan/.local/bin' which is not on PATH.\n", " Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n", "\u001b[33m WARNING: The script strip-hints is installed in '/home/jovyan/.local/bin' which is not on PATH.\n", " Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n", "\u001b[33m WARNING: The scripts dsl-compile, dsl-compile-v2 and kfp are installed in '/home/jovyan/.local/bin' which is not on PATH.\n", " Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n", - "Successfully installed Deprecated-1.2.10 click-7.1.2 docstring-parser-0.7.3 kfp-1.1.1 kfp-pipeline-spec-0.1.2 kfp-server-api-1.0.4 requests-toolbelt-0.9.1 strip-hints-0.1.9 tabulate-0.8.7\n", + "Successfully installed Deprecated-1.2.11 click-7.1.2 docstring-parser-0.7.3 kfp-1.1.1 kfp-pipeline-spec-0.1.6 kfp-server-api-1.4.1 protobuf-3.15.3 requests-toolbelt-0.9.1 strip-hints-0.1.9 tabulate-0.8.9\n", "Defaulting to user installation because normal site-packages is not writeable\n", "Collecting kubeflow-katib==0.10.1\n", " Downloading kubeflow_katib-0.10.1-py3-none-any.whl (113 kB)\n", - "\u001b[K |████████████████████████████████| 113 kB 30.1 MB/s eta 0:00:01\n", - "\u001b[?25hRequirement already satisfied: kubernetes==10.0.1 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (10.0.1)\n", - "Requirement already satisfied: urllib3>=1.15.1 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (1.25.8)\n", + "\u001b[K |████████████████████████████████| 113 kB 16.4 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: certifi>=14.05.14 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (2019.11.28)\n", "Requirement already satisfied: setuptools>=21.0.0 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (45.1.0)\n", - "Requirement already satisfied: certifi>=14.05.14 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (2019.11.28)\n", - "Requirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (2.8.1)\n", "Requirement already satisfied: six>=1.10 in /usr/lib/python3/dist-packages (from kubeflow-katib==0.10.1) (1.11.0)\n", - "Requirement already satisfied: six>=1.10 in /usr/lib/python3/dist-packages (from kubeflow-katib==0.10.1) (1.11.0)\n", - "Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.6/dist-packages (from kubernetes==10.0.1->kubeflow-katib==0.10.1) (1.3.0)\n", + "Requirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (2.8.1)\n", + "Collecting table-logger>=0.3.5\n", + " Downloading table_logger-0.3.6-py3-none-any.whl (14 kB)\n", + "Requirement already satisfied: kubernetes==10.0.1 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (10.0.1)\n", "Requirement already satisfied: urllib3>=1.15.1 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (1.25.8)\n", - "Requirement already satisfied: setuptools>=21.0.0 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (45.1.0)\n", "Requirement already satisfied: pyyaml>=3.12 in /usr/local/lib/python3.6/dist-packages (from kubernetes==10.0.1->kubeflow-katib==0.10.1) (5.3)\n", - "Requirement already satisfied: certifi>=14.05.14 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (2019.11.28)\n", - "Requirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (2.8.1)\n", - "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /usr/local/lib/python3.6/dist-packages (from kubernetes==10.0.1->kubeflow-katib==0.10.1) (0.57.0)\n", - "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from kubernetes==10.0.1->kubeflow-katib==0.10.1) (2.22.0)\n", "Requirement already satisfied: google-auth>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from kubernetes==10.0.1->kubeflow-katib==0.10.1) (1.11.0)\n", - "Requirement already satisfied: six>=1.10 in /usr/lib/python3/dist-packages (from kubeflow-katib==0.10.1) (1.11.0)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from kubernetes==10.0.1->kubeflow-katib==0.10.1) (2.22.0)\n", + "Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.6/dist-packages (from kubernetes==10.0.1->kubeflow-katib==0.10.1) (1.3.0)\n", + "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /usr/local/lib/python3.6/dist-packages (from kubernetes==10.0.1->kubeflow-katib==0.10.1) (0.57.0)\n", + "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.0.1->kubernetes==10.0.1->kubeflow-katib==0.10.1) (4.0.0)\n", "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.0.1->kubernetes==10.0.1->kubeflow-katib==0.10.1) (0.2.8)\n", "Requirement already satisfied: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.0.1->kubernetes==10.0.1->kubeflow-katib==0.10.1) (4.0)\n", - "Requirement already satisfied: setuptools>=21.0.0 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (45.1.0)\n", - "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.0.1->kubernetes==10.0.1->kubeflow-katib==0.10.1) (4.0.0)\n", "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.6/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes==10.0.1->kubeflow-katib==0.10.1) (0.4.8)\n", - "Requirement already satisfied: six>=1.10 in /usr/lib/python3/dist-packages (from kubeflow-katib==0.10.1) (1.11.0)\n", - "Requirement already satisfied: certifi>=14.05.14 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (2019.11.28)\n", + "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from table-logger>=0.3.5->kubeflow-katib==0.10.1) (1.18.1)\n", "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->kubernetes==10.0.1->kubeflow-katib==0.10.1) (3.0.4)\n", "Requirement already satisfied: idna<2.9,>=2.5 in /usr/lib/python3/dist-packages (from requests->kubernetes==10.0.1->kubeflow-katib==0.10.1) (2.6)\n", - "Requirement already satisfied: urllib3>=1.15.1 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (1.25.8)\n", - "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from kubernetes==10.0.1->kubeflow-katib==0.10.1) (2.22.0)\n", "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib->kubernetes==10.0.1->kubeflow-katib==0.10.1) (3.1.0)\n", - "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.6/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes==10.0.1->kubeflow-katib==0.10.1) (0.4.8)\n", - "Collecting table-logger>=0.3.5\n", - " Downloading table_logger-0.3.6-py3-none-any.whl (14 kB)\n", - "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from table-logger>=0.3.5->kubeflow-katib==0.10.1) (1.18.1)\n", - "Requirement already satisfied: six>=1.10 in /usr/lib/python3/dist-packages (from kubeflow-katib==0.10.1) (1.11.0)\n", "Installing collected packages: table-logger, kubeflow-katib\n", - "Successfully installed kubeflow-katib-0.10.1 table-logger-0.3.6\n" + "Successfully installed kubeflow-katib-0.10.1 table-logger-0.3.6\n", + "Defaulting to user installation because normal site-packages is not writeable\n", + "Collecting kfp-tekton==0.4.0\n", + " Downloading kfp-tekton-0.4.0.tar.gz (38 kB)\n", + "Collecting kfp==1.0.4\n", + " Downloading kfp-1.0.4.tar.gz (116 kB)\n", + "\u001b[K |████████████████████████████████| 116 kB 6.7 MB/s eta 0:00:01\n", + "\u001b[?25hCollecting kubernetes==11.0.0\n", + " Downloading kubernetes-11.0.0-py3-none-any.whl (1.5 MB)\n", + "\u001b[K |████████████████████████████████| 1.5 MB 29.9 MB/s eta 0:00:01\n", + "\u001b[?25hRequirement already satisfied: PyYAML in /usr/local/lib/python3.6/dist-packages (from kfp==1.0.4->kfp-tekton==0.4.0) (5.3)\n", + "Requirement already satisfied: google-cloud-storage>=1.13.0 in /usr/local/lib/python3.6/dist-packages (from kfp==1.0.4->kfp-tekton==0.4.0) (1.25.0)\n", + "Requirement already satisfied: google-auth>=1.6.1 in /usr/local/lib/python3.6/dist-packages (from kfp==1.0.4->kfp-tekton==0.4.0) (1.11.0)\n", + "Requirement already satisfied: requests_toolbelt>=0.8.0 in ./.local/lib/python3.6/site-packages (from kfp==1.0.4->kfp-tekton==0.4.0) (0.9.1)\n", + "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.6/dist-packages (from kfp==1.0.4->kfp-tekton==0.4.0) (1.2.2)\n", + "Requirement already satisfied: kfp-server-api<2.0.0,>=0.2.5 in ./.local/lib/python3.6/site-packages (from kfp==1.0.4->kfp-tekton==0.4.0) (1.4.1)\n", + "Requirement already satisfied: jsonschema>=3.0.1 in /usr/local/lib/python3.6/dist-packages (from kfp==1.0.4->kfp-tekton==0.4.0) (3.2.0)\n", + "Requirement already satisfied: tabulate in ./.local/lib/python3.6/site-packages (from kfp==1.0.4->kfp-tekton==0.4.0) (0.8.9)\n", + "Requirement already satisfied: click in ./.local/lib/python3.6/site-packages (from kfp==1.0.4->kfp-tekton==0.4.0) (7.1.2)\n", + "Requirement already satisfied: Deprecated in ./.local/lib/python3.6/site-packages (from kfp==1.0.4->kfp-tekton==0.4.0) (1.2.11)\n", + "Requirement already satisfied: strip-hints in ./.local/lib/python3.6/site-packages (from kfp==1.0.4->kfp-tekton==0.4.0) (0.1.9)\n", + "Requirement already satisfied: urllib3>=1.24.2 in /usr/local/lib/python3.6/dist-packages (from kubernetes==11.0.0->kfp-tekton==0.4.0) (1.25.8)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from kubernetes==11.0.0->kfp-tekton==0.4.0) (2.22.0)\n", + "Requirement already satisfied: certifi>=14.05.14 in /usr/local/lib/python3.6/dist-packages (from kubernetes==11.0.0->kfp-tekton==0.4.0) (2019.11.28)\n", + "Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.6/dist-packages (from kubernetes==11.0.0->kfp-tekton==0.4.0) (1.3.0)\n", + "Requirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.6/dist-packages (from kubernetes==11.0.0->kfp-tekton==0.4.0) (2.8.1)\n", + "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /usr/local/lib/python3.6/dist-packages (from kubernetes==11.0.0->kfp-tekton==0.4.0) (0.57.0)\n", + "Requirement already satisfied: setuptools>=21.0.0 in /usr/local/lib/python3.6/dist-packages (from kubernetes==11.0.0->kfp-tekton==0.4.0) (45.1.0)\n", + "Requirement already satisfied: six>=1.9.0 in /usr/lib/python3/dist-packages (from kubernetes==11.0.0->kfp-tekton==0.4.0) (1.11.0)\n", + "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.6.1->kfp==1.0.4->kfp-tekton==0.4.0) (4.0.0)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.6.1->kfp==1.0.4->kfp-tekton==0.4.0) (0.2.8)\n", + "Requirement already satisfied: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.6.1->kfp==1.0.4->kfp-tekton==0.4.0) (4.0)\n", + "Requirement already satisfied: google-cloud-core<2.0dev,>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from google-cloud-storage>=1.13.0->kfp==1.0.4->kfp-tekton==0.4.0) (1.3.0)\n", + "Requirement already satisfied: google-resumable-media<0.6dev,>=0.5.0 in /usr/local/lib/python3.6/dist-packages (from google-cloud-storage>=1.13.0->kfp==1.0.4->kfp-tekton==0.4.0) (0.5.0)\n", + "Requirement already satisfied: google-api-core<2.0.0dev,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.0.4->kfp-tekton==0.4.0) (1.16.0)\n", + "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.0.4->kfp-tekton==0.4.0) (1.51.0)\n", + "Requirement already satisfied: protobuf>=3.4.0 in ./.local/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.0.4->kfp-tekton==0.4.0) (3.15.3)\n", + "Requirement already satisfied: pytz in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp==1.0.4->kfp-tekton==0.4.0) (2019.3)\n", + "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.6/dist-packages (from jsonschema>=3.0.1->kfp==1.0.4->kfp-tekton==0.4.0) (19.3.0)\n", + "Requirement already satisfied: pyrsistent>=0.14.0 in /usr/local/lib/python3.6/dist-packages (from jsonschema>=3.0.1->kfp==1.0.4->kfp-tekton==0.4.0) (0.15.7)\n", + "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.6/dist-packages (from jsonschema>=3.0.1->kfp==1.0.4->kfp-tekton==0.4.0) (1.4.0)\n", + "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.6/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.6.1->kfp==1.0.4->kfp-tekton==0.4.0) (0.4.8)\n", + "Requirement already satisfied: idna<2.9,>=2.5 in /usr/lib/python3/dist-packages (from requests->kubernetes==11.0.0->kfp-tekton==0.4.0) (2.6)\n", + "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->kubernetes==11.0.0->kfp-tekton==0.4.0) (3.0.4)\n", + "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.6/dist-packages (from Deprecated->kfp==1.0.4->kfp-tekton==0.4.0) (1.11.2)\n", + "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata->jsonschema>=3.0.1->kfp==1.0.4->kfp-tekton==0.4.0) (2.1.0)\n", + "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib->kubernetes==11.0.0->kfp-tekton==0.4.0) (3.1.0)\n", + "Requirement already satisfied: wheel in /usr/lib/python3/dist-packages (from strip-hints->kfp==1.0.4->kfp-tekton==0.4.0) (0.30.0)\n", + "Building wheels for collected packages: kfp-tekton, kfp\n", + " Building wheel for kfp-tekton (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for kfp-tekton: filename=kfp_tekton-0.4.0-py3-none-any.whl size=32814 sha256=ad561c5904811e0bc21faa694abcfd9d679b32ced94c9ab98b34105364183ade\n", + " Stored in directory: /home/jovyan/.cache/pip/wheels/41/f0/6d/bfbc4f2e2321e94795a76942a6f14ed7b5d3d797d6ef9a490b\n", + " Building wheel for kfp (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for kfp: filename=kfp-1.0.4-py3-none-any.whl size=160907 sha256=34188dc0c2538baddacc37015a26a14127ea98ccddb0da3e426326c639cb0a84\n", + " Stored in directory: /home/jovyan/.cache/pip/wheels/e5/a2/42/9bbf340f69f38f67e7c26434cf0e7cd68c3bd4453bf57ece5e\n", + "Successfully built kfp-tekton kfp\n", + "Installing collected packages: kubernetes, kfp, kfp-tekton\n", + " Attempting uninstall: kfp\n", + " Found existing installation: kfp 1.1.1\n", + " Uninstalling kfp-1.1.1:\n", + " Successfully uninstalled kfp-1.1.1\n", + "\u001b[33m WARNING: The scripts dsl-compile and kfp are installed in '/home/jovyan/.local/bin' which is not on PATH.\n", + " Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n", + "\u001b[33m WARNING: The script dsl-compile-tekton is installed in '/home/jovyan/.local/bin' which is not on PATH.\n", + " Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n", + "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "kubeflow-katib 0.10.1 requires kubernetes==10.0.1, but you have kubernetes 11.0.0 which is incompatible.\u001b[0m\n", + "Successfully installed kfp-1.0.4 kfp-tekton-0.4.0 kubernetes-11.0.0\n" ] } ], @@ -201,7 +233,8 @@ "# Update the PIP version.\n", "!python -m pip install --upgrade pip\n", "!pip install kfp==1.1.1\n", - "!pip install kubeflow-katib==0.10.1" + "!pip install kubeflow-katib==0.10.1\n", + "!pip install kfp-tekton==0.4.0" ] }, { @@ -271,7 +304,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -356,7 +389,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -430,7 +463,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -458,7 +491,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -470,7 +503,6 @@ " name=\"Launch Katib early stopping Experiment\",\n", " description=\"An example to launch Katib Experiment with early stopping\"\n", ")\n", - "\n", "def median_stop():\n", " \n", " # Katib launcher component.\n", @@ -502,23 +534,15 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 21, "metadata": { "scrolled": true }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/jovyan/.local/lib/python3.6/site-packages/kfp/dsl/_container_op.py:1028: FutureWarning: Please create reusable components instead of constructing ContainerOp instances directly. Reusable components are shareable, portable and have compatibility and support guarantees. Please see the documentation: https://www.kubeflow.org/docs/pipelines/sdk/component-development/#writing-your-component-definition-file The components can be created manually (or, in case of python, using kfp.components.create_component_from_func or func_to_container_op) and then loaded using kfp.components.load_component_from_file, load_component_from_uri or load_component_from_text: https://kubeflow-pipelines.readthedocs.io/en/latest/source/kfp.components.html#kfp.components.load_component_from_file\n", - " category=FutureWarning,\n" - ] - }, { "data": { "text/html": [ - "Experiment details." + "Experiment link here" ], "text/plain": [ "" @@ -530,7 +554,7 @@ { "data": { "text/html": [ - "Run details." + "Run link here" ], "text/plain": [ "" @@ -542,16 +566,17 @@ { "data": { "text/plain": [ - "RunPipelineResult(run_id=68100cc5-d42d-441e-974e-f61dd40782ef)" + "RunPipelineResult(run_id=b7518e3c-a0c7-4cf0-8fd6-a125694d5117)" ] }, - "execution_count": 6, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "kfp.Client().create_run_from_pipeline_func(median_stop, arguments={})" + "from kfp_tekton._client import TektonClient \n", + "TektonClient().create_run_from_pipeline_func(median_stop, arguments={})" ] }, { diff --git a/samples/katib/katib.py b/samples/katib/katib.py deleted file mode 100644 index 34b9c072df..0000000000 --- a/samples/katib/katib.py +++ /dev/null @@ -1,79 +0,0 @@ -import json -from kfp import components -import kfp.dsl as dsl - -@dsl.pipeline( - name="Launch katib experiment", - description="An example to launch katib experiment." -) -def mnist_hpo( - name="mnist", - namespace="anonymous", - goal=0.99, - parallelTrialCount=3, - maxTrialCount=12, - experimentTimeoutMinutes=60, - deleteAfterDone=True): - objectiveConfig = { - "type": "maximize", - "goal": goal, - "objectiveMetricName": "Validation-accuracy", - "additionalMetricNames": ["accuracy"] - } - algorithmConfig = {"algorithmName" : "random"} - parameters = [ - {"name": "--lr", "parameterType": "double", "feasibleSpace": {"min": "0.01","max": "0.03"}}, - {"name": "--num-layers", "parameterType": "int", "feasibleSpace": {"min": "2", "max": "5"}}, - {"name": "--optimizer", "parameterType": "categorical", "feasibleSpace": {"list": ["sgd", "adam", "ftrl"]}} - ] - rawTemplate = { - "apiVersion": "batch/v1", - "kind": "Job", - "metadata": { - "name": "{{.Trial}}", - "namespace": "{{.NameSpace}}" - }, - "spec": { - "template": { - "spec": { - "restartPolicy": "Never", - "containers": [ - {"name": "{{.Trial}}", - "image": "docker.io/katib/mxnet-mnist-example", - "command": [ - "python /mxnet/example/image-classification/train_mnist.py --batch-size=64 {{- with .HyperParameters}} {{- range .}} {{.Name}}={{.Value}} {{- end}} {{- end}}" - ] - } - ] - } - } - } - } - trialTemplate = { - "goTemplate": { - "rawTemplate": json.dumps(rawTemplate) - } - } - katib_experiment_launcher_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/master/components/kubeflow/katib-launcher/component.yaml') - op1 = katib_experiment_launcher_op( - experiment_name=name, - experiment_namespace=namespace, - parallel_trial_count=parallelTrialCount, - max_trial_count=maxTrialCount, - objective=str(objectiveConfig), - algorithm=str(algorithmConfig), - trial_template=str(trialTemplate), - parameters=str(parameters), - experiment_timeout_minutes=experimentTimeoutMinutes, - delete_finished_experiment=deleteAfterDone) - - op_out = dsl.ContainerOp( - name="my-out-cop", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo hyperparameter: %s" % op1.output], - ) - -if __name__ == '__main__': - from kfp_tekton.compiler import TektonCompiler - TektonCompiler().compile(mnist_hpo, __file__.replace('.py', '.yaml'))