From ec0a5adc8a7c2e5c18da68d1fc03109524d2957b Mon Sep 17 00:00:00 2001 From: J-shang Date: Mon, 22 Feb 2021 12:38:50 +0000 Subject: [PATCH 01/13] add python api example --- .../python_api_quickstart.ipynb | 430 ++++++++++++++++++ nni/experiment/experiment.py | 72 ++- test/nni_test/nnitest/validators.py | 2 +- 3 files changed, 492 insertions(+), 12 deletions(-) create mode 100644 examples/trials/sklearn/classification/python_api_quickstart.ipynb diff --git a/examples/trials/sklearn/classification/python_api_quickstart.ipynb b/examples/trials/sklearn/classification/python_api_quickstart.ipynb new file mode 100644 index 0000000000..be2ec602ec --- /dev/null +++ b/examples/trials/sklearn/classification/python_api_quickstart.ipynb @@ -0,0 +1,430 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "seasonal-rubber", + "metadata": {}, + "source": [ + "# Quick Start with Jupyter Notebook" + ] + }, + { + "cell_type": "markdown", + "id": "technological-script", + "metadata": {}, + "source": [ + "## Start and Manage a New Experiment" + ] + }, + { + "cell_type": "markdown", + "id": "immediate-daily", + "metadata": {}, + "source": [ + "### 1. Initialize Tuner" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "formed-grounds", + "metadata": {}, + "outputs": [], + "source": [ + "from nni.algorithms.hpo.gridsearch_tuner import GridSearchTuner\n", + "tuner = GridSearchTuner()" + ] + }, + { + "cell_type": "markdown", + "id": "reported-somerset", + "metadata": {}, + "source": [ + "### 2. Configure Search Space" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "potential-williams", + "metadata": {}, + "outputs": [], + "source": [ + "search_space = {\n", + " \"C\": {\"_type\":\"quniform\",\"_value\":[0.1, 1, 0.1]},\n", + " \"kernel\": {\"_type\":\"choice\",\"_value\":[\"linear\", \"rbf\", \"poly\", \"sigmoid\"]},\n", + " \"degree\": {\"_type\":\"choice\",\"_value\":[1, 2, 3, 4]},\n", + " \"gamma\": {\"_type\":\"quniform\",\"_value\":[0.01, 0.1, 0.01]},\n", + " \"coef0\": {\"_type\":\"quniform\",\"_value\":[0.01, 0.1, 0.01]}\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "greek-archive", + "metadata": {}, + "source": [ + "### 3. Configure Experiment " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "fiscal-expansion", + "metadata": {}, + "outputs": [], + "source": [ + "from nni.experiment import Experiment\n", + "experiment = Experiment(tuner, 'local')\n", + "experiment.config.experiment_name = 'test'\n", + "experiment.config.trial_concurrency = 2\n", + "experiment.config.max_trial_number = 5\n", + "experiment.config.search_space = search_space\n", + "experiment.config.trial_command = 'python3 main.py'\n", + "experiment.config.trial_code_directory = './'" + ] + }, + { + "cell_type": "markdown", + "id": "received-tattoo", + "metadata": {}, + "source": [ + "### 4. Start Experiment" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "pleasant-patent", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2021-02-22 12:27:11] Creating experiment, Experiment ID: bj025qo4\n", + "[2021-02-22 12:27:11] Connecting IPC pipe...\n", + "[2021-02-22 12:27:15] Statring web server...\n", + "[2021-02-22 12:27:16] Setting up...\n", + "[2021-02-22 12:27:16] Dispatcher started\n", + "[2021-02-22 12:27:16] Web UI URLs: http://127.0.0.1:8081 http://10.0.1.5:8081 http://172.17.0.1:8081\n" + ] + } + ], + "source": [ + "experiment.start(8081)" + ] + }, + { + "cell_type": "markdown", + "id": "miniature-prison", + "metadata": {}, + "source": [ + "### 5. Experiment View & Control" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "animated-english", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'RUNNING'" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "experiment.get_status()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "alpha-ottawa", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[TrialResult(parameter={'coef0': 0.01, 'gamma': 0.01, 'degree': 1, 'kernel': 'linear', 'C': 0.1}, value=0.9866666666666667, trialJobId='B55mT'),\n", + " TrialResult(parameter={'coef0': 0.02, 'gamma': 0.01, 'degree': 1, 'kernel': 'linear', 'C': 0.1}, value=0.9866666666666667, trialJobId='QkhD0')]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "experiment.export_data()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "unique-rendering", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'B55mT': [TrialMetricData(timestamp=1613996853005, trialJobId='B55mT', parameterId='0', type='FINAL', sequence=0, data=0.9866666666666667)],\n", + " 'QkhD0': [TrialMetricData(timestamp=1613996853843, trialJobId='QkhD0', parameterId='1', type='FINAL', sequence=0, data=0.9866666666666667)]}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "experiment.get_job_metrics()" + ] + }, + { + "cell_type": "markdown", + "id": "welsh-difference", + "metadata": {}, + "source": [ + "### 6. Stop Experiment" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "technological-cleanup", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2021-02-22 12:28:16] Stopping experiment, please wait...\n", + "[2021-02-22 12:28:16] Dispatcher exiting...\n", + "[2021-02-22 12:28:17] Experiment stopped\n", + "[2021-02-22 12:28:19] Dispatcher terminiated\n" + ] + } + ], + "source": [ + "experiment.stop()" + ] + }, + { + "cell_type": "markdown", + "id": "white-electron", + "metadata": {}, + "source": [ + "## Connect and Manage an Exist Experiment" + ] + }, + { + "cell_type": "markdown", + "id": "romantic-hartford", + "metadata": {}, + "source": [ + "### 1. Initialize Experiment without Tuner" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "sapphire-stand", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2021-02-22 12:29:01] Tuner not set, wait for connect...\n" + ] + } + ], + "source": [ + "from nni.experiment import Experiment\n", + "experiment = Experiment()" + ] + }, + { + "cell_type": "markdown", + "id": "recent-italic", + "metadata": {}, + "source": [ + "### 2. Connect Experiment" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "statistical-repair", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2021-02-22 12:29:04] Connect to port 8080 success, experiment id is m6CKYlc0, status is RUNNING.\n" + ] + } + ], + "source": [ + "experiment.connect(8080)" + ] + }, + { + "cell_type": "markdown", + "id": "defensive-scratch", + "metadata": {}, + "source": [ + "### 3. Experiment View & Control" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "independent-touch", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'm6CKYlc0',\n", + " 'revision': 4,\n", + " 'execDuration': 13,\n", + " 'logDir': '/home/nnidev/nni-experiments/m6CKYlc0',\n", + " 'nextSequenceId': 1,\n", + " 'params': {'authorName': 'default',\n", + " 'experimentName': 'example_sklearn-classification',\n", + " 'trialConcurrency': 1,\n", + " 'maxExecDuration': 3600,\n", + " 'maxTrialNum': 5,\n", + " 'searchSpace': '{\"C\": {\"_type\": \"uniform\", \"_value\": [0.1, 1]}, \"kernel\": {\"_type\": \"choice\", \"_value\": [\"linear\", \"rbf\", \"poly\", \"sigmoid\"]}, \"degree\": {\"_type\": \"choice\", \"_value\": [1, 2, 3, 4]}, \"gamma\": {\"_type\": \"uniform\", \"_value\": [0.01, 0.1]}, \"coef0\": {\"_type\": \"uniform\", \"_value\": [0.01, 0.1]}}',\n", + " 'trainingServicePlatform': 'local',\n", + " 'tuner': {'builtinTunerName': 'TPE',\n", + " 'classArgs': {'optimize_mode': 'maximize'},\n", + " 'checkpointDir': '/home/nnidev/nni-experiments/m6CKYlc0/checkpoint'},\n", + " 'versionCheck': True,\n", + " 'clusterMetaData': [{'key': 'trial_config',\n", + " 'value': {'command': 'python3 main.py',\n", + " 'codeDir': '/home/nnidev/nni/examples/trials/sklearn/classification/.',\n", + " 'gpuNum': 0}}]},\n", + " 'startTime': 1613996933896}" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "experiment.get_experiment_profile()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "printable-bookmark", + "metadata": {}, + "outputs": [], + "source": [ + "experiment.update_max_trial_number(10)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "marine-serial", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'm6CKYlc0',\n", + " 'revision': 6,\n", + " 'execDuration': 21,\n", + " 'logDir': '/home/nnidev/nni-experiments/m6CKYlc0',\n", + " 'nextSequenceId': 2,\n", + " 'params': {'authorName': 'default',\n", + " 'experimentName': 'example_sklearn-classification',\n", + " 'trialConcurrency': 1,\n", + " 'maxExecDuration': 3600,\n", + " 'maxTrialNum': 10,\n", + " 'searchSpace': '{\"C\": {\"_type\": \"uniform\", \"_value\": [0.1, 1]}, \"kernel\": {\"_type\": \"choice\", \"_value\": [\"linear\", \"rbf\", \"poly\", \"sigmoid\"]}, \"degree\": {\"_type\": \"choice\", \"_value\": [1, 2, 3, 4]}, \"gamma\": {\"_type\": \"uniform\", \"_value\": [0.01, 0.1]}, \"coef0\": {\"_type\": \"uniform\", \"_value\": [0.01, 0.1]}}',\n", + " 'trainingServicePlatform': 'local',\n", + " 'tuner': {'builtinTunerName': 'TPE',\n", + " 'classArgs': {'optimize_mode': 'maximize'},\n", + " 'checkpointDir': '/home/nnidev/nni-experiments/m6CKYlc0/checkpoint'},\n", + " 'versionCheck': True,\n", + " 'clusterMetaData': [{'key': 'trial_config',\n", + " 'value': {'command': 'python3 main.py',\n", + " 'codeDir': '/home/nnidev/nni/examples/trials/sklearn/classification/.',\n", + " 'gpuNum': 0}}]},\n", + " 'startTime': 1613996933896}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "experiment.get_experiment_profile()" + ] + }, + { + "cell_type": "markdown", + "id": "opened-lounge", + "metadata": {}, + "source": [ + "### 4. Stop Experiment" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "emotional-machinery", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2021-02-22 12:29:22] Stopping experiment, please wait...\n", + "[2021-02-22 12:29:22] Experiment stopped\n" + ] + } + ], + "source": [ + "experiment.stop()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nni-dev", + "language": "python", + "name": "nni-dev" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/nni/experiment/experiment.py b/nni/experiment/experiment.py index c8e9e16fa9..a44877e6f2 100644 --- a/nni/experiment/experiment.py +++ b/nni/experiment/experiment.py @@ -204,7 +204,7 @@ def run(self, port: int = 8080, debug: bool = False) -> bool: finally: self.stop() - def connect_experiment(self, port: int): + def connect(self, port: int): """ Connect to an existing experiment. @@ -214,7 +214,14 @@ def connect_experiment(self, port: int): The port of web UI. """ self.port = port - self.get_status() + self.id = self.get_experiment_profile().get('id') + status = self.get_status() + pid = self.get_experiment_metadata(self.id).get('pid') + if pid is None: + _logger.warning('Get experiment pid failed, can not stop experiment by stop().') + else: + self._proc = psutil.Process(pid) + _logger.info('Connect to port {} success, experiment id is {}, status is {}.'.format(port, self.id, status)) def _experiment_rest_get(self, port: int, api: str) -> Any: if self.port is None: @@ -316,6 +323,49 @@ def get_experiment_profile(self): resp = self._experiment_rest_get(self.port, '/experiment') return resp + def get_experiment_metadata(self, exp_id: str): + """ + Return experiment metadata with specified exp_id as a dict. + + Returns + ---------- + dict + The specified experiment metadata. + """ + experiments_metadata = self.get_all_experiments_metadata() + for metadata in experiments_metadata: + if metadata['id'] == exp_id: + return metadata + return {} + + def get_all_experiments_metadata(self): + """ + Return all experiments metadata as a list. + + Returns + ---------- + list + The experiments metadata. + Example format: [ + { + "id": str, + "port": int, + "startTime": int, + "endTime": int, + "status": str, + "platform": str, + "experimentName": str, + "tag": [str], + "pid": int, + "webuiUrl": [str], + "logDir": str + }, + {...} + ] + """ + resp = self._experiment_rest_get(self.port, '/experiments-info') + return resp + def export_data(self): """ Return exported information for all trial jobs. @@ -329,13 +379,13 @@ def export_data(self): return [TrialResult(**trial_result) for trial_result in resp] def _get_query_type(self, key: str): - if key == 'trial_concurrency': + if key == 'trialConcurrency': return '?update_type=TRIAL_CONCURRENCY' - if key == 'max_experiment_duration': + if key == 'maxExecDuration': return '?update_type=MAX_EXEC_DURATION' - if key == 'search_space': + if key == 'searchSpace': return '?update_type=SEARCH_SPACE' - if key == 'max_trial_number': + if key == 'maxTrialNum': return '?update_type=MAX_TRIAL_NUM' def _update_experiment_profile(self, key: str, value: Any): @@ -363,7 +413,7 @@ def update_trial_concurrency(self, value: int): value: int New trial_concurrency value. """ - self._update_experiment_profile('trial_concurrency', value) + self._update_experiment_profile('trialConcurrency', value) def update_max_experiment_duration(self, value: str): """ @@ -375,7 +425,7 @@ def update_max_experiment_duration(self, value: str): Strings like '1m' for one minute or '2h' for two hours. SUFFIX may be 's' for seconds, 'm' for minutes, 'h' for hours or 'd' for days. """ - self._update_experiment_profile('max_experiment_duration', value) + self._update_experiment_profile('maxExecDuration', value) def update_search_space(self, value: dict): """ @@ -387,9 +437,9 @@ def update_search_space(self, value: dict): value: dict New search_space. """ - self._update_experiment_profile('search_space', value) + self._update_experiment_profile('searchSpace', value) - def update_max_trial_number(self, value): + def update_max_trial_number(self, value: int): """ Update an experiment's max_trial_number @@ -398,4 +448,4 @@ def update_max_trial_number(self, value): value: int New max_trial_number value. """ - self._update_experiment_profile('max_trial_number', value) + self._update_experiment_profile('maxTrialNum', value) diff --git a/test/nni_test/nnitest/validators.py b/test/nni_test/nnitest/validators.py index f4a880857c..3352899d07 100644 --- a/test/nni_test/nnitest/validators.py +++ b/test/nni_test/nnitest/validators.py @@ -93,7 +93,7 @@ class NnicliValidator(ITValidator): def __call__(self, rest_endpoint, experiment_dir, nni_source_dir, **kwargs): print(rest_endpoint) exp = Experiment() - exp.connect_experiment(int(rest_endpoint.split(':')[-1])) + exp.connect(int(rest_endpoint.split(':')[-1])) print(exp.get_job_statistics()) print(exp.get_experiment_status()) print(exp.list_trial_jobs()) From b169baf110e95204605c1452523a05ba2c59f8ec Mon Sep 17 00:00:00 2001 From: J-shang Date: Tue, 23 Feb 2021 02:12:17 +0000 Subject: [PATCH 02/13] update doc --- docs/en_US/Tutorial/HowToLaunchFromPython.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/en_US/Tutorial/HowToLaunchFromPython.rst b/docs/en_US/Tutorial/HowToLaunchFromPython.rst index 11d3e63267..8409eddbb4 100644 --- a/docs/en_US/Tutorial/HowToLaunchFromPython.rst +++ b/docs/en_US/Tutorial/HowToLaunchFromPython.rst @@ -61,7 +61,7 @@ Now, you have successfully launched an NNI experiment. And you can type ``localh Example ------- -Below is an example for this new launching approach. You can also find this code in :githublink:`mnist-tfv2/launch.py ` . +Below is an example for this new launching approach. You can also find this code in :githublink:`mnist-tfv2/launch.py `. .. code-block:: python @@ -90,6 +90,14 @@ Below is an example for this new launching approach. You can also find this code experiment.run(8081) +View and Control Experiment with Python API +------------------------------------------- +We migrate the API in `nnicli` to this new launching approach. +Launch the experiment by `start()` instead of `run()`, then you can use these APIs in interactive mode. +`run()` polls the experiment status and will automatically call `stop()` when the experiment finished. +`start()` just launch a new experiment, so you need to manually stop the experiment by calling `stop()`. +An example with Jupyter Notebook has been provided, view :githublink:`python_api_quickstart.ipynb `. + API --- From 8674054b2d00ade15c060c617ecf47ee189bd274 Mon Sep 17 00:00:00 2001 From: J-shang Date: Tue, 23 Feb 2021 02:19:30 +0000 Subject: [PATCH 03/13] update doc --- docs/en_US/Tutorial/HowToLaunchFromPython.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/en_US/Tutorial/HowToLaunchFromPython.rst b/docs/en_US/Tutorial/HowToLaunchFromPython.rst index 8409eddbb4..22d29e17df 100644 --- a/docs/en_US/Tutorial/HowToLaunchFromPython.rst +++ b/docs/en_US/Tutorial/HowToLaunchFromPython.rst @@ -93,9 +93,10 @@ Below is an example for this new launching approach. You can also find this code View and Control Experiment with Python API ------------------------------------------- We migrate the API in `nnicli` to this new launching approach. -Launch the experiment by `start()` instead of `run()`, then you can use these APIs in interactive mode. -`run()` polls the experiment status and will automatically call `stop()` when the experiment finished. -`start()` just launch a new experiment, so you need to manually stop the experiment by calling `stop()`. +Launch the experiment by `Experiment.start()` instead of `Experiment.run()`, then you can use these APIs in interactive mode. +`Experiment.run()` polls the experiment status and will automatically call `Experiment.stop()` when the experiment finished. +`Experiment.start()` just launch a new experiment, so you need to manually stop the experiment by calling `Experiment.stop()`. +If you launch the experiment by `nnictl` and also want to use these APIs, you can use `Experiment.connect()` to connect to an exsit experiment. An example with Jupyter Notebook has been provided, view :githublink:`python_api_quickstart.ipynb `. API From b060ff7c96683389d937735358c9795d2ab4e279 Mon Sep 17 00:00:00 2001 From: J-shang Date: Tue, 23 Feb 2021 02:28:53 +0000 Subject: [PATCH 04/13] fix lint --- nni/experiment/experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nni/experiment/experiment.py b/nni/experiment/experiment.py index a44877e6f2..2dc30e5346 100644 --- a/nni/experiment/experiment.py +++ b/nni/experiment/experiment.py @@ -221,7 +221,7 @@ def connect(self, port: int): _logger.warning('Get experiment pid failed, can not stop experiment by stop().') else: self._proc = psutil.Process(pid) - _logger.info('Connect to port {} success, experiment id is {}, status is {}.'.format(port, self.id, status)) + _logger.info('Connect to port %d success, experiment id is %s, status is %s.' % (port, self.id, status)) def _experiment_rest_get(self, port: int, api: str) -> Any: if self.port is None: From 116c7995bf2c9ec3d42ea4b657c01200b6d6527f Mon Sep 17 00:00:00 2001 From: J-shang Date: Tue, 23 Feb 2021 03:33:11 +0000 Subject: [PATCH 05/13] fix lint --- nni/experiment/experiment.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/nni/experiment/experiment.py b/nni/experiment/experiment.py index 2dc30e5346..c54232a61c 100644 --- a/nni/experiment/experiment.py +++ b/nni/experiment/experiment.py @@ -346,22 +346,6 @@ def get_all_experiments_metadata(self): ---------- list The experiments metadata. - Example format: [ - { - "id": str, - "port": int, - "startTime": int, - "endTime": int, - "status": str, - "platform": str, - "experimentName": str, - "tag": [str], - "pid": int, - "webuiUrl": [str], - "logDir": str - }, - {...} - ] """ resp = self._experiment_rest_get(self.port, '/experiments-info') return resp From 0de1becdb72414e03a83ebbef8314b5cbc3f78fc Mon Sep 17 00:00:00 2001 From: J-shang Date: Tue, 23 Feb 2021 03:44:29 +0000 Subject: [PATCH 06/13] fix lint --- nni/experiment/experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nni/experiment/experiment.py b/nni/experiment/experiment.py index c54232a61c..9f1d5d6107 100644 --- a/nni/experiment/experiment.py +++ b/nni/experiment/experiment.py @@ -221,7 +221,7 @@ def connect(self, port: int): _logger.warning('Get experiment pid failed, can not stop experiment by stop().') else: self._proc = psutil.Process(pid) - _logger.info('Connect to port %d success, experiment id is %s, status is %s.' % (port, self.id, status)) + _logger.info('Connect to port %d success, experiment id is %s, status is %s.', port, self.id, status) def _experiment_rest_get(self, port: int, api: str) -> Any: if self.port is None: From 8464a3fec64c8a6f4b2ce3ba8675ec05459f78f3 Mon Sep 17 00:00:00 2001 From: J-shang Date: Thu, 25 Feb 2021 03:23:36 +0000 Subject: [PATCH 07/13] reset and fix comments --- nni/experiment/experiment.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nni/experiment/experiment.py b/nni/experiment/experiment.py index 9f1d5d6107..77733bfdcc 100644 --- a/nni/experiment/experiment.py +++ b/nni/experiment/experiment.py @@ -255,7 +255,7 @@ def get_trial_job(self, trial_job_id: str): Trial job id. Returns - ---------- + ------- TrialJob A `TrialJob` instance corresponding to `trial_job_id`. """ @@ -267,7 +267,7 @@ def list_trial_jobs(self): Return information for all trial jobs as a list. Returns - ---------- + ------- list List of `TrialJob`. """ @@ -279,7 +279,7 @@ def get_job_statistics(self): Return trial job statistics information as a dict. Returns - ---------- + ------- dict Job statistics information. """ @@ -296,7 +296,7 @@ def get_job_metrics(self, trial_job_id=None): trial job id. if this parameter is None, all trail jobs' metrics will be returned. Returns - ---------- + ------- dict Each key is a trialJobId, the corresponding value is a list of `TrialMetricData`. """ @@ -316,7 +316,7 @@ def get_experiment_profile(self): Return experiment profile as a dict. Returns - ---------- + ------- dict The profile of the experiment. """ @@ -328,7 +328,7 @@ def get_experiment_metadata(self, exp_id: str): Return experiment metadata with specified exp_id as a dict. Returns - ---------- + ------- dict The specified experiment metadata. """ @@ -343,7 +343,7 @@ def get_all_experiments_metadata(self): Return all experiments metadata as a list. Returns - ---------- + ------- list The experiments metadata. """ @@ -355,7 +355,7 @@ def export_data(self): Return exported information for all trial jobs. Returns - ---------- + ------- list List of `TrialResult`. """ From d3fc49f70f5f157604bb94067a20a05e42652420 Mon Sep 17 00:00:00 2001 From: J-shang Date: Thu, 25 Feb 2021 07:51:56 +0000 Subject: [PATCH 08/13] change connect to cls method --- nni/experiment/experiment.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/nni/experiment/experiment.py b/nni/experiment/experiment.py index 77733bfdcc..0848f62a69 100644 --- a/nni/experiment/experiment.py +++ b/nni/experiment/experiment.py @@ -77,16 +77,6 @@ def __init__(self, tuner: Tuner, training_service: Union[str, List[str]]) -> Non """ ... - @overload - def __init__(self) -> None: - """ - Prepare an empty experiment, for `connect_experiment`. - - Use `Experiment.connect_experiment` to manage experiment. - - """ - ... - def __init__(self, tuner=None, config=None, training_service=None): self.config: Optional[ExperimentConfig] = None self.id: Optional[str] = None @@ -204,7 +194,8 @@ def run(self, port: int = 8080, debug: bool = False) -> bool: finally: self.stop() - def connect(self, port: int): + @classmethod + def connect(cls, port: int): """ Connect to an existing experiment. @@ -213,15 +204,17 @@ def connect(self, port: int): port The port of web UI. """ - self.port = port - self.id = self.get_experiment_profile().get('id') - status = self.get_status() - pid = self.get_experiment_metadata(self.id).get('pid') + experiment = Experiment() + experiment.port = port + experiment.id = experiment.get_experiment_profile().get('id') + status = experiment.get_status() + pid = experiment.get_experiment_metadata(experiment.id).get('pid') if pid is None: _logger.warning('Get experiment pid failed, can not stop experiment by stop().') else: - self._proc = psutil.Process(pid) - _logger.info('Connect to port %d success, experiment id is %s, status is %s.', port, self.id, status) + experiment._proc = psutil.Process(pid) + _logger.info('Connect to port %d success, experiment id is %s, status is %s.', port, experiment.id, status) + return experiment def _experiment_rest_get(self, port: int, api: str) -> Any: if self.port is None: From c8b1083f94034aac40c68f89ad8391f08b2b1a67 Mon Sep 17 00:00:00 2001 From: J-shang Date: Thu, 25 Feb 2021 08:50:30 +0000 Subject: [PATCH 09/13] update doc --- docs/en_US/Tutorial/HowToLaunchFromPython.rst | 34 ++- docs/en_US/Tutorial/python_api_connect.ipynb | 188 ++++++++++++++ .../en_US/Tutorial/python_api_start.ipynb | 196 --------------- .../classification/python_api_connect.ipynb | 188 ++++++++++++++ .../classification/python_api_start.ipynb | 234 ++++++++++++++++++ 5 files changed, 632 insertions(+), 208 deletions(-) create mode 100644 docs/en_US/Tutorial/python_api_connect.ipynb rename examples/trials/sklearn/classification/python_api_quickstart.ipynb => docs/en_US/Tutorial/python_api_start.ipynb (50%) create mode 100644 examples/trials/sklearn/classification/python_api_connect.ipynb create mode 100644 examples/trials/sklearn/classification/python_api_start.ipynb diff --git a/docs/en_US/Tutorial/HowToLaunchFromPython.rst b/docs/en_US/Tutorial/HowToLaunchFromPython.rst index 22d29e17df..319a7de2a4 100644 --- a/docs/en_US/Tutorial/HowToLaunchFromPython.rst +++ b/docs/en_US/Tutorial/HowToLaunchFromPython.rst @@ -1,12 +1,12 @@ -**How to Launch an experiment from Python** +**How to Launch an Experiment from Python** =========================================== Overview -------- Since ``nni v2.0``, we provide a new way to launch experiments. Before that, you need to configure the experiment in the yaml configuration file and then use the experiment ``nnictl`` command to launch the experiment. Now, you can also configure and run experiments directly in python file. If you are familiar with python programming, this will undoubtedly bring you more convenience. -How to Use ----------- +Run a New Experiment +-------------------- After successfully installing ``nni``, you can start the experiment with a python script in the following 3 steps. .. @@ -59,8 +59,10 @@ See `parameter configuration <../reference/experiment_config.rst>`__ required by Now, you have successfully launched an NNI experiment. And you can type ``localhost:8081`` in your browser to observe your experiment in real time. +.. Note:: In this way, experiment will run in the foreground and will automatically exit when the experiment finished. If you want to run an experiment in an interactive way, use ``start()`` in Step 3. + Example -------- +^^^^^^^ Below is an example for this new launching approach. You can also find this code in :githublink:`mnist-tfv2/launch.py `. .. code-block:: python @@ -90,14 +92,22 @@ Below is an example for this new launching approach. You can also find this code experiment.run(8081) -View and Control Experiment with Python API -------------------------------------------- -We migrate the API in `nnicli` to this new launching approach. -Launch the experiment by `Experiment.start()` instead of `Experiment.run()`, then you can use these APIs in interactive mode. -`Experiment.run()` polls the experiment status and will automatically call `Experiment.stop()` when the experiment finished. -`Experiment.start()` just launch a new experiment, so you need to manually stop the experiment by calling `Experiment.stop()`. -If you launch the experiment by `nnictl` and also want to use these APIs, you can use `Experiment.connect()` to connect to an exsit experiment. -An example with Jupyter Notebook has been provided, view :githublink:`python_api_quickstart.ipynb `. +Start and Manage a New Experiment +--------------------------------- +We migrate the API in ``nniClient`` to this new launching approach. +Launch the experiment by ``start()`` instead of ``run()``, then you can use these APIs in interactive mode. + +Please refer to `example usage <./python_api_start.rst>`__ and code file :githublink:`python_api_start.ipynb `. + +.. Note:: ``run()`` polls the experiment status and will automatically call ``stop()`` when the experiment finished. ``start()`` just launch a new experiment, so you need to manually stop the experiment by calling ``stop()``. + +Connect and Manage an Exist Experiment +-------------------------------------- +If you launch the experiment by `nnictl` and also want to use these APIs, you can use ``Experiment.connect()`` to connect to an exsit experiment. + +Please refer to `example usage <./python_api_connect.rst>`__ and code file :githublink:`python_api_connect.ipynb `. + +.. Note:: You can use ``stop()`` to stop the experiment when connecting to an exsit experiment. API --- diff --git a/docs/en_US/Tutorial/python_api_connect.ipynb b/docs/en_US/Tutorial/python_api_connect.ipynb new file mode 100644 index 0000000000..81ab8bc18e --- /dev/null +++ b/docs/en_US/Tutorial/python_api_connect.ipynb @@ -0,0 +1,188 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "white-electron", + "metadata": {}, + "source": [ + "## Connect and Manage an Exist Experiment" + ] + }, + { + "cell_type": "markdown", + "id": "recent-italic", + "metadata": {}, + "source": [ + "### 1. Connect Experiment" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "statistical-repair", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2021-02-25 07:50:38] Tuner not set, wait for connect...\n", + "[2021-02-25 07:50:38] Connect to port 8080 success, experiment id is IF0JnfLE, status is RUNNING.\n" + ] + } + ], + "source": [ + "from nni.experiment import Experiment\n", + "experiment = Experiment.connect(8080)" + ] + }, + { + "cell_type": "markdown", + "id": "defensive-scratch", + "metadata": {}, + "source": [ + "### 2. Experiment View & Control" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "independent-touch", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'IF0JnfLE',\n", + " 'revision': 6,\n", + " 'execDuration': 28,\n", + " 'logDir': '/home/ningshang/nni-experiments/IF0JnfLE',\n", + " 'nextSequenceId': 2,\n", + " 'params': {'authorName': 'default',\n", + " 'experimentName': 'example_sklearn-classification',\n", + " 'trialConcurrency': 1,\n", + " 'maxExecDuration': 3600,\n", + " 'maxTrialNum': 5,\n", + " 'searchSpace': '{\"C\": {\"_type\": \"uniform\", \"_value\": [0.1, 1]}, \"kernel\": {\"_type\": \"choice\", \"_value\": [\"linear\", \"rbf\", \"poly\", \"sigmoid\"]}, \"degree\": {\"_type\": \"choice\", \"_value\": [1, 2, 3, 4]}, \"gamma\": {\"_type\": \"uniform\", \"_value\": [0.01, 0.1]}, \"coef0\": {\"_type\": \"uniform\", \"_value\": [0.01, 0.1]}}',\n", + " 'trainingServicePlatform': 'local',\n", + " 'tuner': {'builtinTunerName': 'TPE',\n", + " 'classArgs': {'optimize_mode': 'maximize'},\n", + " 'checkpointDir': '/home/ningshang/nni-experiments/IF0JnfLE/checkpoint'},\n", + " 'versionCheck': True,\n", + " 'clusterMetaData': [{'key': 'trial_config',\n", + " 'value': {'command': 'python3 main.py',\n", + " 'codeDir': '/home/ningshang/nni/examples/trials/sklearn/classification/.',\n", + " 'gpuNum': 0}}]},\n", + " 'startTime': 1614239412494}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "experiment.get_experiment_profile()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "printable-bookmark", + "metadata": {}, + "outputs": [], + "source": [ + "experiment.update_max_trial_number(10)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "marine-serial", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'IF0JnfLE',\n", + " 'revision': 8,\n", + " 'execDuration': 32,\n", + " 'logDir': '/home/ningshang/nni-experiments/IF0JnfLE',\n", + " 'nextSequenceId': 2,\n", + " 'params': {'authorName': 'default',\n", + " 'experimentName': 'example_sklearn-classification',\n", + " 'trialConcurrency': 1,\n", + " 'maxExecDuration': 3600,\n", + " 'maxTrialNum': 10,\n", + " 'searchSpace': '{\"C\": {\"_type\": \"uniform\", \"_value\": [0.1, 1]}, \"kernel\": {\"_type\": \"choice\", \"_value\": [\"linear\", \"rbf\", \"poly\", \"sigmoid\"]}, \"degree\": {\"_type\": \"choice\", \"_value\": [1, 2, 3, 4]}, \"gamma\": {\"_type\": \"uniform\", \"_value\": [0.01, 0.1]}, \"coef0\": {\"_type\": \"uniform\", \"_value\": [0.01, 0.1]}}',\n", + " 'trainingServicePlatform': 'local',\n", + " 'tuner': {'builtinTunerName': 'TPE',\n", + " 'classArgs': {'optimize_mode': 'maximize'},\n", + " 'checkpointDir': '/home/ningshang/nni-experiments/IF0JnfLE/checkpoint'},\n", + " 'versionCheck': True,\n", + " 'clusterMetaData': [{'key': 'trial_config',\n", + " 'value': {'command': 'python3 main.py',\n", + " 'codeDir': '/home/ningshang/nni/examples/trials/sklearn/classification/.',\n", + " 'gpuNum': 0}}]},\n", + " 'startTime': 1614239412494}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "experiment.get_experiment_profile()" + ] + }, + { + "cell_type": "markdown", + "id": "opened-lounge", + "metadata": {}, + "source": [ + "### 3. Stop Experiment" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "emotional-machinery", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2021-02-25 07:50:49] Stopping experiment, please wait...\n", + "[2021-02-25 07:50:49] Experiment stopped\n" + ] + } + ], + "source": [ + "experiment.stop()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nni-dev", + "language": "python", + "name": "nni-dev" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/trials/sklearn/classification/python_api_quickstart.ipynb b/docs/en_US/Tutorial/python_api_start.ipynb similarity index 50% rename from examples/trials/sklearn/classification/python_api_quickstart.ipynb rename to docs/en_US/Tutorial/python_api_start.ipynb index be2ec602ec..1ec31f9069 100644 --- a/examples/trials/sklearn/classification/python_api_quickstart.ipynb +++ b/docs/en_US/Tutorial/python_api_start.ipynb @@ -1,13 +1,5 @@ { "cells": [ - { - "cell_type": "markdown", - "id": "seasonal-rubber", - "metadata": {}, - "source": [ - "# Quick Start with Jupyter Notebook" - ] - }, { "cell_type": "markdown", "id": "technological-script", @@ -216,194 +208,6 @@ "source": [ "experiment.stop()" ] - }, - { - "cell_type": "markdown", - "id": "white-electron", - "metadata": {}, - "source": [ - "## Connect and Manage an Exist Experiment" - ] - }, - { - "cell_type": "markdown", - "id": "romantic-hartford", - "metadata": {}, - "source": [ - "### 1. Initialize Experiment without Tuner" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "sapphire-stand", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[2021-02-22 12:29:01] Tuner not set, wait for connect...\n" - ] - } - ], - "source": [ - "from nni.experiment import Experiment\n", - "experiment = Experiment()" - ] - }, - { - "cell_type": "markdown", - "id": "recent-italic", - "metadata": {}, - "source": [ - "### 2. Connect Experiment" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "statistical-repair", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[2021-02-22 12:29:04] Connect to port 8080 success, experiment id is m6CKYlc0, status is RUNNING.\n" - ] - } - ], - "source": [ - "experiment.connect(8080)" - ] - }, - { - "cell_type": "markdown", - "id": "defensive-scratch", - "metadata": {}, - "source": [ - "### 3. Experiment View & Control" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "independent-touch", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': 'm6CKYlc0',\n", - " 'revision': 4,\n", - " 'execDuration': 13,\n", - " 'logDir': '/home/nnidev/nni-experiments/m6CKYlc0',\n", - " 'nextSequenceId': 1,\n", - " 'params': {'authorName': 'default',\n", - " 'experimentName': 'example_sklearn-classification',\n", - " 'trialConcurrency': 1,\n", - " 'maxExecDuration': 3600,\n", - " 'maxTrialNum': 5,\n", - " 'searchSpace': '{\"C\": {\"_type\": \"uniform\", \"_value\": [0.1, 1]}, \"kernel\": {\"_type\": \"choice\", \"_value\": [\"linear\", \"rbf\", \"poly\", \"sigmoid\"]}, \"degree\": {\"_type\": \"choice\", \"_value\": [1, 2, 3, 4]}, \"gamma\": {\"_type\": \"uniform\", \"_value\": [0.01, 0.1]}, \"coef0\": {\"_type\": \"uniform\", \"_value\": [0.01, 0.1]}}',\n", - " 'trainingServicePlatform': 'local',\n", - " 'tuner': {'builtinTunerName': 'TPE',\n", - " 'classArgs': {'optimize_mode': 'maximize'},\n", - " 'checkpointDir': '/home/nnidev/nni-experiments/m6CKYlc0/checkpoint'},\n", - " 'versionCheck': True,\n", - " 'clusterMetaData': [{'key': 'trial_config',\n", - " 'value': {'command': 'python3 main.py',\n", - " 'codeDir': '/home/nnidev/nni/examples/trials/sklearn/classification/.',\n", - " 'gpuNum': 0}}]},\n", - " 'startTime': 1613996933896}" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "experiment.get_experiment_profile()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "printable-bookmark", - "metadata": {}, - "outputs": [], - "source": [ - "experiment.update_max_trial_number(10)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "marine-serial", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': 'm6CKYlc0',\n", - " 'revision': 6,\n", - " 'execDuration': 21,\n", - " 'logDir': '/home/nnidev/nni-experiments/m6CKYlc0',\n", - " 'nextSequenceId': 2,\n", - " 'params': {'authorName': 'default',\n", - " 'experimentName': 'example_sklearn-classification',\n", - " 'trialConcurrency': 1,\n", - " 'maxExecDuration': 3600,\n", - " 'maxTrialNum': 10,\n", - " 'searchSpace': '{\"C\": {\"_type\": \"uniform\", \"_value\": [0.1, 1]}, \"kernel\": {\"_type\": \"choice\", \"_value\": [\"linear\", \"rbf\", \"poly\", \"sigmoid\"]}, \"degree\": {\"_type\": \"choice\", \"_value\": [1, 2, 3, 4]}, \"gamma\": {\"_type\": \"uniform\", \"_value\": [0.01, 0.1]}, \"coef0\": {\"_type\": \"uniform\", \"_value\": [0.01, 0.1]}}',\n", - " 'trainingServicePlatform': 'local',\n", - " 'tuner': {'builtinTunerName': 'TPE',\n", - " 'classArgs': {'optimize_mode': 'maximize'},\n", - " 'checkpointDir': '/home/nnidev/nni-experiments/m6CKYlc0/checkpoint'},\n", - " 'versionCheck': True,\n", - " 'clusterMetaData': [{'key': 'trial_config',\n", - " 'value': {'command': 'python3 main.py',\n", - " 'codeDir': '/home/nnidev/nni/examples/trials/sklearn/classification/.',\n", - " 'gpuNum': 0}}]},\n", - " 'startTime': 1613996933896}" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "experiment.get_experiment_profile()" - ] - }, - { - "cell_type": "markdown", - "id": "opened-lounge", - "metadata": {}, - "source": [ - "### 4. Stop Experiment" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "emotional-machinery", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[2021-02-22 12:29:22] Stopping experiment, please wait...\n", - "[2021-02-22 12:29:22] Experiment stopped\n" - ] - } - ], - "source": [ - "experiment.stop()" - ] } ], "metadata": { diff --git a/examples/trials/sklearn/classification/python_api_connect.ipynb b/examples/trials/sklearn/classification/python_api_connect.ipynb new file mode 100644 index 0000000000..81ab8bc18e --- /dev/null +++ b/examples/trials/sklearn/classification/python_api_connect.ipynb @@ -0,0 +1,188 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "white-electron", + "metadata": {}, + "source": [ + "## Connect and Manage an Exist Experiment" + ] + }, + { + "cell_type": "markdown", + "id": "recent-italic", + "metadata": {}, + "source": [ + "### 1. Connect Experiment" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "statistical-repair", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2021-02-25 07:50:38] Tuner not set, wait for connect...\n", + "[2021-02-25 07:50:38] Connect to port 8080 success, experiment id is IF0JnfLE, status is RUNNING.\n" + ] + } + ], + "source": [ + "from nni.experiment import Experiment\n", + "experiment = Experiment.connect(8080)" + ] + }, + { + "cell_type": "markdown", + "id": "defensive-scratch", + "metadata": {}, + "source": [ + "### 2. Experiment View & Control" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "independent-touch", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'IF0JnfLE',\n", + " 'revision': 6,\n", + " 'execDuration': 28,\n", + " 'logDir': '/home/ningshang/nni-experiments/IF0JnfLE',\n", + " 'nextSequenceId': 2,\n", + " 'params': {'authorName': 'default',\n", + " 'experimentName': 'example_sklearn-classification',\n", + " 'trialConcurrency': 1,\n", + " 'maxExecDuration': 3600,\n", + " 'maxTrialNum': 5,\n", + " 'searchSpace': '{\"C\": {\"_type\": \"uniform\", \"_value\": [0.1, 1]}, \"kernel\": {\"_type\": \"choice\", \"_value\": [\"linear\", \"rbf\", \"poly\", \"sigmoid\"]}, \"degree\": {\"_type\": \"choice\", \"_value\": [1, 2, 3, 4]}, \"gamma\": {\"_type\": \"uniform\", \"_value\": [0.01, 0.1]}, \"coef0\": {\"_type\": \"uniform\", \"_value\": [0.01, 0.1]}}',\n", + " 'trainingServicePlatform': 'local',\n", + " 'tuner': {'builtinTunerName': 'TPE',\n", + " 'classArgs': {'optimize_mode': 'maximize'},\n", + " 'checkpointDir': '/home/ningshang/nni-experiments/IF0JnfLE/checkpoint'},\n", + " 'versionCheck': True,\n", + " 'clusterMetaData': [{'key': 'trial_config',\n", + " 'value': {'command': 'python3 main.py',\n", + " 'codeDir': '/home/ningshang/nni/examples/trials/sklearn/classification/.',\n", + " 'gpuNum': 0}}]},\n", + " 'startTime': 1614239412494}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "experiment.get_experiment_profile()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "printable-bookmark", + "metadata": {}, + "outputs": [], + "source": [ + "experiment.update_max_trial_number(10)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "marine-serial", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 'IF0JnfLE',\n", + " 'revision': 8,\n", + " 'execDuration': 32,\n", + " 'logDir': '/home/ningshang/nni-experiments/IF0JnfLE',\n", + " 'nextSequenceId': 2,\n", + " 'params': {'authorName': 'default',\n", + " 'experimentName': 'example_sklearn-classification',\n", + " 'trialConcurrency': 1,\n", + " 'maxExecDuration': 3600,\n", + " 'maxTrialNum': 10,\n", + " 'searchSpace': '{\"C\": {\"_type\": \"uniform\", \"_value\": [0.1, 1]}, \"kernel\": {\"_type\": \"choice\", \"_value\": [\"linear\", \"rbf\", \"poly\", \"sigmoid\"]}, \"degree\": {\"_type\": \"choice\", \"_value\": [1, 2, 3, 4]}, \"gamma\": {\"_type\": \"uniform\", \"_value\": [0.01, 0.1]}, \"coef0\": {\"_type\": \"uniform\", \"_value\": [0.01, 0.1]}}',\n", + " 'trainingServicePlatform': 'local',\n", + " 'tuner': {'builtinTunerName': 'TPE',\n", + " 'classArgs': {'optimize_mode': 'maximize'},\n", + " 'checkpointDir': '/home/ningshang/nni-experiments/IF0JnfLE/checkpoint'},\n", + " 'versionCheck': True,\n", + " 'clusterMetaData': [{'key': 'trial_config',\n", + " 'value': {'command': 'python3 main.py',\n", + " 'codeDir': '/home/ningshang/nni/examples/trials/sklearn/classification/.',\n", + " 'gpuNum': 0}}]},\n", + " 'startTime': 1614239412494}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "experiment.get_experiment_profile()" + ] + }, + { + "cell_type": "markdown", + "id": "opened-lounge", + "metadata": {}, + "source": [ + "### 3. Stop Experiment" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "emotional-machinery", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2021-02-25 07:50:49] Stopping experiment, please wait...\n", + "[2021-02-25 07:50:49] Experiment stopped\n" + ] + } + ], + "source": [ + "experiment.stop()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nni-dev", + "language": "python", + "name": "nni-dev" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/trials/sklearn/classification/python_api_start.ipynb b/examples/trials/sklearn/classification/python_api_start.ipynb new file mode 100644 index 0000000000..1ec31f9069 --- /dev/null +++ b/examples/trials/sklearn/classification/python_api_start.ipynb @@ -0,0 +1,234 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "technological-script", + "metadata": {}, + "source": [ + "## Start and Manage a New Experiment" + ] + }, + { + "cell_type": "markdown", + "id": "immediate-daily", + "metadata": {}, + "source": [ + "### 1. Initialize Tuner" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "formed-grounds", + "metadata": {}, + "outputs": [], + "source": [ + "from nni.algorithms.hpo.gridsearch_tuner import GridSearchTuner\n", + "tuner = GridSearchTuner()" + ] + }, + { + "cell_type": "markdown", + "id": "reported-somerset", + "metadata": {}, + "source": [ + "### 2. Configure Search Space" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "potential-williams", + "metadata": {}, + "outputs": [], + "source": [ + "search_space = {\n", + " \"C\": {\"_type\":\"quniform\",\"_value\":[0.1, 1, 0.1]},\n", + " \"kernel\": {\"_type\":\"choice\",\"_value\":[\"linear\", \"rbf\", \"poly\", \"sigmoid\"]},\n", + " \"degree\": {\"_type\":\"choice\",\"_value\":[1, 2, 3, 4]},\n", + " \"gamma\": {\"_type\":\"quniform\",\"_value\":[0.01, 0.1, 0.01]},\n", + " \"coef0\": {\"_type\":\"quniform\",\"_value\":[0.01, 0.1, 0.01]}\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "greek-archive", + "metadata": {}, + "source": [ + "### 3. Configure Experiment " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "fiscal-expansion", + "metadata": {}, + "outputs": [], + "source": [ + "from nni.experiment import Experiment\n", + "experiment = Experiment(tuner, 'local')\n", + "experiment.config.experiment_name = 'test'\n", + "experiment.config.trial_concurrency = 2\n", + "experiment.config.max_trial_number = 5\n", + "experiment.config.search_space = search_space\n", + "experiment.config.trial_command = 'python3 main.py'\n", + "experiment.config.trial_code_directory = './'" + ] + }, + { + "cell_type": "markdown", + "id": "received-tattoo", + "metadata": {}, + "source": [ + "### 4. Start Experiment" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "pleasant-patent", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2021-02-22 12:27:11] Creating experiment, Experiment ID: bj025qo4\n", + "[2021-02-22 12:27:11] Connecting IPC pipe...\n", + "[2021-02-22 12:27:15] Statring web server...\n", + "[2021-02-22 12:27:16] Setting up...\n", + "[2021-02-22 12:27:16] Dispatcher started\n", + "[2021-02-22 12:27:16] Web UI URLs: http://127.0.0.1:8081 http://10.0.1.5:8081 http://172.17.0.1:8081\n" + ] + } + ], + "source": [ + "experiment.start(8081)" + ] + }, + { + "cell_type": "markdown", + "id": "miniature-prison", + "metadata": {}, + "source": [ + "### 5. Experiment View & Control" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "animated-english", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'RUNNING'" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "experiment.get_status()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "alpha-ottawa", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[TrialResult(parameter={'coef0': 0.01, 'gamma': 0.01, 'degree': 1, 'kernel': 'linear', 'C': 0.1}, value=0.9866666666666667, trialJobId='B55mT'),\n", + " TrialResult(parameter={'coef0': 0.02, 'gamma': 0.01, 'degree': 1, 'kernel': 'linear', 'C': 0.1}, value=0.9866666666666667, trialJobId='QkhD0')]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "experiment.export_data()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "unique-rendering", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'B55mT': [TrialMetricData(timestamp=1613996853005, trialJobId='B55mT', parameterId='0', type='FINAL', sequence=0, data=0.9866666666666667)],\n", + " 'QkhD0': [TrialMetricData(timestamp=1613996853843, trialJobId='QkhD0', parameterId='1', type='FINAL', sequence=0, data=0.9866666666666667)]}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "experiment.get_job_metrics()" + ] + }, + { + "cell_type": "markdown", + "id": "welsh-difference", + "metadata": {}, + "source": [ + "### 6. Stop Experiment" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "technological-cleanup", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2021-02-22 12:28:16] Stopping experiment, please wait...\n", + "[2021-02-22 12:28:16] Dispatcher exiting...\n", + "[2021-02-22 12:28:17] Experiment stopped\n", + "[2021-02-22 12:28:19] Dispatcher terminiated\n" + ] + } + ], + "source": [ + "experiment.stop()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nni-dev", + "language": "python", + "name": "nni-dev" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 3b1471b8f417ce68793f49dd4f0af08edb0c5a80 Mon Sep 17 00:00:00 2001 From: J-shang Date: Thu, 25 Feb 2021 08:56:31 +0000 Subject: [PATCH 10/13] fix typo --- docs/en_US/Tutorial/HowToLaunchFromPython.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en_US/Tutorial/HowToLaunchFromPython.rst b/docs/en_US/Tutorial/HowToLaunchFromPython.rst index 319a7de2a4..48b5a3504d 100644 --- a/docs/en_US/Tutorial/HowToLaunchFromPython.rst +++ b/docs/en_US/Tutorial/HowToLaunchFromPython.rst @@ -99,15 +99,15 @@ Launch the experiment by ``start()`` instead of ``run()``, then you can use thes Please refer to `example usage <./python_api_start.rst>`__ and code file :githublink:`python_api_start.ipynb `. -.. Note:: ``run()`` polls the experiment status and will automatically call ``stop()`` when the experiment finished. ``start()`` just launch a new experiment, so you need to manually stop the experiment by calling ``stop()``. +.. Note:: ``run()`` polls the experiment status and will automatically call ``stop()`` when the experiment finished. ``start()`` just launched a new experiment, so you need to manually stop the experiment by calling ``stop()``. Connect and Manage an Exist Experiment -------------------------------------- -If you launch the experiment by `nnictl` and also want to use these APIs, you can use ``Experiment.connect()`` to connect to an exsit experiment. +If you launch the experiment by ``nnictl`` and also want to use these APIs, you can use ``Experiment.connect()`` to connect to an existing experiment. Please refer to `example usage <./python_api_connect.rst>`__ and code file :githublink:`python_api_connect.ipynb `. -.. Note:: You can use ``stop()`` to stop the experiment when connecting to an exsit experiment. +.. Note:: You can use ``stop()`` to stop the experiment when connecting to an existing experiment. API --- From 1fccc0aea0c3627829aec074a6ae5cb6d4c90652 Mon Sep 17 00:00:00 2001 From: J-shang Date: Thu, 25 Feb 2021 08:59:55 +0000 Subject: [PATCH 11/13] update doc --- docs/en_US/Tutorial/HowToLaunchFromPython.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en_US/Tutorial/HowToLaunchFromPython.rst b/docs/en_US/Tutorial/HowToLaunchFromPython.rst index 48b5a3504d..59bb0d4b51 100644 --- a/docs/en_US/Tutorial/HowToLaunchFromPython.rst +++ b/docs/en_US/Tutorial/HowToLaunchFromPython.rst @@ -94,7 +94,7 @@ Below is an example for this new launching approach. You can also find this code Start and Manage a New Experiment --------------------------------- -We migrate the API in ``nniClient`` to this new launching approach. +We migrate the API in ``NNI Client`` to this new launching approach. Launch the experiment by ``start()`` instead of ``run()``, then you can use these APIs in interactive mode. Please refer to `example usage <./python_api_start.rst>`__ and code file :githublink:`python_api_start.ipynb `. From b84919ff8b81acee4f06bf8c3a9bea288662147f Mon Sep 17 00:00:00 2001 From: J-shang Date: Thu, 25 Feb 2021 09:15:38 +0000 Subject: [PATCH 12/13] fix sphinx --- docs/en_US/Tutorial/HowToLaunchFromPython.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/en_US/Tutorial/HowToLaunchFromPython.rst b/docs/en_US/Tutorial/HowToLaunchFromPython.rst index 59bb0d4b51..7bcf4406b8 100644 --- a/docs/en_US/Tutorial/HowToLaunchFromPython.rst +++ b/docs/en_US/Tutorial/HowToLaunchFromPython.rst @@ -1,6 +1,12 @@ **How to Launch an Experiment from Python** =========================================== +.. toctree:: + :hidden: + + Start Usage + Connect Usage + Overview -------- Since ``nni v2.0``, we provide a new way to launch experiments. Before that, you need to configure the experiment in the yaml configuration file and then use the experiment ``nnictl`` command to launch the experiment. Now, you can also configure and run experiments directly in python file. If you are familiar with python programming, this will undoubtedly bring you more convenience. From 5ebe671bc63897a66f6729d6d94a98a7ad506f61 Mon Sep 17 00:00:00 2001 From: J-shang Date: Thu, 25 Feb 2021 09:42:31 +0000 Subject: [PATCH 13/13] fix sphinx --- dependencies/required.txt | 1 + docs/en_US/conf.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dependencies/required.txt b/dependencies/required.txt index dc6320846c..b0fc73b13c 100644 --- a/dependencies/required.txt +++ b/dependencies/required.txt @@ -13,6 +13,7 @@ scikit-learn>=0.23.2 websockets filelock prettytable +ipython dataclasses ; python_version < "3.7" numpy < 1.19.4 ; sys_platform == "win32" numpy < 1.20 ; sys_platform != "win32" and python_version < "3.7" diff --git a/docs/en_US/conf.py b/docs/en_US/conf.py index 10b1c651a2..4486baf816 100644 --- a/docs/en_US/conf.py +++ b/docs/en_US/conf.py @@ -47,6 +47,7 @@ 'sphinx.ext.intersphinx', 'nbsphinx', 'sphinx.ext.extlinks', + 'IPython.sphinxext.ipython_console_highlighting', ] # Add mock modules @@ -72,7 +73,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'Release_v1.0.md'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'Release_v1.0.md', '**.ipynb_checkpoints'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = None