diff --git a/.gitignore b/.gitignore index fd71582a60..3d0e2bee92 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,12 @@ +/nni_node/ +/toolchain/ + +# unit test generated files +/test/model_path/ +/test/temp.json +/test/ut/sdk/*.pth + + # Logs logs *.log diff --git a/Makefile b/Makefile deleted file mode 100644 index 3458bfbea9..0000000000 --- a/Makefile +++ /dev/null @@ -1,261 +0,0 @@ -# Setting variables -SHELL := /bin/bash -PIP_INSTALL := python3 -m pip install -PIP_UNINSTALL := python3 -m pip uninstall - -## Colorful output -_INFO := $(shell echo -e '\033[1;36m') -_WARNING := $(shell echo -e '\033[1;33m') -_END := $(shell echo -e '\033[0m') - -## Detect OS -UNAME_S := $(shell uname -s) -ifeq ($(UNAME_S), Linux) - OS_SPEC := linux - NODE_URL := https://nodejs.org/dist/v10.22.1/node-v10.22.1-linux-x64.tar.xz -else ifeq ($(UNAME_S), Darwin) - OS_SPEC := darwin - NODE_URL := https://nodejs.org/dist/v10.22.1/node-v10.22.1-darwin-x64.tar.xz -else - $(error platform $(UNAME_S) not supported) -endif - -## Install directories - -## For apt-get or pip installed virtualenv -ifdef VIRTUAL_ENV - ROOT_FOLDER ?= $(VIRTUAL_ENV) - BASH_COMP_PREFIX ?= ${HOME}/.bash_completion.d -else - ROOT_FOLDER ?= $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getsitepackages()[0]).parents[2])') - IS_SYS_PYTHON ?= $(shell [[ $(ROOT_FOLDER) == /usr* || $(ROOT_FOLDER) == /Library* ]] && echo TRUE || echo FALSE) - - ifeq ($(shell id -u), 0) # is root - _ROOT := 1 - BASH_COMP_PREFIX ?= /usr/share/bash-completion/completions - else # is normal user - ifeq (TRUE, $(IS_SYS_PYTHON)) - ROOT_FOLDER := $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getusersitepackages()).parents[2])') - PIP_MODE ?= --user - endif - BASH_COMP_PREFIX ?= ${HOME}/.bash_completion.d - endif -endif -BASH_COMP_SCRIPT := $(BASH_COMP_PREFIX)/nnictl - -NNI_INSTALL_PATH ?= $(INSTALL_PREFIX)/nni - -BIN_FOLDER ?= $(ROOT_FOLDER)/bin -NNI_PKG_FOLDER ?= $(ROOT_FOLDER)/nni -NASUI_PKG_FOLDER ?= $(ROOT_FOLDER)/nni/nasui - -## Dependency information -NNI_DEPENDENCY_FOLDER = /tmp/$(USER) -$(shell mkdir -p $(NNI_DEPENDENCY_FOLDER)) -NNI_NODE_TARBALL ?= $(NNI_DEPENDENCY_FOLDER)/nni-node-$(OS_SPEC)-x64.tar.xz -NNI_NODE_FOLDER = $(NNI_DEPENDENCY_FOLDER)/nni-node-$(OS_SPEC)-x64 -NNI_NODE ?= $(BIN_FOLDER)/node -NNI_NPM ?= $(BIN_FOLDER)/npm -NNI_YARN_TARBALL ?= $(NNI_DEPENDENCY_FOLDER)/nni-yarn.tar.gz -NNI_YARN_FOLDER ?= $(NNI_DEPENDENCY_FOLDER)/nni-yarn -NNI_YARN ?= PATH=$(BIN_FOLDER):$${PATH} $(NNI_YARN_FOLDER)/bin/yarn - -## Version number -NNI_VERSION_VALUE = $(shell git describe --tags) -NNI_VERSION_VALUE := $(NNI_VERSION_VALUE:v%=%) -NNI_VERSION_TEMPLATE = 999.0.0-developing - -# Main targets - -.PHONY: build -build: - #$(_INFO) Building NNI Manager $(_END) - cd src/nni_manager && $(NNI_YARN) && $(NNI_YARN) build - cp -rf src/nni_manager/config src/nni_manager/dist/ - #$(_INFO) Building WebUI $(_END) - cd src/webui && $(NNI_YARN) && $(NNI_YARN) build - #$(_INFO) Building NAS UI $(_END) - cd src/nasui && $(NNI_YARN) && $(NNI_YARN) build - -# All-in-one target for non-expert users -# Installs NNI as well as its dependencies, and update bashrc to set PATH -.PHONY: easy-install -easy-install: check-perm -easy-install: install-dependencies -easy-install: build -easy-install: install -easy-install: update-bash-config -easy-install: - #$(_INFO) Complete! $(_END) - -# All-in-one target for developer users -# Install NNI as well as its dependencies, and update bashrc to set PATH -.PHONY: dev-easy-install -dev-easy-install: dev-check-perm -dev-easy-install: install-dependencies -dev-easy-install: build -dev-easy-install: dev-install -dev-easy-install: update-bash-config -dev-easy-install: - #$(_INFO) Complete! $(_END) - -# Standard installation target -# Must be invoked after building -.PHONY: install -install: install-python-modules -install: install-node-modules -install: install-scripts -install: - #$(_INFO) Complete! You may want to add $(BIN_FOLDER) to your PATH environment $(_END) - -# Target for NNI developers -# Creates symlinks instead of copying files -.PHONY: dev-install -dev-install: dev-install-python-modules -dev-install: dev-install-node-modules -dev-install: install-scripts -dev-install: - #$(_INFO) Complete! You may want to add $(BIN_FOLDER) to your PATH environment $(_END) - -.PHONY: uninstall -uninstall: - -cd build && $(PIP_UNINSTALL) -y nni - -rm -rf build - -rm -rf $(NNI_PKG_FOLDER) - -rm -f $(BIN_FOLDER)/node - -rm -f $(BIN_FOLDER)/nnictl - -rm -f $(BASH_COMP_SCRIPT) - -.PHONY: clean -clean: - -rm -rf tools/build - -rm -rf tools/nnictl.egg-info - -rm -rf src/nni_manager/dist - -rm -rf src/nni_manager/node_modules - -rm -rf src/sdk/pynni/build - -rm -rf src/sdk/pynni/nni_sdk.egg-info - -rm -rf src/webui/build - -rm -rf src/webui/node_modules - -rm -rf src/nasui/build - -rm -rf src/nasui/node_modules - -# Main targets end - -# Helper targets - -$(NNI_NODE_TARBALL): - #$(_INFO) Downloading Node.js $(_END) - wget $(NODE_URL) -O $(NNI_NODE_TARBALL) - -$(NNI_YARN_TARBALL): - #$(_INFO) Downloading Yarn $(_END) - wget https://github.com/yarnpkg/yarn/releases/download/v1.22.5/yarn-v1.22.5.tar.gz -O $(NNI_YARN_TARBALL) - -.PHONY: install-dependencies -install-dependencies: $(NNI_NODE_TARBALL) $(NNI_YARN_TARBALL) - #$(_INFO) Extracting Node.js $(_END) - rm -rf $(NNI_NODE_FOLDER) - mkdir $(NNI_NODE_FOLDER) - tar -xf $(NNI_NODE_TARBALL) -C $(NNI_NODE_FOLDER) --strip-components 1 - mkdir -p $(BIN_FOLDER) - rm -f $(NNI_NODE) $(NNI_NPM) - ln -s $(NNI_NODE_FOLDER)/bin/node $(NNI_NODE) - ln -s $(NNI_NODE_FOLDER)/bin/npm $(NNI_NPM) - - #$(_INFO) Extracting Yarn $(_END) - rm -rf $(NNI_YARN_FOLDER) - mkdir $(NNI_YARN_FOLDER) - tar -xf $(NNI_YARN_TARBALL) -C $(NNI_YARN_FOLDER) --strip-components 1 - -.PHONY: install-python-modules -install-python-modules: - #$(_INFO) Installing Python SDK $(_END) - sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' src/sdk/pynni/nni/__init__.py - sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' setup.py && $(PIP_INSTALL) $(PIP_MODE) . - -.PHONY: dev-install-python-modules -dev-install-python-modules: - #$(_INFO) Installing Python SDK $(_END) - mkdir -p build - ln -sfT ../src/sdk/pynni/nni build/nni - ln -sfT ../src/sdk/pycli/nnicli build/nnicli - ln -sfT ../tools/nni_annotation build/nni_annotation - ln -sfT ../tools/nni_cmd build/nni_cmd - ln -sfT ../tools/nni_trial_tool build/nni_trial_tool - ln -sfT ../tools/nni_gpu_tool build/nni_gpu_tool - cp setup.py build/ - cp README.md build/ - sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' build/setup.py - sed -ie 's/src\/sdk\/pynni\/nni/nni/g' build/setup.py - sed -ie 's/tools\///g' build/setup.py - cd build && $(PIP_INSTALL) $(PIP_MODE) -e . - - -.PHONY: install-node-modules -install-node-modules: - #$(_INFO) Installing NNI Package $(_END) - rm -rf $(NNI_PKG_FOLDER) - cp -r src/nni_manager/dist $(NNI_PKG_FOLDER) - cp src/nni_manager/package.json $(NNI_PKG_FOLDER) - sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' $(NNI_PKG_FOLDER)/package.json - $(NNI_YARN) --prod --cwd $(NNI_PKG_FOLDER) - cp -r src/webui/build $(NNI_PKG_FOLDER)/static - # Install nasui - mkdir -p $(NASUI_PKG_FOLDER) - cp -rf src/nasui/build $(NASUI_PKG_FOLDER) - cp src/nasui/server.js $(NASUI_PKG_FOLDER) - - -.PHONY: dev-install-node-modules -dev-install-node-modules: - #$(_INFO) Installing NNI Package $(_END) - ln -sfT ${PWD}/src/nni_manager/dist $(NNI_PKG_FOLDER) - cp src/nni_manager/package.json $(NNI_PKG_FOLDER) - sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' $(NNI_PKG_FOLDER)/package.json - ln -sfT ${PWD}/src/nni_manager/node_modules $(NNI_PKG_FOLDER)/node_modules - ln -sfT ${PWD}/src/webui/build $(NNI_PKG_FOLDER)/static - mkdir -p $(NASUI_PKG_FOLDER) - ln -sfT ${PWD}/src/nasui/build $(NASUI_PKG_FOLDER)/build - ln -sfT ${PWD}/src/nasui/server.js $(NASUI_PKG_FOLDER)/server.js - -.PHONY: install-scripts -install-scripts: - mkdir -p $(BASH_COMP_PREFIX) - install -m644 tools/bash-completion $(BASH_COMP_SCRIPT) - -.PHONY: update-bash-config -ifndef _ROOT -update-bash-config: - #$(_INFO) Updating bash configurations $(_END) - ifeq (, $(shell echo $$PATH | tr ':' '\n' | grep -x '$(BIN_FOLDER)')) # $(BIN_FOLDER) not in PATH - #$(_WARNING) NOTE: adding $(BIN_FOLDER) to PATH in bashrc $(_END) - echo 'export PATH="$$PATH:$(BIN_FOLDER)"' >> ~/.bashrc - endif - ifeq (, $(shell (source ~/.bash_completion ; command -v _nnictl) 2>/dev/null)) # completion not installed - #$(_WARNING) NOTE: adding $(BASH_COMP_SCRIPT) to ~/.bash_completion $(_END) - echo '[[ -f $(BASH_COMP_SCRIPT) ]] && source $(BASH_COMP_SCRIPT)' >> ~/.bash_completion - endif -else -update-bash-config: ; -endif - -.PHONY: check-perm -ifdef _ROOT -check-perm: - #$(_WARNING) Run easy-install as root is not optimal $(_END) - #$(_WARNING) Suggest run as non-privileged user or manually install instead $(_END) - #$(_WARNING) Continue easy-install as root? (y/N) $(_END) - @read CONFIRM && [ "$$CONFIRM" = y ] -else -check-perm: ; -endif - -.PHONY: dev-check-perm -ifdef _ROOT -dev-check-perm: - $(error You should not develop NNI as root) -else -dev-check-perm: ; -endif - -# Helper targets end diff --git a/README.md b/README.md index cbda92b421..41d0a87960 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ The tool manages automated machine learning (AutoML) experiments, **dispatches a * Researchers and data scientists who want to easily **implement and experiment new AutoML algorithms**, may it be: hyperparameter tuning algorithm, neural architect search algorithm or model compression algorithm. * ML Platform owners who want to **support AutoML in their platform**. -### **[NNI v1.8 has been released!](https://github.com/microsoft/nni/releases)  ** +### **[NNI v1.9 has been released!](https://github.com/microsoft/nni/releases)  ** ## **NNI capabilities in a glance** @@ -246,7 +246,7 @@ The following example is built on TensorFlow 1.x. Make sure **TensorFlow 1.x is * Download the examples via clone the source code. ```bash - git clone -b v1.8 https://github.com/Microsoft/nni.git + git clone -b v1.9 https://github.com/Microsoft/nni.git ``` * Run the MNIST example. @@ -294,8 +294,8 @@ You can use these commands to get more information about the experiment * Open the `Web UI url` in your browser, you can view detail information of the experiment and all the submitted trial jobs as shown below. [Here](docs/en_US/Tutorial/WebUI.md) are more Web UI pages. - - + +
drawingdrawingdrawingdrawing
## **Documentation** diff --git a/README.Makefile.md b/archive-del/README.Makefile.md similarity index 100% rename from README.Makefile.md rename to archive-del/README.Makefile.md diff --git a/README.Makefile_zh_CN.md b/archive-del/README.Makefile_zh_CN.md similarity index 100% rename from README.Makefile_zh_CN.md rename to archive-del/README.Makefile_zh_CN.md diff --git a/_config.yml b/archive-del/_config.yml similarity index 100% rename from _config.yml rename to archive-del/_config.yml diff --git a/install.ps1 b/archive-del/install.ps1 similarity index 100% rename from install.ps1 rename to archive-del/install.ps1 diff --git a/install.sh b/archive-del/install.sh similarity index 100% rename from install.sh rename to archive-del/install.sh diff --git a/src/sdk/pycli/setup.py b/archive-del/sdk/pycli/setup.py similarity index 100% rename from src/sdk/pycli/setup.py rename to archive-del/sdk/pycli/setup.py diff --git a/src/sdk/pynni/.gitignore b/archive-del/sdk/pynni/.gitignore similarity index 100% rename from src/sdk/pynni/.gitignore rename to archive-del/sdk/pynni/.gitignore diff --git a/src/sdk/pynni/pylintrc b/archive-del/sdk/pynni/pylintrc similarity index 100% rename from src/sdk/pynni/pylintrc rename to archive-del/sdk/pynni/pylintrc diff --git a/src/sdk/pynni/requirements.txt b/archive-del/sdk/pynni/requirements.txt similarity index 100% rename from src/sdk/pynni/requirements.txt rename to archive-del/sdk/pynni/requirements.txt diff --git a/src/sdk/pynni/setup.py b/archive-del/sdk/pynni/setup.py similarity index 100% rename from src/sdk/pynni/setup.py rename to archive-del/sdk/pynni/setup.py diff --git a/src/sdk/pynni/ut.sh b/archive-del/sdk/pynni/ut.sh similarity index 100% rename from src/sdk/pynni/ut.sh rename to archive-del/sdk/pynni/ut.sh diff --git a/tools/bash-completion b/archive-del/tools/bash-completion similarity index 100% rename from tools/bash-completion rename to archive-del/tools/bash-completion diff --git a/tools/setup.py b/archive-del/tools/setup.py similarity index 100% rename from tools/setup.py rename to archive-del/tools/setup.py diff --git a/uninstall.ps1 b/archive-del/uninstall.ps1 similarity index 100% rename from uninstall.ps1 rename to archive-del/uninstall.ps1 diff --git a/src/sdk/pynni/README.md b/archive-doc/nni/README.md similarity index 100% rename from src/sdk/pynni/README.md rename to archive-doc/nni/README.md diff --git a/src/sdk/pynni/README_zh_CN.md b/archive-doc/nni/README_zh_CN.md similarity index 100% rename from src/sdk/pynni/README_zh_CN.md rename to archive-doc/nni/README_zh_CN.md diff --git a/tools/nni_annotation/README.md b/archive-doc/nni_annotation/README.md similarity index 100% rename from tools/nni_annotation/README.md rename to archive-doc/nni_annotation/README.md diff --git a/tools/nni_annotation/README_zh_CN.md b/archive-doc/nni_annotation/README_zh_CN.md similarity index 100% rename from tools/nni_annotation/README_zh_CN.md rename to archive-doc/nni_annotation/README_zh_CN.md diff --git a/tools/README.md b/archive-doc/tools/README.md similarity index 100% rename from tools/README.md rename to archive-doc/tools/README.md diff --git a/tools/README_zh_CN.md b/archive-doc/tools/README_zh_CN.md similarity index 100% rename from tools/README_zh_CN.md rename to archive-doc/tools/README_zh_CN.md diff --git a/src/sdk/pynni/tests/__init__.py b/archive-ut/nni/tests/__init__.py similarity index 100% rename from src/sdk/pynni/tests/__init__.py rename to archive-ut/nni/tests/__init__.py diff --git a/src/sdk/pynni/tests/assets/classic_nas_search_space.json b/archive-ut/nni/tests/assets/classic_nas_search_space.json similarity index 100% rename from src/sdk/pynni/tests/assets/classic_nas_search_space.json rename to archive-ut/nni/tests/assets/classic_nas_search_space.json diff --git a/src/sdk/pynni/tests/assets/search_space.json b/archive-ut/nni/tests/assets/search_space.json similarity index 100% rename from src/sdk/pynni/tests/assets/search_space.json rename to archive-ut/nni/tests/assets/search_space.json diff --git a/src/sdk/pynni/tests/expect/test_graph_module1.expect b/archive-ut/nni/tests/expect/test_graph_module1.expect similarity index 100% rename from src/sdk/pynni/tests/expect/test_graph_module1.expect rename to archive-ut/nni/tests/expect/test_graph_module1.expect diff --git a/src/sdk/pynni/tests/expect/test_graph_module2.expect b/archive-ut/nni/tests/expect/test_graph_module2.expect similarity index 100% rename from src/sdk/pynni/tests/expect/test_graph_module2.expect rename to archive-ut/nni/tests/expect/test_graph_module2.expect diff --git a/src/sdk/pynni/tests/expect/test_graph_module3.expect b/archive-ut/nni/tests/expect/test_graph_module3.expect similarity index 100% rename from src/sdk/pynni/tests/expect/test_graph_module3.expect rename to archive-ut/nni/tests/expect/test_graph_module3.expect diff --git a/src/sdk/pynni/tests/models/pytorch_models/__init__.py b/archive-ut/nni/tests/models/pytorch_models/__init__.py similarity index 100% rename from src/sdk/pynni/tests/models/pytorch_models/__init__.py rename to archive-ut/nni/tests/models/pytorch_models/__init__.py diff --git a/src/sdk/pynni/tests/models/pytorch_models/layer_choice_only.py b/archive-ut/nni/tests/models/pytorch_models/layer_choice_only.py similarity index 100% rename from src/sdk/pynni/tests/models/pytorch_models/layer_choice_only.py rename to archive-ut/nni/tests/models/pytorch_models/layer_choice_only.py diff --git a/src/sdk/pynni/tests/models/pytorch_models/mobilenet.py b/archive-ut/nni/tests/models/pytorch_models/mobilenet.py similarity index 100% rename from src/sdk/pynni/tests/models/pytorch_models/mobilenet.py rename to archive-ut/nni/tests/models/pytorch_models/mobilenet.py diff --git a/src/sdk/pynni/tests/models/pytorch_models/mutable_scope.py b/archive-ut/nni/tests/models/pytorch_models/mutable_scope.py similarity index 100% rename from src/sdk/pynni/tests/models/pytorch_models/mutable_scope.py rename to archive-ut/nni/tests/models/pytorch_models/mutable_scope.py diff --git a/src/sdk/pynni/tests/models/pytorch_models/naive.py b/archive-ut/nni/tests/models/pytorch_models/naive.py similarity index 100% rename from src/sdk/pynni/tests/models/pytorch_models/naive.py rename to archive-ut/nni/tests/models/pytorch_models/naive.py diff --git a/src/sdk/pynni/tests/models/pytorch_models/nested.py b/archive-ut/nni/tests/models/pytorch_models/nested.py similarity index 100% rename from src/sdk/pynni/tests/models/pytorch_models/nested.py rename to archive-ut/nni/tests/models/pytorch_models/nested.py diff --git a/src/sdk/pynni/tests/test_assessor.py b/archive-ut/nni/tests/test_assessor.py similarity index 100% rename from src/sdk/pynni/tests/test_assessor.py rename to archive-ut/nni/tests/test_assessor.py diff --git a/src/sdk/pynni/tests/test_builtin_tuners.py b/archive-ut/nni/tests/test_builtin_tuners.py similarity index 100% rename from src/sdk/pynni/tests/test_builtin_tuners.py rename to archive-ut/nni/tests/test_builtin_tuners.py diff --git a/src/sdk/pynni/tests/test_compression_utils.py b/archive-ut/nni/tests/test_compression_utils.py similarity index 100% rename from src/sdk/pynni/tests/test_compression_utils.py rename to archive-ut/nni/tests/test_compression_utils.py diff --git a/src/sdk/pynni/tests/test_compressor_tf.py b/archive-ut/nni/tests/test_compressor_tf.py similarity index 100% rename from src/sdk/pynni/tests/test_compressor_tf.py rename to archive-ut/nni/tests/test_compressor_tf.py diff --git a/src/sdk/pynni/tests/test_compressor_torch.py b/archive-ut/nni/tests/test_compressor_torch.py similarity index 100% rename from src/sdk/pynni/tests/test_compressor_torch.py rename to archive-ut/nni/tests/test_compressor_torch.py diff --git a/src/sdk/pynni/tests/test_curvefitting_assessor.py b/archive-ut/nni/tests/test_curvefitting_assessor.py similarity index 100% rename from src/sdk/pynni/tests/test_curvefitting_assessor.py rename to archive-ut/nni/tests/test_curvefitting_assessor.py diff --git a/src/sdk/pynni/tests/test_dependecy_aware.py b/archive-ut/nni/tests/test_dependecy_aware.py similarity index 100% rename from src/sdk/pynni/tests/test_dependecy_aware.py rename to archive-ut/nni/tests/test_dependecy_aware.py diff --git a/src/sdk/pynni/tests/test_evolution_tuner.py b/archive-ut/nni/tests/test_evolution_tuner.py similarity index 100% rename from src/sdk/pynni/tests/test_evolution_tuner.py rename to archive-ut/nni/tests/test_evolution_tuner.py diff --git a/src/sdk/pynni/tests/test_graph_utils.py b/archive-ut/nni/tests/test_graph_utils.py similarity index 100% rename from src/sdk/pynni/tests/test_graph_utils.py rename to archive-ut/nni/tests/test_graph_utils.py diff --git a/src/sdk/pynni/tests/test_hyperopt_tuner.py b/archive-ut/nni/tests/test_hyperopt_tuner.py similarity index 100% rename from src/sdk/pynni/tests/test_hyperopt_tuner.py rename to archive-ut/nni/tests/test_hyperopt_tuner.py diff --git a/src/sdk/pynni/tests/test_model_speedup.py b/archive-ut/nni/tests/test_model_speedup.py similarity index 100% rename from src/sdk/pynni/tests/test_model_speedup.py rename to archive-ut/nni/tests/test_model_speedup.py diff --git a/src/sdk/pynni/tests/test_msg_dispatcher.py b/archive-ut/nni/tests/test_msg_dispatcher.py similarity index 100% rename from src/sdk/pynni/tests/test_msg_dispatcher.py rename to archive-ut/nni/tests/test_msg_dispatcher.py diff --git a/src/sdk/pynni/tests/test_nas.py b/archive-ut/nni/tests/test_nas.py similarity index 100% rename from src/sdk/pynni/tests/test_nas.py rename to archive-ut/nni/tests/test_nas.py diff --git a/src/sdk/pynni/tests/test_networkmorphism_tuner.py b/archive-ut/nni/tests/test_networkmorphism_tuner.py similarity index 100% rename from src/sdk/pynni/tests/test_networkmorphism_tuner.py rename to archive-ut/nni/tests/test_networkmorphism_tuner.py diff --git a/src/sdk/pynni/tests/test_protocol.py b/archive-ut/nni/tests/test_protocol.py similarity index 100% rename from src/sdk/pynni/tests/test_protocol.py rename to archive-ut/nni/tests/test_protocol.py diff --git a/src/sdk/pynni/tests/test_pruners.py b/archive-ut/nni/tests/test_pruners.py similarity index 100% rename from src/sdk/pynni/tests/test_pruners.py rename to archive-ut/nni/tests/test_pruners.py diff --git a/src/sdk/pynni/tests/test_smartparam.py b/archive-ut/nni/tests/test_smartparam.py similarity index 100% rename from src/sdk/pynni/tests/test_smartparam.py rename to archive-ut/nni/tests/test_smartparam.py diff --git a/src/sdk/pynni/tests/test_trial.py b/archive-ut/nni/tests/test_trial.py similarity index 100% rename from src/sdk/pynni/tests/test_trial.py rename to archive-ut/nni/tests/test_trial.py diff --git a/src/sdk/pynni/tests/test_utils.py b/archive-ut/nni/tests/test_utils.py similarity index 100% rename from src/sdk/pynni/tests/test_utils.py rename to archive-ut/nni/tests/test_utils.py diff --git a/tools/nni_annotation/examples/mnist_generated.py b/archive-ut/nni_annotation/examples/mnist_generated.py similarity index 100% rename from tools/nni_annotation/examples/mnist_generated.py rename to archive-ut/nni_annotation/examples/mnist_generated.py diff --git a/tools/nni_annotation/examples/mnist_with_annotation.json b/archive-ut/nni_annotation/examples/mnist_with_annotation.json similarity index 100% rename from tools/nni_annotation/examples/mnist_with_annotation.json rename to archive-ut/nni_annotation/examples/mnist_with_annotation.json diff --git a/tools/nni_annotation/examples/mnist_with_annotation.py b/archive-ut/nni_annotation/examples/mnist_with_annotation.py similarity index 100% rename from tools/nni_annotation/examples/mnist_with_annotation.py rename to archive-ut/nni_annotation/examples/mnist_with_annotation.py diff --git a/tools/nni_annotation/examples/mnist_without_annotation.json b/archive-ut/nni_annotation/examples/mnist_without_annotation.json similarity index 100% rename from tools/nni_annotation/examples/mnist_without_annotation.json rename to archive-ut/nni_annotation/examples/mnist_without_annotation.json diff --git a/tools/nni_annotation/examples/mnist_without_annotation.py b/archive-ut/nni_annotation/examples/mnist_without_annotation.py similarity index 100% rename from tools/nni_annotation/examples/mnist_without_annotation.py rename to archive-ut/nni_annotation/examples/mnist_without_annotation.py diff --git a/tools/nni_annotation/test_annotation.py b/archive-ut/nni_annotation/test_annotation.py similarity index 91% rename from tools/nni_annotation/test_annotation.py rename to archive-ut/nni_annotation/test_annotation.py index 3d88e0bc3c..61523fb240 100644 --- a/tools/nni_annotation/test_annotation.py +++ b/archive-ut/nni_annotation/test_annotation.py @@ -5,12 +5,13 @@ from .__init__ import * +import sys import ast import json import os import shutil import tempfile -from unittest import TestCase, main +from unittest import TestCase, main, skipIf class AnnotationTestCase(TestCase): @@ -26,6 +27,7 @@ def test_search_space_generator(self): with open('testcase/searchspace.json') as f: self.assertEqual(search_space, json.load(f)) + @skipIf(sys.version_info.major == 3 and sys.version_info.minor > 7, "skip for python3.8 temporarily") def test_code_generator(self): code_dir = expand_annotations('testcase/usercode', '_generated/usercode', nas_mode='classic_mode') self.assertEqual(code_dir, '_generated/usercode') diff --git a/tools/nni_annotation/testcase/annotated/dir/simple.py b/archive-ut/nni_annotation/testcase/annotated/dir/simple.py similarity index 100% rename from tools/nni_annotation/testcase/annotated/dir/simple.py rename to archive-ut/nni_annotation/testcase/annotated/dir/simple.py diff --git a/tools/nni_annotation/testcase/annotated/handwrite.py b/archive-ut/nni_annotation/testcase/annotated/handwrite.py similarity index 100% rename from tools/nni_annotation/testcase/annotated/handwrite.py rename to archive-ut/nni_annotation/testcase/annotated/handwrite.py diff --git a/tools/nni_annotation/testcase/annotated/mnist.py b/archive-ut/nni_annotation/testcase/annotated/mnist.py similarity index 100% rename from tools/nni_annotation/testcase/annotated/mnist.py rename to archive-ut/nni_annotation/testcase/annotated/mnist.py diff --git a/tools/nni_annotation/testcase/annotated/nas.py b/archive-ut/nni_annotation/testcase/annotated/nas.py similarity index 100% rename from tools/nni_annotation/testcase/annotated/nas.py rename to archive-ut/nni_annotation/testcase/annotated/nas.py diff --git a/tools/nni_annotation/testcase/annotated/non_annotation/bar.py b/archive-ut/nni_annotation/testcase/annotated/non_annotation/bar.py similarity index 100% rename from tools/nni_annotation/testcase/annotated/non_annotation/bar.py rename to archive-ut/nni_annotation/testcase/annotated/non_annotation/bar.py diff --git a/tools/nni_annotation/testcase/annotated/non_annotation/foo.py b/archive-ut/nni_annotation/testcase/annotated/non_annotation/foo.py similarity index 100% rename from tools/nni_annotation/testcase/annotated/non_annotation/foo.py rename to archive-ut/nni_annotation/testcase/annotated/non_annotation/foo.py diff --git a/tools/nni_annotation/testcase/searchspace.json b/archive-ut/nni_annotation/testcase/searchspace.json similarity index 100% rename from tools/nni_annotation/testcase/searchspace.json rename to archive-ut/nni_annotation/testcase/searchspace.json diff --git a/tools/nni_annotation/testcase/usercode/dir/simple.py b/archive-ut/nni_annotation/testcase/usercode/dir/simple.py similarity index 100% rename from tools/nni_annotation/testcase/usercode/dir/simple.py rename to archive-ut/nni_annotation/testcase/usercode/dir/simple.py diff --git a/tools/nni_annotation/testcase/usercode/mnist.py b/archive-ut/nni_annotation/testcase/usercode/mnist.py similarity index 100% rename from tools/nni_annotation/testcase/usercode/mnist.py rename to archive-ut/nni_annotation/testcase/usercode/mnist.py diff --git a/tools/nni_annotation/testcase/usercode/nas.py b/archive-ut/nni_annotation/testcase/usercode/nas.py similarity index 100% rename from tools/nni_annotation/testcase/usercode/nas.py rename to archive-ut/nni_annotation/testcase/usercode/nas.py diff --git a/tools/nni_annotation/testcase/usercode/non_annotation/bar.py b/archive-ut/nni_annotation/testcase/usercode/non_annotation/bar.py similarity index 100% rename from tools/nni_annotation/testcase/usercode/non_annotation/bar.py rename to archive-ut/nni_annotation/testcase/usercode/non_annotation/bar.py diff --git a/tools/nni_annotation/testcase/usercode/non_annotation/foo.py b/archive-ut/nni_annotation/testcase/usercode/non_annotation/foo.py similarity index 100% rename from tools/nni_annotation/testcase/usercode/non_annotation/foo.py rename to archive-ut/nni_annotation/testcase/usercode/non_annotation/foo.py diff --git a/tools/nni_annotation/testcase/usercode/nonpy.txt b/archive-ut/nni_annotation/testcase/usercode/nonpy.txt similarity index 100% rename from tools/nni_annotation/testcase/usercode/nonpy.txt rename to archive-ut/nni_annotation/testcase/usercode/nonpy.txt diff --git a/tools/nni_cmd/tests/config_files/invalid/custom-tuner-1.yml b/archive-ut/nni_cmd/tests/config_files/invalid/custom-tuner-1.yml similarity index 100% rename from tools/nni_cmd/tests/config_files/invalid/custom-tuner-1.yml rename to archive-ut/nni_cmd/tests/config_files/invalid/custom-tuner-1.yml diff --git a/tools/nni_cmd/tests/config_files/invalid/custom-tuner-2.yml b/archive-ut/nni_cmd/tests/config_files/invalid/custom-tuner-2.yml similarity index 100% rename from tools/nni_cmd/tests/config_files/invalid/custom-tuner-2.yml rename to archive-ut/nni_cmd/tests/config_files/invalid/custom-tuner-2.yml diff --git a/tools/nni_cmd/tests/config_files/invalid/mytuner.py b/archive-ut/nni_cmd/tests/config_files/invalid/mytuner.py similarity index 100% rename from tools/nni_cmd/tests/config_files/invalid/mytuner.py rename to archive-ut/nni_cmd/tests/config_files/invalid/mytuner.py diff --git a/tools/nni_cmd/tests/config_files/invalid/no-tuner.yml b/archive-ut/nni_cmd/tests/config_files/invalid/no-tuner.yml similarity index 100% rename from tools/nni_cmd/tests/config_files/invalid/no-tuner.yml rename to archive-ut/nni_cmd/tests/config_files/invalid/no-tuner.yml diff --git a/tools/nni_cmd/tests/config_files/invalid/search_space.json b/archive-ut/nni_cmd/tests/config_files/invalid/search_space.json similarity index 100% rename from tools/nni_cmd/tests/config_files/invalid/search_space.json rename to archive-ut/nni_cmd/tests/config_files/invalid/search_space.json diff --git a/tools/nni_cmd/tests/config_files/invalid/searchspace-path.yml b/archive-ut/nni_cmd/tests/config_files/invalid/searchspace-path.yml similarity index 100% rename from tools/nni_cmd/tests/config_files/invalid/searchspace-path.yml rename to archive-ut/nni_cmd/tests/config_files/invalid/searchspace-path.yml diff --git a/tools/nni_cmd/tests/config_files/invalid/tuner-wrong-key.yml b/archive-ut/nni_cmd/tests/config_files/invalid/tuner-wrong-key.yml similarity index 100% rename from tools/nni_cmd/tests/config_files/invalid/tuner-wrong-key.yml rename to archive-ut/nni_cmd/tests/config_files/invalid/tuner-wrong-key.yml diff --git a/tools/nni_cmd/tests/config_files/invalid/wrong-class-args.yml b/archive-ut/nni_cmd/tests/config_files/invalid/wrong-class-args.yml similarity index 100% rename from tools/nni_cmd/tests/config_files/invalid/wrong-class-args.yml rename to archive-ut/nni_cmd/tests/config_files/invalid/wrong-class-args.yml diff --git a/tools/nni_cmd/tests/config_files/invalid/wrong-training-service.yml b/archive-ut/nni_cmd/tests/config_files/invalid/wrong-training-service.yml similarity index 100% rename from tools/nni_cmd/tests/config_files/invalid/wrong-training-service.yml rename to archive-ut/nni_cmd/tests/config_files/invalid/wrong-training-service.yml diff --git a/tools/nni_cmd/tests/config_files/test_files/test_json.json b/archive-ut/nni_cmd/tests/config_files/test_files/test_json.json similarity index 100% rename from tools/nni_cmd/tests/config_files/test_files/test_json.json rename to archive-ut/nni_cmd/tests/config_files/test_files/test_json.json diff --git a/tools/nni_cmd/tests/config_files/test_files/test_yaml.yml b/archive-ut/nni_cmd/tests/config_files/test_files/test_yaml.yml similarity index 100% rename from tools/nni_cmd/tests/config_files/test_files/test_yaml.yml rename to archive-ut/nni_cmd/tests/config_files/test_files/test_yaml.yml diff --git a/tools/nni_cmd/tests/config_files/valid/main.py b/archive-ut/nni_cmd/tests/config_files/valid/main.py similarity index 100% rename from tools/nni_cmd/tests/config_files/valid/main.py rename to archive-ut/nni_cmd/tests/config_files/valid/main.py diff --git a/tools/nni_cmd/tests/config_files/valid/search_space.json b/archive-ut/nni_cmd/tests/config_files/valid/search_space.json similarity index 100% rename from tools/nni_cmd/tests/config_files/valid/search_space.json rename to archive-ut/nni_cmd/tests/config_files/valid/search_space.json diff --git a/tools/nni_cmd/tests/config_files/valid/test.yml b/archive-ut/nni_cmd/tests/config_files/valid/test.yml similarity index 100% rename from tools/nni_cmd/tests/config_files/valid/test.yml rename to archive-ut/nni_cmd/tests/config_files/valid/test.yml diff --git a/tools/nni_cmd/tests/mock/experiment.py b/archive-ut/nni_cmd/tests/mock/experiment.py similarity index 100% rename from tools/nni_cmd/tests/mock/experiment.py rename to archive-ut/nni_cmd/tests/mock/experiment.py diff --git a/tools/nni_cmd/tests/mock/nnictl_metadata/.experiment b/archive-ut/nni_cmd/tests/mock/nnictl_metadata/.experiment similarity index 100% rename from tools/nni_cmd/tests/mock/nnictl_metadata/.experiment rename to archive-ut/nni_cmd/tests/mock/nnictl_metadata/.experiment diff --git a/tools/nni_cmd/tests/mock/nnictl_metadata/aGew0x/.config b/archive-ut/nni_cmd/tests/mock/nnictl_metadata/aGew0x/.config similarity index 100% rename from tools/nni_cmd/tests/mock/nnictl_metadata/aGew0x/.config rename to archive-ut/nni_cmd/tests/mock/nnictl_metadata/aGew0x/.config diff --git a/tools/nni_cmd/tests/mock/nnictl_metadata/aGew0x/stderr b/archive-ut/nni_cmd/tests/mock/nnictl_metadata/aGew0x/stderr similarity index 100% rename from tools/nni_cmd/tests/mock/nnictl_metadata/aGew0x/stderr rename to archive-ut/nni_cmd/tests/mock/nnictl_metadata/aGew0x/stderr diff --git a/tools/nni_cmd/tests/mock/nnictl_metadata/aGew0x/stdout b/archive-ut/nni_cmd/tests/mock/nnictl_metadata/aGew0x/stdout similarity index 100% rename from tools/nni_cmd/tests/mock/nnictl_metadata/aGew0x/stdout rename to archive-ut/nni_cmd/tests/mock/nnictl_metadata/aGew0x/stdout diff --git a/tools/nni_cmd/tests/mock/nnictl_metadata/config/.config b/archive-ut/nni_cmd/tests/mock/nnictl_metadata/config/.config similarity index 100% rename from tools/nni_cmd/tests/mock/nnictl_metadata/config/.config rename to archive-ut/nni_cmd/tests/mock/nnictl_metadata/config/.config diff --git a/tools/nni_cmd/tests/mock/restful_server.py b/archive-ut/nni_cmd/tests/mock/restful_server.py similarity index 100% rename from tools/nni_cmd/tests/mock/restful_server.py rename to archive-ut/nni_cmd/tests/mock/restful_server.py diff --git a/tools/nni_cmd/tests/test_common_utils.py b/archive-ut/nni_cmd/tests/test_common_utils.py similarity index 100% rename from tools/nni_cmd/tests/test_common_utils.py rename to archive-ut/nni_cmd/tests/test_common_utils.py diff --git a/tools/nni_cmd/tests/test_config_utils.py b/archive-ut/nni_cmd/tests/test_config_utils.py similarity index 100% rename from tools/nni_cmd/tests/test_config_utils.py rename to archive-ut/nni_cmd/tests/test_config_utils.py diff --git a/tools/nni_cmd/tests/test_config_validation.py b/archive-ut/nni_cmd/tests/test_config_validation.py similarity index 100% rename from tools/nni_cmd/tests/test_config_validation.py rename to archive-ut/nni_cmd/tests/test_config_validation.py diff --git a/tools/nni_cmd/tests/test_nnictl_utils.py b/archive-ut/nni_cmd/tests/test_nnictl_utils.py similarity index 100% rename from tools/nni_cmd/tests/test_nnictl_utils.py rename to archive-ut/nni_cmd/tests/test_nnictl_utils.py diff --git a/tools/nni_trial_tool/test/__init__.py b/archive-ut/nni_trial_tool/test/__init__.py similarity index 100% rename from tools/nni_trial_tool/test/__init__.py rename to archive-ut/nni_trial_tool/test/__init__.py diff --git a/tools/nni_trial_tool/test/test_file_channel.py b/archive-ut/nni_trial_tool/test/test_file_channel.py similarity index 100% rename from tools/nni_trial_tool/test/test_file_channel.py rename to archive-ut/nni_trial_tool/test/test_file_channel.py diff --git a/tools/nni_trial_tool/test/test_hdfsClientUtility.py b/archive-ut/nni_trial_tool/test/test_hdfsClientUtility.py similarity index 100% rename from tools/nni_trial_tool/test/test_hdfsClientUtility.py rename to archive-ut/nni_trial_tool/test/test_hdfsClientUtility.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1448b1b98c..8a20e51b74 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -14,11 +14,11 @@ jobs: echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}" displayName: 'Install python tools' - script: | - source install.sh + python3 setup.py develop displayName: 'Install nni toolkit via source code' - script: | set -e - cd src/nni_manager + cd ts/nni_manager yarn eslint # uncomment following 2 lines to enable webui eslint cd ../webui @@ -45,10 +45,10 @@ jobs: cd test python3 nni_test/nnitest/run_tests.py --config config/pr_tests.yml displayName: 'Simple test' - - script: | - cd docs/en_US/ - sphinx-build -M html . _build -W - displayName: 'Sphinx Documentation Build check' + #- script: | + # cd docs/en_US/ + # sphinx-build -M html . _build -W + # displayName: 'Sphinx Documentation Build check' - job: 'ubuntu_1804_python36_legacy_torch_tf' pool: @@ -64,7 +64,7 @@ jobs: echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}" displayName: 'Install python tools' - script: | - source install.sh + python3 setup.py develop displayName: 'Install nni toolkit via source code' - script: | set -e @@ -78,18 +78,14 @@ jobs: displayName: 'Install dependencies' - script: | set -e - python3 -m pylint --rcfile pylintrc nni_annotation - python3 -m pylint --rcfile pylintrc nni_cmd - python3 -m pylint --rcfile pylintrc nni_gpu_tool - python3 -m pylint --rcfile pylintrc nni_trial_tool python3 -m pylint --rcfile pylintrc nni - python3 -m pylint --rcfile pylintrc nnicli displayName: 'Run pylint' - script: | set -e python3 -m pip install flake8 --user - EXCLUDES=./src/nni_manager/,./src/webui,./tools/nni_annotation/testcase/,./examples/trials/mnist-nas/*/mnist*.py,./examples/trials/nas_cifar10/src/cifar10/general_child.py - python3 -m flake8 . --count --exclude=$EXCLUDES --select=E9,F63,F72,F82 --show-source --statistics + python3 -m flake8 nni --count --select=E9,F63,F72,F82 --show-source --statistics + EXCLUDES=examples/trials/mnist-nas/*/mnist*.py,examples/trials/nas_cifar10/src/cifar10/general_child.py + python3 -m flake8 examples --count --exclude=$EXCLUDES --select=E9,F63,F72,F82 --show-source --statistics displayName: 'Run flake8 tests to find Python syntax errors and undefined names' - script: | cd test @@ -101,58 +97,58 @@ jobs: displayName: 'Simple test' -- job: 'macos_1015_python37' - pool: - vmImage: 'macOS-10.15' - - steps: - - script: | - export PYTHON37_BIN_DIR=/usr/local/Cellar/python@3.7/`ls /usr/local/Cellar/python@3.7`/bin - echo "##vso[task.setvariable variable=PATH]${PYTHON37_BIN_DIR}:${HOME}/Library/Python/3.7/bin:${PATH}" - python3 -m pip install --upgrade pip setuptools - displayName: 'Install python tools' - - script: | - echo "network-timeout 600000" >> ${HOME}/.yarnrc - source install.sh - displayName: 'Install nni toolkit via source code' - - script: | - set -e - # pytorch Mac binary does not support CUDA, default is cpu version - python3 -m pip install torchvision==0.6.0 torch==1.5.0 --user - python3 -m pip install tensorflow==1.15.2 --user - brew install swig@3 - rm -f /usr/local/bin/swig - ln -s /usr/local/opt/swig\@3/bin/swig /usr/local/bin/swig - nnictl package install --name=SMAC - displayName: 'Install dependencies' - - script: | - cd test - source scripts/unittest.sh - displayName: 'Unit test' - - script: | - cd test - python3 nni_test/nnitest/run_tests.py --config config/pr_tests.yml - displayName: 'Simple test' - -- job: 'win2016_python37' - pool: - vmImage: 'vs2017-win2016' - - steps: - - script: | - powershell.exe -file install.ps1 - displayName: 'Install nni toolkit via source code' - - script: | - python -m pip install scikit-learn==0.23.2 --user - python -m pip install keras==2.1.6 --user - python -m pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html --user - python -m pip install tensorflow==1.15.2 --user - displayName: 'Install dependencies' - - script: | - cd test - powershell.exe -file scripts/unittest.ps1 - displayName: 'unit test' - - script: | - cd test - python nni_test/nnitest/run_tests.py --config config/pr_tests.yml - displayName: 'Simple test' +#- job: 'macos_latest_python38' +# pool: +# vmImage: 'macOS-latest' +# +# steps: +# - script: | +# export PYTHON38_BIN_DIR=/usr/local/Cellar/python@3.8/`ls /usr/local/Cellar/python@3.8`/bin +# echo "##vso[task.setvariable variable=PATH]${PYTHON38_BIN_DIR}:${HOME}/Library/Python/3.8/bin:${PATH}" +# python3 -m pip install --upgrade pip setuptools +# displayName: 'Install python tools' +# - script: | +# echo "network-timeout 600000" >> ${HOME}/.yarnrc +# source install.sh +# displayName: 'Install nni toolkit via source code' +# - script: | +# set -e +# # pytorch Mac binary does not support CUDA, default is cpu version +# python3 -m pip install torchvision==0.6.0 torch==1.5.0 --user +# python3 -m pip install tensorflow==2.2 --user +# brew install swig@3 +# rm -f /usr/local/bin/swig +# ln -s /usr/local/opt/swig\@3/bin/swig /usr/local/bin/swig +# nnictl package install --name=SMAC +# displayName: 'Install dependencies' +# - script: | +# cd test +# source scripts/unittest.sh +# displayName: 'Unit test' +# - script: | +# cd test +# python3 nni_test/nnitest/run_tests.py --config config/pr_tests.yml +# displayName: 'Simple test' +# +#- job: 'win2016_python37' +# pool: +# vmImage: 'vs2017-win2016' +# +# steps: +# - script: | +# powershell.exe -file install.ps1 +# displayName: 'Install nni toolkit via source code' +# - script: | +# python -m pip install scikit-learn==0.23.2 --user +# python -m pip install keras==2.1.6 --user +# python -m pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html --user +# python -m pip install tensorflow==1.15.2 --user +# displayName: 'Install dependencies' +# - script: | +# cd test +# powershell.exe -file scripts/unittest.ps1 +# displayName: 'unit test' +# - script: | +# cd test +# python nni_test/nnitest/run_tests.py --config config/pr_tests.yml +# displayName: 'Simple test' diff --git a/deployment/deployment-pipelines.yml b/deployment/deployment-pipelines.yml index 66ba8fddf7..8b05f4e6ff 100644 --- a/deployment/deployment-pipelines.yml +++ b/deployment/deployment-pipelines.yml @@ -113,10 +113,6 @@ jobs: condition: succeeded() pool: vmImage: 'macOS-10.15' - strategy: - matrix: - Python36: - PYTHON_VERSION: '3.6' steps: - script: | python3 -m pip install --upgrade pip setuptools --user @@ -134,10 +130,10 @@ jobs: # NNI build scripts (Makefile) uses branch tag as package version number git tag $(build_version) echo 'building prerelease package...' - PATH=$HOME/Library/Python/3.7/bin:$PATH make version_ts=true build + PATH=$HOME/Library/Python/3.8/bin:$PATH make version_ts=true build else echo 'building release package...' - PATH=$HOME/Library/Python/3.7/bin:$PATH make build + PATH=$HOME/Library/Python/3.8/bin:$PATH make build fi condition: eq( variables['upload_package'], 'true') displayName: 'build nni bdsit_wheel' diff --git a/docs/en_US/Assessor/CustomizeAssessor.md b/docs/en_US/Assessor/CustomizeAssessor.md index 47f3debf1c..1567f4e282 100644 --- a/docs/en_US/Assessor/CustomizeAssessor.md +++ b/docs/en_US/Assessor/CustomizeAssessor.md @@ -57,5 +57,5 @@ Please noted in **2**. The object `trial_history` are exact the object that Tria The working directory of your assessor is `/nni-experiments//log`, which can be retrieved with environment variable `NNI_LOG_DIRECTORY`, More detail example you could see: -> * [medianstop-assessor](https://github.com/Microsoft/nni/tree/master/src/sdk/pynni/nni/medianstop_assessor) -> * [curvefitting-assessor](https://github.com/Microsoft/nni/tree/master/src/sdk/pynni/nni/curvefitting_assessor) \ No newline at end of file +> * [medianstop-assessor](https://github.com/Microsoft/nni/tree/v1.9/src/sdk/pynni/nni/medianstop_assessor) +> * [curvefitting-assessor](https://github.com/Microsoft/nni/tree/v1.9/src/sdk/pynni/nni/curvefitting_assessor) \ No newline at end of file diff --git a/docs/en_US/CommunitySharings/AutoCompletion.md b/docs/en_US/CommunitySharings/AutoCompletion.md index f7da5dabb2..1d8b034625 100644 --- a/docs/en_US/CommunitySharings/AutoCompletion.md +++ b/docs/en_US/CommunitySharings/AutoCompletion.md @@ -18,7 +18,7 @@ For now, auto-completion will not be enabled by default if you install NNI throu cd ~ wget https://raw.githubusercontent.com/microsoft/nni/{nni-version}/tools/bash-completion ``` -Here, {nni-version} should by replaced by the version of NNI, e.g., `master`, `v1.9`. You can also check the latest `bash-completion` script [here](https://github.com/microsoft/nni/blob/master/tools/bash-completion). +Here, {nni-version} should by replaced by the version of NNI, e.g., `master`, `v1.9`. You can also check the latest `bash-completion` script [here](https://github.com/microsoft/nni/blob/v1.9/tools/bash-completion). ### Step 2. Install the script If you are running a root account and want to install this script for all the users diff --git a/docs/en_US/CommunitySharings/ModelCompressionComparison.md b/docs/en_US/CommunitySharings/ModelCompressionComparison.md index 9d79076d4f..5b9ebf8375 100644 --- a/docs/en_US/CommunitySharings/ModelCompressionComparison.md +++ b/docs/en_US/CommunitySharings/ModelCompressionComparison.md @@ -9,7 +9,7 @@ In addition, we provide friendly instructions on the re-implementation of these The experiments are performed with the following pruners/datasets/models: -* Models: [VGG16, ResNet18, ResNet50](https://github.com/microsoft/nni/tree/master/examples/model_compress/models/cifar10) +* Models: [VGG16, ResNet18, ResNet50](https://github.com/microsoft/nni/tree/v1.9/examples/model_compress/models/cifar10) * Datasets: CIFAR-10 @@ -23,7 +23,7 @@ The experiments are performed with the following pruners/datasets/models: For the pruners with scheduling, `L1Filter Pruner` is used as the base algorithm. That is to say, after the sparsities distribution is decided by the scheduling algorithm, `L1Filter Pruner` is used to performn real pruning. - - All the pruners listed above are implemented in [nni](https://github.com/microsoft/nni/tree/master/docs/en_US/Compression/Overview.md). + - All the pruners listed above are implemented in [nni](https://github.com/microsoft/nni/tree/v1.9/docs/en_US/Compression/Overview.md). ## Experiment Result @@ -60,14 +60,14 @@ From the experiment result, we get the following conclusions: * The experiment results are all collected with the default configuration of the pruners in nni, which means that when we call a pruner class in nni, we don't change any default class arguments. -* Both FLOPs and the number of parameters are counted with [Model FLOPs/Parameters Counter](https://github.com/microsoft/nni/tree/master/docs/en_US/Compression/CompressionUtils.md#model-flopsparameters-counter) after [model speed up](https://github.com/microsoft/nni/tree/master/docs/en_US/Compression/ModelSpeedup.md). +* Both FLOPs and the number of parameters are counted with [Model FLOPs/Parameters Counter](https://github.com/microsoft/nni/tree/v1.9/docs/en_US/Compression/CompressionUtils.md#model-flopsparameters-counter) after [model speed up](https://github.com/microsoft/nni/tree/v1.9/docs/en_US/Compression/ModelSpeedup.md). This avoids potential issues of counting them of masked models. -* The experiment code can be found [here]( https://github.com/microsoft/nni/tree/master/examples/model_compress/auto_pruners_torch.py). +* The experiment code can be found [here]( https://github.com/microsoft/nni/tree/v1.9/examples/model_compress/auto_pruners_torch.py). ### Experiment Result Rendering -* If you follow the practice in the [example]( https://github.com/microsoft/nni/tree/master/examples/model_compress/auto_pruners_torch.py), for every single pruning experiment, the experiment result will be saved in JSON format as follows: +* If you follow the practice in the [example]( https://github.com/microsoft/nni/tree/v1.9/examples/model_compress/auto_pruners_torch.py), for every single pruning experiment, the experiment result will be saved in JSON format as follows: ``` json { "performance": {"original": 0.9298, "pruned": 0.1, "speedup": 0.1, "finetuned": 0.7746}, @@ -76,8 +76,8 @@ This avoids potential issues of counting them of masked models. } ``` -* The experiment results are saved [here](https://github.com/microsoft/nni/tree/master/examples/model_compress/comparison_of_pruners). -You can refer to [analyze](https://github.com/microsoft/nni/tree/master/examples/model_compress/comparison_of_pruners/analyze.py) to plot new performance comparison figures. +* The experiment results are saved [here](https://github.com/microsoft/nni/tree/v1.9/examples/model_compress/comparison_of_pruners). +You can refer to [analyze](https://github.com/microsoft/nni/tree/v1.9/examples/model_compress/comparison_of_pruners/analyze.py) to plot new performance comparison figures. ## Contribution diff --git a/docs/en_US/Compression/AutoPruningUsingTuners.md b/docs/en_US/Compression/AutoPruningUsingTuners.md index 36ccf85f06..d9557d820d 100644 --- a/docs/en_US/Compression/AutoPruningUsingTuners.md +++ b/docs/en_US/Compression/AutoPruningUsingTuners.md @@ -13,7 +13,7 @@ pruner = LevelPruner(model, config_list) pruner.compress() ``` -The 'default' op_type stands for the module types defined in [default_layers.py](https://github.com/microsoft/nni/blob/master/src/sdk/pynni/nni/compression/torch/default_layers.py) for pytorch. +The 'default' op_type stands for the module types defined in [default_layers.py](https://github.com/microsoft/nni/blob/v1.9/src/sdk/pynni/nni/compression/torch/default_layers.py) for pytorch. Therefore ```{ 'sparsity': 0.8, 'op_types': ['default'] }```means that **all layers with specified op_types will be compressed with the same 0.8 sparsity**. When ```pruner.compress()``` called, the model is compressed with masks and after that you can normally fine tune this model and **pruned weights won't be updated** which have been masked. diff --git a/docs/en_US/Compression/CompressionUtils.md b/docs/en_US/Compression/CompressionUtils.md index c5f6f2746c..590f02f774 100644 --- a/docs/en_US/Compression/CompressionUtils.md +++ b/docs/en_US/Compression/CompressionUtils.md @@ -120,7 +120,7 @@ from nni.compression.torch.utils.mask_conflict import fix_mask_conflict fixed_mask = fix_mask_conflict('./resnet18_mask', net, data) ``` -### Model FLOPs/Parameters Counter +## Model FLOPs/Parameters Counter We provide a model counter for calculating the model FLOPs and parameters. This counter supports calculating FLOPs/parameters of a normal model without masks, it can also calculates FLOPs/parameters of a model with mask wrappers, which helps users easily check model complexity during model compression on NNI. Note that, for sturctured pruning, we only identify the remained filters according to its mask, which not taking the pruned input channels into consideration, so the calculated FLOPs will be larger than real number (i.e., the number calculated after Model Speedup). ### Usage diff --git a/docs/en_US/Compression/CustomizeCompressor.md b/docs/en_US/Compression/CustomizeCompressor.md index a894bbac7a..e11f17e866 100644 --- a/docs/en_US/Compression/CustomizeCompressor.md +++ b/docs/en_US/Compression/CustomizeCompressor.md @@ -29,7 +29,7 @@ class MyMasker(WeightMasker): return {'weight_mask': mask} ``` -You can reference nni provided [weight masker](https://github.com/microsoft/nni/blob/master/src/sdk/pynni/nni/compression/torch/pruning/structured_pruning.py) implementations to implement your own weight masker. +You can reference nni provided [weight masker](https://github.com/microsoft/nni/blob/v1.9/src/sdk/pynni/nni/compression/torch/pruning/structured_pruning.py) implementations to implement your own weight masker. A basic `pruner` looks likes this: @@ -54,7 +54,7 @@ class MyPruner(Pruner): ``` -Reference nni provided [pruner](https://github.com/microsoft/nni/blob/master/src/sdk/pynni/nni/compression/torch/pruning/one_shot.py) implementations to implement your own pruner class. +Reference nni provided [pruner](https://github.com/microsoft/nni/blob/v1.9/src/sdk/pynni/nni/compression/torch/pruning/one_shot.py) implementations to implement your own pruner class. *** diff --git a/docs/en_US/Compression/Framework.md b/docs/en_US/Compression/Framework.md index 6b271343b5..cc9b2691da 100644 --- a/docs/en_US/Compression/Framework.md +++ b/docs/en_US/Compression/Framework.md @@ -48,7 +48,7 @@ quantizer = DoReFaQuantizer(model, configure_list, optimizer) quantizer.compress() ``` -View [example code](https://github.com/microsoft/nni/tree/master/examples/model_compress) for more information. +View [example code](https://github.com/microsoft/nni/tree/v1.9/examples/model_compress) for more information. `Compressor` class provides some utility methods for subclass and users: diff --git a/docs/en_US/Compression/ModelSpeedup.md b/docs/en_US/Compression/ModelSpeedup.md index 4158532634..6d1eec4cc2 100644 --- a/docs/en_US/Compression/ModelSpeedup.md +++ b/docs/en_US/Compression/ModelSpeedup.md @@ -32,7 +32,7 @@ start = time.time() out = model(dummy_input) print('elapsed time: ', time.time() - start) ``` -For complete examples please refer to [the code](https://github.com/microsoft/nni/tree/master/examples/model_compress/model_speedup.py) +For complete examples please refer to [the code](https://github.com/microsoft/nni/tree/v1.9/examples/model_compress/model_speedup.py) NOTE: The current implementation supports PyTorch 1.3.1 or newer. @@ -44,7 +44,7 @@ For PyTorch we can only replace modules, if functions in `forward` should be rep ## Speedup Results of Examples -The code of these experiments can be found [here](https://github.com/microsoft/nni/tree/master/examples/model_compress/model_speedup.py). +The code of these experiments can be found [here](https://github.com/microsoft/nni/tree/v1.9/examples/model_compress/model_speedup.py). ### slim pruner example diff --git a/docs/en_US/Compression/Overview.md b/docs/en_US/Compression/Overview.md index a0b4d42112..c1c6cb55d2 100644 --- a/docs/en_US/Compression/Overview.md +++ b/docs/en_US/Compression/Overview.md @@ -41,8 +41,9 @@ Pruning algorithms compress the original network by removing redundant weights o | [NetAdapt Pruner](https://nni.readthedocs.io/en/latest/Compression/Pruner.html#netadapt-pruner) | Automatically simplify a pretrained network to meet the resource budget by iterative pruning [Reference Paper](https://arxiv.org/abs/1804.03230) | | [SimulatedAnnealing Pruner](https://nni.readthedocs.io/en/latest/Compression/Pruner.html#simulatedannealing-pruner) | Automatic pruning with a guided heuristic search method, Simulated Annealing algorithm [Reference Paper](https://arxiv.org/abs/1907.03141) | | [AutoCompress Pruner](https://nni.readthedocs.io/en/latest/Compression/Pruner.html#autocompress-pruner) | Automatic pruning by iteratively call SimulatedAnnealing Pruner and ADMM Pruner [Reference Paper](https://arxiv.org/abs/1907.03141) | +| [AMC Pruner](https://nni.readthedocs.io/en/latest/Compression/Pruner.html#amc-pruner) | AMC: AutoML for Model Compression and Acceleration on Mobile Devices [Reference Paper](https://arxiv.org/pdf/1802.03494.pdf) | -You can refer to this [benchmark](https://github.com/microsoft/nni/tree/master/docs/en_US/CommunitySharings/ModelCompressionComparison.md) for the performance of these pruners on some benchmark problems. +You can refer to this [benchmark](https://github.com/microsoft/nni/tree/v1.9/docs/en_US/CommunitySharings/ModelCompressionComparison.md) for the performance of these pruners on some benchmark problems. ### Quantization Algorithms diff --git a/docs/en_US/Compression/Pruner.md b/docs/en_US/Compression/Pruner.md index fe81c3f0b9..1dd6c491c2 100644 --- a/docs/en_US/Compression/Pruner.md +++ b/docs/en_US/Compression/Pruner.md @@ -20,7 +20,7 @@ We provide several pruning algorithms that support fine-grained weight pruning a * [NetAdapt Pruner](#netadapt-pruner) * [SimulatedAnnealing Pruner](#simulatedannealing-pruner) * [AutoCompress Pruner](#autocompress-pruner) -* [AutoML for Model Compression Pruner](#automl-for-model-compression-pruner) +* [AMC Pruner](#amc-pruner) * [Sensitivity Pruner](#sensitivity-pruner) **Others** @@ -102,7 +102,7 @@ We implemented one of the experiments in ['Learning Efficient Convolutional Netw | VGGNet | 6.34/6.40 | 20.04M | | | Pruned-VGGNet | 6.20/6.26 | 2.03M | 88.5% | -The experiments code can be found at [examples/model_compress]( https://github.com/microsoft/nni/tree/master/examples/model_compress/) +The experiments code can be found at [examples/model_compress]( https://github.com/microsoft/nni/tree/v1.9/examples/model_compress/) *** @@ -185,7 +185,7 @@ We implemented one of the experiments in ['PRUNING FILTERS FOR EFFICIENT CONVNET | VGG-16 | 6.75/6.49 | 1.5x10^7 | | | VGG-16-pruned-A | 6.60/6.47 | 5.4x10^6 | 64.0% | -The experiments code can be found at [examples/model_compress]( https://github.com/microsoft/nni/tree/master/examples/model_compress/) +The experiments code can be found at [examples/model_compress]( https://github.com/microsoft/nni/tree/v1.9/examples/model_compress/) *** @@ -242,7 +242,7 @@ pruner.compress() Note: ActivationAPoZRankFilterPruner is used to prune convolutional layers within deep neural networks, therefore the `op_types` field supports only convolutional layers. -You can view [example](https://github.com/microsoft/nni/blob/master/examples/model_compress/model_prune_torch.py) for more information. +You can view [example](https://github.com/microsoft/nni/blob/v1.9/examples/model_compress/model_prune_torch.py) for more information. @@ -277,7 +277,7 @@ pruner.compress() Note: ActivationMeanRankFilterPruner is used to prune convolutional layers within deep neural networks, therefore the `op_types` field supports only convolutional layers. -You can view [example](https://github.com/microsoft/nni/blob/master/examples/model_compress/model_prune_torch.py) for more information. +You can view [example](https://github.com/microsoft/nni/blob/v1.9/examples/model_compress/model_prune_torch.py) for more information. ### User configuration for ActivationMeanRankFilterPruner @@ -376,7 +376,7 @@ PyTorch code ```python pruner.update_epoch(epoch) ``` -You can view [example](https://github.com/microsoft/nni/blob/master/examples/model_compress/model_prune_torch.py) for more information. +You can view [example](https://github.com/microsoft/nni/blob/v1.9/examples/model_compress/model_prune_torch.py) for more information. #### User configuration for AGP Pruner @@ -410,7 +410,7 @@ pruner = NetAdaptPruner(model, config_list, short_term_fine_tuner=short_term_fin pruner.compress() ``` -You can view [example](https://github.com/microsoft/nni/blob/master/examples/model_compress/auto_pruners_torch.py) for more information. +You can view [example](https://github.com/microsoft/nni/blob/v1.9/examples/model_compress/auto_pruners_torch.py) for more information. #### User configuration for NetAdapt Pruner @@ -449,7 +449,7 @@ pruner = SimulatedAnnealingPruner(model, config_list, evaluator=evaluator, base_ pruner.compress() ``` -You can view [example](https://github.com/microsoft/nni/blob/master/examples/model_compress/auto_pruners_torch.py) for more information. +You can view [example](https://github.com/microsoft/nni/blob/v1.9/examples/model_compress/auto_pruners_torch.py) for more information. #### User configuration for SimulatedAnnealing Pruner @@ -485,7 +485,7 @@ pruner = AutoCompressPruner( pruner.compress() ``` -You can view [example](https://github.com/microsoft/nni/blob/master/examples/model_compress/auto_pruners_torch.py) for more information. +You can view [example](https://github.com/microsoft/nni/blob/v1.9/examples/model_compress/auto_pruners_torch.py) for more information. #### User configuration for AutoCompress Pruner @@ -495,9 +495,9 @@ You can view [example](https://github.com/microsoft/nni/blob/master/examples/mod .. autoclass:: nni.compression.torch.AutoCompressPruner ``` -## AutoML for Model Compression Pruner +## AMC Pruner -AutoML for Model Compression Pruner (AMCPruner) leverages reinforcement learning to provide the model compression policy. +AMC pruner leverages reinforcement learning to provide the model compression policy. This learning-based compression policy outperforms conventional rule-based compression policy by having higher compression ratio, better preserving the accuracy and freeing human labor. @@ -519,7 +519,7 @@ pruner = AMCPruner(model, config_list, evaluator, val_loader, flops_ratio=0.5) pruner.compress() ``` -You can view [example](https://github.com/microsoft/nni/blob/master/examples/model_compress/amc/) for more information. +You can view [example](https://github.com/microsoft/nni/blob/v1.9/examples/model_compress/amc/) for more information. #### User configuration for AutoCompress Pruner @@ -537,7 +537,7 @@ We implemented one of the experiments in [AMC: AutoML for Model Compression and | ------------- | --------------| -------------- | ----- | | MobileNet | 70.5% / 69.9% | 89.3% / 89.1% | 50% | -The experiments code can be found at [examples/model_compress]( https://github.com/microsoft/nni/tree/master/examples/model_compress/amc/) +The experiments code can be found at [examples/model_compress]( https://github.com/microsoft/nni/tree/v1.9/examples/model_compress/amc/) ## ADMM Pruner Alternating Direction Method of Multipliers (ADMM) is a mathematical optimization technique, @@ -568,7 +568,7 @@ pruner = ADMMPruner(model, config_list, trainer=trainer, num_iterations=30, epoc pruner.compress() ``` -You can view [example](https://github.com/microsoft/nni/blob/master/examples/model_compress/auto_pruners_torch.py) for more information. +You can view [example](https://github.com/microsoft/nni/blob/v1.9/examples/model_compress/auto_pruners_torch.py) for more information. #### User configuration for ADMM Pruner @@ -624,7 +624,7 @@ The above configuration means that there are 5 times of iterative pruning. As th ### Reproduced Experiment -We try to reproduce the experiment result of the fully connected network on MNIST using the same configuration as in the paper. The code can be referred [here](https://github.com/microsoft/nni/tree/master/examples/model_compress/lottery_torch_mnist_fc.py). In this experiment, we prune 10 times, for each pruning we train the pruned model for 50 epochs. +We try to reproduce the experiment result of the fully connected network on MNIST using the same configuration as in the paper. The code can be referred [here](https://github.com/microsoft/nni/tree/v1.9/examples/model_compress/lottery_torch_mnist_fc.py). In this experiment, we prune 10 times, for each pruning we train the pruned model for 50 epochs. ![](../../img/lottery_ticket_mnist_fc.png) diff --git a/docs/en_US/Compression/Quantizer.md b/docs/en_US/Compression/Quantizer.md index 55fde6db66..bc80e5c137 100644 --- a/docs/en_US/Compression/Quantizer.md +++ b/docs/en_US/Compression/Quantizer.md @@ -129,7 +129,7 @@ quantizer = BNNQuantizer(model, configure_list) model = quantizer.compress() ``` -You can view example [examples/model_compress/BNN_quantizer_cifar10.py]( https://github.com/microsoft/nni/tree/master/examples/model_compress/BNN_quantizer_cifar10.py) for more information. +You can view example [examples/model_compress/BNN_quantizer_cifar10.py]( https://github.com/microsoft/nni/tree/v1.9/examples/model_compress/BNN_quantizer_cifar10.py) for more information. #### User configuration for BNN Quantizer @@ -146,4 +146,4 @@ We implemented one of the experiments in [Binarized Neural Networks: Training De | VGGNet | 86.93% | -The experiments code can be found at [examples/model_compress/BNN_quantizer_cifar10.py]( https://github.com/microsoft/nni/tree/master/examples/model_compress/BNN_quantizer_cifar10.py) \ No newline at end of file +The experiments code can be found at [examples/model_compress/BNN_quantizer_cifar10.py]( https://github.com/microsoft/nni/tree/v1.9/examples/model_compress/BNN_quantizer_cifar10.py) \ No newline at end of file diff --git a/docs/en_US/Compression/QuickStart.md b/docs/en_US/Compression/QuickStart.md index 81c92b9099..f0be1b0f32 100644 --- a/docs/en_US/Compression/QuickStart.md +++ b/docs/en_US/Compression/QuickStart.md @@ -42,7 +42,7 @@ After training, you get accuracy of the pruned model. You can export model weigh pruner.export_model(model_path='pruned_vgg19_cifar10.pth', mask_path='mask_vgg19_cifar10.pth') ``` -The complete code of model compression examples can be found [here](https://github.com/microsoft/nni/blob/master/examples/model_compress/model_prune_torch.py). +The complete code of model compression examples can be found [here](https://github.com/microsoft/nni/blob/v1.9/examples/model_compress/model_prune_torch.py). ### Speed up the model diff --git a/docs/en_US/FeatureEngineering/Overview.md b/docs/en_US/FeatureEngineering/Overview.md index 22eeeb25be..7ae48e57a6 100644 --- a/docs/en_US/FeatureEngineering/Overview.md +++ b/docs/en_US/FeatureEngineering/Overview.md @@ -11,7 +11,7 @@ These selectors are suitable for tabular data(which means it doesn't include ima In addition, those selector only for feature selection. If you want to: 1) generate high-order combined features on nni while doing feature selection; 2) leverage your distributed resources; -you could try this [example](https://github.com/microsoft/nni/tree/master/examples/feature_engineering/auto-feature-engineering). +you could try this [example](https://github.com/microsoft/nni/tree/v1.9/examples/feature_engineering/auto-feature-engineering). ## How to use? @@ -102,7 +102,7 @@ class CustomizedSelector(FeatureSelector): **3. Integrate with Sklearn** `sklearn.pipeline.Pipeline` can connect models in series, such as feature selector, normalization, and classification/regression to form a typical machine learning problem workflow. -The following step could help us to better integrate with sklearn, which means we could treat the customized feature selector as a mudule of the pipeline. +The following step could help us to better integrate with sklearn, which means we could treat the customized feature selector as a module of the pipeline. 1. Inherit the calss _sklearn.base.BaseEstimator_ 1. Implement _get_params_ and _set_params_ function in _BaseEstimator_ @@ -266,6 +266,6 @@ The code could be refenrence `/examples/feature_engineering/gradient_feature_sel ## Reference and Feedback * To [report a bug](https://github.com/microsoft/nni/issues/new?template=bug-report.md) for this feature in GitHub; * To [file a feature or improvement request](https://github.com/microsoft/nni/issues/new?template=enhancement.md) for this feature in GitHub; -* To know more about [Neural Architecture Search with NNI](https://github.com/microsoft/nni/blob/master/docs/en_US/NAS/Overview.md); -* To know more about [Model Compression with NNI](https://github.com/microsoft/nni/blob/master/docs/en_US/Compression/Overview.md); -* To know more about [Hyperparameter Tuning with NNI](https://github.com/microsoft/nni/blob/master/docs/en_US/Tuner/BuiltinTuner.md); +* To know more about [Neural Architecture Search with NNI](https://github.com/microsoft/nni/blob/v1.9/docs/en_US/NAS/Overview.md); +* To know more about [Model Compression with NNI](https://github.com/microsoft/nni/blob/v1.9/docs/en_US/Compression/Overview.md); +* To know more about [Hyperparameter Tuning with NNI](https://github.com/microsoft/nni/blob/v1.9/docs/en_US/Tuner/BuiltinTuner.md); diff --git a/docs/en_US/NAS/Advanced.md b/docs/en_US/NAS/Advanced.md index 86041c6149..063ebe80f6 100644 --- a/docs/en_US/NAS/Advanced.md +++ b/docs/en_US/NAS/Advanced.md @@ -76,9 +76,9 @@ class RandomMutator(Mutator): return self.sample_search() # use the same logic here. you can do something different ``` -The complete example of random mutator can be found [here](https://github.com/microsoft/nni/blob/master/src/sdk/pynni/nni/nas/pytorch/random/mutator.py). +The complete example of random mutator can be found [here](https://github.com/microsoft/nni/blob/v1.9/src/sdk/pynni/nni/nas/pytorch/random/mutator.py). -For advanced usages, e.g., users want to manipulate the way modules in `LayerChoice` are executed, they can inherit `BaseMutator`, and overwrite `on_forward_layer_choice` and `on_forward_input_choice`, which are the callback implementation of `LayerChoice` and `InputChoice` respectively. Users can still use property `mutables` to get all `LayerChoice` and `InputChoice` in the model code. For details, please refer to [reference](https://github.com/microsoft/nni/tree/master/src/sdk/pynni/nni/nas/pytorch) here to learn more. +For advanced usages, e.g., users want to manipulate the way modules in `LayerChoice` are executed, they can inherit `BaseMutator`, and overwrite `on_forward_layer_choice` and `on_forward_input_choice`, which are the callback implementation of `LayerChoice` and `InputChoice` respectively. Users can still use property `mutables` to get all `LayerChoice` and `InputChoice` in the model code. For details, please refer to [reference](https://github.com/microsoft/nni/tree/v1.9/src/sdk/pynni/nni/nas/pytorch) here to learn more. ```eval_rst .. tip:: diff --git a/docs/en_US/NAS/Benchmarks.md b/docs/en_US/NAS/Benchmarks.md index 8d169d9dd0..d90982616c 100644 --- a/docs/en_US/NAS/Benchmarks.md +++ b/docs/en_US/NAS/Benchmarks.md @@ -25,7 +25,7 @@ To avoid storage and legality issues, we do not provide any prepared databases. git clone -b ${NNI_VERSION} https://github.com/microsoft/nni cd nni/examples/nas/benchmarks ``` -Replace `${NNI_VERSION}` with a released version name or branch name, e.g., `v1.8`. +Replace `${NNI_VERSION}` with a released version name or branch name, e.g., `v1.9`. 2. Install dependencies via `pip3 install -r xxx.requirements.txt`. `xxx` can be `nasbench101`, `nasbench201` or `nds`. 3. Generate the database via `./xxx.sh`. The directory that stores the benchmark file can be configured with `NASBENCHMARK_DIR` environment variable, which defaults to `~/.nni/nasbenchmark`. Note that the NAS-Bench-201 dataset will be downloaded from a google drive. diff --git a/docs/en_US/NAS/CDARTS.md b/docs/en_US/NAS/CDARTS.md index 8a1f9d4d9f..3de2da8d44 100644 --- a/docs/en_US/NAS/CDARTS.md +++ b/docs/en_US/NAS/CDARTS.md @@ -19,7 +19,7 @@ This is CDARTS based on the NNI platform, which currently supports CIFAR10 searc ## Examples -[Example code](https://github.com/microsoft/nni/tree/master/examples/nas/cdarts) +[Example code](https://github.com/microsoft/nni/tree/v1.9/examples/nas/cdarts) ```bash # In case NNI code is not cloned. If the code is cloned already, ignore this line and enter code folder. diff --git a/docs/en_US/NAS/ClassicNas.md b/docs/en_US/NAS/ClassicNas.md index 3c6bd2b416..161e62c46a 100644 --- a/docs/en_US/NAS/ClassicNas.md +++ b/docs/en_US/NAS/ClassicNas.md @@ -24,9 +24,9 @@ At this point, trial code is ready. Then, we can prepare an NNI experiment, i.e. A file named `nni_auto_gen_search_space.json` is generated by this command. Then put the path of the generated search space in the field `searchSpacePath` of the experiment config file. The other fields of the config file can be filled by referring [this tutorial](../Tutorial/QuickStart.md). -Currently, we only support [PPO Tuner](../Tuner/BuiltinTuner.md), [Regularized Evolution Tuner](#regulaized-evolution-tuner) and [Random Tuner](https://github.com/microsoft/nni/tree/master/examples/tuners/random_nas_tuner) for classic NAS. More classic NAS algorithms will be supported soon. +Currently, we only support [PPO Tuner](../Tuner/BuiltinTuner.md), [Regularized Evolution Tuner](#regulaized-evolution-tuner) and [Random Tuner](https://github.com/microsoft/nni/tree/v1.9/examples/tuners/random_nas_tuner) for classic NAS. More classic NAS algorithms will be supported soon. -The complete examples can be found [here](https://github.com/microsoft/nni/tree/master/examples/nas/classic_nas) for PyTorch and [here](https://github.com/microsoft/nni/tree/master/examples/nas/classic_nas-tf) for TensorFlow. +The complete examples can be found [here](https://github.com/microsoft/nni/tree/v1.9/examples/nas/classic_nas) for PyTorch and [here](https://github.com/microsoft/nni/tree/v1.9/examples/nas/classic_nas-tf) for TensorFlow. ## Standalone mode for easy debugging diff --git a/docs/en_US/NAS/DARTS.md b/docs/en_US/NAS/DARTS.md index 8c1384cd25..4d9522b842 100644 --- a/docs/en_US/NAS/DARTS.md +++ b/docs/en_US/NAS/DARTS.md @@ -21,7 +21,7 @@ The above-mentioned example is meant to reproduce the results in the paper, we d ### CNN Search Space -[Example code](https://github.com/microsoft/nni/tree/master/examples/nas/darts) +[Example code](https://github.com/microsoft/nni/tree/v1.9/examples/nas/darts) ```bash # In case NNI code is not cloned. If the code is cloned already, ignore this line and enter code folder. diff --git a/docs/en_US/NAS/ENAS.md b/docs/en_US/NAS/ENAS.md index e6bae1ab9a..3119a8df71 100644 --- a/docs/en_US/NAS/ENAS.md +++ b/docs/en_US/NAS/ENAS.md @@ -10,7 +10,7 @@ Implementation on NNI is based on the [official implementation in Tensorflow](ht ### CIFAR10 Macro/Micro Search Space -[Example code](https://github.com/microsoft/nni/tree/master/examples/nas/enas) +[Example code](https://github.com/microsoft/nni/tree/v1.9/examples/nas/enas) ```bash # In case NNI code is not cloned. If the code is cloned already, ignore this line and enter code folder. diff --git a/docs/en_US/NAS/Overview.md b/docs/en_US/NAS/Overview.md index deffe22ed7..2a1e650f73 100644 --- a/docs/en_US/NAS/Overview.md +++ b/docs/en_US/NAS/Overview.md @@ -22,7 +22,7 @@ The procedure of classic NAS algorithms is similar to hyper-parameter tuning, us |Name|Brief Introduction of Algorithm| |---|---| -| [Random Search](https://github.com/microsoft/nni/tree/master/examples/tuners/random_nas_tuner) | Randomly pick a model from search space | +| [Random Search](https://github.com/microsoft/nni/tree/v1.9/examples/tuners/random_nas_tuner) | Randomly pick a model from search space | | [PPO Tuner](https://nni.readthedocs.io/en/latest/Tuner/BuiltinTuner.html#PPOTuner) | PPO Tuner is a Reinforcement Learning tuner based on PPO algorithm. [Reference Paper](https://arxiv.org/abs/1707.06347) | Please refer to [here](ClassicNas.md) for the usage of classic NAS algorithms. diff --git a/docs/en_US/NAS/PDARTS.md b/docs/en_US/NAS/PDARTS.md index e7ebede1c8..c6b27f0015 100644 --- a/docs/en_US/NAS/PDARTS.md +++ b/docs/en_US/NAS/PDARTS.md @@ -2,7 +2,7 @@ ## Examples -[Example code](https://github.com/microsoft/nni/tree/master/examples/nas/pdarts) +[Example code](https://github.com/microsoft/nni/tree/v1.9/examples/nas/pdarts) ```bash # In case NNI code is not cloned. If the code is cloned already, ignore this line and enter code folder. diff --git a/docs/en_US/NAS/Proxylessnas.md b/docs/en_US/NAS/Proxylessnas.md index 982557cfa2..8615297a52 100644 --- a/docs/en_US/NAS/Proxylessnas.md +++ b/docs/en_US/NAS/Proxylessnas.md @@ -19,7 +19,7 @@ trainer = ProxylessNasTrainer(model, trainer.train() trainer.export(args.arch_path) ``` -The complete example code can be found [here](https://github.com/microsoft/nni/tree/master/examples/nas/proxylessnas). +The complete example code can be found [here](https://github.com/microsoft/nni/tree/v1.9/examples/nas/proxylessnas). **Input arguments of ProxylessNasTrainer** @@ -50,7 +50,7 @@ The complete example code can be found [here](https://github.com/microsoft/nni/t The implementation on NNI is based on the [offical implementation](https://github.com/mit-han-lab/ProxylessNAS). The official implementation supports two training approaches: gradient descent and RL based, and support different targeted hardware, including 'mobile', 'cpu', 'gpu8', 'flops'. In our current implementation on NNI, gradient descent training approach is supported, but has not supported different hardwares. The complete support is ongoing. -Below we will describe implementation details. Like other one-shot NAS algorithms on NNI, ProxylessNAS is composed of two parts: *search space* and *training approach*. For users to flexibly define their own search space and use built-in ProxylessNAS training approach, we put the specified search space in [example code](https://github.com/microsoft/nni/tree/master/examples/nas/proxylessnas) using [NNI NAS interface](NasGuide.md), and put the training approach in [SDK](https://github.com/microsoft/nni/tree/master/src/sdk/pynni/nni/nas/pytorch/proxylessnas). +Below we will describe implementation details. Like other one-shot NAS algorithms on NNI, ProxylessNAS is composed of two parts: *search space* and *training approach*. For users to flexibly define their own search space and use built-in ProxylessNAS training approach, we put the specified search space in [example code](https://github.com/microsoft/nni/tree/v1.9/examples/nas/proxylessnas) using [NNI NAS interface](NasGuide.md), and put the training approach in [SDK](https://github.com/microsoft/nni/tree/v1.9/src/sdk/pynni/nni/nas/pytorch/proxylessnas). ![](../../img/proxylessnas.png) diff --git a/docs/en_US/NAS/SPOS.md b/docs/en_US/NAS/SPOS.md index 12e2b83be6..500a386b91 100644 --- a/docs/en_US/NAS/SPOS.md +++ b/docs/en_US/NAS/SPOS.md @@ -10,7 +10,7 @@ Implementation on NNI is based on [official repo](https://github.com/megvii-mode Here is a use case, which is the search space in paper, and the way to use flops limit to perform uniform sampling. -[Example code](https://github.com/microsoft/nni/tree/master/examples/nas/spos) +[Example code](https://github.com/microsoft/nni/tree/v1.9/examples/nas/spos) ### Requirements diff --git a/docs/en_US/NAS/SearchSpaceZoo.md b/docs/en_US/NAS/SearchSpaceZoo.md index 36a519d0a1..4fc3bde23f 100644 --- a/docs/en_US/NAS/SearchSpaceZoo.md +++ b/docs/en_US/NAS/SearchSpaceZoo.md @@ -2,7 +2,7 @@ ## DartsCell -DartsCell is extracted from [CNN model](./DARTS.md) designed [here](https://github.com/microsoft/nni/tree/master/examples/nas/darts). A DartsCell is a directed acyclic graph containing an ordered sequence of N nodes and each node stands for a latent representation (e.g. feature map in a convolutional network). Directed edges from Node 1 to Node 2 are associated with some operations that transform Node 1 and the result is stored on Node 2. The [Candidate operators](#predefined-operations-darts) between nodes is predefined and unchangeable. One edge represents an operation that chosen from the predefined ones to be applied to the starting node of the edge. One cell contains two input nodes, a single output node, and other `n_node` nodes. The input nodes are defined as the cell outputs in the previous two layers. The output of the cell is obtained by applying a reduction operation (e.g. concatenation) to all the intermediate nodes. To make the search space continuous, the categorical choice of a particular operation is relaxed to a softmax over all possible operations. By adjusting the weight of softmax on every node, the operation with the highest probability is chosen to be part of the final structure. A CNN model can be formed by stacking several cells together, which builds a search space. Note that, in DARTS paper all cells in the model share the same structure. +DartsCell is extracted from [CNN model](./DARTS.md) designed [here](https://github.com/microsoft/nni/tree/v1.9/examples/nas/darts). A DartsCell is a directed acyclic graph containing an ordered sequence of N nodes and each node stands for a latent representation (e.g. feature map in a convolutional network). Directed edges from Node 1 to Node 2 are associated with some operations that transform Node 1 and the result is stored on Node 2. The [Candidate operators](#predefined-operations-darts) between nodes is predefined and unchangeable. One edge represents an operation that chosen from the predefined ones to be applied to the starting node of the edge. One cell contains two input nodes, a single output node, and other `n_node` nodes. The input nodes are defined as the cell outputs in the previous two layers. The output of the cell is obtained by applying a reduction operation (e.g. concatenation) to all the intermediate nodes. To make the search space continuous, the categorical choice of a particular operation is relaxed to a softmax over all possible operations. By adjusting the weight of softmax on every node, the operation with the highest probability is chosen to be part of the final structure. A CNN model can be formed by stacking several cells together, which builds a search space. Note that, in DARTS paper all cells in the model share the same structure. One structure in the Darts search space is shown below. Note that, NNI merges the last one of the four intermediate nodes and the output node. @@ -17,7 +17,7 @@ The predefined operators are shown [here](#predefined-operations-darts). ### Example code -[example code](https://github.com/microsoft/nni/tree/master/examples/nas/search_space_zoo/darts_example.py) +[example code](https://github.com/microsoft/nni/tree/v1.9/examples/nas/search_space_zoo/darts_example.py) ```bash git clone https://github.com/Microsoft/nni.git @@ -61,7 +61,7 @@ All supported operators for Darts are listed below. ## ENASMicroLayer -This layer is extracted from the model designed [here](https://github.com/microsoft/nni/tree/master/examples/nas/enas). A model contains several blocks that share the same architecture. A block is made up of some normal layers and reduction layers, `ENASMicroLayer` is a unified implementation of the two types of layers. The only difference between the two layers is that reduction layers apply all operations with `stride=2`. +This layer is extracted from the model designed [here](https://github.com/microsoft/nni/tree/v1.9/examples/nas/enas). A model contains several blocks that share the same architecture. A block is made up of some normal layers and reduction layers, `ENASMicroLayer` is a unified implementation of the two types of layers. The only difference between the two layers is that reduction layers apply all operations with `stride=2`. ENAS Micro employs a DAG with N nodes in one cell, where the nodes represent local computations, and the edges represent the flow of information between the N nodes. One cell contains two input nodes and a single output node. The following nodes choose two previous nodes as input and apply two operations from [predefined ones](#predefined-operations-enas) then add them as the output of this node. For example, Node 4 chooses Node 1 and Node 3 as inputs then applies `MaxPool` and `AvgPool` on the inputs respectively, then adds and sums them as the output of Node 4. Nodes that are not served as input for any other node are viewed as the output of the layer. If there are multiple output nodes, the model will calculate the average of these nodes as the layer output. @@ -80,7 +80,7 @@ The Reduction Layer is made up of two Conv operations followed by BatchNorm, eac ### Example code -[example code](https://github.com/microsoft/nni/tree/master/examples/nas/search_space_zoo/enas_micro_example.py) +[example code](https://github.com/microsoft/nni/tree/v1.9/examples/nas/search_space_zoo/enas_micro_example.py) ```bash git clone https://github.com/Microsoft/nni.git @@ -136,7 +136,7 @@ To describe the whole search space, NNI provides a model, which is built by stac ### Example code -[example code](https://github.com/microsoft/nni/tree/master/examples/nas/search_space_zoo/enas_macro_example.py) +[example code](https://github.com/microsoft/nni/tree/v1.9/examples/nas/search_space_zoo/enas_macro_example.py) ```bash git clone https://github.com/Microsoft/nni.git @@ -187,7 +187,7 @@ The search space of NAS Bench 201 is shown below. ### Example code -[example code](https://github.com/microsoft/nni/tree/master/examples/nas/search_space_zoo/nas_bench_201.py) +[example code](https://github.com/microsoft/nni/tree/v1.9/examples/nas/search_space_zoo/nas_bench_201.py) ```bash # for structure searching diff --git a/docs/en_US/NAS/TextNAS.md b/docs/en_US/NAS/TextNAS.md index 7c455534ec..83969b50b7 100644 --- a/docs/en_US/NAS/TextNAS.md +++ b/docs/en_US/NAS/TextNAS.md @@ -45,7 +45,7 @@ The following link might be helpful for finding and downloading the correspondin ### Search Space -[Example code](https://github.com/microsoft/nni/tree/master/examples/nas/textnas) +[Example code](https://github.com/microsoft/nni/tree/v1.9/examples/nas/textnas) ```bash # In case NNI code is not cloned. If the code is cloned already, ignore this line and enter code folder. diff --git a/docs/en_US/Release.md b/docs/en_US/Release.md index 158e14d693..bfbf766dce 100644 --- a/docs/en_US/Release.md +++ b/docs/en_US/Release.md @@ -1,5 +1,50 @@ # ChangeLog +# Release 1.9 - 10/22/2020 + +## Major updates + +### Neural architecture search +* Support regularized evolution algorithm for NAS scenario (#2802) +* Add NASBench201 in search space zoo (#2766) + +### Model compression +* AMC pruner improvement: support resnet, support reproduction of the experiments (default parameters in our example code) in AMC paper (#2876 #2906) +* Support constraint-aware on some of our pruners to improve model compression efficiency (#2657) +* Support "tf.keras.Sequential" in model compression for TensorFlow (#2887) +* Support customized op in the model flops counter (#2795) +* Support quantizing bias in QAT quantizer (#2914) + +### Training service + +* Support configuring python environment using "preCommand" in remote mode (#2875) +* Support AML training service in Windows (#2882) +* Support reuse mode for remote training service (#2923) + +### WebUI & nnictl + +* The "Overview" page on WebUI is redesigned with new layout (#2914) +* Upgraded node, yarn and FabricUI, and enabled Eslint (#2894 #2873 #2744) +* Add/Remove columns in hyper-parameter chart and trials table in "Trials detail" page (#2900) +* JSON format utility beautify on WebUI (#2863) +* Support nnictl command auto-completion (#2857) + +## UT & IT +* Add integration test for experiment import and export (#2878) +* Add integration test for user installed builtin tuner (#2859) +* Add unit test for nnictl (#2912) + +## Documentation +* Refactor of the document for model compression (#2919) + +## Bug fixes +* Bug fix of naïve evolution tuner, correctly deal with trial fails (#2695) +* Resolve the warning "WARNING (nni.protocol) IPC pipeline not exists, maybe you are importing tuner/assessor from trial code?" (#2864) +* Fix search space issue in experiment save/load (#2886) +* Fix bug in experiment import data (#2878) +* Fix annotation in remote mode (python 3.8 ast update issue) (#2881) +* Support boolean type for "choice" hyper-parameter when customizing trial configuration on WebUI (#3003) + # Release 1.8 - 8/27/2020 ## Major updates @@ -158,7 +203,7 @@ and [examples](https://github.com/microsoft/nni/blob/v1.7/docs/en_US/NAS/Benchma * Improve PBT on failure handling and support experiment resume for PBT #### NAS Updates -* NAS support for TensorFlow 2.0 (preview) [TF2.0 NAS examples](https://github.com/microsoft/nni/tree/master/examples/nas/naive-tf) +* NAS support for TensorFlow 2.0 (preview) [TF2.0 NAS examples](https://github.com/microsoft/nni/tree/v1.9/examples/nas/naive-tf) * Use OrderedDict for LayerChoice * Prettify the format of export * Replace layer choice with selected module after applied fixed architecture @@ -168,7 +213,7 @@ and [examples](https://github.com/microsoft/nni/blob/v1.7/docs/en_US/NAS/Benchma #### Training Service Updates * update pai yaml merge logic -* support windows as remote machine in remote mode [Remote Mode](https://github.com/microsoft/nni/blob/master/docs/en_US/TrainingService/RemoteMachineMode.md#windows) +* support windows as remote machine in remote mode [Remote Mode](https://github.com/microsoft/nni/blob/v1.9/docs/en_US/TrainingService/RemoteMachineMode.md#windows) ### Bug Fix * fix dev install @@ -183,26 +228,26 @@ and [examples](https://github.com/microsoft/nni/blob/v1.7/docs/en_US/NAS/Benchma #### Hyper-Parameter Optimizing -* New tuner: [Population Based Training (PBT)](https://github.com/microsoft/nni/blob/master/docs/en_US/Tuner/PBTTuner.md) +* New tuner: [Population Based Training (PBT)](https://github.com/microsoft/nni/blob/v1.9/docs/en_US/Tuner/PBTTuner.md) * Trials can now report infinity and NaN as result #### Neural Architecture Search -* New NAS algorithm: [TextNAS](https://github.com/microsoft/nni/blob/master/docs/en_US/NAS/TextNAS.md) -* ENAS and DARTS now support [visualization](https://github.com/microsoft/nni/blob/master/docs/en_US/NAS/Visualization.md) through web UI. +* New NAS algorithm: [TextNAS](https://github.com/microsoft/nni/blob/v1.9/docs/en_US/NAS/TextNAS.md) +* ENAS and DARTS now support [visualization](https://github.com/microsoft/nni/blob/v1.9/docs/en_US/NAS/Visualization.md) through web UI. #### Model Compression -* New Pruner: [GradientRankFilterPruner](https://github.com/microsoft/nni/blob/master/docs/en_US/Compressor/Pruner.md#gradientrankfilterpruner) +* New Pruner: [GradientRankFilterPruner](https://github.com/microsoft/nni/blob/v1.9/docs/en_US/Compressor/Pruner.md#gradientrankfilterpruner) * Compressors will validate configuration by default * Refactor: Adding optimizer as an input argument of pruner, for easy support of DataParallel and more efficient iterative pruning. This is a broken change for the usage of iterative pruning algorithms. * Model compression examples are refactored and improved -* Added documentation for [implementing compressing algorithm](https://github.com/microsoft/nni/blob/master/docs/en_US/Compressor/Framework.md) +* Added documentation for [implementing compressing algorithm](https://github.com/microsoft/nni/blob/v1.9/docs/en_US/Compressor/Framework.md) #### Training Service * Kubeflow now supports pytorchjob crd v1 (thanks external contributor @jiapinai) -* Experimental [DLTS](https://github.com/microsoft/nni/blob/master/docs/en_US/TrainingService/DLTSMode.md) support +* Experimental [DLTS](https://github.com/microsoft/nni/blob/v1.9/docs/en_US/TrainingService/DLTSMode.md) support #### Overall Documentation Improvement @@ -355,7 +400,7 @@ and [examples](https://github.com/microsoft/nni/blob/v1.7/docs/en_US/NAS/Benchma - Support Auto-Feature generator & selection -Issue#877 -PR #1387 + Provide auto feature interface + Tuner based on beam search - + [Add Pakdd example](https://github.com/microsoft/nni/tree/master/examples/trials/auto-feature-engineering) + + [Add Pakdd example](https://github.com/microsoft/nni/tree/v1.9/examples/trials/auto-feature-engineering) - Add a parallel algorithm to improve the performance of TPE with large concurrency. -PR #1052 - Support multiphase for hyperband -PR #1257 @@ -602,8 +647,8 @@ and [examples](https://github.com/microsoft/nni/blob/v1.7/docs/en_US/NAS/Benchma ### New examples -* [FashionMnist](https://github.com/microsoft/nni/tree/master/examples/trials/network_morphism), work together with network morphism tuner -* [Distributed MNIST example](https://github.com/microsoft/nni/tree/master/examples/trials/mnist-distributed-pytorch) written in PyTorch +* [FashionMnist](https://github.com/microsoft/nni/tree/v1.9/examples/trials/network_morphism), work together with network morphism tuner +* [Distributed MNIST example](https://github.com/microsoft/nni/tree/v1.9/examples/trials/mnist-distributed-pytorch) written in PyTorch ## Release 0.4 - 12/6/2018 @@ -611,7 +656,7 @@ and [examples](https://github.com/microsoft/nni/blob/v1.7/docs/en_US/NAS/Benchma * [Kubeflow Training service](TrainingService/KubeflowMode.md) * Support tf-operator - * [Distributed trial example](https://github.com/microsoft/nni/tree/master/examples/trials/mnist-distributed/dist_mnist.py) on Kubeflow + * [Distributed trial example](https://github.com/microsoft/nni/tree/v1.9/examples/trials/mnist-distributed/dist_mnist.py) on Kubeflow * [Grid search tuner](Tuner/GridsearchTuner.md) * [Hyperband tuner](Tuner/HyperbandAdvisor.md) * Support launch NNI experiment on MAC @@ -680,8 +725,8 @@ and [examples](https://github.com/microsoft/nni/blob/v1.7/docs/en_US/NAS/Benchma docker pull msranni/nni:latest ``` -* New trial example: [NNI Sklearn Example](https://github.com/microsoft/nni/tree/master/examples/trials/sklearn) -* New competition example: [Kaggle Competition TGS Salt Example](https://github.com/microsoft/nni/tree/master/examples/trials/kaggle-tgs-salt) +* New trial example: [NNI Sklearn Example](https://github.com/microsoft/nni/tree/v1.9/examples/trials/sklearn) +* New competition example: [Kaggle Competition TGS Salt Example](https://github.com/microsoft/nni/tree/v1.9/examples/trials/kaggle-tgs-salt) ### Others diff --git a/docs/en_US/TrainingService/AMLMode.md b/docs/en_US/TrainingService/AMLMode.md index b912efc2b0..ff6f05a836 100644 --- a/docs/en_US/TrainingService/AMLMode.md +++ b/docs/en_US/TrainingService/AMLMode.md @@ -62,7 +62,11 @@ Note: You should set `trainingServicePlatform: aml` in NNI config YAML file if y Compared with [LocalMode](LocalMode.md) trial configuration in aml mode have these additional keys: * image - * required key. The docker image name used in job. The image `msranni/nni` of this example only support GPU computeTargets. + * required key. The docker image name used in job. NNI support image `msranni/nni` for running aml jobs. + ``` + Note: This image is build based on cuda environment, may not be suitable for CPU clusters in AML. + ``` + amlConfig: * subscriptionId @@ -72,11 +76,11 @@ amlConfig: * workspaceName * required key, the workspaceName of your account * computeTarget - * required key, the compute cluster name you want to use in your AML workspace. See Step 6. + * required key, the compute cluster name you want to use in your AML workspace. [refer](https://docs.microsoft.com/en-us/azure/machine-learning/concept-compute-target) See Step 6. * maxTrialNumPerGpu - * optional key, used to specify the max concurrency trial number on a GPU device. + * optional key, default 1. Used to specify the max concurrency trial number on a GPU device. * useActiveGpu - * optional key, used to specify whether to use a GPU if there is another process. By default, NNI will use the GPU only if there is no other active process in the GPU. + * optional key, default false. Used to specify whether to use a GPU if there is another process. By default, NNI will use the GPU only if there is no other active process in the GPU. The required information of amlConfig could be found in the downloaded `config.json` in Step 5. @@ -89,4 +93,4 @@ cd nni/examples/trials/mnist-tfv1 nnictl create --config config_aml.yml ``` -Replace `${NNI_VERSION}` with a released version name or branch name, e.g., `v1.8`. +Replace `${NNI_VERSION}` with a released version name or branch name, e.g., `v1.9`. diff --git a/docs/en_US/TrainingService/KubeflowMode.md b/docs/en_US/TrainingService/KubeflowMode.md index 9097ab7a1f..d84812259b 100644 --- a/docs/en_US/TrainingService/KubeflowMode.md +++ b/docs/en_US/TrainingService/KubeflowMode.md @@ -196,7 +196,7 @@ Trial configuration in kubeflow mode have the following configuration keys: * gpuNum * image * Required key. In kubeflow mode, your trial program will be scheduled by Kubernetes to run in [Pod](https://kubernetes.io/docs/concepts/workloads/pods/pod/). This key is used to specify the Docker image used to create the pod where your trail program will run. - * We already build a docker image [msranni/nni](https://hub.docker.com/r/msranni/nni/) on [Docker Hub](https://hub.docker.com/). It contains NNI python packages, Node modules and javascript artifact files required to start experiment, and all of NNI dependencies. The docker file used to build this image can be found at [here](https://github.com/Microsoft/nni/tree/master/deployment/docker/Dockerfile). You can either use this image directly in your config file, or build your own image based on it. + * We already build a docker image [msranni/nni](https://hub.docker.com/r/msranni/nni/) on [Docker Hub](https://hub.docker.com/). It contains NNI python packages, Node modules and javascript artifact files required to start experiment, and all of NNI dependencies. The docker file used to build this image can be found at [here](https://github.com/Microsoft/nni/tree/v1.9/deployment/docker/Dockerfile). You can either use this image directly in your config file, or build your own image based on it. * privateRegistryAuthPath * Optional field, specify `config.json` file path that holds an authorization token of docker registry, used to pull image from private registry. [Refer](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/). * apiVersion diff --git a/docs/en_US/TrainingService/Overview.md b/docs/en_US/TrainingService/Overview.md index 563cb36405..bd87037f0f 100644 --- a/docs/en_US/TrainingService/Overview.md +++ b/docs/en_US/TrainingService/Overview.md @@ -12,7 +12,7 @@ If the computing resource customers try to use is not listed above, NNI provides Training service needs to be chosen and configured properly in experiment configuration YAML file. Users could refer to the document of each training service for how to write the configuration. Also, [reference](../Tutorial/ExperimentConfig) provides more details on the specification of the experiment configuration file. -Next, users should prepare code directory, which is specified as `codeDir` in config file. Please note that in non-local mode, the code directory will be uploaded to remote or cluster before the experiment. Therefore, we limit the number of files to 2000 and total size to 300MB. If the code directory contains too many files, users can choose which files and subfolders should be excluded by adding a `.nniignore` file that works like a `.gitignore` file. For more details on how to write this file, see [this example](https://github.com/Microsoft/nni/tree/master/examples/trials/mnist-tfv1/.nniignore) and the [git documentation](https://git-scm.com/docs/gitignore#_pattern_format). +Next, users should prepare code directory, which is specified as `codeDir` in config file. Please note that in non-local mode, the code directory will be uploaded to remote or cluster before the experiment. Therefore, we limit the number of files to 2000 and total size to 300MB. If the code directory contains too many files, users can choose which files and subfolders should be excluded by adding a `.nniignore` file that works like a `.gitignore` file. For more details on how to write this file, see [this example](https://github.com/Microsoft/nni/tree/v1.9/examples/trials/mnist-tfv1/.nniignore) and the [git documentation](https://git-scm.com/docs/gitignore#_pattern_format). In case users intend to use large files in their experiment (like large-scaled datasets) and they are not using local mode, they can either: 1) download the data before each trial launches by putting it into trial command; or 2) use a shared storage that is accessible to worker nodes. Usually, training platforms are equipped with shared storage, and NNI allows users to easily use them. Refer to docs of each built-in training service for details. diff --git a/docs/en_US/TrainingService/PaiMode.md b/docs/en_US/TrainingService/PaiMode.md index 585ec37fc9..4e9acd234c 100644 --- a/docs/en_US/TrainingService/PaiMode.md +++ b/docs/en_US/TrainingService/PaiMode.md @@ -104,7 +104,7 @@ Compared with [LocalMode](LocalMode.md) and [RemoteMachineMode](RemoteMachineMod Optional key. In pai mode, your trial program will be scheduled by OpenPAI to run in [Docker container](https://www.docker.com/). This key is used to specify the Docker image used to create the container in which your trial will run. - We already build a docker image [nnimsra/nni](https://hub.docker.com/r/msranni/nni/) on [Docker Hub](https://hub.docker.com/). It contains NNI python packages, Node modules and javascript artifact files required to start experiment, and all of NNI dependencies. The docker file used to build this image can be found at [here](https://github.com/Microsoft/nni/tree/master/deployment/docker/Dockerfile). You can either use this image directly in your config file, or build your own image based on it. If it is not set in trial configuration, it should be set in the config file specified in `paiConfigPath` field. + We already build a docker image [nnimsra/nni](https://hub.docker.com/r/msranni/nni/) on [Docker Hub](https://hub.docker.com/). It contains NNI python packages, Node modules and javascript artifact files required to start experiment, and all of NNI dependencies. The docker file used to build this image can be found at [here](https://github.com/Microsoft/nni/tree/v1.9/deployment/docker/Dockerfile). You can either use this image directly in your config file, or build your own image based on it. If it is not set in trial configuration, it should be set in the config file specified in `paiConfigPath` field. * virtualCluster diff --git a/docs/en_US/TrainingService/PaiYarnMode.md b/docs/en_US/TrainingService/PaiYarnMode.md index 64dda5e465..dcb0a79c3f 100644 --- a/docs/en_US/TrainingService/PaiYarnMode.md +++ b/docs/en_US/TrainingService/PaiYarnMode.md @@ -50,7 +50,7 @@ Compared with [LocalMode](LocalMode.md) and [RemoteMachineMode](RemoteMachineMod * Required key. Should be positive number based on your trial program's memory requirement * image * Required key. In paiYarn mode, your trial program will be scheduled by OpenpaiYarn to run in [Docker container](https://www.docker.com/). This key is used to specify the Docker image used to create the container in which your trial will run. - * We already build a docker image [nnimsra/nni](https://hub.docker.com/r/msranni/nni/) on [Docker Hub](https://hub.docker.com/). It contains NNI python packages, Node modules and javascript artifact files required to start experiment, and all of NNI dependencies. The docker file used to build this image can be found at [here](https://github.com/Microsoft/nni/tree/master/deployment/docker/Dockerfile). You can either use this image directly in your config file, or build your own image based on it. + * We already build a docker image [nnimsra/nni](https://hub.docker.com/r/msranni/nni/) on [Docker Hub](https://hub.docker.com/). It contains NNI python packages, Node modules and javascript artifact files required to start experiment, and all of NNI dependencies. The docker file used to build this image can be found at [here](https://github.com/Microsoft/nni/tree/v1.9/deployment/docker/Dockerfile). You can either use this image directly in your config file, or build your own image based on it. * virtualCluster * Optional key. Set the virtualCluster of OpenpaiYarn. If omitted, the job will run on default virtual cluster. * shmMB diff --git a/docs/en_US/TrialExample/Cifar10Examples.md b/docs/en_US/TrialExample/Cifar10Examples.md index d148a394e1..86a1584f21 100644 --- a/docs/en_US/TrialExample/Cifar10Examples.md +++ b/docs/en_US/TrialExample/Cifar10Examples.md @@ -73,12 +73,12 @@ We are ready for the experiment, let's now **run the config.yml file from your c nnictl create --config nni/examples/trials/cifar10_pytorch/config.yml ``` -[1]: https://github.com/Microsoft/nni/tree/master/examples/trials/cifar10_pytorch +[1]: https://github.com/Microsoft/nni/tree/v1.9/examples/trials/cifar10_pytorch [2]: https://pytorch.org/ [3]: https://www.cs.toronto.edu/~kriz/cifar.html -[4]: https://github.com/Microsoft/nni/tree/master/examples/trials/cifar10_pytorch +[4]: https://github.com/Microsoft/nni/tree/v1.9/examples/trials/cifar10_pytorch [5]: Trials.md -[6]: https://github.com/Microsoft/nni/blob/master/examples/trials/cifar10_pytorch/config.yml -[7]: https://github.com/Microsoft/nni/blob/master/examples/trials/cifar10_pytorch/config_pai.yml -[8]: https://github.com/Microsoft/nni/blob/master/examples/trials/cifar10_pytorch/search_space.json -[9]: https://github.com/Microsoft/nni/blob/master/examples/trials/cifar10_pytorch/main.py +[6]: https://github.com/Microsoft/nni/blob/v1.9/examples/trials/cifar10_pytorch/config.yml +[7]: https://github.com/Microsoft/nni/blob/v1.9/examples/trials/cifar10_pytorch/config_pai.yml +[8]: https://github.com/Microsoft/nni/blob/v1.9/examples/trials/cifar10_pytorch/search_space.json +[9]: https://github.com/Microsoft/nni/blob/v1.9/examples/trials/cifar10_pytorch/main.py diff --git a/docs/en_US/TrialExample/EfficientNet.md b/docs/en_US/TrialExample/EfficientNet.md index 634a1a9593..e22da7e42e 100644 --- a/docs/en_US/TrialExample/EfficientNet.md +++ b/docs/en_US/TrialExample/EfficientNet.md @@ -6,7 +6,7 @@ Use Grid search to find the best combination of alpha, beta and gamma for Effici ## Instructions -[Example code](https://github.com/microsoft/nni/tree/master/examples/trials/efficientnet) +[Example code](https://github.com/microsoft/nni/tree/v1.9/examples/trials/efficientnet) 1. Set your working directory here in the example code directory. 2. Run `git clone https://github.com/ultmaster/EfficientNet-PyTorch` to clone this modified version of [EfficientNet-PyTorch](https://github.com/lukemelas/EfficientNet-PyTorch). The modifications were done to adhere to the original [Tensorflow version](https://github.com/tensorflow/tpu/tree/master/models/official/efficientnet) as close as possible (including EMA, label smoothing and etc.); also added are the part which gets parameters from tuner and reports intermediate/final results. Clone it into `EfficientNet-PyTorch`; the files like `main.py`, `train_imagenet.sh` will appear inside, as specified in the configuration files. diff --git a/docs/en_US/TrialExample/GbdtExample.md b/docs/en_US/TrialExample/GbdtExample.md index 8a154e097c..b3d4279a5c 100644 --- a/docs/en_US/TrialExample/GbdtExample.md +++ b/docs/en_US/TrialExample/GbdtExample.md @@ -39,7 +39,7 @@ Reference link: [lightgbm](https://lightgbm.readthedocs.io/en/latest/Parameters-Tuning.html) and [autoxgoboost](https://github.com/ja-thomas/autoxgboost/blob/master/poster_2018.pdf) ## 2. Task description -Now we come back to our example "auto-gbdt" which run in lightgbm and nni. The data including [train data](https://github.com/Microsoft/nni/blob/master/examples/trials/auto-gbdt/data/regression.train) and [test data](https://github.com/Microsoft/nni/blob/master/examples/trials/auto-gbdt/data/regression.train). +Now we come back to our example "auto-gbdt" which run in lightgbm and nni. The data including [train data](https://github.com/Microsoft/nni/blob/v1.9/examples/trials/auto-gbdt/data/regression.train) and [test data](https://github.com/Microsoft/nni/blob/v1.9/examples/trials/auto-gbdt/data/regression.train). Given the features and label in train data, we train a GBDT regression model and use it to predict. ## 3. How to run in nni @@ -95,7 +95,7 @@ if __name__ == '__main__': ``` ### 3.3 Prepare your search space. -If you like to tune `num_leaves`, `learning_rate`, `bagging_fraction` and `bagging_freq`, you could write a [search_space.json](https://github.com/Microsoft/nni/blob/master/examples/trials/auto-gbdt/search_space.json) as follow: +If you like to tune `num_leaves`, `learning_rate`, `bagging_fraction` and `bagging_freq`, you could write a [search_space.json](https://github.com/Microsoft/nni/blob/v1.9/examples/trials/auto-gbdt/search_space.json) as follow: ```json { diff --git a/docs/en_US/TrialExample/RocksdbExamples.md b/docs/en_US/TrialExample/RocksdbExamples.md index 6e7d36cc32..9ccba176df 100644 --- a/docs/en_US/TrialExample/RocksdbExamples.md +++ b/docs/en_US/TrialExample/RocksdbExamples.md @@ -8,9 +8,9 @@ The performance of RocksDB is highly contingent on its tuning. However, because This example illustrates how to use NNI to search the best configuration of RocksDB for a `fillrandom` benchmark supported by a benchmark tool `db_bench`, which is an official benchmark tool provided by RocksDB itself. Therefore, before running this example, please make sure NNI is installed and [`db_bench`](https://github.com/facebook/rocksdb/wiki/Benchmarking-tools) is in your `PATH`. Please refer to [here](../Tutorial/QuickStart.md) for detailed information about installation and preparing of NNI environment, and [here](https://github.com/facebook/rocksdb/blob/master/INSTALL.md) for compiling RocksDB as well as `db_bench`. -We also provide a simple script [`db_bench_installation.sh`](https://github.com/microsoft/nni/tree/master/examples/trials/systems/rocksdb-fillrandom/db_bench_installation.sh) helping to compile and install `db_bench` as well as its dependencies on Ubuntu. Installing RocksDB on other systems can follow the same procedure. +We also provide a simple script [`db_bench_installation.sh`](https://github.com/microsoft/nni/tree/v1.9/examples/trials/systems/rocksdb-fillrandom/db_bench_installation.sh) helping to compile and install `db_bench` as well as its dependencies on Ubuntu. Installing RocksDB on other systems can follow the same procedure. -*code directory: [`example/trials/systems/rocksdb-fillrandom`](https://github.com/microsoft/nni/tree/master/examples/trials/systems/rocksdb-fillrandom)* +*code directory: [`example/trials/systems/rocksdb-fillrandom`](https://github.com/microsoft/nni/tree/v1.9/examples/trials/systems/rocksdb-fillrandom)* ## Experiment setup @@ -39,7 +39,7 @@ In this example, the search space is specified by a `search_space.json` file as } ``` -*code directory: [`example/trials/systems/rocksdb-fillrandom/search_space.json`](https://github.com/microsoft/nni/tree/master/examples/trials/systems/rocksdb-fillrandom/search_space.json)* +*code directory: [`example/trials/systems/rocksdb-fillrandom/search_space.json`](https://github.com/microsoft/nni/tree/v1.9/examples/trials/systems/rocksdb-fillrandom/search_space.json)* ### Benchmark code @@ -48,7 +48,7 @@ Benchmark code should receive a configuration from NNI manager, and report the c * Use `nni.get_next_parameter()` to get next system configuration. * Use `nni.report_final_result(metric)` to report the benchmark result. -*code directory: [`example/trials/systems/rocksdb-fillrandom/main.py`](https://github.com/microsoft/nni/tree/master/examples/trials/systems/rocksdb-fillrandom/main.py)* +*code directory: [`example/trials/systems/rocksdb-fillrandom/main.py`](https://github.com/microsoft/nni/tree/v1.9/examples/trials/systems/rocksdb-fillrandom/main.py)* ### Config file @@ -56,11 +56,11 @@ One could start a NNI experiment with a config file. A config file for NNI is a Here is an example of tuning RocksDB with SMAC algorithm: -*code directory: [`example/trials/systems/rocksdb-fillrandom/config_smac.yml`](https://github.com/microsoft/nni/tree/master/examples/trials/systems/rocksdb-fillrandom/config_smac.yml)* +*code directory: [`example/trials/systems/rocksdb-fillrandom/config_smac.yml`](https://github.com/microsoft/nni/tree/v1.9/examples/trials/systems/rocksdb-fillrandom/config_smac.yml)* Here is an example of tuning RocksDB with TPE algorithm: -*code directory: [`example/trials/systems/rocksdb-fillrandom/config_tpe.yml`](https://github.com/microsoft/nni/tree/master/examples/trials/systems/rocksdb-fillrandom/config_tpe.yml)* +*code directory: [`example/trials/systems/rocksdb-fillrandom/config_tpe.yml`](https://github.com/microsoft/nni/tree/v1.9/examples/trials/systems/rocksdb-fillrandom/config_tpe.yml)* Other tuners can be easily adopted in the same way. Please refer to [here](../Tuner/BuiltinTuner.md) for more information. @@ -87,7 +87,7 @@ We ran these two examples on the same machine with following details: The detailed experiment results are shown in the below figure. Horizontal axis is sequential order of trials. Vertical axis is the metric, write OPS in this example. Blue dots represent trials for tuning RocksDB with SMAC tuner, and orange dots stand for trials for tuning RocksDB with TPE tuner. -![image](https://github.com/microsoft/nni/tree/master/examples/trials/systems/rocksdb-fillrandom/plot.png) +![image](https://github.com/microsoft/nni/tree/v1.9/examples/trials/systems/rocksdb-fillrandom/plot.png) Following table lists the best trials and corresponding parameters and metric obtained by the two tuners. Unsurprisingly, both of them found the same optimal configuration for `fillrandom` benchmark. diff --git a/docs/en_US/TrialExample/SquadEvolutionExamples.md b/docs/en_US/TrialExample/SquadEvolutionExamples.md index 39d74a955a..f1b7eea6ca 100644 --- a/docs/en_US/TrialExample/SquadEvolutionExamples.md +++ b/docs/en_US/TrialExample/SquadEvolutionExamples.md @@ -99,7 +99,7 @@ useAnnotation: false #Your nni_manager ip nniManagerIp: 10.10.10.10 tuner: - codeDir: https://github.com/Microsoft/nni/tree/master/examples/tuners/ga_customer_tuner + codeDir: https://github.com/Microsoft/nni/tree/v1.9/examples/tuners/ga_customer_tuner classFileName: customer_tuner.py className: CustomerTuner classArgs: diff --git a/docs/en_US/TrialExample/Trials.md b/docs/en_US/TrialExample/Trials.md index 2edf5a3f40..75cc96f611 100644 --- a/docs/en_US/TrialExample/Trials.md +++ b/docs/en_US/TrialExample/Trials.md @@ -140,7 +140,7 @@ nni.get_trial_id # return "STANDALONE" nni.get_sequence_id # return 0 ``` -You can try standalone mode with the [mnist example](https://github.com/microsoft/nni/tree/master/examples/trials/mnist-tfv1). Simply run `python3 mnist.py` under the code directory. The trial code should successfully run with the default hyperparameter values. +You can try standalone mode with the [mnist example](https://github.com/microsoft/nni/tree/v1.9/examples/trials/mnist-tfv1). Simply run `python3 mnist.py` under the code directory. The trial code should successfully run with the default hyperparameter values. For more information on debugging, please refer to [How to Debug](../Tutorial/HowToDebug.md) diff --git a/docs/en_US/Tuner/BohbAdvisor.md b/docs/en_US/Tuner/BohbAdvisor.md index 0e431ee4bb..d27a9958f5 100644 --- a/docs/en_US/Tuner/BohbAdvisor.md +++ b/docs/en_US/Tuner/BohbAdvisor.md @@ -92,7 +92,7 @@ The advisor has a lot of different files, functions, and classes. Here, we will ### MNIST with BOHB -code implementation: [examples/trials/mnist-advisor](https://github.com/Microsoft/nni/tree/master/examples/trials/) +code implementation: [examples/trials/mnist-advisor](https://github.com/Microsoft/nni/tree/v1.9/examples/trials/) We chose BOHB to build a CNN on the MNIST dataset. The following is our experimental final results: diff --git a/docs/en_US/Tuner/BuiltinTuner.md b/docs/en_US/Tuner/BuiltinTuner.md index a1a7a2d44a..0fd30f1ca5 100644 --- a/docs/en_US/Tuner/BuiltinTuner.md +++ b/docs/en_US/Tuner/BuiltinTuner.md @@ -270,11 +270,11 @@ advisor: **Installation** -NetworkMorphism requires [PyTorch](https://pytorch.org/get-started/locally) and [Keras](https://keras.io/#installation), so users should install them first. The corresponding requirements file is [here](https://github.com/microsoft/nni/blob/master/examples/trials/network_morphism/requirements.txt). +NetworkMorphism requires [PyTorch](https://pytorch.org/get-started/locally) and [Keras](https://keras.io/#installation), so users should install them first. The corresponding requirements file is [here](https://github.com/microsoft/nni/blob/v1.9/examples/trials/network_morphism/requirements.txt). **Suggested scenario** -This is suggested when you want to apply deep learning methods to your task but you have no idea how to choose or design a network. You may modify this [example](https://github.com/Microsoft/nni/tree/master/examples/trials/network_morphism/cifar10/cifar10_keras.py) to fit your own dataset and your own data augmentation method. Also you can change the batch size, learning rate, or optimizer. Currently, this tuner only supports the computer vision domain. [Detailed Description](./NetworkmorphismTuner.md) +This is suggested when you want to apply deep learning methods to your task but you have no idea how to choose or design a network. You may modify this [example](https://github.com/Microsoft/nni/tree/v1.9/examples/trials/network_morphism/cifar10/cifar10_keras.py) to fit your own dataset and your own data augmentation method. Also you can change the batch size, learning rate, or optimizer. Currently, this tuner only supports the computer vision domain. [Detailed Description](./NetworkmorphismTuner.md) **classArgs Requirements:** @@ -310,7 +310,7 @@ Note that the only acceptable types of search space types are `quniform`, `unifo **Suggested scenario** -Similar to TPE and SMAC, Metis is a black-box tuner. If your system takes a long time to finish each trial, Metis is more favorable than other approaches such as random search. Furthermore, Metis provides guidance on subsequent trials. Here is an [example](https://github.com/Microsoft/nni/tree/master/examples/trials/auto-gbdt/search_space_metis.json) on the use of Metis. Users only need to send the final result, such as `accuracy`, to the tuner by calling the NNI SDK. [Detailed Description](./MetisTuner.md) +Similar to TPE and SMAC, Metis is a black-box tuner. If your system takes a long time to finish each trial, Metis is more favorable than other approaches such as random search. Furthermore, Metis provides guidance on subsequent trials. Here is an [example](https://github.com/Microsoft/nni/tree/v1.9/examples/trials/auto-gbdt/search_space_metis.json) on the use of Metis. Users only need to send the final result, such as `accuracy`, to the tuner by calling the NNI SDK. [Detailed Description](./MetisTuner.md) **classArgs Requirements:** @@ -425,7 +425,7 @@ Note that the only acceptable types within the search space are `layer_choice` a **Suggested scenario** -PPOTuner is a Reinforcement Learning tuner based on the PPO algorithm. PPOTuner can be used when using the NNI NAS interface to do neural architecture search. In general, the Reinforcement Learning algorithm needs more computing resources, though the PPO algorithm is relatively more efficient than others. It's recommended to use this tuner when you have a large amount of computional resources available. You could try it on a very simple task, such as the [mnist-nas](https://github.com/microsoft/nni/tree/master/examples/trials/mnist-nas) example. [See details](./PPOTuner.md) +PPOTuner is a Reinforcement Learning tuner based on the PPO algorithm. PPOTuner can be used when using the NNI NAS interface to do neural architecture search. In general, the Reinforcement Learning algorithm needs more computing resources, though the PPO algorithm is relatively more efficient than others. It's recommended to use this tuner when you have a large amount of computional resources available. You could try it on a very simple task, such as the [mnist-nas](https://github.com/microsoft/nni/tree/v1.9/examples/trials/mnist-nas) example. [See details](./PPOTuner.md) **classArgs Requirements:** @@ -485,6 +485,6 @@ Note that, to use this tuner, your trial code should be modified accordingly, pl ## **Reference and Feedback** * To [report a bug](https://github.com/microsoft/nni/issues/new?template=bug-report.md) for this feature in GitHub; * To [file a feature or improvement request](https://github.com/microsoft/nni/issues/new?template=enhancement.md) for this feature in GitHub; -* To know more about [Feature Engineering with NNI](https://github.com/microsoft/nni/blob/master/docs/en_US/FeatureEngineering/Overview.md); -* To know more about [NAS with NNI](https://github.com/microsoft/nni/blob/master/docs/en_US/NAS/Overview.md); -* To know more about [Model Compression with NNI](https://github.com/microsoft/nni/blob/master/docs/en_US/Compression/Overview.md); +* To know more about [Feature Engineering with NNI](https://github.com/microsoft/nni/blob/v1.9/docs/en_US/FeatureEngineering/Overview.md); +* To know more about [NAS with NNI](https://github.com/microsoft/nni/blob/v1.9/docs/en_US/NAS/Overview.md); +* To know more about [Model Compression with NNI](https://github.com/microsoft/nni/blob/v1.9/docs/en_US/Compression/Overview.md); diff --git a/docs/en_US/Tuner/CustomizeAdvisor.md b/docs/en_US/Tuner/CustomizeAdvisor.md index b368f281f5..0c8e1c32c9 100644 --- a/docs/en_US/Tuner/CustomizeAdvisor.md +++ b/docs/en_US/Tuner/CustomizeAdvisor.md @@ -37,4 +37,4 @@ advisor: ## Example -Here we provide an [example](https://github.com/microsoft/nni/tree/master/examples/tuners/mnist_keras_customized_advisor). +Here we provide an [example](https://github.com/microsoft/nni/tree/v1.9/examples/tuners/mnist_keras_customized_advisor). diff --git a/docs/en_US/Tuner/CustomizeTuner.md b/docs/en_US/Tuner/CustomizeTuner.md index be7e7c2d6d..aff8cfc5d6 100644 --- a/docs/en_US/Tuner/CustomizeTuner.md +++ b/docs/en_US/Tuner/CustomizeTuner.md @@ -113,10 +113,10 @@ tuner: ``` More detail example you could see: -> * [evolution-tuner](https://github.com/Microsoft/nni/tree/master/src/sdk/pynni/nni/evolution_tuner) -> * [hyperopt-tuner](https://github.com/Microsoft/nni/tree/master/src/sdk/pynni/nni/hyperopt_tuner) -> * [evolution-based-customized-tuner](https://github.com/Microsoft/nni/tree/master/examples/tuners/ga_customer_tuner) +> * [evolution-tuner](https://github.com/Microsoft/nni/tree/v1.9/src/sdk/pynni/nni/evolution_tuner) +> * [hyperopt-tuner](https://github.com/Microsoft/nni/tree/v1.9/src/sdk/pynni/nni/hyperopt_tuner) +> * [evolution-based-customized-tuner](https://github.com/Microsoft/nni/tree/v1.9/examples/tuners/ga_customer_tuner) ### Write a more advanced automl algorithm -The methods above are usually enough to write a general tuner. However, users may also want more methods, for example, intermediate results, trials' state (e.g., the methods in assessor), in order to have a more powerful automl algorithm. Therefore, we have another concept called `advisor` which directly inherits from `MsgDispatcherBase` in [`src/sdk/pynni/nni/msg_dispatcher_base.py`](https://github.com/Microsoft/nni/tree/master/src/sdk/pynni/nni/msg_dispatcher_base.py). Please refer to [here](CustomizeAdvisor.md) for how to write a customized advisor. +The methods above are usually enough to write a general tuner. However, users may also want more methods, for example, intermediate results, trials' state (e.g., the methods in assessor), in order to have a more powerful automl algorithm. Therefore, we have another concept called `advisor` which directly inherits from `MsgDispatcherBase` in [`src/sdk/pynni/nni/msg_dispatcher_base.py`](https://github.com/Microsoft/nni/tree/v1.9/src/sdk/pynni/nni/msg_dispatcher_base.py). Please refer to [here](CustomizeAdvisor.md) for how to write a customized advisor. diff --git a/docs/en_US/Tuner/NetworkmorphismTuner.md b/docs/en_US/Tuner/NetworkmorphismTuner.md index b300ffc37e..aa7a8e77a4 100644 --- a/docs/en_US/Tuner/NetworkmorphismTuner.md +++ b/docs/en_US/Tuner/NetworkmorphismTuner.md @@ -4,7 +4,7 @@ [Autokeras](https://arxiv.org/abs/1806.10282) is a popular autoML tool using Network Morphism. The basic idea of Autokeras is to use Bayesian Regression to estimate the metric of the Neural Network Architecture. Each time, it generates several child networks from father networks. Then it uses a naïve Bayesian regression to estimate its metric value from the history of trained results of network and metric value pairs. Next, it chooses the child which has the best, estimated performance and adds it to the training queue. Inspired by the work of Autokeras and referring to its [code](https://github.com/jhfjhfj1/autokeras), we implemented our Network Morphism method on the NNI platform. -If you want to know more about network morphism trial usage, please see the [Readme.md](https://github.com/Microsoft/nni/blob/master/examples/trials/network_morphism/README.md). +If you want to know more about network morphism trial usage, please see the [Readme.md](https://github.com/Microsoft/nni/blob/v1.9/examples/trials/network_morphism/README.md). ## 2. Usage diff --git a/docs/en_US/Tuner/PBTTuner.md b/docs/en_US/Tuner/PBTTuner.md index 1554039a33..be009b7b03 100644 --- a/docs/en_US/Tuner/PBTTuner.md +++ b/docs/en_US/Tuner/PBTTuner.md @@ -31,7 +31,7 @@ save_path = os.path.join(params['save_checkpoint_dir'], 'model.pth') ... ``` -The complete example code can be found [here](https://github.com/microsoft/nni/tree/master/examples/trials/mnist-pbt-tuner-pytorch). +The complete example code can be found [here](https://github.com/microsoft/nni/tree/v1.9/examples/trials/mnist-pbt-tuner-pytorch). ### Experiment config diff --git a/docs/en_US/Tuner/PPOTuner.md b/docs/en_US/Tuner/PPOTuner.md index 0654a17d83..6d93784436 100644 --- a/docs/en_US/Tuner/PPOTuner.md +++ b/docs/en_US/Tuner/PPOTuner.md @@ -10,12 +10,12 @@ We had successfully tuned the mnist-nas example and has the following result: ![](../../img/ppo_mnist.png) -We also tune [the macro search space for image classification in the enas paper](https://github.com/microsoft/nni/tree/master/examples/trials/nas_cifar10) (with a limited epoch number for each trial, i.e., 8 epochs), which is implemented using the NAS interface and tuned with PPOTuner. Here is Figure 7 from the [enas paper](https://arxiv.org/pdf/1802.03268.pdf) to show what the search space looks like +We also tune [the macro search space for image classification in the enas paper](https://github.com/microsoft/nni/tree/v1.9/examples/trials/nas_cifar10) (with a limited epoch number for each trial, i.e., 8 epochs), which is implemented using the NAS interface and tuned with PPOTuner. Here is Figure 7 from the [enas paper](https://arxiv.org/pdf/1802.03268.pdf) to show what the search space looks like ![](../../img/enas_search_space.png) The figure above was the chosen architecture. Each square is a layer whose operation was chosen from 6 options. Each dashed line is a skip connection, each square layer can choose 0 or 1 skip connections, getting the output from a previous layer. __Note that__, in original macro search space, each square layer could choose any number of skip connections, while in our implementation, it is only allowed to choose 0 or 1. -The results are shown in figure below (see the experimenal config [here](https://github.com/microsoft/nni/blob/master/examples/trials/nas_cifar10/config_ppo.yml): +The results are shown in figure below (see the experimenal config [here](https://github.com/microsoft/nni/blob/v1.9/examples/trials/nas_cifar10/config_ppo.yml): ![](../../img/ppo_cifar10.png) diff --git a/docs/en_US/Tutorial/Contributing.md b/docs/en_US/Tutorial/Contributing.md index ce91b12344..f765e9c07c 100644 --- a/docs/en_US/Tutorial/Contributing.md +++ b/docs/en_US/Tutorial/Contributing.md @@ -47,10 +47,10 @@ A person looking to contribute can take up an issue by claiming it as a comment/ - Internal Guideline on Writing Standards](https://ribokit.github.io/docs/text/) ## Documentation -Our documentation is built with [sphinx](http://sphinx-doc.org/), supporting [Markdown](https://guides.github.com/features/mastering-markdown/) and [reStructuredText](http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html) format. All our documentations are placed under [docs/en_US](https://github.com/Microsoft/nni/tree/master/docs). +Our documentation is built with [sphinx](http://sphinx-doc.org/), supporting [Markdown](https://guides.github.com/features/mastering-markdown/) and [reStructuredText](http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html) format. All our documentations are placed under [docs/en_US](https://github.com/Microsoft/nni/tree/v1.9/docs). * Before submitting the documentation change, please __build homepage locally__: `cd docs/en_US && make html`, then you can see all the built documentation webpage under the folder `docs/en_US/_build/html`. It's also highly recommended taking care of __every WARNING__ during the build, which is very likely the signal of a __deadlink__ and other annoying issues. * For links, please consider using __relative paths__ first. However, if the documentation is written in Markdown format, and: * It's an image link which needs to be formatted with embedded html grammar, please use global URL like `https://user-images.githubusercontent.com/44491713/51381727-e3d0f780-1b4f-11e9-96ab-d26b9198ba65.png`, which can be automatically generated by dragging picture onto [Github Issue](https://github.com/Microsoft/nni/issues/new) Box. - * It cannot be re-formatted by sphinx, such as source code, please use its global URL. For source code that links to our github repo, please use URLs rooted at `https://github.com/Microsoft/nni/tree/master/` ([mnist.py](https://github.com/Microsoft/nni/blob/master/examples/trials/mnist-tfv1/mnist.py) for example). + * It cannot be re-formatted by sphinx, such as source code, please use its global URL. For source code that links to our github repo, please use URLs rooted at `https://github.com/Microsoft/nni/tree/v1.9/` ([mnist.py](https://github.com/Microsoft/nni/blob/v1.9/examples/trials/mnist-tfv1/mnist.py) for example). diff --git a/docs/en_US/Tutorial/InstallationLinux.md b/docs/en_US/Tutorial/InstallationLinux.md index 88e8176ec3..a2f5cc0242 100644 --- a/docs/en_US/Tutorial/InstallationLinux.md +++ b/docs/en_US/Tutorial/InstallationLinux.md @@ -19,14 +19,14 @@ Installation on Linux and macOS follow the same instructions, given below. Prerequisites: `python 64-bit >=3.6`, `git`, `wget` ```bash - git clone -b v1.8 https://github.com/Microsoft/nni.git + git clone -b v1.9 https://github.com/Microsoft/nni.git cd nni ./install.sh ``` ### Use NNI in a docker image - You can also install NNI in a docker image. Please follow the instructions [here](https://github.com/Microsoft/nni/tree/master/deployment/docker/README.md) to build an NNI docker image. The NNI docker image can also be retrieved from Docker Hub through the command `docker pull msranni/nni:latest`. + You can also install NNI in a docker image. Please follow the instructions [here](https://github.com/Microsoft/nni/tree/v1.9/deployment/docker/README.md) to build an NNI docker image. The NNI docker image can also be retrieved from Docker Hub through the command `docker pull msranni/nni:latest`. ## Verify installation @@ -35,7 +35,7 @@ The following example is built on TensorFlow 1.x. Make sure **TensorFlow 1.x is * Download the examples via cloning the source code. ```bash - git clone -b v1.8 https://github.com/Microsoft/nni.git + git clone -b v1.9 https://github.com/Microsoft/nni.git ``` * Run the MNIST example. diff --git a/docs/en_US/Tutorial/InstallationWin.md b/docs/en_US/Tutorial/InstallationWin.md index a16f0b3cdc..53daccba06 100644 --- a/docs/en_US/Tutorial/InstallationWin.md +++ b/docs/en_US/Tutorial/InstallationWin.md @@ -29,7 +29,7 @@ If you want to contribute to NNI, refer to [setup development environment](Setup * From source code ```bat - git clone -b v1.8 https://github.com/Microsoft/nni.git + git clone -b v1.9 https://github.com/Microsoft/nni.git cd nni powershell -ExecutionPolicy Bypass -file install.ps1 ``` @@ -41,7 +41,7 @@ The following example is built on TensorFlow 1.x. Make sure **TensorFlow 1.x is * Clone examples within source code. ```bat - git clone -b v1.8 https://github.com/Microsoft/nni.git + git clone -b v1.9 https://github.com/Microsoft/nni.git ``` * Run the MNIST example. diff --git a/docs/en_US/Tutorial/QuickStart.md b/docs/en_US/Tutorial/QuickStart.md index 1265d968b8..e70fe7ca25 100644 --- a/docs/en_US/Tutorial/QuickStart.md +++ b/docs/en_US/Tutorial/QuickStart.md @@ -71,7 +71,7 @@ if __name__ == '__main__': run_trial(params) ``` -If you want to see the full implementation, please refer to [examples/trials/mnist-tfv1/mnist_before.py](https://github.com/Microsoft/nni/tree/master/examples/trials/mnist-tfv1/mnist_before.py). +If you want to see the full implementation, please refer to [examples/trials/mnist-tfv1/mnist_before.py](https://github.com/Microsoft/nni/tree/v1.9/examples/trials/mnist-tfv1/mnist_before.py). The above code can only try one set of parameters at a time; if we want to tune learning rate, we need to manually modify the hyperparameter and start the trial again and again. @@ -108,7 +108,7 @@ If you want to use NNI to automatically train your model and find the optimal hy + } ``` -*Example: [search_space.json](https://github.com/Microsoft/nni/tree/master/examples/trials/mnist-tfv1/search_space.json)* +*Example: [search_space.json](https://github.com/Microsoft/nni/tree/v1.9/examples/trials/mnist-tfv1/search_space.json)* **Step 2**: Modify your `Trial` file to get the hyperparameter set from NNI and report the final result to NNI. @@ -133,7 +133,7 @@ If you want to use NNI to automatically train your model and find the optimal hy run_trial(params) ``` -*Example: [mnist.py](https://github.com/Microsoft/nni/tree/master/examples/trials/mnist-tfv1/mnist.py)* +*Example: [mnist.py](https://github.com/Microsoft/nni/tree/v1.9/examples/trials/mnist-tfv1/mnist.py)* **Step 3**: Define a `config` file in YAML which declares the `path` to the search space and trial files. It also gives other information such as the tuning algorithm, max trial number, and max duration arguments. @@ -160,9 +160,9 @@ trial: .. Note:: If you are planning to use remote machines or clusters as your :doc:`training service <../TrainingService/Overview>`, to avoid too much pressure on network, we limit the number of files to 2000 and total size to 300MB. If your codeDir contains too many files, you can choose which files and subfolders should be excluded by adding a ``.nniignore`` file that works like a ``.gitignore`` file. For more details on how to write this file, see the `git documentation `_. ``` -*Example: [config.yml](https://github.com/Microsoft/nni/tree/master/examples/trials/mnist-tfv1/config.yml) [.nniignore](https://github.com/Microsoft/nni/tree/master/examples/trials/mnist-tfv1/.nniignore)* +*Example: [config.yml](https://github.com/Microsoft/nni/tree/v1.9/examples/trials/mnist-tfv1/config.yml) [.nniignore](https://github.com/Microsoft/nni/tree/v1.9/examples/trials/mnist-tfv1/.nniignore)* -All the code above is already prepared and stored in [examples/trials/mnist-tfv1/](https://github.com/Microsoft/nni/tree/master/examples/trials/mnist-tfv1). +All the code above is already prepared and stored in [examples/trials/mnist-tfv1/](https://github.com/Microsoft/nni/tree/v1.9/examples/trials/mnist-tfv1). #### Linux and macOS diff --git a/docs/en_US/Tutorial/SearchSpaceSpec.md b/docs/en_US/Tutorial/SearchSpaceSpec.md index 54264be2b5..af07075745 100644 --- a/docs/en_US/Tutorial/SearchSpaceSpec.md +++ b/docs/en_US/Tutorial/SearchSpaceSpec.md @@ -30,7 +30,7 @@ All types of sampling strategies and their parameter are listed here: * `{"_type": "choice", "_value": options}` * The variable's value is one of the options. Here `options` should be a list of numbers or a list of strings. Using arbitrary objects as members of this list (like sublists, a mixture of numbers and strings, or null values) should work in most cases, but may trigger undefined behaviors. - * `options` can also be a nested sub-search-space, this sub-search-space takes effect only when the corresponding element is chosen. The variables in this sub-search-space can be seen as conditional variables. Here is an simple [example of nested search space definition](https://github.com/microsoft/nni/tree/master/examples/trials/mnist-nested-search-space/search_space.json). If an element in the options list is a dict, it is a sub-search-space, and for our built-in tuners you have to add a `_name` key in this dict, which helps you to identify which element is chosen. Accordingly, here is a [sample](https://github.com/microsoft/nni/tree/master/examples/trials/mnist-nested-search-space/sample.json) which users can get from nni with nested search space definition. See the table below for the tuners which support nested search spaces. + * `options` can also be a nested sub-search-space, this sub-search-space takes effect only when the corresponding element is chosen. The variables in this sub-search-space can be seen as conditional variables. Here is an simple [example of nested search space definition](https://github.com/microsoft/nni/tree/v1.9/examples/trials/mnist-nested-search-space/search_space.json). If an element in the options list is a dict, it is a sub-search-space, and for our built-in tuners you have to add a `_name` key in this dict, which helps you to identify which element is chosen. Accordingly, here is a [sample](https://github.com/microsoft/nni/tree/v1.9/examples/trials/mnist-nested-search-space/sample.json) which users can get from nni with nested search space definition. See the table below for the tuners which support nested search spaces. * `{"_type": "randint", "_value": [lower, upper]}` * Choosing a random integer between `lower` (inclusive) and `upper` (exclusive). diff --git a/docs/en_US/Tutorial/WebUI.md b/docs/en_US/Tutorial/WebUI.md index a4eec711b5..b3bef5023f 100644 --- a/docs/en_US/Tutorial/WebUI.md +++ b/docs/en_US/Tutorial/WebUI.md @@ -4,22 +4,26 @@ Click the tab "Overview". -* On the overview tab, you can see the experiment trial profile/search space and the performance of top trials. +* On the overview tab, you can see the experiment information and status and the performance of top trials. If you want to see config and search space, please click the right button "Config" and "Search space". -![](../../img/webui-img/over1.png) -![](../../img/webui-img/over2.png) +![](../../img/webui-img/full-oview.png) * If your experiment has many trials, you can change the refresh interval here. ![](../../img/webui-img/refresh-interval.png) -* You can review and download the experiment results and nni-manager/dispatcher log files from the "View" button. +* You can review and download the experiment results and nni-manager/dispatcher log files from the "Download" button. ![](../../img/webui-img/download.png) + +* You can change some experiment configurations such as maxExecDuration, maxTrialNum and trial concurrency on here. + +![](../../img/webui-img/edit-experiment-param.png) + * You can click the exclamation point in the error box to see a log message if the experiment's status is an error. ![](../../img/webui-img/log-error.png) ![](../../img/webui-img/review-log.png) -* You can click "Feedback" to report any questions. +* You can click "About" to see the version and report any questions. ## View job default metric @@ -35,15 +39,15 @@ Click the tab "Overview". Click the tab "Hyper Parameter" to see the parallel graph. +* You can add/remove axes and drag to swap axes on the chart. * You can select the percentage to see top trials. -* Choose two axis to swap its positions -![](../../img/hyperPara.png) +![](../../img/webui-img/hyperPara.png) ## View Trial Duration Click the tab "Trial Duration" to see the bar graph. -![](../../img/trial_duration.png) +![](../../img/webui-img/trial_duration.png) ## View Trial Intermediate Result Graph Click the tab "Intermediate Result" to see the line graph. @@ -75,14 +79,12 @@ Click the tab "Trials Detail" to see the status of all trials. Specifically: * You can use the button named "Copy as python" to copy the trial's parameters. ![](../../img/webui-img/copyParameter.png) -* If you run on the OpenPAI or Kubeflow platform, you can also see the hdfsLog. +* If you run on the OpenPAI or Kubeflow platform, you can also see the nfs log. ![](../../img/webui-img/detail-pai.png) -* Intermediate Result Graph: you can see the default and other keys in this graph by clicking the operation column button. +* Intermediate Result Graph: you can see the default metric in this graph by clicking the intermediate button. -![](../../img/webui-img/intermediate-btn.png) ![](../../img/webui-img/intermediate.png) * Kill: you can kill a job that status is running. -![](../../img/webui-img/kill-running.png) -![](../../img/webui-img/canceled.png) +![](../../img/webui-img/kill-running.png) \ No newline at end of file diff --git a/docs/en_US/_templates/index.html b/docs/en_US/_templates/index.html index fd4c650bb7..c84be8fc3b 100644 --- a/docs/en_US/_templates/index.html +++ b/docs/en_US/_templates/index.html @@ -107,11 +107,11 @@

NNI capabilities in a glance

  • Examples
      • -
    • MNIST-pytorch
      • +
    • MNIST-pytorch
      • -
    • MNIST-tensorflow
      • +
    • MNIST-tensorflow
      • -
    • MNIST-keras
      • +
    • MNIST-keras
    • Auto-gbdt
    • Cifar10-pytorch
    • Scikit-learn
      • @@ -393,7 +393,7 @@

      External Repositories

    • Run ENAS with NNI
    • Automatic + href="https://github.com/microsoft/nni/blob/v1.9/examples/feature_engineering/auto-feature-engineering/README.md">Automatic Feature Engineering with NNI
    • Related Projects {% endblock %} diff --git a/docs/en_US/builtin_assessor.rst b/docs/en_US/builtin_assessor.rst index f0ba1849b8..ab058bc8d1 100644 --- a/docs/en_US/builtin_assessor.rst +++ b/docs/en_US/builtin_assessor.rst @@ -7,7 +7,7 @@ Assessor receives the intermediate result from a trial and decides whether the t Here is an experimental result of MNIST after using the 'Curvefitting' Assessor in 'maximize' mode. You can see that Assessor successfully **early stopped** many trials with bad hyperparameters in advance. If you use Assessor, you may get better hyperparameters using the same computing resources. -*Implemented code directory: [config_assessor.yml](https://github.com/Microsoft/nni/blob/master/examples/trials/mnist-tfv1/config_assessor.yml)* +*Implemented code directory: [config_assessor.yml](https://github.com/Microsoft/nni/blob/v1.9/examples/trials/mnist-tfv1/config_assessor.yml)* .. image:: ../img/Assessor.png diff --git a/docs/en_US/conf.py b/docs/en_US/conf.py index 0fecda0372..01ed9b6cc8 100644 --- a/docs/en_US/conf.py +++ b/docs/en_US/conf.py @@ -29,7 +29,7 @@ # The short X.Y version version = '' # The full version, including alpha/beta/rc tags -release = 'v1.8' +release = 'v1.9' # -- General configuration --------------------------------------------------- diff --git a/docs/en_US/nnicli_ref.md b/docs/en_US/nnicli_ref.md index 02c8cbbb30..d308ebca77 100644 --- a/docs/en_US/nnicli_ref.md +++ b/docs/en_US/nnicli_ref.md @@ -3,7 +3,7 @@ NNI client is a python API of `nnictl`, which implements the most commonly used commands. Users can use this API to control their experiments, collect experiment results and conduct advanced analyses based on experiment results in python code directly instead of using command line. Here is an example: ``` -from nnicli import Experiment +from nni.experiment import Experiment # create an experiment instance exp = Experiment() @@ -28,14 +28,14 @@ exp.stop_experiment() ## References ```eval_rst -.. autoclass:: nnicli.Experiment +.. autoclass:: nni.experiment.Experiment :members: -.. autoclass:: nnicli.TrialJob +.. autoclass:: nni.experiment.TrialJob :members: -.. autoclass:: nnicli.TrialHyperParameters +.. autoclass:: nni.experiment.TrialHyperParameters :members: -.. autoclass:: nnicli.TrialMetricData +.. autoclass:: nni.experiment.TrialMetricData :members: -.. autoclass:: nnicli.TrialResult +.. autoclass:: nni.experiment.TrialResult :members: ``` diff --git a/docs/img/webui-img/Intermediate-column.png b/docs/img/webui-img/Intermediate-column.png deleted file mode 100644 index da84fc903e..0000000000 Binary files a/docs/img/webui-img/Intermediate-column.png and /dev/null differ diff --git a/docs/img/webui-img/addColumn.png b/docs/img/webui-img/addColumn.png index cd5b0181d0..907ed00d11 100644 Binary files a/docs/img/webui-img/addColumn.png and b/docs/img/webui-img/addColumn.png differ diff --git a/docs/img/webui-img/best-curve.png b/docs/img/webui-img/best-curve.png index cccc495b08..4880e80623 100644 Binary files a/docs/img/webui-img/best-curve.png and b/docs/img/webui-img/best-curve.png differ diff --git a/docs/img/webui-img/canceled.png b/docs/img/webui-img/canceled.png deleted file mode 100644 index 2f8d9527c7..0000000000 Binary files a/docs/img/webui-img/canceled.png and /dev/null differ diff --git a/docs/img/webui-img/compare.png b/docs/img/webui-img/compare.png index 256bd58837..944e3be0ea 100644 Binary files a/docs/img/webui-img/compare.png and b/docs/img/webui-img/compare.png differ diff --git a/docs/img/webui-img/copyParameter.png b/docs/img/webui-img/copyParameter.png index a668e6f326..2dcd8a9620 100644 Binary files a/docs/img/webui-img/copyParameter.png and b/docs/img/webui-img/copyParameter.png differ diff --git a/docs/img/webui-img/default-metric.png b/docs/img/webui-img/default-metric.png index e7bdd18441..3e75951a1e 100644 Binary files a/docs/img/webui-img/default-metric.png and b/docs/img/webui-img/default-metric.png differ diff --git a/docs/img/webui-img/detail-local.png b/docs/img/webui-img/detail-local.png index 6c82666d35..99b4a663ee 100644 Binary files a/docs/img/webui-img/detail-local.png and b/docs/img/webui-img/detail-local.png differ diff --git a/docs/img/webui-img/detail-pai.png b/docs/img/webui-img/detail-pai.png index 1c2439d94c..0a0921d2fd 100644 Binary files a/docs/img/webui-img/detail-pai.png and b/docs/img/webui-img/detail-pai.png differ diff --git a/docs/img/webui-img/download.png b/docs/img/webui-img/download.png index 0c6c13eb92..ad333df5d5 100644 Binary files a/docs/img/webui-img/download.png and b/docs/img/webui-img/download.png differ diff --git a/docs/img/webui-img/edit-experiment-param.png b/docs/img/webui-img/edit-experiment-param.png new file mode 100644 index 0000000000..8fe7d35ac2 Binary files /dev/null and b/docs/img/webui-img/edit-experiment-param.png differ diff --git a/docs/img/webui-img/filter-intermediate.png b/docs/img/webui-img/filter-intermediate.png index 91b51987ea..1ebbb62d3e 100644 Binary files a/docs/img/webui-img/filter-intermediate.png and b/docs/img/webui-img/filter-intermediate.png differ diff --git a/docs/img/webui-img/full-detail.png b/docs/img/webui-img/full-detail.png new file mode 100644 index 0000000000..e61ab33c69 Binary files /dev/null and b/docs/img/webui-img/full-detail.png differ diff --git a/docs/img/webui-img/full-oview.png b/docs/img/webui-img/full-oview.png new file mode 100644 index 0000000000..7200633200 Binary files /dev/null and b/docs/img/webui-img/full-oview.png differ diff --git a/docs/img/webui-img/hyperPara.png b/docs/img/webui-img/hyperPara.png new file mode 100644 index 0000000000..77f9340f35 Binary files /dev/null and b/docs/img/webui-img/hyperPara.png differ diff --git a/docs/img/webui-img/intermediate-btn.png b/docs/img/webui-img/intermediate-btn.png deleted file mode 100644 index 5e14934f93..0000000000 Binary files a/docs/img/webui-img/intermediate-btn.png and /dev/null differ diff --git a/docs/img/webui-img/intermediate.png b/docs/img/webui-img/intermediate.png index 50a67db9f5..bf19500fd0 100644 Binary files a/docs/img/webui-img/intermediate.png and b/docs/img/webui-img/intermediate.png differ diff --git a/docs/img/webui-img/kill-running.png b/docs/img/webui-img/kill-running.png index 4d9b7b812e..193deb1ae7 100644 Binary files a/docs/img/webui-img/kill-running.png and b/docs/img/webui-img/kill-running.png differ diff --git a/docs/img/webui-img/log-error.png b/docs/img/webui-img/log-error.png index c3691fba25..53f4083541 100644 Binary files a/docs/img/webui-img/log-error.png and b/docs/img/webui-img/log-error.png differ diff --git a/docs/img/webui-img/over1.png b/docs/img/webui-img/over1.png index e2c69720fb..733c1044b1 100644 Binary files a/docs/img/webui-img/over1.png and b/docs/img/webui-img/over1.png differ diff --git a/docs/img/webui-img/over2.png b/docs/img/webui-img/over2.png index 3ba7e3c6f4..f5174c4c72 100644 Binary files a/docs/img/webui-img/over2.png and b/docs/img/webui-img/over2.png differ diff --git a/docs/img/webui-img/refresh-interval.png b/docs/img/webui-img/refresh-interval.png index 1e5d759823..03175dd9b9 100644 Binary files a/docs/img/webui-img/refresh-interval.png and b/docs/img/webui-img/refresh-interval.png differ diff --git a/docs/img/webui-img/review-log.png b/docs/img/webui-img/review-log.png index 559f554ce6..f06e2c8972 100644 Binary files a/docs/img/webui-img/review-log.png and b/docs/img/webui-img/review-log.png differ diff --git a/docs/img/webui-img/search-space-button.png b/docs/img/webui-img/search-space-button.png new file mode 100644 index 0000000000..830c6e780c Binary files /dev/null and b/docs/img/webui-img/search-space-button.png differ diff --git a/docs/img/webui-img/search-space.png b/docs/img/webui-img/search-space.png new file mode 100644 index 0000000000..b77294627e Binary files /dev/null and b/docs/img/webui-img/search-space.png differ diff --git a/docs/img/webui-img/search-trial.png b/docs/img/webui-img/search-trial.png index 80ca052894..3da210ab1f 100644 Binary files a/docs/img/webui-img/search-trial.png and b/docs/img/webui-img/search-trial.png differ diff --git a/docs/img/webui-img/select-trial.png b/docs/img/webui-img/select-trial.png index abbe9acaa7..1635220581 100644 Binary files a/docs/img/webui-img/select-trial.png and b/docs/img/webui-img/select-trial.png differ diff --git a/docs/img/webui-img/trial_duration.png b/docs/img/webui-img/trial_duration.png new file mode 100644 index 0000000000..d4bd5e7359 Binary files /dev/null and b/docs/img/webui-img/trial_duration.png differ diff --git a/docs/img/webui-img/trials_intermeidate.png b/docs/img/webui-img/trials_intermeidate.png index 1f980e920a..767a6014ef 100644 Binary files a/docs/img/webui-img/trials_intermeidate.png and b/docs/img/webui-img/trials_intermeidate.png differ diff --git a/examples/model_compress/amc/README.md b/examples/model_compress/amc/README.md new file mode 100644 index 0000000000..e51cc40288 --- /dev/null +++ b/examples/model_compress/amc/README.md @@ -0,0 +1,28 @@ +# AMCPruner Example +This example shows us how to use AMCPruner example. + +## Step 1: train a model for pruning +Run following command to train a mobilenetv2 model: +```bash +python3 amc_train.py --model_type mobilenetv2 --n_epoch 50 +``` +Once finished, saved checkpoint file can be found at: +``` +logs/mobilenetv2_cifar10_train-run1/ckpt.best.pth +``` + +## Pruning with AMCPruner +Run following command to prune the trained model: +```bash +python3 amc_search.py --model_type mobilenetv2 --ckpt logs/mobilenetv2_cifar10_train-run1/ckpt.best.pth +``` +Once finished, pruned model and mask can be found at: +``` +logs/mobilenetv2_cifar10_r0.5_search-run2 +``` + +## Finetune pruned model +Run `amc_train.py` again with `--ckpt` and `--mask` to speedup and finetune the pruned model: +```bash +python3 amc_train.py --model_type mobilenetv2 --ckpt logs/mobilenetv2_cifar10_r0.5_search-run2/best_model.pth --mask logs/mobilenetv2_cifar10_r0.5_search-run2/best_mask.pth --n_epoch 100 +``` diff --git a/examples/model_compress/amc/amc_search.py b/examples/model_compress/amc/amc_search.py index a11459d81c..0bc6b1a969 100644 --- a/examples/model_compress/amc/amc_search.py +++ b/examples/model_compress/amc/amc_search.py @@ -20,7 +20,7 @@ def parse_args(): help='model to prune') parser.add_argument('--dataset', default='cifar10', type=str, choices=['cifar10', 'imagenet'], help='dataset to use (cifar/imagenet)') parser.add_argument('--batch_size', default=50, type=int, help='number of data batch size') - parser.add_argument('--data_root', default='./cifar10', type=str, help='dataset path') + parser.add_argument('--data_root', default='./data', type=str, help='dataset path') parser.add_argument('--flops_ratio', default=0.5, type=float, help='target flops ratio to preserve of the model') parser.add_argument('--lbound', default=0.2, type=float, help='minimum sparsity') parser.add_argument('--rbound', default=1., type=float, help='maximum sparsity') diff --git a/examples/model_compress/amc/amc_train.py b/examples/model_compress/amc/amc_train.py index c105d25785..5fde972392 100644 --- a/examples/model_compress/amc/amc_train.py +++ b/examples/model_compress/amc/amc_train.py @@ -13,6 +13,7 @@ import torch.nn as nn import torch.optim as optim from tensorboardX import SummaryWriter +from torchvision.models import resnet from nni.compression.torch.pruning.amc.lib.net_measure import measure_model from nni.compression.torch.pruning.amc.lib.utils import get_output_folder @@ -27,7 +28,9 @@ def parse_args(): parser = argparse.ArgumentParser(description='AMC train / fine-tune script') - parser.add_argument('--model_type', default='mobilenet', type=str, help='name of the model to train') + parser.add_argument('--model_type', default='mobilenet', type=str, + choices=['mobilenet', 'mobilenetv2', 'resnet18', 'resnet34', 'resnet50'], + help='name of the model to train') parser.add_argument('--dataset', default='cifar10', type=str, help='name of the dataset to train') parser.add_argument('--lr', default=0.05, type=float, help='learning rate') parser.add_argument('--n_gpu', default=4, type=int, help='number of GPUs to use') @@ -62,17 +65,21 @@ def get_model(args): net = MobileNet(n_class=n_class) elif args.model_type == 'mobilenetv2': net = MobileNetV2(n_class=n_class) + elif args.model_type.startswith('resnet'): + net = resnet.__dict__[args.model_type](pretrained=True) + in_features = net.fc.in_features + net.fc = nn.Linear(in_features, n_class) else: raise NotImplementedError if args.ckpt_path is not None: # the checkpoint can be state_dict exported by amc_search.py or saved by amc_train.py print('=> Loading checkpoint {} ..'.format(args.ckpt_path)) - net.load_state_dict(torch.load(args.ckpt_path)) + net.load_state_dict(torch.load(args.ckpt_path, torch.device('cpu'))) if args.mask_path is not None: SZ = 224 if args.dataset == 'imagenet' else 32 data = torch.randn(2, 3, SZ, SZ) - ms = ModelSpeedup(net, data, args.mask_path) + ms = ModelSpeedup(net, data, args.mask_path, torch.device('cpu')) ms.speedup_model() net.to(args.device) @@ -179,11 +186,11 @@ def adjust_learning_rate(optimizer, epoch): return lr def save_checkpoint(state, is_best, checkpoint_dir='.'): - filename = os.path.join(checkpoint_dir, 'ckpt.pth.tar') + filename = os.path.join(checkpoint_dir, 'ckpt.pth') print('=> Saving checkpoint to {}'.format(filename)) torch.save(state, filename) if is_best: - shutil.copyfile(filename, filename.replace('.pth.tar', '.best.pth.tar')) + shutil.copyfile(filename, filename.replace('.pth', '.best.pth')) if __name__ == '__main__': args = parse_args() diff --git a/examples/nas/naive-tf/train.py b/examples/nas/naive-tf/train.py index a4e56a39f8..6b00d4d45e 100644 --- a/examples/nas/naive-tf/train.py +++ b/examples/nas/naive-tf/train.py @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + import tensorflow as tf from tensorflow.keras import Model from tensorflow.keras.layers import (AveragePooling2D, BatchNormalization, Conv2D, Dense, MaxPool2D) @@ -7,8 +10,6 @@ from nni.nas.tensorflow.mutables import LayerChoice, InputChoice from nni.nas.tensorflow.enas import EnasTrainer -tf.get_logger().setLevel('ERROR') - class Net(Model): def __init__(self): @@ -53,35 +54,36 @@ def call(self, x): return x -def accuracy(output, target): - bs = target.shape[0] - predicted = tf.cast(tf.argmax(output, 1), target.dtype) - target = tf.reshape(target, [-1]) - return sum(tf.cast(predicted == target, tf.float32)) / bs +def accuracy(truth, logits): + truth = tf.reshape(truth, -1) + predicted = tf.cast(tf.math.argmax(logits, axis=1), truth.dtype) + equal = tf.cast(predicted == truth, tf.int32) + return tf.math.reduce_sum(equal).numpy() / equal.shape[0] + +def accuracy_metrics(truth, logits): + acc = accuracy(truth, logits) + return {'accuracy': acc} if __name__ == '__main__': cifar10 = tf.keras.datasets.cifar10 - (x_train, y_train), (x_test, y_test) = cifar10.load_data() - x_train, x_test = x_train / 255.0, x_test / 255.0 - split = int(len(x_train) * 0.9) - dataset_train = tf.data.Dataset.from_tensor_slices((x_train[:split], y_train[:split])).batch(64) - dataset_valid = tf.data.Dataset.from_tensor_slices((x_train[split:], y_train[split:])).batch(64) - dataset_test = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(64) + (x_train, y_train), (x_valid, y_valid) = cifar10.load_data() + x_train, x_valid = x_train / 255.0, x_valid / 255.0 + train_set = (x_train, y_train) + valid_set = (x_valid, y_valid) net = Net() + trainer = EnasTrainer( net, - loss=SparseCategoricalCrossentropy(reduction=Reduction.SUM), - metrics=accuracy, + loss=SparseCategoricalCrossentropy(from_logits=True, reduction=Reduction.NONE), + metrics=accuracy_metrics, reward_function=accuracy, optimizer=SGD(learning_rate=0.001, momentum=0.9), batch_size=64, num_epochs=2, - dataset_train=dataset_train, - dataset_valid=dataset_valid, - dataset_test=dataset_test + dataset_train=train_set, + dataset_valid=valid_set ) trainer.train() - #trainer.export('checkpoint') diff --git a/examples/nas/spos/supernet.py b/examples/nas/spos/supernet.py index 3ab717868c..bf6848698f 100644 --- a/examples/nas/spos/supernet.py +++ b/examples/nas/spos/supernet.py @@ -45,6 +45,7 @@ torch.backends.cudnn.deterministic = True model = ShuffleNetV2OneShot() + flops_func = model.get_candidate_flops if args.load_checkpoint: if not args.spos_preprocessing: logger.warning("You might want to use SPOS preprocessing if you are loading their checkpoints.") @@ -52,7 +53,7 @@ model.cuda() if torch.cuda.device_count() > 1: # exclude last gpu, saving for data preprocessing on gpu model = nn.DataParallel(model, device_ids=list(range(0, torch.cuda.device_count() - 1))) - mutator = SPOSSupernetTrainingMutator(model, flops_func=model.module.get_candidate_flops, + mutator = SPOSSupernetTrainingMutator(model, flops_func=flops_func, flops_lb=290E6, flops_ub=360E6) criterion = CrossEntropyLabelSmooth(1000, args.label_smoothing) optimizer = torch.optim.SGD(model.parameters(), lr=args.learning_rate, diff --git a/examples/trials/mnist-pytorch/config_aml.yml b/examples/trials/mnist-pytorch/config_aml.yml index 883a00340e..8a5618606f 100644 --- a/examples/trials/mnist-pytorch/config_aml.yml +++ b/examples/trials/mnist-pytorch/config_aml.yml @@ -17,9 +17,9 @@ tuner: trial: command: python3 mnist.py codeDir: . - computeTarget: ${replace_to_your_computeTarget} image: msranni/nni amlConfig: subscriptionId: ${replace_to_your_subscriptionId} resourceGroup: ${replace_to_your_resourceGroup} workspaceName: ${replace_to_your_workspaceName} + computeTarget: ${replace_to_your_computeTarget} diff --git a/examples/trials/mnist-tfv1/config_aml.yml b/examples/trials/mnist-tfv1/config_aml.yml index 6a556a2f4e..cfc5fcaaf5 100644 --- a/examples/trials/mnist-tfv1/config_aml.yml +++ b/examples/trials/mnist-tfv1/config_aml.yml @@ -17,9 +17,9 @@ tuner: trial: command: python3 mnist.py codeDir: . - computeTarget: ${replace_to_your_computeTarget} image: msranni/nni amlConfig: subscriptionId: ${replace_to_your_subscriptionId} resourceGroup: ${replace_to_your_resourceGroup} workspaceName: ${replace_to_your_workspaceName} + computeTarget: ${replace_to_your_computeTarget} diff --git a/src/sdk/pynni/nni/__init__.py b/nni/__init__.py similarity index 81% rename from src/sdk/pynni/nni/__init__.py rename to nni/__init__.py index f83650eee1..0630571ae6 100644 --- a/src/sdk/pynni/nni/__init__.py +++ b/nni/__init__.py @@ -3,13 +3,13 @@ __version__ = '999.0.0-developing' -from .env_vars import dispatcher_env_vars +from .runtime.env_vars import dispatcher_env_vars from .utils import ClassArgsValidator if dispatcher_env_vars.SDK_PROCESS != 'dispatcher': from .trial import * from .smartparam import * - from .nas_utils import training_update + from .common.nas_utils import training_update class NoMoreTrialError(Exception): def __init__(self, ErrorInfo): diff --git a/src/sdk/pynni/nni/__main__.py b/nni/__main__.py similarity index 94% rename from src/sdk/pynni/nni/__main__.py rename to nni/__main__.py index 003a2bfe41..e3f982f42f 100644 --- a/src/sdk/pynni/nni/__main__.py +++ b/nni/__main__.py @@ -7,9 +7,9 @@ import json import base64 -from .common import enable_multi_thread, enable_multi_phase -from .msg_dispatcher import MsgDispatcher -from .package_utils import create_builtin_class_instance, create_customized_class_instance +from .runtime.common import enable_multi_thread, enable_multi_phase +from .runtime.msg_dispatcher import MsgDispatcher +from .tools.package_utils import create_builtin_class_instance, create_customized_class_instance logger = logging.getLogger('nni.main') logger.debug('START') diff --git a/src/sdk/pynni/nni/compression/torch/pruning/__init__.py b/nni/algorithms/compression/pytorch/pruning/__init__.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/pruning/__init__.py rename to nni/algorithms/compression/pytorch/pruning/__init__.py diff --git a/src/sdk/pynni/nni/compression/torch/pruning/admm_pruner.py b/nni/algorithms/compression/pytorch/pruning/admm_pruner.py similarity index 98% rename from src/sdk/pynni/nni/compression/torch/pruning/admm_pruner.py rename to nni/algorithms/compression/pytorch/pruning/admm_pruner.py index ae1a9ec9dc..f65f1405e1 100644 --- a/src/sdk/pynni/nni/compression/torch/pruning/admm_pruner.py +++ b/nni/algorithms/compression/pytorch/pruning/admm_pruner.py @@ -5,7 +5,7 @@ import torch from schema import And, Optional -from ..utils.config_validation import CompressorSchema +from nni.compression.pytorch.utils.config_validation import CompressorSchema from .constants import MASKER_DICT from .one_shot import OneshotPruner diff --git a/src/sdk/pynni/nni/compression/torch/pruning/agp.py b/nni/algorithms/compression/pytorch/pruning/agp.py similarity index 97% rename from src/sdk/pynni/nni/compression/torch/pruning/agp.py rename to nni/algorithms/compression/pytorch/pruning/agp.py index 3e34ba5aef..ef9ca71635 100644 --- a/src/sdk/pynni/nni/compression/torch/pruning/agp.py +++ b/nni/algorithms/compression/pytorch/pruning/agp.py @@ -13,8 +13,8 @@ import torch from schema import And, Optional from .constants import MASKER_DICT -from ..utils.config_validation import CompressorSchema -from ..compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Pruner __all__ = ['AGPPruner'] diff --git a/src/sdk/pynni/nni/compression/torch/pruning/amc/__init__.py b/nni/algorithms/compression/pytorch/pruning/amc/__init__.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/pruning/amc/__init__.py rename to nni/algorithms/compression/pytorch/pruning/amc/__init__.py diff --git a/src/sdk/pynni/nni/compression/torch/pruning/amc/amc_pruner.py b/nni/algorithms/compression/pytorch/pruning/amc/amc_pruner.py similarity index 99% rename from src/sdk/pynni/nni/compression/torch/pruning/amc/amc_pruner.py rename to nni/algorithms/compression/pytorch/pruning/amc/amc_pruner.py index 540ec3fbbc..c2d12429d5 100644 --- a/src/sdk/pynni/nni/compression/torch/pruning/amc/amc_pruner.py +++ b/nni/algorithms/compression/pytorch/pruning/amc/amc_pruner.py @@ -9,7 +9,7 @@ import torch from torch.utils.tensorboard import SummaryWriter -from nni.compression.torch.compressor import Pruner +from nni.compression.pytorch.compressor import Pruner from .channel_pruning_env import ChannelPruningEnv from .lib.agent import DDPG from .lib.utils import get_output_folder diff --git a/src/sdk/pynni/nni/compression/torch/pruning/amc/channel_pruning_env.py b/nni/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py similarity index 99% rename from src/sdk/pynni/nni/compression/torch/pruning/amc/channel_pruning_env.py rename to nni/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py index eb5e86b5d2..443daf7efb 100644 --- a/src/sdk/pynni/nni/compression/torch/pruning/amc/channel_pruning_env.py +++ b/nni/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py @@ -10,7 +10,7 @@ import torch import torch.nn as nn -from nni.compression.torch.compressor import PrunerModuleWrapper +from nni.compression.pytorch.compressor import PrunerModuleWrapper from .. import AMCWeightMasker _logger = logging.getLogger(__name__) diff --git a/src/sdk/pynni/nni/batch_tuner/__init__.py b/nni/algorithms/compression/pytorch/pruning/amc/lib/__init__.py similarity index 100% rename from src/sdk/pynni/nni/batch_tuner/__init__.py rename to nni/algorithms/compression/pytorch/pruning/amc/lib/__init__.py diff --git a/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/agent.py b/nni/algorithms/compression/pytorch/pruning/amc/lib/agent.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/pruning/amc/lib/agent.py rename to nni/algorithms/compression/pytorch/pruning/amc/lib/agent.py diff --git a/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/memory.py b/nni/algorithms/compression/pytorch/pruning/amc/lib/memory.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/pruning/amc/lib/memory.py rename to nni/algorithms/compression/pytorch/pruning/amc/lib/memory.py diff --git a/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/net_measure.py b/nni/algorithms/compression/pytorch/pruning/amc/lib/net_measure.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/pruning/amc/lib/net_measure.py rename to nni/algorithms/compression/pytorch/pruning/amc/lib/net_measure.py diff --git a/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/utils.py b/nni/algorithms/compression/pytorch/pruning/amc/lib/utils.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/pruning/amc/lib/utils.py rename to nni/algorithms/compression/pytorch/pruning/amc/lib/utils.py diff --git a/src/sdk/pynni/nni/compression/torch/pruning/apply_compression.py b/nni/algorithms/compression/pytorch/pruning/apply_compression.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/pruning/apply_compression.py rename to nni/algorithms/compression/pytorch/pruning/apply_compression.py diff --git a/src/sdk/pynni/nni/compression/torch/pruning/auto_compress_pruner.py b/nni/algorithms/compression/pytorch/pruning/auto_compress_pruner.py similarity index 98% rename from src/sdk/pynni/nni/compression/torch/pruning/auto_compress_pruner.py rename to nni/algorithms/compression/pytorch/pruning/auto_compress_pruner.py index 24db9f2f88..d52c6ec42d 100644 --- a/src/sdk/pynni/nni/compression/torch/pruning/auto_compress_pruner.py +++ b/nni/algorithms/compression/pytorch/pruning/auto_compress_pruner.py @@ -8,10 +8,10 @@ from schema import And, Optional from nni.utils import OptimizeMode -from nni.compression.torch import ModelSpeedup +from nni.compression.pytorch import ModelSpeedup -from ..compressor import Pruner -from ..utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema from .simulated_annealing_pruner import SimulatedAnnealingPruner from .admm_pruner import ADMMPruner diff --git a/src/sdk/pynni/nni/compression/torch/pruning/constants.py b/nni/algorithms/compression/pytorch/pruning/constants.py similarity index 87% rename from src/sdk/pynni/nni/compression/torch/pruning/constants.py rename to nni/algorithms/compression/pytorch/pruning/constants.py index 2d0545d09d..24b84340cc 100644 --- a/src/sdk/pynni/nni/compression/torch/pruning/constants.py +++ b/nni/algorithms/compression/pytorch/pruning/constants.py @@ -2,7 +2,7 @@ # Licensed under the MIT license. -from ..pruning import LevelPrunerMasker, SlimPrunerMasker, L1FilterPrunerMasker, \ +from . import LevelPrunerMasker, SlimPrunerMasker, L1FilterPrunerMasker, \ L2FilterPrunerMasker, FPGMPrunerMasker, TaylorFOWeightFilterPrunerMasker, \ ActivationAPoZRankFilterPrunerMasker, ActivationMeanRankFilterPrunerMasker diff --git a/src/sdk/pynni/nni/compression/torch/pruning/constants_pruner.py b/nni/algorithms/compression/pytorch/pruning/constants_pruner.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/pruning/constants_pruner.py rename to nni/algorithms/compression/pytorch/pruning/constants_pruner.py diff --git a/src/sdk/pynni/nni/compression/torch/pruning/finegrained_pruning.py b/nni/algorithms/compression/pytorch/pruning/finegrained_pruning.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/pruning/finegrained_pruning.py rename to nni/algorithms/compression/pytorch/pruning/finegrained_pruning.py diff --git a/src/sdk/pynni/nni/compression/torch/pruning/lottery_ticket.py b/nni/algorithms/compression/pytorch/pruning/lottery_ticket.py similarity index 97% rename from src/sdk/pynni/nni/compression/torch/pruning/lottery_ticket.py rename to nni/algorithms/compression/pytorch/pruning/lottery_ticket.py index 3a05217753..b0d041dd02 100644 --- a/src/sdk/pynni/nni/compression/torch/pruning/lottery_ticket.py +++ b/nni/algorithms/compression/pytorch/pruning/lottery_ticket.py @@ -5,8 +5,8 @@ import logging import torch from schema import And, Optional -from ..utils.config_validation import CompressorSchema -from ..compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Pruner from .finegrained_pruning import LevelPrunerMasker logger = logging.getLogger('torch pruner') diff --git a/src/sdk/pynni/nni/compression/torch/pruning/net_adapt_pruner.py b/nni/algorithms/compression/pytorch/pruning/net_adapt_pruner.py similarity index 98% rename from src/sdk/pynni/nni/compression/torch/pruning/net_adapt_pruner.py rename to nni/algorithms/compression/pytorch/pruning/net_adapt_pruner.py index 81d0c47d14..6f55234d5b 100644 --- a/src/sdk/pynni/nni/compression/torch/pruning/net_adapt_pruner.py +++ b/nni/algorithms/compression/pytorch/pruning/net_adapt_pruner.py @@ -10,9 +10,9 @@ from nni.utils import OptimizeMode -from ..compressor import Pruner -from ..utils.config_validation import CompressorSchema -from ..utils.num_param_counter import get_total_num_weights +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.utils.num_param_counter import get_total_num_weights from .constants_pruner import PRUNER_DICT diff --git a/src/sdk/pynni/nni/compression/torch/pruning/one_shot.py b/nni/algorithms/compression/pytorch/pruning/one_shot.py similarity index 98% rename from src/sdk/pynni/nni/compression/torch/pruning/one_shot.py rename to nni/algorithms/compression/pytorch/pruning/one_shot.py index 1958af91e2..75e2a7c307 100644 --- a/src/sdk/pynni/nni/compression/torch/pruning/one_shot.py +++ b/nni/algorithms/compression/pytorch/pruning/one_shot.py @@ -3,11 +3,11 @@ import logging from schema import And, Optional, SchemaError -from nni._graph_utils import TorchModuleGraph -from nni.compression.torch.utils.shape_dependency import ChannelDependency, GroupDependency +from nni.common.graph_utils import TorchModuleGraph +from nni.compression.pytorch.utils.shape_dependency import ChannelDependency, GroupDependency from .constants import MASKER_DICT -from ..utils.config_validation import CompressorSchema -from ..compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Pruner __all__ = ['LevelPruner', 'SlimPruner', 'L1FilterPruner', 'L2FilterPruner', 'FPGMPruner', diff --git a/src/sdk/pynni/nni/compression/torch/pruning/sensitivity_pruner.py b/nni/algorithms/compression/pytorch/pruning/sensitivity_pruner.py similarity index 98% rename from src/sdk/pynni/nni/compression/torch/pruning/sensitivity_pruner.py rename to nni/algorithms/compression/pytorch/pruning/sensitivity_pruner.py index ed72b1f542..037e7efc5e 100644 --- a/src/sdk/pynni/nni/compression/torch/pruning/sensitivity_pruner.py +++ b/nni/algorithms/compression/pytorch/pruning/sensitivity_pruner.py @@ -8,10 +8,10 @@ import torch from schema import And, Optional -from ..compressor import Pruner -from ..utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema from .constants_pruner import PRUNER_DICT -from ..utils.sensitivity_analysis import SensitivityAnalysis +from nni.compression.pytorch.utils.sensitivity_analysis import SensitivityAnalysis MAX_PRUNE_RATIO_PER_ITER = 0.95 diff --git a/src/sdk/pynni/nni/compression/torch/pruning/simulated_annealing_pruner.py b/nni/algorithms/compression/pytorch/pruning/simulated_annealing_pruner.py similarity index 99% rename from src/sdk/pynni/nni/compression/torch/pruning/simulated_annealing_pruner.py rename to nni/algorithms/compression/pytorch/pruning/simulated_annealing_pruner.py index accd325d06..91cf160ff3 100644 --- a/src/sdk/pynni/nni/compression/torch/pruning/simulated_annealing_pruner.py +++ b/nni/algorithms/compression/pytorch/pruning/simulated_annealing_pruner.py @@ -12,8 +12,8 @@ from nni.utils import OptimizeMode -from ..compressor import Pruner -from ..utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema from .constants_pruner import PRUNER_DICT diff --git a/src/sdk/pynni/nni/compression/torch/pruning/structured_pruning.py b/nni/algorithms/compression/pytorch/pruning/structured_pruning.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/pruning/structured_pruning.py rename to nni/algorithms/compression/pytorch/pruning/structured_pruning.py diff --git a/src/sdk/pynni/nni/compression/torch/pruning/weight_masker.py b/nni/algorithms/compression/pytorch/pruning/weight_masker.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/pruning/weight_masker.py rename to nni/algorithms/compression/pytorch/pruning/weight_masker.py diff --git a/src/sdk/pynni/nni/compression/torch/quantization/__init__.py b/nni/algorithms/compression/pytorch/quantization/__init__.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/quantization/__init__.py rename to nni/algorithms/compression/pytorch/quantization/__init__.py diff --git a/src/sdk/pynni/nni/compression/torch/quantization/quantizers.py b/nni/algorithms/compression/pytorch/quantization/quantizers.py similarity index 98% rename from src/sdk/pynni/nni/compression/torch/quantization/quantizers.py rename to nni/algorithms/compression/pytorch/quantization/quantizers.py index 5f4ad36d5e..689b3c56b2 100644 --- a/src/sdk/pynni/nni/compression/torch/quantization/quantizers.py +++ b/nni/algorithms/compression/pytorch/quantization/quantizers.py @@ -5,8 +5,8 @@ import copy import torch from schema import Schema, And, Or, Optional -from ..utils.config_validation import CompressorSchema -from ..compressor import Quantizer, QuantGrad, QuantType +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Quantizer, QuantGrad, QuantType __all__ = ['NaiveQuantizer', 'QAT_Quantizer', 'DoReFaQuantizer', 'BNNQuantizer'] diff --git a/src/sdk/pynni/nni/compression/tensorflow/pruning/__init__.py b/nni/algorithms/compression/tensorflow/pruning/__init__.py similarity index 100% rename from src/sdk/pynni/nni/compression/tensorflow/pruning/__init__.py rename to nni/algorithms/compression/tensorflow/pruning/__init__.py diff --git a/src/sdk/pynni/nni/compression/tensorflow/pruning/one_shot.py b/nni/algorithms/compression/tensorflow/pruning/one_shot.py similarity index 97% rename from src/sdk/pynni/nni/compression/tensorflow/pruning/one_shot.py rename to nni/algorithms/compression/tensorflow/pruning/one_shot.py index 13e45d8592..76dc1915ce 100644 --- a/src/sdk/pynni/nni/compression/tensorflow/pruning/one_shot.py +++ b/nni/algorithms/compression/tensorflow/pruning/one_shot.py @@ -1,6 +1,6 @@ import tensorflow as tf -from ..compressor import Pruner +from nni.compression.tensorflow import Pruner __all__ = [ 'OneshotPruner', diff --git a/src/sdk/pynni/nni/feature_engineering/gbdt_selector/__init__.py b/nni/algorithms/feature_engineering/gbdt_selector/__init__.py similarity index 100% rename from src/sdk/pynni/nni/feature_engineering/gbdt_selector/__init__.py rename to nni/algorithms/feature_engineering/gbdt_selector/__init__.py diff --git a/src/sdk/pynni/nni/feature_engineering/gbdt_selector/gbdt_selector.py b/nni/algorithms/feature_engineering/gbdt_selector/gbdt_selector.py similarity index 99% rename from src/sdk/pynni/nni/feature_engineering/gbdt_selector/gbdt_selector.py rename to nni/algorithms/feature_engineering/gbdt_selector/gbdt_selector.py index e5f88989b3..9ee09c25a3 100644 --- a/src/sdk/pynni/nni/feature_engineering/gbdt_selector/gbdt_selector.py +++ b/nni/algorithms/feature_engineering/gbdt_selector/gbdt_selector.py @@ -25,11 +25,12 @@ class GBDTSelector import random from sklearn.model_selection import train_test_split -from nni.feature_engineering.feature_selector import FeatureSelector # pylint: disable=E0401 import lightgbm as lgb +from nni.feature_engineering.feature_selector import FeatureSelector + class GBDTSelector(FeatureSelector): diff --git a/src/sdk/pynni/nni/feature_engineering/gbdt_selector/requirements.txt b/nni/algorithms/feature_engineering/gbdt_selector/requirements.txt similarity index 100% rename from src/sdk/pynni/nni/feature_engineering/gbdt_selector/requirements.txt rename to nni/algorithms/feature_engineering/gbdt_selector/requirements.txt diff --git a/src/sdk/pynni/nni/feature_engineering/gradient_selector/__init__.py b/nni/algorithms/feature_engineering/gradient_selector/__init__.py similarity index 100% rename from src/sdk/pynni/nni/feature_engineering/gradient_selector/__init__.py rename to nni/algorithms/feature_engineering/gradient_selector/__init__.py diff --git a/src/sdk/pynni/nni/feature_engineering/gradient_selector/constants.py b/nni/algorithms/feature_engineering/gradient_selector/constants.py similarity index 100% rename from src/sdk/pynni/nni/feature_engineering/gradient_selector/constants.py rename to nni/algorithms/feature_engineering/gradient_selector/constants.py diff --git a/src/sdk/pynni/nni/feature_engineering/gradient_selector/fginitialize.py b/nni/algorithms/feature_engineering/gradient_selector/fginitialize.py similarity index 100% rename from src/sdk/pynni/nni/feature_engineering/gradient_selector/fginitialize.py rename to nni/algorithms/feature_engineering/gradient_selector/fginitialize.py diff --git a/src/sdk/pynni/nni/feature_engineering/gradient_selector/fgtrain.py b/nni/algorithms/feature_engineering/gradient_selector/fgtrain.py similarity index 100% rename from src/sdk/pynni/nni/feature_engineering/gradient_selector/fgtrain.py rename to nni/algorithms/feature_engineering/gradient_selector/fgtrain.py diff --git a/src/sdk/pynni/nni/feature_engineering/gradient_selector/gradient_selector.py b/nni/algorithms/feature_engineering/gradient_selector/gradient_selector.py similarity index 100% rename from src/sdk/pynni/nni/feature_engineering/gradient_selector/gradient_selector.py rename to nni/algorithms/feature_engineering/gradient_selector/gradient_selector.py diff --git a/src/sdk/pynni/nni/feature_engineering/gradient_selector/learnability.py b/nni/algorithms/feature_engineering/gradient_selector/learnability.py similarity index 100% rename from src/sdk/pynni/nni/feature_engineering/gradient_selector/learnability.py rename to nni/algorithms/feature_engineering/gradient_selector/learnability.py diff --git a/src/sdk/pynni/nni/feature_engineering/gradient_selector/requirements.txt b/nni/algorithms/feature_engineering/gradient_selector/requirements.txt similarity index 100% rename from src/sdk/pynni/nni/feature_engineering/gradient_selector/requirements.txt rename to nni/algorithms/feature_engineering/gradient_selector/requirements.txt diff --git a/src/sdk/pynni/nni/feature_engineering/gradient_selector/syssettings.py b/nni/algorithms/feature_engineering/gradient_selector/syssettings.py similarity index 100% rename from src/sdk/pynni/nni/feature_engineering/gradient_selector/syssettings.py rename to nni/algorithms/feature_engineering/gradient_selector/syssettings.py diff --git a/src/sdk/pynni/nni/feature_engineering/gradient_selector/utils.py b/nni/algorithms/feature_engineering/gradient_selector/utils.py similarity index 100% rename from src/sdk/pynni/nni/feature_engineering/gradient_selector/utils.py rename to nni/algorithms/feature_engineering/gradient_selector/utils.py diff --git a/src/sdk/pynni/nni/bohb_advisor/__init__.py b/nni/algorithms/hpo/batch_tuner/__init__.py similarity index 100% rename from src/sdk/pynni/nni/bohb_advisor/__init__.py rename to nni/algorithms/hpo/batch_tuner/__init__.py diff --git a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py b/nni/algorithms/hpo/batch_tuner/batch_tuner.py similarity index 100% rename from src/sdk/pynni/nni/batch_tuner/batch_tuner.py rename to nni/algorithms/hpo/batch_tuner/batch_tuner.py diff --git a/src/sdk/pynni/nni/compression/__init__.py b/nni/algorithms/hpo/bohb_advisor/__init__.py similarity index 100% rename from src/sdk/pynni/nni/compression/__init__.py rename to nni/algorithms/hpo/bohb_advisor/__init__.py diff --git a/src/sdk/pynni/nni/bohb_advisor/bohb_advisor.py b/nni/algorithms/hpo/bohb_advisor/bohb_advisor.py similarity index 99% rename from src/sdk/pynni/nni/bohb_advisor/bohb_advisor.py rename to nni/algorithms/hpo/bohb_advisor/bohb_advisor.py index 40a78ada69..73687abc5c 100644 --- a/src/sdk/pynni/nni/bohb_advisor/bohb_advisor.py +++ b/nni/algorithms/hpo/bohb_advisor/bohb_advisor.py @@ -14,10 +14,10 @@ import ConfigSpace.hyperparameters as CSH from nni import ClassArgsValidator -from nni.protocol import CommandType, send -from nni.msg_dispatcher_base import MsgDispatcherBase +from nni.runtime.protocol import CommandType, send +from nni.runtime.msg_dispatcher_base import MsgDispatcherBase from nni.utils import OptimizeMode, MetricType, extract_scalar_reward -from nni.common import multi_phase_enabled +from nni.runtime.common import multi_phase_enabled from .config_generator import CG_BOHB diff --git a/src/sdk/pynni/nni/bohb_advisor/config_generator.py b/nni/algorithms/hpo/bohb_advisor/config_generator.py similarity index 100% rename from src/sdk/pynni/nni/bohb_advisor/config_generator.py rename to nni/algorithms/hpo/bohb_advisor/config_generator.py diff --git a/src/sdk/pynni/nni/bohb_advisor/requirements.txt b/nni/algorithms/hpo/bohb_advisor/requirements.txt similarity index 100% rename from src/sdk/pynni/nni/bohb_advisor/requirements.txt rename to nni/algorithms/hpo/bohb_advisor/requirements.txt diff --git a/src/sdk/pynni/nni/curvefitting_assessor/__init__.py b/nni/algorithms/hpo/curvefitting_assessor/__init__.py similarity index 100% rename from src/sdk/pynni/nni/curvefitting_assessor/__init__.py rename to nni/algorithms/hpo/curvefitting_assessor/__init__.py diff --git a/src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py b/nni/algorithms/hpo/curvefitting_assessor/curvefitting_assessor.py similarity index 100% rename from src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py rename to nni/algorithms/hpo/curvefitting_assessor/curvefitting_assessor.py diff --git a/src/sdk/pynni/nni/curvefitting_assessor/curvefunctions.py b/nni/algorithms/hpo/curvefitting_assessor/curvefunctions.py similarity index 100% rename from src/sdk/pynni/nni/curvefitting_assessor/curvefunctions.py rename to nni/algorithms/hpo/curvefitting_assessor/curvefunctions.py diff --git a/src/sdk/pynni/nni/curvefitting_assessor/model_factory.py b/nni/algorithms/hpo/curvefitting_assessor/model_factory.py similarity index 100% rename from src/sdk/pynni/nni/curvefitting_assessor/model_factory.py rename to nni/algorithms/hpo/curvefitting_assessor/model_factory.py diff --git a/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/__init__.py b/nni/algorithms/hpo/evolution_tuner/__init__.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/pruning/amc/lib/__init__.py rename to nni/algorithms/hpo/evolution_tuner/__init__.py diff --git a/src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py b/nni/algorithms/hpo/evolution_tuner/evolution_tuner.py similarity index 100% rename from src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py rename to nni/algorithms/hpo/evolution_tuner/evolution_tuner.py diff --git a/src/sdk/pynni/nni/compression/torch/utils/__init__.py b/nni/algorithms/hpo/gp_tuner/__init__.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/utils/__init__.py rename to nni/algorithms/hpo/gp_tuner/__init__.py diff --git a/src/sdk/pynni/nni/gp_tuner/gp_tuner.py b/nni/algorithms/hpo/gp_tuner/gp_tuner.py similarity index 100% rename from src/sdk/pynni/nni/gp_tuner/gp_tuner.py rename to nni/algorithms/hpo/gp_tuner/gp_tuner.py diff --git a/src/sdk/pynni/nni/gp_tuner/target_space.py b/nni/algorithms/hpo/gp_tuner/target_space.py similarity index 100% rename from src/sdk/pynni/nni/gp_tuner/target_space.py rename to nni/algorithms/hpo/gp_tuner/target_space.py diff --git a/src/sdk/pynni/nni/gp_tuner/util.py b/nni/algorithms/hpo/gp_tuner/util.py similarity index 100% rename from src/sdk/pynni/nni/gp_tuner/util.py rename to nni/algorithms/hpo/gp_tuner/util.py diff --git a/src/sdk/pynni/nni/gridsearch_tuner/__init__.py b/nni/algorithms/hpo/gridsearch_tuner/__init__.py similarity index 100% rename from src/sdk/pynni/nni/gridsearch_tuner/__init__.py rename to nni/algorithms/hpo/gridsearch_tuner/__init__.py diff --git a/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py b/nni/algorithms/hpo/gridsearch_tuner/gridsearch_tuner.py similarity index 100% rename from src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py rename to nni/algorithms/hpo/gridsearch_tuner/gridsearch_tuner.py diff --git a/src/sdk/pynni/nni/evolution_tuner/__init__.py b/nni/algorithms/hpo/hyperband_advisor/__init__.py similarity index 100% rename from src/sdk/pynni/nni/evolution_tuner/__init__.py rename to nni/algorithms/hpo/hyperband_advisor/__init__.py diff --git a/src/sdk/pynni/nni/hyperband_advisor/hyperband_advisor.py b/nni/algorithms/hpo/hyperband_advisor/hyperband_advisor.py similarity index 99% rename from src/sdk/pynni/nni/hyperband_advisor/hyperband_advisor.py rename to nni/algorithms/hpo/hyperband_advisor/hyperband_advisor.py index 2b13d8f3c3..72e2c28e1a 100644 --- a/src/sdk/pynni/nni/hyperband_advisor/hyperband_advisor.py +++ b/nni/algorithms/hpo/hyperband_advisor/hyperband_advisor.py @@ -15,9 +15,9 @@ from schema import Schema, Optional from nni import ClassArgsValidator -from nni.common import multi_phase_enabled -from nni.msg_dispatcher_base import MsgDispatcherBase -from nni.protocol import CommandType, send +from nni.runtime.common import multi_phase_enabled +from nni.runtime.msg_dispatcher_base import MsgDispatcherBase +from nni.runtime.protocol import CommandType, send from nni.utils import NodeType, OptimizeMode, MetricType, extract_scalar_reward from nni import parameter_expressions diff --git a/src/sdk/pynni/nni/hyperband_advisor/requirements.txt b/nni/algorithms/hpo/hyperband_advisor/requirements.txt similarity index 100% rename from src/sdk/pynni/nni/hyperband_advisor/requirements.txt rename to nni/algorithms/hpo/hyperband_advisor/requirements.txt diff --git a/src/sdk/pynni/nni/gp_tuner/__init__.py b/nni/algorithms/hpo/hyperopt_tuner/__init__.py similarity index 100% rename from src/sdk/pynni/nni/gp_tuner/__init__.py rename to nni/algorithms/hpo/hyperopt_tuner/__init__.py diff --git a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py b/nni/algorithms/hpo/hyperopt_tuner/hyperopt_tuner.py similarity index 100% rename from src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py rename to nni/algorithms/hpo/hyperopt_tuner/hyperopt_tuner.py diff --git a/src/sdk/pynni/nni/medianstop_assessor/__init__.py b/nni/algorithms/hpo/medianstop_assessor/__init__.py similarity index 100% rename from src/sdk/pynni/nni/medianstop_assessor/__init__.py rename to nni/algorithms/hpo/medianstop_assessor/__init__.py diff --git a/src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py b/nni/algorithms/hpo/medianstop_assessor/medianstop_assessor.py similarity index 100% rename from src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py rename to nni/algorithms/hpo/medianstop_assessor/medianstop_assessor.py diff --git a/src/sdk/pynni/nni/medianstop_assessor/test.py b/nni/algorithms/hpo/medianstop_assessor/test.py similarity index 100% rename from src/sdk/pynni/nni/medianstop_assessor/test.py rename to nni/algorithms/hpo/medianstop_assessor/test.py diff --git a/src/sdk/pynni/nni/metis_tuner/Regression_GMM/CreateModel.py b/nni/algorithms/hpo/metis_tuner/Regression_GMM/CreateModel.py similarity index 100% rename from src/sdk/pynni/nni/metis_tuner/Regression_GMM/CreateModel.py rename to nni/algorithms/hpo/metis_tuner/Regression_GMM/CreateModel.py diff --git a/src/sdk/pynni/nni/metis_tuner/Regression_GMM/Selection.py b/nni/algorithms/hpo/metis_tuner/Regression_GMM/Selection.py similarity index 95% rename from src/sdk/pynni/nni/metis_tuner/Regression_GMM/Selection.py rename to nni/algorithms/hpo/metis_tuner/Regression_GMM/Selection.py index 9b0c78860a..0bca96647d 100644 --- a/src/sdk/pynni/nni/metis_tuner/Regression_GMM/Selection.py +++ b/nni/algorithms/hpo/metis_tuner/Regression_GMM/Selection.py @@ -5,8 +5,8 @@ import random import sys -import nni.metis_tuner.lib_acquisition_function as lib_acquisition_function -import nni.metis_tuner.lib_constraint_summation as lib_constraint_summation +from .. import lib_acquisition_function +from .. import lib_constraint_summation sys.path.insert(1, os.path.join(sys.path[0], '..')) diff --git a/src/sdk/pynni/nni/hyperband_advisor/__init__.py b/nni/algorithms/hpo/metis_tuner/Regression_GMM/__init__.py similarity index 100% rename from src/sdk/pynni/nni/hyperband_advisor/__init__.py rename to nni/algorithms/hpo/metis_tuner/Regression_GMM/__init__.py diff --git a/src/sdk/pynni/nni/metis_tuner/Regression_GP/CreateModel.py b/nni/algorithms/hpo/metis_tuner/Regression_GP/CreateModel.py similarity index 100% rename from src/sdk/pynni/nni/metis_tuner/Regression_GP/CreateModel.py rename to nni/algorithms/hpo/metis_tuner/Regression_GP/CreateModel.py diff --git a/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py b/nni/algorithms/hpo/metis_tuner/Regression_GP/OutlierDetection.py similarity index 96% rename from src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py rename to nni/algorithms/hpo/metis_tuner/Regression_GP/OutlierDetection.py index 43234ca3e4..f07a93dd3e 100644 --- a/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py +++ b/nni/algorithms/hpo/metis_tuner/Regression_GP/OutlierDetection.py @@ -9,8 +9,8 @@ import sys from multiprocessing.dummy import Pool as ThreadPool -import nni.metis_tuner.Regression_GP.CreateModel as gp_create_model -import nni.metis_tuner.Regression_GP.Prediction as gp_prediction +from . import CreateModel as gp_create_model +from . import Prediction as gp_prediction sys.path.insert(1, os.path.join(sys.path[0], '..')) diff --git a/src/sdk/pynni/nni/metis_tuner/Regression_GP/Prediction.py b/nni/algorithms/hpo/metis_tuner/Regression_GP/Prediction.py similarity index 100% rename from src/sdk/pynni/nni/metis_tuner/Regression_GP/Prediction.py rename to nni/algorithms/hpo/metis_tuner/Regression_GP/Prediction.py diff --git a/src/sdk/pynni/nni/metis_tuner/Regression_GP/Selection.py b/nni/algorithms/hpo/metis_tuner/Regression_GP/Selection.py similarity index 93% rename from src/sdk/pynni/nni/metis_tuner/Regression_GP/Selection.py rename to nni/algorithms/hpo/metis_tuner/Regression_GP/Selection.py index 0cf19f88f6..68383ed0f2 100644 --- a/src/sdk/pynni/nni/metis_tuner/Regression_GP/Selection.py +++ b/nni/algorithms/hpo/metis_tuner/Regression_GP/Selection.py @@ -5,10 +5,10 @@ import random import sys -import nni.metis_tuner.lib_acquisition_function as lib_acquisition_function -import nni.metis_tuner.lib_constraint_summation as lib_constraint_summation -import nni.metis_tuner.lib_data as lib_data -import nni.metis_tuner.Regression_GP.Prediction as gp_prediction +from .. import lib_acquisition_function +from .. import lib_constraint_summation +from .. import lib_data +from . import Prediction as gp_prediction sys.path.insert(1, os.path.join(sys.path[0], '..')) diff --git a/src/sdk/pynni/nni/hyperopt_tuner/__init__.py b/nni/algorithms/hpo/metis_tuner/Regression_GP/__init__.py similarity index 100% rename from src/sdk/pynni/nni/hyperopt_tuner/__init__.py rename to nni/algorithms/hpo/metis_tuner/Regression_GP/__init__.py diff --git a/src/sdk/pynni/nni/metis_tuner/Regression_GMM/__init__.py b/nni/algorithms/hpo/metis_tuner/__init__.py similarity index 100% rename from src/sdk/pynni/nni/metis_tuner/Regression_GMM/__init__.py rename to nni/algorithms/hpo/metis_tuner/__init__.py diff --git a/src/sdk/pynni/nni/metis_tuner/lib_acquisition_function.py b/nni/algorithms/hpo/metis_tuner/lib_acquisition_function.py similarity index 99% rename from src/sdk/pynni/nni/metis_tuner/lib_acquisition_function.py rename to nni/algorithms/hpo/metis_tuner/lib_acquisition_function.py index a59de4891b..f1b1edfe01 100644 --- a/src/sdk/pynni/nni/metis_tuner/lib_acquisition_function.py +++ b/nni/algorithms/hpo/metis_tuner/lib_acquisition_function.py @@ -11,7 +11,7 @@ from scipy.stats import norm from scipy.optimize import minimize -import nni.metis_tuner.lib_data as lib_data +from . import lib_data def next_hyperparameter_expected_improvement(fun_prediction, diff --git a/src/sdk/pynni/nni/metis_tuner/lib_constraint_summation.py b/nni/algorithms/hpo/metis_tuner/lib_constraint_summation.py similarity index 100% rename from src/sdk/pynni/nni/metis_tuner/lib_constraint_summation.py rename to nni/algorithms/hpo/metis_tuner/lib_constraint_summation.py diff --git a/src/sdk/pynni/nni/metis_tuner/lib_data.py b/nni/algorithms/hpo/metis_tuner/lib_data.py similarity index 100% rename from src/sdk/pynni/nni/metis_tuner/lib_data.py rename to nni/algorithms/hpo/metis_tuner/lib_data.py diff --git a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py b/nni/algorithms/hpo/metis_tuner/metis_tuner.py similarity index 97% rename from src/sdk/pynni/nni/metis_tuner/metis_tuner.py rename to nni/algorithms/hpo/metis_tuner/metis_tuner.py index cdd50dbcf3..1a0670f5e5 100644 --- a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py +++ b/nni/algorithms/hpo/metis_tuner/metis_tuner.py @@ -15,14 +15,14 @@ from schema import Schema, Optional from nni import ClassArgsValidator -import nni.metis_tuner.lib_constraint_summation as lib_constraint_summation -import nni.metis_tuner.lib_data as lib_data -import nni.metis_tuner.Regression_GMM.CreateModel as gmm_create_model -import nni.metis_tuner.Regression_GMM.Selection as gmm_selection -import nni.metis_tuner.Regression_GP.CreateModel as gp_create_model -import nni.metis_tuner.Regression_GP.OutlierDetection as gp_outlier_detection -import nni.metis_tuner.Regression_GP.Prediction as gp_prediction -import nni.metis_tuner.Regression_GP.Selection as gp_selection +from . import lib_constraint_summation +from . import lib_data +from .Regression_GMM import CreateModel as gmm_create_model +from .Regression_GMM import Selection as gmm_selection +from .Regression_GP import CreateModel as gp_create_model +from .Regression_GP import OutlierDetection as gp_outlier_detection +from .Regression_GP import Prediction as gp_prediction +from .Regression_GP import Selection as gp_selection from nni.tuner import Tuner from nni.utils import OptimizeMode, extract_scalar_reward diff --git a/src/sdk/pynni/nni/metis_tuner/requirments.txt b/nni/algorithms/hpo/metis_tuner/requirments.txt similarity index 100% rename from src/sdk/pynni/nni/metis_tuner/requirments.txt rename to nni/algorithms/hpo/metis_tuner/requirments.txt diff --git a/src/sdk/pynni/nni/metis_tuner/Regression_GP/__init__.py b/nni/algorithms/hpo/networkmorphism_tuner/__init__.py similarity index 100% rename from src/sdk/pynni/nni/metis_tuner/Regression_GP/__init__.py rename to nni/algorithms/hpo/networkmorphism_tuner/__init__.py diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/bayesian.py b/nni/algorithms/hpo/networkmorphism_tuner/bayesian.py similarity index 98% rename from src/sdk/pynni/nni/networkmorphism_tuner/bayesian.py rename to nni/algorithms/hpo/networkmorphism_tuner/bayesian.py index 8d838337fd..54c1996dc7 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/bayesian.py +++ b/nni/algorithms/hpo/networkmorphism_tuner/bayesian.py @@ -13,9 +13,9 @@ from sklearn.metrics.pairwise import rbf_kernel from nni.utils import OptimizeMode -from nni.networkmorphism_tuner.graph_transformer import transform -from nni.networkmorphism_tuner.utils import Constant -from nni.networkmorphism_tuner.layers import is_layer +from .graph_transformer import transform +from .utils import Constant +from .layers import is_layer def layer_distance(a, b): diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/graph.py b/nni/algorithms/hpo/networkmorphism_tuner/graph.py similarity index 99% rename from src/sdk/pynni/nni/networkmorphism_tuner/graph.py rename to nni/algorithms/hpo/networkmorphism_tuner/graph.py index 30adc4aab5..9c96b6c2f0 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/graph.py +++ b/nni/algorithms/hpo/networkmorphism_tuner/graph.py @@ -9,7 +9,7 @@ import numpy as np import torch -from nni.networkmorphism_tuner.layer_transformer import ( +from .layer_transformer import ( add_noise, wider_bn, wider_next_conv, @@ -20,7 +20,7 @@ init_conv_weight, init_bn_weight, ) -from nni.networkmorphism_tuner.layers import ( +from .layers import ( StubAdd, StubConcatenate, StubReLU, @@ -36,7 +36,7 @@ layer_description_extractor, layer_description_builder, ) -from nni.networkmorphism_tuner.utils import Constant +from .utils import Constant class NetworkDescriptor: diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/graph_transformer.py b/nni/algorithms/hpo/networkmorphism_tuner/graph_transformer.py similarity index 96% rename from src/sdk/pynni/nni/networkmorphism_tuner/graph_transformer.py rename to nni/algorithms/hpo/networkmorphism_tuner/graph_transformer.py index d29668e94d..b03112d63b 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/graph_transformer.py +++ b/nni/algorithms/hpo/networkmorphism_tuner/graph_transformer.py @@ -5,8 +5,8 @@ from random import randrange, sample -from nni.networkmorphism_tuner.graph import NetworkDescriptor -from nni.networkmorphism_tuner.layers import ( +from .graph import NetworkDescriptor +from .layers import ( StubDense, StubReLU, get_batch_norm_class, @@ -15,7 +15,7 @@ get_pooling_class, is_layer, ) -from nni.networkmorphism_tuner.utils import Constant +from .utils import Constant def to_wider_graph(graph): diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/layer_transformer.py b/nni/algorithms/hpo/networkmorphism_tuner/layer_transformer.py similarity index 99% rename from src/sdk/pynni/nni/networkmorphism_tuner/layer_transformer.py rename to nni/algorithms/hpo/networkmorphism_tuner/layer_transformer.py index c3642dadc7..6ffd1b20fb 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/layer_transformer.py +++ b/nni/algorithms/hpo/networkmorphism_tuner/layer_transformer.py @@ -3,7 +3,7 @@ import numpy as np -from nni.networkmorphism_tuner.layers import ( +from .layers import ( StubDense, StubReLU, get_batch_norm_class, diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/layers.py b/nni/algorithms/hpo/networkmorphism_tuner/layers.py similarity index 99% rename from src/sdk/pynni/nni/networkmorphism_tuner/layers.py rename to nni/algorithms/hpo/networkmorphism_tuner/layers.py index 365561299c..a96c87b780 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/layers.py +++ b/nni/algorithms/hpo/networkmorphism_tuner/layers.py @@ -7,7 +7,7 @@ import torch from torch import nn from torch.nn import functional -from nni.networkmorphism_tuner.utils import Constant +from .utils import Constant class AvgPool(nn.Module): diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py b/nni/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py similarity index 97% rename from src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py rename to nni/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py index 50706756dd..6a73cad3c6 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py +++ b/nni/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py @@ -10,10 +10,10 @@ from schema import Optional, Schema from nni.tuner import Tuner from nni.utils import OptimizeMode, extract_scalar_reward -from nni.networkmorphism_tuner.bayesian import BayesianOptimizer -from nni.networkmorphism_tuner.nn import CnnGenerator, MlpGenerator -from nni.networkmorphism_tuner.utils import Constant -from nni.networkmorphism_tuner.graph import graph_to_json, json_to_graph +from .bayesian import BayesianOptimizer +from .nn import CnnGenerator, MlpGenerator +from .utils import Constant +from .graph import graph_to_json, json_to_graph from nni import ClassArgsValidator logger = logging.getLogger("NetworkMorphism_AutoML") diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/nn.py b/nni/algorithms/hpo/networkmorphism_tuner/nn.py similarity index 97% rename from src/sdk/pynni/nni/networkmorphism_tuner/nn.py rename to nni/algorithms/hpo/networkmorphism_tuner/nn.py index 14061682cc..9e0072f9b3 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/nn.py +++ b/nni/algorithms/hpo/networkmorphism_tuner/nn.py @@ -3,14 +3,14 @@ from abc import abstractmethod -from nni.networkmorphism_tuner.graph import Graph -from nni.networkmorphism_tuner.layers import (StubDense, StubDropout1d, +from .graph import Graph +from .layers import (StubDense, StubDropout1d, StubReLU, get_batch_norm_class, get_conv_class, get_dropout_class, get_global_avg_pooling_class, get_pooling_class) -from nni.networkmorphism_tuner.utils import Constant +from .utils import Constant class NetworkGenerator: diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/utils.py b/nni/algorithms/hpo/networkmorphism_tuner/utils.py similarity index 100% rename from src/sdk/pynni/nni/networkmorphism_tuner/utils.py rename to nni/algorithms/hpo/networkmorphism_tuner/utils.py diff --git a/src/sdk/pynni/nni/metis_tuner/__init__.py b/nni/algorithms/hpo/pbt_tuner/__init__.py similarity index 100% rename from src/sdk/pynni/nni/metis_tuner/__init__.py rename to nni/algorithms/hpo/pbt_tuner/__init__.py diff --git a/src/sdk/pynni/nni/pbt_tuner/pbt_tuner.py b/nni/algorithms/hpo/pbt_tuner/pbt_tuner.py similarity index 100% rename from src/sdk/pynni/nni/pbt_tuner/pbt_tuner.py rename to nni/algorithms/hpo/pbt_tuner/pbt_tuner.py diff --git a/src/sdk/pynni/nni/ppo_tuner/__init__.py b/nni/algorithms/hpo/ppo_tuner/__init__.py similarity index 100% rename from src/sdk/pynni/nni/ppo_tuner/__init__.py rename to nni/algorithms/hpo/ppo_tuner/__init__.py diff --git a/src/sdk/pynni/nni/ppo_tuner/distri.py b/nni/algorithms/hpo/ppo_tuner/distri.py similarity index 100% rename from src/sdk/pynni/nni/ppo_tuner/distri.py rename to nni/algorithms/hpo/ppo_tuner/distri.py diff --git a/src/sdk/pynni/nni/ppo_tuner/model.py b/nni/algorithms/hpo/ppo_tuner/model.py similarity index 100% rename from src/sdk/pynni/nni/ppo_tuner/model.py rename to nni/algorithms/hpo/ppo_tuner/model.py diff --git a/src/sdk/pynni/nni/ppo_tuner/policy.py b/nni/algorithms/hpo/ppo_tuner/policy.py similarity index 100% rename from src/sdk/pynni/nni/ppo_tuner/policy.py rename to nni/algorithms/hpo/ppo_tuner/policy.py diff --git a/src/sdk/pynni/nni/ppo_tuner/ppo_tuner.py b/nni/algorithms/hpo/ppo_tuner/ppo_tuner.py similarity index 100% rename from src/sdk/pynni/nni/ppo_tuner/ppo_tuner.py rename to nni/algorithms/hpo/ppo_tuner/ppo_tuner.py diff --git a/src/sdk/pynni/nni/ppo_tuner/requirements.txt b/nni/algorithms/hpo/ppo_tuner/requirements.txt similarity index 100% rename from src/sdk/pynni/nni/ppo_tuner/requirements.txt rename to nni/algorithms/hpo/ppo_tuner/requirements.txt diff --git a/src/sdk/pynni/nni/ppo_tuner/util.py b/nni/algorithms/hpo/ppo_tuner/util.py similarity index 100% rename from src/sdk/pynni/nni/ppo_tuner/util.py rename to nni/algorithms/hpo/ppo_tuner/util.py diff --git a/src/sdk/pynni/nni/regularized_evolution_tuner/__init__.py b/nni/algorithms/hpo/regularized_evolution_tuner/__init__.py similarity index 100% rename from src/sdk/pynni/nni/regularized_evolution_tuner/__init__.py rename to nni/algorithms/hpo/regularized_evolution_tuner/__init__.py diff --git a/src/sdk/pynni/nni/regularized_evolution_tuner/regularized_evolution_tuner.py b/nni/algorithms/hpo/regularized_evolution_tuner/regularized_evolution_tuner.py similarity index 100% rename from src/sdk/pynni/nni/regularized_evolution_tuner/regularized_evolution_tuner.py rename to nni/algorithms/hpo/regularized_evolution_tuner/regularized_evolution_tuner.py index a466df195b..ac756c3388 100644 --- a/src/sdk/pynni/nni/regularized_evolution_tuner/regularized_evolution_tuner.py +++ b/nni/algorithms/hpo/regularized_evolution_tuner/regularized_evolution_tuner.py @@ -3,8 +3,8 @@ import random from collections import deque -import nni from schema import Schema, Optional +import nni from nni.tuner import Tuner from nni import ClassArgsValidator from nni.utils import OptimizeMode, extract_scalar_reward diff --git a/src/sdk/pynni/nni/smac_tuner/__init__.py b/nni/algorithms/hpo/smac_tuner/__init__.py similarity index 100% rename from src/sdk/pynni/nni/smac_tuner/__init__.py rename to nni/algorithms/hpo/smac_tuner/__init__.py diff --git a/src/sdk/pynni/nni/smac_tuner/convert_ss_to_scenario.py b/nni/algorithms/hpo/smac_tuner/convert_ss_to_scenario.py similarity index 100% rename from src/sdk/pynni/nni/smac_tuner/convert_ss_to_scenario.py rename to nni/algorithms/hpo/smac_tuner/convert_ss_to_scenario.py diff --git a/src/sdk/pynni/nni/smac_tuner/requirements.txt b/nni/algorithms/hpo/smac_tuner/requirements.txt similarity index 100% rename from src/sdk/pynni/nni/smac_tuner/requirements.txt rename to nni/algorithms/hpo/smac_tuner/requirements.txt diff --git a/src/sdk/pynni/nni/smac_tuner/smac_tuner.py b/nni/algorithms/hpo/smac_tuner/smac_tuner.py similarity index 100% rename from src/sdk/pynni/nni/smac_tuner/smac_tuner.py rename to nni/algorithms/hpo/smac_tuner/smac_tuner.py diff --git a/src/sdk/pynni/nni/nas/pytorch/cdarts/__init__.py b/nni/algorithms/nas/pytorch/cdarts/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/cdarts/__init__.py rename to nni/algorithms/nas/pytorch/cdarts/__init__.py diff --git a/src/sdk/pynni/nni/nas/pytorch/cdarts/mutator.py b/nni/algorithms/nas/pytorch/cdarts/mutator.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/cdarts/mutator.py rename to nni/algorithms/nas/pytorch/cdarts/mutator.py diff --git a/src/sdk/pynni/nni/nas/pytorch/cdarts/trainer.py b/nni/algorithms/nas/pytorch/cdarts/trainer.py similarity index 98% rename from src/sdk/pynni/nni/nas/pytorch/cdarts/trainer.py rename to nni/algorithms/nas/pytorch/cdarts/trainer.py index fa6d09e3e7..1a5174216f 100644 --- a/src/sdk/pynni/nni/nas/pytorch/cdarts/trainer.py +++ b/nni/algorithms/nas/pytorch/cdarts/trainer.py @@ -10,7 +10,7 @@ import torch.nn.functional as F import apex # pylint: disable=import-error from apex.parallel import DistributedDataParallel # pylint: disable=import-error -from nni.nas.pytorch.cdarts import RegularizedDartsMutator, RegularizedMutatorParallel, DartsDiscreteMutator # pylint: disable=wrong-import-order +from .mutator import RegularizedDartsMutator, RegularizedMutatorParallel, DartsDiscreteMutator # pylint: disable=wrong-import-order from nni.nas.pytorch.utils import AverageMeterGroup # pylint: disable=wrong-import-order from .utils import CyclicIterator, TorchTensorEncoder, accuracy, reduce_metrics diff --git a/src/sdk/pynni/nni/nas/pytorch/cdarts/utils.py b/nni/algorithms/nas/pytorch/cdarts/utils.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/cdarts/utils.py rename to nni/algorithms/nas/pytorch/cdarts/utils.py diff --git a/src/sdk/pynni/nni/nas/pytorch/classic_nas/__init__.py b/nni/algorithms/nas/pytorch/classic_nas/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/classic_nas/__init__.py rename to nni/algorithms/nas/pytorch/classic_nas/__init__.py diff --git a/src/sdk/pynni/nni/nas/pytorch/classic_nas/mutator.py b/nni/algorithms/nas/pytorch/classic_nas/mutator.py similarity index 99% rename from src/sdk/pynni/nni/nas/pytorch/classic_nas/mutator.py rename to nni/algorithms/nas/pytorch/classic_nas/mutator.py index e018e758d1..7254a8b0b4 100644 --- a/src/sdk/pynni/nni/nas/pytorch/classic_nas/mutator.py +++ b/nni/algorithms/nas/pytorch/classic_nas/mutator.py @@ -9,7 +9,7 @@ import torch import nni -from nni.env_vars import trial_env_vars +from nni.runtime.env_vars import trial_env_vars from nni.nas.pytorch.mutables import LayerChoice, InputChoice, MutableScope from nni.nas.pytorch.mutator import Mutator diff --git a/src/sdk/pynni/nni/nas/pytorch/darts/__init__.py b/nni/algorithms/nas/pytorch/darts/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/darts/__init__.py rename to nni/algorithms/nas/pytorch/darts/__init__.py diff --git a/src/sdk/pynni/nni/nas/pytorch/darts/mutator.py b/nni/algorithms/nas/pytorch/darts/mutator.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/darts/mutator.py rename to nni/algorithms/nas/pytorch/darts/mutator.py diff --git a/src/sdk/pynni/nni/nas/pytorch/darts/trainer.py b/nni/algorithms/nas/pytorch/darts/trainer.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/darts/trainer.py rename to nni/algorithms/nas/pytorch/darts/trainer.py diff --git a/src/sdk/pynni/nni/nas/pytorch/enas/__init__.py b/nni/algorithms/nas/pytorch/enas/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/enas/__init__.py rename to nni/algorithms/nas/pytorch/enas/__init__.py diff --git a/src/sdk/pynni/nni/nas/pytorch/enas/mutator.py b/nni/algorithms/nas/pytorch/enas/mutator.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/enas/mutator.py rename to nni/algorithms/nas/pytorch/enas/mutator.py diff --git a/src/sdk/pynni/nni/nas/pytorch/enas/trainer.py b/nni/algorithms/nas/pytorch/enas/trainer.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/enas/trainer.py rename to nni/algorithms/nas/pytorch/enas/trainer.py diff --git a/src/sdk/pynni/nni/nas/pytorch/pdarts/__init__.py b/nni/algorithms/nas/pytorch/pdarts/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/pdarts/__init__.py rename to nni/algorithms/nas/pytorch/pdarts/__init__.py diff --git a/src/sdk/pynni/nni/nas/pytorch/pdarts/mutator.py b/nni/algorithms/nas/pytorch/pdarts/mutator.py similarity index 98% rename from src/sdk/pynni/nni/nas/pytorch/pdarts/mutator.py rename to nni/algorithms/nas/pytorch/pdarts/mutator.py index 5bafba4042..09ad51c5e4 100644 --- a/src/sdk/pynni/nni/nas/pytorch/pdarts/mutator.py +++ b/nni/algorithms/nas/pytorch/pdarts/mutator.py @@ -7,7 +7,7 @@ import torch from torch import nn -from nni.nas.pytorch.darts import DartsMutator +from nni.algorithms.nas.pytorch.darts import DartsMutator from nni.nas.pytorch.mutables import LayerChoice diff --git a/src/sdk/pynni/nni/nas/pytorch/pdarts/trainer.py b/nni/algorithms/nas/pytorch/pdarts/trainer.py similarity index 98% rename from src/sdk/pynni/nni/nas/pytorch/pdarts/trainer.py rename to nni/algorithms/nas/pytorch/pdarts/trainer.py index 8d3ad64dda..7f23a6e222 100644 --- a/src/sdk/pynni/nni/nas/pytorch/pdarts/trainer.py +++ b/nni/algorithms/nas/pytorch/pdarts/trainer.py @@ -5,7 +5,7 @@ import logging from nni.nas.pytorch.callbacks import LRSchedulerCallback -from nni.nas.pytorch.darts import DartsTrainer +from nni.algorithms.nas.pytorch.darts import DartsTrainer from nni.nas.pytorch.trainer import BaseTrainer, TorchTensorEncoder from .mutator import PdartsMutator diff --git a/src/sdk/pynni/nni/nas/pytorch/proxylessnas/__init__.py b/nni/algorithms/nas/pytorch/proxylessnas/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/proxylessnas/__init__.py rename to nni/algorithms/nas/pytorch/proxylessnas/__init__.py diff --git a/src/sdk/pynni/nni/nas/pytorch/proxylessnas/mutator.py b/nni/algorithms/nas/pytorch/proxylessnas/mutator.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/proxylessnas/mutator.py rename to nni/algorithms/nas/pytorch/proxylessnas/mutator.py diff --git a/src/sdk/pynni/nni/nas/pytorch/proxylessnas/trainer.py b/nni/algorithms/nas/pytorch/proxylessnas/trainer.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/proxylessnas/trainer.py rename to nni/algorithms/nas/pytorch/proxylessnas/trainer.py diff --git a/src/sdk/pynni/nni/nas/pytorch/proxylessnas/utils.py b/nni/algorithms/nas/pytorch/proxylessnas/utils.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/proxylessnas/utils.py rename to nni/algorithms/nas/pytorch/proxylessnas/utils.py diff --git a/src/sdk/pynni/nni/nas/pytorch/random/__init__.py b/nni/algorithms/nas/pytorch/random/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/random/__init__.py rename to nni/algorithms/nas/pytorch/random/__init__.py diff --git a/src/sdk/pynni/nni/nas/pytorch/random/mutator.py b/nni/algorithms/nas/pytorch/random/mutator.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/random/mutator.py rename to nni/algorithms/nas/pytorch/random/mutator.py diff --git a/src/sdk/pynni/nni/nas/pytorch/spos/__init__.py b/nni/algorithms/nas/pytorch/spos/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/spos/__init__.py rename to nni/algorithms/nas/pytorch/spos/__init__.py diff --git a/src/sdk/pynni/nni/nas/pytorch/spos/evolution.py b/nni/algorithms/nas/pytorch/spos/evolution.py similarity index 99% rename from src/sdk/pynni/nni/nas/pytorch/spos/evolution.py rename to nni/algorithms/nas/pytorch/spos/evolution.py index 9042d30474..bd099e276e 100644 --- a/src/sdk/pynni/nni/nas/pytorch/spos/evolution.py +++ b/nni/algorithms/nas/pytorch/spos/evolution.py @@ -9,7 +9,7 @@ import numpy as np from nni.tuner import Tuner -from nni.nas.pytorch.classic_nas.mutator import LAYER_CHOICE, INPUT_CHOICE +from nni.algorithms.nas.pytorch.classic_nas.mutator import LAYER_CHOICE, INPUT_CHOICE _logger = logging.getLogger(__name__) diff --git a/src/sdk/pynni/nni/nas/pytorch/spos/mutator.py b/nni/algorithms/nas/pytorch/spos/mutator.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/spos/mutator.py rename to nni/algorithms/nas/pytorch/spos/mutator.py diff --git a/src/sdk/pynni/nni/nas/pytorch/spos/trainer.py b/nni/algorithms/nas/pytorch/spos/trainer.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/spos/trainer.py rename to nni/algorithms/nas/pytorch/spos/trainer.py diff --git a/src/sdk/pynni/nni/nas/tensorflow/classic_nas/__init__.py b/nni/algorithms/nas/tensorflow/classic_nas/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/tensorflow/classic_nas/__init__.py rename to nni/algorithms/nas/tensorflow/classic_nas/__init__.py diff --git a/src/sdk/pynni/nni/nas/tensorflow/classic_nas/mutator.py b/nni/algorithms/nas/tensorflow/classic_nas/mutator.py similarity index 99% rename from src/sdk/pynni/nni/nas/tensorflow/classic_nas/mutator.py rename to nni/algorithms/nas/tensorflow/classic_nas/mutator.py index 4b9212f342..fad4987fed 100644 --- a/src/sdk/pynni/nni/nas/tensorflow/classic_nas/mutator.py +++ b/nni/algorithms/nas/tensorflow/classic_nas/mutator.py @@ -9,7 +9,7 @@ import tensorflow as tf import nni -from nni.env_vars import trial_env_vars +from nni.runtime.env_vars import trial_env_vars from nni.nas.tensorflow.mutables import LayerChoice, InputChoice, MutableScope from nni.nas.tensorflow.mutator import Mutator diff --git a/src/sdk/pynni/nni/nas/tensorflow/enas/__init__.py b/nni/algorithms/nas/tensorflow/enas/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/tensorflow/enas/__init__.py rename to nni/algorithms/nas/tensorflow/enas/__init__.py diff --git a/src/sdk/pynni/nni/nas/tensorflow/enas/mutator.py b/nni/algorithms/nas/tensorflow/enas/mutator.py similarity index 100% rename from src/sdk/pynni/nni/nas/tensorflow/enas/mutator.py rename to nni/algorithms/nas/tensorflow/enas/mutator.py diff --git a/nni/algorithms/nas/tensorflow/enas/trainer.py b/nni/algorithms/nas/tensorflow/enas/trainer.py new file mode 100644 index 0000000000..c20f20ab1a --- /dev/null +++ b/nni/algorithms/nas/tensorflow/enas/trainer.py @@ -0,0 +1,203 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import tensorflow as tf +from tensorflow.keras.optimizers import Adam + +from nni.nas.tensorflow.utils import AverageMeterGroup, fill_zero_grads + +from .mutator import EnasMutator + +logger = logging.getLogger(__name__) + + +class EnasTrainer: + def __init__( + self, + model, + loss, + metrics, + reward_function, + optimizer, + batch_size, + num_epochs, + dataset_train, + dataset_valid, + log_frequency=100, + entropy_weight=0.0001, + skip_weight=0.8, + baseline_decay=0.999, + child_steps=500, + mutator_lr=0.00035, + mutator_steps=50, + mutator_steps_aggregate=20, + aux_weight=0.4, + test_arc_per_epoch=1, + ): + self.model = model + self.loss = loss + self.metrics = metrics + self.reward_function = reward_function + self.optimizer = optimizer + self.batch_size = batch_size + self.num_epochs = num_epochs + + x, y = dataset_train + split = int(len(x) * 0.9) + self.train_set = tf.data.Dataset.from_tensor_slices((x[:split], y[:split])) + self.valid_set = tf.data.Dataset.from_tensor_slices((x[split:], y[split:])) + self.test_set = tf.data.Dataset.from_tensor_slices(dataset_valid) + + self.log_frequency = log_frequency + self.entropy_weight = entropy_weight + self.skip_weight = skip_weight + self.baseline_decay = baseline_decay + self.child_steps = child_steps + self.mutator_lr = mutator_lr + self.mutator_steps = mutator_steps + self.mutator_steps_aggregate = mutator_steps_aggregate + self.aux_weight = aux_weight + self.test_arc_per_epoch = test_arc_per_epoch + + self.mutator = EnasMutator(model) + self.mutator_optim = Adam(learning_rate=self.mutator_lr) + + self.baseline = 0.0 + + def train(self, validate=True): + for epoch in range(self.num_epochs): + logger.info("Epoch %d Training", epoch + 1) + self.train_one_epoch(epoch) + logger.info("Epoch %d Validating", epoch + 1) + self.validate_one_epoch(epoch) + + def validate(self): + self.validate_one_epoch(-1) + + def train_one_epoch(self, epoch): + train_loader, valid_loader = self._create_train_loader() + + # Sample model and train + meters = AverageMeterGroup() + + for step in range(1, self.child_steps + 1): + x, y = next(train_loader) + self.mutator.reset() + + with tf.GradientTape() as tape: + logits = self.model(x, training=True) + if isinstance(logits, tuple): + logits, aux_logits = logits + aux_loss = self.loss(aux_logits, y) + else: + aux_loss = 0.0 + metrics = self.metrics(y, logits) + loss = self.loss(y, logits) + self.aux_weight * aux_loss + + grads = tape.gradient(loss, self.model.trainable_weights) + grads = fill_zero_grads(grads, self.model.trainable_weights) + grads, _ = tf.clip_by_global_norm(grads, 5.0) + self.optimizer.apply_gradients(zip(grads, self.model.trainable_weights)) + + metrics["loss"] = tf.reduce_mean(loss).numpy() + meters.update(metrics) + + if self.log_frequency and step % self.log_frequency == 0: + logger.info( + "Model Epoch [%d/%d] Step [%d/%d] %s", + epoch + 1, + self.num_epochs, + step, + self.child_steps, + meters, + ) + + # Train sampler (mutator) + meters = AverageMeterGroup() + for mutator_step in range(1, self.mutator_steps + 1): + grads_list = [] + for step in range(1, self.mutator_steps_aggregate + 1): + with tf.GradientTape() as tape: + x, y = next(valid_loader) + self.mutator.reset() + + logits = self.model(x, training=False) + metrics = self.metrics(y, logits) + reward = ( + self.reward_function(y, logits) + + self.entropy_weight * self.mutator.sample_entropy + ) + self.baseline = self.baseline * self.baseline_decay + reward * ( + 1 - self.baseline_decay + ) + loss = self.mutator.sample_log_prob * (reward - self.baseline) + loss += self.skip_weight * self.mutator.sample_skip_penalty + + meters.update( + { + "reward": reward, + "loss": tf.reduce_mean(loss).numpy(), + "ent": self.mutator.sample_entropy.numpy(), + "log_prob": self.mutator.sample_log_prob.numpy(), + "baseline": self.baseline, + "skip": self.mutator.sample_skip_penalty, + } + ) + + cur_step = step + (mutator_step - 1) * self.mutator_steps_aggregate + if self.log_frequency and cur_step % self.log_frequency == 0: + logger.info( + "RL Epoch [%d/%d] Step [%d/%d] [%d/%d] %s", + epoch + 1, + self.num_epochs, + mutator_step, + self.mutator_steps, + step, + self.mutator_steps_aggregate, + meters, + ) + + grads = tape.gradient(loss, self.mutator.trainable_weights) + grads = fill_zero_grads(grads, self.mutator.trainable_weights) + grads_list.append(grads) + total_grads = [ + tf.math.add_n(weight_grads) for weight_grads in zip(*grads_list) + ] + total_grads, _ = tf.clip_by_global_norm(total_grads, 5.0) + self.mutator_optim.apply_gradients( + zip(total_grads, self.mutator.trainable_weights) + ) + + def validate_one_epoch(self, epoch): + test_loader = self._create_validate_loader() + + for arc_id in range(self.test_arc_per_epoch): + meters = AverageMeterGroup() + for x, y in test_loader: + self.mutator.reset() + logits = self.model(x, training=False) + if isinstance(logits, tuple): + logits, _ = logits + metrics = self.metrics(y, logits) + loss = self.loss(y, logits) + metrics["loss"] = tf.reduce_mean(loss).numpy() + meters.update(metrics) + + logger.info( + "Test Epoch [%d/%d] Arc [%d/%d] Summary %s", + epoch + 1, + self.num_epochs, + arc_id + 1, + self.test_arc_per_epoch, + meters.summary(), + ) + + def _create_train_loader(self): + train_set = self.train_set.shuffle(1000000).repeat().batch(self.batch_size) + test_set = self.valid_set.shuffle(1000000).repeat().batch(self.batch_size) + return iter(train_set), iter(test_set) + + def _create_validate_loader(self): + return iter(self.test_set.shuffle(1000000).batch(self.batch_size)) diff --git a/src/sdk/pynni/nni/assessor.py b/nni/assessor.py similarity index 96% rename from src/sdk/pynni/nni/assessor.py rename to nni/assessor.py index 9094de21e7..b70995cbad 100644 --- a/src/sdk/pynni/nni/assessor.py +++ b/nni/assessor.py @@ -50,8 +50,8 @@ class Assessor(Recoverable): See Also -------- Builtin assessors: - :class:`~nni.medianstop_assessor.MedianstopAssessor` - :class:`~nni.curvefitting_assessor.CurvefittingAssessor` + :class:`~nni.algorithms.hpo.medianstop_assessor.MedianstopAssessor` + :class:`~nni.algorithms.hpo.curvefitting_assessor.CurvefittingAssessor` """ def assess_trial(self, trial_job_id, trial_history): diff --git a/src/sdk/pynni/nni/_graph_utils.py b/nni/common/graph_utils.py similarity index 100% rename from src/sdk/pynni/nni/_graph_utils.py rename to nni/common/graph_utils.py diff --git a/src/sdk/pynni/nni/nas_utils.py b/nni/common/nas_utils.py similarity index 99% rename from src/sdk/pynni/nni/nas_utils.py rename to nni/common/nas_utils.py index 147faf1ec1..d7f050ec9b 100644 --- a/src/sdk/pynni/nni/nas_utils.py +++ b/nni/common/nas_utils.py @@ -4,7 +4,7 @@ import functools import logging -from . import trial +from .. import trial _logger = logging.getLogger(__name__) diff --git a/src/sdk/pynni/nni/nas/__init__.py b/nni/compression/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/__init__.py rename to nni/compression/__init__.py diff --git a/src/sdk/pynni/nni/compression/torch/__init__.py b/nni/compression/pytorch/__init__.py similarity index 76% rename from src/sdk/pynni/nni/compression/torch/__init__.py rename to nni/compression/pytorch/__init__.py index 4b55c0f5de..e20f284fef 100644 --- a/src/sdk/pynni/nni/compression/torch/__init__.py +++ b/nni/compression/pytorch/__init__.py @@ -2,6 +2,4 @@ # Licensed under the MIT license. from .speedup import ModelSpeedup -from .pruning import * -from .quantization import * from .compressor import Compressor, Pruner, Quantizer diff --git a/src/sdk/pynni/nni/compression/torch/compressor.py b/nni/compression/pytorch/compressor.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/compressor.py rename to nni/compression/pytorch/compressor.py diff --git a/src/sdk/pynni/nni/compression/torch/default_layers.py b/nni/compression/pytorch/default_layers.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/default_layers.py rename to nni/compression/pytorch/default_layers.py diff --git a/src/sdk/pynni/nni/compression/torch/speedup/__init__.py b/nni/compression/pytorch/speedup/__init__.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/speedup/__init__.py rename to nni/compression/pytorch/speedup/__init__.py diff --git a/src/sdk/pynni/nni/compression/torch/speedup/compress_modules.py b/nni/compression/pytorch/speedup/compress_modules.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/speedup/compress_modules.py rename to nni/compression/pytorch/speedup/compress_modules.py diff --git a/src/sdk/pynni/nni/compression/torch/speedup/compressor.py b/nni/compression/pytorch/speedup/compressor.py similarity index 97% rename from src/sdk/pynni/nni/compression/torch/speedup/compressor.py rename to nni/compression/pytorch/speedup/compressor.py index a9ae7aef54..0f308bbf18 100644 --- a/src/sdk/pynni/nni/compression/torch/speedup/compressor.py +++ b/nni/compression/pytorch/speedup/compressor.py @@ -3,8 +3,8 @@ import logging import torch -from nni.compression.torch.utils.mask_conflict import fix_mask_conflict -from nni.compression.torch.utils.utils import get_module_by_name +from nni.compression.pytorch.utils.mask_conflict import fix_mask_conflict +from nni.compression.pytorch.utils.utils import get_module_by_name from .compress_modules import replace_module from .infer_shape import ModuleMasks, infer_from_mask, infer_from_inshape, infer_from_outshape, set_conv_prune_dim @@ -29,7 +29,7 @@ def __init__(self, model, dummy_input, masks_file, map_location=None): map_location : str the device on which masks are placed, same to map_location in ```torch.load``` """ - from nni._graph_utils import build_module_graph + from nni.common.graph_utils import build_module_graph self.bound_model = model self.masks = torch.load(masks_file, map_location) diff --git a/src/sdk/pynni/nni/compression/torch/speedup/infer_shape.py b/nni/compression/pytorch/speedup/infer_shape.py similarity index 99% rename from src/sdk/pynni/nni/compression/torch/speedup/infer_shape.py rename to nni/compression/pytorch/speedup/infer_shape.py index 5d636c8784..252ede28af 100644 --- a/src/sdk/pynni/nni/compression/torch/speedup/infer_shape.py +++ b/nni/compression/pytorch/speedup/infer_shape.py @@ -573,7 +573,7 @@ def view_inshape(module_masks, mask, shape): step_size = shape['in_shape'][2] * shape['in_shape'][3] for loc in mask.mask_index[1]: index.extend([loc * step_size + i for i in range(step_size)]) - output_cmask.add_index_mask(dim=1, index=torch.tensor(index)) # pylint: disable=not-callable + output_cmask.add_index_mask(dim=1, index=torch.tensor(index).to(mask.mask_index[1].device)) # pylint: disable=not-callable module_masks.set_output_mask(output_cmask) return output_cmask @@ -609,7 +609,7 @@ def view_outshape(module_masks, mask, shape): step_size = shape['in_shape'][2] * shape['in_shape'][3] for loc in mask.mask_index[1]: index.extend([loc * step_size + i for i in range(step_size)]) - input_cmask.add_index_mask(dim=1, index=torch.tensor(index)) # pylint: disable=not-callable + input_cmask.add_index_mask(dim=1, index=torch.tensor(index).to(mask.mask_index[1].device)) # pylint: disable=not-callable module_masks.set_input_mask(input_cmask) return input_cmask @@ -870,7 +870,7 @@ def convert_to_coarse_mask(mask, dim=0): if index is None: return None, None, None else: - index = torch.LongTensor(index).to(weight_mask.device) + index = index.long().to(weight_mask.device) weight_cmask = CoarseMask(num_dim=4) weight_cmask.add_index_mask(dim=dim, index=index) bias_cmask = None diff --git a/src/sdk/pynni/nni/nas/benchmarks/__init__.py b/nni/compression/pytorch/utils/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/benchmarks/__init__.py rename to nni/compression/pytorch/utils/__init__.py diff --git a/src/sdk/pynni/nni/compression/torch/utils/config_validation.py b/nni/compression/pytorch/utils/config_validation.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/utils/config_validation.py rename to nni/compression/pytorch/utils/config_validation.py diff --git a/src/sdk/pynni/nni/compression/torch/utils/counter.py b/nni/compression/pytorch/utils/counter.py similarity index 98% rename from src/sdk/pynni/nni/compression/torch/utils/counter.py rename to nni/compression/pytorch/utils/counter.py index f4a3db7aa7..6061e8a8d0 100644 --- a/src/sdk/pynni/nni/compression/torch/utils/counter.py +++ b/nni/compression/pytorch/utils/counter.py @@ -3,7 +3,7 @@ import torch import torch.nn as nn -from nni.compression.torch.compressor import PrunerModuleWrapper +from nni.compression.pytorch.compressor import PrunerModuleWrapper try: from thop import profile @@ -132,4 +132,4 @@ def count_linear_mask(m, x, y): nn.Conv2d: count_convNd_mask, nn.Conv3d: count_convNd_mask, nn.Linear: count_linear_mask, -} \ No newline at end of file +} diff --git a/src/sdk/pynni/nni/compression/torch/utils/mask_conflict.py b/nni/compression/pytorch/utils/mask_conflict.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/utils/mask_conflict.py rename to nni/compression/pytorch/utils/mask_conflict.py diff --git a/src/sdk/pynni/nni/compression/torch/utils/num_param_counter.py b/nni/compression/pytorch/utils/num_param_counter.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/utils/num_param_counter.py rename to nni/compression/pytorch/utils/num_param_counter.py diff --git a/src/sdk/pynni/nni/compression/torch/utils/sensitivity_analysis.py b/nni/compression/pytorch/utils/sensitivity_analysis.py similarity index 98% rename from src/sdk/pynni/nni/compression/torch/utils/sensitivity_analysis.py rename to nni/compression/pytorch/utils/sensitivity_analysis.py index ef58237006..a36a523feb 100644 --- a/src/sdk/pynni/nni/compression/torch/utils/sensitivity_analysis.py +++ b/nni/compression/pytorch/utils/sensitivity_analysis.py @@ -9,7 +9,7 @@ import numpy as np import torch.nn as nn -from ..pruning.constants_pruner import PRUNER_DICT +# FIXME: I don't know where "utils" should be SUPPORTED_OP_NAME = ['Conv2d', 'Conv1d'] SUPPORTED_OP_TYPE = [getattr(nn, name) for name in SUPPORTED_OP_NAME] @@ -63,6 +63,8 @@ def __init__(self, model, val_func, sparsities=None, prune_type='l1', early_stop This value is effective only when the early_stop_mode is set. """ + from nni.algorithms.compression.pytorch.pruning.constants_pruner import PRUNER_DICT + self.model = model self.val_func = val_func self.target_layer = OrderedDict() diff --git a/src/sdk/pynni/nni/compression/torch/utils/shape_dependency.py b/nni/compression/pytorch/utils/shape_dependency.py similarity index 99% rename from src/sdk/pynni/nni/compression/torch/utils/shape_dependency.py rename to nni/compression/pytorch/utils/shape_dependency.py index a238848d86..3c2b63cd0b 100644 --- a/src/sdk/pynni/nni/compression/torch/utils/shape_dependency.py +++ b/nni/compression/pytorch/utils/shape_dependency.py @@ -17,7 +17,7 @@ def __init__(self, model=None, dummy_input=None, traced_model=None): """ Build the graph for the model. """ - from nni._graph_utils import TorchModuleGraph + from nni.common.graph_utils import TorchModuleGraph # check if the input is legal if traced_model is None: diff --git a/src/sdk/pynni/nni/compression/torch/utils/utils.py b/nni/compression/pytorch/utils/utils.py similarity index 100% rename from src/sdk/pynni/nni/compression/torch/utils/utils.py rename to nni/compression/pytorch/utils/utils.py diff --git a/src/sdk/pynni/nni/compression/tensorflow/__init__.py b/nni/compression/tensorflow/__init__.py similarity index 83% rename from src/sdk/pynni/nni/compression/tensorflow/__init__.py rename to nni/compression/tensorflow/__init__.py index 00d41ee55b..d05fade2f1 100644 --- a/src/sdk/pynni/nni/compression/tensorflow/__init__.py +++ b/nni/compression/tensorflow/__init__.py @@ -2,4 +2,3 @@ # Licensed under the MIT license. from .compressor import Compressor, Pruner -from .pruning import * diff --git a/src/sdk/pynni/nni/compression/tensorflow/compressor.py b/nni/compression/tensorflow/compressor.py similarity index 100% rename from src/sdk/pynni/nni/compression/tensorflow/compressor.py rename to nni/compression/tensorflow/compressor.py diff --git a/src/sdk/pynni/nni/compression/tensorflow/default_layers.py b/nni/compression/tensorflow/default_layers.py similarity index 100% rename from src/sdk/pynni/nni/compression/tensorflow/default_layers.py rename to nni/compression/tensorflow/default_layers.py diff --git a/src/sdk/pycli/nnicli/__init__.py b/nni/experiment/__init__.py similarity index 100% rename from src/sdk/pycli/nnicli/__init__.py rename to nni/experiment/__init__.py diff --git a/src/sdk/pycli/nnicli/nni_client.py b/nni/experiment/nni_client.py similarity index 98% rename from src/sdk/pycli/nnicli/nni_client.py rename to nni/experiment/nni_client.py index 571e2bd036..1dddafd219 100644 --- a/src/sdk/pycli/nnicli/nni_client.py +++ b/nni/experiment/nni_client.py @@ -5,7 +5,7 @@ Example: -from nnicli import Experiment +from nni.experiment import Experiment exp = Experiment() exp.start_experiment('../../../../examples/trials/mnist-pytorch/config.yml') @@ -196,16 +196,16 @@ class TrialJob: Trial job id. status: str Job status. - hyperParameters: list of `nnicli.TrialHyperParameters` - See `nnicli.TrialHyperParameters`. + hyperParameters: list of `nni.experiment.TrialHyperParameters` + See `nni.experiment.TrialHyperParameters`. logPath: str Log path. startTime: int Job start time (timestamp). endTime: int Job end time (timestamp). - finalMetricData: list of `nnicli.TrialMetricData` - See `nnicli.TrialMetricData`. + finalMetricData: list of `nni.experiment.TrialMetricData` + See `nni.experiment.TrialMetricData`. parameter_index: int Parameter index. """ diff --git a/src/sdk/pynni/nni/feature_engineering/__init__.py b/nni/feature_engineering/__init__.py similarity index 100% rename from src/sdk/pynni/nni/feature_engineering/__init__.py rename to nni/feature_engineering/__init__.py diff --git a/src/sdk/pynni/nni/feature_engineering/feature_selector.py b/nni/feature_engineering/feature_selector.py similarity index 100% rename from src/sdk/pynni/nni/feature_engineering/feature_selector.py rename to nni/feature_engineering/feature_selector.py diff --git a/src/sdk/pynni/nni/nas/pytorch/__init__.py b/nni/nas/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/__init__.py rename to nni/nas/__init__.py diff --git a/src/sdk/pynni/nni/nas/tensorflow/__init__.py b/nni/nas/benchmarks/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/tensorflow/__init__.py rename to nni/nas/benchmarks/__init__.py diff --git a/src/sdk/pynni/nni/nas/benchmarks/constants.py b/nni/nas/benchmarks/constants.py similarity index 100% rename from src/sdk/pynni/nni/nas/benchmarks/constants.py rename to nni/nas/benchmarks/constants.py diff --git a/src/sdk/pynni/nni/nas/benchmarks/nasbench101/__init__.py b/nni/nas/benchmarks/nasbench101/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/benchmarks/nasbench101/__init__.py rename to nni/nas/benchmarks/nasbench101/__init__.py diff --git a/src/sdk/pynni/nni/nas/benchmarks/nasbench101/constants.py b/nni/nas/benchmarks/nasbench101/constants.py similarity index 100% rename from src/sdk/pynni/nni/nas/benchmarks/nasbench101/constants.py rename to nni/nas/benchmarks/nasbench101/constants.py diff --git a/src/sdk/pynni/nni/nas/benchmarks/nasbench101/db_gen.py b/nni/nas/benchmarks/nasbench101/db_gen.py similarity index 100% rename from src/sdk/pynni/nni/nas/benchmarks/nasbench101/db_gen.py rename to nni/nas/benchmarks/nasbench101/db_gen.py diff --git a/src/sdk/pynni/nni/nas/benchmarks/nasbench101/graph_util.py b/nni/nas/benchmarks/nasbench101/graph_util.py similarity index 100% rename from src/sdk/pynni/nni/nas/benchmarks/nasbench101/graph_util.py rename to nni/nas/benchmarks/nasbench101/graph_util.py diff --git a/src/sdk/pynni/nni/nas/benchmarks/nasbench101/model.py b/nni/nas/benchmarks/nasbench101/model.py similarity index 100% rename from src/sdk/pynni/nni/nas/benchmarks/nasbench101/model.py rename to nni/nas/benchmarks/nasbench101/model.py diff --git a/src/sdk/pynni/nni/nas/benchmarks/nasbench101/query.py b/nni/nas/benchmarks/nasbench101/query.py similarity index 100% rename from src/sdk/pynni/nni/nas/benchmarks/nasbench101/query.py rename to nni/nas/benchmarks/nasbench101/query.py diff --git a/src/sdk/pynni/nni/nas/benchmarks/nasbench201/__init__.py b/nni/nas/benchmarks/nasbench201/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/benchmarks/nasbench201/__init__.py rename to nni/nas/benchmarks/nasbench201/__init__.py diff --git a/src/sdk/pynni/nni/nas/benchmarks/nasbench201/constants.py b/nni/nas/benchmarks/nasbench201/constants.py similarity index 100% rename from src/sdk/pynni/nni/nas/benchmarks/nasbench201/constants.py rename to nni/nas/benchmarks/nasbench201/constants.py diff --git a/src/sdk/pynni/nni/nas/benchmarks/nasbench201/db_gen.py b/nni/nas/benchmarks/nasbench201/db_gen.py similarity index 100% rename from src/sdk/pynni/nni/nas/benchmarks/nasbench201/db_gen.py rename to nni/nas/benchmarks/nasbench201/db_gen.py diff --git a/src/sdk/pynni/nni/nas/benchmarks/nasbench201/model.py b/nni/nas/benchmarks/nasbench201/model.py similarity index 98% rename from src/sdk/pynni/nni/nas/benchmarks/nasbench201/model.py rename to nni/nas/benchmarks/nasbench201/model.py index 3b898de7c8..b390a43091 100644 --- a/src/sdk/pynni/nni/nas/benchmarks/nasbench201/model.py +++ b/nni/nas/benchmarks/nasbench201/model.py @@ -31,7 +31,7 @@ class Nb201TrialConfig(Model): Dataset used for training and evaluation. NAS-Bench-201 provides the following 4 options: ``cifar10-valid`` (training data is splited into 25k for training and 25k for validation, validation data is used for test), ``cifar10`` (training data is used in training, validation - data is splited into 25k for validation and 25k for testing), ``cifar100`` (same protocol as ``cifar10``), + data is splited into 5k for validation and 5k for testing), ``cifar100`` (same protocol as ``cifar10``), and ``imagenet16-120`` (a subset of 120 classes in ImageNet, downscaled to 16x16, using training data for training, 6k images from validation set for validation and the other 6k for testing). """ diff --git a/src/sdk/pynni/nni/nas/benchmarks/nasbench201/query.py b/nni/nas/benchmarks/nasbench201/query.py similarity index 100% rename from src/sdk/pynni/nni/nas/benchmarks/nasbench201/query.py rename to nni/nas/benchmarks/nasbench201/query.py diff --git a/src/sdk/pynni/nni/nas/benchmarks/nds/__init__.py b/nni/nas/benchmarks/nds/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/benchmarks/nds/__init__.py rename to nni/nas/benchmarks/nds/__init__.py diff --git a/src/sdk/pynni/nni/nas/benchmarks/nds/constants.py b/nni/nas/benchmarks/nds/constants.py similarity index 100% rename from src/sdk/pynni/nni/nas/benchmarks/nds/constants.py rename to nni/nas/benchmarks/nds/constants.py diff --git a/src/sdk/pynni/nni/nas/benchmarks/nds/db_gen.py b/nni/nas/benchmarks/nds/db_gen.py similarity index 100% rename from src/sdk/pynni/nni/nas/benchmarks/nds/db_gen.py rename to nni/nas/benchmarks/nds/db_gen.py diff --git a/src/sdk/pynni/nni/nas/benchmarks/nds/model.py b/nni/nas/benchmarks/nds/model.py similarity index 100% rename from src/sdk/pynni/nni/nas/benchmarks/nds/model.py rename to nni/nas/benchmarks/nds/model.py diff --git a/src/sdk/pynni/nni/nas/benchmarks/nds/query.py b/nni/nas/benchmarks/nds/query.py similarity index 100% rename from src/sdk/pynni/nni/nas/benchmarks/nds/query.py rename to nni/nas/benchmarks/nds/query.py diff --git a/src/sdk/pynni/nni/nas/benchmarks/utils.py b/nni/nas/benchmarks/utils.py similarity index 100% rename from src/sdk/pynni/nni/nas/benchmarks/utils.py rename to nni/nas/benchmarks/utils.py diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/__init__.py b/nni/nas/pytorch/__init__.py similarity index 100% rename from src/sdk/pynni/nni/networkmorphism_tuner/__init__.py rename to nni/nas/pytorch/__init__.py diff --git a/src/sdk/pynni/nni/nas/pytorch/base_mutator.py b/nni/nas/pytorch/base_mutator.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/base_mutator.py rename to nni/nas/pytorch/base_mutator.py diff --git a/src/sdk/pynni/nni/nas/pytorch/base_trainer.py b/nni/nas/pytorch/base_trainer.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/base_trainer.py rename to nni/nas/pytorch/base_trainer.py diff --git a/src/sdk/pynni/nni/nas/pytorch/callbacks.py b/nni/nas/pytorch/callbacks.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/callbacks.py rename to nni/nas/pytorch/callbacks.py diff --git a/src/sdk/pynni/nni/nas/pytorch/fixed.py b/nni/nas/pytorch/fixed.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/fixed.py rename to nni/nas/pytorch/fixed.py diff --git a/src/sdk/pynni/nni/nas/pytorch/mutables.py b/nni/nas/pytorch/mutables.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/mutables.py rename to nni/nas/pytorch/mutables.py diff --git a/src/sdk/pynni/nni/nas/pytorch/mutator.py b/nni/nas/pytorch/mutator.py similarity index 99% rename from src/sdk/pynni/nni/nas/pytorch/mutator.py rename to nni/nas/pytorch/mutator.py index 1e97ab2123..1845ba73b2 100644 --- a/src/sdk/pynni/nni/nas/pytorch/mutator.py +++ b/nni/nas/pytorch/mutator.py @@ -107,7 +107,7 @@ def graph(self, inputs): """ if not torch.__version__.startswith("1.4"): logger.warning("Graph is only tested with PyTorch 1.4. Other versions might not work.") - from nni._graph_utils import build_graph + from nni.common.graph_utils import build_graph from google.protobuf import json_format # protobuf should be installed as long as tensorboard is installed try: diff --git a/src/sdk/pynni/nni/nas/pytorch/nasbench201/__init__.py b/nni/nas/pytorch/nasbench201/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/nasbench201/__init__.py rename to nni/nas/pytorch/nasbench201/__init__.py diff --git a/src/sdk/pynni/nni/nas/pytorch/nasbench201/nasbench201.py b/nni/nas/pytorch/nasbench201/nasbench201.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/nasbench201/nasbench201.py rename to nni/nas/pytorch/nasbench201/nasbench201.py diff --git a/src/sdk/pynni/nni/nas/pytorch/nasbench201/nasbench201_ops.py b/nni/nas/pytorch/nasbench201/nasbench201_ops.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/nasbench201/nasbench201_ops.py rename to nni/nas/pytorch/nasbench201/nasbench201_ops.py diff --git a/src/sdk/pynni/nni/nas/pytorch/search_space_zoo/__init__.py b/nni/nas/pytorch/search_space_zoo/__init__.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/search_space_zoo/__init__.py rename to nni/nas/pytorch/search_space_zoo/__init__.py diff --git a/src/sdk/pynni/nni/nas/pytorch/search_space_zoo/darts_cell.py b/nni/nas/pytorch/search_space_zoo/darts_cell.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/search_space_zoo/darts_cell.py rename to nni/nas/pytorch/search_space_zoo/darts_cell.py diff --git a/src/sdk/pynni/nni/nas/pytorch/search_space_zoo/darts_ops.py b/nni/nas/pytorch/search_space_zoo/darts_ops.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/search_space_zoo/darts_ops.py rename to nni/nas/pytorch/search_space_zoo/darts_ops.py diff --git a/src/sdk/pynni/nni/nas/pytorch/search_space_zoo/enas_cell.py b/nni/nas/pytorch/search_space_zoo/enas_cell.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/search_space_zoo/enas_cell.py rename to nni/nas/pytorch/search_space_zoo/enas_cell.py diff --git a/src/sdk/pynni/nni/nas/pytorch/search_space_zoo/enas_ops.py b/nni/nas/pytorch/search_space_zoo/enas_ops.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/search_space_zoo/enas_ops.py rename to nni/nas/pytorch/search_space_zoo/enas_ops.py diff --git a/src/sdk/pynni/nni/nas/pytorch/trainer.py b/nni/nas/pytorch/trainer.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/trainer.py rename to nni/nas/pytorch/trainer.py diff --git a/src/sdk/pynni/nni/nas/pytorch/utils.py b/nni/nas/pytorch/utils.py similarity index 100% rename from src/sdk/pynni/nni/nas/pytorch/utils.py rename to nni/nas/pytorch/utils.py diff --git a/src/sdk/pynni/nni/pbt_tuner/__init__.py b/nni/nas/tensorflow/__init__.py similarity index 100% rename from src/sdk/pynni/nni/pbt_tuner/__init__.py rename to nni/nas/tensorflow/__init__.py diff --git a/src/sdk/pynni/nni/nas/tensorflow/base_mutator.py b/nni/nas/tensorflow/base_mutator.py similarity index 100% rename from src/sdk/pynni/nni/nas/tensorflow/base_mutator.py rename to nni/nas/tensorflow/base_mutator.py diff --git a/src/sdk/pynni/nni/nas/tensorflow/mutables.py b/nni/nas/tensorflow/mutables.py similarity index 100% rename from src/sdk/pynni/nni/nas/tensorflow/mutables.py rename to nni/nas/tensorflow/mutables.py diff --git a/src/sdk/pynni/nni/nas/tensorflow/mutator.py b/nni/nas/tensorflow/mutator.py similarity index 91% rename from src/sdk/pynni/nni/nas/tensorflow/mutator.py rename to nni/nas/tensorflow/mutator.py index 20c57f9405..378cd5915d 100644 --- a/src/sdk/pynni/nni/nas/tensorflow/mutator.py +++ b/nni/nas/tensorflow/mutator.py @@ -66,7 +66,12 @@ def _tensor_reduction(self, reduction_type, tensor_list): if reduction_type == 'mean': return sum(tensor_list) / len(tensor_list) if reduction_type == 'concat': - return tf.concat(tensor_list, axis=0) + image_data_format = tf.keras.backend.image_data_format() + if image_data_format == "channels_first": + axis = 0 + else: + axis = -1 + return tf.concat(tensor_list, axis=axis) raise ValueError('Unrecognized reduction policy: "{}'.format(reduction_type)) def _get_decision(self, mutable): diff --git a/src/sdk/pynni/nni/nas/tensorflow/utils.py b/nni/nas/tensorflow/utils.py similarity index 100% rename from src/sdk/pynni/nni/nas/tensorflow/utils.py rename to nni/nas/tensorflow/utils.py diff --git a/src/sdk/pynni/nni/parameter_expressions.py b/nni/parameter_expressions.py similarity index 100% rename from src/sdk/pynni/nni/parameter_expressions.py rename to nni/parameter_expressions.py diff --git a/src/sdk/pynni/nni/recoverable.py b/nni/recoverable.py similarity index 100% rename from src/sdk/pynni/nni/recoverable.py rename to nni/recoverable.py diff --git a/src/sdk/pynni/nni/common.py b/nni/runtime/common.py similarity index 82% rename from src/sdk/pynni/nni/common.py rename to nni/runtime/common.py index 97532dae5d..ec5ef10162 100644 --- a/src/sdk/pynni/nni/common.py +++ b/nni/runtime/common.py @@ -4,6 +4,7 @@ from datetime import datetime from io import TextIOBase import logging +import os import sys import time @@ -17,6 +18,11 @@ _time_format = '%m/%d/%Y, %I:%M:%S %p' +# FIXME +# This hotfix the bug that querying installed tuners with `package_utils` will activate dispatcher logger. +# This behavior depends on underlying implementation of `nnictl` and is likely to break in future. +_logger_initialized = False + class _LoggerFileWrapper(TextIOBase): def __init__(self, logger_file): self.file = logger_file @@ -33,6 +39,14 @@ def init_logger(logger_file_path, log_level_name='info'): This will redirect anything from logging.getLogger() as well as stdout to specified file. logger_file_path: path of logger file (path-like object). """ + global _logger_initialized + if _logger_initialized: + return + _logger_initialized = True + + if os.environ.get('NNI_PLATFORM') == 'unittest': + return # fixme: launching logic needs refactor + log_level = log_level_map.get(log_level_name, logging.INFO) logger_file = open(logger_file_path, 'w') fmt = '[%(asctime)s] %(levelname)s (%(name)s/%(threadName)s) %(message)s' @@ -55,6 +69,11 @@ def init_standalone_logger(): Initialize root logger for standalone mode. This will set NNI's log level to INFO and print its log to stdout. """ + global _logger_initialized + if _logger_initialized: + return + _logger_initialized = True + fmt = '[%(asctime)s] %(levelname)s (%(name)s) %(message)s' formatter = logging.Formatter(fmt, _time_format) handler = logging.StreamHandler(sys.stdout) diff --git a/src/sdk/pynni/nni/env_vars.py b/nni/runtime/env_vars.py similarity index 100% rename from src/sdk/pynni/nni/env_vars.py rename to nni/runtime/env_vars.py diff --git a/src/sdk/pynni/nni/msg_dispatcher.py b/nni/runtime/msg_dispatcher.py similarity index 99% rename from src/sdk/pynni/nni/msg_dispatcher.py rename to nni/runtime/msg_dispatcher.py index 7e3232e4f9..4f24294fc7 100644 --- a/src/sdk/pynni/nni/msg_dispatcher.py +++ b/nni/runtime/msg_dispatcher.py @@ -8,10 +8,10 @@ from nni import NoMoreTrialError from .protocol import CommandType, send from .msg_dispatcher_base import MsgDispatcherBase -from .assessor import AssessResult +from nni.assessor import AssessResult from .common import multi_thread_enabled, multi_phase_enabled from .env_vars import dispatcher_env_vars -from .utils import MetricType, to_json +from ..utils import MetricType, to_json _logger = logging.getLogger(__name__) diff --git a/src/sdk/pynni/nni/msg_dispatcher_base.py b/nni/runtime/msg_dispatcher_base.py similarity index 99% rename from src/sdk/pynni/nni/msg_dispatcher_base.py rename to nni/runtime/msg_dispatcher_base.py index e323257d7f..66af52df28 100644 --- a/src/sdk/pynni/nni/msg_dispatcher_base.py +++ b/nni/runtime/msg_dispatcher_base.py @@ -9,8 +9,8 @@ from .common import multi_thread_enabled from .env_vars import dispatcher_env_vars -from .utils import init_dispatcher_logger -from .recoverable import Recoverable +from ..utils import init_dispatcher_logger +from ..recoverable import Recoverable from .protocol import CommandType, receive init_dispatcher_logger() diff --git a/src/sdk/pynni/nni/platform/__init__.py b/nni/runtime/platform/__init__.py similarity index 100% rename from src/sdk/pynni/nni/platform/__init__.py rename to nni/runtime/platform/__init__.py diff --git a/src/sdk/pynni/nni/platform/local.py b/nni/runtime/platform/local.py similarity index 98% rename from src/sdk/pynni/nni/platform/local.py rename to nni/runtime/platform/local.py index a25f27546f..5d8124d3ff 100644 --- a/src/sdk/pynni/nni/platform/local.py +++ b/nni/runtime/platform/local.py @@ -9,7 +9,7 @@ from ..common import init_logger from ..env_vars import trial_env_vars -from ..utils import to_json +from nni.utils import to_json _sysdir = trial_env_vars.NNI_SYS_DIR if not os.path.exists(os.path.join(_sysdir, '.nni')): diff --git a/src/sdk/pynni/nni/platform/standalone.py b/nni/runtime/platform/standalone.py similarity index 100% rename from src/sdk/pynni/nni/platform/standalone.py rename to nni/runtime/platform/standalone.py diff --git a/src/sdk/pynni/nni/platform/test.py b/nni/runtime/platform/test.py similarity index 100% rename from src/sdk/pynni/nni/platform/test.py rename to nni/runtime/platform/test.py diff --git a/src/sdk/pynni/nni/protocol.py b/nni/runtime/protocol.py similarity index 93% rename from src/sdk/pynni/nni/protocol.py rename to nni/runtime/protocol.py index ca2d7069d2..57ea7fbc0b 100644 --- a/src/sdk/pynni/nni/protocol.py +++ b/nni/runtime/protocol.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. import logging +import os import threading from enum import Enum @@ -27,8 +28,9 @@ class CommandType(Enum): _lock = threading.Lock() try: - _in_file = open(3, 'rb') - _out_file = open(4, 'wb') + if os.environ.get('NNI_PLATFORM') != 'unittest': + _in_file = open(3, 'rb') + _out_file = open(4, 'wb') except OSError: _msg = 'IPC pipeline not exists, maybe you are importing tuner/assessor from trial code?' logging.getLogger(__name__).warning(_msg) diff --git a/src/sdk/pynni/nni/smartparam.py b/nni/smartparam.py similarity index 97% rename from src/sdk/pynni/nni/smartparam.py rename to nni/smartparam.py index 5841714e11..dde0ac2bd6 100644 --- a/src/sdk/pynni/nni/smartparam.py +++ b/nni/smartparam.py @@ -3,10 +3,10 @@ import numpy as np -from .env_vars import trial_env_vars +from .runtime.env_vars import trial_env_vars from . import trial from . import parameter_expressions as param_exp -from .nas_utils import classic_mode, enas_mode, oneshot_mode, darts_mode +from .common.nas_utils import classic_mode, enas_mode, oneshot_mode, darts_mode __all__ = [ diff --git a/tools/nni_annotation/.gitignore b/nni/tools/annotation/.gitignore similarity index 100% rename from tools/nni_annotation/.gitignore rename to nni/tools/annotation/.gitignore diff --git a/tools/nni_annotation/__init__.py b/nni/tools/annotation/__init__.py similarity index 100% rename from tools/nni_annotation/__init__.py rename to nni/tools/annotation/__init__.py diff --git a/tools/nni_annotation/code_generator.py b/nni/tools/annotation/code_generator.py similarity index 100% rename from tools/nni_annotation/code_generator.py rename to nni/tools/annotation/code_generator.py diff --git a/tools/nni_annotation/search_space_generator.py b/nni/tools/annotation/search_space_generator.py similarity index 100% rename from tools/nni_annotation/search_space_generator.py rename to nni/tools/annotation/search_space_generator.py diff --git a/tools/nni_annotation/specific_code_generator.py b/nni/tools/annotation/specific_code_generator.py similarity index 99% rename from tools/nni_annotation/specific_code_generator.py rename to nni/tools/annotation/specific_code_generator.py index 511f7dba05..08659594ea 100644 --- a/tools/nni_annotation/specific_code_generator.py +++ b/nni/tools/annotation/specific_code_generator.py @@ -3,7 +3,7 @@ import ast import astor -from nni_cmd.common_utils import print_warning +from nni.tools.nnictl.common_utils import print_warning from .utils import ast_Num, ast_Str diff --git a/tools/nni_annotation/utils.py b/nni/tools/annotation/utils.py similarity index 100% rename from tools/nni_annotation/utils.py rename to nni/tools/annotation/utils.py diff --git a/tools/nni_cmd/__init__.py b/nni/tools/gpu_tool/__init__.py similarity index 100% rename from tools/nni_cmd/__init__.py rename to nni/tools/gpu_tool/__init__.py diff --git a/tools/nni_gpu_tool/gpu_metrics_collector.py b/nni/tools/gpu_tool/gpu_metrics_collector.py similarity index 100% rename from tools/nni_gpu_tool/gpu_metrics_collector.py rename to nni/tools/gpu_tool/gpu_metrics_collector.py diff --git a/tools/nni_gpu_tool/__init__.py b/nni/tools/nnictl/__init__.py similarity index 100% rename from tools/nni_gpu_tool/__init__.py rename to nni/tools/nnictl/__init__.py diff --git a/tools/nni_cmd/command_utils.py b/nni/tools/nnictl/command_utils.py similarity index 100% rename from tools/nni_cmd/command_utils.py rename to nni/tools/nnictl/command_utils.py diff --git a/tools/nni_cmd/common_utils.py b/nni/tools/nnictl/common_utils.py similarity index 100% rename from tools/nni_cmd/common_utils.py rename to nni/tools/nnictl/common_utils.py diff --git a/tools/nni_cmd/config_schema.py b/nni/tools/nnictl/config_schema.py similarity index 99% rename from tools/nni_cmd/config_schema.py rename to nni/tools/nnictl/config_schema.py index 3c8115a5e8..d320163595 100644 --- a/tools/nni_cmd/config_schema.py +++ b/nni/tools/nnictl/config_schema.py @@ -6,7 +6,7 @@ import os import netifaces from schema import Schema, And, Optional, Regex, Or, SchemaError -from nni.package_utils import create_validator_instance, get_all_builtin_names, get_builtin_algo_meta +from nni.tools.package_utils import create_validator_instance, get_all_builtin_names, get_builtin_algo_meta from .constants import SCHEMA_TYPE_ERROR, SCHEMA_RANGE_ERROR, SCHEMA_PATH_ERROR from .common_utils import get_yml_content, print_warning diff --git a/tools/nni_cmd/config_utils.py b/nni/tools/nnictl/config_utils.py similarity index 100% rename from tools/nni_cmd/config_utils.py rename to nni/tools/nnictl/config_utils.py diff --git a/tools/nni_cmd/constants.py b/nni/tools/nnictl/constants.py similarity index 100% rename from tools/nni_cmd/constants.py rename to nni/tools/nnictl/constants.py diff --git a/tools/nni_cmd/launcher.py b/nni/tools/nnictl/launcher.py similarity index 97% rename from tools/nni_cmd/launcher.py rename to nni/tools/nnictl/launcher.py index c47e4971e8..fa0aa3baab 100644 --- a/tools/nni_cmd/launcher.py +++ b/nni/tools/nnictl/launcher.py @@ -9,8 +9,9 @@ import time import tempfile from subprocess import Popen, check_call, CalledProcessError, PIPE, STDOUT -from nni_annotation import expand_annotations, generate_search_space -from nni.package_utils import get_builtin_module_class_name, get_nni_installation_path +from nni.tools.annotation import expand_annotations, generate_search_space +from nni.tools.package_utils import get_builtin_module_class_name +import nni_node from .launcher_utils import validate_all_content from .rest_utils import rest_put, rest_post, check_rest_server, check_response from .url_utils import cluster_metadata_url, experiment_url, get_local_urls @@ -52,15 +53,16 @@ def start_rest_server(port, platform, mode, config_file_name, foreground=False, print_normal('Starting restful server...') - entry_dir = get_nni_installation_path() + entry_dir = nni_node.__path__[0] if (not entry_dir) or (not os.path.exists(entry_dir)): print_error('Fail to find nni under python library') exit(1) entry_file = os.path.join(entry_dir, 'main.js') - node_command = 'node' if sys.platform == 'win32': - node_command = os.path.join(entry_dir[:-3], 'Scripts', 'node.exe') + node_command = os.path.join(entry_dir, 'node.exe') + else: + node_command = 'node' cmds = [node_command, '--max-old-space-size=4096', entry_file, '--port', str(port), '--mode', platform] if mode == 'view': cmds += ['--start_mode', 'resume'] @@ -138,7 +140,10 @@ def set_remote_config(experiment_config, port, config_file_name): '''Call setClusterMetadata to pass trial''' #set machine_list request_data = dict() - request_data['remote_config'] = experiment_config['remoteConfig'] + if experiment_config.get('remoteConfig'): + request_data['remote_config'] = experiment_config['remoteConfig'] + else: + request_data['remote_config'] = {'reuse': False} request_data['machine_list'] = experiment_config['machineList'] if request_data['machine_list']: for i in range(len(request_data['machine_list'])): @@ -368,6 +373,11 @@ def set_experiment(experiment_config, mode, port, config_file_name): {'key': 'frameworkcontroller_config', 'value': experiment_config['frameworkcontrollerConfig']}) request_data['clusterMetaData'].append( {'key': 'trial_config', 'value': experiment_config['trial']}) + elif experiment_config['trainingServicePlatform'] == 'aml': + request_data['clusterMetaData'].append( + {'key': 'aml_config', 'value': experiment_config['amlConfig']}) + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) response = rest_post(experiment_url(port), json.dumps(request_data), REST_TIME_OUT, show_error=True) if check_response(response): return response diff --git a/tools/nni_cmd/launcher_utils.py b/nni/tools/nnictl/launcher_utils.py similarity index 100% rename from tools/nni_cmd/launcher_utils.py rename to nni/tools/nnictl/launcher_utils.py diff --git a/tools/nni_cmd/nnictl.py b/nni/tools/nnictl/nnictl.py similarity index 100% rename from tools/nni_cmd/nnictl.py rename to nni/tools/nnictl/nnictl.py diff --git a/tools/nni_cmd/nnictl_utils.py b/nni/tools/nnictl/nnictl_utils.py similarity index 99% rename from tools/nni_cmd/nnictl_utils.py rename to nni/tools/nnictl/nnictl_utils.py index 4c01d2be09..3427d698b0 100644 --- a/tools/nni_cmd/nnictl_utils.py +++ b/nni/tools/nnictl/nnictl_utils.py @@ -13,8 +13,8 @@ from datetime import datetime, timezone from subprocess import Popen from pyhdfs import HdfsClient -from nni.package_utils import get_nni_installation_path -from nni_annotation import expand_annotations +from nni.tools.annotation import expand_annotations +import nni_node from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url, export_data_url, metric_data_url from .config_utils import Config, Experiments @@ -424,13 +424,14 @@ def webui_nas(args): '''launch nas ui''' print_normal('Starting NAS UI...') try: - entry_dir = get_nni_installation_path() + entry_dir = nni_node.__path__[0] entry_file = os.path.join(entry_dir, 'nasui', 'server.js') - node_command = 'node' if sys.platform == 'win32': - node_command = os.path.join(entry_dir[:-3], 'Scripts', 'node.exe') + node_command = os.path.join(entry_dir, 'node.exe') + else: + node_command = 'node' cmds = [node_command, '--max-old-space-size=4096', entry_file, '--port', str(args.port), '--logdir', args.logdir] - subprocess.run(cmds) + subprocess.run(cmds, cwd=entry_dir) except KeyboardInterrupt: pass diff --git a/tools/nni_cmd/package_management.py b/nni/tools/nnictl/package_management.py similarity index 96% rename from tools/nni_cmd/package_management.py rename to nni/tools/nnictl/package_management.py index 942665b97a..5eef340752 100644 --- a/tools/nni_cmd/package_management.py +++ b/nni/tools/nnictl/package_management.py @@ -6,7 +6,7 @@ import json import pkginfo import nni -from nni.package_utils import read_installed_package_meta, get_installed_package_meta, \ +from nni.tools.package_utils import read_installed_package_meta, get_installed_package_meta, \ write_package_meta, get_builtin_algo_meta, get_not_installable_builtin_names, ALGO_TYPES from .constants import INSTALLABLE_PACKAGE_META @@ -19,7 +19,7 @@ def install_by_name(package_name): if package_name not in INSTALLABLE_PACKAGE_META: raise RuntimeError('{} is not found in installable packages!'.format(package_name)) - requirements_path = os.path.join(nni.__path__[0], INSTALLABLE_PACKAGE_META[package_name]['code_sub_dir'], 'requirements.txt') + requirements_path = os.path.join(nni.__path__[0], 'algorithms/hpo', INSTALLABLE_PACKAGE_META[package_name]['code_sub_dir'], 'requirements.txt') assert os.path.exists(requirements_path) return install_requirements_command(requirements_path) diff --git a/tools/nni_cmd/rest_utils.py b/nni/tools/nnictl/rest_utils.py similarity index 100% rename from tools/nni_cmd/rest_utils.py rename to nni/tools/nnictl/rest_utils.py diff --git a/tools/nni_cmd/ssh_utils.py b/nni/tools/nnictl/ssh_utils.py similarity index 100% rename from tools/nni_cmd/ssh_utils.py rename to nni/tools/nnictl/ssh_utils.py diff --git a/tools/nni_cmd/tensorboard_utils.py b/nni/tools/nnictl/tensorboard_utils.py similarity index 100% rename from tools/nni_cmd/tensorboard_utils.py rename to nni/tools/nnictl/tensorboard_utils.py diff --git a/tools/nni_cmd/updater.py b/nni/tools/nnictl/updater.py similarity index 100% rename from tools/nni_cmd/updater.py rename to nni/tools/nnictl/updater.py diff --git a/tools/nni_cmd/url_utils.py b/nni/tools/nnictl/url_utils.py similarity index 100% rename from tools/nni_cmd/url_utils.py rename to nni/tools/nnictl/url_utils.py diff --git a/src/sdk/pynni/nni/package_utils.py b/nni/tools/package_utils/__init__.py similarity index 84% rename from src/sdk/pynni/nni/package_utils.py rename to nni/tools/package_utils/__init__.py index 308913c6b7..9a1c004053 100644 --- a/src/sdk/pynni/nni/package_utils.py +++ b/nni/tools/package_utils/__init__.py @@ -1,14 +1,15 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -import os -import site -import sys from collections import defaultdict -from pathlib import Path import importlib +import os +from pathlib import Path +import sys + import ruamel.yaml as yaml +import nni from .constants import BuiltinAlgorithms ALGO_TYPES = ['tuners', 'assessors', 'advisors'] @@ -286,50 +287,9 @@ def create_customized_class_instance(class_params): return instance -def get_nni_installation_parent_dir(): - ''' Find nni installation parent directory - ''' - if os.getenv('VIRTUAL_ENV'): - # if 'virtualenv' package is used, `site` has not attr getsitepackages, so we will instead use VIRTUAL_ENV - # Note that conda venv will not have VIRTUAL_ENV - python_dir = os.getenv('VIRTUAL_ENV') - else: - python_sitepackage = site.getsitepackages()[0] - # If system-wide python is used, we will give priority to using `local sitepackage`--"usersitepackages()" given - # that nni exists there - if python_sitepackage.startswith('/usr') or python_sitepackage.startswith('/Library'): - python_dir = _try_installation_path_sequentially(site.getusersitepackages(), *site.getsitepackages()) - else: - python_dir = _try_installation_path_sequentially(*site.getsitepackages(), site.getusersitepackages()) - return python_dir - -def _try_installation_path_sequentially(*sitepackages): - '''Try different installation path sequentially util nni is found. - Return None if nothing is found - ''' - for sitepackage in sitepackages: - path = Path(sitepackage) - if len(path.parents) > 2 and (path.parents[2] / 'nni' / 'main.js').is_file(): - return str(path.parents[2]) - if (path / 'nni' / 'main.js').is_file(): - return str(path) - return None - -def get_nni_installation_path(): - ''' Find nni installation directory - ''' - parent_dir = get_nni_installation_parent_dir() - if parent_dir: - entry_file = os.path.join(parent_dir, 'nni', 'main.js') - if os.path.isfile(entry_file): - return os.path.join(parent_dir, 'nni') - return None - -def get_nni_config_dir(): - return os.path.join(get_nni_installation_path(), 'config') - def get_package_config_path(): - config_dir = get_nni_config_dir() + # FIXME: this might not be the desired location + config_dir = Path(nni.__path__[0]).parent / 'nni_config' if not os.path.exists(config_dir): os.makedirs(config_dir, exist_ok=True) return os.path.join(config_dir, 'installed_packages.yml') diff --git a/nni/tools/package_utils/constants.py b/nni/tools/package_utils/constants.py new file mode 100644 index 0000000000..952310b5f9 --- /dev/null +++ b/nni/tools/package_utils/constants.py @@ -0,0 +1,91 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +BuiltinAlgorithms = { + 'tuners': [ + { + 'name': 'TPE', + 'class_name': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', + 'class_args': { + 'algorithm_name': 'tpe' + }, + 'class_args_validator': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' + }, + { + 'name': 'Random', + 'class_name': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', + 'class_args': { + 'algorithm_name': 'random_search' + }, + 'accept_class_args': False, + 'class_args_validator': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' + }, + { + 'name': 'Anneal', + 'class_name': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', + 'class_args': { + 'algorithm_name': 'anneal' + }, + 'class_args_validator': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' + }, + { + 'name': 'Evolution', + 'class_name': 'nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionTuner', + 'class_args_validator': 'nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionClassArgsValidator' + }, + { + 'name': 'BatchTuner', + 'class_name': 'nni.algorithms.hpo.batch_tuner.batch_tuner.BatchTuner', + 'accept_class_args': False, + }, + { + 'name': 'GridSearch', + 'class_name': 'nni.algorithms.hpo.gridsearch_tuner.gridsearch_tuner.GridSearchTuner', + 'accept_class_args': False, + }, + { + 'name': 'NetworkMorphism', + 'class_name': 'nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner', + 'class_args_validator': 'nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismClassArgsValidator' + }, + { + 'name': 'MetisTuner', + 'class_name': 'nni.algorithms.hpo.metis_tuner.metis_tuner.MetisTuner', + 'class_args_validator': 'nni.algorithms.hpo.metis_tuner.metis_tuner.MetisClassArgsValidator' + }, + { + 'name': 'GPTuner', + 'class_name': 'nni.algorithms.hpo.gp_tuner.gp_tuner.GPTuner', + 'class_args_validator': 'nni.algorithms.hpo.gp_tuner.gp_tuner.GPClassArgsValidator' + }, + { + 'name': 'PBTTuner', + 'class_name': 'nni.algorithms.hpo.pbt_tuner.pbt_tuner.PBTTuner', + 'class_args_validator': 'nni.algorithms.hpo.pbt_tuner.pbt_tuner.PBTClassArgsValidator' + }, + { + 'name': 'RegularizedEvolutionTuner', + 'class_name': 'nni.algorithms.hpo.regularized_evolution_tuner.regularized_evolution_tuner.RegularizedEvolutionTuner', + 'class_args_validator': 'nni.algorithms.hpo.regularized_evolution_tuner.regularized_evolution_tuner.EvolutionClassArgsValidator' + } + ], + 'assessors': [ + { + 'name': 'Medianstop', + 'class_name': 'nni.algorithms.hpo.medianstop_assessor.medianstop_assessor.MedianstopAssessor', + 'class_args_validator': 'nni.algorithms.hpo.medianstop_assessor.medianstop_assessor.MedianstopClassArgsValidator' + }, + { + 'name': 'Curvefitting', + 'class_name': 'nni.algorithms.hpo.curvefitting_assessor.curvefitting_assessor.CurvefittingAssessor', + 'class_args_validator': 'nni.algorithms.hpo.curvefitting_assessor.curvefitting_assessor.CurvefittingClassArgsValidator' + }, + ], + 'advisors': [ + { + 'name': 'Hyperband', + 'class_name': 'nni.algorithms.hpo.hyperband_advisor.hyperband_advisor.Hyperband', + 'class_args_validator': 'nni.algorithms.hpo.hyperband_advisor.hyperband_advisor.HyperbandClassArgsValidator' + } + ] +} diff --git a/tools/nni_trial_tool/__init__.py b/nni/tools/trial_tool/__init__.py similarity index 100% rename from tools/nni_trial_tool/__init__.py rename to nni/tools/trial_tool/__init__.py diff --git a/tools/nni_trial_tool/aml_channel.py b/nni/tools/trial_tool/aml_channel.py similarity index 100% rename from tools/nni_trial_tool/aml_channel.py rename to nni/tools/trial_tool/aml_channel.py diff --git a/tools/nni_trial_tool/base_channel.py b/nni/tools/trial_tool/base_channel.py similarity index 100% rename from tools/nni_trial_tool/base_channel.py rename to nni/tools/trial_tool/base_channel.py diff --git a/tools/nni_trial_tool/commands.py b/nni/tools/trial_tool/commands.py similarity index 100% rename from tools/nni_trial_tool/commands.py rename to nni/tools/trial_tool/commands.py diff --git a/tools/nni_trial_tool/constants.py b/nni/tools/trial_tool/constants.py similarity index 100% rename from tools/nni_trial_tool/constants.py rename to nni/tools/trial_tool/constants.py diff --git a/tools/nni_trial_tool/file_channel.py b/nni/tools/trial_tool/file_channel.py similarity index 100% rename from tools/nni_trial_tool/file_channel.py rename to nni/tools/trial_tool/file_channel.py diff --git a/tools/nni_trial_tool/gpu.py b/nni/tools/trial_tool/gpu.py similarity index 100% rename from tools/nni_trial_tool/gpu.py rename to nni/tools/trial_tool/gpu.py diff --git a/tools/nni_trial_tool/hdfsClientUtility.py b/nni/tools/trial_tool/hdfsClientUtility.py similarity index 100% rename from tools/nni_trial_tool/hdfsClientUtility.py rename to nni/tools/trial_tool/hdfsClientUtility.py diff --git a/tools/nni_trial_tool/log_utils.py b/nni/tools/trial_tool/log_utils.py similarity index 100% rename from tools/nni_trial_tool/log_utils.py rename to nni/tools/trial_tool/log_utils.py diff --git a/tools/nni_trial_tool/rest_utils.py b/nni/tools/trial_tool/rest_utils.py similarity index 100% rename from tools/nni_trial_tool/rest_utils.py rename to nni/tools/trial_tool/rest_utils.py diff --git a/tools/nni_trial_tool/trial.py b/nni/tools/trial_tool/trial.py similarity index 91% rename from tools/nni_trial_tool/trial.py rename to nni/tools/trial_tool/trial.py index 6f81b2427e..037b210cb0 100644 --- a/tools/nni_trial_tool/trial.py +++ b/nni/tools/trial_tool/trial.py @@ -137,10 +137,15 @@ def is_running(self): def kill(self, trial_id=None): if trial_id == self.id or trial_id is None: if self.process is not None: - nni_log(LogType.Info, "%s: killing trial" % self.name) - for child in psutil.Process(self.process.pid).children(True): - child.kill() - self.process.kill() + try: + nni_log(LogType.Info, "%s: killing trial" % self.name) + for child in psutil.Process(self.process.pid).children(True): + child.kill() + self.process.kill() + except psutil.NoSuchProcess: + nni_log(LogType.Info, "kill trial %s failed: %s does not exist!" % (trial_id, self.process.pid)) + except Exception as ex: + nni_log(LogType.Error, "kill trial %s failed: %s " % (trial_id, str(ex))) self.cleanup() def cleanup(self): diff --git a/tools/nni_trial_tool/trial_keeper.py b/nni/tools/trial_tool/trial_keeper.py similarity index 100% rename from tools/nni_trial_tool/trial_keeper.py rename to nni/tools/trial_tool/trial_keeper.py diff --git a/tools/nni_trial_tool/trial_runner.py b/nni/tools/trial_tool/trial_runner.py similarity index 100% rename from tools/nni_trial_tool/trial_runner.py rename to nni/tools/trial_tool/trial_runner.py diff --git a/tools/nni_trial_tool/url_utils.py b/nni/tools/trial_tool/url_utils.py similarity index 100% rename from tools/nni_trial_tool/url_utils.py rename to nni/tools/trial_tool/url_utils.py diff --git a/tools/nni_trial_tool/web_channel.py b/nni/tools/trial_tool/web_channel.py similarity index 100% rename from tools/nni_trial_tool/web_channel.py rename to nni/tools/trial_tool/web_channel.py diff --git a/src/sdk/pynni/nni/trial.py b/nni/trial.py similarity index 98% rename from src/sdk/pynni/nni/trial.py rename to nni/trial.py index d4851316f5..cdb2b1e683 100644 --- a/src/sdk/pynni/nni/trial.py +++ b/nni/trial.py @@ -2,8 +2,8 @@ # Licensed under the MIT license. from .utils import to_json -from .env_vars import trial_env_vars -from . import platform +from .runtime.env_vars import trial_env_vars +from .runtime import platform __all__ = [ diff --git a/src/sdk/pynni/nni/tuner.py b/nni/tuner.py similarity index 93% rename from src/sdk/pynni/nni/tuner.py rename to nni/tuner.py index 0f8bb1e352..4fbcc011d0 100644 --- a/src/sdk/pynni/nni/tuner.py +++ b/nni/tuner.py @@ -57,14 +57,14 @@ class Tuner(Recoverable): See Also -------- Builtin tuners: - :class:`~nni.hyperopt_tuner.hyperopt_tuner.HyperoptTuner` - :class:`~nni.evolution_tuner.evolution_tuner.EvolutionTuner` - :class:`~nni.smac_tuner.SMACTuner` - :class:`~nni.gridsearch_tuner.GridSearchTuner` - :class:`~nni.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner` - :class:`~nni.metis_tuner.mets_tuner.MetisTuner` - :class:`~nni.ppo_tuner.PPOTuner` - :class:`~nni.gp_tuner.gp_tuner.GPTuner` + :class:`~nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner` + :class:`~nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionTuner` + :class:`~nni.algorithms.hpo.smac_tuner.SMACTuner` + :class:`~nni.algorithms.hpo.gridsearch_tuner.GridSearchTuner` + :class:`~nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner` + :class:`~nni.algorithms.hpo.metis_tuner.mets_tuner.MetisTuner` + :class:`~nni.algorithms.hpo.ppo_tuner.PPOTuner` + :class:`~nni.algorithms.hpo.gp_tuner.gp_tuner.GPTuner` """ def generate_parameters(self, parameter_id, **kwargs): diff --git a/src/sdk/pynni/nni/utils.py b/nni/utils.py similarity index 99% rename from src/sdk/pynni/nni/utils.py rename to nni/utils.py index 0346aec9fc..fe782dae2c 100644 --- a/src/sdk/pynni/nni/utils.py +++ b/nni/utils.py @@ -9,8 +9,8 @@ from schema import And from . import parameter_expressions -from .common import init_logger -from .env_vars import dispatcher_env_vars +from .runtime.common import init_logger +from .runtime.env_vars import dispatcher_env_vars to_json = functools.partial(json_tricks.dumps, allow_nan=True) diff --git a/nni_node/__init__.py b/nni_node/__init__.py new file mode 100644 index 0000000000..4db8f4050b --- /dev/null +++ b/nni_node/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +TypeScript modules of NNI. + +As a python package this only contains "package data". +""" diff --git a/setup.py b/setup.py index 99f183e80d..0011d0b20e 100644 --- a/setup.py +++ b/setup.py @@ -1,55 +1,202 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +""" +Script for installation and distribution. + +You can use environment variable `NNI_RELEASE` to set release version. + +If release version is not set, default to a development build whose version string will be `999.dev0`. + + +## Development ## + +Build and install for development: + + $ python setup.py develop + +Uninstall: + + $ pip uninstall nni + +Remove generated files: (use "--all" to remove toolchain and built wheel) + + $ python setup.py clean [--all] + +Build TypeScript modules without install: + + $ python setup.py build_ts + + +## Release ## + +Build wheel package: + + $ NNI_RELEASE=2.0 python setup.py build_ts + $ NNI_RELEASE=2.0 python setup.py bdist_wheel -p manylinux1_x86_64 + +Where "2.0" is version string and "manylinux1_x86_64" is platform. +The platform may also be "macosx_10_9_x86_64" or "win_amd64". + +`build_ts` must be manually invoked before `bdist_wheel`, +or setuptools cannot locate JS files which should be packed into wheel. +""" + +from distutils.cmd import Command +from distutils.command.build import build +from distutils.command.clean import clean +import glob import os -from setuptools import setup, find_packages - -def read(fname): - return open(os.path.join(os.path.dirname(__file__), fname), encoding='utf-8').read() - -setup( - name = 'nni', - version = '999.0.0-developing', - author = 'Microsoft NNI Team', - author_email = 'nni@microsoft.com', - description = 'Neural Network Intelligence project', - long_description = read('README.md'), - license = 'MIT', - url = 'https://github.com/Microsoft/nni', - - packages = find_packages('src/sdk/pynni', exclude=['tests']) + find_packages('src/sdk/pycli') + find_packages('tools'), - package_dir = { - 'nni': 'src/sdk/pynni/nni', - 'nnicli': 'src/sdk/pycli/nnicli', - 'nni_annotation': 'tools/nni_annotation', - 'nni_cmd': 'tools/nni_cmd', - 'nni_trial_tool':'tools/nni_trial_tool', - 'nni_gpu_tool':'tools/nni_gpu_tool' - }, - package_data = {'nni': ['**/requirements.txt']}, - python_requires = '>=3.6', - install_requires = [ - 'astor', - 'hyperopt==0.1.2', - 'json_tricks', - 'netifaces', - 'numpy', - 'psutil', - 'ruamel.yaml', - 'requests', - 'responses', - 'scipy', - 'schema', - 'PythonWebHDFS', - 'colorama', - 'scikit-learn>=0.23.2', - 'pkginfo', - 'websockets' - ], - - entry_points = { - 'console_scripts' : [ - 'nnictl = nni_cmd.nnictl:parse_args' - ] - } -) +import shutil + +import setuptools +from setuptools.command.develop import develop + +import setup_ts + + +dependencies = [ + 'astor', + 'hyperopt==0.1.2', + 'json_tricks', + 'netifaces', + 'numpy', + 'psutil', + 'ruamel.yaml', + 'requests', + 'responses', + 'scipy', + 'schema', + 'PythonWebHDFS', + 'colorama', + 'scikit-learn>=0.23.2', + 'pkginfo', + 'websockets' +] + + +release = os.environ.get('NNI_RELEASE') + +def _setup(): + setuptools.setup( + name = 'nni', + version = release or '999.dev0', + description = 'Neural Network Intelligence project', + long_description = open('README.md', encoding='utf-8').read(), + long_description_content_type = 'text/markdown', + url = 'https://github.com/Microsoft/nni', + author = 'Microsoft NNI Team', + author_email = 'nni@microsoft.com', + license = 'MIT', + classifiers = [ + 'License :: OSI Approved :: MIT License', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: Microsoft :: Windows :: Windows 10', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 3 :: Only', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + ], + + packages = _find_python_packages(), + package_data = { + 'nni': ['**/requirements.txt'], + 'nni_node': _find_node_files() # note: this does not work before building + }, + + python_requires = '>=3.6', + install_requires = dependencies, + setup_requires = ['requests'], + + entry_points = { + 'console_scripts' : [ + 'nnictl = nni.tools.nnictl.nnictl:parse_args' + ] + }, + + cmdclass = { + 'build': Build, + 'build_ts': BuildTs, + 'clean': Clean, + 'develop': Develop, + } + ) + + +def _find_python_packages(): + packages = [] + for dirpath, dirnames, filenames in os.walk('nni'): + if '/__pycache__' not in dirpath: + packages.append(dirpath.replace('/', '.')) + return sorted(packages) + ['nni_node'] + +def _find_node_files(): + files = [] + for dirpath, dirnames, filenames in os.walk('nni_node'): + for filename in filenames: + files.append((dirpath + '/' + filename)[len('nni_node/'):]) + files.remove('__init__.py') + return sorted(files) + + +class BuildTs(Command): + description = 'build TypeScript modules' + + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + setup_ts.build(release) + +class Build(build): + def run(self): + assert release, 'Please set environment variable "NNI_RELEASE="' + assert os.path.isfile('nni_node/main.js'), 'Please run "build_ts" before "build"' + assert not os.path.islink('nni_node/main.js'), 'This is a development build' + super().run() + +class Develop(develop): + def finalize_options(self): + self.user = True # always use `develop --user` + super().finalize_options() + + def run(self): + setup_ts.build(release=None) + super().run() + +class Clean(clean): + def finalize_options(self): + self._all = self.all + self.all = True # always use `clean --all` + super().finalize_options() + + def run(self): + super().run() + setup_ts.clean(self._all) + _clean_temp_files() + shutil.rmtree('nni.egg-info', ignore_errors=True) + if self._all: + shutil.rmtree('dist', ignore_errors=True) + + +def _clean_temp_files(): + for pattern in _temp_files: + for path in glob.glob(pattern): + if os.path.islink(path) or os.path.isfile(path): + os.remove(path) + else: + shutil.rmtree(path) + +_temp_files = [ + # unit test + 'test/model_path/', + 'test/temp.json', + 'test/ut/sdk/*.pth' +] + + +_setup() diff --git a/setup_ts.py b/setup_ts.py new file mode 100644 index 0000000000..5872ce7bbd --- /dev/null +++ b/setup_ts.py @@ -0,0 +1,229 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Script for building TypeScript modules. +This script is called by `setup.py` and common users should avoid using this directly. + +It compiles TypeScript source files in `ts` directory, +and copies (or links) JavaScript output as well as dependencies to `nni_node`. + +You can set environment `GLOBAL_TOOLCHAIN=1` to use global node and yarn, if you know what you are doing. +""" + +from io import BytesIO +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tarfile +from zipfile import ZipFile + + +node_version = 'v10.22.1' +yarn_version = 'v1.22.10' + + +def build(release): + """ + Compile TypeScript modules and copy or symlink to nni_node directory. + + `release` is the version number without leading letter "v". + + If `release` is None or empty, this is a development build and uses symlinks; + otherwise this is a release build and copies files instead. + """ + if release or not os.environ.get('GLOBAL_TOOLCHAIN'): + download_toolchain() + compile_ts() + if release: + copy_nni_node(release) + else: + symlink_nni_node() + +def clean(clean_all=False): + """ + Remove TypeScript-related intermediate files. + Python intermediate files are not touched here. + """ + clear_nni_node() + for path in generated_directories: + shutil.rmtree(path, ignore_errors=True) + if clean_all: + shutil.rmtree('toolchain', ignore_errors=True) + Path('nni_node', node_executable).unlink() + + +if sys.platform == 'linux' or sys.platform == 'darwin': + node_executable = 'node' + node_spec = f'node-{node_version}-{sys.platform}-x64' + node_download_url = f'https://nodejs.org/dist/latest-v10.x/{node_spec}.tar.xz' + node_extractor = lambda data: tarfile.open(fileobj=BytesIO(data), mode='r:xz') + node_executable_in_tarball = 'bin/node' + +elif sys.platform == 'win32': + node_executable = 'node.exe' + node_spec = f'node-{node_version}-win-x64' + node_download_url = f'https://nodejs.org/dist/latest-v10.x/{node_spec}.zip' + node_extractor = lambda data: ZipFile(BytesIO(data)) + node_executable_in_tarball = 'node.exe' + +else: + raise RuntimeError('Unsupported system') + +yarn_executable = 'yarn' if sys.platform != 'win32' else 'yarn.cmd' +yarn_download_url = f'https://github.com/yarnpkg/yarn/releases/download/{yarn_version}/yarn-{yarn_version}.tar.gz' + + +def download_toolchain(): + """ + Download and extract node and yarn, + then copy node executable to nni_node directory. + """ + if Path('nni_node', node_executable).is_file(): + return + Path('toolchain').mkdir(exist_ok=True) + import requests # place it here so setup.py can install it before importing + + _print(f'Downloading node.js from {node_download_url}') + resp = requests.get(node_download_url) + resp.raise_for_status() + _print('Extracting node.js') + tarball = node_extractor(resp.content) + tarball.extractall('toolchain') + shutil.rmtree('toolchain/node', ignore_errors=True) + Path('toolchain', node_spec).rename('toolchain/node') + + _print(f'Downloading yarn from {yarn_download_url}') + resp = requests.get(yarn_download_url) + resp.raise_for_status() + _print('Extracting yarn') + tarball = tarfile.open(fileobj=BytesIO(resp.content), mode='r:gz') + tarball.extractall('toolchain') + shutil.rmtree('toolchain/yarn', ignore_errors=True) + Path(f'toolchain/yarn-{yarn_version}').rename('toolchain/yarn') + + src = Path('toolchain/node', node_executable_in_tarball) + dst = Path('nni_node', node_executable) + shutil.copyfile(src, dst) + + +def compile_ts(): + """ + Use yarn to download dependencies and compile TypeScript code. + """ + _print('Building NNI manager') + _yarn('ts/nni_manager') + _yarn('ts/nni_manager', 'build') + # todo: I don't think these should be here + shutil.rmtree('ts/nni_manager/dist/config', ignore_errors=True) + shutil.copytree('ts/nni_manager/config', 'ts/nni_manager/dist/config') + + _print('Building web UI') + _yarn('ts/webui') + _yarn('ts/webui', 'build') + + _print('Building NAS UI') + _yarn('ts/nasui') + _yarn('ts/nasui', 'build') + + +def symlink_nni_node(): + """ + Create symlinks to compiled JS files. + If you manually modify and compile TS source files you don't need to install again. + """ + _print('Creating symlinks') + clear_nni_node() + + for path in Path('ts/nni_manager/dist').iterdir(): + _symlink(path, Path('nni_node', path.name)) + _symlink('ts/nni_manager/package.json', 'nni_node/package.json') + _symlink('ts/nni_manager/node_modules', 'nni_node/node_modules') + + _symlink('ts/webui/build', 'nni_node/static') + + Path('nni_node/nasui').mkdir(exist_ok=True) + _symlink('ts/nasui/build', 'nni_node/nasui/build') + _symlink('ts/nasui/server.js', 'nni_node/nasui/server.js') + + +def copy_nni_node(version): + """ + Copy compiled JS files to nni_node. + This is meant for building release package, so you need to provide version string. + The version will written to `package.json` in nni_node directory, + while `package.json` in ts directory will be left unchanged. + """ + _print('Copying files') + clear_nni_node() + + # copytree(..., dirs_exist_ok=True) is not supported by Python 3.6 + for path in Path('ts/nni_manager/dist').iterdir(): + if path.is_file(): + shutil.copyfile(path, Path('nni_node', path.name)) + else: + shutil.copytree(path, Path('nni_node', path.name)) + + package_json = json.load(open('ts/nni_manager/package.json')) + if version.count('.') == 1: # node.js semver requires at least three parts + version = version + '.0' + package_json['version'] = version + json.dump(package_json, open('nni_node/package.json', 'w'), indent=2) + + _yarn('ts/nni_manager', '--prod', '--cwd', str(Path('nni_node').resolve())) + + shutil.copytree('ts/webui/build', 'nni_node/static') + + Path('nni_node/nasui').mkdir(exist_ok=True) + shutil.copytree('ts/nasui/build', 'nni_node/nasui/build') + shutil.copyfile('ts/nasui/server.js', 'nni_node/nasui/server.js') + + +def clear_nni_node(): + """ + Remove compiled files in nni_node. + Use `clean()` if you what to remove files in ts as well. + """ + for path in Path('nni_node').iterdir(): + if path.name not in ('__init__.py', 'node', 'node.exe'): + if path.is_symlink() or path.is_file(): + path.unlink() + else: + shutil.rmtree(path) + + +_yarn_env = dict(os.environ) +_yarn_env['PATH'] = str(Path('nni_node').resolve()) + ':' + os.environ['PATH'] +_yarn_path = Path('toolchain/yarn/bin', yarn_executable).resolve() + +def _yarn(path, *args): + if os.environ.get('GLOBAL_TOOLCHAIN'): + subprocess.run(['yarn', *args], cwd=path, check=True) + else: + subprocess.run([_yarn_path, *args], cwd=path, check=True, env=_yarn_env) + + +def _symlink(target_file, link_location): + target = Path(target_file) + link = Path(link_location) + relative = os.path.relpath(target, link.parent) + link.symlink_to(relative, target.is_dir()) + + +def _print(*args): + print('\033[1;36m# ', end='') + print(*args, end='') + print('\033[0m') + + +generated_directories = [ + 'ts/nni_manager/dist', + 'ts/nni_manager/node_modules', + 'ts/webui/build', + 'ts/webui/node_modules', + 'ts/nasui/build', + 'ts/nasui/node_modules', +] diff --git a/src/sdk/pynni/nni/constants.py b/src/sdk/pynni/nni/constants.py deleted file mode 100644 index 3e98f6eb6f..0000000000 --- a/src/sdk/pynni/nni/constants.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -BuiltinAlgorithms = { - 'tuners': [ - { - 'name': 'TPE', - 'class_name': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', - 'class_args': { - 'algorithm_name': 'tpe' - }, - 'class_args_validator': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' - }, - { - 'name': 'Random', - 'class_name': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', - 'class_args': { - 'algorithm_name': 'random_search' - }, - 'accept_class_args': False, - 'class_args_validator': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' - }, - { - 'name': 'Anneal', - 'class_name': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', - 'class_args': { - 'algorithm_name': 'anneal' - }, - 'class_args_validator': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' - }, - { - 'name': 'Evolution', - 'class_name': 'nni.evolution_tuner.evolution_tuner.EvolutionTuner', - 'class_args_validator': 'nni.evolution_tuner.evolution_tuner.EvolutionClassArgsValidator' - }, - { - 'name': 'BatchTuner', - 'class_name': 'nni.batch_tuner.batch_tuner.BatchTuner', - 'accept_class_args': False, - }, - { - 'name': 'GridSearch', - 'class_name': 'nni.gridsearch_tuner.gridsearch_tuner.GridSearchTuner', - 'accept_class_args': False, - }, - { - 'name': 'NetworkMorphism', - 'class_name': 'nni.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner', - 'class_args_validator': 'nni.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismClassArgsValidator' - }, - { - 'name': 'MetisTuner', - 'class_name': 'nni.metis_tuner.metis_tuner.MetisTuner', - 'class_args_validator': 'nni.metis_tuner.metis_tuner.MetisClassArgsValidator' - }, - { - 'name': 'GPTuner', - 'class_name': 'nni.gp_tuner.gp_tuner.GPTuner', - 'class_args_validator': 'nni.gp_tuner.gp_tuner.GPClassArgsValidator' - }, - { - 'name': 'PBTTuner', - 'class_name': 'nni.pbt_tuner.pbt_tuner.PBTTuner', - 'class_args_validator': 'nni.pbt_tuner.pbt_tuner.PBTClassArgsValidator' - }, - { - 'name': 'RegularizedEvolutionTuner', - 'class_name': 'nni.regularized_evolution_tuner.regularized_evolution_tuner.RegularizedEvolutionTuner', - 'class_args_validator': 'nni.regularized_evolution_tuner.regularized_evolution_tuner.EvolutionClassArgsValidator' - } - ], - 'assessors': [ - { - 'name': 'Medianstop', - 'class_name': 'nni.medianstop_assessor.medianstop_assessor.MedianstopAssessor', - 'class_args_validator': 'nni.medianstop_assessor.medianstop_assessor.MedianstopClassArgsValidator' - }, - { - 'name': 'Curvefitting', - 'class_name': 'nni.curvefitting_assessor.curvefitting_assessor.CurvefittingAssessor', - 'class_args_validator': 'nni.curvefitting_assessor.curvefitting_assessor.CurvefittingClassArgsValidator' - }, - ], - 'advisors': [ - { - 'name': 'Hyperband', - 'class_name': 'nni.hyperband_advisor.hyperband_advisor.Hyperband', - 'class_args_validator': 'nni.hyperband_advisor.hyperband_advisor.HyperbandClassArgsValidator' - } - ] -} diff --git a/src/sdk/pynni/nni/nas/tensorflow/enas/trainer.py b/src/sdk/pynni/nni/nas/tensorflow/enas/trainer.py deleted file mode 100644 index a9645e9203..0000000000 --- a/src/sdk/pynni/nni/nas/tensorflow/enas/trainer.py +++ /dev/null @@ -1,158 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import logging - -import tensorflow as tf -from tensorflow.keras.optimizers import Adam - -from nni.nas.tensorflow.utils import AverageMeterGroup, fill_zero_grads - -from .mutator import EnasMutator - -logger = logging.getLogger(__name__) - - -log_frequency = 100 -entropy_weight = 0.0001 -skip_weight = 0.8 -baseline_decay = 0.999 -child_steps = 500 -mutator_lr = 0.00035 -mutator_steps = 50 -mutator_steps_aggregate = 20 -aux_weight = 0.4 -test_arc_per_epoch = 1 - - -class EnasTrainer: - def __init__(self, model, loss, metrics, reward_function, optimizer, batch_size, num_epochs, - dataset_train, dataset_valid): - self.model = model - self.loss = loss - self.metrics = metrics - self.reward_function = reward_function - self.optimizer = optimizer - self.batch_size = batch_size - self.num_epochs = num_epochs - - x, y = dataset_train - split = int(len(x) * 0.9) - self.train_set = tf.data.Dataset.from_tensor_slices((x[:split], y[:split])) - self.valid_set = tf.data.Dataset.from_tensor_slices((x[split:], y[split:])) - self.test_set = tf.data.Dataset.from_tensor_slices(dataset_valid) - - self.mutator = EnasMutator(model) - self.mutator_optim = Adam(learning_rate=mutator_lr) - - self.baseline = 0. - - - def train(self, validate=True): - for epoch in range(self.num_epochs): - logger.info("Epoch %d Training", epoch + 1) - self.train_one_epoch(epoch) - logger.info("Epoch %d Validating", epoch + 1) - self.validate_one_epoch(epoch) - - def validate(self): - self.validate_one_epoch(-1) - - - def train_one_epoch(self, epoch): - train_loader, valid_loader = self._create_train_loader() - - # Sample model and train - meters = AverageMeterGroup() - - for step in range(1, child_steps + 1): - x, y = next(train_loader) - self.mutator.reset() - - with tf.GradientTape() as tape: - logits = self.model(x, training=True) - if isinstance(logits, tuple): - logits, aux_logits = logits - aux_loss = self.loss(aux_logits, y) - else: - aux_loss = 0. - metrics = self.metrics(y, logits) - loss = self.loss(y, logits) + aux_weight * aux_loss - - grads = tape.gradient(loss, self.model.trainable_weights) - grads = fill_zero_grads(grads, self.model.trainable_weights) - grads, _ = tf.clip_by_global_norm(grads, 5.0) - self.optimizer.apply_gradients(zip(grads, self.model.trainable_weights)) - - metrics['loss'] = tf.reduce_mean(loss).numpy() - meters.update(metrics) - - if log_frequency and step % log_frequency == 0: - logger.info("Model Epoch [%d/%d] Step [%d/%d] %s", epoch + 1, - self.num_epochs, step, child_steps, meters) - - # Train sampler (mutator) - meters = AverageMeterGroup() - for mutator_step in range(1, mutator_steps + 1): - grads_list = [] - for step in range(1, mutator_steps_aggregate + 1): - with tf.GradientTape() as tape: - x, y = next(valid_loader) - self.mutator.reset() - - logits = self.model(x, training=False) - metrics = self.metrics(y, logits) - reward = self.reward_function(y, logits) + entropy_weight * self.mutator.sample_entropy - self.baseline = self.baseline * baseline_decay + reward * (1 - baseline_decay) - loss = self.mutator.sample_log_prob * (reward - self.baseline) - loss += skip_weight * self.mutator.sample_skip_penalty - - meters.update({ - 'reward': reward, - 'loss': tf.reduce_mean(loss).numpy(), - 'ent': self.mutator.sample_entropy.numpy(), - 'log_prob': self.mutator.sample_log_prob.numpy(), - 'baseline': self.baseline, - 'skip': self.mutator.sample_skip_penalty, - }) - - cur_step = step + (mutator_step - 1) * mutator_steps_aggregate - if log_frequency and cur_step % log_frequency == 0: - logger.info("RL Epoch [%d/%d] Step [%d/%d] [%d/%d] %s", epoch + 1, self.num_epochs, - mutator_step, mutator_steps, step, mutator_steps_aggregate, - meters) - - grads = tape.gradient(loss, self.mutator.trainable_weights) - grads = fill_zero_grads(grads, self.mutator.trainable_weights) - grads_list.append(grads) - total_grads = [tf.math.add_n(weight_grads) for weight_grads in zip(*grads_list)] - total_grads, _ = tf.clip_by_global_norm(total_grads, 5.0) - self.mutator_optim.apply_gradients(zip(total_grads, self.mutator.trainable_weights)) - - def validate_one_epoch(self, epoch): - test_loader = self._create_validate_loader() - - for arc_id in range(test_arc_per_epoch): - meters = AverageMeterGroup() - for x, y in test_loader: - self.mutator.reset() - logits = self.model(x, training=False) - if isinstance(logits, tuple): - logits, _ = logits - metrics = self.metrics(y, logits) - loss = self.loss(y, logits) - metrics['loss'] = tf.reduce_mean(loss).numpy() - meters.update(metrics) - - logger.info("Test Epoch [%d/%d] Arc [%d/%d] Summary %s", - epoch + 1, self.num_epochs, arc_id + 1, test_arc_per_epoch, - meters.summary()) - - - def _create_train_loader(self): - train_set = self.train_set.shuffle(1000000).repeat().batch(self.batch_size) - test_set = self.valid_set.shuffle(1000000).repeat().batch(self.batch_size) - return iter(train_set), iter(test_set) - - def _create_validate_loader(self): - return iter(self.test_set.shuffle(1000000).batch(self.batch_size)) diff --git a/src/webui/src/components/Overview.tsx b/src/webui/src/components/Overview.tsx deleted file mode 100644 index a875d2efaa..0000000000 --- a/src/webui/src/components/Overview.tsx +++ /dev/null @@ -1,238 +0,0 @@ -import * as React from 'react'; -import { Stack, IStackTokens, Dropdown } from '@fluentui/react'; -import { EXPERIMENT, TRIALS } from '../static/datamodel'; -import { Trial } from '../static/model/trial'; -import { AppContext } from '../App'; -import { Title1 } from './overview/Title1'; -import SuccessTable from './overview/SuccessTable'; -import Progressed from './overview/Progress'; -import Accuracy from './overview/Accuracy'; -import SearchSpace from './overview/SearchSpace'; -import { BasicInfo } from './overview/BasicInfo'; -import TrialInfo from './overview/TrialProfile'; -import '../static/style/overview.scss'; -import '../static/style/logPath.scss'; - -const stackTokens: IStackTokens = { - childrenGap: 30 -}; - -const entriesOption = [ - { key: '10', text: 'Display top 10 trials' }, - { key: '20', text: 'Display top 20 trials' }, - { key: '30', text: 'Display top 30 trials' }, - { key: '50', text: 'Display top 50 trials' }, - { key: '100', text: 'Display top 100 trials' } -]; - -interface OverviewState { - trialConcurrency: number; -} - -export const TitleContext = React.createContext({ - text: '', - icon: '', - fontColor: '' -}); - -class Overview extends React.Component<{}, OverviewState> { - static contextType = AppContext; - - constructor(props) { - super(props); - this.state = { - trialConcurrency: EXPERIMENT.trialConcurrency - }; - } - - clickMaxTop = (event: React.SyntheticEvent): void => { - event.stopPropagation(); - // #999 panel active bgcolor; #b3b3b3 as usual - const { changeMetricGraphMode } = this.context; - changeMetricGraphMode('max'); - }; - - clickMinTop = (event: React.SyntheticEvent): void => { - event.stopPropagation(); - const { changeMetricGraphMode } = this.context; - changeMetricGraphMode('min'); - }; - - changeConcurrency = (val: number): void => { - this.setState({ trialConcurrency: val }); - }; - - // updateEntries = (event: React.FormEvent, item: IDropdownOption | undefined): void => { - updateEntries = (event: React.FormEvent, item: any): void => { - if (item !== undefined) { - this.context.changeEntries(item.key); - } - }; - - render(): React.ReactNode { - const { trialConcurrency } = this.state; - const bestTrials = this.findBestTrials(); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const bestAccuracy = bestTrials.length > 0 ? bestTrials[0].accuracy! : NaN; - const accuracyGraphData = this.generateAccuracyGraph(bestTrials); - const noDataMessage = bestTrials.length > 0 ? '' : 'No data'; - return ( - - {(value): React.ReactNode => { - const { experimentUpdateBroadcast, metricGraphMode, bestTrialEntries } = value; - const titleMaxbgcolor = metricGraphMode === 'max' ? '#333' : '#b3b3b3'; - const titleMinbgcolor = metricGraphMode === 'min' ? '#333' : '#b3b3b3'; - return ( -
      - {/* status and experiment block */} - - - - - - - - - {/* status block */} - - - - - - - {/* experiment parameters search space tuner assessor... */} - - - - - - - - - - - - - - {/* the scroll bar all the trial profile in the searchSpace div*/} -
      - -
      -       -       -       - - - -
      - - - -
      -
      - - - -
      -
      - -
      -       - -
      - -
      -
      - trial.info.id)} /> -
      -       -       -
      - ); - }} -       - ); - } - - private findBestTrials(): Trial[] { - const bestTrials = TRIALS.sort(); - const { bestTrialEntries, metricGraphMode } = this.context; - if (metricGraphMode === 'max') { - bestTrials.reverse().splice(JSON.parse(bestTrialEntries)); - } else { - bestTrials.splice(JSON.parse(bestTrialEntries)); - } - return bestTrials; - } - - private generateAccuracyGraph(bestTrials: Trial[]): object { - const xSequence = bestTrials.map(trial => trial.sequenceId); - const ySequence = bestTrials.map(trial => trial.accuracy); - - return { - // support max show 0.0000000 - grid: { - left: 67, - right: 40 - }, - tooltip: { - trigger: 'item' - }, - xAxis: { - name: 'Trial', - type: 'category', - data: xSequence - }, - yAxis: { - name: 'Default metric', - type: 'value', - scale: true, - data: ySequence - }, - series: [ - { - symbolSize: 6, - type: 'scatter', - data: ySequence - } - ] - }; - } -} - -export default Overview; diff --git a/src/webui/src/components/TrialsDetail.tsx b/src/webui/src/components/TrialsDetail.tsx deleted file mode 100644 index b22d6d2841..0000000000 --- a/src/webui/src/components/TrialsDetail.tsx +++ /dev/null @@ -1,208 +0,0 @@ -import * as React from 'react'; -import { Stack, StackItem, Pivot, PivotItem, Dropdown, IDropdownOption, DefaultButton } from '@fluentui/react'; -import { EXPERIMENT, TRIALS } from '../static/datamodel'; -import { Trial } from '../static/model/trial'; -import { AppContext } from '../App'; -import { tableListIcon } from './buttons/Icon'; -import DefaultPoint from './trial-detail/DefaultMetricPoint'; -import Duration from './trial-detail/Duration'; -import Para from './trial-detail/Para'; -import Intermediate from './trial-detail/Intermediate'; -import TableList from './trial-detail/TableList'; -import '../static/style/trialsDetail.scss'; -import '../static/style/search.scss'; - -const searchOptions = [ - { key: 'id', text: 'Id' }, - { key: 'Trial No.', text: 'Trial No.' }, - { key: 'status', text: 'Status' }, - { key: 'parameters', text: 'Parameters' } -]; - -interface TrialDetailState { - tablePageSize: number; // table components val - whichChart: string; - searchType: string; - searchFilter: (trial: Trial) => boolean; -} - -class TrialsDetail extends React.Component<{}, TrialDetailState> { - static contextType = AppContext; - public interAccuracy = 0; - public interAllTableList = 2; - - public tableList!: TableList | null; - public searchInput!: HTMLInputElement | null; - - constructor(props) { - super(props); - this.state = { - tablePageSize: 20, - whichChart: 'Default metric', - searchType: 'id', - // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/explicit-function-return-type - searchFilter: trial => true - }; - } - - // search a trial by trial No. | trial id | Parameters | Status - searchTrial = (event: React.ChangeEvent): void => { - const targetValue = event.target.value; - // eslint-disable-next-line @typescript-eslint/no-unused-vars - let filter = (trial: Trial): boolean => true; - if (!targetValue.trim()) { - this.setState({ searchFilter: filter }); - return; - } - switch (this.state.searchType) { - case 'id': - filter = (trial): boolean => trial.info.id.toUpperCase().includes(targetValue.toUpperCase()); - break; - case 'Trial No.': - filter = (trial): boolean => trial.info.sequenceId.toString() === targetValue; - break; - case 'status': - filter = (trial): boolean => trial.info.status.toUpperCase().includes(targetValue.toUpperCase()); - break; - case 'parameters': - // TODO: support filters like `x: 2` (instead of `"x": 2`) - filter = (trial): boolean => JSON.stringify(trial.info.hyperParameters, null, 4).includes(targetValue); - break; - default: - alert(`Unexpected search filter ${this.state.searchType}`); - } - this.setState({ searchFilter: filter }); - }; - - handleTablePageSizeSelect = (event: React.FormEvent, item: IDropdownOption | undefined): void => { - if (item !== undefined) { - this.setState({ tablePageSize: item.text === 'all' ? -1 : parseInt(item.text, 10) }); - } - }; - - handleWhichTabs = (item: any): void => { - this.setState({ whichChart: item.props.headerText }); - }; - - updateSearchFilterType = (event: React.FormEvent, item: IDropdownOption | undefined): void => { - // clear input value and re-render table - if (item !== undefined) { - if (this.searchInput !== null) { - this.searchInput.value = ''; - } - this.setState(() => ({ searchType: item.key.toString() })); - } - }; - - render(): React.ReactNode { - const { tablePageSize, whichChart, searchType } = this.state; - const source = TRIALS.filter(this.state.searchFilter); - const trialIds = TRIALS.filter(this.state.searchFilter).map(trial => trial.id); - - return ( - - {(value): React.ReactNode => ( - -
      - - {/* doesn't work*/} - - - - - - {/* */} - - - - - - {/* */} - - - - {/* */} - - {/* *why this graph has small footprint? */} - - - -
      - {/* trial table list */} -
      - - {tableListIcon} - Trial jobs - - - - { - if (this.tableList) { - this.tableList.compareBtn(); - } - }} - /> - - - - { - if (this.tableList) { - this.tableList.addColumn(); - } - }} - /> - - (this.searchInput = text)} - /> - - - - trial.tableRecord)} - columnList={value.columnList} - changeColumn={value.changeColumn} - trialsUpdateBroadcast={this.context.trialsUpdateBroadcast} - // TODO: change any to specific type - ref={(tabList): any => (this.tableList = tabList)} - /> -
      -       - )} -       - ); - } -} - -export default TrialsDetail; diff --git a/src/webui/src/components/modals/ChangeColumnComponent.tsx b/src/webui/src/components/modals/ChangeColumnComponent.tsx deleted file mode 100644 index a4691b8d40..0000000000 --- a/src/webui/src/components/modals/ChangeColumnComponent.tsx +++ /dev/null @@ -1,153 +0,0 @@ -import * as React from 'react'; -import { Dialog, DialogType, DialogFooter, Checkbox, PrimaryButton, DefaultButton } from '@fluentui/react'; -import { OPERATION } from '../../static/const'; - -interface ChangeColumnState { - userSelectColumnList: string[]; - originSelectColumnList: string[]; -} - -interface ChangeColumnProps { - isHideDialog: boolean; - showColumn: string[]; // all column List - selectedColumn: string[]; // user selected column list - changeColumn: (val: string[]) => void; - hideShowColumnDialog: () => void; -} - -interface CheckBoxItems { - label: string; - checked: boolean; - onChange: () => void; -} -class ChangeColumnComponent extends React.Component { - constructor(props: ChangeColumnProps) { - super(props); - this.state = { - userSelectColumnList: this.props.selectedColumn, - originSelectColumnList: this.props.selectedColumn - }; - } - - makeChangeHandler = (label: string): any => { - return (ev: any, checked: boolean): void => this.onCheckboxChange(ev, label, checked); - }; - - onCheckboxChange = ( - ev: React.FormEvent | undefined, - label: string, - val?: boolean - ): void => { - const source: string[] = JSON.parse(JSON.stringify(this.state.userSelectColumnList)); - if (val === true) { - if (!source.includes(label)) { - source.push(label); - this.setState(() => ({ userSelectColumnList: source })); - } - } else { - if (source.includes(label)) { - // remove from source - const result = source.filter(item => item !== label); - this.setState(() => ({ userSelectColumnList: result })); - } - } - }; - - saveUserSelectColumn = (): void => { - const { userSelectColumnList } = this.state; - const { showColumn } = this.props; - // sort by Trial No. | ID | Duration | Start Time | End Time | ... - const sortColumn: string[] = []; - /** - * - * TODO: use this function to refactor sort column - * search space might orderless - showColumn.map(item => { - userSelectColumnList.map(key => { - if (item === key || key.includes('search space')) { - if (!sortColumn.includes(key)) { - sortColumn.push(key); - } - } - }); - }); - */ - // push ![Operation] ![search space] column - showColumn.map(item => { - userSelectColumnList.map(key => { - if (item === key && item !== OPERATION) { - sortColumn.push(key); - } - }); - }); - // push search space key - userSelectColumnList.map(index => { - if (index.includes('search space')) { - if (!sortColumn.includes(index)) { - sortColumn.push(index); - } - } - }); - // push Operation - if (userSelectColumnList.includes(OPERATION)) { - sortColumn.push(OPERATION); - } - this.props.changeColumn(sortColumn); - this.hideDialog(); // hide dialog - }; - - hideDialog = (): void => { - this.props.hideShowColumnDialog(); - }; - - // user exit dialog - cancelOption = (): void => { - // reset select column - const { originSelectColumnList } = this.state; - this.setState({ userSelectColumnList: originSelectColumnList }, () => { - this.hideDialog(); - }); - }; - - render(): React.ReactNode { - const { showColumn, isHideDialog } = this.props; - const { userSelectColumnList } = this.state; - const renderOptions: Array = []; - showColumn.map(item => { - if (userSelectColumnList.includes(item)) { - // selected column name - renderOptions.push({ label: item, checked: true, onChange: this.makeChangeHandler(item) }); - } else { - renderOptions.push({ label: item, checked: false, onChange: this.makeChangeHandler(item) }); - } - }); - return ( -
      - -
      - ); - } -} - -export default ChangeColumnComponent; diff --git a/src/webui/src/components/modals/Compare.tsx b/src/webui/src/components/modals/Compare.tsx deleted file mode 100644 index daed24526c..0000000000 --- a/src/webui/src/components/modals/Compare.tsx +++ /dev/null @@ -1,254 +0,0 @@ -import * as React from 'react'; -import { Stack, Modal, IconButton, IDragOptions, ContextualMenu } from '@fluentui/react'; -import ReactEcharts from 'echarts-for-react'; -import IntermediateVal from '../public-child/IntermediateVal'; -import { TRIALS } from '../../static/datamodel'; -import { TableRecord, Intermedia, TooltipForIntermediate } from '../../static/interface'; -import { contentStyles, iconButtonStyles } from '../buttons/ModalTheme'; -import '../../static/style/compare.scss'; - -const dragOptions: IDragOptions = { - moveMenuItemText: 'Move', - closeMenuItemText: 'Close', - menu: ContextualMenu -}; - -// the modal of trial compare -interface CompareProps { - compareStacks: Array; - cancelFunc: () => void; -} - -class Compare extends React.Component { - public _isCompareMount!: boolean; - constructor(props: CompareProps) { - super(props); - } - - intermediate = (): React.ReactNode => { - const { compareStacks } = this.props; - const trialIntermediate: Array = []; - const idsList: string[] = []; - compareStacks.forEach(element => { - const trial = TRIALS.getTrial(element.id); - trialIntermediate.push({ - name: element.id, - data: trial.description.intermediate, - type: 'line', - hyperPara: trial.description.parameters - }); - idsList.push(element.id); - }); - // find max intermediate number - trialIntermediate.sort((a, b) => { - return b.data.length - a.data.length; - }); - const legend: string[] = []; - // max length - const length = trialIntermediate[0] !== undefined ? trialIntermediate[0].data.length : 0; - const xAxis: number[] = []; - trialIntermediate.forEach(element => { - legend.push(element.name); - }); - for (let i = 1; i <= length; i++) { - xAxis.push(i); - } - const option = { - tooltip: { - trigger: 'item', - enterable: true, - position: function(point: number[], data: TooltipForIntermediate): number[] { - if (data.dataIndex < length / 2) { - return [point[0], 80]; - } else { - return [point[0] - 300, 80]; - } - }, - formatter: function(data: TooltipForIntermediate): React.ReactNode { - const trialId = data.seriesName; - let obj = {}; - const temp = trialIntermediate.find(key => key.name === trialId); - if (temp !== undefined) { - obj = temp.hyperPara; - } - return ( - '
      ' + - '
      Trial ID: ' + - trialId + - '
      ' + - '
      Intermediate: ' + - data.data + - '
      ' + - '
      Parameters: ' + - '
      ' +
-                        JSON.stringify(obj, null, 4) +
-                        '
      ' + - '
      ' + - '
      ' - ); - } - }, - grid: { - left: '5%', - top: 40, - containLabel: true - }, - legend: { - type: 'scroll', - right: 40, - left: idsList.length > 6 ? 80 : null, - data: idsList - }, - xAxis: { - type: 'category', - // name: '# Intermediate', - boundaryGap: false, - data: xAxis - }, - yAxis: { - type: 'value', - name: 'Metric', - scale: true - }, - series: trialIntermediate - }; - return ( - - ); - }; - - // render table column --- - initColumn = (): React.ReactNode => { - const idList: string[] = []; - const sequenceIdList: number[] = []; - const durationList: number[] = []; - - const compareStacks = this.props.compareStacks.map(tableRecord => TRIALS.getTrial(tableRecord.id)); - - const parameterList: Array = []; - let parameterKeys: string[] = []; - if (compareStacks.length !== 0) { - parameterKeys = Object.keys(compareStacks[0].description.parameters); - } - compareStacks.forEach(temp => { - idList.push(temp.id); - sequenceIdList.push(temp.sequenceId); - durationList.push(temp.duration); - parameterList.push(temp.description.parameters); - }); - let isComplexSearchSpace; - if (parameterList.length > 0) { - isComplexSearchSpace = typeof parameterList[0][parameterKeys[0]] === 'object' ? true : false; - } - const width = this.getWebUIWidth(); - let scrollClass; - if (width > 1200) { - scrollClass = idList.length > 3 ? 'flex' : ''; - } else if (width < 700) { - scrollClass = idList.length > 1 ? 'flex' : ''; - } else { - scrollClass = idList.length > 2 ? 'flex' : ''; - } - return ( - - - - - {Object.keys(idList).map(key => ( - - ))} - - - - {Object.keys(sequenceIdList).map(key => ( - - ))} - - - - {Object.keys(compareStacks).map(index => ( - - ))} - - - - {Object.keys(durationList).map(index => ( - - ))} - - {isComplexSearchSpace - ? null - : Object.keys(parameterKeys).map(index => ( - - - {Object.keys(parameterList).map(key => ( - - ))} - - ))} - -
      Id - {idList[key]} -
      Trial No. - {sequenceIdList[key]} -
      Default metric - -
      duration - {durationList[index]} -
      - {parameterKeys[index]} - - {parameterList[key][parameterKeys[index]]} -
      - ); - }; - - getWebUIWidth = (): number => { - return window.innerWidth; - }; - - componentDidMount(): void { - this._isCompareMount = true; - } - - componentWillUnmount(): void { - this._isCompareMount = false; - } - - render(): React.ReactNode { - const { cancelFunc } = this.props; - - return ( - -
      -
      - Compare trials - -
      - - {this.intermediate()} - # Intermediate result - - {this.initColumn()} -
      -       - ); - } -} - -export default Compare; diff --git a/src/webui/src/components/overview/BasicInfo.tsx b/src/webui/src/components/overview/BasicInfo.tsx deleted file mode 100644 index 6ef6505191..0000000000 --- a/src/webui/src/components/overview/BasicInfo.tsx +++ /dev/null @@ -1,43 +0,0 @@ -import React from 'react'; -import { Stack, TooltipHost } from '@fluentui/react'; -import { EXPERIMENT } from '../../static/datamodel'; -import { formatTimestamp } from '../../static/function'; - -export const BasicInfo = (): any => ( - - -

      Name

      -
      {EXPERIMENT.profile.params.experimentName}
      -       - -

      ID

      -
      {EXPERIMENT.profile.id}
      -       - -

      Start time

      -
      {formatTimestamp(EXPERIMENT.profile.startTime)}
      -       - -

      End time

      -
      {formatTimestamp(EXPERIMENT.profile.endTime)}
      -       - -

      Log directory

      -
      - - {/* show logDir */} - {EXPERIMENT.profile.logDir || 'unknown'} - -
      -       - -

      Training platform

      -
      {EXPERIMENT.profile.params.trainingServicePlatform}
      -       -       -); diff --git a/src/webui/src/components/overview/NumInput.tsx b/src/webui/src/components/overview/NumInput.tsx deleted file mode 100644 index 6264923aa6..0000000000 --- a/src/webui/src/components/overview/NumInput.tsx +++ /dev/null @@ -1,65 +0,0 @@ -import * as React from 'react'; -import { Stack, PrimaryButton } from '@fluentui/react'; - -interface ConcurrencyInputProps { - value: number; - updateValue: (val: string) => void; -} - -interface ConcurrencyInputStates { - editting: boolean; -} - -class ConcurrencyInput extends React.Component { - private input = React.createRef(); - - constructor(props: ConcurrencyInputProps) { - super(props); - this.state = { editting: false }; - } - - save = (): void => { - if (this.input.current !== null) { - this.props.updateValue(this.input.current.value); - this.setState({ editting: false }); - } - }; - - cancel = (): void => { - this.setState({ editting: false }); - }; - - edit = (): void => { - this.setState({ editting: true }); - }; - - render(): React.ReactNode { - if (this.state.editting) { - return ( - - - - - - ); - } else { - return ( - - - - - ); - } - } -} - -export default ConcurrencyInput; diff --git a/src/webui/src/components/overview/Progress.tsx b/src/webui/src/components/overview/Progress.tsx deleted file mode 100644 index 4bc5139f0c..0000000000 --- a/src/webui/src/components/overview/Progress.tsx +++ /dev/null @@ -1,315 +0,0 @@ -import * as React from 'react'; -import { - Stack, - Callout, - Link, - IconButton, - FontWeights, - mergeStyleSets, - getId, - getTheme, - StackItem, - TooltipHost -} from '@fluentui/react'; -import axios from 'axios'; -import { MANAGER_IP, CONCURRENCYTOOLTIP } from '../../static/const'; -import { EXPERIMENT, TRIALS } from '../../static/datamodel'; -import { convertTime } from '../../static/function'; -import ConcurrencyInput from './NumInput'; -import ProgressBar from './ProgressItem'; -import LogDrawer from '../modals/LogPanel'; -import MessageInfo from '../modals/MessageInfo'; -import { infoIcon } from '../buttons/Icon'; -import '../../static/style/progress.scss'; -import '../../static/style/probar.scss'; -interface ProgressProps { - concurrency: number; - bestAccuracy: number; - changeConcurrency: (val: number) => void; - experimentUpdateBroadcast: number; -} - -interface ProgressState { - isShowLogDrawer: boolean; - isCalloutVisible?: boolean; - isShowSucceedInfo: boolean; - info: string; - typeInfo: string; -} - -const itemStyles: React.CSSProperties = { - height: 50, - width: 100 -}; -const theme = getTheme(); -const styles = mergeStyleSets({ - buttonArea: { - verticalAlign: 'top', - display: 'inline-block', - textAlign: 'center', - // margin: '0 100px', - minWidth: 30, - height: 30 - }, - callout: { - maxWidth: 300 - }, - header: { - padding: '18px 24px 12px' - }, - title: [ - theme.fonts.xLarge, - { - margin: 0, - color: theme.palette.neutralPrimary, - fontWeight: FontWeights.semilight - } - ], - inner: { - height: '100%', - padding: '0 24px 20px' - }, - actions: { - position: 'relative', - marginTop: 20, - width: '100%', - whiteSpace: 'nowrap' - }, - subtext: [ - theme.fonts.small, - { - margin: 0, - color: theme.palette.neutralPrimary, - fontWeight: FontWeights.semilight - } - ], - link: [ - theme.fonts.medium, - { - color: theme.palette.neutralPrimary - } - ] -}); - -class Progressed extends React.Component { - private menuButtonElement!: HTMLDivElement | null; - private labelId: string = getId('callout-label'); - private descriptionId: string = getId('callout-description'); - constructor(props: ProgressProps) { - super(props); - this.state = { - isShowLogDrawer: false, - isCalloutVisible: false, - isShowSucceedInfo: false, - info: '', - typeInfo: 'success' - }; - } - - hideSucceedInfo = (): void => { - this.setState(() => ({ isShowSucceedInfo: false })); - }; - - /** - * info: message content - * typeInfo: message type: success | error... - * continuousTime: show time, 2000ms - */ - showMessageInfo = (info: string, typeInfo: string): void => { - this.setState(() => ({ - info, - typeInfo, - isShowSucceedInfo: true - })); - setTimeout(this.hideSucceedInfo, 2000); - }; - - editTrialConcurrency = async (userInput: string): Promise => { - if (!userInput.match(/^[1-9]\d*$/)) { - this.showMessageInfo('Please enter a positive integer!', 'error'); - return; - } - const newConcurrency = parseInt(userInput, 10); - if (newConcurrency === this.props.concurrency) { - this.showMessageInfo('Trial concurrency has not changed', 'error'); - return; - } - - const newProfile = Object.assign({}, EXPERIMENT.profile); - newProfile.params.trialConcurrency = newConcurrency; - - // rest api, modify trial concurrency value - try { - const res = await axios.put(`${MANAGER_IP}/experiment`, newProfile, { - // eslint-disable-next-line @typescript-eslint/camelcase - params: { update_type: 'TRIAL_CONCURRENCY' } - }); - if (res.status === 200) { - this.showMessageInfo('Successfully updated trial concurrency', 'success'); - // NOTE: should we do this earlier in favor of poor networks? - this.props.changeConcurrency(newConcurrency); - } - } catch (error) { - if (error.response && error.response.data.error) { - this.showMessageInfo(`Failed to update trial concurrency\n${error.response.data.error}`, 'error'); - } else if (error.response) { - this.showMessageInfo( - `Failed to update trial concurrency\nServer responsed ${error.response.status}`, - 'error' - ); - } else if (error.message) { - this.showMessageInfo(`Failed to update trial concurrency\n${error.message}`, 'error'); - } else { - this.showMessageInfo(`Failed to update trial concurrency\nUnknown error`, 'error'); - } - } - }; - - isShowDrawer = (): void => { - this.setState({ isShowLogDrawer: true }); - }; - - closeDrawer = (): void => { - this.setState({ isShowLogDrawer: false }); - }; - - onDismiss = (): void => { - this.setState({ isCalloutVisible: false }); - }; - - onShow = (): void => { - this.setState({ isCalloutVisible: true }); - }; - - render(): React.ReactNode { - const { bestAccuracy } = this.props; - const { isShowLogDrawer, isCalloutVisible, isShowSucceedInfo, info, typeInfo } = this.state; - - const count = TRIALS.countStatus(); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const stoppedCount = count.get('USER_CANCELED')! + count.get('SYS_CANCELED')! + count.get('EARLY_STOPPED')!; - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const bar2 = count.get('RUNNING')! + count.get('SUCCEEDED')! + count.get('FAILED')! + stoppedCount; - // support type [0, 1], not 98% - const bar2Percent = bar2 / EXPERIMENT.profile.params.maxTrialNum; - const percent = EXPERIMENT.profile.execDuration / EXPERIMENT.profile.params.maxExecDuration; - const remaining = convertTime(EXPERIMENT.profile.params.maxExecDuration - EXPERIMENT.profile.execDuration); - const maxDuration = convertTime(EXPERIMENT.profile.params.maxExecDuration); - const maxTrialNum = EXPERIMENT.profile.params.maxTrialNum; - const execDuration = convertTime(EXPERIMENT.profile.execDuration); - - return ( - - -

      Status

      - - {EXPERIMENT.status} - {EXPERIMENT.status === 'ERROR' ? ( -
      -
      (this.menuButtonElement = val)}> - -
      - {isCalloutVisible && ( - -
      -

      - Error -

      -
      -
      -

      - {EXPERIMENT.error} -

      -
      - - Learn about - -
      -
      -       - )} -
      - ) : null} -       -       - - - - -

      Best metric

      -
      {isNaN(bestAccuracy) ? 'N/A' : bestAccuracy.toFixed(6)}
      -       - - {isShowSucceedInfo && } - -       - - -

      Spent

      -
      {execDuration}
      -       - -

      Remaining

      -
      {remaining}
      -       - - {/* modify concurrency */} - -

      - Concurrency{infoIcon} -

      -       - -       - -       - -
      -

      Running

      -
      {count.get('RUNNING')}
      -
      -
      -

      Succeeded

      -
      {count.get('SUCCEEDED')}
      -
      -
      -

      Stopped

      -
      {stoppedCount}
      -
      -
      -

      Failed

      -
      {count.get('FAILED')}
      -
      -       - {/* learn about click -> default active key is dispatcher. */} - {isShowLogDrawer ? : null} -       - ); - } -} - -export default Progressed; diff --git a/src/webui/src/components/overview/ProgressItem.tsx b/src/webui/src/components/overview/ProgressItem.tsx deleted file mode 100644 index 849171bb57..0000000000 --- a/src/webui/src/components/overview/ProgressItem.tsx +++ /dev/null @@ -1,42 +0,0 @@ -import * as React from 'react'; -import { Stack, StackItem, ProgressIndicator } from '@fluentui/react'; - -interface ProItemProps { - who: string; - percent: number; - description: string; - maxString: string; - bgclass: string; -} - -class ProgressBar extends React.Component { - constructor(props: ProItemProps) { - super(props); - } - - render(): React.ReactNode { - const { who, percent, description, maxString, bgclass } = this.props; - return ( -
      - -
      {who}
      -
      - - - 0 - - {maxString} - - -
      -
      - {description} -
      -       -
      -
      - ); - } -} - -export default ProgressBar; diff --git a/src/webui/src/components/overview/SearchSpace.tsx b/src/webui/src/components/overview/SearchSpace.tsx deleted file mode 100644 index 9532f8b812..0000000000 --- a/src/webui/src/components/overview/SearchSpace.tsx +++ /dev/null @@ -1,30 +0,0 @@ -import * as React from 'react'; -import MonacoEditor from 'react-monaco-editor'; -import { MONACO } from '../../static/const'; - -interface SearchspaceProps { - searchSpace: object; -} - -class SearchSpace extends React.Component { - constructor(props: SearchspaceProps) { - super(props); - } - - render(): React.ReactNode { - const { searchSpace } = this.props; - return ( -
      - -
      - ); - } -} - -export default SearchSpace; diff --git a/src/webui/src/components/overview/Title1.tsx b/src/webui/src/components/overview/Title1.tsx deleted file mode 100644 index dd89dd8fa0..0000000000 --- a/src/webui/src/components/overview/Title1.tsx +++ /dev/null @@ -1,15 +0,0 @@ -import * as React from 'react'; -import { Stack } from '@fluentui/react'; -import { TitleContext } from '../Overview'; -import '../../static/style/overviewTitle.scss'; - -export const Title1 = (): any => ( - - {(value): React.ReactNode => ( - - icon - {value.text} - - )} - -); diff --git a/src/webui/src/components/overview/TrialProfile.tsx b/src/webui/src/components/overview/TrialProfile.tsx deleted file mode 100644 index 6c49ad4907..0000000000 --- a/src/webui/src/components/overview/TrialProfile.tsx +++ /dev/null @@ -1,49 +0,0 @@ -import * as React from 'react'; -import MonacoEditor from 'react-monaco-editor'; -import { MONACO } from '../../static/const'; -import { EXPERIMENT } from '../../static/datamodel'; - -interface TrialInfoProps { - experimentUpdateBroadcast: number; - concurrency: number; -} - -class TrialInfo extends React.Component { - constructor(props: TrialInfoProps) { - super(props); - } - - render(): React.ReactNode { - const blacklist = [ - 'id', - 'logDir', - 'startTime', - 'endTime', - 'experimentName', - 'searchSpace', - 'trainingServicePlatform' - ]; - const filter = (key: string, val: any): any => { - if (key === 'trialConcurrency') { - return this.props.concurrency; - } - return blacklist.includes(key) ? undefined : val; - }; - const profile = JSON.stringify(EXPERIMENT.profile, filter, 2); - - return ( -
      - -
      - ); - } -} - -export default TrialInfo; diff --git a/src/webui/src/components/trial-detail/TableList.tsx b/src/webui/src/components/trial-detail/TableList.tsx deleted file mode 100644 index 47687f7142..0000000000 --- a/src/webui/src/components/trial-detail/TableList.tsx +++ /dev/null @@ -1,795 +0,0 @@ -import React, { lazy } from 'react'; -import axios from 'axios'; -import ReactEcharts from 'echarts-for-react'; -import { - Stack, - Dropdown, - DetailsList, - IDetailsListProps, - DetailsListLayoutMode, - PrimaryButton, - Modal, - IDropdownOption, - IColumn, - Selection, - SelectionMode, - IconButton, - TooltipHost, - IStackTokens -} from '@fluentui/react'; -import ReactPaginate from 'react-paginate'; -import { LineChart, blocked, copy } from '../buttons/Icon'; -import { MANAGER_IP, COLUMNPro } from '../../static/const'; -import { convertDuration, formatTimestamp, intermediateGraphOption, parseMetrics } from '../../static/function'; -import { EXPERIMENT, TRIALS } from '../../static/datamodel'; -import { TableRecord, TrialJobInfo } from '../../static/interface'; -const Details = lazy(() => import('../overview/Details')); -const ChangeColumnComponent = lazy(() => import('../modals/ChangeColumnComponent')); -const Compare = lazy(() => import('../modals/Compare')); -const KillJob = lazy(() => import('../modals/Killjob')); -const Customize = lazy(() => import('../modals/CustomizedTrial')); -import { contentStyles, iconButtonStyles } from '../buttons/ModalTheme'; -import '../../static/style/search.scss'; -import '../../static/style/tableStatus.css'; -import '../../static/style/logPath.scss'; -import '../../static/style/table.scss'; -import '../../static/style/button.scss'; -import '../../static/style/openRow.scss'; -import '../../static/style/pagination.scss'; - -const echarts = require('echarts/lib/echarts'); -require('echarts/lib/chart/line'); -require('echarts/lib/component/tooltip'); -require('echarts/lib/component/title'); -echarts.registerTheme('my_theme', { - color: '#3c8dbc' -}); - -const horizontalGapStackTokens: IStackTokens = { - childrenGap: 20, - padding: 10 -}; - -interface TableListProps { - pageSize: number; - tableSource: Array; - columnList: string[]; // user select columnKeys - changeColumn: (val: string[]) => void; - trialsUpdateBroadcast: number; -} - -interface SortInfo { - field: string; - isDescend?: boolean; -} - -interface TableListState { - intermediateOption: object; - modalVisible: boolean; - isObjFinal: boolean; - isShowColumn: boolean; - selectRows: Array; - isShowCompareModal: boolean; - selectedRowKeys: string[] | number[]; - intermediateData: Array; // a trial's intermediate results (include dict) - intermediateId: string; - intermediateOtherKeys: string[]; - isShowCustomizedModal: boolean; - copyTrialId: string; // user copy trial to submit a new customized trial - isCalloutVisible: boolean; // kill job button callout [kill or not kill job window] - intermediateKey: string; // intermeidate modal: which key is choosed. - isExpand: boolean; - modalIntermediateWidth: number; - modalIntermediateHeight: number; - tableColumns: IColumn[]; - allColumnList: string[]; - tableSourceForSort: Array; - sortMessage: SortInfo; - offset: number; - tablePerPage: Array; - perPage: number; - currentPage: number; - pageCount: number; -} - -class TableList extends React.Component { - public intervalTrialLog = 10; - public trialId!: string; - - constructor(props: TableListProps) { - super(props); - - this.state = { - intermediateOption: {}, - modalVisible: false, - isObjFinal: false, - isShowColumn: false, - isShowCompareModal: false, - selectRows: [], - selectedRowKeys: [], // close selected trial message after modal closed - intermediateData: [], - intermediateId: '', - intermediateOtherKeys: [], - isShowCustomizedModal: false, - isCalloutVisible: false, - copyTrialId: '', - intermediateKey: 'default', - isExpand: false, - modalIntermediateWidth: window.innerWidth, - modalIntermediateHeight: window.innerHeight, - tableColumns: this.initTableColumnList(this.props.columnList), - allColumnList: this.getAllColumnKeys(), - sortMessage: { field: '', isDescend: false }, - offset: 0, - tablePerPage: [], - perPage: 20, - currentPage: 0, - pageCount: 0, - tableSourceForSort: this.props.tableSource - }; - } - - // sort for table column - onColumnClick = (ev: React.MouseEvent, getColumn: IColumn): void => { - const { tableColumns } = this.state; - const newColumns: IColumn[] = tableColumns.slice(); - const currColumn: IColumn = newColumns.filter(item => getColumn.key === item.key)[0]; - newColumns.forEach((newCol: IColumn) => { - if (newCol === currColumn) { - currColumn.isSortedDescending = !currColumn.isSortedDescending; - currColumn.isSorted = true; - } else { - newCol.isSorted = false; - newCol.isSortedDescending = true; - } - }); - - this.setState( - { - tableColumns: newColumns, - sortMessage: { field: getColumn.key, isDescend: currColumn.isSortedDescending } - }, - () => { - this.updateData(); - } - ); - }; - - AccuracyColumnConfig: any = { - name: 'Default metric', - className: 'leftTitle', - key: 'latestAccuracy', - fieldName: 'latestAccuracy', - minWidth: 200, - maxWidth: 300, - isResizable: true, - data: 'number', - onColumnClick: this.onColumnClick, - onRender: (item): React.ReactNode => ( - -
      {item.formattedLatestAccuracy}
      -       - ) - }; - - SequenceIdColumnConfig: any = { - name: 'Trial No.', - key: 'sequenceId', - fieldName: 'sequenceId', - minWidth: 80, - maxWidth: 240, - className: 'tableHead', - data: 'number', - onColumnClick: this.onColumnClick - }; - - IdColumnConfig: any = { - name: 'ID', - key: 'id', - fieldName: 'id', - minWidth: 150, - maxWidth: 200, - isResizable: true, - data: 'string', - onColumnClick: this.onColumnClick, - className: 'tableHead leftTitle' - }; - - StartTimeColumnConfig: any = { - name: 'Start time', - key: 'startTime', - fieldName: 'startTime', - minWidth: 150, - maxWidth: 400, - isResizable: true, - data: 'number', - onColumnClick: this.onColumnClick, - onRender: (record): React.ReactNode => {formatTimestamp(record.startTime)} - }; - - EndTimeColumnConfig: any = { - name: 'End time', - key: 'endTime', - fieldName: 'endTime', - minWidth: 200, - maxWidth: 400, - isResizable: true, - data: 'number', - onColumnClick: this.onColumnClick, - onRender: (record): React.ReactNode => {formatTimestamp(record.endTime, '--')} - }; - - DurationColumnConfig: any = { - name: 'Duration', - key: 'duration', - fieldName: 'duration', - minWidth: 150, - maxWidth: 300, - isResizable: true, - data: 'number', - onColumnClick: this.onColumnClick, - onRender: (record): React.ReactNode => {convertDuration(record.duration)} - }; - - StatusColumnConfig: any = { - name: 'Status', - key: 'status', - fieldName: 'status', - className: 'tableStatus', - minWidth: 150, - maxWidth: 250, - isResizable: true, - data: 'string', - onColumnClick: this.onColumnClick, - onRender: (record): React.ReactNode => {record.status} - }; - - IntermediateCountColumnConfig: any = { - name: 'Intermediate result', - dataIndex: 'intermediateCount', - fieldName: 'intermediateCount', - minWidth: 150, - maxWidth: 200, - isResizable: true, - data: 'number', - onColumnClick: this.onColumnClick, - onRender: (record): React.ReactNode => {`#${record.intermediateCount}`} - }; - - showIntermediateModal = async (record: TrialJobInfo, event: React.SyntheticEvent): Promise => { - event.preventDefault(); - event.stopPropagation(); - const res = await axios.get(`${MANAGER_IP}/metric-data/${record.id}`); - if (res.status === 200) { - const intermediateArr: number[] = []; - // support intermediate result is dict because the last intermediate result is - // final result in a succeed trial, it may be a dict. - // get intermediate result dict keys array - const { intermediateKey } = this.state; - const otherkeys: string[] = []; - const metricDatas = res.data; - if (metricDatas.length !== 0) { - // just add type=number keys - const intermediateMetrics = parseMetrics(metricDatas[0].data); - for (const key in intermediateMetrics) { - if (typeof intermediateMetrics[key] === 'number') { - otherkeys.push(key); - } - } - } - // intermediateArr just store default val - metricDatas.map(item => { - if (item.type === 'PERIODICAL') { - const temp = parseMetrics(item.data); - if (typeof temp === 'object') { - intermediateArr.push(temp[intermediateKey]); - } else { - intermediateArr.push(temp); - } - } - }); - const intermediate = intermediateGraphOption(intermediateArr, record.id); - this.setState({ - intermediateData: res.data, // store origin intermediate data for a trial - intermediateOption: intermediate, - intermediateOtherKeys: otherkeys, - intermediateId: record.id - }); - } - this.setState({ modalVisible: true }); - }; - - // intermediate button click -> intermediate graph for each trial - // support intermediate is dict - selectOtherKeys = (event: React.FormEvent, item?: IDropdownOption): void => { - if (item !== undefined) { - const value = item.text; - const isShowDefault: boolean = value === 'default' ? true : false; - const { intermediateData, intermediateId } = this.state; - const intermediateArr: number[] = []; - // just watch default key-val - if (isShowDefault === true) { - Object.keys(intermediateData).map(item => { - if (intermediateData[item].type === 'PERIODICAL') { - const temp = parseMetrics(intermediateData[item].data); - if (typeof temp === 'object') { - intermediateArr.push(temp[value]); - } else { - intermediateArr.push(temp); - } - } - }); - } else { - Object.keys(intermediateData).map(item => { - const temp = parseMetrics(intermediateData[item].data); - if (typeof temp === 'object') { - intermediateArr.push(temp[value]); - } - }); - } - const intermediate = intermediateGraphOption(intermediateArr, intermediateId); - // re-render - this.setState({ - intermediateKey: value, - intermediateOption: intermediate - }); - } - }; - - hideIntermediateModal = (): void => { - this.setState({ - modalVisible: false - }); - }; - - hideShowColumnModal = (): void => { - this.setState(() => ({ isShowColumn: false })); - }; - - // click add column btn, just show the modal of addcolumn - addColumn = (): void => { - // show user select check button - this.setState(() => ({ isShowColumn: true })); - }; - - fillSelectedRowsTostate = (selected: number[] | string[], selectedRows: Array): void => { - this.setState({ selectRows: selectedRows, selectedRowKeys: selected }); - }; - - // open Compare-modal - compareBtn = (): void => { - const { selectRows } = this.state; - if (selectRows.length === 0) { - alert('Please select datas you want to compare!'); - } else { - this.setState({ isShowCompareModal: true }); - } - }; - - // close Compare-modal - hideCompareModal = (): void => { - // close modal. clear select rows data, clear selected track - this.setState({ isShowCompareModal: false, selectedRowKeys: [], selectRows: [] }); - }; - - // open customized trial modal - private setCustomizedTrial = (trialId: string, event: React.SyntheticEvent): void => { - event.preventDefault(); - event.stopPropagation(); - this.setState({ - isShowCustomizedModal: true, - copyTrialId: trialId - }); - }; - - private closeCustomizedTrial = (): void => { - this.setState({ - isShowCustomizedModal: false, - copyTrialId: '' - }); - }; - - private onWindowResize = (): void => { - this.setState(() => ({ - modalIntermediateHeight: window.innerHeight, - modalIntermediateWidth: window.innerWidth - })); - }; - - private onRenderRow: IDetailsListProps['onRenderRow'] = props => { - if (props) { - return
      ; - } - return null; - }; - - private getSelectedRows = new Selection({ - onSelectionChanged: (): void => { - this.setState(() => ({ selectRows: this.getSelectedRows.getSelection() })); - } - }); - - // trial parameters & dict final keys & Trial No. Id ... - private getAllColumnKeys = (): string[] => { - const tableSource: Array = JSON.parse(JSON.stringify(this.props.tableSource)); - // parameter as table column - const parameterStr: string[] = []; - if (!EXPERIMENT.isNestedExp()) { - if (tableSource.length > 0) { - const trialMess = TRIALS.getTrial(tableSource[0].id); - const trial = trialMess.description.parameters; - const parameterColumn: string[] = Object.keys(trial); - parameterColumn.forEach(value => { - parameterStr.push(`${value} (search space)`); - }); - } - } - // concat trial all final keys and remove dup "default" val, return list - const finalKeysList = TRIALS.finalKeys().filter(item => item !== 'default'); - return COLUMNPro.concat(parameterStr).concat(finalKeysList); - }; - - // get IColumn[] - // when user click [Add Column] need to use the function - private initTableColumnList = (columnList: string[]): IColumn[] => { - // const { columnList } = this.props; - const disabledAddCustomizedTrial = ['DONE', 'ERROR', 'STOPPED'].includes(EXPERIMENT.status); - const showColumn: IColumn[] = []; - for (const item of columnList) { - const paraColumn = item.match(/ \(search space\)$/); - let result; - if (paraColumn !== null) { - result = paraColumn.input; - } - switch (item) { - case 'Trial No.': - showColumn.push(this.SequenceIdColumnConfig); - break; - case 'ID': - showColumn.push(this.IdColumnConfig); - break; - case 'Start time': - showColumn.push(this.StartTimeColumnConfig); - break; - case 'End time': - showColumn.push(this.EndTimeColumnConfig); - break; - case 'Duration': - showColumn.push(this.DurationColumnConfig); - break; - case 'Status': - showColumn.push(this.StatusColumnConfig); - break; - case 'Intermediate result': - showColumn.push(this.IntermediateCountColumnConfig); - break; - case 'Default': - showColumn.push(this.AccuracyColumnConfig); - break; - case 'Operation': - showColumn.push({ - name: 'Operation', - key: 'operation', - fieldName: 'operation', - minWidth: 160, - maxWidth: 200, - isResizable: true, - className: 'detail-table', - onRender: (record: any) => { - const trialStatus = record.status; - const flag: boolean = trialStatus === 'RUNNING' || trialStatus === 'UNKNOWN' ? false : true; - return ( - - {/* see intermediate result graph */} - - {LineChart} - - {/* kill job */} - {flag ? ( - - {blocked} - - ) : ( - - )} - {/* Add a new trial-customized trial */} - - {copy} - - - ); - } - }); - break; - case result: - // remove SEARCH_SPACE title - // const realItem = item.replace(' (search space)', ''); - showColumn.push({ - name: item.replace(' (search space)', ''), - key: item, - fieldName: item, - minWidth: 150, - onRender: (record: TableRecord) => { - const eachTrial = TRIALS.getTrial(record.id); - return {eachTrial.description.parameters[item.replace(' (search space)', '')]}; - } - }); - break; - default: - showColumn.push({ - name: item, - key: item, - fieldName: item, - minWidth: 100, - onRender: (record: TableRecord) => { - const accDictionary = record.accDictionary; - let other = ''; - if (accDictionary !== undefined) { - other = accDictionary[item].toString(); - } - return ( - -
      {other}
      -       - ); - } - }); - } - } - return showColumn; - }; - - componentDidMount(): void { - window.addEventListener('resize', this.onWindowResize); - this.updateData(); - } - - componentDidUpdate(prevProps: TableListProps): void { - if ( - this.props.columnList !== prevProps.columnList || - this.props.tableSource !== prevProps.tableSource || - prevProps.trialsUpdateBroadcast !== this.props.trialsUpdateBroadcast - ) { - const { columnList } = this.props; - this.setState( - { - tableColumns: this.initTableColumnList(columnList), - allColumnList: this.getAllColumnKeys() - }, - () => { - this.updateData(); - } - ); - } - } - - // slice all table data into current page data - updateData(): void { - const tableSource: Array = this.props.tableSource; - const { offset, perPage, sortMessage } = this.state; - - if (sortMessage.field !== '') { - tableSource.sort(function(a, b): any { - if ( - a[sortMessage.field] === undefined || - Object.is(a[sortMessage.field], NaN) || - Object.is(a[sortMessage.field], Infinity) || - Object.is(a[sortMessage.field], -Infinity) || - typeof a[sortMessage.field] === 'object' - ) { - return 1; - } - if ( - b[sortMessage.field] === undefined || - Object.is(b[sortMessage.field], NaN) || - Object.is(b[sortMessage.field], Infinity) || - Object.is(b[sortMessage.field], -Infinity) || - typeof b[sortMessage.field] === 'object' - ) { - return -1; - } - return (sortMessage.isDescend - ? a[sortMessage.field] < b[sortMessage.field] - : a[sortMessage.field] > b[sortMessage.field]) - ? 1 - : -1; - }); - } - - const tableSlice = tableSource.slice(offset, offset + perPage); - const curPageCount = Math.ceil(tableSource.length / perPage); - this.setState({ - tablePerPage: tableSlice, - pageCount: curPageCount - }); - } - - // update data when click the page index of pagination - handlePageClick = (evt: any): void => { - const selectedPage = evt.selected; - const offset = selectedPage * this.state.perPage; - - this.setState( - { - currentPage: selectedPage, - offset: offset - }, - () => { - this.updateData(); - } - ); - }; - - // update per page items when click the dropdown of pagination - updatePerPage = (event: React.FormEvent, item: IDropdownOption | undefined): void => { - const { pageCount } = this.state; - - if (item !== undefined) { - const currentPerPage = item.key === 'all' ? this.props.tableSource.length : Number(item.key); - const currentPageCount = this.props.tableSource.length <= currentPerPage ? 1 : pageCount; - - this.setState( - { - perPage: currentPerPage, - offset: 0, - currentPage: 0, - pageCount: currentPageCount - }, - () => { - this.updateData(); - } - ); - } - }; - - render(): React.ReactNode { - const { - intermediateKey, - modalIntermediateWidth, - modalIntermediateHeight, - tableColumns, - allColumnList, - isShowColumn, - modalVisible, - selectRows, - isShowCompareModal, - intermediateOtherKeys, - isShowCustomizedModal, - copyTrialId, - intermediateOption, - tablePerPage - } = this.state; - const { columnList } = this.props; - const perPageOptions = [ - { key: '10', text: '10 items per page' }, - { key: '20', text: '20 items per page' }, - { key: '50', text: '50 items per page' }, - { key: 'all', text: 'All items' } - ]; - - return ( - -
      - - - - - - '} - breakLabel={'...'} - breakClassName={'break'} - pageCount={this.state.pageCount} - marginPagesDisplayed={2} - pageRangeDisplayed={2} - onPageChange={this.handlePageClick} - containerClassName={this.props.tableSource.length == 0 ? 'pagination hidden' : 'pagination'} - subContainerClassName={'pages pagination'} - disableInitialCallback={false} - activeClassName={'active'} - forcePage={this.state.currentPage} - /> - -
      - {/* Intermediate Result Modal */} - -
      - Intermediate result - -
      - {intermediateOtherKeys.length > 1 ? ( - - { - return { - key: key, - text: intermediateOtherKeys[item] - }; - })} - onChange={this.selectOtherKeys} - /> - - ) : null} -
      - -
      #Intermediate result
      -
      -       - {/* Add Column Modal */} - {isShowColumn && ( - - )} - {/* compare trials based message */} - {isShowCompareModal && } - {/* clone trial parameters and could submit a customized trial */} - -       - ); - } -} - -export default TableList; diff --git a/src/webui/src/static/img/icon/1.png b/src/webui/src/static/img/icon/1.png deleted file mode 100644 index bb16591ccf..0000000000 Binary files a/src/webui/src/static/img/icon/1.png and /dev/null differ diff --git a/src/webui/src/static/img/icon/10.png b/src/webui/src/static/img/icon/10.png deleted file mode 100644 index 73d482d77d..0000000000 Binary files a/src/webui/src/static/img/icon/10.png and /dev/null differ diff --git a/src/webui/src/static/img/icon/11.png b/src/webui/src/static/img/icon/11.png deleted file mode 100644 index 8d2da0c764..0000000000 Binary files a/src/webui/src/static/img/icon/11.png and /dev/null differ diff --git a/src/webui/src/static/img/icon/2.png b/src/webui/src/static/img/icon/2.png deleted file mode 100644 index 7a830558aa..0000000000 Binary files a/src/webui/src/static/img/icon/2.png and /dev/null differ diff --git a/src/webui/src/static/img/icon/3.png b/src/webui/src/static/img/icon/3.png deleted file mode 100644 index 9a70d1622d..0000000000 Binary files a/src/webui/src/static/img/icon/3.png and /dev/null differ diff --git a/src/webui/src/static/img/icon/4.png b/src/webui/src/static/img/icon/4.png deleted file mode 100644 index daa7101860..0000000000 Binary files a/src/webui/src/static/img/icon/4.png and /dev/null differ diff --git a/src/webui/src/static/img/icon/5.png b/src/webui/src/static/img/icon/5.png deleted file mode 100644 index 7026fc4905..0000000000 Binary files a/src/webui/src/static/img/icon/5.png and /dev/null differ diff --git a/src/webui/src/static/img/icon/6.png b/src/webui/src/static/img/icon/6.png deleted file mode 100644 index 0cfaef4ba6..0000000000 Binary files a/src/webui/src/static/img/icon/6.png and /dev/null differ diff --git a/src/webui/src/static/img/icon/7.png b/src/webui/src/static/img/icon/7.png deleted file mode 100644 index 43b2a85943..0000000000 Binary files a/src/webui/src/static/img/icon/7.png and /dev/null differ diff --git a/src/webui/src/static/img/icon/8.png b/src/webui/src/static/img/icon/8.png deleted file mode 100644 index db0140f614..0000000000 Binary files a/src/webui/src/static/img/icon/8.png and /dev/null differ diff --git a/src/webui/src/static/img/icon/9.png b/src/webui/src/static/img/icon/9.png deleted file mode 100644 index 8072740a9e..0000000000 Binary files a/src/webui/src/static/img/icon/9.png and /dev/null differ diff --git a/src/webui/src/static/img/icon/download.png b/src/webui/src/static/img/icon/download.png deleted file mode 100644 index 4182f1a70e..0000000000 Binary files a/src/webui/src/static/img/icon/download.png and /dev/null differ diff --git a/src/webui/src/static/img/icon/issue.png b/src/webui/src/static/img/icon/issue.png deleted file mode 100644 index 0966645fc7..0000000000 Binary files a/src/webui/src/static/img/icon/issue.png and /dev/null differ diff --git a/src/webui/src/static/img/icon/max.png b/src/webui/src/static/img/icon/max.png deleted file mode 100644 index f32d9d55f4..0000000000 Binary files a/src/webui/src/static/img/icon/max.png and /dev/null differ diff --git a/src/webui/src/static/img/icon/min.png b/src/webui/src/static/img/icon/min.png deleted file mode 100644 index 4a19000c25..0000000000 Binary files a/src/webui/src/static/img/icon/min.png and /dev/null differ diff --git a/src/webui/src/static/img/icon/ques.png b/src/webui/src/static/img/icon/ques.png deleted file mode 100644 index baf4d0a45e..0000000000 Binary files a/src/webui/src/static/img/icon/ques.png and /dev/null differ diff --git a/src/webui/src/static/img/logo.png b/src/webui/src/static/img/logo.png deleted file mode 100644 index 20dd992b66..0000000000 Binary files a/src/webui/src/static/img/logo.png and /dev/null differ diff --git a/src/webui/src/static/style/overview.scss b/src/webui/src/static/style/overview.scss deleted file mode 100644 index 7a42427b9d..0000000000 --- a/src/webui/src/static/style/overview.scss +++ /dev/null @@ -1,67 +0,0 @@ - -/* new style */ -.overMessage { - height: 446px; -} - -.blockPadding { - padding: 10px 20px; -} - -.commonTableStyle { - padding: 15px 20px; - height: 100%; - min-width: 500px; - overflow-y: auto; -} - -.padItem { - padding: 10px 20px 0 20px; -} - -.searchSpace { - line-height: 22px; - font-size: 14px; - padding: 15px 0; - color: #212121; - width: 95%; -} - -.nowrap { - overflow: hidden; - white-space: nowrap; - text-overflow: ellipsis; -} - -.main { - margin: 9px 0; -} - -.profile { - pre { - overflow: inherit; - } -} - -.link { - margin-bottom: 10px; -} - -.info { - position: relative; - top: 15px; - left: 10px; - - span { - color: #333; - font-size: 14px; - } -} - -/* overview-succeed-graph */ -.showMess { - position: absolute; - top: 50%; - left: 50%; - transform: translate(-50%, -50%); -} diff --git a/src/webui/src/static/style/probar.scss b/src/webui/src/static/style/probar.scss deleted file mode 100644 index fcf1ca2a14..0000000000 --- a/src/webui/src/static/style/probar.scss +++ /dev/null @@ -1,61 +0,0 @@ -#barBack { - /* status: 'INITIALIZED' | 'RUNNING' | 'ERROR' | 'STOPPING' | 'STOPPED' | 'DONE' */ - - /* status: 'TUNER_NO_MORE_TRIAL' | 'NO_MORE_TRIAL' */ - .RUNNING, - .STOPPING, - .INITIALIZED, - .NO_MORE_TRIAL, - .TUNER_NO_MORE_TRIAL { - /* specific status color */ - color: #0071bc; - - /* progress- duration & trial numbers span */ - .ms-ProgressIndicator-progressBar { - background-color: #0071bc; - } - } - - .DONE, - .STOPPED { - color: #009245; - - .ms-ProgressIndicator-progressBar { - background-color: #009245; - } - } - - .ERROR { - color: #eb0716; - - .ms-ProgressIndicator-progressBar { - background-color: #eb0716; - } - } -} - -.errorBtn { - margin-left: 15px; - display: inline-block; - width: 18px; - height: 18px; - line-height: 18px; - text-align: center; - font-size: 14px; - border: 1px solid #4d4d4d; - border-radius: 50%; - background-color: #4d4d4d; - color: #fff; -} - -.errorBtn:hover { - cursor: pointer; -} - -.errors { - width: 240px; - font-size: 14px; - color: #212121; - word-wrap: break-word; - word-break: normal; -} diff --git a/src/webui/src/static/style/progress.scss b/src/webui/src/static/style/progress.scss deleted file mode 100644 index f6c5b25010..0000000000 --- a/src/webui/src/static/style/progress.scss +++ /dev/null @@ -1,124 +0,0 @@ -.progress { - margin: 15px 17px; - - .status { - color: #0573bc; - font-size: 20px; - font-weight: 600; - margin-top: 5px; - - .status-text { - display: inline-block; - line-height: 30px; - } - } - - .probar { - width: 100%; - height: 34px; - margin-top: 15px; - - .showProgress { - width: 300px; - height: 30px; - } - - .name { - width: 178px; - box-sizing: border-box; - line-height: 30px; - text-align: center; - color: #fff; - background-color: #999; - border: 2px solid #e6e6e6; - border-top-left-radius: 12px; - border-bottom-left-radius: 12px; - } - - .boundary { - width: 100%; - line-height: 24px; - font-size: 12px; - color: #212121; - - .right { - text-align: right; - } - } - - .description { - line-height: 34px; - margin-left: 6px; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; - } - } - - &-info { - margin-left: 4px; - position: relative; - top: 2px; - } -} - -/* basic experiment message style */ -.basic { - line-height: 24px; - font-family: "Segoe UI", Tahoma, Geneva, Verdana, sans-serif; - - p { - font-size: 14px; - color: #212121; - } - - div { - font-size: 16px; - color: #0573bc; - } -} - -.colorOfbasic { - div { - color: #0573bc; - } - - .time { - overflow: hidden; - white-space: nowrap; - text-overflow: ellipsis; - } -} - -.mess { - margin: 10px 0; -} - -.inputBox { - height: 32px; - margin-top: 5px; - - .concurrencyInput { - width: 40px; - padding-left: 8px; - outline: none; - border: 1px solid #ccc; - } -} - -.lineBasic { - padding-bottom: 14px; - border-bottom: 1px solid #ccc; -} - -/* office-fabric-ui progressIndicator */ -.ms-ProgressIndicator-itemProgress { - padding: 0; - border: 2px solid #e6e6e6; - border-radius: 0 12px 12px 0 !important; -} - -.cursor, -.cursor:hover { - cursor: pointer; -} diff --git a/src/webui/src/static/style/succTable.scss b/src/webui/src/static/style/succTable.scss deleted file mode 100644 index 86b89e3ff9..0000000000 --- a/src/webui/src/static/style/succTable.scss +++ /dev/null @@ -1,20 +0,0 @@ -#succTable { - height: 404px; - overflow: auto; - position: relative; - - .succTable-tooltip { - position: absolute; - top: 40%; - left: 17%; - - .link { - margin-left: 15px; - - a { - font-weight: 500; - color: blue; - } - } - } -} diff --git a/test/.coveragerc b/test/.coveragerc index fdcb4e097d..1565dbf27b 100644 --- a/test/.coveragerc +++ b/test/.coveragerc @@ -3,7 +3,7 @@ branch = True parallel = True data_file = ${COVERAGE_DATA_FILE} -source = nni, nni_cmd, nni_trial_tool +source = nni, nni.tools.cmd, nni.tools.trial_tool concurrency = multiprocessing diff --git a/test/config/integration_tests.yml b/test/config/integration_tests.yml index a4c50788ca..40f6a39165 100644 --- a/test/config/integration_tests.yml +++ b/test/config/integration_tests.yml @@ -147,8 +147,8 @@ testCases: config: maxTrialNum: 4 trialConcurrency: 4 - launchCommand: python3 -c 'from nnicli import Experiment; exp = Experiment(); exp.start_experiment("$configFile")' - stopCommand: python3 -c 'from nnicli import Experiment; exp = Experiment(); exp.connect_experiment("http://localhost:8080/"); exp.stop_experiment()' + launchCommand: python3 -c 'from nni.experiment import Experiment; exp = Experiment(); exp.start_experiment("$configFile")' + stopCommand: python3 -c 'from nni.experiment import Experiment; exp = Experiment(); exp.connect_experiment("http://localhost:8080/"); exp.stop_experiment()' validator: class: NnicliValidator platform: linux darwin diff --git a/test/config/integration_tests_tf2.yml b/test/config/integration_tests_tf2.yml index 2002f36367..8bc74f9b7a 100644 --- a/test/config/integration_tests_tf2.yml +++ b/test/config/integration_tests_tf2.yml @@ -110,8 +110,8 @@ testCases: config: maxTrialNum: 4 trialConcurrency: 4 - launchCommand: python3 -c 'from nnicli import Experiment; exp = Experiment(); exp.start_experiment("$configFile")' - stopCommand: python3 -c 'from nnicli import Experiment; exp = Experiment(); exp.connect_experiment("http://localhost:8080/"); exp.stop_experiment()' + launchCommand: python3 -c 'from nni.experiment import Experiment; exp = Experiment(); exp.start_experiment("$configFile")' + stopCommand: python3 -c 'from nni.experiment import Experiment; exp = Experiment(); exp.connect_experiment("http://localhost:8080/"); exp.stop_experiment()' validator: class: NnicliValidator platform: linux darwin diff --git a/test/config/pr_tests.yml b/test/config/pr_tests.yml index f82143b836..1e8ee8862a 100644 --- a/test/config/pr_tests.yml +++ b/test/config/pr_tests.yml @@ -47,8 +47,8 @@ testCases: config: maxTrialNum: 4 trialConcurrency: 4 - launchCommand: python3 -c 'from nnicli import Experiment; exp = Experiment(); exp.start_experiment("$configFile")' - stopCommand: python3 -c 'from nnicli import Experiment; exp = Experiment(); exp.connect_experiment("http://localhost:8080/"); exp.stop_experiment()' + launchCommand: python3 -c 'from nni.experiment import Experiment; exp = Experiment(); exp.start_experiment("$configFile")' + stopCommand: python3 -c 'from nni.experiment import Experiment; exp = Experiment(); exp.connect_experiment("http://localhost:8080/"); exp.stop_experiment()' validator: class: NnicliValidator platform: linux darwin diff --git a/test/config/training_service.yml b/test/config/training_service.yml index 998d39abaa..2365747355 100644 --- a/test/config/training_service.yml +++ b/test/config/training_service.yml @@ -95,6 +95,8 @@ pai: containerNFSMountPath: paiStorageConfigName: remote: + remoteConfig: + reuse: false machineList: - ip: passwd: diff --git a/test/nni_test/nnitest/generate_ts_config.py b/test/nni_test/nnitest/generate_ts_config.py index c11fb0d213..76376a3412 100644 --- a/test/nni_test/nnitest/generate_ts_config.py +++ b/test/nni_test/nnitest/generate_ts_config.py @@ -86,6 +86,8 @@ def update_training_service_config(args): config[args.ts]['machineList'][0]['port'] = args.remote_port if args.remote_pwd is not None: config[args.ts]['machineList'][0]['passwd'] = args.remote_pwd + if args.remote_reuse is not None: + config[args.ts]['remoteConfig']['reuse'] = args.remote_reuse.lower() == 'true' dump_yml_content(TRAINING_SERVICE_FILE, config) @@ -119,6 +121,7 @@ def update_training_service_config(args): parser.add_argument("--remote_pwd", type=str) parser.add_argument("--remote_host", type=str) parser.add_argument("--remote_port", type=int) + parser.add_argument("--remote_reuse", type=str) args = parser.parse_args() update_training_service_config(args) diff --git a/test/nni_test/nnitest/validators.py b/test/nni_test/nnitest/validators.py index ff349ebc2b..2df7342eb6 100644 --- a/test/nni_test/nnitest/validators.py +++ b/test/nni_test/nnitest/validators.py @@ -6,8 +6,8 @@ import subprocess import json import requests -from nnicli import Experiment -from nni_cmd.updater import load_search_space +from nni.experiment import Experiment +from nni.tools.nnictl.updater import load_search_space from utils import METRICS_URL, GET_IMPORTED_DATA_URL diff --git a/test/pipelines/pipelines-it-installation.yml b/test/pipelines/pipelines-it-installation.yml index a4fb28587b..85df9bfad4 100644 --- a/test/pipelines/pipelines-it-installation.yml +++ b/test/pipelines/pipelines-it-installation.yml @@ -14,13 +14,9 @@ jobs: python3 -m pip install --upgrade nni --user displayName: 'Install nni' -- job: 'pip_install_macOS_python36' +- job: 'pip_install_macOS_python3' pool: - vmImage: 'macOS-10.13' - strategy: - matrix: - Python36: - PYTHON_VERSION: '3.6' + vmImage: 'macOS-latest' steps: - script: | diff --git a/test/pipelines/pipelines-it-remote-linux-to-linux.yml b/test/pipelines/pipelines-it-remote-linux-to-linux.yml index b29567c3d4..9023eee52c 100644 --- a/test/pipelines/pipelines-it-remote-linux-to-linux.yml +++ b/test/pipelines/pipelines-it-remote-linux-to-linux.yml @@ -62,7 +62,7 @@ jobs: - script: | set -e cd test - python3 nni_test/nnitest/generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) \ + python3 nni_test/nnitest/generate_ts_config.py --ts remote --remote_reuse $(remote_reuse) --remote_user $(docker_user) --remote_host $(remote_host) \ --remote_port $(cat port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip) cat config/training_service.yml PATH=$HOME/.local/bin:$PATH python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote diff --git a/test/pipelines/pipelines-it-remote-windows-to-linux.yml b/test/pipelines/pipelines-it-remote-windows-to-linux.yml index 710d0e405e..03f2126a95 100644 --- a/test/pipelines/pipelines-it-remote-windows-to-linux.yml +++ b/test/pipelines/pipelines-it-remote-windows-to-linux.yml @@ -48,7 +48,7 @@ jobs: displayName: 'Get docker port' - powershell: | cd test - python nni_test/nnitest/generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) --remote_port $(Get-Content port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip) + python nni_test/nnitest/generate_ts_config.py --ts remote --remote_reuse $(remote_reuse) --remote_user $(docker_user) --remote_host $(remote_host) --remote_port $(Get-Content port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip) Get-Content config/training_service.yml python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote --exclude cifar10 displayName: 'integration test' diff --git a/test/scripts/unittest.sh b/test/scripts/unittest.sh index ee66d86c85..68adbcefdb 100644 --- a/test/scripts/unittest.sh +++ b/test/scripts/unittest.sh @@ -7,8 +7,8 @@ CWD=${PWD} ## ------Run annotation test------ echo "" echo "===========================Testing: nni_annotation===========================" -cd ${CWD}/../tools/ -python3 -m unittest -v nni_annotation/test_annotation.py +#cd ${CWD}/../tools/ +#python3 -m unittest -v nni_annotation/test_annotation.py ## Export certain environment variables for unittest code to work export NNI_TRIAL_JOB_ID=test_trial_job_id @@ -17,23 +17,23 @@ export NNI_PLATFORM=unittest ## ------Run sdk test------ echo "" echo "===========================Testing: nni_sdk===========================" -cd ${CWD}/../src/sdk/pynni/ -python3 -m unittest discover -v tests +#cd ${CWD}/../src/sdk/pynni/ +#python3 -m unittest discover -v tests # -------------For typescript unittest------------- -cd ${CWD}/../src/nni_manager +#cd ${CWD}/../ts/nni_manager echo "" echo "===========================Testing: nni_manager===========================" -npm run test +#npm run test # -------------For NASUI unittest------------- -cd ${CWD}/../src/nasui +#cd ${CWD}/../ts/nasui echo "" echo "===========================Testing: nasui===========================" -CI=true npm test +#CI=true npm test ## ------Run nnictl unit test------ echo "" echo "===========================Testing: nnictl===========================" -cd ${CWD}/../tools/nni_cmd/ -python3 -m unittest discover -v tests +#cd ${CWD}/../tools/nni_cmd/ +#python3 -m unittest discover -v tests diff --git a/src/nasui/.gitignore b/ts/nasui/.gitignore similarity index 100% rename from src/nasui/.gitignore rename to ts/nasui/.gitignore diff --git a/src/nasui/assets/darts/graph.json b/ts/nasui/assets/darts/graph.json similarity index 100% rename from src/nasui/assets/darts/graph.json rename to ts/nasui/assets/darts/graph.json diff --git a/src/nasui/assets/darts/log b/ts/nasui/assets/darts/log similarity index 100% rename from src/nasui/assets/darts/log rename to ts/nasui/assets/darts/log diff --git a/src/nasui/assets/naive/graph.json b/ts/nasui/assets/naive/graph.json similarity index 100% rename from src/nasui/assets/naive/graph.json rename to ts/nasui/assets/naive/graph.json diff --git a/src/nasui/assets/naive/log b/ts/nasui/assets/naive/log similarity index 100% rename from src/nasui/assets/naive/log rename to ts/nasui/assets/naive/log diff --git a/src/nasui/package.json b/ts/nasui/package.json similarity index 100% rename from src/nasui/package.json rename to ts/nasui/package.json diff --git a/src/nasui/public/icon.png b/ts/nasui/public/icon.png similarity index 100% rename from src/nasui/public/icon.png rename to ts/nasui/public/icon.png diff --git a/src/nasui/public/index.html b/ts/nasui/public/index.html similarity index 100% rename from src/nasui/public/index.html rename to ts/nasui/public/index.html diff --git a/src/nasui/server.js b/ts/nasui/server.js similarity index 100% rename from src/nasui/server.js rename to ts/nasui/server.js diff --git a/src/nasui/src/App.css b/ts/nasui/src/App.css similarity index 100% rename from src/nasui/src/App.css rename to ts/nasui/src/App.css diff --git a/src/nasui/src/App.tsx b/ts/nasui/src/App.tsx similarity index 100% rename from src/nasui/src/App.tsx rename to ts/nasui/src/App.tsx diff --git a/src/nasui/src/Chart.tsx b/ts/nasui/src/Chart.tsx similarity index 100% rename from src/nasui/src/Chart.tsx rename to ts/nasui/src/Chart.tsx diff --git a/src/nasui/src/__tests__/graphUtils.ts b/ts/nasui/src/__tests__/graphUtils.ts similarity index 100% rename from src/nasui/src/__tests__/graphUtils.ts rename to ts/nasui/src/__tests__/graphUtils.ts diff --git a/src/nasui/src/graphUtils.ts b/ts/nasui/src/graphUtils.ts similarity index 100% rename from src/nasui/src/graphUtils.ts rename to ts/nasui/src/graphUtils.ts diff --git a/src/nasui/src/index.css b/ts/nasui/src/index.css similarity index 100% rename from src/nasui/src/index.css rename to ts/nasui/src/index.css diff --git a/src/nasui/src/index.tsx b/ts/nasui/src/index.tsx similarity index 100% rename from src/nasui/src/index.tsx rename to ts/nasui/src/index.tsx diff --git a/src/nasui/src/react-app-env.d.ts b/ts/nasui/src/react-app-env.d.ts similarity index 100% rename from src/nasui/src/react-app-env.d.ts rename to ts/nasui/src/react-app-env.d.ts diff --git a/src/nasui/src/serviceWorker.ts b/ts/nasui/src/serviceWorker.ts similarity index 100% rename from src/nasui/src/serviceWorker.ts rename to ts/nasui/src/serviceWorker.ts diff --git a/src/nasui/tsconfig.json b/ts/nasui/tsconfig.json similarity index 100% rename from src/nasui/tsconfig.json rename to ts/nasui/tsconfig.json diff --git a/src/nasui/types/cytoscape-dagre/index.d.ts b/ts/nasui/types/cytoscape-dagre/index.d.ts similarity index 100% rename from src/nasui/types/cytoscape-dagre/index.d.ts rename to ts/nasui/types/cytoscape-dagre/index.d.ts diff --git a/src/nasui/yarn.lock b/ts/nasui/yarn.lock similarity index 100% rename from src/nasui/yarn.lock rename to ts/nasui/yarn.lock diff --git a/src/nni_manager/.eslintrc b/ts/nni_manager/.eslintrc similarity index 100% rename from src/nni_manager/.eslintrc rename to ts/nni_manager/.eslintrc diff --git a/src/nni_manager/.gitignore b/ts/nni_manager/.gitignore similarity index 100% rename from src/nni_manager/.gitignore rename to ts/nni_manager/.gitignore diff --git a/src/nni_manager/common/component.ts b/ts/nni_manager/common/component.ts similarity index 100% rename from src/nni_manager/common/component.ts rename to ts/nni_manager/common/component.ts diff --git a/src/nni_manager/common/datastore.ts b/ts/nni_manager/common/datastore.ts similarity index 100% rename from src/nni_manager/common/datastore.ts rename to ts/nni_manager/common/datastore.ts diff --git a/src/nni_manager/common/errors.ts b/ts/nni_manager/common/errors.ts similarity index 100% rename from src/nni_manager/common/errors.ts rename to ts/nni_manager/common/errors.ts diff --git a/src/nni_manager/common/experimentStartupInfo.ts b/ts/nni_manager/common/experimentStartupInfo.ts similarity index 100% rename from src/nni_manager/common/experimentStartupInfo.ts rename to ts/nni_manager/common/experimentStartupInfo.ts diff --git a/src/nni_manager/common/log.ts b/ts/nni_manager/common/log.ts similarity index 100% rename from src/nni_manager/common/log.ts rename to ts/nni_manager/common/log.ts diff --git a/src/nni_manager/common/manager.ts b/ts/nni_manager/common/manager.ts similarity index 100% rename from src/nni_manager/common/manager.ts rename to ts/nni_manager/common/manager.ts diff --git a/src/nni_manager/common/observableTimer.ts b/ts/nni_manager/common/observableTimer.ts similarity index 100% rename from src/nni_manager/common/observableTimer.ts rename to ts/nni_manager/common/observableTimer.ts diff --git a/src/nni_manager/common/restServer.ts b/ts/nni_manager/common/restServer.ts similarity index 100% rename from src/nni_manager/common/restServer.ts rename to ts/nni_manager/common/restServer.ts diff --git a/src/nni_manager/common/trainingService.ts b/ts/nni_manager/common/trainingService.ts similarity index 100% rename from src/nni_manager/common/trainingService.ts rename to ts/nni_manager/common/trainingService.ts diff --git a/src/nni_manager/common/utils.ts b/ts/nni_manager/common/utils.ts similarity index 100% rename from src/nni_manager/common/utils.ts rename to ts/nni_manager/common/utils.ts diff --git a/src/nni_manager/config/aml/amlUtil.py b/ts/nni_manager/config/aml/amlUtil.py similarity index 100% rename from src/nni_manager/config/aml/amlUtil.py rename to ts/nni_manager/config/aml/amlUtil.py diff --git a/src/nni_manager/config/frameworkcontroller/frameworkcontrollerjob-crd-v1.json b/ts/nni_manager/config/frameworkcontroller/frameworkcontrollerjob-crd-v1.json similarity index 100% rename from src/nni_manager/config/frameworkcontroller/frameworkcontrollerjob-crd-v1.json rename to ts/nni_manager/config/frameworkcontroller/frameworkcontrollerjob-crd-v1.json diff --git a/src/nni_manager/config/kubeflow/pytorchjob-crd-v1.json b/ts/nni_manager/config/kubeflow/pytorchjob-crd-v1.json similarity index 100% rename from src/nni_manager/config/kubeflow/pytorchjob-crd-v1.json rename to ts/nni_manager/config/kubeflow/pytorchjob-crd-v1.json diff --git a/src/nni_manager/config/kubeflow/pytorchjob-crd-v1alpha2.json b/ts/nni_manager/config/kubeflow/pytorchjob-crd-v1alpha2.json similarity index 100% rename from src/nni_manager/config/kubeflow/pytorchjob-crd-v1alpha2.json rename to ts/nni_manager/config/kubeflow/pytorchjob-crd-v1alpha2.json diff --git a/src/nni_manager/config/kubeflow/pytorchjob-crd-v1beta1.json b/ts/nni_manager/config/kubeflow/pytorchjob-crd-v1beta1.json similarity index 100% rename from src/nni_manager/config/kubeflow/pytorchjob-crd-v1beta1.json rename to ts/nni_manager/config/kubeflow/pytorchjob-crd-v1beta1.json diff --git a/src/nni_manager/config/kubeflow/pytorchjob-crd-v1beta2.json b/ts/nni_manager/config/kubeflow/pytorchjob-crd-v1beta2.json similarity index 100% rename from src/nni_manager/config/kubeflow/pytorchjob-crd-v1beta2.json rename to ts/nni_manager/config/kubeflow/pytorchjob-crd-v1beta2.json diff --git a/src/nni_manager/config/kubeflow/tfjob-crd-v1.json b/ts/nni_manager/config/kubeflow/tfjob-crd-v1.json similarity index 100% rename from src/nni_manager/config/kubeflow/tfjob-crd-v1.json rename to ts/nni_manager/config/kubeflow/tfjob-crd-v1.json diff --git a/src/nni_manager/config/kubeflow/tfjob-crd-v1alpha2.json b/ts/nni_manager/config/kubeflow/tfjob-crd-v1alpha2.json similarity index 100% rename from src/nni_manager/config/kubeflow/tfjob-crd-v1alpha2.json rename to ts/nni_manager/config/kubeflow/tfjob-crd-v1alpha2.json diff --git a/src/nni_manager/config/kubeflow/tfjob-crd-v1beta1.json b/ts/nni_manager/config/kubeflow/tfjob-crd-v1beta1.json similarity index 100% rename from src/nni_manager/config/kubeflow/tfjob-crd-v1beta1.json rename to ts/nni_manager/config/kubeflow/tfjob-crd-v1beta1.json diff --git a/src/nni_manager/config/kubeflow/tfjob-crd-v1beta2.json b/ts/nni_manager/config/kubeflow/tfjob-crd-v1beta2.json similarity index 100% rename from src/nni_manager/config/kubeflow/tfjob-crd-v1beta2.json rename to ts/nni_manager/config/kubeflow/tfjob-crd-v1beta2.json diff --git a/src/nni_manager/core/commands.ts b/ts/nni_manager/core/commands.ts similarity index 100% rename from src/nni_manager/core/commands.ts rename to ts/nni_manager/core/commands.ts diff --git a/src/nni_manager/core/ipcInterface.ts b/ts/nni_manager/core/ipcInterface.ts similarity index 100% rename from src/nni_manager/core/ipcInterface.ts rename to ts/nni_manager/core/ipcInterface.ts diff --git a/src/nni_manager/core/nniDataStore.ts b/ts/nni_manager/core/nniDataStore.ts similarity index 100% rename from src/nni_manager/core/nniDataStore.ts rename to ts/nni_manager/core/nniDataStore.ts diff --git a/src/nni_manager/core/nnimanager.ts b/ts/nni_manager/core/nnimanager.ts similarity index 100% rename from src/nni_manager/core/nnimanager.ts rename to ts/nni_manager/core/nnimanager.ts diff --git a/src/nni_manager/core/sqlDatabase.ts b/ts/nni_manager/core/sqlDatabase.ts similarity index 100% rename from src/nni_manager/core/sqlDatabase.ts rename to ts/nni_manager/core/sqlDatabase.ts diff --git a/src/nni_manager/core/test/assessor.py b/ts/nni_manager/core/test/assessor.py similarity index 100% rename from src/nni_manager/core/test/assessor.py rename to ts/nni_manager/core/test/assessor.py diff --git a/src/nni_manager/core/test/dataStore.test.ts b/ts/nni_manager/core/test/dataStore.test.ts similarity index 100% rename from src/nni_manager/core/test/dataStore.test.ts rename to ts/nni_manager/core/test/dataStore.test.ts diff --git a/src/nni_manager/core/test/dummy_assessor.py b/ts/nni_manager/core/test/dummy_assessor.py similarity index 100% rename from src/nni_manager/core/test/dummy_assessor.py rename to ts/nni_manager/core/test/dummy_assessor.py diff --git a/src/nni_manager/core/test/dummy_tuner.py b/ts/nni_manager/core/test/dummy_tuner.py similarity index 100% rename from src/nni_manager/core/test/dummy_tuner.py rename to ts/nni_manager/core/test/dummy_tuner.py diff --git a/src/nni_manager/core/test/import_all.test.ts b/ts/nni_manager/core/test/import_all.test.ts similarity index 100% rename from src/nni_manager/core/test/import_all.test.ts rename to ts/nni_manager/core/test/import_all.test.ts diff --git a/src/nni_manager/core/test/ipcInterface.test.ts b/ts/nni_manager/core/test/ipcInterface.test.ts similarity index 100% rename from src/nni_manager/core/test/ipcInterface.test.ts rename to ts/nni_manager/core/test/ipcInterface.test.ts diff --git a/src/nni_manager/core/test/ipcInterfaceTerminate.test.ts b/ts/nni_manager/core/test/ipcInterfaceTerminate.test.ts similarity index 100% rename from src/nni_manager/core/test/ipcInterfaceTerminate.test.ts rename to ts/nni_manager/core/test/ipcInterfaceTerminate.test.ts diff --git a/src/nni_manager/core/test/mockedDatastore.ts b/ts/nni_manager/core/test/mockedDatastore.ts similarity index 100% rename from src/nni_manager/core/test/mockedDatastore.ts rename to ts/nni_manager/core/test/mockedDatastore.ts diff --git a/src/nni_manager/core/test/mockedTrainingService.ts b/ts/nni_manager/core/test/mockedTrainingService.ts similarity index 100% rename from src/nni_manager/core/test/mockedTrainingService.ts rename to ts/nni_manager/core/test/mockedTrainingService.ts diff --git a/src/nni_manager/core/test/nnimanager.test.ts b/ts/nni_manager/core/test/nnimanager.test.ts similarity index 100% rename from src/nni_manager/core/test/nnimanager.test.ts rename to ts/nni_manager/core/test/nnimanager.test.ts diff --git a/src/nni_manager/core/test/sqlDatabase.test.ts b/ts/nni_manager/core/test/sqlDatabase.test.ts similarity index 100% rename from src/nni_manager/core/test/sqlDatabase.test.ts rename to ts/nni_manager/core/test/sqlDatabase.test.ts diff --git a/src/nni_manager/main.ts b/ts/nni_manager/main.ts similarity index 100% rename from src/nni_manager/main.ts rename to ts/nni_manager/main.ts diff --git a/src/nni_manager/package.json b/ts/nni_manager/package.json similarity index 99% rename from src/nni_manager/package.json rename to ts/nni_manager/package.json index 11f4adfc5b..aabb58e6d9 100644 --- a/src/nni_manager/package.json +++ b/ts/nni_manager/package.json @@ -12,7 +12,6 @@ "license": "MIT", "dependencies": { "azure-storage": "^2.10.2", - "chai-as-promised": "^7.1.1", "child-process-promise": "^2.2.1", "express": "^4.16.3", "express-joi-validator": "^2.0.0", @@ -21,7 +20,7 @@ "kubernetes-client": "^6.5.0", "python-shell": "^2.0.1", "rx": "^4.1.0", - "sqlite3": "^4.0.2", + "sqlite3": "^5.0.0", "ssh2": "^0.6.1", "stream-buffers": "^3.0.2", "tail-stream": "^0.3.4", @@ -53,6 +52,7 @@ "@typescript-eslint/eslint-plugin": "^2.10.0", "@typescript-eslint/parser": "^2.10.0", "chai": "^4.1.2", + "chai-as-promised": "^7.1.1", "eslint": "^6.7.2", "glob": "^7.1.3", "mocha": "^8.1.3", diff --git a/src/nni_manager/rest_server/nniRestServer.ts b/ts/nni_manager/rest_server/nniRestServer.ts similarity index 100% rename from src/nni_manager/rest_server/nniRestServer.ts rename to ts/nni_manager/rest_server/nniRestServer.ts diff --git a/src/nni_manager/rest_server/restHandler.ts b/ts/nni_manager/rest_server/restHandler.ts similarity index 100% rename from src/nni_manager/rest_server/restHandler.ts rename to ts/nni_manager/rest_server/restHandler.ts diff --git a/src/nni_manager/rest_server/restValidationSchemas.ts b/ts/nni_manager/rest_server/restValidationSchemas.ts similarity index 100% rename from src/nni_manager/rest_server/restValidationSchemas.ts rename to ts/nni_manager/rest_server/restValidationSchemas.ts diff --git a/src/nni_manager/rest_server/test/mockedNNIManager.ts b/ts/nni_manager/rest_server/test/mockedNNIManager.ts similarity index 100% rename from src/nni_manager/rest_server/test/mockedNNIManager.ts rename to ts/nni_manager/rest_server/test/mockedNNIManager.ts diff --git a/src/nni_manager/rest_server/test/restserver.test.ts b/ts/nni_manager/rest_server/test/restserver.test.ts similarity index 100% rename from src/nni_manager/rest_server/test/restserver.test.ts rename to ts/nni_manager/rest_server/test/restserver.test.ts diff --git a/src/nni_manager/training_service/common/clusterJobRestServer.ts b/ts/nni_manager/training_service/common/clusterJobRestServer.ts similarity index 100% rename from src/nni_manager/training_service/common/clusterJobRestServer.ts rename to ts/nni_manager/training_service/common/clusterJobRestServer.ts diff --git a/src/nni_manager/training_service/common/containerJobData.ts b/ts/nni_manager/training_service/common/containerJobData.ts similarity index 100% rename from src/nni_manager/training_service/common/containerJobData.ts rename to ts/nni_manager/training_service/common/containerJobData.ts diff --git a/src/nni_manager/training_service/common/gpuData.ts b/ts/nni_manager/training_service/common/gpuData.ts similarity index 95% rename from src/nni_manager/training_service/common/gpuData.ts rename to ts/nni_manager/training_service/common/gpuData.ts index 835569255e..8767baf6e4 100644 --- a/src/nni_manager/training_service/common/gpuData.ts +++ b/ts/nni_manager/training_service/common/gpuData.ts @@ -79,6 +79,6 @@ export function parseGpuIndices(gpuIndices?: string): Set | undefined { export const GPU_INFO_COLLECTOR_FORMAT_WINDOWS: string = ` $env:METRIC_OUTPUT_DIR="{0}" -$app = Start-Process "python" -ArgumentList "-m nni_gpu_tool.gpu_metrics_collector" -passthru -NoNewWindow +$app = Start-Process "python" -ArgumentList "-m nni.tools.gpu_tool.gpu_metrics_collector" -passthru -NoNewWindow Write $app.ID | Out-File {1} -NoNewline -encoding utf8 `; diff --git a/src/nni_manager/training_service/common/jobMetrics.ts b/ts/nni_manager/training_service/common/jobMetrics.ts similarity index 100% rename from src/nni_manager/training_service/common/jobMetrics.ts rename to ts/nni_manager/training_service/common/jobMetrics.ts diff --git a/src/nni_manager/training_service/common/trialConfig.ts b/ts/nni_manager/training_service/common/trialConfig.ts similarity index 100% rename from src/nni_manager/training_service/common/trialConfig.ts rename to ts/nni_manager/training_service/common/trialConfig.ts diff --git a/src/nni_manager/training_service/common/trialConfigMetadataKey.ts b/ts/nni_manager/training_service/common/trialConfigMetadataKey.ts similarity index 100% rename from src/nni_manager/training_service/common/trialConfigMetadataKey.ts rename to ts/nni_manager/training_service/common/trialConfigMetadataKey.ts diff --git a/src/nni_manager/training_service/common/util.ts b/ts/nni_manager/training_service/common/util.ts similarity index 99% rename from src/nni_manager/training_service/common/util.ts rename to ts/nni_manager/training_service/common/util.ts index 61bd3cb622..791d7dcebb 100644 --- a/src/nni_manager/training_service/common/util.ts +++ b/ts/nni_manager/training_service/common/util.ts @@ -236,7 +236,7 @@ export function getScriptName(fileNamePrefix: string): string { } export function getGpuMetricsCollectorBashScriptContent(scriptFolder: string): string { - return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni_gpu_tool.gpu_metrics_collector`; + return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni.tools.gpu_tool.gpu_metrics_collector`; } export function runGpuMetricsCollector(scriptFolder: string): void { diff --git a/src/nni_manager/training_service/dlts/dltsClusterConfig.ts b/ts/nni_manager/training_service/dlts/dltsClusterConfig.ts similarity index 100% rename from src/nni_manager/training_service/dlts/dltsClusterConfig.ts rename to ts/nni_manager/training_service/dlts/dltsClusterConfig.ts diff --git a/src/nni_manager/training_service/dlts/dltsData.ts b/ts/nni_manager/training_service/dlts/dltsData.ts similarity index 82% rename from src/nni_manager/training_service/dlts/dltsData.ts rename to ts/nni_manager/training_service/dlts/dltsData.ts index f658414535..3a59c3d8d2 100644 --- a/src/nni_manager/training_service/dlts/dltsData.ts +++ b/ts/nni_manager/training_service/dlts/dltsData.ts @@ -4,5 +4,5 @@ export const DLTS_TRIAL_COMMAND_FORMAT: string = `export NNI_PLATFORM=dlts NNI_SYS_DIR={0} NNI_OUTPUT_DIR={1} NNI_TRIAL_JOB_ID={2} NNI_EXP_ID={3} NNI_TRIAL_SEQ_ID={4} MULTI_PHASE={5} \ && cd $NNI_SYS_DIR && sh install_nni.sh \ -&& cd '{6}' && python3 -m nni_trial_tool.trial_keeper --trial_command '{7}' \ +&& cd '{6}' && python3 -m nni.tools.trial_tool.trial_keeper --trial_command '{7}' \ --nnimanager_ip '{8}' --nnimanager_port '{9}' --nni_manager_version '{10}' --log_collection '{11}'`; diff --git a/src/nni_manager/training_service/dlts/dltsJobConfig.ts b/ts/nni_manager/training_service/dlts/dltsJobConfig.ts similarity index 100% rename from src/nni_manager/training_service/dlts/dltsJobConfig.ts rename to ts/nni_manager/training_service/dlts/dltsJobConfig.ts diff --git a/src/nni_manager/training_service/dlts/dltsJobRestServer.ts b/ts/nni_manager/training_service/dlts/dltsJobRestServer.ts similarity index 100% rename from src/nni_manager/training_service/dlts/dltsJobRestServer.ts rename to ts/nni_manager/training_service/dlts/dltsJobRestServer.ts diff --git a/src/nni_manager/training_service/dlts/dltsTrainingService.ts b/ts/nni_manager/training_service/dlts/dltsTrainingService.ts similarity index 100% rename from src/nni_manager/training_service/dlts/dltsTrainingService.ts rename to ts/nni_manager/training_service/dlts/dltsTrainingService.ts diff --git a/src/nni_manager/training_service/dlts/dltsTrialConfig.ts b/ts/nni_manager/training_service/dlts/dltsTrialConfig.ts similarity index 100% rename from src/nni_manager/training_service/dlts/dltsTrialConfig.ts rename to ts/nni_manager/training_service/dlts/dltsTrialConfig.ts diff --git a/src/nni_manager/training_service/dlts/dltsTrialJobDetail.ts b/ts/nni_manager/training_service/dlts/dltsTrialJobDetail.ts similarity index 100% rename from src/nni_manager/training_service/dlts/dltsTrialJobDetail.ts rename to ts/nni_manager/training_service/dlts/dltsTrialJobDetail.ts diff --git a/src/nni_manager/training_service/kubernetes/azureStorageClientUtils.ts b/ts/nni_manager/training_service/kubernetes/azureStorageClientUtils.ts similarity index 100% rename from src/nni_manager/training_service/kubernetes/azureStorageClientUtils.ts rename to ts/nni_manager/training_service/kubernetes/azureStorageClientUtils.ts diff --git a/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerApiClient.ts b/ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerApiClient.ts similarity index 100% rename from src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerApiClient.ts rename to ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerApiClient.ts diff --git a/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerConfig.ts b/ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerConfig.ts similarity index 100% rename from src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerConfig.ts rename to ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerConfig.ts diff --git a/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerJobInfoCollector.ts b/ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerJobInfoCollector.ts similarity index 100% rename from src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerJobInfoCollector.ts rename to ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerJobInfoCollector.ts diff --git a/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerJobRestServer.ts b/ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerJobRestServer.ts similarity index 100% rename from src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerJobRestServer.ts rename to ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerJobRestServer.ts diff --git a/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts b/ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts similarity index 100% rename from src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts rename to ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts diff --git a/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowApiClient.ts b/ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowApiClient.ts similarity index 100% rename from src/nni_manager/training_service/kubernetes/kubeflow/kubeflowApiClient.ts rename to ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowApiClient.ts diff --git a/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowConfig.ts b/ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowConfig.ts similarity index 100% rename from src/nni_manager/training_service/kubernetes/kubeflow/kubeflowConfig.ts rename to ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowConfig.ts diff --git a/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowJobInfoCollector.ts b/ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowJobInfoCollector.ts similarity index 100% rename from src/nni_manager/training_service/kubernetes/kubeflow/kubeflowJobInfoCollector.ts rename to ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowJobInfoCollector.ts diff --git a/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowJobRestServer.ts b/ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowJobRestServer.ts similarity index 100% rename from src/nni_manager/training_service/kubernetes/kubeflow/kubeflowJobRestServer.ts rename to ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowJobRestServer.ts diff --git a/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts b/ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts similarity index 100% rename from src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts rename to ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts diff --git a/src/nni_manager/training_service/kubernetes/kubernetesApiClient.ts b/ts/nni_manager/training_service/kubernetes/kubernetesApiClient.ts similarity index 100% rename from src/nni_manager/training_service/kubernetes/kubernetesApiClient.ts rename to ts/nni_manager/training_service/kubernetes/kubernetesApiClient.ts diff --git a/src/nni_manager/training_service/kubernetes/kubernetesConfig.ts b/ts/nni_manager/training_service/kubernetes/kubernetesConfig.ts similarity index 100% rename from src/nni_manager/training_service/kubernetes/kubernetesConfig.ts rename to ts/nni_manager/training_service/kubernetes/kubernetesConfig.ts diff --git a/src/nni_manager/training_service/kubernetes/kubernetesData.ts b/ts/nni_manager/training_service/kubernetes/kubernetesData.ts similarity index 93% rename from src/nni_manager/training_service/kubernetes/kubernetesData.ts rename to ts/nni_manager/training_service/kubernetes/kubernetesData.ts index 6f0f3f4342..eb96202aed 100644 --- a/src/nni_manager/training_service/kubernetes/kubernetesData.ts +++ b/ts/nni_manager/training_service/kubernetes/kubernetesData.ts @@ -52,5 +52,5 @@ mkdir -p $NNI_OUTPUT_DIR cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR/code sh $NNI_SYS_DIR/install_nni.sh cd $NNI_SYS_DIR/code -python3 -m nni_trial_tool.trial_keeper --trial_command '{8}' --nnimanager_ip {9} --nnimanager_port {10} \ +python3 -m nni.tools.trial_tool.trial_keeper --trial_command '{8}' --nnimanager_ip {9} --nnimanager_port {10} \ --nni_manager_version '{11}' --log_collection '{12}' 1>$NNI_OUTPUT_DIR/trialkeeper_stdout 2>$NNI_OUTPUT_DIR/trialkeeper_stderr`; diff --git a/src/nni_manager/training_service/kubernetes/kubernetesJobInfoCollector.ts b/ts/nni_manager/training_service/kubernetes/kubernetesJobInfoCollector.ts similarity index 100% rename from src/nni_manager/training_service/kubernetes/kubernetesJobInfoCollector.ts rename to ts/nni_manager/training_service/kubernetes/kubernetesJobInfoCollector.ts diff --git a/src/nni_manager/training_service/kubernetes/kubernetesJobRestServer.ts b/ts/nni_manager/training_service/kubernetes/kubernetesJobRestServer.ts similarity index 100% rename from src/nni_manager/training_service/kubernetes/kubernetesJobRestServer.ts rename to ts/nni_manager/training_service/kubernetes/kubernetesJobRestServer.ts diff --git a/src/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts b/ts/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts similarity index 100% rename from src/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts rename to ts/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts diff --git a/src/nni_manager/training_service/local/gpuScheduler.ts b/ts/nni_manager/training_service/local/gpuScheduler.ts similarity index 100% rename from src/nni_manager/training_service/local/gpuScheduler.ts rename to ts/nni_manager/training_service/local/gpuScheduler.ts diff --git a/src/nni_manager/training_service/local/localTrainingService.ts b/ts/nni_manager/training_service/local/localTrainingService.ts similarity index 100% rename from src/nni_manager/training_service/local/localTrainingService.ts rename to ts/nni_manager/training_service/local/localTrainingService.ts diff --git a/src/nni_manager/training_service/pai/paiConfig.ts b/ts/nni_manager/training_service/pai/paiConfig.ts similarity index 100% rename from src/nni_manager/training_service/pai/paiConfig.ts rename to ts/nni_manager/training_service/pai/paiConfig.ts diff --git a/src/nni_manager/training_service/pai/paiJobInfoCollector.ts b/ts/nni_manager/training_service/pai/paiJobInfoCollector.ts similarity index 100% rename from src/nni_manager/training_service/pai/paiJobInfoCollector.ts rename to ts/nni_manager/training_service/pai/paiJobInfoCollector.ts diff --git a/src/nni_manager/training_service/pai/paiJobRestServer.ts b/ts/nni_manager/training_service/pai/paiJobRestServer.ts similarity index 100% rename from src/nni_manager/training_service/pai/paiJobRestServer.ts rename to ts/nni_manager/training_service/pai/paiJobRestServer.ts diff --git a/src/nni_manager/training_service/pai/paiK8S/paiK8SConfig.ts b/ts/nni_manager/training_service/pai/paiK8S/paiK8SConfig.ts similarity index 100% rename from src/nni_manager/training_service/pai/paiK8S/paiK8SConfig.ts rename to ts/nni_manager/training_service/pai/paiK8S/paiK8SConfig.ts diff --git a/src/nni_manager/training_service/pai/paiK8S/paiK8SData.ts b/ts/nni_manager/training_service/pai/paiK8S/paiK8SData.ts similarity index 83% rename from src/nni_manager/training_service/pai/paiK8S/paiK8SData.ts rename to ts/nni_manager/training_service/pai/paiK8S/paiK8SData.ts index 2c6b9f3d66..1d4aef9423 100644 --- a/src/nni_manager/training_service/pai/paiK8S/paiK8SData.ts +++ b/ts/nni_manager/training_service/pai/paiK8S/paiK8SData.ts @@ -16,5 +16,5 @@ fi`; export const PAI_K8S_TRIAL_COMMAND_FORMAT: string = `export NNI_PLATFORM=pai NNI_SYS_DIR={0} NNI_OUTPUT_DIR={1} NNI_TRIAL_JOB_ID={2} NNI_EXP_ID={3} NNI_TRIAL_SEQ_ID={4} MULTI_PHASE={5} \ && NNI_CODE_DIR={6} && mkdir -p $NNI_SYS_DIR/code && cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR/code && sh $NNI_SYS_DIR/install_nni.sh \ -&& cd $NNI_SYS_DIR/code && python3 -m nni_trial_tool.trial_keeper --trial_command '{7}' --nnimanager_ip '{8}' --nnimanager_port '{9}' \ +&& cd $NNI_SYS_DIR/code && python3 -m nni.tools.trial_tool.trial_keeper --trial_command '{7}' --nnimanager_ip '{8}' --nnimanager_port '{9}' \ --nni_manager_version '{10}' --log_collection '{11}' | tee $NNI_OUTPUT_DIR/trial.log`; diff --git a/src/nni_manager/training_service/pai/paiK8S/paiK8STrainingService.ts b/ts/nni_manager/training_service/pai/paiK8S/paiK8STrainingService.ts similarity index 100% rename from src/nni_manager/training_service/pai/paiK8S/paiK8STrainingService.ts rename to ts/nni_manager/training_service/pai/paiK8S/paiK8STrainingService.ts diff --git a/src/nni_manager/training_service/pai/paiTrainingService.ts b/ts/nni_manager/training_service/pai/paiTrainingService.ts similarity index 100% rename from src/nni_manager/training_service/pai/paiTrainingService.ts rename to ts/nni_manager/training_service/pai/paiTrainingService.ts diff --git a/src/nni_manager/training_service/pai/paiYarn/hdfsClientUtility.ts b/ts/nni_manager/training_service/pai/paiYarn/hdfsClientUtility.ts similarity index 100% rename from src/nni_manager/training_service/pai/paiYarn/hdfsClientUtility.ts rename to ts/nni_manager/training_service/pai/paiYarn/hdfsClientUtility.ts diff --git a/src/nni_manager/training_service/pai/paiYarn/paiYarnConfig.ts b/ts/nni_manager/training_service/pai/paiYarn/paiYarnConfig.ts similarity index 100% rename from src/nni_manager/training_service/pai/paiYarn/paiYarnConfig.ts rename to ts/nni_manager/training_service/pai/paiYarn/paiYarnConfig.ts diff --git a/src/nni_manager/training_service/pai/paiYarn/paiYarnData.ts b/ts/nni_manager/training_service/pai/paiYarn/paiYarnData.ts similarity index 100% rename from src/nni_manager/training_service/pai/paiYarn/paiYarnData.ts rename to ts/nni_manager/training_service/pai/paiYarn/paiYarnData.ts diff --git a/src/nni_manager/training_service/pai/paiYarn/paiYarnTrainingService.ts b/ts/nni_manager/training_service/pai/paiYarn/paiYarnTrainingService.ts similarity index 100% rename from src/nni_manager/training_service/pai/paiYarn/paiYarnTrainingService.ts rename to ts/nni_manager/training_service/pai/paiYarn/paiYarnTrainingService.ts diff --git a/src/nni_manager/training_service/pai/paiYarn/paiYarnTrialConfig.ts b/ts/nni_manager/training_service/pai/paiYarn/paiYarnTrialConfig.ts similarity index 100% rename from src/nni_manager/training_service/pai/paiYarn/paiYarnTrialConfig.ts rename to ts/nni_manager/training_service/pai/paiYarn/paiYarnTrialConfig.ts diff --git a/src/nni_manager/training_service/remote_machine/extends/linuxCommands.ts b/ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts similarity index 93% rename from src/nni_manager/training_service/remote_machine/extends/linuxCommands.ts rename to ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts index 0dab8a63ea..6c95c4d0bf 100644 --- a/src/nni_manager/training_service/remote_machine/extends/linuxCommands.ts +++ b/ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts @@ -26,7 +26,7 @@ class LinuxCommands extends OsCommands { cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR/code sh $NNI_SYS_DIR/install_nni.sh cd $NNI_SYS_DIR/code - python3 -m nni_trial_tool.trial_keeper --trial_command '${cudaVisibleSetting} ${command}' --nnimanager_ip '${nniManagerAddress}' \ + python3 -m nni.tools.trial_tool.trial_keeper --trial_command '${cudaVisibleSetting} ${command}' --nnimanager_ip '${nniManagerAddress}' \ --nnimanager_port '${nniManagerPort}' --nni_manager_version '${nniManagerVersion}' \ --job_id_file ${jobIdFileName} \ --log_collection '${logCollection}' 1>$NNI_OUTPUT_DIR/trialkeeper_stdout 2>$NNI_OUTPUT_DIR/trialkeeper_stderr @@ -34,7 +34,7 @@ class LinuxCommands extends OsCommands { } public generateGpuStatsScript(scriptFolder: string): string { - return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni_gpu_tool.gpu_metrics_collector`; + return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni.tools.gpu_tool.gpu_metrics_collector`; } public createFolder(folderName: string, sharedFolder: boolean = false): string { @@ -136,6 +136,10 @@ class LinuxCommands extends OsCommands { return `${preCommand} && ${command}`; } } + + public fileExistCommand(filePath: string): string { + return `test -e ${filePath} && echo True || echo False`; + } } export { LinuxCommands }; diff --git a/src/nni_manager/training_service/remote_machine/extends/windowsCommands.ts b/ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts similarity index 87% rename from src/nni_manager/training_service/remote_machine/extends/windowsCommands.ts rename to ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts index 2a81157e29..c75653b8be 100644 --- a/src/nni_manager/training_service/remote_machine/extends/windowsCommands.ts +++ b/ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts @@ -38,7 +38,7 @@ class WindowsCommands extends OsCommands { ) echo starting script - python -m nni_trial_tool.trial_keeper --trial_command "${command}" --nnimanager_ip "${nniManagerAddress}" --nnimanager_port "${nniManagerPort}" --nni_manager_version "${nniManagerVersion}" --log_collection "${logCollection}" --job_id_file ${jobIdFileName} 1>%NNI_OUTPUT_DIR%/trialkeeper_stdout 2>%NNI_OUTPUT_DIR%/trialkeeper_stderr + python -m nni.tools.trial_tool.trial_keeper --trial_command "${command}" --nnimanager_ip "${nniManagerAddress}" --nnimanager_port "${nniManagerPort}" --nni_manager_version "${nniManagerVersion}" --log_collection "${logCollection}" --job_id_file ${jobIdFileName} 1>%NNI_OUTPUT_DIR%/trialkeeper_stdout 2>%NNI_OUTPUT_DIR%/trialkeeper_stderr echo save exit code(%ERRORLEVEL%) and time echo|set /p="%ERRORLEVEL% " > ${exitCodeFile} @@ -46,7 +46,7 @@ class WindowsCommands extends OsCommands { } public generateGpuStatsScript(scriptFolder: string): string { - return `powershell -command $env:Path=If($env:prePath){$env:prePath}Else{$env:Path};$env:METRIC_OUTPUT_DIR='${scriptFolder}';$app = Start-Process -FilePath python -NoNewWindow -passthru -ArgumentList '-m nni_gpu_tool.gpu_metrics_collector' -RedirectStandardOutput ${scriptFolder}\\scriptstdout -RedirectStandardError ${scriptFolder}\\scriptstderr;Write $PID ^| Out-File ${scriptFolder}\\pid -NoNewline -encoding utf8;wait-process $app.ID`; + return `powershell -command $env:Path=If($env:prePath){$env:prePath}Else{$env:Path};$env:METRIC_OUTPUT_DIR='${scriptFolder}';$app = Start-Process -FilePath python -NoNewWindow -passthru -ArgumentList '-m nni.tools.gpu_tool.gpu_metrics_collector' -RedirectStandardOutput ${scriptFolder}\\scriptstdout -RedirectStandardError ${scriptFolder}\\scriptstderr;Write $PID ^| Out-File ${scriptFolder}\\pid -NoNewline -encoding utf8;wait-process $app.ID`; } public createFolder(folderName: string, sharedFolder: boolean = false): string { @@ -130,6 +130,10 @@ class WindowsCommands extends OsCommands { return `${preCommand} && set prePath=%path% && ${command}`; } } + + public fileExistCommand(filePath: string): string { + return `powershell Test-Path ${filePath} -PathType Leaf`; + } } export { WindowsCommands }; diff --git a/src/nni_manager/training_service/remote_machine/gpuScheduler.ts b/ts/nni_manager/training_service/remote_machine/gpuScheduler.ts similarity index 100% rename from src/nni_manager/training_service/remote_machine/gpuScheduler.ts rename to ts/nni_manager/training_service/remote_machine/gpuScheduler.ts diff --git a/src/nni_manager/training_service/remote_machine/osCommands.ts b/ts/nni_manager/training_service/remote_machine/osCommands.ts similarity index 96% rename from src/nni_manager/training_service/remote_machine/osCommands.ts rename to ts/nni_manager/training_service/remote_machine/osCommands.ts index cb110c6694..769135d9fc 100644 --- a/src/nni_manager/training_service/remote_machine/osCommands.ts +++ b/ts/nni_manager/training_service/remote_machine/osCommands.ts @@ -29,6 +29,7 @@ abstract class OsCommands { public abstract extractFile(tarFileName: string, targetFolder: string): string; public abstract executeScript(script: string, isFile: boolean): string; public abstract addPreCommand(preCommand: string | undefined, command: string | undefined): string | undefined; + public abstract fileExistCommand(filePath: string): string | undefined; public joinPath(...paths: string[]): string { let dir: string = paths.filter((path: any) => path !== '').join(this.pathSpliter); diff --git a/src/nni_manager/training_service/remote_machine/remoteMachineData.ts b/ts/nni_manager/training_service/remote_machine/remoteMachineData.ts similarity index 100% rename from src/nni_manager/training_service/remote_machine/remoteMachineData.ts rename to ts/nni_manager/training_service/remote_machine/remoteMachineData.ts diff --git a/src/nni_manager/training_service/remote_machine/remoteMachineJobRestServer.ts b/ts/nni_manager/training_service/remote_machine/remoteMachineJobRestServer.ts similarity index 100% rename from src/nni_manager/training_service/remote_machine/remoteMachineJobRestServer.ts rename to ts/nni_manager/training_service/remote_machine/remoteMachineJobRestServer.ts diff --git a/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts b/ts/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts similarity index 100% rename from src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts rename to ts/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts diff --git a/src/nni_manager/training_service/remote_machine/shellExecutor.ts b/ts/nni_manager/training_service/remote_machine/shellExecutor.ts similarity index 98% rename from src/nni_manager/training_service/remote_machine/shellExecutor.ts rename to ts/nni_manager/training_service/remote_machine/shellExecutor.ts index b093dfcd0e..d262ea500f 100644 --- a/src/nni_manager/training_service/remote_machine/shellExecutor.ts +++ b/ts/nni_manager/training_service/remote_machine/shellExecutor.ts @@ -238,6 +238,12 @@ class ShellExecutor { return commandResult.exitCode == 0; } + public async fileExist(filePath: string): Promise { + const commandText = this.osCommands && this.osCommands.fileExistCommand(filePath); + const commandResult = await this.execute(commandText); + return commandResult.stdout !== undefined && commandResult.stdout.trim() === 'True'; + } + public async extractFile(tarFileName: string, targetFolder: string): Promise { const commandText = this.osCommands && this.osCommands.extractFile(tarFileName, targetFolder); const commandResult = await this.execute(commandText); diff --git a/src/nni_manager/training_service/remote_machine/test/linuxCommands.test.ts b/ts/nni_manager/training_service/remote_machine/test/linuxCommands.test.ts similarity index 100% rename from src/nni_manager/training_service/remote_machine/test/linuxCommands.test.ts rename to ts/nni_manager/training_service/remote_machine/test/linuxCommands.test.ts diff --git a/src/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts b/ts/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts similarity index 100% rename from src/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts rename to ts/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts diff --git a/src/nni_manager/training_service/remote_machine/test/windowsCommands.test.ts b/ts/nni_manager/training_service/remote_machine/test/windowsCommands.test.ts similarity index 100% rename from src/nni_manager/training_service/remote_machine/test/windowsCommands.test.ts rename to ts/nni_manager/training_service/remote_machine/test/windowsCommands.test.ts diff --git a/src/nni_manager/training_service/reusable/aml/amlClient.ts b/ts/nni_manager/training_service/reusable/aml/amlClient.ts similarity index 100% rename from src/nni_manager/training_service/reusable/aml/amlClient.ts rename to ts/nni_manager/training_service/reusable/aml/amlClient.ts diff --git a/src/nni_manager/training_service/reusable/aml/amlConfig.ts b/ts/nni_manager/training_service/reusable/aml/amlConfig.ts similarity index 100% rename from src/nni_manager/training_service/reusable/aml/amlConfig.ts rename to ts/nni_manager/training_service/reusable/aml/amlConfig.ts diff --git a/src/nni_manager/training_service/reusable/channels/amlCommandChannel.ts b/ts/nni_manager/training_service/reusable/channels/amlCommandChannel.ts similarity index 100% rename from src/nni_manager/training_service/reusable/channels/amlCommandChannel.ts rename to ts/nni_manager/training_service/reusable/channels/amlCommandChannel.ts diff --git a/src/nni_manager/training_service/reusable/channels/fileCommandChannel.ts b/ts/nni_manager/training_service/reusable/channels/fileCommandChannel.ts similarity index 100% rename from src/nni_manager/training_service/reusable/channels/fileCommandChannel.ts rename to ts/nni_manager/training_service/reusable/channels/fileCommandChannel.ts diff --git a/src/nni_manager/training_service/reusable/channels/webCommandChannel.ts b/ts/nni_manager/training_service/reusable/channels/webCommandChannel.ts similarity index 100% rename from src/nni_manager/training_service/reusable/channels/webCommandChannel.ts rename to ts/nni_manager/training_service/reusable/channels/webCommandChannel.ts diff --git a/src/nni_manager/training_service/reusable/commandChannel.ts b/ts/nni_manager/training_service/reusable/commandChannel.ts similarity index 100% rename from src/nni_manager/training_service/reusable/commandChannel.ts rename to ts/nni_manager/training_service/reusable/commandChannel.ts diff --git a/src/nni_manager/training_service/reusable/environment.ts b/ts/nni_manager/training_service/reusable/environment.ts similarity index 100% rename from src/nni_manager/training_service/reusable/environment.ts rename to ts/nni_manager/training_service/reusable/environment.ts diff --git a/src/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts b/ts/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts similarity index 100% rename from src/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts rename to ts/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts diff --git a/src/nni_manager/training_service/reusable/environments/openPaiEnvironmentService.ts b/ts/nni_manager/training_service/reusable/environments/openPaiEnvironmentService.ts similarity index 100% rename from src/nni_manager/training_service/reusable/environments/openPaiEnvironmentService.ts rename to ts/nni_manager/training_service/reusable/environments/openPaiEnvironmentService.ts diff --git a/src/nni_manager/training_service/reusable/environments/remoteEnvironmentService.ts b/ts/nni_manager/training_service/reusable/environments/remoteEnvironmentService.ts similarity index 85% rename from src/nni_manager/training_service/reusable/environments/remoteEnvironmentService.ts rename to ts/nni_manager/training_service/reusable/environments/remoteEnvironmentService.ts index 2988a34896..fb5b3c789f 100644 --- a/src/nni_manager/training_service/reusable/environments/remoteEnvironmentService.ts +++ b/ts/nni_manager/training_service/reusable/environments/remoteEnvironmentService.ts @@ -137,40 +137,43 @@ export class RemoteEnvironmentService extends EnvironmentService { private async refreshEnvironment(environment: EnvironmentInformation): Promise { const executor = await this.getExecutor(environment.id); - const jobpidPath: string = `${environment.runnerWorkingFolder}/pid`; - const runnerReturnCodeFilePath: string = `${environment.runnerWorkingFolder}/code`; - if (fs.existsSync(jobpidPath)) { - /* eslint-disable require-atomic-updates */ - try { - const isAlive = await executor.isProcessAlive(jobpidPath); - // if the process of jobpid is not alive any more - if (!isAlive) { - const remoteEnvironment: RemoteMachineEnvironmentInformation = environment as RemoteMachineEnvironmentInformation; - if (remoteEnvironment.rmMachineMeta === undefined) { - throw new Error(`${remoteEnvironment.id} machine meta not initialized!`); - } - this.log.info(`pid in ${remoteEnvironment.rmMachineMeta.ip}:${jobpidPath} is not alive!`); - if (fs.existsSync(runnerReturnCodeFilePath)) { - const runnerReturnCode: string = await executor.getRemoteFileContent(runnerReturnCodeFilePath); - const match: RegExpMatchArray | null = runnerReturnCode.trim() - .match(/^-?(\d+)\s+(\d+)$/); - if (match !== null) { - const { 1: code } = match; - // Update trial job's status based on result code - if (parseInt(code, 10) === 0) { - environment.setStatus('SUCCEEDED'); - } else { - environment.setStatus('FAILED'); - } - this.releaseEnvironmentResource(environment); - } + const jobpidPath: string = `${environment.runnerWorkingFolder}/pid`; + const runnerReturnCodeFilePath: string = `${environment.runnerWorkingFolder}/code`; + /* eslint-disable require-atomic-updates */ + try { + // check if pid file exist + const pidExist = await executor.fileExist(jobpidPath); + if (!pidExist) { + return; + } + const isAlive = await executor.isProcessAlive(jobpidPath); + environment.status = 'RUNNING'; + // if the process of jobpid is not alive any more + if (!isAlive) { + const remoteEnvironment: RemoteMachineEnvironmentInformation = environment as RemoteMachineEnvironmentInformation; + if (remoteEnvironment.rmMachineMeta === undefined) { + throw new Error(`${remoteEnvironment.id} machine meta not initialized!`); + } + this.log.info(`pid in ${remoteEnvironment.rmMachineMeta.ip}:${jobpidPath} is not alive!`); + if (fs.existsSync(runnerReturnCodeFilePath)) { + const runnerReturnCode: string = await executor.getRemoteFileContent(runnerReturnCodeFilePath); + const match: RegExpMatchArray | null = runnerReturnCode.trim() + .match(/^-?(\d+)\s+(\d+)$/); + if (match !== null) { + const { 1: code } = match; + // Update trial job's status based on result code + if (parseInt(code, 10) === 0) { + environment.setStatus('SUCCEEDED'); + } else { + environment.setStatus('FAILED'); } + this.releaseEnvironmentResource(environment); } - } catch (error) { - this.releaseEnvironmentResource(environment); - this.log.error(`Update job status exception, error is ${error.message}`); } } + } catch (error) { + this.log.error(`Update job status exception, error is ${error.message}`); + } } public async refreshEnvironmentsStatus(environments: EnvironmentInformation[]): Promise { @@ -245,6 +248,7 @@ export class RemoteEnvironmentService extends EnvironmentService { 'envs', environment.id) environment.command = `cd ${environment.runnerWorkingFolder} && \ ${environment.command} --job_pid_file ${environment.runnerWorkingFolder}/pid \ +1>${environment.runnerWorkingFolder}/trialrunner_stdout 2>${environment.runnerWorkingFolder}/trialrunner_stderr \ && echo $? \`date +%s%3N\` >${environment.runnerWorkingFolder}/code`; return Promise.resolve(true); } @@ -266,7 +270,6 @@ ${environment.command} --job_pid_file ${environment.runnerWorkingFolder}/pid \ // Execute command in remote machine executor.executeScript(executor.joinPath(environment.runnerWorkingFolder, executor.getScriptName("run")), true, false); - environment.status = 'RUNNING'; if (environment.rmMachineMeta === undefined) { throw new Error(`${environment.id} rmMachineMeta not initialized!`); } diff --git a/src/nni_manager/training_service/reusable/gpuScheduler.ts b/ts/nni_manager/training_service/reusable/gpuScheduler.ts similarity index 100% rename from src/nni_manager/training_service/reusable/gpuScheduler.ts rename to ts/nni_manager/training_service/reusable/gpuScheduler.ts diff --git a/src/nni_manager/training_service/reusable/remote/remoteConfig.ts b/ts/nni_manager/training_service/reusable/remote/remoteConfig.ts similarity index 100% rename from src/nni_manager/training_service/reusable/remote/remoteConfig.ts rename to ts/nni_manager/training_service/reusable/remote/remoteConfig.ts diff --git a/src/nni_manager/training_service/reusable/routerTrainingService.ts b/ts/nni_manager/training_service/reusable/routerTrainingService.ts similarity index 100% rename from src/nni_manager/training_service/reusable/routerTrainingService.ts rename to ts/nni_manager/training_service/reusable/routerTrainingService.ts diff --git a/src/nni_manager/training_service/reusable/storageService.ts b/ts/nni_manager/training_service/reusable/storageService.ts similarity index 100% rename from src/nni_manager/training_service/reusable/storageService.ts rename to ts/nni_manager/training_service/reusable/storageService.ts diff --git a/src/nni_manager/training_service/reusable/storages/mountedStorageService.ts b/ts/nni_manager/training_service/reusable/storages/mountedStorageService.ts similarity index 100% rename from src/nni_manager/training_service/reusable/storages/mountedStorageService.ts rename to ts/nni_manager/training_service/reusable/storages/mountedStorageService.ts diff --git a/src/nni_manager/training_service/reusable/test/amlClient.test.ts b/ts/nni_manager/training_service/reusable/test/amlClient.test.ts similarity index 100% rename from src/nni_manager/training_service/reusable/test/amlClient.test.ts rename to ts/nni_manager/training_service/reusable/test/amlClient.test.ts diff --git a/src/nni_manager/training_service/reusable/test/mountedStorageService.test.ts b/ts/nni_manager/training_service/reusable/test/mountedStorageService.test.ts similarity index 100% rename from src/nni_manager/training_service/reusable/test/mountedStorageService.test.ts rename to ts/nni_manager/training_service/reusable/test/mountedStorageService.test.ts diff --git a/src/nni_manager/training_service/reusable/test/trialDispatcher.test.ts b/ts/nni_manager/training_service/reusable/test/trialDispatcher.test.ts similarity index 100% rename from src/nni_manager/training_service/reusable/test/trialDispatcher.test.ts rename to ts/nni_manager/training_service/reusable/test/trialDispatcher.test.ts diff --git a/src/nni_manager/training_service/reusable/test/utCommandChannel.ts b/ts/nni_manager/training_service/reusable/test/utCommandChannel.ts similarity index 100% rename from src/nni_manager/training_service/reusable/test/utCommandChannel.ts rename to ts/nni_manager/training_service/reusable/test/utCommandChannel.ts diff --git a/src/nni_manager/training_service/reusable/test/utEnvironmentService.ts b/ts/nni_manager/training_service/reusable/test/utEnvironmentService.ts similarity index 100% rename from src/nni_manager/training_service/reusable/test/utEnvironmentService.ts rename to ts/nni_manager/training_service/reusable/test/utEnvironmentService.ts diff --git a/src/nni_manager/training_service/reusable/trial.ts b/ts/nni_manager/training_service/reusable/trial.ts similarity index 100% rename from src/nni_manager/training_service/reusable/trial.ts rename to ts/nni_manager/training_service/reusable/trial.ts diff --git a/src/nni_manager/training_service/reusable/trialDispatcher.ts b/ts/nni_manager/training_service/reusable/trialDispatcher.ts similarity index 98% rename from src/nni_manager/training_service/reusable/trialDispatcher.ts rename to ts/nni_manager/training_service/reusable/trialDispatcher.ts index a56f6d8330..79cac7c10e 100644 --- a/src/nni_manager/training_service/reusable/trialDispatcher.ts +++ b/ts/nni_manager/training_service/reusable/trialDispatcher.ts @@ -228,6 +228,7 @@ class TrialDispatcher implements TrainingService { const runnerSettings = storageService.joinPath(envDir, "settings.json"); await storageService.save(JSON.stringify(this.runnerSettings), runnerSettings); + // FIXME: what the hell is this? if (this.isDeveloping) { let trialToolsPath = path.join(__dirname, "../../../../../tools/nni_trial_tool"); if (false === fs.existsSync(trialToolsPath)) { @@ -598,7 +599,7 @@ class TrialDispatcher implements TrainingService { const envName = `nni_exp_${this.experimentId}_env_${envId}`; const environment = environmentService.createEnvironmentInformation(envId, envName); - environment.command = `sh ../install_nni.sh && python3 -m nni_trial_tool.trial_runner`; + environment.command = `sh ../install_nni.sh && python3 -m nni.tools.trial_tool.trial_runner`; if (this.isDeveloping) { environment.command = "[ -d \"nni_trial_tool\" ] && echo \"nni_trial_tool exists already\" || (mkdir ./nni_trial_tool && tar -xof ../nni_trial_tool.tar.gz -C ./nni_trial_tool) && pip3 install websockets && " + environment.command; @@ -662,19 +663,22 @@ class TrialDispatcher implements TrainingService { trial.status = "RUNNING"; await this.commandChannel.sendCommand(trial.environment, NEW_TRIAL_JOB, trial.settings); } - + + /** + * release the trial assigned environment resources + * @param trial + */ private releaseEnvironment(trial: TrialDetail): void { - if (undefined === trial.environment) { - throw new Error(`TrialDispatcher: environment is not assigned to trial ${trial.id}, and cannot be released!`); - } - if (trial.environment.runningTrialCount <= 0) { - throw new Error(`TrialDispatcher: environment ${trial.environment.id} has no counted running trial!`); + if (trial.environment !== undefined) { + if (trial.environment.runningTrialCount <= 0) { + throw new Error(`TrialDispatcher: environment ${trial.environment.id} has no counted running trial!`); + } + trial.environment.runningTrialCount--; + trial.environment = undefined; } if (true === this.enableGpuScheduler) { this.gpuScheduler.removeGpuReservation(trial); } - trial.environment.runningTrialCount--; - trial.environment = undefined; } private async handleMetricData(trialId: string, data: any): Promise { diff --git a/src/nni_manager/training_service/test/fileUtility.test.ts b/ts/nni_manager/training_service/test/fileUtility.test.ts similarity index 100% rename from src/nni_manager/training_service/test/fileUtility.test.ts rename to ts/nni_manager/training_service/test/fileUtility.test.ts diff --git a/src/nni_manager/training_service/test/hdfsClientUtility.test.ts b/ts/nni_manager/training_service/test/hdfsClientUtility.test.ts similarity index 100% rename from src/nni_manager/training_service/test/hdfsClientUtility.test.ts rename to ts/nni_manager/training_service/test/hdfsClientUtility.test.ts diff --git a/src/nni_manager/training_service/test/kubeflowTrainingService.test.ts b/ts/nni_manager/training_service/test/kubeflowTrainingService.test.ts similarity index 100% rename from src/nni_manager/training_service/test/kubeflowTrainingService.test.ts rename to ts/nni_manager/training_service/test/kubeflowTrainingService.test.ts diff --git a/src/nni_manager/training_service/test/localTrainingService.test.ts b/ts/nni_manager/training_service/test/localTrainingService.test.ts similarity index 100% rename from src/nni_manager/training_service/test/localTrainingService.test.ts rename to ts/nni_manager/training_service/test/localTrainingService.test.ts diff --git a/src/nni_manager/training_service/test/mockedTrial.py b/ts/nni_manager/training_service/test/mockedTrial.py similarity index 100% rename from src/nni_manager/training_service/test/mockedTrial.py rename to ts/nni_manager/training_service/test/mockedTrial.py diff --git a/src/nni_manager/training_service/test/paiYarnTrainingService.test.ts b/ts/nni_manager/training_service/test/paiYarnTrainingService.test.ts similarity index 100% rename from src/nni_manager/training_service/test/paiYarnTrainingService.test.ts rename to ts/nni_manager/training_service/test/paiYarnTrainingService.test.ts diff --git a/src/nni_manager/training_service/test/remoteMachineTrainingService.test.ts b/ts/nni_manager/training_service/test/remoteMachineTrainingService.test.ts similarity index 100% rename from src/nni_manager/training_service/test/remoteMachineTrainingService.test.ts rename to ts/nni_manager/training_service/test/remoteMachineTrainingService.test.ts diff --git a/src/nni_manager/tsconfig.json b/ts/nni_manager/tsconfig.json similarity index 100% rename from src/nni_manager/tsconfig.json rename to ts/nni_manager/tsconfig.json diff --git a/src/nni_manager/types/child-process-promise/index.d.ts b/ts/nni_manager/types/child-process-promise/index.d.ts similarity index 100% rename from src/nni_manager/types/child-process-promise/index.d.ts rename to ts/nni_manager/types/child-process-promise/index.d.ts diff --git a/src/nni_manager/types/tail-stream/index.d.ts b/ts/nni_manager/types/tail-stream/index.d.ts similarity index 100% rename from src/nni_manager/types/tail-stream/index.d.ts rename to ts/nni_manager/types/tail-stream/index.d.ts diff --git a/src/nni_manager/types/webhdfs/index.d.ts b/ts/nni_manager/types/webhdfs/index.d.ts similarity index 100% rename from src/nni_manager/types/webhdfs/index.d.ts rename to ts/nni_manager/types/webhdfs/index.d.ts diff --git a/src/nni_manager/yarn.lock b/ts/nni_manager/yarn.lock similarity index 97% rename from src/nni_manager/yarn.lock rename to ts/nni_manager/yarn.lock index ee386f23d0..a7b150a83b 100644 --- a/src/nni_manager/yarn.lock +++ b/ts/nni_manager/yarn.lock @@ -789,6 +789,13 @@ binary-extensions@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.0.0.tgz#23c0df14f6a88077f5f986c0d167ec03c3d5537c" +block-stream@*: + version "0.0.9" + resolved "https://registry.yarnpkg.com/block-stream/-/block-stream-0.0.9.tgz#13ebfe778a03205cfe03751481ebb4b3300c126a" + integrity sha1-E+v+d4oDIFz+A3UUgeu0szAMEmo= + dependencies: + inherits "~2.0.0" + bluebird@^3.5.1, bluebird@^3.5.3, bluebird@^3.5.5: version "3.7.2" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" @@ -1089,10 +1096,10 @@ cli-width@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-2.2.0.tgz#ff19ede8a9a5e579324147b0c11f0fbcbabed639" -cliui@^7.0.0: - version "7.0.1" - resolved "https://registry.yarnpkg.com/cliui/-/cliui-7.0.1.tgz#a4cb67aad45cd83d8d05128fc9f4d8fbb887e6b3" - integrity sha512-rcvHOWyGyid6I1WjT/3NatKj2kDt9OdSHSXpyLXaMWFbKpGACNW8pRhhdPUq9MWUOdwn8Rz9AVETjF4105rZZQ== +cliui@^7.0.2: + version "7.0.3" + resolved "https://registry.yarnpkg.com/cliui/-/cliui-7.0.3.tgz#ef180f26c8d9bff3927ee52428bfec2090427981" + integrity sha512-Gj3QHTkVMPKqwP3f7B4KPkBZRMR9r4rfi5bXFpg1a+Svvj8l7q5CnkBkVQzfxT5DFSsGk2+PascOgL0JYkL2kw== dependencies: string-width "^4.2.0" strip-ansi "^6.0.0" @@ -1329,7 +1336,7 @@ debug@^3.1.0: dependencies: ms "^2.1.1" -debuglog@^1.0.1: +debuglog@*, debuglog@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492" @@ -1599,10 +1606,10 @@ es6-promisify@^5.0.0: dependencies: es6-promise "^4.0.3" -escalade@^3.0.2: - version "3.1.0" - resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.0.tgz#e8e2d7c7a8b76f6ee64c2181d6b8151441602d4e" - integrity sha512-mAk+hPSO8fLDkhV7V0dXazH5pDc6MrjBTPyD3VeKzxnVFjH1MIxbCdqGZB9O8+EwWakZs3ZCbDS4IpRt79V1ig== +escalade@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" + integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw== escape-html@~1.0.3: version "1.0.3" @@ -1993,6 +2000,16 @@ fsevents@~2.1.2: resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.1.3.tgz#fb738703ae8d2f9fe900c33836ddebee8b97f23e" integrity sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ== +fstream@^1.0.0, fstream@^1.0.12: + version "1.0.12" + resolved "https://registry.yarnpkg.com/fstream/-/fstream-1.0.12.tgz#4e8ba8ee2d48be4f7d0de505455548eae5932045" + integrity sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg== + dependencies: + graceful-fs "^4.1.2" + inherits "~2.0.0" + mkdirp ">=0.5 0" + rimraf "2" + function-bind@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" @@ -2375,7 +2392,7 @@ import-lazy@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/import-lazy/-/import-lazy-2.1.0.tgz#05698e3d45c88e8d7e9d92cb0584e77f096f3e43" -imurmurhash@^0.1.4: +imurmurhash@*, imurmurhash@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" integrity sha1-khi5srkoojixPcT7a21XbyMUU+o= @@ -2400,7 +2417,7 @@ inflight@^1.0.4, inflight@~1.0.6: once "^1.3.0" wrappy "1" -inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.3: +inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.0, inherits@~2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" @@ -3057,6 +3074,11 @@ lockfile@^1.0.4: dependencies: signal-exit "^3.0.2" +lodash._baseindexof@*: + version "3.1.0" + resolved "https://registry.yarnpkg.com/lodash._baseindexof/-/lodash._baseindexof-3.1.0.tgz#fe52b53a1c6761e42618d654e4a25789ed61822c" + integrity sha1-/lK1OhxnYeQmGNZU5KJXie1hgiw= + lodash._baseuniq@~4.6.0: version "4.6.0" resolved "https://registry.yarnpkg.com/lodash._baseuniq/-/lodash._baseuniq-4.6.0.tgz#0ebb44e456814af7905c6212fa2c9b2d51b841e8" @@ -3064,10 +3086,32 @@ lodash._baseuniq@~4.6.0: lodash._createset "~4.0.0" lodash._root "~3.0.0" +lodash._bindcallback@*: + version "3.0.1" + resolved "https://registry.yarnpkg.com/lodash._bindcallback/-/lodash._bindcallback-3.0.1.tgz#e531c27644cf8b57a99e17ed95b35c748789392e" + integrity sha1-5THCdkTPi1epnhftlbNcdIeJOS4= + +lodash._cacheindexof@*: + version "3.0.2" + resolved "https://registry.yarnpkg.com/lodash._cacheindexof/-/lodash._cacheindexof-3.0.2.tgz#3dc69ac82498d2ee5e3ce56091bafd2adc7bde92" + integrity sha1-PcaayCSY0u5ePOVgkbr9Ktx73pI= + +lodash._createcache@*: + version "3.1.2" + resolved "https://registry.yarnpkg.com/lodash._createcache/-/lodash._createcache-3.1.2.tgz#56d6a064017625e79ebca6b8018e17440bdcf093" + integrity sha1-VtagZAF2JeeevKa4AY4XRAvc8JM= + dependencies: + lodash._getnative "^3.0.0" + lodash._createset@~4.0.0: version "4.0.3" resolved "https://registry.yarnpkg.com/lodash._createset/-/lodash._createset-4.0.3.tgz#0f4659fbb09d75194fa9e2b88a6644d363c9fe26" +lodash._getnative@*, lodash._getnative@^3.0.0: + version "3.9.1" + resolved "https://registry.yarnpkg.com/lodash._getnative/-/lodash._getnative-3.9.1.tgz#570bc7dede46d61cdcde687d65d3eecbaa3aaff5" + integrity sha1-VwvH3t5G1hzc3mh9ZdPuy6o6r/U= + lodash._root@~3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/lodash._root/-/lodash._root-3.0.1.tgz#fba1c4524c19ee9a5f8136b4609f017cf4ded692" @@ -3116,6 +3160,11 @@ lodash.pick@^4.4.0: version "4.4.0" resolved "https://registry.yarnpkg.com/lodash.pick/-/lodash.pick-4.4.0.tgz#52f05610fff9ded422611441ed1fc123a03001b3" +lodash.restparam@*: + version "3.6.1" + resolved "https://registry.yarnpkg.com/lodash.restparam/-/lodash.restparam-3.6.1.tgz#936a4e309ef330a7645ed4145986c85ae5b20805" + integrity sha1-k2pOMJ7zMKdkXtQUWYbIWuWyCAU= + lodash.unescape@4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/lodash.unescape/-/lodash.unescape-4.0.1.tgz#bf2249886ce514cda112fae9218cdc065211fc9c" @@ -3342,19 +3391,19 @@ mississippi@^3.0.0: stream-each "^1.1.0" through2 "^2.0.0" -mkdirp@^0.5.0, mkdirp@~0.5.0: - version "0.5.3" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.3.tgz#5a514b7179259287952881e94410ec5465659f8c" - dependencies: - minimist "^1.2.5" - -mkdirp@^0.5.1, mkdirp@^0.5.5: +"mkdirp@>=0.5 0", mkdirp@^0.5.1, mkdirp@^0.5.5: version "0.5.5" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ== dependencies: minimist "^1.2.5" +mkdirp@^0.5.0, mkdirp@~0.5.0: + version "0.5.3" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.3.tgz#5a514b7179259287952881e94410ec5465659f8c" + dependencies: + minimist "^1.2.5" + mkdirp@^1.0.3: version "1.0.4" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" @@ -3420,10 +3469,6 @@ mute-stream@0.0.8, mute-stream@~0.0.4: version "0.0.8" resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.8.tgz#1630c42b2251ff81e2a283de96a5497ea92e5e0d" -nan@~2.10.0: - version "2.10.0" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.10.0.tgz#96d0cd610ebd58d4b4de9cc0c6828cda99c7548f" - natural-compare@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" @@ -3444,6 +3489,11 @@ nice-try@^1.0.4: version "1.0.5" resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" +node-addon-api@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-2.0.0.tgz#f9afb8d777a91525244b01775ea0ddbe1125483b" + integrity sha512-ASCL5U13as7HhOExbT6OlWJJUV/lLzL2voOSP1UVehpRD8FbSrSDjfScK/KwAvVTI5AS6r4VwbOMlIqtvRidnA== + node-fetch-npm@^2.0.2: version "2.0.4" resolved "https://registry.yarnpkg.com/node-fetch-npm/-/node-fetch-npm-2.0.4.tgz#6507d0e17a9ec0be3bec516958a497cec54bf5a4" @@ -3458,6 +3508,24 @@ node-forge@^0.10.0, node-forge@^0.7.6: resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3" integrity sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA== +node-gyp@3.x: + version "3.8.0" + resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-3.8.0.tgz#540304261c330e80d0d5edce253a68cb3964218c" + integrity sha512-3g8lYefrRRzvGeSowdJKAKyks8oUpLEd/DyPV4eMhVlhJ0aNaZqIrNUIPuEWWTAoPqyFkfGrM67MC69baqn6vA== + dependencies: + fstream "^1.0.0" + glob "^7.0.3" + graceful-fs "^4.1.2" + mkdirp "^0.5.0" + nopt "2 || 3" + npmlog "0 || 1 || 2 || 3 || 4" + osenv "0" + request "^2.87.0" + rimraf "2" + semver "~5.3.0" + tar "^2.0.0" + which "1" + node-gyp@^5.0.2, node-gyp@^5.1.0: version "5.1.1" resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-5.1.1.tgz#eb915f7b631c937d282e33aed44cb7a025f62a3e" @@ -3495,9 +3563,10 @@ node-jose@^1.1.0: node-forge "^0.7.6" uuid "^3.3.2" -node-pre-gyp@^0.10.3: - version "0.10.3" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.10.3.tgz#3070040716afdc778747b61b6887bf78880b80fc" +node-pre-gyp@^0.11.0: + version "0.11.0" + resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz#db1f33215272f692cd38f03238e3e9b47c5dd054" + integrity sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q== dependencies: detect-libc "^1.0.2" mkdirp "^0.5.1" @@ -3533,6 +3602,13 @@ node.flow@1.2.3: dependencies: node.extend "1.0.8" +"nopt@2 || 3": + version "3.0.6" + resolved "https://registry.yarnpkg.com/nopt/-/nopt-3.0.6.tgz#c6465dbf08abcd4db359317f79ac68a646b28ff9" + integrity sha1-xkZdvwirzU2zWTF/eaxopkayj/k= + dependencies: + abbrev "1" + nopt@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/nopt/-/nopt-4.0.1.tgz#d0d4685afd5415193c8c7505602d0d17cd64474d" @@ -3679,10 +3755,11 @@ npm-run-path@^2.0.0: path-key "^2.0.0" npm-user-validate@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/npm-user-validate/-/npm-user-validate-1.0.0.tgz#8ceca0f5cea04d4e93519ef72d0557a75122e951" + version "1.0.1" + resolved "https://registry.yarnpkg.com/npm-user-validate/-/npm-user-validate-1.0.1.tgz#31428fc5475fe8416023f178c0ab47935ad8c561" + integrity sha512-uQwcd/tY+h1jnEaze6cdX/LrhWhoBxfSknxentoqmIuStxUExxjWd3ULMLFPiFUrZKbOVMowH6Jq2FRWfmhcEw== -npm@5.1.0, npm@^6.14.8: +npm@5.1.0, npm@>=6.14.8: version "6.14.8" resolved "https://registry.yarnpkg.com/npm/-/npm-6.14.8.tgz#64ef754345639bc035982ec3f609353c8539033c" integrity sha512-HBZVBMYs5blsj94GTeQZel7s9odVuuSUHy1+AlZh7rPVux1os2ashvEGLy/STNK7vUjbrCg5Kq9/GXisJgdf6A== @@ -3803,7 +3880,7 @@ npm@5.1.0, npm@^6.14.8: worker-farm "^1.7.0" write-file-atomic "^2.4.3" -npmlog@^4.0.2, npmlog@^4.1.2, npmlog@~4.1.2: +"npmlog@0 || 1 || 2 || 3 || 4", npmlog@^4.0.2, npmlog@^4.1.2, npmlog@~4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" dependencies: @@ -3953,7 +4030,7 @@ os-tmpdir@^1.0.0, os-tmpdir@~1.0.2: resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" integrity sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ= -osenv@^0.1.4, osenv@^0.1.5: +osenv@0, osenv@^0.1.4, osenv@^0.1.5: version "0.1.5" resolved "https://registry.yarnpkg.com/osenv/-/osenv-0.1.5.tgz#85cdfafaeb28e8677f416e287592b5f3f49ea410" integrity sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g== @@ -4661,19 +4738,19 @@ retry@^0.12.0: resolved "https://registry.yarnpkg.com/retry/-/retry-0.12.0.tgz#1b42a6266a21f07421d1b0b54b7dc167b01c013b" integrity sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs= -rimraf@2.6.3: - version "2.6.3" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab" - dependencies: - glob "^7.1.3" - -rimraf@^2.5.2, rimraf@^2.5.4, rimraf@^2.6.3, rimraf@^2.7.1: +rimraf@2, rimraf@^2.5.2, rimraf@^2.5.4, rimraf@^2.6.3, rimraf@^2.7.1: version "2.7.1" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w== dependencies: glob "^7.1.3" +rimraf@2.6.3: + version "2.6.3" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab" + dependencies: + glob "^7.1.3" + rimraf@^2.6.1, rimraf@^2.6.2: version "2.6.2" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.2.tgz#2ed8150d24a16ea8651e6d6ef0f47c4158ce7a36" @@ -4763,6 +4840,11 @@ semver@^6.0.0, semver@^6.1.2, semver@^6.3.0: version "6.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" +semver@~5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" + integrity sha1-myzl094C0XxgEq0yaqa00M9U+U8= + send@0.16.2: version "0.16.2" resolved "https://registry.yarnpkg.com/send/-/send-0.16.2.tgz#6ecca1e0f8c156d141597559848df64730a6bbc1" @@ -4951,13 +5033,15 @@ sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" -sqlite3@^4.0.2: - version "4.0.2" - resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-4.0.2.tgz#1bbeb68b03ead5d499e42a3a1b140064791c5a64" +sqlite3@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-5.0.0.tgz#1bfef2151c6bc48a3ab1a6c126088bb8dd233566" + integrity sha512-rjvqHFUaSGnzxDy2AHCwhHy6Zp6MNJzCPGYju4kD8yi6bze4d1/zMTg6C7JI49b7/EM7jKMTvyfN/4ylBKdwfw== dependencies: - nan "~2.10.0" - node-pre-gyp "^0.10.3" - request "^2.87.0" + node-addon-api "2.0.0" + node-pre-gyp "^0.11.0" + optionalDependencies: + node-gyp "3.x" ssh2-streams@~0.2.0: version "0.2.1" @@ -5189,6 +5273,15 @@ tail-stream@^0.3.4: version "0.3.4" resolved "https://registry.yarnpkg.com/tail-stream/-/tail-stream-0.3.4.tgz#bc675a20e92732b1a6a7cc65d6be66f7817fd5c1" +tar@^2.0.0: + version "2.2.2" + resolved "https://registry.yarnpkg.com/tar/-/tar-2.2.2.tgz#0ca8848562c7299b8b446ff6a4d60cdbb23edc40" + integrity sha512-FCEhQ/4rE1zYv9rYXJw/msRqsnmlje5jHP6huWeBZ704jUTy02c5AZyWujpMR1ax6mVw9NyJMfuK2CMDWVIfgA== + dependencies: + block-stream "*" + fstream "^1.0.12" + inherits "2" + tar@^4: version "4.4.4" resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.4.tgz#ec8409fae9f665a4355cc3b4087d0820232bb8cd" @@ -5544,15 +5637,15 @@ webhdfs@^1.2.0: extend "^3.0.0" request "^2.74.0" -which@2.0.2, which@^2.0.1: - version "2.0.2" - resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" +which@1, which@^1.2.9, which@^1.3.0, which@^1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" dependencies: isexe "^2.0.0" -which@^1.2.9, which@^1.3.0, which@^1.3.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" +which@2.0.2, which@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" dependencies: isexe "^2.0.0" @@ -5664,10 +5757,10 @@ y18n@^4.0.0: resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.0.tgz#95ef94f85ecc81d007c264e190a120f0a3c8566b" integrity sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w== -y18n@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.1.tgz#1ad2a7eddfa8bce7caa2e1f6b5da96c39d99d571" - integrity sha512-/jJ831jEs4vGDbYPQp4yGKDYPSCCEQ45uZWJHE1AoYBzqdZi8+LDWas0z4HrmJXmKdpFsTiowSHXdxyFhpmdMg== +y18n@^5.0.2: + version "5.0.4" + resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.4.tgz#0ab2db89dd5873b5ec4682d8e703e833373ea897" + integrity sha512-deLOfD+RvFgrpAmSZgfGdWYE+OKyHcVHaRQ7NphG/63scpRvTHHeQMAxGGvaLVGJ+HYVcCXlzcTK0ZehFf+eHQ== yallist@^2.1.2: version "2.1.2" @@ -5686,10 +5779,10 @@ yallist@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" -yargs-parser@13.1.2, yargs-parser@^20.0.0, yargs-parser@^20.2.0: - version "20.2.0" - resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.0.tgz#944791ca2be2e08ddadd3d87e9de4c6484338605" - integrity sha512-2agPoRFPoIcFzOIp6656gcvsg2ohtscpw2OINr/q46+Sq41xz2OYLqx5HRHabmFU1OARIPAYH5uteICE7mn/5A== +yargs-parser@13.1.2, yargs-parser@>=20.2.0, yargs-parser@^20.2.2: + version "20.2.3" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.3.tgz#92419ba867b858c868acf8bae9bf74af0dd0ce26" + integrity sha512-emOFRT9WVHw03QSvN5qor9QQT9+sw5vwxfYweivSMHTcAXPefwVae2FjO7JJjj8hCE4CzPOPeFM83VwT29HCww== yargs-unparser@1.6.1: version "1.6.1" @@ -5702,18 +5795,18 @@ yargs-unparser@1.6.1: is-plain-obj "^1.1.0" yargs "^14.2.3" -yargs@13.3.2, yargs@^11.0.0, yargs@^14.2.3, yargs@^15.0.2, yargs@^16.0.3, yargs@^8.0.2: - version "16.0.3" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.0.3.tgz#7a919b9e43c90f80d4a142a89795e85399a7e54c" - integrity sha512-6+nLw8xa9uK1BOEOykaiYAJVh6/CjxWXK/q9b5FpRgNslt8s22F2xMBqVIKgCRjNgGvGPBy8Vog7WN7yh4amtA== +yargs@13.3.2, yargs@>=16.0.3, yargs@^11.0.0, yargs@^14.2.3, yargs@^15.0.2, yargs@^8.0.2: + version "16.1.0" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.1.0.tgz#fc333fe4791660eace5a894b39d42f851cd48f2a" + integrity sha512-upWFJOmDdHN0syLuESuvXDmrRcWd1QafJolHskzaw79uZa7/x53gxQKiR07W59GWY1tFhhU/Th9DrtSfpS782g== dependencies: - cliui "^7.0.0" - escalade "^3.0.2" + cliui "^7.0.2" + escalade "^3.1.1" get-caller-file "^2.0.5" require-directory "^2.1.1" string-width "^4.2.0" - y18n "^5.0.1" - yargs-parser "^20.0.0" + y18n "^5.0.2" + yargs-parser "^20.2.2" yn@^2.0.0: version "2.0.0" diff --git a/src/webui/.eslintignore b/ts/webui/.eslintignore similarity index 100% rename from src/webui/.eslintignore rename to ts/webui/.eslintignore diff --git a/src/webui/.eslintrc b/ts/webui/.eslintrc similarity index 94% rename from src/webui/.eslintrc rename to ts/webui/.eslintrc index 9fb5393e3e..e220b1f541 100644 --- a/src/webui/.eslintrc +++ b/ts/webui/.eslintrc @@ -28,6 +28,7 @@ "@typescript-eslint/no-inferrable-types": 0, "@typescript-eslint/no-use-before-define": [2, "nofunc"], "@typescript-eslint/no-var-requires": 0, + "@typescript-eslint/no-unused-vars": [2, { "argsIgnorePattern": "^_" }], "arrow-parens": [2, "as-needed"], "no-inner-declarations": 0, "no-empty": 2, diff --git a/src/webui/.gitignore b/ts/webui/.gitignore similarity index 100% rename from src/webui/.gitignore rename to ts/webui/.gitignore diff --git a/src/webui/.stylelintrc.json b/ts/webui/.stylelintrc.json similarity index 100% rename from src/webui/.stylelintrc.json rename to ts/webui/.stylelintrc.json diff --git a/src/webui/LICENSE b/ts/webui/LICENSE similarity index 100% rename from src/webui/LICENSE rename to ts/webui/LICENSE diff --git a/src/webui/README.md b/ts/webui/README.md similarity index 100% rename from src/webui/README.md rename to ts/webui/README.md diff --git a/src/webui/README_zh_CN.md b/ts/webui/README_zh_CN.md similarity index 100% rename from src/webui/README_zh_CN.md rename to ts/webui/README_zh_CN.md diff --git a/src/webui/config/env.js b/ts/webui/config/env.js similarity index 100% rename from src/webui/config/env.js rename to ts/webui/config/env.js diff --git a/src/webui/config/jest/cssTransform.js b/ts/webui/config/jest/cssTransform.js similarity index 100% rename from src/webui/config/jest/cssTransform.js rename to ts/webui/config/jest/cssTransform.js diff --git a/src/webui/config/jest/fileTransform.js b/ts/webui/config/jest/fileTransform.js similarity index 100% rename from src/webui/config/jest/fileTransform.js rename to ts/webui/config/jest/fileTransform.js diff --git a/src/webui/config/modules.js b/ts/webui/config/modules.js similarity index 100% rename from src/webui/config/modules.js rename to ts/webui/config/modules.js diff --git a/src/webui/config/paths.js b/ts/webui/config/paths.js similarity index 100% rename from src/webui/config/paths.js rename to ts/webui/config/paths.js diff --git a/src/webui/config/webpack.config.js b/ts/webui/config/webpack.config.js similarity index 100% rename from src/webui/config/webpack.config.js rename to ts/webui/config/webpack.config.js diff --git a/src/webui/config/webpackDevServer.config.js b/ts/webui/config/webpackDevServer.config.js similarity index 100% rename from src/webui/config/webpackDevServer.config.js rename to ts/webui/config/webpackDevServer.config.js diff --git a/src/webui/mock/all-types-metric.json b/ts/webui/mock/all-types-metric.json similarity index 100% rename from src/webui/mock/all-types-metric.json rename to ts/webui/mock/all-types-metric.json diff --git a/src/webui/mock/dict-metrics-100.json b/ts/webui/mock/dict-metrics-100.json similarity index 100% rename from src/webui/mock/dict-metrics-100.json rename to ts/webui/mock/dict-metrics-100.json diff --git a/src/webui/mock/mnist-tfv1-nested.json b/ts/webui/mock/mnist-tfv1-nested.json similarity index 100% rename from src/webui/mock/mnist-tfv1-nested.json rename to ts/webui/mock/mnist-tfv1-nested.json diff --git a/src/webui/mock/mnist-tfv1-running.json b/ts/webui/mock/mnist-tfv1-running.json similarity index 100% rename from src/webui/mock/mnist-tfv1-running.json rename to ts/webui/mock/mnist-tfv1-running.json diff --git a/src/webui/mock/mnist-tfv2-stage0.json b/ts/webui/mock/mnist-tfv2-stage0.json similarity index 100% rename from src/webui/mock/mnist-tfv2-stage0.json rename to ts/webui/mock/mnist-tfv2-stage0.json diff --git a/src/webui/mock/mnist-tfv2-stage1.json b/ts/webui/mock/mnist-tfv2-stage1.json similarity index 100% rename from src/webui/mock/mnist-tfv2-stage1.json rename to ts/webui/mock/mnist-tfv2-stage1.json diff --git a/src/webui/mock/mnist-tfv2-stage2.json b/ts/webui/mock/mnist-tfv2-stage2.json similarity index 100% rename from src/webui/mock/mnist-tfv2-stage2.json rename to ts/webui/mock/mnist-tfv2-stage2.json diff --git a/src/webui/package.json b/ts/webui/package.json similarity index 100% rename from src/webui/package.json rename to ts/webui/package.json diff --git a/src/webui/prettier.config.js b/ts/webui/prettier.config.js similarity index 100% rename from src/webui/prettier.config.js rename to ts/webui/prettier.config.js diff --git a/src/webui/public/favicon.ico b/ts/webui/public/favicon.ico similarity index 100% rename from src/webui/public/favicon.ico rename to ts/webui/public/favicon.ico diff --git a/src/webui/public/icon.png b/ts/webui/public/icon.png similarity index 100% rename from src/webui/public/icon.png rename to ts/webui/public/icon.png diff --git a/src/webui/public/index.html b/ts/webui/public/index.html similarity index 100% rename from src/webui/public/index.html rename to ts/webui/public/index.html diff --git a/src/webui/public/manifest.json b/ts/webui/public/manifest.json similarity index 100% rename from src/webui/public/manifest.json rename to ts/webui/public/manifest.json diff --git a/src/webui/scripts/build.js b/ts/webui/scripts/build.js similarity index 100% rename from src/webui/scripts/build.js rename to ts/webui/scripts/build.js diff --git a/src/webui/scripts/newmock.js b/ts/webui/scripts/newmock.js similarity index 100% rename from src/webui/scripts/newmock.js rename to ts/webui/scripts/newmock.js diff --git a/src/webui/scripts/server.js b/ts/webui/scripts/server.js similarity index 100% rename from src/webui/scripts/server.js rename to ts/webui/scripts/server.js diff --git a/src/webui/scripts/start.js b/ts/webui/scripts/start.js similarity index 100% rename from src/webui/scripts/start.js rename to ts/webui/scripts/start.js diff --git a/src/webui/scripts/test.js b/ts/webui/scripts/test.js similarity index 100% rename from src/webui/scripts/test.js rename to ts/webui/scripts/test.js diff --git a/src/webui/src/App.scss b/ts/webui/src/App.scss similarity index 93% rename from src/webui/src/App.scss rename to ts/webui/src/App.scss index 7121c4e6fe..3811759ea8 100644 --- a/src/webui/src/App.scss +++ b/ts/webui/src/App.scss @@ -26,9 +26,9 @@ } .content { - width: 89%; - min-width: 1024px; + width: 87%; margin: 0 auto; + min-width: 1200px; margin-top: 74px; margin-bottom: 30px; } @@ -53,7 +53,7 @@ .ms-Callout-main { p { font-weight: 500; - color: #333; + color: #fff; } } diff --git a/src/webui/src/App.test.tsx b/ts/webui/src/App.test.tsx similarity index 100% rename from src/webui/src/App.test.tsx rename to ts/webui/src/App.test.tsx diff --git a/src/webui/src/App.tsx b/ts/webui/src/App.tsx similarity index 79% rename from src/webui/src/App.tsx rename to ts/webui/src/App.tsx index 08818e64be..e2a2489cc7 100644 --- a/src/webui/src/App.tsx +++ b/ts/webui/src/App.tsx @@ -4,6 +4,7 @@ import { COLUMN } from './static/const'; import { EXPERIMENT, TRIALS } from './static/datamodel'; import NavCon from './components/NavCon'; import MessageInfo from './components/modals/MessageInfo'; +import { TrialConfigButton } from './components/public-child/config/TrialConfigButton'; import './App.scss'; interface AppState { @@ -11,6 +12,7 @@ interface AppState { columnList: string[]; experimentUpdateBroadcast: number; trialsUpdateBroadcast: number; + maxDurationUnit: string; metricGraphMode: 'max' | 'min'; // tuner's optimize_mode filed isillegalFinal: boolean; expWarningMessage: string; @@ -25,17 +27,21 @@ export const AppContext = React.createContext({ trialsUpdateBroadcast: 0, metricGraphMode: 'max', bestTrialEntries: '10', + maxDurationUnit: 'm', // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars changeColumn: (val: string[]) => {}, // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars changeMetricGraphMode: (val: 'max' | 'min') => {}, // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars - changeEntries: (val: string) => {} + changeMaxDurationUnit: (val: string) => {}, + // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars + changeEntries: (val: string) => {}, + // eslint-disable-next-line @typescript-eslint/no-empty-function, @typescript-eslint/no-unused-vars + updateOverviewPage: () => {} }); class App extends React.Component<{}, AppState> { private timerId!: number | undefined; - private dataFormatimer!: number; private firstLoad: boolean = false; // when click refresh selector options constructor(props: {}) { super(props); @@ -45,6 +51,7 @@ class App extends React.Component<{}, AppState> { experimentUpdateBroadcast: 0, trialsUpdateBroadcast: 0, metricGraphMode: 'max', + maxDurationUnit: 'm', isillegalFinal: false, expWarningMessage: '', bestTrialEntries: '10', @@ -60,35 +67,8 @@ class App extends React.Component<{}, AppState> { metricGraphMode: EXPERIMENT.optimizeMode === 'minimize' ? 'min' : 'max' })); this.timerId = window.setTimeout(this.refresh, this.state.interval * 100); - // final result is legal - // get a succeed trial，see final result data's format - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - this.dataFormatimer = window.setInterval(this.getFinalDataFormat, this.state.interval * 1000); } - getFinalDataFormat = (): void => { - for (let i = 0; this.state.isillegalFinal === false; i++) { - if (TRIALS.succeededTrials()[0] !== undefined && TRIALS.succeededTrials()[0].final !== undefined) { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const oneSucceedTrial = JSON.parse(JSON.parse(TRIALS.succeededTrials()[0].final!.data)); - if (typeof oneSucceedTrial === 'number' || oneSucceedTrial.hasOwnProperty('default')) { - window.clearInterval(this.dataFormatimer); - break; - } else { - // illegal final data - this.setState(() => ({ - isillegalFinal: true, - expWarningMessage: - 'WebUI support final result as number and dictornary includes default keys, your experiment final result is illegal, please check your data.' - })); - window.clearInterval(this.dataFormatimer); - } - } else { - break; - } - } - }; - changeInterval = (interval: number): void => { window.clearTimeout(this.timerId); if (interval === 0) { @@ -116,6 +96,17 @@ class App extends React.Component<{}, AppState> { this.setState({ bestTrialEntries: entries }); }; + // overview max duration unit + changeMaxDurationUnit = (unit: string): void => { + this.setState({ maxDurationUnit: unit }); + }; + + updateOverviewPage = (): void => { + this.setState(state => ({ + experimentUpdateBroadcast: state.experimentUpdateBroadcast + 1 + })); + }; + shouldComponentUpdate(nextProps: any, nextState: AppState): boolean { if (!(nextState.isUpdate || nextState.isUpdate === undefined)) { nextState.isUpdate = true; @@ -133,7 +124,8 @@ class App extends React.Component<{}, AppState> { metricGraphMode, isillegalFinal, expWarningMessage, - bestTrialEntries + bestTrialEntries, + maxDurationUnit } = this.state; if (experimentUpdateBroadcast === 0 || trialsUpdateBroadcast === 0) { return null; // TODO: render a loading page @@ -155,6 +147,25 @@ class App extends React.Component<{}, AppState> { + {/* search space & config */} + + + {/* if api has error field, show error message */} {errorList.map( (item, key) => @@ -177,9 +188,12 @@ class App extends React.Component<{}, AppState> { experimentUpdateBroadcast, trialsUpdateBroadcast, metricGraphMode, + maxDurationUnit, + changeMaxDurationUnit: this.changeMaxDurationUnit, changeMetricGraphMode: this.changeMetricGraphMode, bestTrialEntries, - changeEntries: this.changeEntries + changeEntries: this.changeEntries, + updateOverviewPage: this.updateOverviewPage }} > {this.props.children} diff --git a/src/webui/src/components/NavCon.tsx b/ts/webui/src/components/NavCon.tsx similarity index 100% rename from src/webui/src/components/NavCon.tsx rename to ts/webui/src/components/NavCon.tsx diff --git a/ts/webui/src/components/Overview.tsx b/ts/webui/src/components/Overview.tsx new file mode 100644 index 0000000000..3557ba1f73 --- /dev/null +++ b/ts/webui/src/components/Overview.tsx @@ -0,0 +1,250 @@ +import * as React from 'react'; +import { Stack, Icon, Dropdown, DefaultButton } from '@fluentui/react'; +import { EXPERIMENT, TRIALS } from '../static/datamodel'; +import { Trial } from '../static/model/trial'; +import { AppContext } from '../App'; +import { Title } from './overview/Title'; +import SuccessTable from './overview/table/SuccessTable'; +import Accuracy from './overview/Accuracy'; +import { ReBasicInfo } from './overview/experiment/BasicInfo'; +import { ExpDuration } from './overview/count/ExpDuration'; +import { ExpDurationContext } from './overview/count/ExpDurationContext'; +import { TrialCount } from './overview/count/TrialCount'; +import { Command1 } from './overview/command/Command1'; +import { Command2 } from './overview/command/Command2'; +import { TitleContext } from './overview/TitleContext'; +import { itemStyle1, itemStyleSucceed, itemStyle2, entriesOption } from './overview/overviewConst'; +import '../static/style/overview/overview.scss'; +import '../static/style/logPath.scss'; + +interface OverviewState { + trialConcurrency: number; +} + +export const BestMetricContext = React.createContext({ + bestAccuracy: 0 +}); + +class Overview extends React.Component<{}, OverviewState> { + static contextType = AppContext; + context!: React.ContextType; + + constructor(props) { + super(props); + this.state = { + trialConcurrency: EXPERIMENT.trialConcurrency + }; + } + + clickMaxTop = (event: React.SyntheticEvent): void => { + event.stopPropagation(); + // #999 panel active bgcolor; #b3b3b3 as usual + const { changeMetricGraphMode } = this.context; + changeMetricGraphMode('max'); + }; + + clickMinTop = (event: React.SyntheticEvent): void => { + event.stopPropagation(); + const { changeMetricGraphMode } = this.context; + changeMetricGraphMode('min'); + }; + + updateEntries = (event: React.FormEvent, item: any): void => { + if (item !== undefined) { + this.context.changeEntries(item.key); + } + }; + + render(): React.ReactNode { + const bestTrials = this.findBestTrials(); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const bestAccuracy = bestTrials.length > 0 ? bestTrials[0].accuracy! : NaN; + const accuracyGraphData = this.generateAccuracyGraph(bestTrials); + const noDataMessage = bestTrials.length > 0 ? '' : 'No data'; + + const maxExecDuration = EXPERIMENT.profile.params.maxExecDuration; + const execDuration = EXPERIMENT.profile.execDuration; + return ( + + {(value): React.ReactNode => { + const { + metricGraphMode, + bestTrialEntries, + maxDurationUnit, + updateOverviewPage, + changeMaxDurationUnit + } = value; + const maxActive = metricGraphMode === 'max' ? 'active' : ''; + const minActive = metricGraphMode === 'min' ? 'active' : ''; + return ( +
      +
      + {/* exp params */} +
      + + + </TitleContext.Provider> + <BestMetricContext.Provider value={{ bestAccuracy: bestAccuracy }}> + <ReBasicInfo /> + </BestMetricContext.Provider> + </div> + {/* duration & trial numbers */} + <div className='overviewProgress'> + <div className='duration'> + <TitleContext.Provider value={{ text: 'Duration', icon: 'Timer' }}> + <Title /> + </TitleContext.Provider> + <ExpDurationContext.Provider + value={{ + maxExecDuration, + execDuration, + updateOverviewPage, + maxDurationUnit, + changeMaxDurationUnit + }} + > + <ExpDuration /> + </ExpDurationContext.Provider> + </div> + <div className='trialCount'> + <TitleContext.Provider value={{ text: 'Trial numbers', icon: 'NumberSymbol' }}> + <Title /> + </TitleContext.Provider> + <ExpDurationContext.Provider + value={{ + maxExecDuration, + execDuration, + updateOverviewPage, + maxDurationUnit, + changeMaxDurationUnit + }} + > + <TrialCount /> + </ExpDurationContext.Provider> + </div> + </div> + {/* table */} + <div className='overviewTable'> + <Stack horizontal> + <div style={itemStyleSucceed}> + <TitleContext.Provider value={{ text: 'Top trials', icon: 'BulletedList' }}> + <Title /> + </TitleContext.Provider> + </div> + <div className='topTrialTitle'> + <Stack horizontal horizontalAlign='end'> + <DefaultButton + onClick={this.clickMaxTop} + className={maxActive} + styles={{ root: { minWidth: 70, padding: 0 } }} + > + <Icon iconName='Market' /> + <span className='max'>Max</span> + </DefaultButton> + <div className='mincenter'> + <DefaultButton + onClick={this.clickMinTop} + className={minActive} + styles={{ root: { minWidth: 70, padding: 0 } }} + > + <Icon iconName='MarketDown' /> + <span className='max'>Min</span> + </DefaultButton> + </div> + <div> + <Stack horizontal> + <div className='chooseEntry'>Display top</div> + <div> + <Dropdown + selectedKey={bestTrialEntries} + options={entriesOption} + onChange={this.updateEntries} + styles={{ root: { width: 70 } }} + /> + </div> + </Stack> + </div> + </Stack> + </div> + </Stack> + <SuccessTable trialIds={bestTrials.map(trial => trial.info.id)} /> + </div> + <div className='overviewCommand1'> + <Command1 /> + </div> + <div className='overviewCommand2'> + <Command2 /> + </div> + <div className='overviewChart'> + <Stack horizontal> + <div style={itemStyle1}> + <TitleContext.Provider + value={{ text: 'Trial metric chart', icon: 'HomeGroup' }} + > + <Title /> + </TitleContext.Provider> + </div> + <div style={itemStyle2}> + <Stack className='maxmin' horizontal> + <div className='circle' /> + <div>{`Top ${this.context.metricGraphMode}imal trials`}</div> + </Stack> + </div> + </Stack> + <Accuracy accuracyData={accuracyGraphData} accNodata={noDataMessage} height={380} /> + </div> + </div> + </div> + ); + }} + </AppContext.Consumer> + ); + } + + private findBestTrials(): Trial[] { + const bestTrials = TRIALS.sort(); + const { bestTrialEntries, metricGraphMode } = this.context; + if (metricGraphMode === 'max') { + bestTrials.reverse().splice(JSON.parse(bestTrialEntries)); + } else { + bestTrials.splice(JSON.parse(bestTrialEntries)); + } + return bestTrials; + } + + private generateAccuracyGraph(bestTrials: Trial[]): object { + const xSequence = bestTrials.map(trial => trial.sequenceId); + const ySequence = bestTrials.map(trial => trial.accuracy); + + return { + // support max show 0.0000000 + grid: { + left: 67, + right: 40 + }, + tooltip: { + trigger: 'item' + }, + xAxis: { + name: 'Trial', + type: 'category', + data: xSequence + }, + yAxis: { + name: 'Default metric', + type: 'value', + scale: true, + data: ySequence + }, + series: [ + { + symbolSize: 6, + type: 'scatter', + data: ySequence + } + ] + }; + } +} + +export default Overview; diff --git a/ts/webui/src/components/TrialsDetail.tsx b/ts/webui/src/components/TrialsDetail.tsx new file mode 100644 index 0000000000..5b84a66a85 --- /dev/null +++ b/ts/webui/src/components/TrialsDetail.tsx @@ -0,0 +1,98 @@ +import * as React from 'react'; +import { Stack, Pivot, PivotItem } from '@fluentui/react'; +import { EXPERIMENT, TRIALS } from '../static/datamodel'; +import { AppContext } from '../App'; +import DefaultPoint from './trial-detail/DefaultMetricPoint'; +import Duration from './trial-detail/Duration'; +import Para from './trial-detail/Para'; +import Intermediate from './trial-detail/Intermediate'; +import TableList from './trial-detail/TableList'; +import '../static/style/trialsDetail.scss'; +import '../static/style/search.scss'; + +interface TrialDetailState { + whichChart: string; +} + +class TrialsDetail extends React.Component<{}, TrialDetailState> { + static contextType = AppContext; + context!: React.ContextType<typeof AppContext>; + public interAccuracy = 0; + public interAllTableList = 2; + + public tableList!: TableList | null; + public searchInput!: HTMLInputElement | null; + + constructor(props) { + super(props); + this.state = { + whichChart: 'Default metric' + }; + } + + handleWhichTabs = (item: any): void => { + this.setState({ whichChart: item.props.headerText }); + }; + + render(): React.ReactNode { + const { whichChart } = this.state; + const source = TRIALS.toArray(); + const trialIds = TRIALS.toArray().map(trial => trial.id); + + return ( + <AppContext.Consumer> + {(_value): React.ReactNode => ( + <React.Fragment> + <div className='trial' id='tabsty'> + <Pivot + defaultSelectedKey={'0'} + className='detial-title' + onLinkClick={this.handleWhichTabs} + selectedKey={whichChart} + > + {/* <PivotItem tab={this.titleOfacc} key="1"> doesn't work*/} + <PivotItem headerText='Default metric' itemIcon='HomeGroup' key='Default metric'> + <Stack className='graph'> + <DefaultPoint trialIds={trialIds} visible={whichChart === 'Default metric'} /> + </Stack> + </PivotItem> + {/* <PivotItem tab={this.titleOfhyper} key="2"> */} + <PivotItem headerText='Hyper-parameter' itemIcon='Equalizer' key='Hyper-parameter'> + <Stack className='graph'> + <Para + trials={source} + searchSpace={EXPERIMENT.searchSpaceNew} + whichChart={whichChart} + /> + </Stack> + </PivotItem> + {/* <PivotItem tab={this.titleOfDuration} key="3"> */} + <PivotItem headerText='Duration' itemIcon='BarChartHorizontal' key='Duration'> + <Duration source={source} whichChart={whichChart} /> + </PivotItem> + {/* <PivotItem tab={this.titleOfIntermediate} key="4"> */} + <PivotItem + headerText='Intermediate result' + itemIcon='StackedLineChart' + key='Intermediate result' + > + {/* *why this graph has small footprint? */} + <Intermediate source={source} whichChart={whichChart} /> + </PivotItem> + </Pivot> + </div> + {/* trial table list */} + <div style={{ backgroundColor: '#fff', marginTop: 10 }}> + <TableList + tableSource={source} + trialsUpdateBroadcast={this.context.trialsUpdateBroadcast} + /> + </div> + </React.Fragment> + )} + </AppContext.Consumer> + ); + } +} + +export default TrialsDetail; diff --git a/src/webui/src/components/buttons/Icon.tsx b/ts/webui/src/components/buttons/Icon.tsx similarity index 80% rename from src/webui/src/components/buttons/Icon.tsx rename to ts/webui/src/components/buttons/Icon.tsx index 4d77758feb..b552de115d 100644 --- a/src/webui/src/components/buttons/Icon.tsx +++ b/ts/webui/src/components/buttons/Icon.tsx @@ -11,11 +11,14 @@ const copy = <Icon iconName='Copy' />; const tableListIcon = <Icon iconName='BulletedList' />; const downLoadIcon = { iconName: 'Download' }; const infoIconAbout = { iconName: 'info' }; -const timeIcon = { iconName: 'Refresh' }; +const timeIcon = { iconName: 'ReminderTime' }; const disableUpdates = { iconName: 'DisableUpdates' }; const requency = { iconName: 'Timer' }; const closeTimer = { iconName: 'Blocked2' }; const LineChart = <Icon iconName='LineChart' />; +const Edit = <Icon iconName='Edit' />; +const CheckMark = <Icon iconName='CheckMark' />; +const Cancel = <Icon iconName='Cancel' />; export { infoIcon, @@ -31,5 +34,8 @@ export { disableUpdates, requency, closeTimer, - LineChart + LineChart, + Edit, + CheckMark, + Cancel }; diff --git a/src/webui/src/components/buttons/ModalTheme.tsx b/ts/webui/src/components/buttons/ModalTheme.tsx similarity index 100% rename from src/webui/src/components/buttons/ModalTheme.tsx rename to ts/webui/src/components/buttons/ModalTheme.tsx diff --git a/ts/webui/src/components/modals/ChangeColumnComponent.tsx b/ts/webui/src/components/modals/ChangeColumnComponent.tsx new file mode 100644 index 0000000000..8fbc10980c --- /dev/null +++ b/ts/webui/src/components/modals/ChangeColumnComponent.tsx @@ -0,0 +1,120 @@ +import * as React from 'react'; +import { Dialog, DialogType, DialogFooter, Checkbox, PrimaryButton, DefaultButton } from '@fluentui/react'; + +interface ChangeColumnState { + // buffer, not saved yet + currentSelected: string[]; +} + +interface ChangeColumnProps { + allColumns: SimpleColumn[]; // all column List + selectedColumns: string[]; // user selected column list + onSelectedChange: (val: string[]) => void; + onHideDialog: () => void; + minSelected?: number; +} + +interface SimpleColumn { + key: string; // key for management + name: string; // name to display +} + +interface CheckBoxItems { + label: string; + checked: boolean; + onChange: () => void; +} + +class ChangeColumnComponent extends React.Component<ChangeColumnProps, ChangeColumnState> { + constructor(props: ChangeColumnProps) { + super(props); + this.state = { + currentSelected: this.props.selectedColumns + }; + } + + makeChangeHandler = (label: string): any => { + return (ev: any, checked: boolean): void => this.onCheckboxChange(ev, label, checked); + }; + + onCheckboxChange = ( + ev: React.FormEvent<HTMLElement | HTMLInputElement> | undefined, + label: string, + val?: boolean + ): void => { + const source: string[] = [...this.state.currentSelected]; + if (val === true) { + if (!source.includes(label)) { + source.push(label); + this.setState({ currentSelected: source }); + } + } else { + // remove from source + const result = source.filter(item => item !== label); + this.setState({ currentSelected: result }); + } + }; + + saveUserSelectColumn = (): void => { + const { currentSelected } = this.state; + const { allColumns, onSelectedChange } = this.props; + const selectedColumns = allColumns.map(column => column.key).filter(key => currentSelected.includes(key)); + onSelectedChange(selectedColumns); + this.hideDialog(); + }; + + // user exit dialog + cancelOption = (): void => { + // reset select column + this.setState({ currentSelected: this.props.selectedColumns }, () => { + this.hideDialog(); + }); + }; + + private hideDialog = (): void => { + this.props.onHideDialog(); + }; + + render(): React.ReactNode { + const { allColumns, minSelected } = this.props; + const { currentSelected } = this.state; + return ( + <div> + <Dialog + hidden={false} + dialogContentProps={{ + type: DialogType.largeHeader, + title: 'Customize columns', + subText: 'You can choose which columns you wish to see.' + }} + modalProps={{ + isBlocking: false, + styles: { main: { maxWidth: 450 } } + }} + > + <div className='columns-height'> + {allColumns.map(item => ( + <Checkbox + key={item.key} + label={item.name} + checked={currentSelected.includes(item.key)} + onChange={this.makeChangeHandler(item.key)} + styles={{ root: { marginBottom: 8 } }} + /> + ))} + </div> + <DialogFooter> + <PrimaryButton + text='Save' + onClick={this.saveUserSelectColumn} + disabled={currentSelected.length < (minSelected === undefined ? 1 : minSelected)} + /> + <DefaultButton text='Cancel' onClick={this.cancelOption} /> + </DialogFooter> + </Dialog> + </div> + ); + } +} + +export default ChangeColumnComponent; diff --git a/ts/webui/src/components/modals/Compare.tsx b/ts/webui/src/components/modals/Compare.tsx new file mode 100644 index 0000000000..936c2714d3 --- /dev/null +++ b/ts/webui/src/components/modals/Compare.tsx @@ -0,0 +1,245 @@ +import * as React from 'react'; +import { renderToString } from 'react-dom/server'; +import { Stack, Modal, IconButton, IDragOptions, ContextualMenu } from '@fluentui/react'; +import ReactEcharts from 'echarts-for-react'; +import { TooltipForIntermediate, TableObj, SingleAxis } from '../../static/interface'; +import { contentStyles, iconButtonStyles } from '../buttons/ModalTheme'; +import '../../static/style/compare.scss'; +import { convertDuration, parseMetrics } from '../../static/function'; +import { EXPERIMENT, TRIALS } from '../../static/datamodel'; + +function _getWebUIWidth(): number { + return window.innerWidth; +} + +const dragOptions: IDragOptions = { + moveMenuItemText: 'Move', + closeMenuItemText: 'Close', + menu: ContextualMenu +}; + +// TODO: this should be refactored to the common modules +// copied from trial.ts +function _parseIntermediates(trial: TableObj): number[] { + const intermediates: number[] = []; + for (const metric of trial.intermediates) { + if (metric === undefined) { + break; + } + const parsedMetric = parseMetrics(metric.data); + if (typeof parsedMetric === 'object') { + // TODO: should handle more types of metric keys + intermediates.push(parsedMetric.default); + } else { + intermediates.push(parsedMetric); + } + } + return intermediates; +} + +interface Item { + id: string; + sequenceId: number; + duration: string; + parameters: Map<string, any>; + metrics: Map<string, any>; + intermediates: number[]; +} + +interface CompareProps { + trials: TableObj[]; + title: string; + showDetails: boolean; + onHideDialog: () => void; +} + +class Compare extends React.Component<CompareProps, {}> { + constructor(props: CompareProps) { + super(props); + } + + private _generateTooltipSummary(row: Item, metricKey: string): string { + return renderToString( + <div className='tooldetailAccuracy'> + <div>Trial ID: {row.id}</div> + <div>Default metric: {row.metrics.get(metricKey) || 'N/A'}</div> + </div> + ); + } + + private _intermediates(items: Item[], metricKey: string): React.ReactNode { + // Precondition: make sure `items` is not empty + const xAxisMax = Math.max(...items.map(item => item.intermediates.length)); + const xAxis = Array(xAxisMax) + .fill(0) + .map((_, i) => i + 1); // [1, 2, 3, ..., xAxisMax] + const dataForEchart = items.map(item => ({ + name: item.id, + data: item.intermediates, + type: 'line' + })); + const legend = dataForEchart.map(item => item.name); + const option = { + tooltip: { + trigger: 'item', + enterable: true, + position: (point: number[], data: TooltipForIntermediate): [number, number] => { + if (data.dataIndex < length / 2) { + return [point[0], 80]; + } else { + return [point[0] - 300, 80]; + } + }, + formatter: (data: TooltipForIntermediate): string => { + const item = items.find(k => k.id === data.seriesName) as Item; + return this._generateTooltipSummary(item, metricKey); + } + }, + grid: { + left: '5%', + top: 40, + containLabel: true + }, + legend: { + type: 'scroll', + right: 40, + left: legend.length > 6 ? 80 : null, + data: legend + }, + xAxis: { + type: 'category', + boundaryGap: false, + data: xAxis + }, + yAxis: { + type: 'value', + name: 'Metric', + scale: true + }, + series: dataForEchart + }; + return ( + <ReactEcharts + option={option} + style={{ width: '100%', height: 418, margin: '0 auto' }} + notMerge={true} // update now + /> + ); + } + + private _renderRow( + key: string, + rowName: string, + className: string, + items: Item[], + formatter: (item: Item) => string + ): React.ReactNode { + return ( + <tr key={key}> + <td className='column'>{rowName}</td> + {items.map(item => ( + <td className={className} key={item.id}> + {formatter(item)} + </td> + ))} + </tr> + ); + } + + private _overlapKeys(s: Map<string, any>[]): string[] { + // Calculate the overlapped keys for multiple + const intersection: string[] = []; + for (const i of s[0].keys()) { + let inAll = true; + for (const t of s) { + if (!Array.from(t.keys()).includes(i)) { + inAll = false; + break; + } + } + if (inAll) { + intersection.push(i); + } + } + return intersection; + } + + // render table column --- + private _columns(items: Item[]): React.ReactNode { + // Precondition: make sure `items` is not empty + const width = _getWebUIWidth(); + let scrollClass: string = ''; + if (width > 1200) { + scrollClass = items.length > 3 ? 'flex' : ''; + } else if (width < 700) { + scrollClass = items.length > 1 ? 'flex' : ''; + } else { + scrollClass = items.length > 2 ? 'flex' : ''; + } + const parameterKeys = this._overlapKeys(items.map(item => item.parameters)); + const metricKeys = this._overlapKeys(items.map(item => item.metrics)); + return ( + <table className={`compare-modal-table ${scrollClass}`}> + <tbody> + {this._renderRow('id', 'ID', 'value idList', items, item => item.id)} + {this._renderRow('trialnum', 'Trial No.', 'value', items, item => item.sequenceId.toString())} + {this._renderRow('duration', 'Duration', 'value', items, item => item.duration)} + {parameterKeys.map(k => + this._renderRow(`space_${k}`, k, 'value', items, item => item.parameters.get(k)) + )} + {metricKeys.map(k => + this._renderRow(`metrics_${k}`, `Metric: ${k}`, 'value', items, item => item.metrics.get(k)) + )} + </tbody> + </table> + ); + } + + render(): React.ReactNode { + const { onHideDialog, trials, title, showDetails } = this.props; + const flatten = (m: Map<SingleAxis, any>): Map<string, any> => { + return new Map(Array.from(m).map(([key, value]) => [key.baseName, value])); + }; + const inferredSearchSpace = TRIALS.inferredSearchSpace(EXPERIMENT.searchSpaceNew); + const items: Item[] = trials.map(trial => ({ + id: trial.id, + sequenceId: trial.sequenceId, + duration: convertDuration(trial.duration), + parameters: flatten(trial.parameters(inferredSearchSpace)), + metrics: flatten(trial.metrics(TRIALS.inferredMetricSpace())), + intermediates: _parseIntermediates(trial) + })); + const metricKeys = this._overlapKeys(items.map(item => item.metrics)); + const defaultMetricKey = !metricKeys || metricKeys.includes('default') ? 'default' : metricKeys[0]; + + return ( + <Modal + isOpen={true} + containerClassName={contentStyles.container} + className='compare-modal' + allowTouchBodyScroll={true} + dragOptions={dragOptions} + onDismiss={onHideDialog} + > + <div> + <div className={contentStyles.header}> + <span>{title}</span> + <IconButton + styles={iconButtonStyles} + iconProps={{ iconName: 'Cancel' }} + ariaLabel='Close popup modal' + onClick={onHideDialog} + /> + </div> + <Stack className='compare-modal-intermediate'> + {this._intermediates(items, defaultMetricKey)} + <Stack className='compare-yAxis'># Intermediate result</Stack> + </Stack> + {showDetails && <Stack>{this._columns(items)}</Stack>} + </div> + </Modal> + ); + } +} + +export default Compare; diff --git a/src/webui/src/components/modals/CustomizedTrial.tsx b/ts/webui/src/components/modals/CustomizedTrial.tsx similarity index 97% rename from src/webui/src/components/modals/CustomizedTrial.tsx rename to ts/webui/src/components/modals/CustomizedTrial.tsx index 86d4ed3c50..0479ad7610 100644 --- a/src/webui/src/components/modals/CustomizedTrial.tsx +++ b/ts/webui/src/components/modals/CustomizedTrial.tsx @@ -60,7 +60,10 @@ class Customize extends React.Component<CustomizeProps, CustomizeState> { Object.keys(customized).map(item => { if (item !== 'tag') { // unified data type - if (typeof copyTrialParameter[item] === 'number' && typeof customized[item] === 'string') { + if ( + (typeof copyTrialParameter[item] === 'number' && typeof customized[item] === 'string') || + (typeof copyTrialParameter[item] === 'boolean' && typeof customized[item] === 'string') + ) { customized[item] = JSON.parse(customized[item]); } if (searchSpace[item] === undefined) { diff --git a/src/webui/src/components/modals/ExperimentPanel.tsx b/ts/webui/src/components/modals/ExperimentPanel.tsx similarity index 100% rename from src/webui/src/components/modals/ExperimentPanel.tsx rename to ts/webui/src/components/modals/ExperimentPanel.tsx diff --git a/src/webui/src/components/modals/Killjob.tsx b/ts/webui/src/components/modals/Killjob.tsx similarity index 93% rename from src/webui/src/components/modals/Killjob.tsx rename to ts/webui/src/components/modals/Killjob.tsx index e35be32cfa..0a567815a8 100644 --- a/src/webui/src/components/modals/Killjob.tsx +++ b/ts/webui/src/components/modals/Killjob.tsx @@ -112,11 +112,15 @@ class KillJob extends React.Component<KillJobProps, KillJobState> { setInitialFocus={true} > <div className={styles.header}> - <p className={styles.title}>Kill trial</p> + <p className={styles.title} style={{ color: '#333' }}> + Kill trial + </p> </div> <div className={styles.inner}> <div> - <p className={styles.subtext}>{prompString}</p> + <p className={styles.subtext} style={{ color: '#333' }}> + {prompString} + </p> </div> </div> <FocusZone> diff --git a/src/webui/src/components/modals/LogPanel.tsx b/ts/webui/src/components/modals/LogPanel.tsx similarity index 100% rename from src/webui/src/components/modals/LogPanel.tsx rename to ts/webui/src/components/modals/LogPanel.tsx diff --git a/src/webui/src/components/modals/MessageInfo.tsx b/ts/webui/src/components/modals/MessageInfo.tsx similarity index 100% rename from src/webui/src/components/modals/MessageInfo.tsx rename to ts/webui/src/components/modals/MessageInfo.tsx diff --git a/src/webui/src/components/modals/customized.scss b/ts/webui/src/components/modals/customized.scss similarity index 100% rename from src/webui/src/components/modals/customized.scss rename to ts/webui/src/components/modals/customized.scss diff --git a/src/webui/src/components/overview/Accuracy.tsx b/ts/webui/src/components/overview/Accuracy.tsx similarity index 95% rename from src/webui/src/components/overview/Accuracy.tsx rename to ts/webui/src/components/overview/Accuracy.tsx index 2f2fd9253b..395e04d364 100644 --- a/src/webui/src/components/overview/Accuracy.tsx +++ b/ts/webui/src/components/overview/Accuracy.tsx @@ -22,7 +22,7 @@ class Accuracy extends React.Component<AccuracyProps, {}> { render(): React.ReactNode { const { accNodata, accuracyData, height } = this.props; return ( - <div> + <div style={{ position: 'relative' }}> <ReactEcharts option={accuracyData} style={{ diff --git a/ts/webui/src/components/overview/Title.tsx b/ts/webui/src/components/overview/Title.tsx new file mode 100644 index 0000000000..06367c049d --- /dev/null +++ b/ts/webui/src/components/overview/Title.tsx @@ -0,0 +1,16 @@ +import React from 'react'; +import { Stack, Icon, initializeIcons } from '@fluentui/react'; +import { TitleContext } from './TitleContext'; +import '../../static/style/overview/overviewTitle.scss'; +initializeIcons(); + +export const Title = (): any => ( + <TitleContext.Consumer> + {(value): React.ReactNode => ( + <Stack horizontal className='panelTitle'> + <Icon iconName={value.icon} /> + <span>{value.text}</span> + </Stack> + )} + </TitleContext.Consumer> +); diff --git a/ts/webui/src/components/overview/TitleContext.tsx b/ts/webui/src/components/overview/TitleContext.tsx new file mode 100644 index 0000000000..5f7a1b8e27 --- /dev/null +++ b/ts/webui/src/components/overview/TitleContext.tsx @@ -0,0 +1,6 @@ +import * as React from 'react'; + +export const TitleContext = React.createContext({ + text: '', + icon: '' +}); diff --git a/ts/webui/src/components/overview/command/Command1.tsx b/ts/webui/src/components/overview/command/Command1.tsx new file mode 100644 index 0000000000..fc78a795fb --- /dev/null +++ b/ts/webui/src/components/overview/command/Command1.tsx @@ -0,0 +1,40 @@ +import React from 'react'; +import { EXPERIMENT } from '../../../static/datamodel'; +import '../../../static/style/overview/command.scss'; + +export const Command1 = (): any => { + const tuner = EXPERIMENT.profile.params.tuner; + const advisor = EXPERIMENT.profile.params.advisor; + const assessor = EXPERIMENT.profile.params.assessor; + let title = ''; + let builtinName = ''; + if (tuner !== undefined) { + title = title.concat('Tuner'); + if (tuner.builtinTunerName !== undefined) { + builtinName = builtinName.concat(tuner.builtinTunerName); + } + } + if (advisor !== undefined) { + title = title.concat('/ Assessor'); + if (advisor.builtinAdvisorName !== undefined) { + builtinName = builtinName.concat(advisor.builtinAdvisorName); + } + } + if (assessor !== undefined) { + title = title.concat('/ Addvisor'); + if (assessor.builtinAssessorName !== undefined) { + builtinName = builtinName.concat(assessor.builtinAssessorName); + } + } + + return ( + <div className='basic'> + <div> + <p className='command'>Training platform</p> + <div className='nowrap'>{EXPERIMENT.profile.params.trainingServicePlatform}</div> + <p className='lineMargin'>{title}</p> + <div className='nowrap'>{builtinName}</div> + </div> + </div> + ); +}; diff --git a/ts/webui/src/components/overview/command/Command2.tsx b/ts/webui/src/components/overview/command/Command2.tsx new file mode 100644 index 0000000000..d6cf4725e7 --- /dev/null +++ b/ts/webui/src/components/overview/command/Command2.tsx @@ -0,0 +1,65 @@ +import React from 'react'; +import { TooltipHost, DirectionalHint } from '@fluentui/react'; +import { EXPERIMENT } from '../../../static/datamodel'; +import { TOOLTIP_BACKGROUND_COLOR } from '../../../static/const'; +import '../../../static/style/overview/command.scss'; + +export const Command2 = (): any => { + const clusterMetaData = EXPERIMENT.profile.params.clusterMetaData; + let trialCommand = 'unknown'; + + if (clusterMetaData !== undefined) { + for (const item of clusterMetaData) { + if (item.key === 'command') { + trialCommand = item.value as string; + } + if (item.key === 'trial_config') { + if (typeof item.value === 'object' && 'command' in item.value) { + trialCommand = item.value.command as string; + } + } + } + } + return ( + <div className='basic'> + <p className='command'>Log directory</p> + <div className='nowrap'> + <TooltipHost + content={EXPERIMENT.profile.logDir || 'unknown'} + className='nowrap' + directionalHint={DirectionalHint.bottomCenter} + tooltipProps={{ + calloutProps: { + styles: { + beak: { background: TOOLTIP_BACKGROUND_COLOR }, + beakCurtain: { background: TOOLTIP_BACKGROUND_COLOR }, + calloutMain: { background: TOOLTIP_BACKGROUND_COLOR } + } + } + }} + > + {EXPERIMENT.profile.logDir || 'unknown'} + </TooltipHost> + </div> + <p className='lineMargin'>Trial command</p> + <div className='nowrap'> + <TooltipHost + content={trialCommand || 'unknown'} + className='nowrap' + directionalHint={DirectionalHint.bottomCenter} + tooltipProps={{ + calloutProps: { + styles: { + beak: { background: TOOLTIP_BACKGROUND_COLOR }, + beakCurtain: { background: TOOLTIP_BACKGROUND_COLOR }, + calloutMain: { background: TOOLTIP_BACKGROUND_COLOR } + } + } + }} + > + {trialCommand || 'unknown'} + </TooltipHost> + </div> + </div> + ); +}; diff --git a/ts/webui/src/components/overview/count/EditExperimentParam.tsx b/ts/webui/src/components/overview/count/EditExperimentParam.tsx new file mode 100644 index 0000000000..07eee048a4 --- /dev/null +++ b/ts/webui/src/components/overview/count/EditExperimentParam.tsx @@ -0,0 +1,218 @@ +import React, { useState, useCallback, useContext } from 'react'; +import axios from 'axios'; +import { Dropdown } from '@fluentui/react'; +import { EXPERIMENT } from '../../../static/datamodel'; +import { AppContext } from '../../../App'; +import { EditExpeParamContext } from './context'; +import { durationUnit } from '../overviewConst'; +import { MANAGER_IP, MAX_TRIAL_NUMBERS } from '../../../static/const'; +import { Edit, CheckMark, Cancel } from '../../buttons/Icon'; +import MessageInfo from '../../modals/MessageInfo'; +import '../../../static/style/overview/count.scss'; + +const DurationInputRef = React.createRef<HTMLInputElement>(); + +export const EditExperimentParam = (): any => { + const [isShowPencil, setShowPencil] = useState(true); + const [isShowSucceedInfo, setShowSucceedInfo] = useState(false); + const [typeInfo, setTypeInfo] = useState(''); + const [info, setInfo] = useState(''); + const showPencil = useCallback(() => { + setShowPencil(true); + }, []); + const hidePencil = useCallback(() => { + setShowPencil(false); + }, []); + const showSucceedInfo = useCallback(() => setShowSucceedInfo(true), []); + const hideSucceedInfo = useCallback(() => { + setShowSucceedInfo(false); + }, []); + const { title, field, editType, maxExecDuration, maxTrialNum, trialConcurrency, updateOverviewPage } = useContext( + EditExpeParamContext + ); + const { maxDurationUnit, changeMaxDurationUnit } = useContext(AppContext); + const [unit, setUnit] = useState(maxDurationUnit); + let defaultVal = ''; + let editVal = ''; + if (title === 'Max duration') { + defaultVal = maxExecDuration; + editVal = maxExecDuration; + } else if (title === MAX_TRIAL_NUMBERS) { + defaultVal = maxTrialNum.toString(); + editVal = maxTrialNum.toString(); + } else { + defaultVal = trialConcurrency.toString(); + editVal = trialConcurrency.toString(); + } + const [editInputVal, setEditValInput] = useState(editVal); + + function setInputVal(event: any): void { + setEditValInput(event.target.value); + } + + function showMessageInfo(info: string, typeInfo: string): any { + setInfo(info); + setTypeInfo(typeInfo); + showSucceedInfo(); + setTimeout(hideSucceedInfo, 2000); + } + + function updateUnit(event: React.FormEvent<HTMLDivElement>, item: any): void { + if (item !== undefined) { + setUnit(item.key); + } + } + + async function confirmEdit(): Promise<void> { + const isMaxDuration = title === 'Max duration'; + const newProfile = Object.assign({}, EXPERIMENT.profile); + let beforeParam = ''; + if (isMaxDuration) { + if (!editInputVal.match(/^\d+(?=\.{0,1}\d+$|$)/)) { + showMessageInfo('Please enter a number!', 'error'); + setEditValInput(defaultVal); + return; + } + } else { + if (!editInputVal.match(/^[1-9]\d*$/)) { + showMessageInfo('Please enter a positive integer!', 'error'); + setEditValInput(defaultVal); + return; + } + } + if (isMaxDuration) { + beforeParam = maxExecDuration; + } else if (title === MAX_TRIAL_NUMBERS) { + beforeParam = maxTrialNum.toString(); + } else { + beforeParam = trialConcurrency.toString(); + } + + if (editInputVal === beforeParam) { + if (isMaxDuration) { + if (maxDurationUnit === unit) { + showMessageInfo(`Trial ${field} has not changed`, 'error'); + return; + } + } else { + showMessageInfo(`Trial ${field} has not changed`, 'error'); + return; + } + } + if (isMaxDuration) { + const maxDura = JSON.parse(editInputVal); + if (unit === 'm') { + newProfile.params[field] = maxDura * 60; + } else if (unit === 'h') { + newProfile.params[field] = maxDura * 3600; + } else { + newProfile.params[field] = maxDura * 24 * 60 * 60; + } + } else { + newProfile.params[field] = parseInt(editInputVal, 10); + } + // rest api, modify trial concurrency value + try { + const res = await axios.put(`${MANAGER_IP}/experiment`, newProfile, { + // eslint-disable-next-line @typescript-eslint/camelcase + params: { update_type: editType } + }); + if (res.status === 200) { + showMessageInfo(`Successfully updated experiment's ${field}`, 'success'); + changeMaxDurationUnit(unit); + } + } catch (error) { + if (error.response && error.response.data.error) { + showMessageInfo(`Failed to update trial ${field}\n${error.response.data.error}`, 'error'); + } else if (error.response) { + showMessageInfo(`Failed to update trial ${field}\nServer responsed ${error.response.status}`, 'error'); + } else if (error.message) { + showMessageInfo(`Failed to update trial ${field}\n${error.message}`, 'error'); + } else { + showMessageInfo(`Failed to update trial ${field}\nUnknown error`, 'error'); + } + setEditValInput(defaultVal); + } + showPencil(); + updateOverviewPage(); + } + + function cancelEdit(): void { + setEditValInput(defaultVal); + showPencil(); + setUnit(maxDurationUnit); + } + + function convertUnit(val: string): string { + if (val === 'd') { + return 'day'; + } else if (val === 'h') { + return 'hour'; + } else if (val === 'm') { + return 'min'; + } else { + return val; + } + } + + return ( + <AppContext.Consumer> + {(values): React.ReactNode => { + return ( + <EditExpeParamContext.Consumer> + {(value): React.ReactNode => { + let editClassName = ''; + if (value.field === 'maxExecDuration') { + editClassName = isShowPencil ? 'noEditDuration' : 'editDuration'; + } + return ( + <React.Fragment> + <div className={`${editClassName} editparam`}> + <span>{value.title}</span> + <input + className={`${value.field} editparam-Input`} + ref={DurationInputRef} + disabled={isShowPencil ? true : false} + value={editInputVal} + onChange={setInputVal} + /> + {isShowPencil && title === 'Max duration' && ( + <span>{convertUnit(values.maxDurationUnit)}</span> + )} + {!isShowPencil && title === 'Max duration' && ( + <Dropdown + selectedKey={unit} + options={durationUnit} + className='editparam-dropdown' + onChange={updateUnit} + /> + )} + {isShowPencil && ( + <span className='edit' onClick={hidePencil}> + {Edit} + </span> + )} + {!isShowPencil && ( + <span className='series'> + <span className='confirm' onClick={confirmEdit}> + {CheckMark} + </span> + <span className='cancel' onClick={cancelEdit}> + {Cancel} + </span> + </span> + )} + + {isShowSucceedInfo && ( + <MessageInfo className='info' typeInfo={typeInfo} info={info} /> + )} + </div> + </React.Fragment> + ); + }} + </EditExpeParamContext.Consumer> + ); + }} + </AppContext.Consumer> + ); +}; diff --git a/ts/webui/src/components/overview/count/ExpDuration.tsx b/ts/webui/src/components/overview/count/ExpDuration.tsx new file mode 100644 index 0000000000..12a1d723ff --- /dev/null +++ b/ts/webui/src/components/overview/count/ExpDuration.tsx @@ -0,0 +1,64 @@ +import React from 'react'; +import { Stack, ProgressIndicator, TooltipHost, DirectionalHint } from '@fluentui/react'; +import { EXPERIMENT } from '../../../static/datamodel'; +import { CONTROLTYPE, TOOLTIP_BACKGROUND_COLOR } from '../../../static/const'; +import { convertDuration, convertTimeAsUnit } from '../../../static/function'; +import { EditExperimentParam } from './EditExperimentParam'; +import { ExpDurationContext } from './ExpDurationContext'; +import { EditExpeParamContext } from './context'; +import { durationItem1, durationItem2 } from './commonStyle'; +import '../../../static/style/overview/count.scss'; + +export const ExpDuration = (): any => ( + <ExpDurationContext.Consumer> + {(value): React.ReactNode => { + const { maxExecDuration, execDuration, maxDurationUnit, updateOverviewPage } = value; + const tooltip = maxExecDuration - execDuration; + const percent = execDuration / maxExecDuration; + const execDurationStr = convertDuration(execDuration); + const maxExecDurationStr = convertTimeAsUnit(maxDurationUnit, maxExecDuration).toString(); + return ( + <Stack horizontal className='ExpDuration'> + <div style={durationItem1}> + <TooltipHost + content={`${convertDuration(tooltip)} remaining`} + directionalHint={DirectionalHint.bottomCenter} + tooltipProps={{ + calloutProps: { + styles: { + beak: { background: TOOLTIP_BACKGROUND_COLOR }, + beakCurtain: { background: TOOLTIP_BACKGROUND_COLOR }, + calloutMain: { background: TOOLTIP_BACKGROUND_COLOR } + } + } + }} + > + <ProgressIndicator className={EXPERIMENT.status} percentComplete={percent} barHeight={15} /> + </TooltipHost> + {/* execDuration / maxDuration: 20min / 1h */} + <div className='exp-progress'> + <span className={`${EXPERIMENT.status} bold`}>{execDurationStr}</span> + <span className='joiner'>/</span> + <span>{`${maxExecDurationStr} ${maxDurationUnit}`}</span> + </div> + </div> + <div style={durationItem2}> + <EditExpeParamContext.Provider + value={{ + editType: CONTROLTYPE[0], + field: 'maxExecDuration', + title: 'Max duration', + maxExecDuration: maxExecDurationStr, + maxTrialNum: EXPERIMENT.profile.params.maxTrialNum, + trialConcurrency: EXPERIMENT.profile.params.trialConcurrency, + updateOverviewPage + }} + > + <EditExperimentParam /> + </EditExpeParamContext.Provider> + </div> + </Stack> + ); + }} + </ExpDurationContext.Consumer> +); diff --git a/ts/webui/src/components/overview/count/ExpDurationContext.tsx b/ts/webui/src/components/overview/count/ExpDurationContext.tsx new file mode 100644 index 0000000000..6f0c77d265 --- /dev/null +++ b/ts/webui/src/components/overview/count/ExpDurationContext.tsx @@ -0,0 +1,10 @@ +import React from 'react'; +export const ExpDurationContext = React.createContext({ + maxExecDuration: 0, + execDuration: 0, + // eslint-disable-next-line @typescript-eslint/no-empty-function + updateOverviewPage: (): void => {}, + maxDurationUnit: 'm', + // eslint-disable-next-line @typescript-eslint/no-empty-function + changeMaxDurationUnit: (_val: 'd' | 'h' | 'm'): void => {} +}); diff --git a/ts/webui/src/components/overview/count/TrialCount.tsx b/ts/webui/src/components/overview/count/TrialCount.tsx new file mode 100644 index 0000000000..f5846bdae0 --- /dev/null +++ b/ts/webui/src/components/overview/count/TrialCount.tsx @@ -0,0 +1,111 @@ +import * as React from 'react'; +import { Stack, TooltipHost, ProgressIndicator, DirectionalHint } from '@fluentui/react'; +import { EXPERIMENT, TRIALS } from '../../../static/datamodel'; +import { CONTROLTYPE, TOOLTIP_BACKGROUND_COLOR, MAX_TRIAL_NUMBERS } from '../../../static/const'; +import { EditExperimentParam } from './EditExperimentParam'; +import { EditExpeParamContext } from './context'; +import { ExpDurationContext } from './ExpDurationContext'; +import { trialCountItem1, trialCountItem2 } from './commonStyle'; + +export const TrialCount = (): any => { + const count = TRIALS.countStatus(); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const stoppedCount = count.get('USER_CANCELED')! + count.get('SYS_CANCELED')! + count.get('EARLY_STOPPED')!; + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const bar2 = count.get('RUNNING')! + count.get('SUCCEEDED')! + count.get('FAILED')! + stoppedCount; + const maxTrialNum = EXPERIMENT.profile.params.maxTrialNum; + // support type [0, 1], not 98% + const bar2Percent = bar2 / maxTrialNum; + return ( + <ExpDurationContext.Consumer> + {(value): React.ReactNode => { + const { updateOverviewPage } = value; + return ( + <React.Fragment> + <Stack horizontal horizontalAlign='space-between' className='ExpDuration'> + <div style={trialCountItem1}> + <TooltipHost + content={bar2.toString()} + directionalHint={DirectionalHint.bottomCenter} + tooltipProps={{ + calloutProps: { + styles: { + beak: { background: TOOLTIP_BACKGROUND_COLOR }, + beakCurtain: { background: TOOLTIP_BACKGROUND_COLOR }, + calloutMain: { background: TOOLTIP_BACKGROUND_COLOR } + } + } + }} + > + <ProgressIndicator + className={EXPERIMENT.status} + percentComplete={bar2Percent} + barHeight={15} + /> + </TooltipHost> + <div className='exp-progress'> + <span className={`${EXPERIMENT.status} bold`}>{bar2}</span> + <span className='joiner'>/</span> + <span>{maxTrialNum}</span> + </div> + </div> + <div style={trialCountItem2}> + <EditExpeParamContext.Provider + value={{ + title: MAX_TRIAL_NUMBERS, + field: 'maxTrialNum', + editType: CONTROLTYPE[1], + maxExecDuration: '', + maxTrialNum: EXPERIMENT.profile.params.maxTrialNum, + trialConcurrency: EXPERIMENT.profile.params.trialConcurrency, + updateOverviewPage + }} + > + <div className='maxTrialNum'> + <EditExperimentParam /> + </div> + </EditExpeParamContext.Provider> + <EditExpeParamContext.Provider + value={{ + title: 'Concurrency', + field: 'trialConcurrency', + editType: CONTROLTYPE[2], + // maxExecDuration: EXPERIMENT.profile.params.maxExecDuration, + maxExecDuration: '', + maxTrialNum: EXPERIMENT.profile.params.maxTrialNum, + trialConcurrency: EXPERIMENT.profile.params.trialConcurrency, + updateOverviewPage + }} + > + <EditExperimentParam /> + </EditExpeParamContext.Provider> + </div> + </Stack> + <Stack horizontal horizontalAlign='space-between' className='mess'> + <div className='basic'> + <p>Running</p> + <div>{count.get('RUNNING')}</div> + </div> + <div className='basic'> + <p>Succeeded</p> + <div>{count.get('SUCCEEDED')}</div> + </div> + <div className='basic'> + <p>Stopped</p> + <div>{stoppedCount}</div> + </div> + <div className='basic'> + <p>Failed</p> + <div>{count.get('FAILED')}</div> + </div> + <div className='basic'> + <p>Waiting</p> + <div>{count.get('WAITING')}</div> + </div> + </Stack> + </React.Fragment> + ); + }} + </ExpDurationContext.Consumer> + ); +}; diff --git a/ts/webui/src/components/overview/count/commonStyle.ts b/ts/webui/src/components/overview/count/commonStyle.ts new file mode 100644 index 0000000000..2138aa7a5c --- /dev/null +++ b/ts/webui/src/components/overview/count/commonStyle.ts @@ -0,0 +1,16 @@ +const durationItem1: React.CSSProperties = { + width: '33%' +}; +const durationItem2: React.CSSProperties = { + width: '52%', + paddingLeft: '15%' +}; + +const trialCountItem1: React.CSSProperties = { + width: '33%' +}; +const trialCountItem2: React.CSSProperties = { + width: '52%' +}; + +export { durationItem1, durationItem2, trialCountItem1, trialCountItem2 }; diff --git a/ts/webui/src/components/overview/count/context.tsx b/ts/webui/src/components/overview/count/context.tsx new file mode 100644 index 0000000000..7d11f8008c --- /dev/null +++ b/ts/webui/src/components/overview/count/context.tsx @@ -0,0 +1,17 @@ +import React from 'react'; +/*** + * const CONTROLTYPE = ['MAX_EXEC_DURATION', 'MAX_TRIAL_NUM', 'TRIAL_CONCURRENCY', 'SEARCH_SPACE']; + * [0], 'MAX_EXEC_DURATION', params.maxExecDuration + * [1], 'MAX_TRIAL_NUM', params.maxTrialNum + * [2], 'TRIAL_CONCURRENCY', params.trialConcurrency + */ +export const EditExpeParamContext = React.createContext({ + editType: '', + field: '', + title: '', + maxExecDuration: '', + maxTrialNum: 0, + trialConcurrency: 0, + // eslint-disable-next-line @typescript-eslint/no-empty-function + updateOverviewPage: (): void => {} +}); diff --git a/ts/webui/src/components/overview/experiment/BasicInfo.tsx b/ts/webui/src/components/overview/experiment/BasicInfo.tsx new file mode 100644 index 0000000000..605b9ed025 --- /dev/null +++ b/ts/webui/src/components/overview/experiment/BasicInfo.tsx @@ -0,0 +1,100 @@ +import React, { useState, useCallback } from 'react'; +import { Stack, Callout, Link, IconButton } from '@fluentui/react'; +import LogDrawer from '../../modals/LogPanel'; +import { EXPERIMENT } from '../../../static/datamodel'; +import { formatTimestamp } from '../../../static/function'; +import { useId } from '@uifabric/react-hooks'; +import { BestMetricContext } from '../../Overview'; +import { styles } from './basicInfoStyles'; +import '../../../static/style/progress/progress.scss'; +import '../../../static/style/progress/probar.scss'; + +export const ReBasicInfo = (): any => { + const labelId: string = useId('callout-label'); + const descriptionId: string = useId('callout-description'); + const ref = React.createRef<HTMLDivElement>(); + const [isCalloutVisible, setCalloutVisible] = useState(false); + const [isShowLogDrawer, setShowLogDrawer] = useState(false); + const onDismiss = useCallback(() => setCalloutVisible(false), []); + const showCallout = useCallback(() => setCalloutVisible(true), []); + + const closeLogDrawer = useCallback(() => setShowLogDrawer(false), []); + const ShowLogDrawer = useCallback(() => setShowLogDrawer(true), []); + + return ( + <div> + <div className='basic'> + <p> + ID: <span>{EXPERIMENT.profile.id}</span> + </p> + <div>{EXPERIMENT.profile.params.experimentName}</div> + </div> + <div className='basic'> + <p>Status</p> + <Stack horizontal className='status'> + <span className={`${EXPERIMENT.status} status-text`}>{EXPERIMENT.status}</span> + {EXPERIMENT.status === 'ERROR' ? ( + <div> + <div className={styles.buttonArea} ref={ref}> + <IconButton + iconProps={{ iconName: 'info' }} + onClick={isCalloutVisible ? onDismiss : showCallout} + /> + </div> + {isCalloutVisible && ( + <Callout + className={styles.callout} + ariaLabelledBy={labelId} + ariaDescribedBy={descriptionId} + role='alertdialog' + gapSpace={0} + target={ref} + onDismiss={onDismiss} + setInitialFocus={true} + > + <div className={styles.header}> + <p className={styles.title} id={labelId} style={{ color: '#333' }}> + Error + </p> + </div> + <div className={styles.inner}> + <p className={styles.subtext} id={descriptionId} style={{ color: '#333' }}> + {EXPERIMENT.error} + </p> + <div className={styles.actions}> + <Link className={styles.link} onClick={ShowLogDrawer}> + Learn about + </Link> + </div> + </div> + </Callout> + )} + </div> + ) : null} + </Stack> + </div> + <div className='basic'> + <BestMetricContext.Consumer> + {(value): React.ReactNode => ( + <Stack className='bestMetric'> + <p>Best metric</p> + <div className={EXPERIMENT.status}> + {isNaN(value.bestAccuracy) ? 'N/A' : value.bestAccuracy.toFixed(6)} + </div> + </Stack> + )} + </BestMetricContext.Consumer> + </div> + <div className='basic'> + <p>Start time</p> + <div className='nowrap'>{formatTimestamp(EXPERIMENT.profile.startTime)}</div> + </div> + <div className='basic'> + <p>End time</p> + <div className='nowrap'>{formatTimestamp(EXPERIMENT.profile.endTime)}</div> + </div> + {/* learn about click -> default active key is dispatcher. */} + {isShowLogDrawer ? <LogDrawer closeDrawer={closeLogDrawer} activeTab='dispatcher' /> : null} + </div> + ); +}; diff --git a/ts/webui/src/components/overview/experiment/basicInfoStyles.ts b/ts/webui/src/components/overview/experiment/basicInfoStyles.ts new file mode 100644 index 0000000000..05c9f819bc --- /dev/null +++ b/ts/webui/src/components/overview/experiment/basicInfoStyles.ts @@ -0,0 +1,52 @@ +import { FontWeights, mergeStyleSets, getTheme } from '@fluentui/react'; + +const theme = getTheme(); + +export const styles = mergeStyleSets({ + buttonArea: { + verticalAlign: 'top', + display: 'inline-block', + textAlign: 'center', + // margin: '0 100px', + minWidth: 30, + height: 30 + }, + callout: { + maxWidth: 300 + }, + header: { + padding: '18px 24px 12px' + }, + title: [ + theme.fonts.xLarge, + { + margin: 0, + color: theme.palette.neutralPrimary, + fontWeight: FontWeights.semilight + } + ], + inner: { + height: '100%', + padding: '0 24px 20px' + }, + actions: { + position: 'relative', + marginTop: 20, + width: '100%', + whiteSpace: 'nowrap' + }, + subtext: [ + theme.fonts.small, + { + margin: 0, + color: theme.palette.neutralPrimary, + fontWeight: FontWeights.semilight + } + ], + link: [ + theme.fonts.medium, + { + color: theme.palette.neutralPrimary + } + ] +}); diff --git a/ts/webui/src/components/overview/overviewConst.ts b/ts/webui/src/components/overview/overviewConst.ts new file mode 100644 index 0000000000..073e9c2735 --- /dev/null +++ b/ts/webui/src/components/overview/overviewConst.ts @@ -0,0 +1,27 @@ +const itemStyle1: React.CSSProperties = { + width: '75%' +}; +const itemStyleSucceed: React.CSSProperties = { + width: '28%' +}; + +const itemStyle2: React.CSSProperties = { + height: 38 +}; + +// top trials entries +const entriesOption = [ + { key: '10', text: '10' }, + { key: '20', text: '20' }, + { key: '30', text: '30' }, + { key: '50', text: '40' }, + { key: '100', text: '100' } +]; + +const durationUnit = [ + { key: 'm', text: 'min' }, + { key: 'h', text: 'hour' }, + { key: 'd', text: 'day' } +]; + +export { itemStyle1, itemStyleSucceed, itemStyle2, entriesOption, durationUnit }; diff --git a/src/webui/src/components/overview/Details.tsx b/ts/webui/src/components/overview/table/Details.tsx similarity index 94% rename from src/webui/src/components/overview/Details.tsx rename to ts/webui/src/components/overview/table/Details.tsx index 65407b7965..bf11ddb789 100644 --- a/src/webui/src/components/overview/Details.tsx +++ b/ts/webui/src/components/overview/table/Details.tsx @@ -1,6 +1,6 @@ import * as React from 'react'; import { DetailsRow, IDetailsRowBaseProps } from '@fluentui/react'; -import OpenRow from '../public-child/OpenRow'; +import OpenRow from '../../public-child/OpenRow'; interface DetailsProps { detailsProps: IDetailsRowBaseProps; diff --git a/src/webui/src/components/overview/SuccessTable.tsx b/ts/webui/src/components/overview/table/SuccessTable.tsx similarity index 67% rename from src/webui/src/components/overview/SuccessTable.tsx rename to ts/webui/src/components/overview/table/SuccessTable.tsx index 67c3ae8908..424ea75d5f 100644 --- a/src/webui/src/components/overview/SuccessTable.tsx +++ b/ts/webui/src/components/overview/table/SuccessTable.tsx @@ -1,12 +1,13 @@ import * as React from 'react'; import { DetailsList, IDetailsListProps, IColumn } from '@fluentui/react'; -import DefaultMetric from '../public-child/DefaultMetric'; +import DefaultMetric from '../../public-child/DefaultMetric'; import Details from './Details'; -import { convertDuration } from '../../static/function'; -import { TRIALS } from '../../static/datamodel'; -import { DETAILTABS } from '../stateless-component/NNItabs'; -import '../../static/style/succTable.scss'; -import '../../static/style/openRow.scss'; +import { convertDuration } from '../../../static/function'; +import { TRIALS } from '../../../static/datamodel'; +import { DETAILTABS } from '../../stateless-component/NNItabs'; +import '../../../static/style/succTable.scss'; +import '../../../static/style/tableStatus.css'; +import '../../../static/style/openRow.scss'; interface SuccessTableProps { trialIds: string[]; @@ -15,12 +16,17 @@ interface SuccessTableProps { interface SuccessTableState { columns: IColumn[]; source: Array<any>; + innerWidth: number; } class SuccessTable extends React.Component<SuccessTableProps, SuccessTableState> { constructor(props: SuccessTableProps) { super(props); - this.state = { columns: this.columns, source: TRIALS.table(this.props.trialIds) }; + this.state = { + columns: this.columns, + source: TRIALS.table(this.props.trialIds), + innerWidth: window.innerWidth + }; } private onRenderRow: IDetailsListProps['onRenderRow'] = props => { @@ -57,12 +63,10 @@ class SuccessTable extends React.Component<SuccessTableProps, SuccessTableState> } tooltipStr = ( - <div> - <p>The experiment is running, please wait for the final metric patiently.</p> - <div className='link'> - You could also find status of trial job with <span>{DETAILTABS}</span> button. - </div> - </div> + <React.Fragment> + The experiment is running, please wait for the final metric patiently. You could also find status of trial + job with <span>{DETAILTABS}</span> button. + </React.Fragment> ); columns = [ @@ -70,34 +74,36 @@ class SuccessTable extends React.Component<SuccessTableProps, SuccessTableState> name: 'Trial No.', key: 'sequenceId', fieldName: 'sequenceId', // required! - minWidth: 60, - maxWidth: 120, + minWidth: 50, + maxWidth: 87, isResizable: true, data: 'number', - onColumnClick: this.onColumnClick + onColumnClick: this.onColumnClick, + onRender: (item: any): React.ReactNode => <div className='succeed-padding'>{item.sequenceId}</div> }, { name: 'ID', key: 'id', fieldName: 'id', - minWidth: 80, - maxWidth: 100, + minWidth: 50, + maxWidth: 87, isResizable: true, className: 'tableHead leftTitle', data: 'string', - onColumnClick: this.onColumnClick + onColumnClick: this.onColumnClick, + onRender: (item: any): React.ReactNode => <div className='succeed-padding'>{item.id}</div> }, { name: 'Duration', key: 'duration', - minWidth: 100, - maxWidth: 210, + minWidth: 65, + maxWidth: 150, isResizable: true, fieldName: 'duration', data: 'number', onColumnClick: this.onColumnClick, onRender: (item: any): React.ReactNode => ( - <div className='durationsty'> + <div className='durationsty succeed-padding'> <div>{convertDuration(item.duration)}</div> </div> ) @@ -105,29 +111,38 @@ class SuccessTable extends React.Component<SuccessTableProps, SuccessTableState> { name: 'Status', key: 'status', - minWidth: 140, - maxWidth: 210, + minWidth: 80, + maxWidth: 150, isResizable: true, fieldName: 'status', - onRender: (item: any): React.ReactNode => { - return <div className={`${item.status} commonStyle`}>{item.status}</div>; - } + onRender: (item: any): React.ReactNode => ( + <div className={`${item.status} commonStyle succeed-padding`}>{item.status}</div> + ) }, { name: 'Default metric', key: 'accuracy', fieldName: 'accuracy', - minWidth: 120, - maxWidth: 360, + minWidth: 100, + maxWidth: 160, isResizable: true, data: 'number', onColumnClick: this.onColumnClick, - onRender: (item: any): React.ReactNode => { - return <DefaultMetric trialId={item.id} />; - } + onRender: (item: any): React.ReactNode => <DefaultMetric trialId={item.id} /> } ]; + setInnerWidth = (): void => { + this.setState(() => ({ innerWidth: window.innerWidth })); + }; + + componentDidMount(): void { + window.addEventListener('resize', this.setInnerWidth); + } + componentWillUnmount(): void { + window.removeEventListener('resize', this.setInnerWidth); + } + componentDidUpdate(prevProps: SuccessTableProps): void { if (this.props.trialIds !== prevProps.trialIds) { const { trialIds } = this.props; diff --git a/src/webui/src/components/public-child/DefaultMetric.tsx b/ts/webui/src/components/public-child/DefaultMetric.tsx similarity index 80% rename from src/webui/src/components/public-child/DefaultMetric.tsx rename to ts/webui/src/components/public-child/DefaultMetric.tsx index 388a7f1d18..bbd81c901a 100644 --- a/src/webui/src/components/public-child/DefaultMetric.tsx +++ b/ts/webui/src/components/public-child/DefaultMetric.tsx @@ -13,7 +13,7 @@ class DefaultMetric extends React.Component<DefaultMetricProps, {}> { render(): React.ReactNode { const accuracy = TRIALS.getTrial(this.props.trialId).accuracy; - return <div>{accuracy !== undefined ? formatAccuracy(accuracy) : '--'}</div>; + return <div className='succeed-padding'>{accuracy !== undefined ? formatAccuracy(accuracy) : '--'}</div>; } } diff --git a/ts/webui/src/components/public-child/ExpandableDetails.tsx b/ts/webui/src/components/public-child/ExpandableDetails.tsx new file mode 100644 index 0000000000..34f010f215 --- /dev/null +++ b/ts/webui/src/components/public-child/ExpandableDetails.tsx @@ -0,0 +1,22 @@ +import * as React from 'react'; +import { DetailsRow, IDetailsRowBaseProps } from '@fluentui/react'; +import OpenRow from '../public-child/OpenRow'; + +interface ExpandableDetailsProps { + detailsProps: IDetailsRowBaseProps; + isExpand: boolean; +} + +class ExpandableDetails extends React.Component<ExpandableDetailsProps, {}> { + render(): React.ReactNode { + const { detailsProps, isExpand } = this.props; + return ( + <div> + <DetailsRow {...detailsProps} /> + {isExpand && <OpenRow trialId={detailsProps.item.id} />} + </div> + ); + } +} + +export default ExpandableDetails; diff --git a/src/webui/src/components/public-child/IntermediateVal.tsx b/ts/webui/src/components/public-child/IntermediateVal.tsx similarity index 100% rename from src/webui/src/components/public-child/IntermediateVal.tsx rename to ts/webui/src/components/public-child/IntermediateVal.tsx diff --git a/src/webui/src/components/public-child/LogPathChild.tsx b/ts/webui/src/components/public-child/LogPathChild.tsx similarity index 100% rename from src/webui/src/components/public-child/LogPathChild.tsx rename to ts/webui/src/components/public-child/LogPathChild.tsx diff --git a/src/webui/src/components/public-child/MonacoEditor.tsx b/ts/webui/src/components/public-child/MonacoEditor.tsx similarity index 100% rename from src/webui/src/components/public-child/MonacoEditor.tsx rename to ts/webui/src/components/public-child/MonacoEditor.tsx diff --git a/src/webui/src/components/public-child/OpenRow.tsx b/ts/webui/src/components/public-child/OpenRow.tsx similarity index 99% rename from src/webui/src/components/public-child/OpenRow.tsx rename to ts/webui/src/components/public-child/OpenRow.tsx index b54b24aa5d..459ae54801 100644 --- a/src/webui/src/components/public-child/OpenRow.tsx +++ b/ts/webui/src/components/public-child/OpenRow.tsx @@ -8,7 +8,7 @@ import JSONTree from 'react-json-tree'; import PaiTrialLog from '../public-child/PaiTrialLog'; import TrialLog from '../public-child/TrialLog'; import MessageInfo from '../modals/MessageInfo'; -import '../../static/style/overview.scss'; +import '../../static/style/overview/overview.scss'; import '../../static/style/copyParameter.scss'; import '../../static/style/openRow.scss'; diff --git a/ts/webui/src/components/public-child/PaginationTable.tsx b/ts/webui/src/components/public-child/PaginationTable.tsx new file mode 100644 index 0000000000..9379968e95 --- /dev/null +++ b/ts/webui/src/components/public-child/PaginationTable.tsx @@ -0,0 +1,120 @@ +import * as React from 'react'; +import { DetailsList, Dropdown, Icon, IDetailsListProps, IDropdownOption, IStackTokens, Stack } from '@fluentui/react'; +import ReactPaginate from 'react-paginate'; + +interface PaginationTableState { + itemsPerPage: number; + currentPage: number; + itemsOnPage: any[]; // this needs to be stored in state to prevent re-rendering +} + +const horizontalGapStackTokens: IStackTokens = { + childrenGap: 20, + padding: 10 +}; + +function _currentTableOffset(perPage: number, currentPage: number, source: any[]): number { + return perPage === -1 ? 0 : Math.min(currentPage, Math.floor((source.length - 1) / perPage)) * perPage; +} + +function _obtainPaginationSlice(perPage: number, currentPage: number, source: any[]): any[] { + if (perPage === -1) { + return source; + } else { + const offset = _currentTableOffset(perPage, currentPage, source); + return source.slice(offset, offset + perPage); + } +} + +class PaginationTable extends React.PureComponent<IDetailsListProps, PaginationTableState> { + constructor(props: IDetailsListProps) { + super(props); + this.state = { + itemsPerPage: 20, + currentPage: 0, + itemsOnPage: [] + }; + } + + private _onItemsPerPageSelect(event: React.FormEvent<HTMLDivElement>, item: IDropdownOption | undefined): void { + if (item !== undefined) { + const { items } = this.props; + // use current offset to calculate the next `current_page` + const currentOffset = _currentTableOffset(this.state.itemsPerPage, this.state.currentPage, items); + const itemsPerPage = item.key as number; + const currentPage = Math.floor(currentOffset / itemsPerPage); + this.setState({ + itemsPerPage: itemsPerPage, + currentPage: currentPage, + itemsOnPage: _obtainPaginationSlice(itemsPerPage, currentPage, this.props.items) + }); + } + } + + private _onPageSelect(event: any): void { + const currentPage = event.selected; + this.setState({ + currentPage: currentPage, + itemsOnPage: _obtainPaginationSlice(this.state.itemsPerPage, currentPage, this.props.items) + }); + } + + componentDidUpdate(prevProps: IDetailsListProps): void { + if (prevProps.items !== this.props.items) { + this.setState({ + itemsOnPage: _obtainPaginationSlice(this.state.itemsPerPage, this.state.currentPage, this.props.items) + }); + } + } + + render(): React.ReactNode { + const { itemsPerPage, itemsOnPage } = this.state; + const detailListProps = { + ...this.props, + items: itemsOnPage + }; + const itemsCount = this.props.items.length; + const pageCount = itemsPerPage === -1 ? 1 : Math.ceil(itemsCount / itemsPerPage); + const perPageOptions = [ + { key: 10, text: '10 items per page' }, + { key: 20, text: '20 items per page' }, + { key: 50, text: '50 items per page' }, + { key: -1, text: 'All items' } + ]; + return ( + <div> + <DetailsList {...detailListProps} /> + <Stack + horizontal + horizontalAlign='end' + verticalAlign='baseline' + styles={{ root: { padding: 10 } }} + tokens={horizontalGapStackTokens} + > + <Dropdown + selectedKey={itemsPerPage} + options={perPageOptions} + onChange={this._onItemsPerPageSelect.bind(this)} + styles={{ dropdown: { width: 150 } }} + /> + <ReactPaginate + previousLabel={<Icon aria-hidden={true} iconName='ChevronLeft' />} + nextLabel={<Icon aria-hidden={true} iconName='ChevronRight' />} + breakLabel={'...'} + breakClassName={'break'} + pageCount={pageCount} + marginPagesDisplayed={2} + pageRangeDisplayed={2} + onPageChange={this._onPageSelect.bind(this)} + containerClassName={itemsCount === 0 ? 'pagination hidden' : 'pagination'} + subContainerClassName={'pages pagination'} + disableInitialCallback={false} + activeClassName={'active'} + /> + </Stack> + </div> + ); + } +} + +export default PaginationTable; diff --git a/src/webui/src/components/public-child/PaiTrialChild.tsx b/ts/webui/src/components/public-child/PaiTrialChild.tsx similarity index 100% rename from src/webui/src/components/public-child/PaiTrialChild.tsx rename to ts/webui/src/components/public-child/PaiTrialChild.tsx diff --git a/src/webui/src/components/public-child/PaiTrialLog.tsx b/ts/webui/src/components/public-child/PaiTrialLog.tsx similarity index 100% rename from src/webui/src/components/public-child/PaiTrialLog.tsx rename to ts/webui/src/components/public-child/PaiTrialLog.tsx diff --git a/src/webui/src/components/public-child/TrialLog.tsx b/ts/webui/src/components/public-child/TrialLog.tsx similarity index 100% rename from src/webui/src/components/public-child/TrialLog.tsx rename to ts/webui/src/components/public-child/TrialLog.tsx diff --git a/ts/webui/src/components/public-child/config/TrialConfigButton.tsx b/ts/webui/src/components/public-child/config/TrialConfigButton.tsx new file mode 100644 index 0000000000..3c5742db1f --- /dev/null +++ b/ts/webui/src/components/public-child/config/TrialConfigButton.tsx @@ -0,0 +1,27 @@ +import React, { useState, useCallback } from 'react'; +import { DefaultButton, Stack } from '@fluentui/react'; +import TrialConfigPanel from './TrialConfigPanel'; +import '../../../static/style/overview/config.scss'; + +export const TrialConfigButton = (): any => { + const [isShowConfigPanel, setShowConfigPanle] = useState(false); + const [activeTab, setActiveTab] = useState('1'); + const hideConfigPanel = useCallback(() => setShowConfigPanle(false), []); + const showTrialConfigpPanel = useCallback(() => { + setShowConfigPanle(true); + setActiveTab('config'); + }, []); + const showSearchSpacePanel = useCallback(() => { + setShowConfigPanle(true); + setActiveTab('search space'); + }, []); + return ( + <React.Fragment> + <Stack className='config'> + <DefaultButton text='Search space' onClick={showSearchSpacePanel} /> + <DefaultButton text='Config' onClick={showTrialConfigpPanel} /> + </Stack> + {isShowConfigPanel && <TrialConfigPanel hideConfigPanel={hideConfigPanel} activeTab={activeTab} />} + </React.Fragment> + ); +}; diff --git a/ts/webui/src/components/public-child/config/TrialConfigPanel.tsx b/ts/webui/src/components/public-child/config/TrialConfigPanel.tsx new file mode 100644 index 0000000000..24ac550ba4 --- /dev/null +++ b/ts/webui/src/components/public-child/config/TrialConfigPanel.tsx @@ -0,0 +1,124 @@ +import * as React from 'react'; +import { Stack, Panel, Pivot, PivotItem, PrimaryButton } from '@fluentui/react'; +import { EXPERIMENT } from '../../../static/datamodel'; +import MonacoEditor from 'react-monaco-editor'; +import { MONACO } from '../../../static/const'; +import { AppContext } from '../../../App'; +import { convertDuration, convertTimeAsUnit } from '../../../static/function'; +import { prettyStringify } from '../../../static/json_util'; +import lodash from 'lodash'; +import '../../../static/style/logDrawer.scss'; + +interface LogDrawerProps { + hideConfigPanel: () => void; + activeTab?: string; +} + +interface LogDrawerState { + panelInnerHeight: number; + innerWidth: number; +} + +class TrialConfigPanel extends React.Component<LogDrawerProps, LogDrawerState> { + constructor(props: LogDrawerProps) { + super(props); + + this.state = { + panelInnerHeight: window.innerHeight, + innerWidth: window.innerWidth + }; + } + + // use arrow function for change window size met error: this.setState is not a function + setLogDrawerHeight = (): void => { + this.setState(() => ({ panelInnerHeight: window.innerHeight, innerWidth: window.innerWidth })); + }; + + async componentDidMount(): Promise<void> { + window.addEventListener('resize', this.setLogDrawerHeight); + } + + componentWillUnmount(): void { + window.removeEventListener('resize', this.setLogDrawerHeight); + } + + render(): React.ReactNode { + const { hideConfigPanel, activeTab } = this.props; + const { panelInnerHeight, innerWidth } = this.state; + // [marginTop 16px] + [Search space 46px] + + // button[height: 32px, marginTop: 45px, marginBottom: 25px] + [padding-bottom: 20px] + const monacoEditorHeight = panelInnerHeight - 184; + const blacklist = [ + 'id', + 'logDir', + 'startTime', + 'endTime', + 'experimentName', + 'searchSpace', + 'trainingServicePlatform' + ]; + const filter = (key: string, val: any): any => { + return blacklist.includes(key) ? undefined : val; + }; + const profile = lodash.cloneDeep(EXPERIMENT.profile); + profile.execDuration = convertDuration(profile.execDuration); + + const prettyWidth = innerWidth > 1400 ? 100 : 60; + + return ( + <AppContext.Consumer> + {(value): React.ReactNode => { + const unit = value.maxDurationUnit; + profile.params.maxExecDuration = `${convertTimeAsUnit( + unit, + profile.params.maxExecDuration + )}${unit}`; + const showProfile = JSON.stringify(profile, filter, 2); + return ( + <Stack> + <Panel + isOpen={true} + hasCloseButton={false} + isFooterAtBottom={true} + isLightDismiss={true} + onLightDismissClick={hideConfigPanel} + > + <div className='log-tab-body'> + <Pivot + initialSelectedKey={activeTab} + style={{ minHeight: 190, paddingTop: '16px' }} + > + <PivotItem headerText='Search space' itemKey='search space'> + <MonacoEditor + height={monacoEditorHeight} + language='json' + theme='vs-light' + value={prettyStringify(EXPERIMENT.searchSpace, prettyWidth, 2)} + options={MONACO} + /> + </PivotItem> + <PivotItem headerText='Config' itemKey='config'> + <div className='profile'> + <MonacoEditor + width='100%' + height={monacoEditorHeight} + language='json' + theme='vs-light' + value={showProfile} + options={MONACO} + /> + </div> + </PivotItem> + </Pivot> + </div> + <PrimaryButton text='Close' className='configClose' onClick={hideConfigPanel} /> + </Panel> + </Stack> + ); + }} + </AppContext.Consumer> + ); + } +} + +export default TrialConfigPanel; diff --git a/src/webui/src/components/stateless-component/NNItabs.tsx b/ts/webui/src/components/stateless-component/NNItabs.tsx similarity index 82% rename from src/webui/src/components/stateless-component/NNItabs.tsx rename to ts/webui/src/components/stateless-component/NNItabs.tsx index a733da4fdd..36156f35e6 100644 --- a/src/webui/src/components/stateless-component/NNItabs.tsx +++ b/ts/webui/src/components/stateless-component/NNItabs.tsx @@ -15,7 +15,7 @@ const DETAILTABS = ( const NNILOGO = ( <NavLink to={'/oview'}> - <img src={require('../../static/img/logo2.png')} alt='NNI logo' style={{ height: 40 }} /> + <img src={require('../../static/img/logo.png')} alt='NNI logo' style={{ height: 40 }} /> </NavLink> ); diff --git a/src/webui/src/components/trial-detail/DefaultMetricPoint.tsx b/ts/webui/src/components/trial-detail/DefaultMetricPoint.tsx similarity index 100% rename from src/webui/src/components/trial-detail/DefaultMetricPoint.tsx rename to ts/webui/src/components/trial-detail/DefaultMetricPoint.tsx diff --git a/src/webui/src/components/trial-detail/Duration.tsx b/ts/webui/src/components/trial-detail/Duration.tsx similarity index 100% rename from src/webui/src/components/trial-detail/Duration.tsx rename to ts/webui/src/components/trial-detail/Duration.tsx diff --git a/src/webui/src/components/trial-detail/Intermediate.tsx b/ts/webui/src/components/trial-detail/Intermediate.tsx similarity index 100% rename from src/webui/src/components/trial-detail/Intermediate.tsx rename to ts/webui/src/components/trial-detail/Intermediate.tsx diff --git a/src/webui/src/components/trial-detail/Para.tsx b/ts/webui/src/components/trial-detail/Para.tsx similarity index 81% rename from src/webui/src/components/trial-detail/Para.tsx rename to ts/webui/src/components/trial-detail/Para.tsx index 31f33207b7..cc3d3a8e9b 100644 --- a/src/webui/src/components/trial-detail/Para.tsx +++ b/ts/webui/src/components/trial-detail/Para.tsx @@ -1,5 +1,5 @@ import * as d3 from 'd3'; -import { Dropdown, IDropdownOption, Stack } from '@fluentui/react'; +import { Dropdown, IDropdownOption, Stack, DefaultButton } from '@fluentui/react'; import ParCoords from 'parcoord-es'; import 'parcoord-es/dist/parcoords.css'; import * as React from 'react'; @@ -9,12 +9,16 @@ import { filterByStatus } from '../../static/function'; import { TableObj, SingleAxis, MultipleAxes } from '../../static/interface'; import '../../static/style/button.scss'; import '../../static/style/para.scss'; +import ChangeColumnComponent from '../modals/ChangeColumnComponent'; interface ParaState { dimName: string[]; selectedPercent: string; primaryMetricKey: string; noChart: boolean; + customizeColumnsDialogVisible: boolean; + availableDimensions: string[]; + chosenDimensions: string[]; } interface ParaProps { @@ -45,7 +49,10 @@ class Para extends React.Component<ParaProps, ParaState> { dimName: [], primaryMetricKey: 'default', selectedPercent: '1', - noChart: true + noChart: true, + customizeColumnsDialogVisible: false, + availableDimensions: [], + chosenDimensions: [] }; } @@ -82,11 +89,24 @@ class Para extends React.Component<ParaProps, ParaState> { } render(): React.ReactNode { - const { selectedPercent, noChart } = this.state; + const { + selectedPercent, + noChart, + customizeColumnsDialogVisible, + availableDimensions, + chosenDimensions + } = this.state; return ( <div className='parameter'> <Stack horizontal className='para-filter' horizontalAlign='end'> + <DefaultButton + text='Add/Remove axes' + onClick={(): void => { + this.setState({ customizeColumnsDialogVisible: true }); + }} + styles={{ root: { marginRight: 10 } }} + /> <Dropdown selectedKey={selectedPercent} onChange={this.percentNum} @@ -101,6 +121,21 @@ class Para extends React.Component<ParaProps, ParaState> { /> {this.finalKeysDropdown()} </Stack> + {customizeColumnsDialogVisible && availableDimensions.length > 0 && ( + <ChangeColumnComponent + selectedColumns={chosenDimensions} + allColumns={availableDimensions.map(dim => ({ key: dim, name: dim }))} + onSelectedChange={(selected: string[]): void => { + this.setState({ chosenDimensions: selected }, () => { + this.renderParallelCoordinates(); + }); + }} + onHideDialog={(): void => { + this.setState({ customizeColumnsDialogVisible: false }); + }} + minSelected={2} + /> + )} <div className='parcoords' style={this.chartMulineStyle} ref={this.paraRef} /> {noChart && <div className='nodata'>No data</div>} </div> @@ -143,13 +178,13 @@ class Para extends React.Component<ParaProps, ParaState> { private renderParallelCoordinates(): void { const { searchSpace } = this.props; const percent = parseFloat(this.state.selectedPercent); - const { primaryMetricKey } = this.state; + const { primaryMetricKey, chosenDimensions } = this.state; const inferredSearchSpace = TRIALS.inferredSearchSpace(searchSpace); const inferredMetricSpace = TRIALS.inferredMetricSpace(); let convertedTrials = this.getTrialsAsObjectList(inferredSearchSpace, inferredMetricSpace); - const dimensions: [any, any][] = []; + const dimensions: [string, any][] = []; let colorDim: string | undefined = undefined, colorScale: any = undefined; // treat every axis as numeric to fit for brush @@ -213,7 +248,11 @@ class Para extends React.Component<ParaProps, ParaState> { } this.pcs .data(convertedTrials) - .dimensions(dimensions.reduce((obj, entry) => ({ ...obj, [entry[0]]: entry[1] }), {})); + .dimensions( + dimensions + .filter(([d, _]) => chosenDimensions.length === 0 || chosenDimensions.includes(d)) + .reduce((obj, entry) => ({ ...obj, [entry[0]]: entry[1] }), {}) + ); if (firstRun) { this.pcs .margin(this.innerChartMargins) @@ -230,6 +269,12 @@ class Para extends React.Component<ParaProps, ParaState> { if (firstRun) { this.setState({ noChart: false }); } + + // set new available dims + this.setState({ + availableDimensions: dimensions.map(e => e[0]), + chosenDimensions: chosenDimensions.length === 0 ? dimensions.map(e => e[0]) : chosenDimensions + }); } private getTrialsAsObjectList(inferredSearchSpace: MultipleAxes, inferredMetricSpace: MultipleAxes): {}[] { diff --git a/ts/webui/src/components/trial-detail/TableList.tsx b/ts/webui/src/components/trial-detail/TableList.tsx new file mode 100644 index 0000000000..6752f78b3a --- /dev/null +++ b/ts/webui/src/components/trial-detail/TableList.tsx @@ -0,0 +1,589 @@ +import { + DefaultButton, + Dropdown, + IColumn, + Icon, + IDropdownOption, + PrimaryButton, + Selection, + SelectionMode, + Stack, + StackItem, + TooltipHost, + DirectionalHint +} from '@fluentui/react'; +import React from 'react'; +import { EXPERIMENT, TRIALS } from '../../static/datamodel'; +import { TOOLTIP_BACKGROUND_COLOR } from '../../static/const'; +import { convertDuration, formatTimestamp } from '../../static/function'; +import { TableObj } from '../../static/interface'; +import '../../static/style/search.scss'; +import '../../static/style/tableStatus.css'; +import '../../static/style/logPath.scss'; +import '../../static/style/table.scss'; +import '../../static/style/button.scss'; +import '../../static/style/logPath.scss'; +import '../../static/style/openRow.scss'; +import '../../static/style/pagination.scss'; +import '../../static/style/search.scss'; +import '../../static/style/table.scss'; +import '../../static/style/tableStatus.css'; +import '../../static/style/overview/overviewTitle.scss'; +import { blocked, copy, LineChart, tableListIcon } from '../buttons/Icon'; +import ChangeColumnComponent from '../modals/ChangeColumnComponent'; +import Compare from '../modals/Compare'; +import Customize from '../modals/CustomizedTrial'; +import KillJob from '../modals/Killjob'; +import ExpandableDetails from '../public-child/ExpandableDetails'; +import PaginationTable from '../public-child/PaginationTable'; +import { Trial } from '../../static/model/trial'; + +const echarts = require('echarts/lib/echarts'); +require('echarts/lib/chart/line'); +require('echarts/lib/component/tooltip'); +require('echarts/lib/component/title'); +echarts.registerTheme('my_theme', { + color: '#3c8dbc' +}); + +type SearchOptionType = 'id' | 'trialnum' | 'status' | 'parameters'; +const searchOptionLiterals = { + id: 'ID', + trialnum: 'Trial No.', + status: 'Status', + parameters: 'Parameters' +}; + +const defaultDisplayedColumns = ['sequenceId', 'id', 'duration', 'status', 'latestAccuracy']; + +interface SortInfo { + field: string; + isDescend?: boolean; +} + +function _copyAndSort<T>(items: T[], columnKey: string, isSortedDescending?: boolean): any { + const key = columnKey as keyof T; + return items.slice(0).sort(function(a: T, b: T): any { + if ( + a[key] === undefined || + Object.is(a[key], NaN) || + Object.is(a[key], Infinity) || + Object.is(a[key], -Infinity) || + typeof a[key] === 'object' + ) { + return 1; + } + if ( + b[key] === undefined || + Object.is(b[key], NaN) || + Object.is(b[key], Infinity) || + Object.is(b[key], -Infinity) || + typeof b[key] === 'object' + ) { + return -1; + } + return (isSortedDescending ? a[key] < b[key] : a[key] > b[key]) ? 1 : -1; + }); +} + +function _inferColumnTitle(columnKey: string): string { + if (columnKey === 'sequenceId') { + return 'Trial No.'; + } else if (columnKey === 'id') { + return 'ID'; + } else if (columnKey === 'intermediateCount') { + return 'Intermediate results (#)'; + } else if (columnKey.startsWith('space/')) { + return columnKey.split('/', 2)[1] + ' (space)'; + } else if (columnKey === 'latestAccuracy') { + return 'Default metric'; // to align with the original design + } else if (columnKey.startsWith('metric/')) { + return columnKey.split('/', 2)[1] + ' (metric)'; + } else if (columnKey.startsWith('_')) { + return columnKey; + } else { + // camel case to verbose form + const withSpace = columnKey.replace(/[A-Z]/g, letter => ` ${letter.toLowerCase()}`); + return withSpace.charAt(0).toUpperCase() + withSpace.slice(1); + } +} + +interface TableListProps { + tableSource: TableObj[]; + trialsUpdateBroadcast: number; +} + +interface TableListState { + displayedItems: any[]; + displayedColumns: string[]; + columns: IColumn[]; + searchType: SearchOptionType; + searchText: string; + selectedRowIds: string[]; + customizeColumnsDialogVisible: boolean; + compareDialogVisible: boolean; + intermediateDialogTrial: TableObj | undefined; + copiedTrialId: string | undefined; + sortInfo: SortInfo; +} + +class TableList extends React.Component<TableListProps, TableListState> { + private _selection: Selection; + private _expandedTrialIds: Set<string>; + + constructor(props: TableListProps) { + super(props); + + this.state = { + displayedItems: [], + displayedColumns: defaultDisplayedColumns, + columns: [], + searchType: 'id', + searchText: '', + customizeColumnsDialogVisible: false, + compareDialogVisible: false, + selectedRowIds: [], + intermediateDialogTrial: undefined, + copiedTrialId: undefined, + sortInfo: { field: '', isDescend: true } + }; + + this._selection = new Selection({ + onSelectionChanged: (): void => { + this.setState({ + selectedRowIds: this._selection.getSelection().map(s => (s as any).id) + }); + } + }); + + this._expandedTrialIds = new Set<string>(); + } + + /* Search related methods */ + + // This functions as the filter for the final trials displayed in the current table + private _filterTrials(trials: TableObj[]): TableObj[] { + const { searchText, searchType } = this.state; + // search a trial by Trial No. | Trial ID | Parameters | Status + let searchFilter = (_: TableObj): boolean => true; // eslint-disable-line no-unused-vars + if (searchText.trim()) { + if (searchType === 'id') { + searchFilter = (trial): boolean => trial.id.toUpperCase().includes(searchText.toUpperCase()); + } else if (searchType === 'trialnum') { + searchFilter = (trial): boolean => trial.sequenceId.toString() === searchText; + } else if (searchType === 'status') { + searchFilter = (trial): boolean => trial.status.toUpperCase().includes(searchText.toUpperCase()); + } else if (searchType === 'parameters') { + // TODO: support filters like `x: 2` (instead of `'x': 2`) + searchFilter = (trial): boolean => JSON.stringify(trial.description.parameters).includes(searchText); + } + } + return trials.filter(searchFilter); + } + + private _updateSearchFilterType(_event: React.FormEvent<HTMLDivElement>, item: IDropdownOption | undefined): void { + if (item !== undefined) { + const value = item.key.toString(); + if (searchOptionLiterals.hasOwnProperty(value)) { + this.setState({ searchType: value as SearchOptionType }, this._updateTableSource); + } + } + } + + private _updateSearchText(ev: React.ChangeEvent<HTMLInputElement>): void { + this.setState({ searchText: ev.target.value }, this._updateTableSource); + } + + /* Table basic function related methods */ + + private _onColumnClick(ev: React.MouseEvent<HTMLElement>, column: IColumn): void { + // handle the click events on table header (do sorting) + const { columns } = this.state; + const newColumns: IColumn[] = columns.slice(); + const currColumn: IColumn = newColumns.filter(currCol => column.key === currCol.key)[0]; + const isSortedDescending = !currColumn.isSortedDescending; + this.setState( + { + sortInfo: { field: column.key, isDescend: isSortedDescending } + }, + this._updateTableSource + ); + } + + private _trialsToTableItems(trials: TableObj[]): any[] { + // TODO: use search space and metrics space from TRIALS will cause update issues. + const searchSpace = TRIALS.inferredSearchSpace(EXPERIMENT.searchSpaceNew); + const metricSpace = TRIALS.inferredMetricSpace(); + const items = trials.map(trial => { + const ret = { + sequenceId: trial.sequenceId, + id: trial.id, + startTime: (trial as Trial).info.startTime, // FIXME: why do we need info here? + endTime: (trial as Trial).info.endTime, + duration: trial.duration, + status: trial.status, + intermediateCount: trial.intermediates.length, + _expandDetails: this._expandedTrialIds.has(trial.id) // hidden field names should start with `_` + }; + for (const [k, v] of trial.parameters(searchSpace)) { + ret[`space/${k.baseName}`] = v; + } + for (const [k, v] of trial.metrics(metricSpace)) { + ret[`metric/${k.baseName}`] = v; + } + ret['latestAccuracy'] = (trial as Trial).latestAccuracy; + ret['_formattedLatestAccuracy'] = (trial as Trial).formatLatestAccuracy(); + return ret; + }); + + const { sortInfo } = this.state; + if (sortInfo.field !== '') { + return _copyAndSort(items, sortInfo.field, sortInfo.isDescend); + } else { + return items; + } + } + + private _buildColumnsFromTableItems(tableItems: any[]): IColumn[] { + // extra column, for a icon to expand the trial details panel + const columns: IColumn[] = [ + { + key: '_expand', + name: '', + onRender: (item): any => { + return ( + <Icon + aria-hidden={true} + iconName='ChevronRight' + styles={{ + root: { + transition: 'all 0.2s', + transform: `rotate(${item._expandDetails ? 90 : 0}deg)` + } + }} + onClick={(event): void => { + event.stopPropagation(); + const newItem: any = { ...item, _expandDetails: !item._expandDetails }; + if (newItem._expandDetails) { + // preserve to be restored when refreshed + this._expandedTrialIds.add(newItem.id); + } else { + this._expandedTrialIds.delete(newItem.id); + } + const newItems = this.state.displayedItems.map(item => + item.id === newItem.id ? newItem : item + ); + this.setState({ + displayedItems: newItems + }); + }} + onMouseDown={(e): void => { + e.stopPropagation(); + }} + onMouseUp={(e): void => { + e.stopPropagation(); + }} + /> + ); + }, + fieldName: 'expand', + isResizable: false, + minWidth: 20, + maxWidth: 20 + } + ]; + // looking at the first row only for now + for (const k of Object.keys(tableItems[0])) { + if (k === 'metric/default') { + // FIXME: default metric is hacked as latestAccuracy currently + continue; + } + const columnTitle = _inferColumnTitle(k); + // TODO: add blacklist + // 0.85: tableWidth / screen + const widths = window.innerWidth * 0.85; + columns.push({ + name: columnTitle, + key: k, + fieldName: k, + minWidth: widths * 0.12, + maxWidth: widths * 0.19, + isResizable: true, + onColumnClick: this._onColumnClick.bind(this), + ...(k === 'status' && { + // color status + onRender: (record): React.ReactNode => ( + <span className={`${record.status} commonStyle`}>{record.status}</span> + ) + }), + ...((k.startsWith('metric/') || k.startsWith('space/')) && { + // show tooltip + onRender: (record): React.ReactNode => ( + <TooltipHost + content={record[k]} + directionalHint={DirectionalHint.bottomCenter} + tooltipProps={{ + calloutProps: { + styles: { + beak: { background: TOOLTIP_BACKGROUND_COLOR }, + beakCurtain: { background: TOOLTIP_BACKGROUND_COLOR }, + calloutMain: { background: TOOLTIP_BACKGROUND_COLOR } + } + } + }} + > + <div className='ellipsis'>{record[k]}</div> + </TooltipHost> + ) + }), + ...(k === 'latestAccuracy' && { + // FIXME: this is ad-hoc + onRender: (record): React.ReactNode => ( + <TooltipHost + content={record._formattedLatestAccuracy} + directionalHint={DirectionalHint.bottomCenter} + tooltipProps={{ + calloutProps: { + styles: { + beak: { background: TOOLTIP_BACKGROUND_COLOR }, + beakCurtain: { background: TOOLTIP_BACKGROUND_COLOR }, + calloutMain: { background: TOOLTIP_BACKGROUND_COLOR } + } + } + }} + > + <div className='ellipsis'>{record._formattedLatestAccuracy}</div> + </TooltipHost> + ) + }), + ...(['startTime', 'endTime'].includes(k) && { + onRender: (record): React.ReactNode => <span>{formatTimestamp(record[k], '--')}</span> + }), + ...(k === 'duration' && { + onRender: (record): React.ReactNode => ( + <span className='durationsty'>{convertDuration(record[k])}</span> + ) + }) + }); + } + // operations column + columns.push({ + name: 'Operation', + key: '_operation', + fieldName: 'operation', + minWidth: 150, + maxWidth: 160, + isResizable: true, + className: 'detail-table', + onRender: this._renderOperationColumn.bind(this) + }); + + const { sortInfo } = this.state; + for (const column of columns) { + if (column.key === sortInfo.field) { + column.isSorted = true; + column.isSortedDescending = sortInfo.isDescend; + } else { + column.isSorted = false; + column.isSortedDescending = true; + } + } + return columns; + } + + private _updateTableSource(): void { + // call this method when trials or the computation of trial filter has changed + const items = this._trialsToTableItems(this._filterTrials(this.props.tableSource)); + if (items.length > 0) { + const columns = this._buildColumnsFromTableItems(items); + this.setState({ + displayedItems: items, + columns: columns + }); + } else { + this.setState({ + displayedItems: [], + columns: [] + }); + } + } + + private _updateDisplayedColumns(displayedColumns: string[]): void { + this.setState({ + displayedColumns: displayedColumns + }); + } + + private _renderOperationColumn(record: any): React.ReactNode { + const runningTrial: boolean = ['RUNNING', 'UNKNOWN'].includes(record.status) ? false : true; + const disabledAddCustomizedTrial = ['DONE', 'ERROR', 'STOPPED'].includes(EXPERIMENT.status); + return ( + <Stack className='detail-button' horizontal> + <PrimaryButton + className='detail-button-operation' + title='Intermediate' + onClick={(): void => { + const { tableSource } = this.props; + const trial = tableSource.find(trial => trial.id === record.id) as TableObj; + this.setState({ intermediateDialogTrial: trial }); + }} + > + {LineChart} + </PrimaryButton> + {runningTrial ? ( + <PrimaryButton className='detail-button-operation' disabled={true} title='kill'> + {blocked} + </PrimaryButton> + ) : ( + <KillJob trial={record} /> + )} + <PrimaryButton + className='detail-button-operation' + title='Customized trial' + onClick={(): void => { + this.setState({ copiedTrialId: record.id }); + }} + disabled={disabledAddCustomizedTrial} + > + {copy} + </PrimaryButton> + </Stack> + ); + } + + componentDidUpdate(prevProps: TableListProps): void { + if (this.props.tableSource !== prevProps.tableSource) { + this._updateTableSource(); + } + } + + componentDidMount(): void { + this._updateTableSource(); + } + + render(): React.ReactNode { + const { + displayedItems, + columns, + searchType, + customizeColumnsDialogVisible, + compareDialogVisible, + displayedColumns, + selectedRowIds, + intermediateDialogTrial, + copiedTrialId + } = this.state; + + return ( + <div id='tableList'> + <Stack horizontal className='panelTitle' style={{ marginTop: 10 }}> + <span style={{ marginRight: 12 }}>{tableListIcon}</span> + <span>Trial jobs</span> + </Stack> + <Stack horizontal className='allList'> + <StackItem grow={50}> + <DefaultButton + text='Compare' + className='allList-compare' + onClick={(): void => { + this.setState({ compareDialogVisible: true }); + }} + disabled={selectedRowIds.length === 0} + /> + </StackItem> + <StackItem grow={50}> + <Stack horizontal horizontalAlign='end' className='allList'> + <DefaultButton + className='allList-button-gap' + text='Add/Remove columns' + onClick={(): void => { + this.setState({ customizeColumnsDialogVisible: true }); + }} + /> + <Dropdown + selectedKey={searchType} + options={Object.entries(searchOptionLiterals).map(([k, v]) => ({ + key: k, + text: v + }))} + onChange={this._updateSearchFilterType.bind(this)} + styles={{ root: { width: 150 } }} + /> + <input + type='text' + className='allList-search-input' + placeholder={`Search by ${ + ['id', 'trialnum'].includes(searchType) + ? searchOptionLiterals[searchType] + : searchType + }`} + onChange={this._updateSearchText.bind(this)} + style={{ width: 230 }} + /> + </Stack> + </StackItem> + </Stack> + {columns && displayedItems && ( + <PaginationTable + columns={columns.filter( + column => + displayedColumns.includes(column.key) || ['_expand', '_operation'].includes(column.key) + )} + items={displayedItems} + compact={true} + selection={this._selection} + selectionMode={SelectionMode.multiple} + selectionPreservedOnEmptyClick={true} + onRenderRow={(props): any => { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return <ExpandableDetails detailsProps={props!} isExpand={props!.item._expandDetails} />; + }} + /> + )} + {compareDialogVisible && ( + <Compare + title='Compare trials' + showDetails={true} + trials={this.props.tableSource.filter(trial => selectedRowIds.includes(trial.id))} + onHideDialog={(): void => { + this.setState({ compareDialogVisible: false }); + }} + /> + )} + {intermediateDialogTrial !== undefined && ( + <Compare + title='Intermediate results' + showDetails={false} + trials={[intermediateDialogTrial]} + onHideDialog={(): void => { + this.setState({ intermediateDialogTrial: undefined }); + }} + /> + )} + {customizeColumnsDialogVisible && ( + <ChangeColumnComponent + selectedColumns={displayedColumns} + allColumns={columns + .filter(column => !column.key.startsWith('_')) + .map(column => ({ key: column.key, name: column.name }))} + onSelectedChange={this._updateDisplayedColumns.bind(this)} + onHideDialog={(): void => { + this.setState({ customizeColumnsDialogVisible: false }); + }} + /> + )} + {/* Clone a trial and customize a set of new parameters */} + {/* visible is done inside because prompt is needed even when the dialog is closed */} + <Customize + visible={copiedTrialId !== undefined} + copyTrialId={copiedTrialId || ''} + closeCustomizeModal={(): void => { + this.setState({ copiedTrialId: undefined }); + }} + /> + </div> + ); + } +} + +export default TableList; diff --git a/src/webui/src/index.css b/ts/webui/src/index.css similarity index 100% rename from src/webui/src/index.css rename to ts/webui/src/index.css diff --git a/src/webui/src/index.tsx b/ts/webui/src/index.tsx similarity index 66% rename from src/webui/src/index.tsx rename to ts/webui/src/index.tsx index 50e4ec0521..af67396b86 100644 --- a/src/webui/src/index.tsx +++ b/ts/webui/src/index.tsx @@ -1,20 +1,27 @@ import React, { lazy, Suspense } from 'react'; import ReactDOM from 'react-dom'; import App from './App'; -import { BrowserRouter as Router, Route, Switch, Redirect } from 'react-router-dom'; +import { BrowserRouter as Router, Route, Switch } from 'react-router-dom'; const Overview = lazy(() => import('./components/Overview')); const TrialsDetail = lazy(() => import('./components/TrialsDetail')); import './index.css'; +import './static/style/loading.scss'; import * as serviceWorker from './serviceWorker'; ReactDOM.render( <Router> <App> <Switch> - <Suspense fallback={null}> + <Suspense + fallback={ + <div className='loading'> + <img src={require('./static/img/loading.gif')} /> + </div> + } + > + <Route path='/' component={Overview} exact /> <Route path='/oview' component={Overview} /> <Route path='/detail' component={TrialsDetail} /> - <Route path='/' render={(): React.ReactNode => <Redirect to={{ pathname: '/oview' }} />} /> </Suspense> </Switch> </App> diff --git a/src/webui/src/logo.svg b/ts/webui/src/logo.svg similarity index 100% rename from src/webui/src/logo.svg rename to ts/webui/src/logo.svg diff --git a/src/webui/src/react-app-env.d.ts b/ts/webui/src/react-app-env.d.ts similarity index 100% rename from src/webui/src/react-app-env.d.ts rename to ts/webui/src/react-app-env.d.ts diff --git a/src/webui/src/serviceWorker.ts b/ts/webui/src/serviceWorker.ts similarity index 100% rename from src/webui/src/serviceWorker.ts rename to ts/webui/src/serviceWorker.ts diff --git a/src/webui/src/static/const.ts b/ts/webui/src/static/const.ts similarity index 85% rename from src/webui/src/static/const.ts rename to ts/webui/src/static/const.ts index f35826b5e0..7f4d6a79c5 100644 --- a/src/webui/src/static/const.ts +++ b/ts/webui/src/static/const.ts @@ -15,7 +15,7 @@ const trialJobStatus = [ 'SYS_CANCELED', 'EARLY_STOPPED' ]; -const CONTROLTYPE = ['SEARCH_SPACE', 'TRIAL_CONCURRENCY', 'MAX_EXEC_DURATION']; +const CONTROLTYPE = ['MAX_EXEC_DURATION', 'MAX_TRIAL_NUM', 'TRIAL_CONCURRENCY', 'SEARCH_SPACE']; const MONACO = { readOnly: true, automaticLayout: true, @@ -57,6 +57,9 @@ const SUPPORTED_SEARCH_SPACE_TYPE = [ 'qlognormal' ]; +const TOOLTIP_BACKGROUND_COLOR = '#484848'; +const MAX_TRIAL_NUMBERS = 'Max trial No.'; + export { MANAGER_IP, DOWNLOAD_IP, @@ -71,5 +74,7 @@ export { METRIC_GROUP_UPDATE_THRESHOLD, METRIC_GROUP_UPDATE_SIZE, CONCURRENCYTOOLTIP, - SUPPORTED_SEARCH_SPACE_TYPE + SUPPORTED_SEARCH_SPACE_TYPE, + TOOLTIP_BACKGROUND_COLOR, + MAX_TRIAL_NUMBERS }; diff --git a/src/webui/src/static/datamodel.ts b/ts/webui/src/static/datamodel.ts similarity index 100% rename from src/webui/src/static/datamodel.ts rename to ts/webui/src/static/datamodel.ts diff --git a/src/webui/src/static/function.ts b/ts/webui/src/static/function.ts similarity index 89% rename from src/webui/src/static/function.ts rename to ts/webui/src/static/function.ts index 58729f9c3c..63dfe557ed 100644 --- a/src/webui/src/static/function.ts +++ b/ts/webui/src/static/function.ts @@ -30,26 +30,45 @@ const convertTime = (num: number): string => { }; // trial's duration, accurate to seconds for example 10min 30s -const convertDuration = (num: number): string => { - if (num < 1) { - return '0s'; +const convertDuration = (seconds: number): string => { + let str = ''; + + const d = Math.floor(seconds / (24 * 3600)); + if (d > 0) { + str += `${d}d `; } - const hour = Math.floor(num / 3600); - const minute = Math.floor((num / 60) % 60); - const second = Math.floor(num % 60); - const result: string[] = []; - if (hour > 0) { - result.push(`${hour}h`); + seconds -= d * 24 * 3600; + + const h = Math.floor(seconds / 3600); + if (h > 0) { + str += `${h}h `; } - if (minute > 0) { - result.push(`${minute}min`); + seconds -= h * 3600; + + const m = Math.floor(seconds / 60); + if (m > 0) { + str += `${m}m `; } - if (second > 0) { - result.push(`${second}s`); + seconds -= m * 60; + + if (seconds > 0) { + str += `${Math.floor(seconds)}s`; } - return result.join(' '); + return str ? str : '0s'; }; +// according the unit(d,h,m) to convert duration +function convertTimeAsUnit(unit: string, value: number): number { + let divisor = 1; + if (unit === 'h') { + divisor = 3600; + } else if (unit === 'm') { + divisor = 60; + } else { + divisor = 24 * 3600; + } + return value / divisor; +} function parseMetrics(metricData: string): any { if (metricData.includes('NaN') || metricData.includes('Infinity')) { return JSON5.parse(JSON5.parse(metricData)); @@ -246,6 +265,7 @@ function formatComplexTypeValue(value: any): string | number { export { convertTime, convertDuration, + convertTimeAsUnit, getFinalResult, getFinal, downFile, diff --git a/ts/webui/src/static/img/loading.gif b/ts/webui/src/static/img/loading.gif new file mode 100644 index 0000000000..624dd9f7c4 Binary files /dev/null and b/ts/webui/src/static/img/loading.gif differ diff --git a/src/webui/src/static/img/logo2.png b/ts/webui/src/static/img/logo.png similarity index 100% rename from src/webui/src/static/img/logo2.png rename to ts/webui/src/static/img/logo.png diff --git a/src/webui/src/static/interface.ts b/ts/webui/src/static/interface.ts similarity index 97% rename from src/webui/src/static/interface.ts rename to ts/webui/src/static/interface.ts index 400d33eee2..5494460358 100644 --- a/src/webui/src/static/interface.ts +++ b/ts/webui/src/static/interface.ts @@ -33,6 +33,7 @@ interface TableObj { color?: string; startTime?: number; endTime?: number; + intermediates: (MetricDataRecord | undefined)[]; parameters(axes: MultipleAxes): Map<SingleAxis, any>; metrics(axes: MultipleAxes): Map<SingleAxis, any>; } @@ -183,10 +184,14 @@ interface ExperimentParams { }; clusterMetaData?: { key: string; - value: string; + value: string | ClusterItem; }[]; } +interface ClusterItem { + command?: string; +} + interface ExperimentProfile { params: ExperimentParams; id: string; diff --git a/src/webui/src/static/json_util.ts b/ts/webui/src/static/json_util.ts similarity index 100% rename from src/webui/src/static/json_util.ts rename to ts/webui/src/static/json_util.ts diff --git a/src/webui/src/static/model/experiment.ts b/ts/webui/src/static/model/experiment.ts similarity index 100% rename from src/webui/src/static/model/experiment.ts rename to ts/webui/src/static/model/experiment.ts diff --git a/src/webui/src/static/model/searchspace.ts b/ts/webui/src/static/model/searchspace.ts similarity index 100% rename from src/webui/src/static/model/searchspace.ts rename to ts/webui/src/static/model/searchspace.ts diff --git a/src/webui/src/static/model/trial.ts b/ts/webui/src/static/model/trial.ts similarity index 95% rename from src/webui/src/static/model/trial.ts rename to ts/webui/src/static/model/trial.ts index 1578a6c4b2..c33888144d 100644 --- a/src/webui/src/static/model/trial.ts +++ b/ts/webui/src/static/model/trial.ts @@ -60,7 +60,7 @@ function inferTrialParameters( class Trial implements TableObj { private metricsInitialized: boolean = false; private infoField: TrialJobInfo | undefined; - private intermediates: (MetricDataRecord | undefined)[] = []; + public intermediates: (MetricDataRecord | undefined)[] = []; public final: MetricDataRecord | undefined; private finalAcc: number | undefined; @@ -224,24 +224,29 @@ class Trial implements TableObj { } public parameters(axes: MultipleAxes): Map<SingleAxis, any> { + const ret = new Map<SingleAxis, any>(Array.from(axes.axes.values()).map(k => [k, null])); if (this.info === undefined || this.info.hyperParameters === undefined) { - throw new Map(); + throw ret; } else { const tempHyper = this.info.hyperParameters; let params = JSON.parse(tempHyper[tempHyper.length - 1]).parameters; if (typeof params === 'string') { params = JSON.parse(params); } - const [result, unexpectedEntries] = inferTrialParameters(params, axes); + const [updated, unexpectedEntries] = inferTrialParameters(params, axes); if (unexpectedEntries.size) { throw unexpectedEntries; } - return result; + for (const [k, v] of updated) { + ret.set(k, v); + } + return ret; } } public metrics(space: MultipleAxes): Map<SingleAxis, any> { - const ret = new Map<SingleAxis, any>(); + // set default value: null + const ret = new Map<SingleAxis, any>(Array.from(space.axes.values()).map(k => [k, null])); const unexpectedEntries = new Map<string, any>(); if (this.acc === undefined) { return ret; diff --git a/src/webui/src/static/model/trialmanager.ts b/ts/webui/src/static/model/trialmanager.ts similarity index 100% rename from src/webui/src/static/model/trialmanager.ts rename to ts/webui/src/static/model/trialmanager.ts diff --git a/src/webui/src/static/style/button.scss b/ts/webui/src/static/style/button.scss similarity index 100% rename from src/webui/src/static/style/button.scss rename to ts/webui/src/static/style/button.scss diff --git a/src/webui/src/static/style/compare.scss b/ts/webui/src/static/style/compare.scss similarity index 100% rename from src/webui/src/static/style/compare.scss rename to ts/webui/src/static/style/compare.scss diff --git a/src/webui/src/static/style/copyParameter.scss b/ts/webui/src/static/style/copyParameter.scss similarity index 100% rename from src/webui/src/static/style/copyParameter.scss rename to ts/webui/src/static/style/copyParameter.scss diff --git a/src/webui/src/static/style/icon.scss b/ts/webui/src/static/style/icon.scss similarity index 100% rename from src/webui/src/static/style/icon.scss rename to ts/webui/src/static/style/icon.scss diff --git a/ts/webui/src/static/style/loading.scss b/ts/webui/src/static/style/loading.scss new file mode 100644 index 0000000000..4b845c47b9 --- /dev/null +++ b/ts/webui/src/static/style/loading.scss @@ -0,0 +1,12 @@ +.loading { + width: 80%; + height: 100%; + margin: 0 auto; + + img { + width: 40%; + position: fixed; + top: 30%; + left: 30%; + } +} diff --git a/src/webui/src/static/style/logDrawer.scss b/ts/webui/src/static/style/logDrawer.scss similarity index 90% rename from src/webui/src/static/style/logDrawer.scss rename to ts/webui/src/static/style/logDrawer.scss index d4dcca201d..d4c3aaf240 100644 --- a/src/webui/src/static/style/logDrawer.scss +++ b/ts/webui/src/static/style/logDrawer.scss @@ -28,3 +28,7 @@ text-align: right; } } + +.configClose { + margin: 45px 0 25px 0; +} diff --git a/src/webui/src/static/style/logPath.scss b/ts/webui/src/static/style/logPath.scss similarity index 100% rename from src/webui/src/static/style/logPath.scss rename to ts/webui/src/static/style/logPath.scss diff --git a/src/webui/src/static/style/nav/nav.scss b/ts/webui/src/static/style/nav/nav.scss similarity index 100% rename from src/webui/src/static/style/nav/nav.scss rename to ts/webui/src/static/style/nav/nav.scss diff --git a/src/webui/src/static/style/openRow.scss b/ts/webui/src/static/style/openRow.scss similarity index 100% rename from src/webui/src/static/style/openRow.scss rename to ts/webui/src/static/style/openRow.scss diff --git a/ts/webui/src/static/style/overview/command.scss b/ts/webui/src/static/style/overview/command.scss new file mode 100644 index 0000000000..8547e2bbeb --- /dev/null +++ b/ts/webui/src/static/style/overview/command.scss @@ -0,0 +1,14 @@ +.overviewCommand1, +.overviewCommand2 { + .command { + margin-top: 0; + font-weight: normal; + } +} + +.basic { + .lineMargin { + margin-top: 20px; + font-weight: normal; + } +} diff --git a/ts/webui/src/static/style/overview/config.scss b/ts/webui/src/static/style/overview/config.scss new file mode 100644 index 0000000000..e6fa74c98d --- /dev/null +++ b/ts/webui/src/static/style/overview/config.scss @@ -0,0 +1,30 @@ +.config { + position: fixed; + right: 0; + z-index: 1000; + + .ms-Button--default { + padding: 0 8px; + margin: 0 0 12px 0; + border: none; + box-shadow: 0 3px 3px rgba(0, 0, 0, 0.08); + border-radius: 18px 0 0 18px; + font-size: 12px; + text-align: left; + + .ms-Button-label { + font-weight: normal; + } + } + + .ms-Button--default:hover { + background-color: #0071bc; + color: #fff; + } +} + +.ms-Fabric { + .ms-Panel-commands { + margin: 0; + } +} diff --git a/ts/webui/src/static/style/overview/count.scss b/ts/webui/src/static/style/overview/count.scss new file mode 100644 index 0000000000..5069e40fd4 --- /dev/null +++ b/ts/webui/src/static/style/overview/count.scss @@ -0,0 +1,117 @@ +$seriesIconMargin: 8px; + +.ExpDuration { + margin-top: 28px; + + span:hover { + cursor: pointer; + } + + .maxTrialNum { + margin-bottom: 10px; + } +} + +.exp-progress { + margin-top: 10px; + + .bold { + font-weight: 500; + } + + .joiner { + padding: 0 3px; + } +} + +.maxTrialNum { + .editparam { + position: relative; + top: -7px; + } +} + +.noEditDuration { + position: relative; + top: -7px; +} + +.editDuration { + position: relative; + top: -17px; +} + +.editparam { + &-Input { + width: 42px; + height: 32px; + padding-right: 5px; + text-align: right; + outline: none; + border: none; + border-bottom: 1px solid #ccc; + } + + .maxExecDuration { + width: 36px; + } + + &-dropdown { + width: 48px; + display: inline-block; + position: relative; + top: 13px; + left: 4px; + margin-right: 3px; + } +} + +.ExpDuration .series .confirm { + margin: 0 6px; +} + +.series { + position: relative; + top: 5px; + + i { + font-size: 20px; + font-weight: 700; + } + + .confirm { + margin: 0 $seriesIconMargin; + + i { + color: green; + } + } +} + +.cancel i { + color: red; + font-size: 16px; +} + +.edit i { + margin-left: 4px; +} + +.overview input:disabled { + background: transparent; + border: none; +} + +.info { + position: absolute; + z-index: 999; + left: 0; +} + +.mess { + margin-top: 20px; + + .basic p { + margin-top: 0; + } +} diff --git a/ts/webui/src/static/style/overview/overview.scss b/ts/webui/src/static/style/overview/overview.scss new file mode 100644 index 0000000000..5cad95f167 --- /dev/null +++ b/ts/webui/src/static/style/overview/overview.scss @@ -0,0 +1,156 @@ +$boxPadding: 20px; +$boxBorderRadius: 5px; +$boxGapPadding: 10px; + +.wrapper { + display: grid; + grid-template-columns: repeat(9, 1fr); + grid-auto-rows: 97px; + + > div { + background: #fff; + padding: $boxPadding; + border-radius: $boxBorderRadius; + box-sizing: border-box; + } + + .overviewProgress { + grid-column: 2 / 6; + grid-row: 1 / 5; + display: grid; + grid-auto-rows: 70px; + margin: 0 $boxGapPadding; + padding: 0; + background: transparent; + + .duration, + .trialCount { + background: #fff; + padding: $boxPadding; + border-radius: $boxBorderRadius; + box-sizing: border-box; + + /* for alert message tooltip position */ + position: relative; + } + + .duration { + // grid-row: 1 / 3; + height: 139px; + } + + .trialCount { + margin-top: 79px; + height: 239px; + } + } + + .overviewCommand1, + .overviewCommand2 { + border-radius: 0; + } + + .overviewCommand1 { + grid-column-start: 1; + border-radius: $boxBorderRadius 0 0 $boxBorderRadius; + } + + .overviewCommand2 { + grid-column: 2 / 6; + margin-right: 10px; + padding-left: 30px; + border-radius: 0 $boxBorderRadius $boxBorderRadius 0; + } +} + +.overviewBasicInfo { + grid-column-start: 1; + grid-row: 1 / 5; + z-index: 2; +} + +.basic { + line-height: 21px; + font-family: "Segoe UI", Tahoma, Geneva, Verdana, sans-serif; + + p { + font-size: 14px; + color: #8f8f8f; + margin-top: 20px; + + span { + color: #484848; + } + } + + div { + font-size: 16px; + font-weight: 500; + color: #484848; + } + + .nowrap { + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; + } +} + +.overviewTable { + grid-column: 6 / 10; + grid-row: 1 / 11; + overflow: hidden; + + .topTrialTitle { + width: 72%; + } + + .active { + background: #d2d0ce; + } + + .max { + margin-left: 7px; + } + + .mincenter { + margin: 0 13px 0 10px; + } + + .chooseEntry { + margin-right: 10px; + line-height: 30px; + } +} + +.overviewCommand1, +.overviewCommand2 { + height: 144px; + overflow: hidden; + margin-top: 10px; +} + +$circle: 10px; +$bgblue: #0071bc; + +.overviewChart { + grid-column: 1 / 6; + grid-row: 7 / 11; + margin-right: $boxGapPadding; + margin-top: -29px; + + .circle { + width: $circle; + height: $circle; + border-radius: 50%; + background-color: $bgblue; + margin-top: 6px; + margin-right: 18px; + } +} + +.showMess { + position: absolute; + top: 42%; + left: 48%; +} diff --git a/src/webui/src/static/style/overviewTitle.scss b/ts/webui/src/static/style/overview/overviewTitle.scss similarity index 54% rename from src/webui/src/static/style/overviewTitle.scss rename to ts/webui/src/static/style/overview/overviewTitle.scss index c080c8e271..7182a0e1a9 100644 --- a/src/webui/src/static/style/overviewTitle.scss +++ b/ts/webui/src/static/style/overview/overviewTitle.scss @@ -1,31 +1,15 @@ -$iconPaddingVal: 14px; +$iconPaddingVal: 20px; .panelTitle { - font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; - width: 100%; - height: 38px; - padding: 4px $iconPaddingVal; - box-sizing: border-box; - - img { - height: 22px; - - /* (38 - 22 ) / 2 */ - margin-top: 8px; - - /* icon right */ - padding: 0 $iconPaddingVal 0 0; - } - span { font-size: 18px; font-weight: 600; color: #333; - line-height: 38px; } i { font-size: 24px; + margin-right: 20px; color: #545454; } } @@ -45,7 +29,7 @@ $iconPaddingVal: 14px; } .minTitle { - margin-right: $iconPaddingVal; + // margin-right: $iconPaddingVal; border-right: 2px solid #fff; } diff --git a/src/webui/src/static/style/pagination.scss b/ts/webui/src/static/style/pagination.scss similarity index 100% rename from src/webui/src/static/style/pagination.scss rename to ts/webui/src/static/style/pagination.scss diff --git a/src/webui/src/static/style/para.scss b/ts/webui/src/static/style/para.scss similarity index 100% rename from src/webui/src/static/style/para.scss rename to ts/webui/src/static/style/para.scss diff --git a/ts/webui/src/static/style/progress/probar.scss b/ts/webui/src/static/style/progress/probar.scss new file mode 100644 index 0000000000..734f8b12d8 --- /dev/null +++ b/ts/webui/src/static/style/progress/probar.scss @@ -0,0 +1,55 @@ +/* status: 'INITIALIZED' | 'RUNNING' | 'ERROR' | 'STOPPING' | 'STOPPED' | 'DONE' */ +$running: #0071bc; +$done: #00ad56; +$error: #a4262c; + +/* status: 'TUNER_NO_MORE_TRIAL' | 'NO_MORE_TRIAL' */ +.RUNNING, +.STOPPING, +.INITIALIZED, +.NO_MORE_TRIAL, +.TUNER_NO_MORE_TRIAL { + /* specific status color */ + color: $running; + + /* progress- duration & trial numbers span */ + .ms-ProgressIndicator-progressBar { + background-color: $running; + } +} + +.DONE, +.STOPPED { + color: $done; + + .ms-ProgressIndicator-progressBar { + background-color: $done; + } +} + +.ERROR { + color: $error; + + .ms-ProgressIndicator-progressBar { + background-color: $error; + } +} + +.bestMetric { + .DONE, + .STOPPED { + color: $done; + } + + .ERROR { + color: $error; + } + + .RUNNING, + .STOPPING, + .INITIALIZED, + .NO_MORE_TRIAL, + .TUNER_NO_MORE_TRIAL { + color: $running; + } +} diff --git a/ts/webui/src/static/style/progress/progress.scss b/ts/webui/src/static/style/progress/progress.scss new file mode 100644 index 0000000000..a2182903ac --- /dev/null +++ b/ts/webui/src/static/style/progress/progress.scss @@ -0,0 +1,69 @@ + +.status { + color: #0573bc; + font-size: 20px; + font-weight: 600; + + .status-text { + display: inline-block; + line-height: 30px; + } +} + +.probar { + width: 100%; + height: 34px; + margin-top: 15px; + + .showProgress { + width: 300px; + height: 30px; + } + + .name { + width: 178px; + box-sizing: border-box; + line-height: 30px; + text-align: center; + color: #fff; + background-color: #999; + border: 2px solid #e6e6e6; + border-top-left-radius: 12px; + border-bottom-left-radius: 12px; + } + + .boundary { + width: 100%; + line-height: 24px; + font-size: 12px; + color: #212121; + + .right { + text-align: right; + } + } + + .description { + line-height: 34px; + margin-left: 6px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } +} + +.inputBox { + height: 32px; + margin-top: 5px; +} + +/* office-fabric-ui progressIndicator */ +.ms-ProgressIndicator-itemProgress { + padding: 0; + border: 2px solid #e6e6e6; +} + +.cursor, +.cursor:hover { + cursor: pointer; +} diff --git a/src/webui/src/static/style/search.scss b/ts/webui/src/static/style/search.scss similarity index 100% rename from src/webui/src/static/style/search.scss rename to ts/webui/src/static/style/search.scss diff --git a/ts/webui/src/static/style/succTable.scss b/ts/webui/src/static/style/succTable.scss new file mode 100644 index 0000000000..dfa32a4ba3 --- /dev/null +++ b/ts/webui/src/static/style/succTable.scss @@ -0,0 +1,24 @@ +#succTable { + min-height: 400px; + max-height: 1000px; + overflow-y: auto; + position: relative; + + .succTable-tooltip { + width: 90%; + position: absolute; + left: 50%; + top: 50%; + transform: translate(-50%, -50%); + + a { + font-weight: 500; + color: blue; + } + } + + .succeed-padding { + padding-left: 6px; + box-sizing: border-box; + } +} diff --git a/src/webui/src/static/style/table.scss b/ts/webui/src/static/style/table.scss similarity index 97% rename from src/webui/src/static/style/table.scss rename to ts/webui/src/static/style/table.scss index 14d6712f23..054c7a63e9 100644 --- a/src/webui/src/static/style/table.scss +++ b/ts/webui/src/static/style/table.scss @@ -58,7 +58,7 @@ } .detail-table { - padding: 5px 0 0 0; + padding-top: 5px; } .columns-height { diff --git a/src/webui/src/static/style/tableStatus.css b/ts/webui/src/static/style/tableStatus.css similarity index 79% rename from src/webui/src/static/style/tableStatus.css rename to ts/webui/src/static/style/tableStatus.css index 9cf6bc57cf..58f1346723 100644 --- a/src/webui/src/static/style/tableStatus.css +++ b/ts/webui/src/static/style/tableStatus.css @@ -5,11 +5,11 @@ } .FAILED { - color: #dd4b39; + color: #a4262c; } .SUCCEEDED { - color: #00a445; + color: #00ad56; } .UNKNOWN { @@ -21,7 +21,7 @@ } .WAITING { - color: #fdc401; + color: #d29200; } .EARLY_STOPPED { diff --git a/src/webui/src/static/style/trialsDetail.scss b/ts/webui/src/static/style/trialsDetail.scss similarity index 94% rename from src/webui/src/static/style/trialsDetail.scss rename to ts/webui/src/static/style/trialsDetail.scss index 24e97b5c93..72f665e944 100644 --- a/src/webui/src/static/style/trialsDetail.scss +++ b/ts/webui/src/static/style/trialsDetail.scss @@ -57,6 +57,14 @@ } /* table list all */ +.bulletedList { + background: #fff; + + .title { + margin-top: 18px; + margin-left: 14px; + } +} #tableList { width: 96%; diff --git a/src/webui/tsconfig.json b/ts/webui/tsconfig.json similarity index 100% rename from src/webui/tsconfig.json rename to ts/webui/tsconfig.json diff --git a/src/webui/yarn.lock b/ts/webui/yarn.lock similarity index 96% rename from src/webui/yarn.lock rename to ts/webui/yarn.lock index 235cda2f82..da648044f5 100644 --- a/src/webui/yarn.lock +++ b/ts/webui/yarn.lock @@ -1163,68 +1163,66 @@ resolved "https://registry.yarnpkg.com/@csstools/normalize.css/-/normalize.css-9.0.1.tgz#c27b391d8457d1e893f1eddeaf5e5412d12ffbb5" integrity sha512-6It2EVfGskxZCQhuykrfnALg7oVeiI6KclWSmGDqB0AiInVrTGB9Jp9i4/Ad21u9Jde/voVQz6eFX/eSg/UsPA== -"@fluentui/date-time-utilities@^7.8.0": - version "7.8.0" - resolved "https://registry.yarnpkg.com/@fluentui/date-time-utilities/-/date-time-utilities-7.8.0.tgz#4fe384a19aba3508492e9d17680fba76a21a6144" - integrity sha512-qzlTp3t+PghebJsLK9JwZr91qBRZ/fOml8TQCIjdtsEn4mH6/ciCwir7Fj8iOEkwwTC0iKsEr1jfsITtJKWSmA== +"@fluentui/date-time-utilities@^7.9.0": + version "7.9.0" + resolved "https://registry.yarnpkg.com/@fluentui/date-time-utilities/-/date-time-utilities-7.9.0.tgz#4c8570b3af8bc654963ecb034d0fd100010e7d6d" + integrity sha512-D8p5WWeonqRO1EgIvo7WSlX1rcm87r2VQd62zTJPQImx8rpwc77CRI+iAvfxyVHRZMdt4Qk6Jq99dUaudPWaZw== dependencies: - "@uifabric/set-version" "^7.0.22" + "@uifabric/set-version" "^7.0.23" tslib "^1.10.0" -"@fluentui/keyboard-key@^0.2.11": - version "0.2.11" - resolved "https://registry.yarnpkg.com/@fluentui/keyboard-key/-/keyboard-key-0.2.11.tgz#97cbea8641c144ac5230fae0daf6e10c4f4228d4" - integrity sha512-4iG5vNlRD3PrLaMfDTimHNOP1x6V3Dad0fXOwtvBBMsfdySPE9zlY+G+0aqCFcgjymOriFOk0ANZVBpK7XrnTg== +"@fluentui/dom-utilities@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@fluentui/dom-utilities/-/dom-utilities-1.1.1.tgz#b0bbab665fe726f245800bb9e7883b1ceb54248b" + integrity sha512-w40gi8fzCpwa7U8cONiuu8rszPStkVOL/weDf5pCbYEb1gdaV7MDPSNkgM6IV0Kz+k017noDgK9Fv4ru1Dwz1g== dependencies: + "@uifabric/set-version" "^7.0.23" tslib "^1.10.0" -"@fluentui/react-focus@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@fluentui/react-focus/-/react-focus-7.16.0.tgz#51e5d4b22fbe91d557115b4c9402a64214b784ec" - integrity sha512-TwB4Av7ID70ejisDIGkCZGKOxlquSazr6W+9Jv1JQAvsBLuj5XOspFJH4/Igjniw1LeO9QmAvFZeh/XRShiObw== +"@fluentui/keyboard-key@^0.2.12": + version "0.2.12" + resolved "https://registry.yarnpkg.com/@fluentui/keyboard-key/-/keyboard-key-0.2.12.tgz#74eddf4657c164193b6c8855746e691af466441a" + integrity sha512-t3yIbbPKJubb22vQ/FIWwS9vFAzaPYzFxKWPHVWLtxs/P+5yL+LD3B16DRtYreWAdl9CZvEbos58ChLZ0KHwSQ== dependencies: - "@fluentui/keyboard-key" "^0.2.11" - "@uifabric/merge-styles" "^7.18.0" - "@uifabric/set-version" "^7.0.22" - "@uifabric/styling" "^7.16.0" - "@uifabric/utilities" "^7.31.0" tslib "^1.10.0" -"@fluentui/react-icons@^0.3.0": - version "0.3.0" - resolved "https://registry.yarnpkg.com/@fluentui/react-icons/-/react-icons-0.3.0.tgz#4acc45e0ecef69860e15d161f26363a6e711282a" - integrity sha512-QDrIhLUpkxM5q5bFE4pkweJJsfaNswdfDoc+X6cmr22Dii+anDFTpOcePEJiTtILxdlEmH0968OcZ4BGZF/joQ== +"@fluentui/react-focus@^7.16.10": + version "7.16.10" + resolved "https://registry.yarnpkg.com/@fluentui/react-focus/-/react-focus-7.16.10.tgz#e82e0719f48877bfc7220d359db4859c4d10b99d" + integrity sha512-+4aP36KjD2RrijRBr6aPYNiBm9M9+33DOHpAdcE0K93TToLIlQ/WIwZGDNbM/6dPFD6vgUj+ya5rvfy6sbibjw== dependencies: - "@microsoft/load-themed-styles" "^1.10.26" - "@uifabric/set-version" "^7.0.22" - "@uifabric/utilities" "^7.31.0" + "@fluentui/keyboard-key" "^0.2.12" + "@uifabric/merge-styles" "^7.19.1" + "@uifabric/set-version" "^7.0.23" + "@uifabric/styling" "^7.16.10" + "@uifabric/utilities" "^7.32.4" tslib "^1.10.0" -"@fluentui/react-window-provider@^0.3.0": - version "0.3.0" - resolved "https://registry.yarnpkg.com/@fluentui/react-window-provider/-/react-window-provider-0.3.0.tgz#6dc90a9826306816ddb5b2961e48546c1cb3d93f" - integrity sha512-zHeH1PAmoFB3OkgU8f51E3oGHrL/CjuvdPBlb1SGpByUGNihAgH/CwvJohV9z0yPDkq+chfNUmxWA/oDVpbh6Q== +"@fluentui/react-window-provider@^0.3.3": + version "0.3.3" + resolved "https://registry.yarnpkg.com/@fluentui/react-window-provider/-/react-window-provider-0.3.3.tgz#2950c4e19b28f24079d0e9d6a7a7e4d5b79ad115" + integrity sha512-MVPf2hqOQ17LAZsuvGcr3oOHksAskUm+fCYdXFhbVoAgsCDVTIuH6i8XgHFd6YjBtzjZmI4+k/3NTQfDqBX8EQ== dependencies: - "@uifabric/set-version" "^7.0.22" + "@uifabric/set-version" "^7.0.23" tslib "^1.10.0" "@fluentui/react@^7.135.0": - version "7.137.1" - resolved "https://registry.yarnpkg.com/@fluentui/react/-/react-7.137.1.tgz#c69878285de5397ec8b50917b9129e50ac3c91c0" - integrity sha512-u9qlBobDBjqwOK3vN7N7NoS53RuPXvcpVMOaK5oPRBMdCTsEDcK9pZj/OrKDSGVKmnPRGpoAdNAgsx34QaOfyQ== + version "7.145.0" + resolved "https://registry.yarnpkg.com/@fluentui/react/-/react-7.145.0.tgz#d9ff288cbaf13a61c5d0d36d19eaa693519e4e4c" + integrity sha512-RNYTWwRQhIFKoMHmDkMLF48KcH91brHV/WXCEboIxsmtHm4HyYVG4Zpjib6UG2XOS1zk9sBefHao/zTx3pdG8Q== dependencies: - "@uifabric/set-version" "^7.0.22" - office-ui-fabric-react "^7.137.1" + "@uifabric/set-version" "^7.0.23" + office-ui-fabric-react "^7.145.0" tslib "^1.10.0" -"@fluentui/theme@^0.2.0": - version "0.2.0" - resolved "https://registry.yarnpkg.com/@fluentui/theme/-/theme-0.2.0.tgz#15e7536e4b8bbbe96132687ef76921687af4200c" - integrity sha512-L5OwqymTdxTVwBa3ifkQLF1A3YU6WMkMTk6PFIMfOonjt8LJ3yMMqZgz+KQR3sF/tzgy4wzW4oBmrxtTfJS2EQ== +"@fluentui/theme@^1.3.0": + version "1.3.0" + resolved "https://registry.yarnpkg.com/@fluentui/theme/-/theme-1.3.0.tgz#15a65c6ff20c11a1a7550919b1a71b9c922e9255" + integrity sha512-OVN3yPdJShOGhTakalI5DC9+8myOhLPKzY/ZlICnNoKhh7cbPVrvdCnv2fLcDyZYMQg0CnB3oAaghBHiIW3jmA== dependencies: - "@uifabric/merge-styles" "^7.18.0" - "@uifabric/set-version" "^7.0.22" - "@uifabric/utilities" "^7.31.0" + "@uifabric/merge-styles" "^7.19.1" + "@uifabric/set-version" "^7.0.23" + "@uifabric/utilities" "^7.32.4" tslib "^1.10.0" "@hapi/address@2.x.x": @@ -1416,10 +1414,10 @@ "@types/istanbul-reports" "^1.1.1" "@types/yargs" "^13.0.0" -"@jest/types@^26.3.0": - version "26.3.0" - resolved "https://registry.yarnpkg.com/@jest/types/-/types-26.3.0.tgz#97627bf4bdb72c55346eef98e3b3f7ddc4941f71" - integrity sha512-BDPG23U0qDeAvU4f99haztXwdAg3hz4El95LkAM+tHAqqhiVzRpEGHHU8EDxT/AnxOrA65YjLBwDahdJ9pTLJQ== +"@jest/types@^26.5.2": + version "26.5.2" + resolved "https://registry.yarnpkg.com/@jest/types/-/types-26.5.2.tgz#44c24f30c8ee6c7f492ead9ec3f3c62a5289756d" + integrity sha512-QDs5d0gYiyetI8q+2xWdkixVQMklReZr4ltw7GFDtb4fuJIBCE6mzj2LnitGqCuAlLap6wPyb8fpoHgwZz5fdg== dependencies: "@types/istanbul-lib-coverage" "^2.0.0" "@types/istanbul-reports" "^3.0.0" @@ -1428,9 +1426,9 @@ chalk "^4.0.0" "@microsoft/load-themed-styles@^1.10.26": - version "1.10.90" - resolved "https://registry.yarnpkg.com/@microsoft/load-themed-styles/-/load-themed-styles-1.10.90.tgz#b9bfc10a9bfe3098efefcc3f842991ca6a029dd8" - integrity sha512-nJaQ6Yij68+GeaSt5ZgK70l5dn2aNGlc9EeES2B054anZTmmBhfaGpciIoSyPZrOg6PSAJjGWTXwd1ovMKOk1w== + version "1.10.108" + resolved "https://registry.yarnpkg.com/@microsoft/load-themed-styles/-/load-themed-styles-1.10.108.tgz#29dfb22e75a50e46a64cb338e2dc680b395f5f01" + integrity sha512-ETEADhZJwttKGPABitB7lDD/22ULL8AG1SKrn9uhYxjF1w1IaS6YS/yCPlDVALeh20zF8LJEEZ2MGb7DgVVYUw== "@mrmlnc/readdir-enhanced@^2.2.1": version "2.2.1" @@ -1585,9 +1583,9 @@ loader-utils "^1.1.0" "@types/babel__core@^7.1.0": - version "7.1.9" - resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.9.tgz#77e59d438522a6fb898fa43dc3455c6e72f3963d" - integrity sha512-sY2RsIJ5rpER1u3/aQ8OFSI7qGIy8o1NEEbgb2UaJcvOtXOMpd39ko723NBpjQFg9SIX7TXtjejZVGeIMLhoOw== + version "7.1.10" + resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.10.tgz#ca58fc195dd9734e77e57c6f2df565623636ab40" + integrity sha512-x8OM8XzITIMyiwl5Vmo2B1cR1S1Ipkyv4mdlbJjMa1lmuKvKY9FrBbEANIaMlnWn5Rf7uO+rC/VgYabNkE17Hw== dependencies: "@babel/parser" "^7.1.0" "@babel/types" "^7.0.0" @@ -1596,252 +1594,242 @@ "@types/babel__traverse" "*" "@types/babel__generator@*": - version "7.6.1" - resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.1.tgz#4901767b397e8711aeb99df8d396d7ba7b7f0e04" - integrity sha512-bBKm+2VPJcMRVwNhxKu8W+5/zT7pwNEqeokFOmbvVSqGzFneNxYcEBro9Ac7/N9tlsaPYnZLK8J1LWKkMsLAew== + version "7.6.2" + resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.2.tgz#f3d71178e187858f7c45e30380f8f1b7415a12d8" + integrity sha512-MdSJnBjl+bdwkLskZ3NGFp9YcXGx5ggLpQQPqtgakVhsWK0hTtNYhjpZLlWQTviGTvF8at+Bvli3jV7faPdgeQ== dependencies: "@babel/types" "^7.0.0" "@types/babel__template@*": - version "7.0.2" - resolved "https://registry.yarnpkg.com/@types/babel__template/-/babel__template-7.0.2.tgz#4ff63d6b52eddac1de7b975a5223ed32ecea9307" - integrity sha512-/K6zCpeW7Imzgab2bLkLEbz0+1JlFSrUMdw7KoIIu+IUdu51GWaBZpd3y1VXGVXzynvGa4DaIaxNZHiON3GXUg== + version "7.0.3" + resolved "https://registry.yarnpkg.com/@types/babel__template/-/babel__template-7.0.3.tgz#b8aaeba0a45caca7b56a5de9459872dde3727214" + integrity sha512-uCoznIPDmnickEi6D0v11SBpW0OuVqHJCa7syXqQHy5uktSCreIlt0iglsCnmvz8yCb38hGcWeseA8cWJSwv5Q== dependencies: "@babel/parser" "^7.1.0" "@babel/types" "^7.0.0" "@types/babel__traverse@*", "@types/babel__traverse@^7.0.6": - version "7.0.14" - resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.0.14.tgz#e99da8c075d4fb098c774ba65dabf7dc9954bd13" - integrity sha512-8w9szzKs14ZtBVuP6Wn7nMLRJ0D6dfB0VEBEyRgxrZ/Ln49aNMykrghM2FaNn4FJRzNppCSa0Rv9pBRM5Xc3wg== + version "7.0.15" + resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.0.15.tgz#db9e4238931eb69ef8aab0ad6523d4d4caa39d03" + integrity sha512-Pzh9O3sTK8V6I1olsXpCfj2k/ygO2q1X0vhhnDrEQyYLHZesWz+zMZMVcwXLCYf0U36EtmyYaFGPfXlTtDHe3A== dependencies: "@babel/types" "^7.3.0" -"@types/color-name@^1.1.1": - version "1.1.1" - resolved "https://registry.yarnpkg.com/@types/color-name/-/color-name-1.1.1.tgz#1c1261bbeaa10a8055bbc5d8ab84b7b2afc846a0" - integrity sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ== - -"@types/d3-array@*": - version "2.0.0" - resolved "https://registry.yarnpkg.com/@types/d3-array/-/d3-array-2.0.0.tgz#a0d63a296a2d8435a9ec59393dcac746c6174a96" - integrity sha512-rGqfPVowNDTszSFvwoZIXvrPG7s/qKzm9piCRIH6xwTTRu7pPZ3ootULFnPkTt74B6i5lN0FpLQL24qGOw1uZA== - "@types/d3-array@^1": version "1.2.7" resolved "https://registry.yarnpkg.com/@types/d3-array/-/d3-array-1.2.7.tgz#34dc654d34fc058c41c31dbca1ed68071a8fcc17" integrity sha512-51vHWuUyDOi+8XuwPrTw3cFqyh2Slg9y8COYkRfjCPG9TfYqY0hoNPzv/8BrcAy0FeQBzqEo/D/8Nk2caOQJnA== -"@types/d3-axis@*": - version "1.0.12" - resolved "https://registry.yarnpkg.com/@types/d3-axis/-/d3-axis-1.0.12.tgz#8c124edfcc02f3b3a9cdaa2a28b8a20341401799" - integrity sha512-BZISgSD5M8TgURyNtcPAmUB9sk490CO1Thb6/gIn0WZTt3Y50IssX+2Z0vTccoqZksUDTep0b+o4ofXslvNbqg== +"@types/d3-axis@^1": + version "1.0.14" + resolved "https://registry.yarnpkg.com/@types/d3-axis/-/d3-axis-1.0.14.tgz#4ff27eb94fab10efbda6c972e1fbb26ea696655b" + integrity sha512-wZAKX/dtFT5t5iuCaiU0QL0BWB19TE6h7C7kgfBVyoka7zidQWvf8E9zQTJ5bNPBQxd0+JmplNqwy1M8O8FOjA== dependencies: - "@types/d3-selection" "*" + "@types/d3-selection" "^1" -"@types/d3-brush@*": - version "1.1.1" - resolved "https://registry.yarnpkg.com/@types/d3-brush/-/d3-brush-1.1.1.tgz#906875ce42db22fc9cde6d1fb2808f17ecd2ea93" - integrity sha512-Exx14trm/q2cskHyMjCrdDllOQ35r1/pmZXaOIt8bBHwYNk722vWY3VxHvN0jdFFX7p2iL3+gD+cGny/aEmhlw== +"@types/d3-brush@^1": + version "1.1.4" + resolved "https://registry.yarnpkg.com/@types/d3-brush/-/d3-brush-1.1.4.tgz#0b5cc9c57476d0144b991228b44664e08494b7f3" + integrity sha512-2t8CgWaha9PsPdSZJ9m6Jl4awqf3DGIXek2e7gfheyfP2R0a/18MX+wuLHx+LyI1Ad7lxDsPWcswKD0XhQEjmg== dependencies: - "@types/d3-selection" "*" + "@types/d3-selection" "^1" -"@types/d3-chord@*": - version "1.0.9" - resolved "https://registry.yarnpkg.com/@types/d3-chord/-/d3-chord-1.0.9.tgz#ccc5de03ff079025491b7aa6b750670a140b45ae" - integrity sha512-UA6lI9CVW5cT5Ku/RV4hxoFn4mKySHm7HEgodtfRthAj1lt9rKZEPon58vyYfk+HIAm33DtJJgZwMXy2QgyPXw== +"@types/d3-chord@^1": + version "1.0.10" + resolved "https://registry.yarnpkg.com/@types/d3-chord/-/d3-chord-1.0.10.tgz#4c14ca40f61b89a3c615d63f5a34fcc81390805c" + integrity sha512-U6YojfET6ITL1/bUJo+/Lh3pMV9XPAfOWwbshl3y3RlgAX9VO/Bxa13IMAylZIDY4VsA3Gkh29kZP1AcAeyoYA== "@types/d3-collection@*": version "1.0.8" resolved "https://registry.yarnpkg.com/@types/d3-collection/-/d3-collection-1.0.8.tgz#aa9552c570a96e33c132e0fd20e331f64baa9dd5" integrity sha512-y5lGlazdc0HNO0F3UUX2DPE7OmYvd9Kcym4hXwrJcNUkDaypR5pX+apuMikl9LfTxKItJsY9KYvzBulpCKyvuQ== -"@types/d3-color@*": - version "1.2.2" - resolved "https://registry.yarnpkg.com/@types/d3-color/-/d3-color-1.2.2.tgz#80cf7cfff7401587b8f89307ba36fe4a576bc7cf" - integrity sha512-6pBxzJ8ZP3dYEQ4YjQ+NVbQaOflfgXq/JbDiS99oLobM2o72uAST4q6yPxHv6FOTCRC/n35ktuo8pvw/S4M7sw== +"@types/d3-color@^1": + version "1.4.1" + resolved "https://registry.yarnpkg.com/@types/d3-color/-/d3-color-1.4.1.tgz#0d9746c84dfef28807b2989eed4f37b2575e1f33" + integrity sha512-xkPLi+gbgUU9ED6QX4g6jqYL2KCB0/3AlM+ncMGqn49OgH0gFMY/ITGqPF8HwEiLzJaC+2L0I+gNwBgABv1Pvg== -"@types/d3-contour@*": - version "1.3.0" - resolved "https://registry.yarnpkg.com/@types/d3-contour/-/d3-contour-1.3.0.tgz#1a408b121fa5e341f715e3055303ef3079fc7eb0" - integrity sha512-AUCUIjEnC5lCGBM9hS+MryRaFLIrPls4Rbv6ktqbd+TK/RXZPwOy9rtBWmGpbeXcSOYCJTUDwNJuEnmYPJRxHQ== +"@types/d3-contour@^1": + version "1.3.1" + resolved "https://registry.yarnpkg.com/@types/d3-contour/-/d3-contour-1.3.1.tgz#589dc3eec14168eea7e31edd1e3bbe246cc9d626" + integrity sha512-wWwsM/3NfKTRBdH00cSf+XlsaHlNTkvH66PgDedobyvKQZ4sJrXXpr16LXvDnAal4B67v8JGrWDgyx6dqqKLuQ== dependencies: - "@types/d3-array" "*" + "@types/d3-array" "^1" "@types/geojson" "*" -"@types/d3-dispatch@*": - version "1.0.8" - resolved "https://registry.yarnpkg.com/@types/d3-dispatch/-/d3-dispatch-1.0.8.tgz#eaeb2ad089d6a0d2685dfa2f2cbbfb7509aae014" - integrity sha512-lCDtqoYez0TgFN3FljBXrz2icqeSzD0gufGook6DPBia+NOh2TBfogjHIsmNa/a+ZOewlHtq4cgLY80O1uLymw== +"@types/d3-dispatch@^1": + version "1.0.9" + resolved "https://registry.yarnpkg.com/@types/d3-dispatch/-/d3-dispatch-1.0.9.tgz#c5a180f1e251de853b399cfbfbb6dd7f8bf842ae" + integrity sha512-zJ44YgjqALmyps+II7b1mZLhrtfV/FOxw9owT87mrweGWcg+WK5oiJX2M3SYJ0XUAExBduarysfgbR11YxzojQ== -"@types/d3-drag@*": - version "1.2.3" - resolved "https://registry.yarnpkg.com/@types/d3-drag/-/d3-drag-1.2.3.tgz#d8ddccca28e939e9c689bea6f40a937e48c39051" - integrity sha512-rWB5SPvkYVxW3sqUxHOJUZwifD0KqvKwvt1bhNqcLpW6Azsd0BJgRNcyVW8GAferaAk5r8dzeZnf9zKlg9+xMQ== +"@types/d3-drag@^1": + version "1.2.5" + resolved "https://registry.yarnpkg.com/@types/d3-drag/-/d3-drag-1.2.5.tgz#0b1b852cb41577075aa625ae6149379ea6c34dfd" + integrity sha512-7NeTnfolst1Js3Vs7myctBkmJWu6DMI3k597AaHUX98saHjHWJ6vouT83UrpE+xfbSceHV+8A0JgxuwgqgmqWw== dependencies: - "@types/d3-selection" "*" + "@types/d3-selection" "^1" -"@types/d3-dsv@*": - version "1.0.36" - resolved "https://registry.yarnpkg.com/@types/d3-dsv/-/d3-dsv-1.0.36.tgz#e91129d7c02b1b814838d001e921e8b9a67153d0" - integrity sha512-jbIWQ27QJcBNMZbQv0NSQMHnBDCmxghAxePxgyiPH1XPCRkOsTBei7jcdi3fDrUCGpCV3lKrSZFSlOkhUQVClA== +"@types/d3-dsv@^1": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@types/d3-dsv/-/d3-dsv-1.2.1.tgz#1524fee9f19d689c2f76aa0e24e230762bf96994" + integrity sha512-LLmJmjiqp/fTNEdij5bIwUJ6P6TVNk5hKM9/uk5RPO2YNgEu9XvKO0dJ7Iqd3psEdmZN1m7gB1bOsjr4HmO2BA== -"@types/d3-ease@*": - version "1.0.9" - resolved "https://registry.yarnpkg.com/@types/d3-ease/-/d3-ease-1.0.9.tgz#1dd849bd7edef6426e915e220ed9970db5ea4e04" - integrity sha512-U5ADevQ+W6fy32FVZZC9EXallcV/Mi12A5Tkd0My5MrC7T8soMQEhlDAg88XUWm0zoCQlB4XV0en/24LvuDB4Q== +"@types/d3-ease@^1": + version "1.0.10" + resolved "https://registry.yarnpkg.com/@types/d3-ease/-/d3-ease-1.0.10.tgz#09910e8558439b6038a7ed620650e510394ffa6d" + integrity sha512-fMFTCzd8DOwruE9zlu2O8ci5ct+U5jkGcDS+cH+HCidnJlDs0MZ+TuSVCFtEzh4E5MasItwy+HvgoFtxPHa5Cw== -"@types/d3-fetch@*": - version "1.1.5" - resolved "https://registry.yarnpkg.com/@types/d3-fetch/-/d3-fetch-1.1.5.tgz#51601f79dd4653b5d84e6a3176d78145e065db5e" - integrity sha512-o9c0ItT5/Gl3wbNuVpzRnYX1t3RghzeWAjHUVLuyZJudiTxC4f/fC0ZPFWLQ2lVY8pAMmxpV8TJ6ETYCgPeI3A== +"@types/d3-fetch@^1": + version "1.2.2" + resolved "https://registry.yarnpkg.com/@types/d3-fetch/-/d3-fetch-1.2.2.tgz#b93bfe248b8b761af82f4dac57959c989f67da3e" + integrity sha512-rtFs92GugtV/NpiJQd0WsmGLcg52tIL0uF0bKbbJg231pR9JEb6HT4AUwrtuLq3lOeKdLBhsjV14qb0pMmd0Aw== dependencies: - "@types/d3-dsv" "*" + "@types/d3-dsv" "^1" -"@types/d3-force@*": - version "1.2.1" - resolved "https://registry.yarnpkg.com/@types/d3-force/-/d3-force-1.2.1.tgz#c28803ea36fe29788db69efa0ad6c2dc09544e83" - integrity sha512-jqK+I36uz4kTBjyk39meed5y31Ab+tXYN/x1dn3nZEus9yOHCLc+VrcIYLc/aSQ0Y7tMPRlIhLetulME76EiiA== +"@types/d3-force@^1": + version "1.2.2" + resolved "https://registry.yarnpkg.com/@types/d3-force/-/d3-force-1.2.2.tgz#6337a146dbdf2781f5dde5bb491a646fd03d7bc4" + integrity sha512-TN7KO7sk0tJauedIt0q20RQRFo4V3v97pJKO/TDK40X3LaPM1aXRM2+zFF+nRMtseEiszg4KffudhjR8a3+4cg== -"@types/d3-format@*": - version "1.3.1" - resolved "https://registry.yarnpkg.com/@types/d3-format/-/d3-format-1.3.1.tgz#35bf88264bd6bcda39251165bb827f67879c4384" - integrity sha512-KAWvReOKMDreaAwOjdfQMm0HjcUMlQG47GwqdVKgmm20vTd2pucj0a70c3gUSHrnsmo6H2AMrkBsZU2UhJLq8A== +"@types/d3-format@^1": + version "1.4.1" + resolved "https://registry.yarnpkg.com/@types/d3-format/-/d3-format-1.4.1.tgz#1e657a219e4b1e3931508a610d570bdec8ecdd9d" + integrity sha512-ss9G2snEKmp2In5Z3T0Jpqv8QaDBc2xHltBw83KjnV5B5w+Iwphbvq5ph/Xnu4d03fmmsdt+o1aWch379rxIbA== -"@types/d3-geo@*": - version "1.11.1" - resolved "https://registry.yarnpkg.com/@types/d3-geo/-/d3-geo-1.11.1.tgz#e96ec91f16221d87507fec66b2cc889f52d2493e" - integrity sha512-Ox8WWOG3igDRoep/dNsGbOiSJYdUG3ew/6z0ETvHyAtXZVBjOE0S96zSSmzgl0gqQ3RdZjn2eeJOj9oRcMZPkQ== +"@types/d3-geo@^1": + version "1.12.1" + resolved "https://registry.yarnpkg.com/@types/d3-geo/-/d3-geo-1.12.1.tgz#bec8692ffee9f60e18483af9008f92d4a8428118" + integrity sha512-8+gyGFyMCXIHtnMNKQDT++tZ4XYFXgiP5NK7mcv34aYXA16GQFiBBITjKzxghpO8QNVceOd9rUn1JY92WLNGQw== dependencies: "@types/geojson" "*" -"@types/d3-hierarchy@*": - version "1.1.6" - resolved "https://registry.yarnpkg.com/@types/d3-hierarchy/-/d3-hierarchy-1.1.6.tgz#4c017521900813ea524c9ecb8d7985ec26a9ad9a" - integrity sha512-vvSaIDf/Ov0o3KwMT+1M8+WbnnlRiGjlGD5uvk83a1mPCTd/E5x12bUJ/oP55+wUY/4Kb5kc67rVpVGJ2KUHxg== +"@types/d3-hierarchy@^1": + version "1.1.7" + resolved "https://registry.yarnpkg.com/@types/d3-hierarchy/-/d3-hierarchy-1.1.7.tgz#14a57b0539f8929015f8ad96490de50a16211040" + integrity sha512-fvht6DOYKzqmXjMb/+xfgkmrWM4SD7rMA/ZbM+gGwr9ZTuIDfky95J8CARtaJo/ExeWyS0xGVdL2gqno2zrQ0Q== -"@types/d3-interpolate@*": - version "1.3.1" - resolved "https://registry.yarnpkg.com/@types/d3-interpolate/-/d3-interpolate-1.3.1.tgz#1c280511f622de9b0b47d463fa55f9a4fd6f5fc8" - integrity sha512-z8Zmi08XVwe8e62vP6wcA+CNuRhpuUU5XPEfqpG0hRypDE5BWNthQHB1UNWWDB7ojCbGaN4qBdsWp5kWxhT1IQ== +"@types/d3-interpolate@^1": + version "1.4.2" + resolved "https://registry.yarnpkg.com/@types/d3-interpolate/-/d3-interpolate-1.4.2.tgz#88902a205f682773a517612299a44699285eed7b" + integrity sha512-ylycts6llFf8yAEs1tXzx2loxxzDZHseuhPokrqKprTQSTcD3JbJI1omZP1rphsELZO3Q+of3ff0ZS7+O6yVzg== dependencies: - "@types/d3-color" "*" + "@types/d3-color" "^1" -"@types/d3-path@*": - version "1.0.8" - resolved "https://registry.yarnpkg.com/@types/d3-path/-/d3-path-1.0.8.tgz#48e6945a8ff43ee0a1ce85c8cfa2337de85c7c79" - integrity sha512-AZGHWslq/oApTAHu9+yH/Bnk63y9oFOMROtqPAtxl5uB6qm1x2lueWdVEjsjjV3Qc2+QfuzKIwIR5MvVBakfzA== +"@types/d3-path@^1": + version "1.0.9" + resolved "https://registry.yarnpkg.com/@types/d3-path/-/d3-path-1.0.9.tgz#73526b150d14cd96e701597cbf346cfd1fd4a58c" + integrity sha512-NaIeSIBiFgSC6IGUBjZWcscUJEq7vpVu7KthHN8eieTV9d9MqkSOZLH4chq1PmcKy06PNe3axLeKmRIyxJ+PZQ== -"@types/d3-polygon@*": - version "1.0.7" - resolved "https://registry.yarnpkg.com/@types/d3-polygon/-/d3-polygon-1.0.7.tgz#7b3947aa2d48287ff535230d3d396668ab17bfdf" - integrity sha512-Xuw0eSjQQKs8jTiNbntWH0S+Xp+JyhqxmQ0YAQ3rDu6c3kKMFfgsaGN7Jv5u3zG6yVX/AsLP/Xs/QRjmi9g43Q== +"@types/d3-polygon@^1": + version "1.0.8" + resolved "https://registry.yarnpkg.com/@types/d3-polygon/-/d3-polygon-1.0.8.tgz#127ee83fccda5bf57384011da90f31367fea1530" + integrity sha512-1TOJPXCBJC9V3+K3tGbTqD/CsqLyv/YkTXAcwdsZzxqw5cvpdnCuDl42M4Dvi8XzMxZNCT9pL4ibrK2n4VmAcw== -"@types/d3-quadtree@*": - version "1.0.7" - resolved "https://registry.yarnpkg.com/@types/d3-quadtree/-/d3-quadtree-1.0.7.tgz#8e29464ff5b326f6612c1428d9362b4b35de2b70" - integrity sha512-0ajFawWicfjsaCLh6NzxOyVDYhQAmMFbsiI3MPGLInorauHFEh9/Cl6UHNf+kt/J1jfoxKY/ZJaKAoDpbvde5Q== +"@types/d3-quadtree@^1": + version "1.0.8" + resolved "https://registry.yarnpkg.com/@types/d3-quadtree/-/d3-quadtree-1.0.8.tgz#980998eb20d5e1c1494089ad9a8466a0e98825a7" + integrity sha512-FuqYiexeSQZlc+IcGAVK8jSJKDFKHcSf/jx8rqJUUVx6rzv7ecQiXKyatrLHHh3W4CAvgNeVI23JKgk4+x2wFg== -"@types/d3-random@*": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@types/d3-random/-/d3-random-1.1.2.tgz#6f77e8b7bb64ac393f92d33fe8f71038bc4f3cde" - integrity sha512-Jui+Zn28pQw/3EayPKaN4c/PqTvqNbIPjHkgIIFnxne1FdwNjfHtAIsZIBMKlquQNrrMjFzCrlF2gPs3xckqaA== +"@types/d3-random@^1": + version "1.1.3" + resolved "https://registry.yarnpkg.com/@types/d3-random/-/d3-random-1.1.3.tgz#8f7fdc23f92d1561e0694eb49567e8ab50537a19" + integrity sha512-XXR+ZbFCoOd4peXSMYJzwk0/elP37WWAzS/DG+90eilzVbUSsgKhBcWqylGWe+lA2ubgr7afWAOBaBxRgMUrBQ== -"@types/d3-scale-chromatic@*": - version "1.5.0" - resolved "https://registry.yarnpkg.com/@types/d3-scale-chromatic/-/d3-scale-chromatic-1.5.0.tgz#315367557d51b823bec848614fac095325613fc3" - integrity sha512-9/D7cOBKdZdTCPc6re0HeSUFBM0aFzdNdmYggUWT9SRRiYSOa6Ys2xdTwHKgc1WS3gGfwTMatBOdWCS863REsg== +"@types/d3-scale-chromatic@^1": + version "1.5.1" + resolved "https://registry.yarnpkg.com/@types/d3-scale-chromatic/-/d3-scale-chromatic-1.5.1.tgz#e2b7c3401e5c13809f831911eb820e444f4fc67a" + integrity sha512-7FtJYrmXTEWLykShjYhoGuDNR/Bda0+tstZMkFj4RRxUEryv16AGh3be21tqg84B6KfEwiZyEpBcTyPyU+GWjg== -"@types/d3-scale@*": - version "2.2.0" - resolved "https://registry.yarnpkg.com/@types/d3-scale/-/d3-scale-2.2.0.tgz#e5987a2857365823eb26ed5eb21bc566c4dcf1c0" - integrity sha512-oQFanN0/PiR2oySHfj+zAAkK1/p4LD32Nt1TMVmzk+bYHk7vgIg/iTXQWitp1cIkDw4LMdcgvO63wL+mNs47YA== +"@types/d3-scale@^2": + version "2.2.4" + resolved "https://registry.yarnpkg.com/@types/d3-scale/-/d3-scale-2.2.4.tgz#ca0d4b84d2f88fe058480f81354d14041a667b96" + integrity sha512-wkQXT+IfgfAnKB5rtS1qMJg3FS32r1rVFHvqtiqk8pX8o5aQR3VwX1P7ErHjzNIicTlkWsaMiUTrYB+E75HFeA== dependencies: - "@types/d3-time" "*" + "@types/d3-time" "^1" -"@types/d3-selection@*": - version "1.4.2" - resolved "https://registry.yarnpkg.com/@types/d3-selection/-/d3-selection-1.4.2.tgz#72dcd61a3aeb9ce3e8d443e3bef7685ffea3413f" - integrity sha512-ksY8UxvTXpzD91Dy3D9zZg98yF2ZEPMKJd8ZQJlZt1QH3Xxr08s6fESEdC2l0Kbe6Xd9VhaoJX06cRaMR1lEnA== +"@types/d3-selection@^1": + version "1.4.3" + resolved "https://registry.yarnpkg.com/@types/d3-selection/-/d3-selection-1.4.3.tgz#36928bbe64eb8e0bbcbaa01fb05c21ff6c71fa93" + integrity sha512-GjKQWVZO6Sa96HiKO6R93VBE8DUW+DDkFpIMf9vpY5S78qZTlRRSNUsHr/afDpF7TvLDV7VxrUFOWW7vdIlYkA== -"@types/d3-shape@*": - version "1.3.2" - resolved "https://registry.yarnpkg.com/@types/d3-shape/-/d3-shape-1.3.2.tgz#a41d9d6b10d02e221696b240caf0b5d0f5a588ec" - integrity sha512-LtD8EaNYCaBRzHzaAiIPrfcL3DdIysc81dkGlQvv7WQP3+YXV7b0JJTtR1U3bzeRieS603KF4wUo+ZkJVenh8w== +"@types/d3-shape@^1": + version "1.3.4" + resolved "https://registry.yarnpkg.com/@types/d3-shape/-/d3-shape-1.3.4.tgz#5a6d8c3026ba8e8a1a985bda8da40acfc9b7b079" + integrity sha512-fxmOjs+UqNQGpztD5BOo+KriE0jLFrBP4Ct++0QExv/xfDOT1cpcMxgsZ+5qPmnR0t+GjbwAe1Um1PHpv3G4oA== dependencies: - "@types/d3-path" "*" + "@types/d3-path" "^1" -"@types/d3-time-format@*": - version "2.1.1" - resolved "https://registry.yarnpkg.com/@types/d3-time-format/-/d3-time-format-2.1.1.tgz#dd2c79ec4575f1355484ab6b10407824668eba42" - integrity sha512-tJSyXta8ZyJ52wDDHA96JEsvkbL6jl7wowGmuf45+fAkj5Y+SQOnz0N7/H68OWmPshPsAaWMQh+GAws44IzH3g== +"@types/d3-time-format@^2": + version "2.3.1" + resolved "https://registry.yarnpkg.com/@types/d3-time-format/-/d3-time-format-2.3.1.tgz#87a30e4513b9d1d53b920327a361f87255bf3372" + integrity sha512-fck0Z9RGfIQn3GJIEKVrp15h9m6Vlg0d5XXeiE/6+CQiBmMDZxfR21XtjEPuDeg7gC3bBM0SdieA5XF3GW1wKA== -"@types/d3-time@*": - version "1.0.10" - resolved "https://registry.yarnpkg.com/@types/d3-time/-/d3-time-1.0.10.tgz#d338c7feac93a98a32aac875d1100f92c7b61f4f" - integrity sha512-aKf62rRQafDQmSiv1NylKhIMmznsjRN+MnXRXTqHoqm0U/UZzVpdrtRnSIfdiLS616OuC1soYeX1dBg2n1u8Xw== +"@types/d3-time@^1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@types/d3-time/-/d3-time-1.1.1.tgz#6cf3a4242c3bbac00440dfb8ba7884f16bedfcbf" + integrity sha512-ULX7LoqXTCYtM+tLYOaeAJK7IwCT+4Gxlm2MaH0ErKLi07R5lh8NHCAyWcDkCCmx1AfRcBEV6H9QE9R25uP7jw== -"@types/d3-timer@*": - version "1.0.9" - resolved "https://registry.yarnpkg.com/@types/d3-timer/-/d3-timer-1.0.9.tgz#aed1bde0cf18920d33f5d44839d73de393633fd3" - integrity sha512-WvfJ3LFxBbWjqRGz9n7GJt08RrTHPJDVsIwwoCMROlqF+iDacYiAFjf9oqnq0mXpb2juA2N/qjKP+MKdal3YNQ== +"@types/d3-timer@^1": + version "1.0.10" + resolved "https://registry.yarnpkg.com/@types/d3-timer/-/d3-timer-1.0.10.tgz#329c51c2c931f44ed0acff78b8c84571acf0ed21" + integrity sha512-ZnAbquVqy+4ZjdW0cY6URp+qF/AzTVNda2jYyOzpR2cPT35FTXl78s15Bomph9+ckOiI1TtkljnWkwbIGAb6rg== -"@types/d3-transition@*": - version "1.1.6" - resolved "https://registry.yarnpkg.com/@types/d3-transition/-/d3-transition-1.1.6.tgz#7e52da29749d874866cc803fad13925713a372da" - integrity sha512-/F+O2r4oz4G9ATIH3cuSCMGphAnl7VDx7SbENEK0NlI/FE8Jx2oiIrv0uTrpg7yF/AmuWbqp7AGdEHAPIh24Gg== +"@types/d3-transition@^1": + version "1.3.1" + resolved "https://registry.yarnpkg.com/@types/d3-transition/-/d3-transition-1.3.1.tgz#5d658eea2db17684daa04eda81d7db9824d3463f" + integrity sha512-U9CpMlTL/NlqdGXBlHYxTZwbmy/vN1cFv8TuAIFPX+xOW/1iChbeJBY2xmINhDQfkGJbgkH4IovafCwI1ZDrgg== dependencies: - "@types/d3-selection" "*" + "@types/d3-selection" "^1" "@types/d3-voronoi@*": version "1.1.9" resolved "https://registry.yarnpkg.com/@types/d3-voronoi/-/d3-voronoi-1.1.9.tgz#7bbc210818a3a5c5e0bafb051420df206617c9e5" integrity sha512-DExNQkaHd1F3dFPvGA/Aw2NGyjMln6E9QzsiqOcBgnE+VInYnFBHBBySbZQts6z6xD+5jTfKCP7M4OqMyVjdwQ== -"@types/d3-zoom@*": - version "1.7.5" - resolved "https://registry.yarnpkg.com/@types/d3-zoom/-/d3-zoom-1.7.5.tgz#1fd7f3497a10f8cfaaefcf2f879767ec775fc3a3" - integrity sha512-G0lpZjlvmv+fI2/dg2whWFbUUmMC3dy4xoeaGOXnaUmOSnms1q9QtlkRq5MXZt1/7LcKwzgmKdEjPVLVq5dH5Q== +"@types/d3-zoom@^1": + version "1.8.2" + resolved "https://registry.yarnpkg.com/@types/d3-zoom/-/d3-zoom-1.8.2.tgz#187d33f9ffa59811ce93b2eacd32d92c1ef03f16" + integrity sha512-rU0LirorUxkLxEHSzkFs7pPC0KWsxRGc0sHrxEDR0/iQq+7/xpNkKuuOOwthlgvOtpOvtTLJ2JFOD6Kr0Si4Uw== dependencies: - "@types/d3-interpolate" "*" - "@types/d3-selection" "*" + "@types/d3-interpolate" "^1" + "@types/d3-selection" "^1" "@types/d3@^5.7.2": - version "5.7.2" - resolved "https://registry.yarnpkg.com/@types/d3/-/d3-5.7.2.tgz#52235eb71a1d3ca171d6dca52a58f5ccbe0254cc" - integrity sha512-7/wClB8ycneWGy3jdvLfXKTd5SoTg9hji7IdJ0RuO9xTY54YpJ8zlcFADcXhY1J3kCBwxp+/1jeN6a5OMwgYOw== + version "5.16.3" + resolved "https://registry.yarnpkg.com/@types/d3/-/d3-5.16.3.tgz#265d506a1b61f558084f2c660f8dd2c93a6d16c8" + integrity sha512-s3wrhYhu25XZQ5p1hI9gEMSX5bx7lg9hAmi0+i5r3v75Gz1zRTgB2Q0psx+SO+4K0AO/PPJ1pnHCz64pANN/4w== dependencies: "@types/d3-array" "^1" - "@types/d3-axis" "*" - "@types/d3-brush" "*" - "@types/d3-chord" "*" + "@types/d3-axis" "^1" + "@types/d3-brush" "^1" + "@types/d3-chord" "^1" "@types/d3-collection" "*" - "@types/d3-color" "*" - "@types/d3-contour" "*" - "@types/d3-dispatch" "*" - "@types/d3-drag" "*" - "@types/d3-dsv" "*" - "@types/d3-ease" "*" - "@types/d3-fetch" "*" - "@types/d3-force" "*" - "@types/d3-format" "*" - "@types/d3-geo" "*" - "@types/d3-hierarchy" "*" - "@types/d3-interpolate" "*" - "@types/d3-path" "*" - "@types/d3-polygon" "*" - "@types/d3-quadtree" "*" - "@types/d3-random" "*" - "@types/d3-scale" "*" - "@types/d3-scale-chromatic" "*" - "@types/d3-selection" "*" - "@types/d3-shape" "*" - "@types/d3-time" "*" - "@types/d3-time-format" "*" - "@types/d3-timer" "*" - "@types/d3-transition" "*" + "@types/d3-color" "^1" + "@types/d3-contour" "^1" + "@types/d3-dispatch" "^1" + "@types/d3-drag" "^1" + "@types/d3-dsv" "^1" + "@types/d3-ease" "^1" + "@types/d3-fetch" "^1" + "@types/d3-force" "^1" + "@types/d3-format" "^1" + "@types/d3-geo" "^1" + "@types/d3-hierarchy" "^1" + "@types/d3-interpolate" "^1" + "@types/d3-path" "^1" + "@types/d3-polygon" "^1" + "@types/d3-quadtree" "^1" + "@types/d3-random" "^1" + "@types/d3-scale" "^2" + "@types/d3-scale-chromatic" "^1" + "@types/d3-selection" "^1" + "@types/d3-shape" "^1" + "@types/d3-time" "^1" + "@types/d3-time-format" "^2" + "@types/d3-timer" "^1" + "@types/d3-transition" "^1" "@types/d3-voronoi" "*" - "@types/d3-zoom" "*" + "@types/d3-zoom" "^1" "@types/eslint-visitor-keys@^1.0.0": version "1.0.0" @@ -1910,14 +1898,14 @@ integrity sha1-aaI6OtKcrwCX8G7aWbNh7i8GOfY= "@types/node@*": - version "14.10.1" - resolved "https://registry.yarnpkg.com/@types/node/-/node-14.10.1.tgz#cc323bad8e8a533d4822f45ce4e5326f36e42177" - integrity sha512-aYNbO+FZ/3KGeQCEkNhHFRIzBOUgc7QvcVNKXbfnhDkSfwUv91JsQQa10rDgKSTSLkXZ1UIyPe4FJJNVgw1xWQ== + version "14.11.7" + resolved "https://registry.yarnpkg.com/@types/node/-/node-14.11.7.tgz#4d9673b8331ce262628ea89d9ef3964b6c1e5489" + integrity sha512-hSEXknS4KiayUdZ7401J/T6eykXHJkDEipnyQMJ4/GstK4kWjbHnwXlcpvIWfPKiEH1JU96DkbzJ1nHRmpmKLw== "@types/node@^10.14.2": - version "10.17.32" - resolved "https://registry.yarnpkg.com/@types/node/-/node-10.17.32.tgz#4ef6ff8b842ea0eb3fcbc4331489f4ae64036fa8" - integrity sha512-EUq+cjH/3KCzQHikGnNbWAGe548IFLSm93Vl8xA7EuYEEATiyOVDyEVuGkowL7c9V69FF/RiZSAOCFPApMs/ig== + version "10.17.38" + resolved "https://registry.yarnpkg.com/@types/node/-/node-10.17.38.tgz#951dc5ba517af0381bd8134636f93bcd93ae3e1b" + integrity sha512-pIQORpqlV+QwNix8K1lMmyS7fp80MkQruXp3eMCYAliS3Z1RMYkd4Wf22+iaKLffkngtlIkhiuXOdwLq1zrclg== "@types/normalize-package-data@^2.4.0": version "2.4.0" @@ -1947,9 +1935,9 @@ "@types/react" "*" "@types/react@*", "@types/react@^15.x || ^16.x": - version "16.9.49" - resolved "https://registry.yarnpkg.com/@types/react/-/react-16.9.49.tgz#09db021cf8089aba0cdb12a49f8021a69cce4872" - integrity sha512-DtLFjSj0OYAdVLBbyjhuV9CdGVHCkHn2R+xr3XkBvK2rS1Y1tkc14XSGjYgm5Fjjr90AxH9tiSzc1pCFMGO06g== + version "16.9.51" + resolved "https://registry.yarnpkg.com/@types/react/-/react-16.9.51.tgz#f8aa51ffa9996f1387f63686696d9b59713d2b60" + integrity sha512-lQa12IyO+DMlnSZ3+AGHRUiUcpK47aakMMoBG8f7HGxJT8Yfe+WE128HIXaHOHVPReAW0oDS3KAI0JI2DDe1PQ== dependencies: "@types/prop-types" "*" csstype "^3.0.2" @@ -1978,16 +1966,16 @@ integrity sha512-FA/BWv8t8ZWJ+gEOnLLd8ygxH/2UFbAvgEonyfN6yWGLKc7zVjbpl2Y4CTjid9h2RfgPP6SEt6uHwEOply00yw== "@types/yargs@^13.0.0": - version "13.0.10" - resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-13.0.10.tgz#e77bf3fc73c781d48c2eb541f87c453e321e5f4b" - integrity sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ== + version "13.0.11" + resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-13.0.11.tgz#def2f0c93e4bdf2c61d7e34899b17e34be28d3b1" + integrity sha512-NRqD6T4gktUrDi1o1wLH3EKC1o2caCr7/wR87ODcbVITQF106OM3sFN92ysZ++wqelOd1CTzatnOBRDYYG6wGQ== dependencies: "@types/yargs-parser" "*" "@types/yargs@^15.0.0": - version "15.0.5" - resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-15.0.5.tgz#947e9a6561483bdee9adffc983e91a6902af8b79" - integrity sha512-Dk/IDOPtOgubt/IaevIUbTgV7doaKkoorvOyYM2CMwuDyP89bekI7H4xLIwunNYiK9jhCkmc6pUrJk3cj2AB9w== + version "15.0.8" + resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-15.0.8.tgz#7644904cad7427eb704331ea9bf1ee5499b82e23" + integrity sha512-b0BYzFUzBpOhPjpl1wtAHU994jBeKF4TKVlT7ssFv44T617XNcPdRoG4AzHLVshLzlrF7i3lTelH7UbuNYV58Q== dependencies: "@types/yargs-parser" "*" @@ -2034,70 +2022,71 @@ semver "^7.3.2" tsutils "^3.17.1" -"@uifabric/foundation@^7.9.0": - version "7.9.0" - resolved "https://registry.yarnpkg.com/@uifabric/foundation/-/foundation-7.9.0.tgz#b7d8333e1c972a22336bfde747fc9f07df24f05c" - integrity sha512-4gdPwx0HV59aWvPKIw6yIje60CYwJZmTJsdLRXqh8Np1tWp2EsbA8U/IHQL1V91rEN0DlQhASqCDFhfQoD1jUw== +"@uifabric/foundation@^7.9.10": + version "7.9.10" + resolved "https://registry.yarnpkg.com/@uifabric/foundation/-/foundation-7.9.10.tgz#3855fe1490e20eb2ac5c8bfb428d7488dd6989bf" + integrity sha512-DeKN+beoqn6HsMW+Ezici1umj7j8aaNykOhLjqg11XRevJh7ifKQOrpePXM+m13B2VAdm2nPyfO3d7eYFvu7Zw== dependencies: - "@uifabric/merge-styles" "^7.18.0" - "@uifabric/set-version" "^7.0.22" - "@uifabric/styling" "^7.16.0" - "@uifabric/utilities" "^7.31.0" + "@uifabric/merge-styles" "^7.19.1" + "@uifabric/set-version" "^7.0.23" + "@uifabric/styling" "^7.16.10" + "@uifabric/utilities" "^7.32.4" tslib "^1.10.0" -"@uifabric/icons@^7.5.0": - version "7.5.0" - resolved "https://registry.yarnpkg.com/@uifabric/icons/-/icons-7.5.0.tgz#49b54b1d2eb235073b824f702fbf895676103012" - integrity sha512-RINA9VkajlCnlwEBbqiwLBaaGn38Bf9UvjOuwrrja4B9lclLeQzZuGrG/kp7CnDtIO+LC19OYl/7E3jW7yIihA== +"@uifabric/icons@^7.5.9": + version "7.5.9" + resolved "https://registry.yarnpkg.com/@uifabric/icons/-/icons-7.5.9.tgz#08f5772a0eb7e224fda3434dd3ab29143f21517a" + integrity sha512-kiFw2hm2++OwcVT8ZkHm/UIsyA+4CXBgttmtMaEXMB6/VSt6mfAc+3gs0ehnfXbrBTLUIM9omxXUrrtULSWLTw== dependencies: - "@uifabric/set-version" "^7.0.22" - "@uifabric/styling" "^7.16.0" + "@uifabric/set-version" "^7.0.23" + "@uifabric/styling" "^7.16.10" tslib "^1.10.0" -"@uifabric/merge-styles@^7.18.0": - version "7.18.0" - resolved "https://registry.yarnpkg.com/@uifabric/merge-styles/-/merge-styles-7.18.0.tgz#66c5d0e0a4ce03791d0ff356634b51ae1de4e36a" - integrity sha512-805WIbN7lAJATXKxZjjRbIgN7raRMwWYWeDkJJ52PCPuCesOvbpdr0GkH8rC6GQ7EB0MB7YM2i6Fiye7SFewbw== +"@uifabric/merge-styles@^7.19.1": + version "7.19.1" + resolved "https://registry.yarnpkg.com/@uifabric/merge-styles/-/merge-styles-7.19.1.tgz#446b3da48ce9925d1649edd0fc232221974b00c9" + integrity sha512-yqUwmk62Kgu216QNPE9vOfS3h0kiSbTvoqM5QcZi+IzpqsBOlzZx3A9Er9UiDaqHRd5lsYF5pO/jeUULmBWF/A== dependencies: - "@uifabric/set-version" "^7.0.22" + "@uifabric/set-version" "^7.0.23" tslib "^1.10.0" -"@uifabric/react-hooks@^7.13.0": - version "7.13.0" - resolved "https://registry.yarnpkg.com/@uifabric/react-hooks/-/react-hooks-7.13.0.tgz#fc1da8da1d337ebc52b7e032d6053d953fbc15d3" - integrity sha512-A7K0YFHHr10hB/txsqpPX6PhNhHEv8U7JpY3O81oqlZwSsHroJklQdd897JkAYJUUR+gWe2kQyXkkcaMDdqXjg== +"@uifabric/react-hooks@^7.13.6": + version "7.13.6" + resolved "https://registry.yarnpkg.com/@uifabric/react-hooks/-/react-hooks-7.13.6.tgz#0626791fe283b235b8ddfedc51f693b50b2792b6" + integrity sha512-DvfphxrTsjo3oYioRZ0D8mXdpxWQhhIHeWk1cfdq0MVGqRyKM+cm++9pJpvItFvnTNba38jzKdggqRSUWi+clg== dependencies: - "@fluentui/react-window-provider" "^0.3.0" - "@uifabric/set-version" "^7.0.22" - "@uifabric/utilities" "^7.31.0" + "@fluentui/react-window-provider" "^0.3.3" + "@uifabric/set-version" "^7.0.23" + "@uifabric/utilities" "^7.32.4" tslib "^1.10.0" -"@uifabric/set-version@^7.0.22": - version "7.0.22" - resolved "https://registry.yarnpkg.com/@uifabric/set-version/-/set-version-7.0.22.tgz#f2d5235bb10927a7024b23ccef52070371349dbb" - integrity sha512-IG35UNJNxqI7NC2eYuobGTD+v4W0VHQcC3bYd5Na9EgoC9jVgguS8n6EXUtP/lC1vJEYEyPEZdVwhPxKw4F4Sw== +"@uifabric/set-version@^7.0.23": + version "7.0.23" + resolved "https://registry.yarnpkg.com/@uifabric/set-version/-/set-version-7.0.23.tgz#bfe10b6ba17a2518704cca856bdba8adbc11ffb0" + integrity sha512-9E+YKtnH2kyMKnK9XZZsqyM8OCxEJIIfxtaThTlQpYOzrWAGJxQADFbZ7+Usi0U2xHnWNPFROjq+B9ocEzhqMA== dependencies: tslib "^1.10.0" -"@uifabric/styling@^7.16.0": - version "7.16.0" - resolved "https://registry.yarnpkg.com/@uifabric/styling/-/styling-7.16.0.tgz#428226d184965d549ebbd110ac33e115b3136b6f" - integrity sha512-yO404ws8Wl4fjBwG4T58TFJjeU/K+SpWJ4w+WeNMDlJcYe4zXaWn2lbOJtKtQiMl324RAEjFSOuNkXzPb0uOGQ== +"@uifabric/styling@^7.16.10": + version "7.16.10" + resolved "https://registry.yarnpkg.com/@uifabric/styling/-/styling-7.16.10.tgz#c9f25c4a269b2a75214928376088dbbd20820e65" + integrity sha512-qhTJME41VM63paw690xp9SxD3NJP7a4YGUQTifLU0q2GM0e7AMQVDPQ+Cd8Fv9YBS1lJKHi069hVRwSfBejlwg== dependencies: - "@fluentui/theme" "^0.2.0" + "@fluentui/theme" "^1.3.0" "@microsoft/load-themed-styles" "^1.10.26" - "@uifabric/merge-styles" "^7.18.0" - "@uifabric/set-version" "^7.0.22" - "@uifabric/utilities" "^7.31.0" + "@uifabric/merge-styles" "^7.19.1" + "@uifabric/set-version" "^7.0.23" + "@uifabric/utilities" "^7.32.4" tslib "^1.10.0" -"@uifabric/utilities@^7.31.0": - version "7.31.0" - resolved "https://registry.yarnpkg.com/@uifabric/utilities/-/utilities-7.31.0.tgz#600ea931b643c7cbcc38d6466748379227d4fbc7" - integrity sha512-m4Yeyn4gyW7xS8LvOnCesokPModYS2YuE9GQmO++MDZ/vC5RRNlvlyktUZDuxCZ84cNCiXyTQ8nImBaPGnxHVQ== +"@uifabric/utilities@^7.32.4": + version "7.32.4" + resolved "https://registry.yarnpkg.com/@uifabric/utilities/-/utilities-7.32.4.tgz#6e784c29101742196da69d0b60e63cf193c10c71" + integrity sha512-IKZc03uyfTrgcCGSPAUWfFl9CyxzpSrxrYwAuO4NgDEySdOlkYBRwsRUh6UwWfM+sJdhg7Y3nupzNU3VSC/arg== dependencies: - "@uifabric/merge-styles" "^7.18.0" - "@uifabric/set-version" "^7.0.22" + "@fluentui/dom-utilities" "^1.1.1" + "@uifabric/merge-styles" "^7.19.1" + "@uifabric/set-version" "^7.0.23" prop-types "^15.7.2" tslib "^1.10.0" @@ -2312,9 +2301,9 @@ acorn@^5.2.1, acorn@^5.5.3: integrity sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg== acorn@^6.0.1, acorn@^6.0.4, acorn@^6.0.5, acorn@^6.0.7: - version "6.4.1" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-6.4.1.tgz#531e58ba3f51b9dacb9a6646ca4debf5b14ca474" - integrity sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA== + version "6.4.2" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-6.4.2.tgz#35866fd710528e92de10cf06016498e47e39e1e6" + integrity sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ== address@1.1.2, address@^1.0.1: version "1.1.2" @@ -2427,11 +2416,10 @@ ansi-styles@^3.2.0, ansi-styles@^3.2.1: color-convert "^1.9.0" ansi-styles@^4.0.0, ansi-styles@^4.1.0: - version "4.2.1" - resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.2.1.tgz#90ae75c424d008d2624c5bf29ead3177ebfcf359" - integrity sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA== + version "4.3.0" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937" + integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== dependencies: - "@types/color-name" "^1.1.1" color-convert "^2.0.1" ansicolors@~0.3.2: @@ -3139,13 +3127,13 @@ browserslist@4.7.0: node-releases "^1.1.29" browserslist@^4.0.0, browserslist@^4.1.1, browserslist@^4.12.0, browserslist@^4.4.2, browserslist@^4.5.2, browserslist@^4.8.5: - version "4.14.2" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.14.2.tgz#1b3cec458a1ba87588cc5e9be62f19b6d48813ce" - integrity sha512-HI4lPveGKUR0x2StIz+2FXfDk9SfVMrxn6PLh1JeGUwcuoDkdKZebWiyLRJ68iIPDpMI4JLVDf7S7XzslgWOhw== + version "4.14.5" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.14.5.tgz#1c751461a102ddc60e40993639b709be7f2c4015" + integrity sha512-Z+vsCZIvCBvqLoYkBFTwEYH3v5MCQbsAjp50ERycpOjnPmolg1Gjy4+KaWWpm8QOJt9GHkhdqAl14NpCX73CWA== dependencies: - caniuse-lite "^1.0.30001125" - electron-to-chromium "^1.3.564" - escalade "^3.0.2" + caniuse-lite "^1.0.30001135" + electron-to-chromium "^1.3.571" + escalade "^3.1.0" node-releases "^1.1.61" bser@2.1.1: @@ -3349,10 +3337,10 @@ caniuse-api@^3.0.0: lodash.memoize "^4.1.2" lodash.uniq "^4.5.0" -caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000939, caniuse-lite@^1.0.30000989, caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001125: - version "1.0.30001129" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001129.tgz#e6514b94c0ef50f98cf7476daa91228ddd2ef7bc" - integrity sha512-9945fTVKS810DZITpsAbuhQG7Lam0tEfVbZlsBaCFZaszepbryrArS05PWmJSBQ6mta+v9iz0pUIAbW1eBILIg== +caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000939, caniuse-lite@^1.0.30000989, caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001135: + version "1.0.30001146" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001146.tgz#c61fcb1474520c1462913689201fb292ba6f447c" + integrity sha512-VAy5RHDfTJhpxnDdp2n40GPPLp3KqNrXz1QqFv4J64HvArKs8nuNMOWkB3ICOaBTU/Aj4rYAo/ytdQDDFF/Pug== capture-exit@^2.0.0: version "2.0.0" @@ -4133,9 +4121,9 @@ css-what@2.1: integrity sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg== css-what@^3.2.1: - version "3.3.0" - resolved "https://registry.yarnpkg.com/css-what/-/css-what-3.3.0.tgz#10fec696a9ece2e591ac772d759aacabac38cd39" - integrity sha512-pv9JPyatiPaQ6pf4OvD/dbfm0o5LviWmwxNWzblYf/1u9QZd0ihV+PMwy5jdQWQ3349kZmKEx9WXuSka2dM4cg== + version "3.4.1" + resolved "https://registry.yarnpkg.com/css-what/-/css-what-3.4.1.tgz#81cb70b609e4b1351b1e54cbc90fd9c54af86e2e" + integrity sha512-wHOppVDKl4vTAOWzJt5Ek37Sgd9qq1Bmj/T1OjvicWbU5W7ru7Pqbn0Jdqii3Drx/h+dixHKXNhZYx7blthL7g== cssdb@^4.3.0: version "4.4.0" @@ -4557,11 +4545,11 @@ debug@^3.1.0, debug@^3.1.1, debug@^3.2.5: ms "^2.1.1" debug@^4.0.1, debug@^4.1.0, debug@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.1.1.tgz#3b72260255109c6b589cee050f1d516139664791" - integrity sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw== + version "4.2.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.2.0.tgz#7f150f93920e94c58f5574c2fd01a3110effe7f1" + integrity sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg== dependencies: - ms "^2.1.1" + ms "2.1.2" debuglog@^1.0.1: version "1.0.1" @@ -4741,10 +4729,10 @@ diff-sequences@^24.9.0: resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-24.9.0.tgz#5715d6244e2aa65f48bba0bc972db0b0b11e95b5" integrity sha512-Dj6Wk3tWyTE+Fo1rW8v0Xhwk80um6yFYKbuAxc9c3EZxIHFDYwbi34Uk42u1CdnIiVorvt4RmlSDjIPyzGC2ew== -diff-sequences@^26.3.0: - version "26.3.0" - resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-26.3.0.tgz#62a59b1b29ab7fd27cef2a33ae52abe73042d0a2" - integrity sha512-5j5vdRcw3CNctePNYN0Wy2e/JbWT6cAYnXv5OuqPhDpyCGc0uLu2TK0zOCJWNB9kOIfYMSpIulRaDgIi4HJ6Ig== +diff-sequences@^26.5.0: + version "26.5.0" + resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-26.5.0.tgz#ef766cf09d43ed40406611f11c6d8d9dd8b2fefd" + integrity sha512-ZXx86srb/iYy6jG71k++wBN9P9J05UNQ5hQHQd9MtMPvcqXPx/vKU69jfHV637D00Q2gSgPk2D+jSx3l1lDW/Q== diffie-hellman@^5.0.0: version "5.0.3" @@ -4954,10 +4942,10 @@ ee-first@1.1.1: resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= -electron-to-chromium@^1.3.247, electron-to-chromium@^1.3.564: - version "1.3.567" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.567.tgz#7a404288952ac990e447a7a86470d460ea953b8f" - integrity sha512-1aKkw0Hha1Bw9JA5K5PT5eFXC/TXbkJvUfNSNEciPUMgSIsRJZM1hF2GUEAGZpAbgvd8En21EA+Lv820KOhvqA== +electron-to-chromium@^1.3.247, electron-to-chromium@^1.3.571: + version "1.3.578" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.578.tgz#e6671936f4571a874eb26e2e833aa0b2c0b776e0" + integrity sha512-z4gU6dA1CbBJsAErW5swTGAaU2TBzc2mPAonJb00zqW1rOraDo2zfBMDRvaz9cVic+0JEZiYbHWPw/fTaZlG2Q== elliptic@^6.5.3: version "6.5.3" @@ -5063,37 +5051,37 @@ error-ex@^1.2.0, error-ex@^1.3.1: is-arrayish "^0.2.1" es-abstract@^1.17.0, es-abstract@^1.17.0-next.1, es-abstract@^1.17.2, es-abstract@^1.17.5: - version "1.17.6" - resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.17.6.tgz#9142071707857b2cacc7b89ecb670316c3e2d52a" - integrity sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw== + version "1.17.7" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.17.7.tgz#a4de61b2f66989fc7421676c1cb9787573ace54c" + integrity sha512-VBl/gnfcJ7OercKA9MVaegWsBHFjV492syMudcnQZvt/Dw8ezpcOHYZXa/J96O8vx+g4x65YKhxOwDUh63aS5g== dependencies: es-to-primitive "^1.2.1" function-bind "^1.1.1" has "^1.0.3" has-symbols "^1.0.1" - is-callable "^1.2.0" - is-regex "^1.1.0" - object-inspect "^1.7.0" + is-callable "^1.2.2" + is-regex "^1.1.1" + object-inspect "^1.8.0" object-keys "^1.1.1" - object.assign "^4.1.0" + object.assign "^4.1.1" string.prototype.trimend "^1.0.1" string.prototype.trimstart "^1.0.1" -es-abstract@^1.18.0-next.0: - version "1.18.0-next.0" - resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.18.0-next.0.tgz#b302834927e624d8e5837ed48224291f2c66e6fc" - integrity sha512-elZXTZXKn51hUBdJjSZGYRujuzilgXo8vSPQzjGYXLvSlGiCo8VO8ZGV3kjo9a0WNJJ57hENagwbtlRuHuzkcQ== +es-abstract@^1.18.0-next.0, es-abstract@^1.18.0-next.1: + version "1.18.0-next.1" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.18.0-next.1.tgz#6e3a0a4bda717e5023ab3b8e90bec36108d22c68" + integrity sha512-I4UGspA0wpZXWENrdA0uHbnhte683t3qT/1VFH9aX2dA5PPSf6QW5HHXf5HImaqPmjXaVeVk4RGWnaylmV7uAA== dependencies: es-to-primitive "^1.2.1" function-bind "^1.1.1" has "^1.0.3" has-symbols "^1.0.1" - is-callable "^1.2.0" + is-callable "^1.2.2" is-negative-zero "^2.0.0" is-regex "^1.1.1" object-inspect "^1.8.0" object-keys "^1.1.1" - object.assign "^4.1.0" + object.assign "^4.1.1" string.prototype.trimend "^1.0.1" string.prototype.trimstart "^1.0.1" @@ -5119,9 +5107,14 @@ es6-promisify@^5.0.0: es6-promise "^4.0.3" escalade@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.0.2.tgz#6a580d70edb87880f22b4c91d0d56078df6962c4" - integrity sha512-gPYAU37hYCUhW5euPeR+Y74F7BL+IBsV93j5cvGriSaD1aG6MGsqsV1yamRdrWrb2j3aiZvb0X+UBOWpx3JWtQ== + version "3.1.1" + resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" + integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw== + +escalade@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.0.tgz#e8e2d7c7a8b76f6ee64c2181d6b8151441602d4e" + integrity sha512-mAk+hPSO8fLDkhV7V0dXazH5pDc6MrjBTPyD3VeKzxnVFjH1MIxbCdqGZB9O8+EwWakZs3ZCbDS4IpRt79V1ig== escape-html@~1.0.3: version "1.0.3" @@ -5146,9 +5139,9 @@ escodegen@^1.11.0, escodegen@^1.9.1: source-map "~0.6.1" eslint-config-prettier@^6.1.0: - version "6.11.0" - resolved "https://registry.yarnpkg.com/eslint-config-prettier/-/eslint-config-prettier-6.11.0.tgz#f6d2238c1290d01c859a8b5c1f7d352a0b0da8b1" - integrity sha512-oB8cpLWSAjOVFEJhhyMZh6NOEOtBVziaqdDQ86+qhDHFbZXoRTM7pNSvFRfW/W/L/LrQ38C99J5CGuRBBzBsdA== + version "6.12.0" + resolved "https://registry.yarnpkg.com/eslint-config-prettier/-/eslint-config-prettier-6.12.0.tgz#9eb2bccff727db1c52104f0b49e87ea46605a0d2" + integrity sha512-9jWPlFlgNwRUYVoujvWTQ1aMO8o6648r+K7qU7K5Jmkbyqav1fuEZC0COYpGBxyiAJb65Ra9hrmFx19xRGwXWw== dependencies: get-stdin "^6.0.0" @@ -6922,10 +6915,10 @@ is-buffer@^2.0.0: resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-2.0.4.tgz#3e572f23c8411a5cfd9557c849e3665e0b290623" integrity sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A== -is-callable@^1.1.4, is-callable@^1.2.0: - version "1.2.1" - resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.1.tgz#4d1e21a4f437509d25ce55f8184350771421c96d" - integrity sha512-wliAfSzx6V+6WfMOmus1xy0XvSgf/dlStkvTfq7F0g4bOIW0PSUbnyse3NhDwdyYS1ozfUtAAySqTws3z9Eqgg== +is-callable@^1.1.4, is-callable@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.2.tgz#c7c6715cd22d4ddb48d3e19970223aceabb080d9" + integrity sha512-dnMqspv5nU3LoewK2N/y7KLtxtakvTuaCsU9FU50/QDmdbHNy/4/JuRtMHqRU22o3q+W89YQndQEeCVwK+3qrA== is-ci@^1.0.10: version "1.2.1" @@ -7151,7 +7144,7 @@ is-redirect@^1.0.0: resolved "https://registry.yarnpkg.com/is-redirect/-/is-redirect-1.0.0.tgz#1d03dded53bd8db0f30c26e4f95d36fc7c87dc24" integrity sha1-HQPd7VO9jbDzDCbk+V02/HyH3CQ= -is-regex@^1.0.4, is-regex@^1.1.0, is-regex@^1.1.1: +is-regex@^1.0.4, is-regex@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.1.1.tgz#c6f98aacc546f6cec5468a07b7b153ab564a57b9" integrity sha512-1+QkEcxiLlB7VEyFtyBg94e08OAsvq7FUBgApTq/w2ymCLyKJgDPsybBENVtA7XCQEgEXxKPonG+mvYRxh/LIg== @@ -7366,14 +7359,14 @@ jest-config@^24.9.0: realpath-native "^1.1.0" jest-diff@*: - version "26.4.2" - resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-26.4.2.tgz#a1b7b303bcc534aabdb3bd4a7caf594ac059f5aa" - integrity sha512-6T1XQY8U28WH0Z5rGpQ+VqZSZz8EN8rZcBtfvXaOkbwxIEeRre6qnuZQlbY1AJ4MKDxQF8EkrCvK+hL/VkyYLQ== + version "26.5.2" + resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-26.5.2.tgz#8e26cb32dc598e8b8a1b9deff55316f8313c8053" + integrity sha512-HCSWDUGwsov5oTlGzrRM+UPJI/Dpqi9jzeV0fdRNi3Ch5bnoXhnyJMmVg2juv9081zLIy3HGPI5mcuGgXM2xRA== dependencies: chalk "^4.0.0" - diff-sequences "^26.3.0" + diff-sequences "^26.5.0" jest-get-type "^26.3.0" - pretty-format "^26.4.2" + pretty-format "^26.5.2" jest-diff@^24.9.0: version "24.9.0" @@ -8661,11 +8654,16 @@ miller-rabin@^4.0.0: bn.js "^4.0.0" brorand "^1.0.1" -mime-db@1.44.0, "mime-db@>= 1.43.0 < 2": +mime-db@1.44.0: version "1.44.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92" integrity sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg== +"mime-db@>= 1.43.0 < 2": + version "1.45.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.45.0.tgz#cceeda21ccd7c3a745eba2decd55d4b73e7879ea" + integrity sha512-CkqLUxUk15hofLoLyljJSrukZi8mAtgd+yE5uO4tqRZsdsAJKv0O+rFMhVDRJgozy+yG6md5KwuXhD4ocIoP+w== + mime-types@^2.1.12, mime-types@~2.1.17, mime-types@~2.1.19, mime-types@~2.1.24: version "2.1.27" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f" @@ -8796,9 +8794,9 @@ mixin-object@^2.0.1: minimist "^1.2.5" monaco-editor-webpack-plugin@^1.8.1: - version "1.9.0" - resolved "https://registry.yarnpkg.com/monaco-editor-webpack-plugin/-/monaco-editor-webpack-plugin-1.9.0.tgz#5b547281b9f404057dc5d8c5722390df9ac90be6" - integrity sha512-tOiiToc94E1sb50BgZ8q8WK/bxus77SRrwCqIpAB5er3cpX78SULbEBY4YPOB8kDolOzKRt30WIHG/D6gz69Ww== + version "1.9.1" + resolved "https://registry.yarnpkg.com/monaco-editor-webpack-plugin/-/monaco-editor-webpack-plugin-1.9.1.tgz#eb4bbb1c5e5bfb554541c1ae1542e74c2a9f43fd" + integrity sha512-x7fx1w3i/uwZERIgztHAAK3VQMsL8+ku0lFXXbO81hKDg8IieACqjGEa2mqEueg0c/fX+wd0oI+75wB19KJAsA== dependencies: loader-utils "^1.2.3" @@ -8829,7 +8827,7 @@ ms@2.1.1: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a" integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg== -ms@^2.0.0, ms@^2.1.1: +ms@2.1.2, ms@^2.0.0, ms@^2.1.1: version "2.1.2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== @@ -9388,18 +9386,18 @@ object-hash@^1.1.4: resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-1.3.1.tgz#fde452098a951cb145f039bb7d455449ddc126df" integrity sha512-OSuu/pU4ENM9kmREg0BdNrUDIl1heYa4mBZacJc+vVWz4GtAwu7jO8s4AIt2aGRUTqxykpWzI3Oqnsm13tTMDA== -object-inspect@^1.7.0, object-inspect@^1.8.0: +object-inspect@^1.8.0: version "1.8.0" resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.8.0.tgz#df807e5ecf53a609cc6bfe93eac3cc7be5b3a9d0" integrity sha512-jLdtEOB112fORuypAyl/50VRVIBIdVQOSUUGQHzJ4xBSbit81zRarz7GThkEFZy1RceYrWYcPcBFPQwHyAc1gA== object-is@^1.0.1: - version "1.1.2" - resolved "https://registry.yarnpkg.com/object-is/-/object-is-1.1.2.tgz#c5d2e87ff9e119f78b7a088441519e2eec1573b6" - integrity sha512-5lHCz+0uufF6wZ7CRFWJN3hp8Jqblpgve06U5CMQ3f//6iDjPr2PEo9MWCjEssDsa+UZEL4PkFpr+BMop6aKzQ== + version "1.1.3" + resolved "https://registry.yarnpkg.com/object-is/-/object-is-1.1.3.tgz#2e3b9e65560137455ee3bd62aec4d90a2ea1cc81" + integrity sha512-teyqLvFWzLkq5B9ki8FVWA902UER2qkxmdA4nLf+wjOLAWgxzCWZNCxpDq9MvE8MmhWNr+I8w3BN49Vx36Y6Xg== dependencies: define-properties "^1.1.3" - es-abstract "^1.17.5" + es-abstract "^1.18.0-next.1" object-keys@^1.0.12, object-keys@^1.1.1: version "1.1.1" @@ -9413,7 +9411,7 @@ object-visit@^1.0.0: dependencies: isobject "^3.0.0" -object.assign@^4.1.0: +object.assign@^4.1.0, object.assign@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.1.tgz#303867a666cdd41936ecdedfb1f8f3e32a478cdd" integrity sha512-VT/cxmx5yaoHSOTSyrCygIDFco+RsibY2NM0a4RdEeY/4KgqezwFtK1yr3U67xYhqJSlASm2pKhLVzPj2lr4bA== @@ -9463,23 +9461,22 @@ obuf@^1.0.0, obuf@^1.1.2: resolved "https://registry.yarnpkg.com/obuf/-/obuf-1.1.2.tgz#09bea3343d41859ebd446292d11c9d4db619084e" integrity sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg== -office-ui-fabric-react@^7.137.1: - version "7.137.1" - resolved "https://registry.yarnpkg.com/office-ui-fabric-react/-/office-ui-fabric-react-7.137.1.tgz#58b3c1d2254afb3d7ab1f780cf1fb6929c42fafe" - integrity sha512-6m1M9m5psaJqhWwroVp1L0G8q66scLMsQRi+gQ/fY3f75ipGSABGeQ86d4P0ilKaYcLVYIzosocPIZKLbBVpnQ== +office-ui-fabric-react@^7.145.0: + version "7.145.0" + resolved "https://registry.yarnpkg.com/office-ui-fabric-react/-/office-ui-fabric-react-7.145.0.tgz#e86a5fa08a2a3528f2a692bec88ea55c76b238b3" + integrity sha512-+eAo/bZpnTXmirwsPZxIbPhjQk+C34f0W/8tAHYWYASrRio9DWWDB0PRsSGUNrn1QubWxjudW294fovkewPeAQ== dependencies: - "@fluentui/date-time-utilities" "^7.8.0" - "@fluentui/react-focus" "^7.16.0" - "@fluentui/react-icons" "^0.3.0" - "@fluentui/react-window-provider" "^0.3.0" + "@fluentui/date-time-utilities" "^7.9.0" + "@fluentui/react-focus" "^7.16.10" + "@fluentui/react-window-provider" "^0.3.3" "@microsoft/load-themed-styles" "^1.10.26" - "@uifabric/foundation" "^7.9.0" - "@uifabric/icons" "^7.5.0" - "@uifabric/merge-styles" "^7.18.0" - "@uifabric/react-hooks" "^7.13.0" - "@uifabric/set-version" "^7.0.22" - "@uifabric/styling" "^7.16.0" - "@uifabric/utilities" "^7.31.0" + "@uifabric/foundation" "^7.9.10" + "@uifabric/icons" "^7.5.9" + "@uifabric/merge-styles" "^7.19.1" + "@uifabric/react-hooks" "^7.13.6" + "@uifabric/set-version" "^7.0.23" + "@uifabric/styling" "^7.16.10" + "@uifabric/utilities" "^7.32.4" prop-types "^15.7.2" tslib "^1.10.0" @@ -10018,9 +10015,9 @@ postcss-browser-comments@^2.0.0: postcss "^7.0.2" postcss-calc@^7.0.1: - version "7.0.4" - resolved "https://registry.yarnpkg.com/postcss-calc/-/postcss-calc-7.0.4.tgz#5e177ddb417341e6d4a193c5d9fd8ada79094f8b" - integrity sha512-0I79VRAd1UTkaHzY9w83P39YGO/M3bG7/tNLrHGEunBolfoGM0hSjrGvjoeaj0JE/zIw5GsI2KZ0UwDJqv5hjw== + version "7.0.5" + resolved "https://registry.yarnpkg.com/postcss-calc/-/postcss-calc-7.0.5.tgz#f8a6e99f12e619c2ebc23cf6c486fdc15860933e" + integrity sha512-1tKHutbGtLtEZF6PT4JSihCHfIVldU72mZ8SdZHIYriIZ9fh9k9aWSppaT8rHsyI3dX+KSR+W+Ix9BMY3AODrg== dependencies: postcss "^7.0.27" postcss-selector-parser "^6.0.2" @@ -10237,9 +10234,9 @@ postcss-less@^3.1.4: postcss "^7.0.14" postcss-load-config@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/postcss-load-config/-/postcss-load-config-2.1.0.tgz#c84d692b7bb7b41ddced94ee62e8ab31b417b003" - integrity sha512-4pV3JJVPLd5+RueiVVB+gFOAa7GWc25XQcMp86Zexzke69mKf6Nx9LRcQywdz7yZI9n1udOxmLuAwTBypypF8Q== + version "2.1.2" + resolved "https://registry.yarnpkg.com/postcss-load-config/-/postcss-load-config-2.1.2.tgz#c5ea504f2c4aef33c7359a34de3573772ad7502a" + integrity sha512-/rDeGV6vMUo3mwJZmeHfEDvwnTKKqQ0S7OHUi/kJvvtx3aWtyWG2/0ZWnzCt2keEclwN6Tf0DST2v9kITdOKYw== dependencies: cosmiconfig "^5.0.0" import-cwd "^2.0.0" @@ -10643,13 +10640,14 @@ postcss-selector-parser@^5.0.0-rc.3, postcss-selector-parser@^5.0.0-rc.4: uniq "^1.0.1" postcss-selector-parser@^6.0.0, postcss-selector-parser@^6.0.2: - version "6.0.2" - resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.2.tgz#934cf799d016c83411859e09dcecade01286ec5c" - integrity sha512-36P2QR59jDTOAiIkqEprfJDsoNrvwFei3eCqKd1Y0tUsBimsq39BLp7RD+JWny3WgB1zGhJX8XVePwm9k4wdBg== + version "6.0.4" + resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.4.tgz#56075a1380a04604c38b063ea7767a129af5c2b3" + integrity sha512-gjMeXBempyInaBqpp8gODmwZ52WaYsVOsfr4L4lDQ7n3ncD6mEyySiDtgzCT+NYC0mmeOLvtsF8iaEf0YT6dBw== dependencies: cssesc "^3.0.0" indexes-of "^1.0.1" uniq "^1.0.1" + util-deprecate "^1.0.2" postcss-svgo@^4.0.2: version "4.0.2" @@ -10694,19 +10692,10 @@ postcss-values-parser@^2.0.0, postcss-values-parser@^2.0.1: indexes-of "^1.0.1" uniq "^1.0.1" -postcss@^7.0.0, postcss@^7.0.1, postcss@^7.0.14, postcss@^7.0.17, postcss@^7.0.2, postcss@^7.0.27, postcss@^7.0.32, postcss@^7.0.5, postcss@^7.0.6: - version "7.0.32" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.32.tgz#4310d6ee347053da3433db2be492883d62cec59d" - integrity sha512-03eXong5NLnNCD05xscnGKGDZ98CyzoqPSMjOe6SuoQY7Z2hIj0Ld1g/O/UQRuOle2aRtiIRDg9tDcTGAkLfKw== - dependencies: - chalk "^2.4.2" - source-map "^0.6.1" - supports-color "^6.1.0" - -postcss@^7.0.21, postcss@^7.0.26: - version "7.0.34" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.34.tgz#f2baf57c36010df7de4009940f21532c16d65c20" - integrity sha512-H/7V2VeNScX9KE83GDrDZNiGT1m2H+UTnlinIzhjlLX9hfMUn1mHNnGeX81a1c8JSBdBvqk7c2ZOG6ZPn5itGw== +postcss@^7.0.0, postcss@^7.0.1, postcss@^7.0.14, postcss@^7.0.17, postcss@^7.0.2, postcss@^7.0.21, postcss@^7.0.26, postcss@^7.0.27, postcss@^7.0.32, postcss@^7.0.5, postcss@^7.0.6: + version "7.0.35" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.35.tgz#d2be00b998f7f211d8a276974079f2e92b970e24" + integrity sha512-3QT8bBJeX/S5zKTTjTCIjRF3If4avAT6kqxcASlTWEtAFCb9NH0OUxNDfgZSWdP5fJnBYCMEWkIFfWeugjzYMg== dependencies: chalk "^2.4.2" source-map "^0.6.1" @@ -10757,12 +10746,12 @@ pretty-format@^24.9.0: ansi-styles "^3.2.0" react-is "^16.8.4" -pretty-format@^26.4.2: - version "26.4.2" - resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-26.4.2.tgz#d081d032b398e801e2012af2df1214ef75a81237" - integrity sha512-zK6Gd8zDsEiVydOCGLkoBoZuqv8VTiHyAbKznXe/gaph/DAeZOmit9yMfgIz5adIgAMMs5XfoYSwAX3jcCO1tA== +pretty-format@^26.5.2: + version "26.5.2" + resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-26.5.2.tgz#5d896acfdaa09210683d34b6dc0e6e21423cd3e1" + integrity sha512-VizyV669eqESlkOikKJI8Ryxl/kPpbdLwNdPs2GrbQs18MpySB5S0Yo0N7zkg2xTRiFq4CFw8ct5Vg4a0xP0og== dependencies: - "@jest/types" "^26.3.0" + "@jest/types" "^26.5.2" ansi-regex "^5.0.0" ansi-styles "^4.0.0" react-is "^16.12.0" @@ -10944,9 +10933,9 @@ qs@~6.5.2: integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA== query-string@^6.8.2: - version "6.13.2" - resolved "https://registry.yarnpkg.com/query-string/-/query-string-6.13.2.tgz#3585aa9412c957cbd358fd5eaca7466f05586dda" - integrity sha512-BMmDaUiLDFU1hlM38jTFcRt7HYiGP/zt1sRzrIWm5zpeEuO1rkbPS0ELI3uehoLuuhHDCS8u8lhFN3fEN4JzPQ== + version "6.13.5" + resolved "https://registry.yarnpkg.com/query-string/-/query-string-6.13.5.tgz#99e95e2fb7021db90a6f373f990c0c814b3812d8" + integrity sha512-svk3xg9qHR39P3JlHuD7g3nRnyay5mHbrPctEBDUxUkHRifPHXJDhBUycdCC0NBjXoDf44Gb+IsOZL1Uwn8M/Q== dependencies: decode-uri-component "^0.2.0" split-on-first "^1.0.0" @@ -11158,9 +11147,9 @@ react-router@5.2.0, react-router@^5.2.0: tiny-warning "^1.0.0" react-table@^7.0.0-rc.15: - version "7.5.0" - resolved "https://registry.yarnpkg.com/react-table/-/react-table-7.5.0.tgz#87326d92cdffbbf970f49b395bcafc3f84989ddf" - integrity sha512-hLsbNqLJkkYiATVteM8mthP8y5vnLPB2qdi9FeGZjsvb2m5vxj6cavIgk35oulvBmYD6Kox0HFfI332HPZpC7w== + version "7.5.2" + resolved "https://registry.yarnpkg.com/react-table/-/react-table-7.5.2.tgz#d82ceee3d4d40b119159bce1708f084a95d3435c" + integrity sha512-qiceR/gQUOBsO/q1z1wZ3RbRvkRt39Gbzo631HiPuWmo+eTmTgaXDqLGzCmU+bOr81PB6kDxXhoWQR8hiWaUJA== react@^0.13.3: version "0.13.3" @@ -11458,9 +11447,9 @@ regexpp@^3.0.0: integrity sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q== regexpu-core@^4.7.0: - version "4.7.0" - resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-4.7.0.tgz#fcbf458c50431b0bb7b45d6967b8192d91f3d938" - integrity sha512-TQ4KXRnIn6tz6tjnrXEkD/sshygKH/j5KzK86X8MkeHyZ8qst/LZ89j3X4/8HEIfHANTFIP/AbXakeRhWIl5YQ== + version "4.7.1" + resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-4.7.1.tgz#2dea5a9a07233298fbf0db91fa9abc4c6e0f8ad6" + integrity sha512-ywH2VUraA44DZQuRKzARmw6S66mr48pQVva4LBeRhcOltJ6hExvWly5ZjFLYo67xbIxb6W1q4bAGtgfEl20zfQ== dependencies: regenerate "^1.4.0" regenerate-unicode-properties "^8.2.0" @@ -11899,11 +11888,11 @@ select-hose@^2.0.0: integrity sha1-Yl2GWPhlr0Psliv8N2o3NZpJlMo= selfsigned@^1.9.1: - version "1.10.7" - resolved "https://registry.yarnpkg.com/selfsigned/-/selfsigned-1.10.7.tgz#da5819fd049d5574f28e88a9bcc6dbc6e6f3906b" - integrity sha512-8M3wBCzeWIJnQfl43IKwOmC4H/RAp50S8DF60znzjW5GVqTcSe2vWclt7hmYVPkKPlHWOu5EaWOMZ2Y6W8ZXTA== + version "1.10.8" + resolved "https://registry.yarnpkg.com/selfsigned/-/selfsigned-1.10.8.tgz#0d17208b7d12c33f8eac85c41835f27fc3d81a30" + integrity sha512-2P4PtieJeEwVgTU9QEcwIRDQ/mXJLX8/+I3ur+Pg16nS8oNbrGxEso9NyYWy8NAmXiNl4dlAp5MwoNeCWzON4w== dependencies: - node-forge "0.9.0" + node-forge "^0.10.0" semver-diff@^2.0.0: version "2.1.0" @@ -12677,9 +12666,9 @@ stylelint-config-standard@^20.0.0: stylelint-config-recommended "^3.0.0" stylelint@^13.7.0: - version "13.7.1" - resolved "https://registry.yarnpkg.com/stylelint/-/stylelint-13.7.1.tgz#bee97ee78d778a3f1dbe3f7397b76414973e263e" - integrity sha512-qzqazcyRxrSRdmFuO0/SZOJ+LyCxYy0pwcvaOBBnl8/2VfHSMrtNIE+AnyJoyq6uKb+mt+hlgmVrvVi6G6XHfQ== + version "13.7.2" + resolved "https://registry.yarnpkg.com/stylelint/-/stylelint-13.7.2.tgz#6f3c58eea4077680ed0ceb0d064b22b100970486" + integrity sha512-mmieorkfmO+ZA6CNDu1ic9qpt4tFvH2QUB7vqXgrMVHe5ENU69q7YDq0YUg/UHLuCsZOWhUAvcMcLzLDIERzSg== dependencies: "@stylelint/postcss-css-in-js" "^0.37.2" "@stylelint/postcss-markdown" "^0.36.1" @@ -13095,9 +13084,9 @@ ts-pnp@^1.0.0: integrity sha512-csd+vJOb/gkzvcCHgTGSChYpy5f1/XKNsmvBGO4JXS+z1v2HobugDz4s1IeFXM3wZB44uczs+eazB5Q/ccdhQw== tslib@^1.10.0, tslib@^1.8.1, tslib@^1.9.0: - version "1.13.0" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.13.0.tgz#c881e13cc7015894ed914862d276436fa9a47043" - integrity sha512-i/6DQjL8Xf3be4K/E6Wgpekn5Qasl1usyw++dAA35Ue5orEn65VIxOA+YvNNl9HV3qv70T7CNwjODHZrLwvd1Q== + version "1.14.0" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.0.tgz#d624983f3e2c5e0b55307c3dd6c86acd737622c6" + integrity sha512-+Zw5lu0D9tvBMjGP8LpvMb0u2WW2QV3y+D8mO6J+cNzCYIN4sVy43Bf9vl92nqFahutN0I8zHa7cc4vihIshnw== tsutils@^3.17.1: version "3.17.1" @@ -13418,7 +13407,7 @@ use@^3.1.0: resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f" integrity sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ== -util-deprecate@^1.0.1, util-deprecate@~1.0.1: +util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=