From b3a085d85b20f5325928ed9b5962f46bdaedda10 Mon Sep 17 00:00:00 2001 From: Chi Song <27178119+squirrelsc@users.noreply.github.com> Date: Mon, 18 May 2020 16:48:27 +0800 Subject: [PATCH 01/14] Support dev install on windows (#2416) update development document. --- .../Tutorial/SetupNniDeveloperEnvironment.md | 69 ++++------ install.ps1 | 123 ++++++++++++------ uninstall.ps1 | 25 ++-- 3 files changed, 126 insertions(+), 91 deletions(-) diff --git a/docs/en_US/Tutorial/SetupNniDeveloperEnvironment.md b/docs/en_US/Tutorial/SetupNniDeveloperEnvironment.md index 00657f28e3..d4e1cfc072 100644 --- a/docs/en_US/Tutorial/SetupNniDeveloperEnvironment.md +++ b/docs/en_US/Tutorial/SetupNniDeveloperEnvironment.md @@ -1,76 +1,59 @@ -**Set up NNI developer environment** +# Setup NNI development environment -=== +NNI development environment supports Ubuntu 1604 (or above), and Windows 10 with Python3 64bit. -## Best practice for debug NNI source code +## Installation -For debugging NNI source code, your development environment should be under Ubuntu 16.04 (or above) system with python 3 and pip 3 installed, then follow the below steps. +The installation steps are similar with installing from source code. But the installation links to code directory, so that code changes can be applied to installation as easy as possible. -### 1. Clone the source code +### 1. Clone source code -Run the command - -``` +```bash git clone https://github.com/Microsoft/nni.git ``` -to clone the source code +Note, if you want to contribute code back, it needs to fork your own NNI repo, and clone from there. -### 2. Prepare the debug environment and install dependencies +### 2. Install from source code -Change directory to the source code folder, then run the command +#### Ubuntu +```bash +make dev-easy-install ``` -make install-dependencies -``` - -to install the dependent tools for the environment - -### 3. Build source code -Run the command +#### Windows +```bat +powershell -ExecutionPolicy Bypass -file install.ps1 -Development ``` -make build -``` - -to build the source code -### 4. Install NNI to development environment - -Run the command - -``` -make dev-install -``` - -to install the distribution content to development environment, and create cli scripts - -### 5. Check if the environment is ready +### 3. Check if the environment is ready Now, you can try to start an experiment to check if your environment is ready. For example, run the command -``` -nnictl create --config ~/nni/examples/trials/mnist-tfv1/config.yml +```bash +nnictl create --config examples/trials/mnist-tfv1/config.yml ``` And open WebUI to check if everything is OK -### 6. Redeploy - -After the code changes, it may need to redeploy. It depends on what kind of code changed. +### 4. Reload changes #### Python -It doesn't need to redeploy, but the nnictl may need to be restarted. +Nothing to do, the code is already linked to package folders. #### TypeScript -* If `src/nni_manager` is changed, run `yarn watch` continually under this folder. It will rebuild code instantly. The nnictl may need to be restarted to reload NNI manager. +* If `src/nni_manager` is changed, run `yarn watch` under this folder. It will watch and build code continually. The `nnictl` need to be restarted to reload NNI manager. * If `src/webui` or `src/nasui` are changed, run `yarn start` under the corresponding folder. The web UI will refresh automatically if code is changed. +### 5. Submit Pull Request + +All changes are merged to master branch from your forked repo. The description of Pull Request must be meaningful, and useful. + +We will review the changes as soon as possible. Once it passes review, we will merge it to master branch. ---- -At last, wish you have a wonderful day. -For more contribution guidelines on making PR's or issues to NNI source code, you can refer to our [Contributing](Contributing.md) document. +For more contribution guidelines and coding styles, you can refer to the [contributing document](Contributing.md). diff --git a/install.ps1 b/install.ps1 index f61a3a7046..9f03454036 100644 --- a/install.ps1 +++ b/install.ps1 @@ -1,12 +1,14 @@ +param ([Switch] $Development) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + $install_node = $true $install_yarn = $true -if([Environment]::Is64BitOperatingSystem){ +if ([Environment]::Is64BitOperatingSystem) { $OS_VERSION = 'win64' } -else{ +else { $OS_VERSION = 'win32' } # nodejs @@ -15,58 +17,58 @@ $yarnUrl = "https://yarnpkg.com/latest.tar.gz" $unzipNodeDir = "node-v*" $unzipYarnDir = "yarn-v*" -$NNI_DEPENDENCY_FOLDER = [System.IO.Path]::GetTempPath()+$env:USERNAME +$NNI_DEPENDENCY_FOLDER = [System.IO.Path]::GetTempPath() + $env:USERNAME $WHICH_PYTHON = where.exe python -if($WHICH_PYTHON -eq $null){ +if ($WHICH_PYTHON -eq $null) { throw "Can not find python" } -else{ +else { $pyVersion = & python -V 2>&1 - $pyVersion = ([string]$pyVersion).substring(7,3) - if([double]$pyVersion -lt 3.5){ + $pyVersion = ([string]$pyVersion).substring(7, 3) + if ([double]$pyVersion -lt 3.5) { throw "python version should >= 3.5" } } $WHICH_PIP = where.exe pip -if($WHICH_PIP -eq $null){ +if ($WHICH_PIP -eq $null) { throw "Can not find pip" } $env:PYTHONIOENCODING = "UTF-8" -if($env:VIRTUAL_ENV){ +if ($env:VIRTUAL_ENV) { $NNI_PYTHON3 = $env:VIRTUAL_ENV + "\Scripts" $NNI_PKG_FOLDER = $env:VIRTUAL_ENV + "\nni" $NNI_PYTHON_SCRIPTS = $NNI_PYTHON3 } -else{ +else { $NNI_PYTHON3 = $(python -c 'import site; from pathlib import Path; print(Path(site.getsitepackages()[0]))') $NNI_PKG_FOLDER = $NNI_PYTHON3 + "\nni" - $NNI_PYTHON_SCRIPTS = $NNI_PYTHON3 + "\Scripts" + $NNI_PYTHON_SCRIPTS = $NNI_PYTHON3 + "\Scripts" } -$PIP_INSTALL = """$NNI_PYTHON3\python"" -m pip install ." +$PIP_INSTALL = """$NNI_PYTHON3\python"" -m pip install " -if(!(Test-Path $NNI_DEPENDENCY_FOLDER)){ +if (!(Test-Path $NNI_DEPENDENCY_FOLDER)) { New-Item $NNI_DEPENDENCY_FOLDER -ItemType Directory } -$NNI_NODE_ZIP = $NNI_DEPENDENCY_FOLDER+"\nni-node.zip" -$NNI_NODE_FOLDER = $NNI_DEPENDENCY_FOLDER+"\nni-node" -$NNI_YARN_TARBALL = $NNI_DEPENDENCY_FOLDER+"\nni-yarn.tar.gz" -$NNI_YARN_FOLDER = $NNI_DEPENDENCY_FOLDER+"\nni-yarn" -$NNI_YARN = $NNI_YARN_FOLDER +"\bin\yarn" +$NNI_NODE_ZIP = $NNI_DEPENDENCY_FOLDER + "\nni-node.zip" +$NNI_NODE_FOLDER = $NNI_DEPENDENCY_FOLDER + "\nni-node" +$NNI_YARN_TARBALL = $NNI_DEPENDENCY_FOLDER + "\nni-yarn.tar.gz" +$NNI_YARN_FOLDER = $NNI_DEPENDENCY_FOLDER + "\nni-yarn" +$NNI_YARN = $NNI_YARN_FOLDER + "\bin\yarn" ## Version number $NNI_VERSION_VALUE = $(git describe --tags) $NNI_VERSION_TEMPLATE = "999.0.0-developing" -if(!(Test-Path $NNI_NODE_ZIP)){ +if (!(Test-Path $NNI_NODE_ZIP)) { Write-Host "Downloading Node..." (New-Object Net.WebClient).DownloadFile($nodeUrl, $NNI_NODE_ZIP) } -if(!(Test-Path $NNI_YARN_TARBALL)){ +if (!(Test-Path $NNI_YARN_TARBALL)) { Write-Host "Downloading Yarn..." (New-Object Net.WebClient).DownloadFile($yarnUrl, $NNI_YARN_TARBALL) } @@ -74,27 +76,30 @@ if(!(Test-Path $NNI_YARN_TARBALL)){ $NNI_YARN_TARBALL = $NNI_YARN_TARBALL -split '\\' -join '\\' $NNI_DEPENDENCY_FOLDER = $NNI_DEPENDENCY_FOLDER -split '\\' -join '\\' $SCRIPT_PATH = $NNI_DEPENDENCY_FOLDER + '\extract.py' -$SCRIPT = "import tarfile", - ("tar = tarfile.open(""{0}"")" -f $NNI_YARN_TARBALL), - ("tar.extractall(""{0}"")" -f $NNI_DEPENDENCY_FOLDER), +$SCRIPT = "import tarfile", + ("tar = tarfile.open(""{0}"")" -f $NNI_YARN_TARBALL), + ("tar.extractall(""{0}"")" -f $NNI_DEPENDENCY_FOLDER), "tar.close()" [System.IO.File]::WriteAllLines($SCRIPT_PATH, $SCRIPT) Add-Type -AssemblyName System.IO.Compression.FileSystem -function Unzip{ +function Unzip { param([string]$zipfile, [string]$outpath) [System.IO.Compression.ZipFile]::ExtractToDirectory($zipfile, $outpath) } if ($install_node) { ### nodejs install - if(!(Test-Path $NNI_NODE_FOLDER)){ + if (!(Test-Path $NNI_NODE_FOLDER)) { Unzip $NNI_NODE_ZIP $NNI_DEPENDENCY_FOLDER $unzipNodeDir = Get-ChildItem "$NNI_DEPENDENCY_FOLDER\$unzipNodeDir" Rename-Item $unzipNodeDir "nni-node" } Copy-Item "$NNI_NODE_FOLDER\node.exe" $NNI_PYTHON_SCRIPTS -Recurse -Force +} + +if ($install_yarn) { ### yarn install - if(!(Test-Path $NNI_YARN_FOLDER)){ + if (!(Test-Path $NNI_YARN_FOLDER)) { cmd /C """$NNI_PYTHON3\python""" $SCRIPT_PATH $unzipYarnDir = Get-ChildItem "$NNI_DEPENDENCY_FOLDER\$unzipYarnDir" Rename-Item $unzipYarnDir "nni-yarn" @@ -104,10 +109,40 @@ if ($install_node) { ## install-python-modules: ### Installing Python SDK (Get-Content setup.py).replace($NNI_VERSION_TEMPLATE, $NNI_VERSION_VALUE) | Set-Content setup.py -cmd /c $PIP_INSTALL + +if ($Development) { + $PYTHON_BUILD = "build" + if (Test-Path $PYTHON_BUILD) { + # To compat with file and links. + cmd /c rmdir /s /q $PYTHON_BUILD + } + New-Item $PYTHON_BUILD -ItemType Directory + New-Item -ItemType Junction -Path "$($PYTHON_BUILD)\nni" -Target "src\sdk\pynni\nni" + New-Item -ItemType Junction -Path "$($PYTHON_BUILD)\nnicli" -Target "src\sdk\pycli\nnicli" + New-Item -ItemType Junction -Path "$($PYTHON_BUILD)\nni_annotation" -Target "tools\nni_annotation" + New-Item -ItemType Junction -Path "$($PYTHON_BUILD)\nni_cmd" -Target "tools\nni_cmd" + New-Item -ItemType Junction -Path "$($PYTHON_BUILD)\nni_trial_tool" -Target "tools\nni_trial_tool" + New-Item -ItemType Junction -Path "$($PYTHON_BUILD)\nni_gpu_tool" -Target "tools\nni_gpu_tool" + + Copy-Item setup.py $PYTHON_BUILD + Copy-Item README.md $PYTHON_BUILD + + Push-Location build + #update folders in setup file + (Get-Content setup.py).replace("src/sdk/pynni/", "") | Set-Content setup.py + (Get-Content setup.py).replace("src/sdk/pycli/", "") | Set-Content setup.py + (Get-Content setup.py).replace("src/sdk/pynni", ".") | Set-Content setup.py + (Get-Content setup.py).replace("tools/", "") | Set-Content setup.py + # install current folder. + cmd /c $PIP_INSTALL -e . + Pop-Location +} +else { + cmd /c $PIP_INSTALL . +} # Building NNI Manager -$env:PATH=$NNI_PYTHON_SCRIPTS+';'+$env:PATH +$env:PATH = $NNI_PYTHON_SCRIPTS + ';' + $env:PATH cd src\nni_manager cmd /c $NNI_YARN cmd /c $NNI_YARN build @@ -124,17 +159,31 @@ cmd /c $NNI_YARN build cd ..\.. ## install-node-modules -if(!(Test-Path $NNI_PKG_FOLDER)){ - New-Item $NNI_PKG_FOLDER -ItemType Directory +if (Test-Path $NNI_PKG_FOLDER) { + # it needs to remove the whole folder for following copy. + cmd /c rmdir /s /q $NNI_PKG_FOLDER +} + +$NNI_PKG_FOLDER_STATIC = $NNI_PKG_FOLDER + "\static" +$NASUI_PKG_FOLDER = $NNI_PKG_FOLDER + "\nasui" + +if ($Development) { + New-Item -ItemType Junction -Path $($NNI_PKG_FOLDER) -Target "src\nni_manager\dist" + New-Item -ItemType Junction -Path "$($NNI_PKG_FOLDER)\node_modules" -Target "src\nni_manager\node_modules" + New-Item -ItemType Junction -Path $($NNI_PKG_FOLDER_STATIC) -Target "src\webui\build" + New-Item -ItemType Junction -Path $($NASUI_PKG_FOLDER) -Target "src\nasui\build" } -Remove-Item $NNI_PKG_FOLDER -Recurse -Force -Copy-Item "src\nni_manager\dist" $NNI_PKG_FOLDER -Recurse +else { + Copy-Item "src\nni_manager\dist" $NNI_PKG_FOLDER -Recurse + Copy-Item "src\webui\build" $NNI_PKG_FOLDER_STATIC -Recurse + Copy-Item "src\nasui\build" $NASUI_PKG_FOLDER -Recurse +} + Copy-Item "src\nni_manager\package.json" $NNI_PKG_FOLDER $PKG_JSON = $NNI_PKG_FOLDER + "\package.json" (Get-Content $PKG_JSON).replace($NNI_VERSION_TEMPLATE, $NNI_VERSION_VALUE) | Set-Content $PKG_JSON -cmd /c $NNI_YARN --prod --cwd $NNI_PKG_FOLDER -$NNI_PKG_FOLDER_STATIC = $NNI_PKG_FOLDER + "\static" -$NASUI_PKG_FOLDER = $NNI_PKG_FOLDER + "\nasui" -Copy-Item "src\webui\build" $NNI_PKG_FOLDER_STATIC -Recurse -Copy-Item "src\nasui\build" $NASUI_PKG_FOLDER -Recurse Copy-Item "src\nasui\server.js" $NASUI_PKG_FOLDER -Recurse + +if (!$Development) { + cmd /c $NNI_YARN --prod --cwd $NNI_PKG_FOLDER +} diff --git a/uninstall.ps1 b/uninstall.ps1 index 0da8f2d7f6..29f8e23483 100644 --- a/uninstall.ps1 +++ b/uninstall.ps1 @@ -4,12 +4,12 @@ $env:PYTHONIOENCODING = "UTF-8" if($env:VIRTUAL_ENV){ $NNI_PYTHON3 = $env:VIRTUAL_ENV + "\Scripts" $NNI_PKG_FOLDER = $env:VIRTUAL_ENV + "\nni" - Remove-Item "$NNI_PYTHON3\node.exe" -Force + cmd /c del "$NNI_PYTHON3\node.exe" } else{ $NNI_PYTHON3 = $(python -c 'import site; from pathlib import Path; print(Path(site.getsitepackages()[0]))') $NNI_PKG_FOLDER = $NNI_PYTHON3 + "\nni" - Remove-Item "$NNI_PYTHON3\Scripts\node.exe" -Force + cmd /c del "$NNI_PYTHON3\Scripts\node.exe" } $PIP_UNINSTALL = """$NNI_PYTHON3\python"" -m pip uninstall -y " @@ -17,13 +17,16 @@ $NNI_NODE_FOLDER = $NNI_DEPENDENCY_FOLDER+"\nni-node" $NNI_YARN_FOLDER = $NNI_DEPENDENCY_FOLDER+"\nni-yarn" # uninstall -Remove-Item $NNI_PKG_FOLDER -Recurse -Force -cmd /C $PIP_UNINSTALL "nni" +cmd /c rmdir /s /q $NNI_PKG_FOLDER +cmd /c $PIP_UNINSTALL "nni" -# clean -Remove-Item "src/nni_manager/dist" -Recurse -Force -Remove-Item "src/nni_manager/node_modules" -Recurse -Force -Remove-Item "src/webui/build" -Recurse -Force -Remove-Item "src/webui/node_modules" -Recurse -Force -Remove-Item $NNI_YARN_FOLDER -Recurse -Force -Remove-Item $NNI_NODE_FOLDER -Recurse -Force +# clean up +cmd /c rmdir /s /q "build" +cmd /c rmdir /s /q "src\nni_manager\dist" +cmd /c rmdir /s /q "src\nni_manager\node_modules" +cmd /c rmdir /s /q "src\webui\build" +cmd /c rmdir /s /q "src\webui\node_modules" +cmd /c rmdir /s /q "src\nasui\build" +cmd /c rmdir /s /q "src\nasui\node_modules" +cmd /c rmdir /s /q $NNI_YARN_FOLDER +cmd /c rmdir /s /q $NNI_NODE_FOLDER From 6f19d3ca185ea012d715d1ba24fac714486c382e Mon Sep 17 00:00:00 2001 From: liuzhe-lz <40699903+liuzhe-lz@users.noreply.github.com> Date: Mon, 18 May 2020 19:00:18 +0800 Subject: [PATCH 02/14] fix security alert (#2443) --- src/webui/package.json | 2 +- src/webui/yarn.lock | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/webui/package.json b/src/webui/package.json index d7400e4f39..ca89200352 100644 --- a/src/webui/package.json +++ b/src/webui/package.json @@ -65,7 +65,6 @@ "@typescript-eslint/eslint-plugin": "^2.11.0", "@typescript-eslint/parser": "^2.11.0", "@uifabric/fluent-theme": "^0.16.7", - "npx": "^10.2.0", "eslint": "^5.16.0", "eslint-config-react-app": "^4.0.0", "eslint-loader": "2.1.2", @@ -74,6 +73,7 @@ "eslint-plugin-jsx-a11y": "6.2.1", "eslint-plugin-react": "7.12.4", "eslint-plugin-react-hooks": "^1.5.0", + "npx": "^10.2.0", "typescript": "3.4.5" }, "scripts": { diff --git a/src/webui/yarn.lock b/src/webui/yarn.lock index 3053a14da9..f6b3ed1cc8 100644 --- a/src/webui/yarn.lock +++ b/src/webui/yarn.lock @@ -5371,8 +5371,9 @@ kind-of@^5.0.0: resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-5.1.0.tgz#729c91e2d857b7a419a1f9aa65685c4c33f5845d" kind-of@^6.0.0, kind-of@^6.0.2: - version "6.0.2" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.2.tgz#01146b36a6218e64e58f3a8d66de5d7fc6f6d051" + version "6.0.3" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" + integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== last-call-webpack-plugin@^3.0.0: version "3.0.0" From 58cd8c76e1b42e049dab818715288c1d4c8d06c1 Mon Sep 17 00:00:00 2001 From: liuzhe-lz <40699903+liuzhe-lz@users.noreply.github.com> Date: Tue, 19 May 2020 08:18:29 +0800 Subject: [PATCH 03/14] Increase IPC message length to 10^14 (#2425) --- src/nni_manager/core/ipcInterface.ts | 14 ++++------- src/nni_manager/core/test/assessor.py | 4 ++-- .../core/test/ipcInterface.test.ts | 24 +++---------------- src/sdk/pynni/nni/protocol.py | 7 +++--- src/sdk/pynni/tests/test_protocol.py | 17 ++++--------- 5 files changed, 17 insertions(+), 49 deletions(-) diff --git a/src/nni_manager/core/ipcInterface.ts b/src/nni_manager/core/ipcInterface.ts index 340af109bc..e7c45beec6 100644 --- a/src/nni_manager/core/ipcInterface.ts +++ b/src/nni_manager/core/ipcInterface.ts @@ -23,11 +23,7 @@ const ipcIncomingFd: number = 4; */ function encodeCommand(commandType: string, content: string): Buffer { const contentBuffer: Buffer = Buffer.from(content); - if (contentBuffer.length >= 1_000_000) { - throw new RangeError('Command too long'); - } - const contentLengthBuffer: Buffer = Buffer.from(contentBuffer.length.toString().padStart(6, '0')); - + const contentLengthBuffer: Buffer = Buffer.from(contentBuffer.length.toString().padStart(14, '0')); return Buffer.concat([Buffer.from(commandType), contentLengthBuffer, contentBuffer]); } @@ -43,12 +39,12 @@ function decodeCommand(data: Buffer): [boolean, string, string, Buffer] { return [false, '', '', data]; } const commandType: string = data.slice(0, 2).toString(); - const contentLength: number = parseInt(data.slice(2, 8).toString(), 10); - if (data.length < contentLength + 8) { + const contentLength: number = parseInt(data.slice(2, 16).toString(), 10); + if (data.length < contentLength + 16) { return [false, '', '', data]; } - const content: string = data.slice(8, contentLength + 8).toString(); - const remain: Buffer = data.slice(contentLength + 8); + const content: string = data.slice(16, contentLength + 16).toString(); + const remain: Buffer = data.slice(contentLength + 16); return [true, commandType, content, remain]; } diff --git a/src/nni_manager/core/test/assessor.py b/src/nni_manager/core/test/assessor.py index 50d1949da3..004283cb51 100644 --- a/src/nni_manager/core/test/assessor.py +++ b/src/nni_manager/core/test/assessor.py @@ -8,13 +8,13 @@ def send(command, data): command = command.encode('utf8') data = data.encode('utf8') - msg = b'%b%06d%b' % (command, len(data), data) + msg = b'%b%14d%b' % (command, len(data), data) _out_file.write(msg) _out_file.flush() def receive(): - header = _in_file.read(8) + header = _in_file.read(16) l = int(header[2:]) command = header[:2].decode('utf8') data = _in_file.read(l).decode('utf8') diff --git a/src/nni_manager/core/test/ipcInterface.test.ts b/src/nni_manager/core/test/ipcInterface.test.ts index 1742303379..4ddeda8a1f 100644 --- a/src/nni_manager/core/test/ipcInterface.test.ts +++ b/src/nni_manager/core/test/ipcInterface.test.ts @@ -14,7 +14,6 @@ import { NNIError } from '../../common/errors'; let sentCommands: { [key: string]: string }[] = []; const receivedCommands: { [key: string]: string }[] = []; -let commandTooLong: Error | undefined; let rejectCommandType: Error | undefined; function runProcess(): Promise { @@ -54,14 +53,7 @@ function runProcess(): Promise { // Command #2: ok dispatcher.sendCommand('ME', '123'); - // Command #3: too long - try { - dispatcher.sendCommand('ME', 'x'.repeat(1_000_000)); - } catch (error) { - commandTooLong = error; - } - - // Command #4: FE is not tuner/assessor command, test the exception type of send non-valid command + // Command #3: FE is not tuner/assessor command, test the exception type of send non-valid command try { dispatcher.sendCommand('FE', '1'); } catch (error) { @@ -88,21 +80,11 @@ describe('core/protocol', (): void => { }); it('sendCommand() should work without content', (): void => { - assert.equal(sentCommands[0], '(\'IN\', \'\')'); + assert.equal(sentCommands[0], "('IN', '')"); }); it('sendCommand() should work with content', (): void => { - assert.equal(sentCommands[1], '(\'ME\', \'123\')'); - }); - - it('sendCommand() should throw on too long command', (): void => { - if (commandTooLong === undefined) { - assert.fail('Should throw error') - } else { - const err: Error | undefined = (commandTooLong).cause; - assert(err && err.name === 'RangeError'); - assert(err && err.message === 'Command too long'); - } + assert.equal(sentCommands[1], "('ME', '123')"); }); it('sendCommand() should throw on wrong command type', (): void => { diff --git a/src/sdk/pynni/nni/protocol.py b/src/sdk/pynni/nni/protocol.py index e7330f78cd..ca2d7069d2 100644 --- a/src/sdk/pynni/nni/protocol.py +++ b/src/sdk/pynni/nni/protocol.py @@ -43,8 +43,7 @@ def send(command, data): try: _lock.acquire() data = data.encode('utf8') - assert len(data) < 1000000, 'Command too long' - msg = b'%b%06d%b' % (command.value, len(data), data) + msg = b'%b%014d%b' % (command.value, len(data), data) logging.getLogger(__name__).debug('Sending command, data: [%s]', msg) _out_file.write(msg) _out_file.flush() @@ -56,9 +55,9 @@ def receive(): """Receive a command from Training Service. Returns a tuple of command (CommandType) and payload (str) """ - header = _in_file.read(8) + header = _in_file.read(16) logging.getLogger(__name__).debug('Received command, header: [%s]', header) - if header is None or len(header) < 8: + if header is None or len(header) < 16: # Pipe EOF encountered logging.getLogger(__name__).debug('Pipe EOF encountered') return None, None diff --git a/src/sdk/pynni/tests/test_protocol.py b/src/sdk/pynni/tests/test_protocol.py index cd59144624..5c9ee78eaf 100644 --- a/src/sdk/pynni/tests/test_protocol.py +++ b/src/sdk/pynni/tests/test_protocol.py @@ -20,30 +20,21 @@ class ProtocolTestCase(TestCase): def test_send_en(self): out_file = _prepare_send() send(CommandType.NewTrialJob, 'CONTENT') - self.assertEqual(out_file.getvalue(), b'TR000007CONTENT') + self.assertEqual(out_file.getvalue(), b'TR00000000000007CONTENT') def test_send_zh(self): out_file = _prepare_send() send(CommandType.NewTrialJob, '你好') - self.assertEqual(out_file.getvalue(), 'TR000006你好'.encode('utf8')) - - def test_send_too_large(self): - _prepare_send() - exception = None - try: - send(CommandType.NewTrialJob, ' ' * 1000000) - except AssertionError as e: - exception = e - self.assertIsNotNone(exception) + self.assertEqual(out_file.getvalue(), 'TR00000000000006你好'.encode('utf8')) def test_receive_en(self): - _prepare_receive(b'IN000005hello') + _prepare_receive(b'IN00000000000005hello') command, data = receive() self.assertIs(command, CommandType.Initialize) self.assertEqual(data, 'hello') def test_receive_zh(self): - _prepare_receive('IN000006世界'.encode('utf8')) + _prepare_receive('IN00000000000006世界'.encode('utf8')) command, data = receive() self.assertIs(command, CommandType.Initialize) self.assertEqual(data, '世界') From 10f5bc5ea639b3db5c50c68b2171d33c49837662 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Tue, 19 May 2020 10:47:40 +0800 Subject: [PATCH 04/14] Update doc for PAI mode (#2444) --- docs/en_US/TrainingService/PaiMode.md | 12 ++++++------ docs/img/pai_job_submission_page.jpg | Bin 127488 -> 39763 bytes docs/img/pai_profile.jpg | Bin 0 -> 18093 bytes docs/img/pai_token.jpg | Bin 0 -> 18383 bytes docs/img/pai_token_profile.jpg | Bin 55722 -> 0 bytes 5 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 docs/img/pai_profile.jpg create mode 100644 docs/img/pai_token.jpg delete mode 100644 docs/img/pai_token_profile.jpg diff --git a/docs/en_US/TrainingService/PaiMode.md b/docs/en_US/TrainingService/PaiMode.md index c608cc970a..53046cdfcd 100644 --- a/docs/en_US/TrainingService/PaiMode.md +++ b/docs/en_US/TrainingService/PaiMode.md @@ -7,9 +7,9 @@ Step 1. Install NNI, follow the install guide [here](../Tutorial/QuickStart.md). Step 2. Get PAI token. Click `My profile` button in the top-right side of PAI's webprotal. -![](../../img/pai_token_button.jpg) -Find the token management region, copy one of the token as your account token. -![](../../img/pai_token_profile.jpg) +![](../../img/pai_profile.jpg) +Click `copy` button in the page to copy a jwt token. +![](../../img/pai_token.jpg) Step 3. Mount NFS storage to local machine. Click `Submit job` button in PAI's webportal. @@ -19,7 +19,7 @@ Step 3. Mount NFS storage to local machine. The `DEFAULT_STORAGE`field is the path to be mounted in PAI's container when a job is started. The `Preview container paths` is the NFS host and path that PAI provided, you need to mount the corresponding host and path to your local machine first, then NNI could use the PAI's NFS storage. For example, use the following command: ``` -sudo mount nfs://gcr-openpai-infra02:/pai/data /local/mnt +sudo mount -t nfs4 gcr-openpai-infra02:/pai/data /local/mnt ``` Then the `/data` folder in container will be mounted to `/local/mnt` folder in your local machine. You could use the following configuration in your NNI's config file: @@ -66,7 +66,7 @@ trial: virtualCluster: default nniManagerNFSMountPath: /home/user/mnt containerNFSMountPath: /mnt/data/user - paiStoragePlugin: team_wise + paiStoragePlugin: teamwise_storage # Configuration to access OpenPAI Cluster paiConfig: userName: your_pai_nni_user @@ -74,7 +74,7 @@ paiConfig: host: 10.1.1.1 ``` -Note: You should set `trainingServicePlatform: pai` in NNI config YAML file if you want to start experiment in pai mode. +Note: You should set `trainingServicePlatform: pai` in NNI config YAML file if you want to start experiment in pai mode. The host field in configuration file is PAI's job submission page uri, like `10.10.5.1`, the default http protocol in NNI is `http`, if your PAI's cluster enabled https, please use the uri in `https://10.10.5.1` format. Compared with [LocalMode](LocalMode.md) and [RemoteMachineMode](RemoteMachineMode.md), trial configuration in pai mode have these additional keys: * cpuNum diff --git a/docs/img/pai_job_submission_page.jpg b/docs/img/pai_job_submission_page.jpg index f49a1c267eea2a48b986e52c97180b06a524264d..377a66f593c937b1b264d9e838ddb69b96320dd4 100644 GIT binary patch literal 39763 zcmeFZcTkgQ*f)y0iY^GNAgF+>q9R?2w1mX9p&&#lBHcn21eB5hA<;!)0fAKzq$Cj) zA|Sm*TBM0mMVi!@ARsja2!xPwp5VG0-2GXYJ zc7j%}UnBgYGg`&Z?c?t^H(|o&4oO$PJRbF7tDleo)t1^%?^`u5Az4-ggiKZ~S3q!$ z*H`R5T9!-=AC72u zt;T>I_*bGWZykldtdA*Y50GWhv@3|l649{E+;TM*OEyF^u)k#9cYJ6siWa*RMk10q zb9hpaB5O%<_8iu9{V;uz_+H0+9hMr-MiGNzmxnVWMmvcW%<4HIS$oHo#qtHO?Zn&U z8L}LDtt=Q@i9ffrc!z#809-ZcPd@emzH6Vss z^du+;_D8>yAxFW4bB)_I4Py?PYQ7^8b#=QBkhpxN7O0s>QAjYgRjtyjWDAKNY)&tk zkC9mpcdYiwL`J4}8geFU=2lJ`)|GJQ>YTKT5m|v_cLhC0JHe57Ir5}j`~60=p9Xkw zIbww9rv)~*e=8>%4v*ePbXyq{3bhrh9f3zn1g8Aij)a0C1W1UQ$&s3M9^v7l+R#bt zXSU8Q@unp%)i@PsnG7$zmLt<8k#ekI^lJFrN-!mtj@r$k$Zo;i&f=iwNP$s!t6B~w zv`*DvH)OFF7T%aWGpQUCR6?PwubG3S$d&V zrKz%#wF{!)qq=sq;#qOaVQG=zSQYye*V*8s)6ANWXj$}PN4^R{|u4JvI+9A~`Cg=u!%6Vm} z;nCFbPpB`WK9$_-#;2Q{MK8%tO|n-%3f7vCf}MSk?XeERIwAfOEQrrk$J~XP;AAX% zaw0z@c;>cQrv1{(v{kQQ>TOxj4y+^Ro^`uY-o5Y`utZx$aR91H*q$^a8eRq6ZP!Qb zrsB4}PV0{*N8~RXmxXB(H5rSY?QgheVNG5DcEpMwZn~9 zsg2Bl61X9O*^!kLcu0B}eX!EQXRxCB(yD0|2an$Z)m+Sc9!I7H?-O=C9F>ZdH2^Wh zJ7vg~6eR;zAuLd+Sw_HWf@rpL`nx?Umo6CIP7vB!C5=qhSwK#pLB&ABRWS}UkWnEx;m9!!blQk8Gh0aenqMcxf0axyR zYwd3HHhJus%6TKiSOLDSyEhg^g=Hbi`8DoWi+iMt?N?(1biazC0u{=`G2GpI&${fOJ+Vb94)oc;23C*e6r>c>HN zNm*r7#W2|;gffx)+to`xv|?6>r8$x^kx`S&aZ$lu3YaQO;GuE}&XPYUukD?CNCFn| zF|+A)**!EJi01?F2Wq%W0VT`Fi3ny?9aI`wo5|rCbL~B8TDhsaS25=!et(I*)jOd{ zavVN1IJctWJPN|zfLi&`{Km?%sh5m~mP@FJtX<|syI_KKCdRE-xMC)@=J?DRFYm9_ zKP&(MWGHk=42G8BbccJ14fGX;lVS1wqiTiH6Si zK8&7C#^Lrn@9Ju*9(ubAMh=E`LG~Ye_i4({kT4c833l8TzMO|FpR{)j)(1QIYd{hD z$fwMt-=pxo4$RBCcJ?e|wR=+Dy9eR2kY7|^kB`>iD=2Vt85>SgQY8M^&lTy?!WF2a zx*fFf+NDXKAM^#I`FWKin4`X4+n~#2gJ%5dYaSaT(3j?V+V^?6(kzs;pYcF+n0tcJ z5k;nb$z8gEdlSLL=gfF{ee^G1tjXsZvgNE^E!tMe2C5Gdo(C`=Md2MxE8It!^loqC zb{aJVo_=b|Ie{`!xtc{KC9^<~1yHwqzG+yfj;M&$dNl%4B>K$GS~he_ahp}h{^N&C z(Zf2mYN+YbIVy)$JvY@0Krsalo*}sAP$t}x8qC$GpBf*&zweQ1^0}BXPDyV?Bj;j|VpC%}}eT%|rT3h{4z}eUl@YEY@H+aR-X_yzD+e zY<Os*hK#A=9b!El7aj|4jK@x23udb}XzJKkVFo*tW`ok#pE#U7%v zC%M&XsX-GMGRdn#lSK?#DI4IR=D14SkV!5a&A>rX!ShJ|Ji*CVBaxp*0pG1#`2U4W z>Hl`B_6WVS*a;Kdp`Zq3cEs{+MnW-C{^5zZpxVfPQn`5PhPbn7=M1i%{->c)4CUO5 z4==F!d;&dWNIPieW(K?bPuBPF(R0L31+7f^SN{xm_C3G{#gn_lF8wpyi!ZR!i zru5HvX14)$eE%^0-mZVfQ_%bW`0}J|zy<}}n7PN^1_@x(Q(-?yGBsKWD>5X3qAnBn z4zpI#yC8YUoz|K^$fxfKz?<~hmITfRcp^6#JK8yG4BO`F0C(yp8|H#Efz)f(GCWH8 z#1FELBO!HESQjM4*fty@e{W>ja^iL3;F&tj^w8$)saHEd0GWxZ%XaGDi`K zGMlg%{V+Vojlkc$0Lc+Q9ic1<3;mesaB8Nn2h?h0ZSnZ*z>+YIdb=-PpKaZHUyhCz z-DQ8ENd5Ns^q;fWT0raQJ6w?sR7-j4Cpa06lb!RixCo+7;I#|Kob+BJlFHb+OW0t%P^Mj7L9u zBxnW==H@RvG!*1IpfeP)*CH%d{mdRS+jD{6t&c5g(kTiY_)MH-^Pb@l+`Cslc!g;O z6=9DsACJ;L~w3s}cB!26I+@+@o{zhkn@LkODS?9fQw|!q-fQ;hybG zGW_HV-MBcs-7lda!lviznDMT!>5?%!JG=P8YR{ocU?(i}SEGaA>$N zTThu-8OMo288xC?fNC7@^W^c`NKI*}nUXNW*nKkVX8=CG z2;RVJ_#ys#!-(|Gg6oaqFy5yEE$7!J9KFzXt71gdi)x}3}W1#Cv?0TFFr7I zHcPr`PlMQ+a!r6d>}M1H4@zDjm}dy{A^3adrkV4@KBKc;6PL!N(7>-vvO|A@y)ve& zH7emUiNyHyDQzUbor)&c~jSu{^lTnS$?W2jtoNp>}Jg2*cHX7Etb!FV4zCOX;0Qp7keK=*4SB%CGGCSIC3WottM&F9hl2B#gk!jIn^wSGm&_X zb5FbP7ya!`rD+2L+@wC}%AwIimLj7MDBE}g-^u8|2)4|G9I{Xf0udcX;bRRJ;r;g_7#3T_xPmVbHtkGKF!nsk`$e?d!6WI zuB7S;vu2St)ij3>bphb=wN@@-G7&G|@7wc?e8sFcgn-I32wJN%l?(zGln^=9}>pMjF0R?r^ylpWDjuFTy> zW@>zZn3^`F5QXIzc(N}|y!m-vSnom2T#Fl*!Ip>4t31a^MB;w6(VGoRd*+t8ySxy6 zd460<0!E)IyPfcqrH0e-GLWYC-`!rNt?R-#$ZIL%`$aQQQdmhY5%I z<-f*o_&?BoiHPK4`q{%isKBY*e0cZ2oT31+^WkBB6#$7Wz|Qz|^8c?7GXJKs{PvK( z7Et)WdWETqTWj6z%56%kFV~W>v^U z?#Oa*_|UFg#t`5a=O5c&h0P=J@31$?@kO~phGvb2ZhtA)tW!vwHQgs~c7h*MZ*Xu3 zW+SNXkZhh2zMl5_8RAjGdKfCoHfV?|R_0$rH^qMD2O6pu0-zq3sNZ^FM+wN&meXfm zo75|lDI@;~x~}I(`muu2PNbp!}3P~DjP9-^<1!SI8gIlGQ~Grv@k)2_=48uru= z%?1f$kQ_NO%TD3Jt21w=2 z@q3Iay(jUYow*x$C-j=h^LxyZ)U3`f5l>0UKmUoOYvZwYmPi#$kclZgllhL7VB*R%mD z{8XrL7YsH%v#%QWo@mJ2NL_K}oK99){&E8R3TSX zvHnmFxmd`1%5j`c0;%hg|Gf$s2pbDfec9B7?)z;@WSaNBt+KoAVspw&t=^j0 z3mYu6^L0?+9~(+YGb_P$Nr^*ya#zO0$SdkZAen+L_%N_UUwupz_DCw+A6InuQ)^Q;!E(lSkCo-ZqpNa4n=8G1LJ|T7bjDD&mJhb$ycp%G zGmpFmu-le9vN|(FjAe9A(~2(9v}laUWm~A{m!MbWYGWyXYp;z4K$5LsRJZy#NkVmi zJ~YP)>Y0kAw~ajO4?I}tzx;J-L?IHN)CT{YP7O=pczVyfW+{hcxkDPvbvZttAV-9+ zsjjZ7Z|)FV4imK)w^cg8M181GP0USgBp^R4!CtBAYly>Qsr^<;of%y5PyG#@&EzzD zc-V4J8z@N#a0Fr*^R8|Xr;++Pm=%a2stv~L#ufmX6c$tNhGb<7B%o*Qb-?Pne)O9N z4pp=zKq?t*C~)VQvm*BGFY6?G=7vLdTQ_7%cJe~1$9%^$egz^By+ zh#ljVBGv^$!p0fanc(bJ`Z(>52msfO1?!Ggd2$zTvT=@zW)53LLqJ$+9XZM>HShL9@}2GHBKgfBuP z?IUyj#5af9L(p@KC)C16@NK4$9l4piN++JtCVWO@^HLk5m8v?k z20n0ya+bBV0%HUO8V2fN;D&j3S4ONX`v#*bZxD(s_s?*2Su}pC7^-pQm4>j;FOUv+0mzQrN++3^jGr7q2 z$|(Gk;Sf08-ms45A0QzXg-@Fb^w9Mlhg(VdK@mqi^L8oHVLzXK-n&o*+Gj`KR@I^5 zKc(ZiTUs-RIiksyfzfxWTWj^mik^(^2pkMM3?Ryt9iG4;m`ccGru&4rX6~J@lIkb# zcm2vfaVTL+#0c~BiSnhEP_PVOG%GfPV7ob0c=Ott+jox#?!sP{Ou`&aYQbCiz*PT!txTlDsST(d$0O z-VOYoc-*X_4WbqLJew z@rTDjh0PVlUbt!>H#_is$=ci*rSXDE^P?k?f20k7COQ@9lpA04+o}-B?A7YSyI|o& z4YLGc5!qwKN)-#iW%z(ZVGMUg@a^=NS2^rybXfpR91AjAwdk6BR^V_xTEg*U*;Nnf z63_I4W;X~#{c1~FT$CeiLoapQkPHe%F81V8A5NNIAc;68otzLC6Z+$eyyM{nq34Hc zeay!D)UF-oU6^sQoVW;L+uVO=9SQ@R9C5q$*1i$3(#=(qfvl9qC}Fb}g)$}?O!qjb z74cO+`=Si-3~*Ep%I2Vb7?T6~rzbSYg<({jMaJt{n*Mo?%(3@w>hU06CCY}(=*Z`0 z198t|vuR5TBv2RZKs_}{3SW3HcfDHB{lzrF+h^_tCG-8tFnrnQsyb7~$ivM6OrI?W z;_M1|xOjw|R`{HA?$r#u+bEMcQ^gpbV|@M`-{?JDcDp}l2jJZ9(NpU085DQMd?QNq zlHYtsewBVKA*H;(50IVY8ThDGQOv4;34rT92d256{>cwX?hF)xMya_Kar+fYYIrfd zaZC8&2Nm0+vB3gw4!ko1&SSjcj94O36UkB{Z4-s6A4$d{j!X{(8gu8k<$27mPUIt? zTC_9jE^Tw&g|tWdxekhp#^D{5Ob&J1W)nJf^qEs;*KSTw8aV@IPDpAG zT-tRY&TA@bYHuDXU9Q(4up(hLloS>C@*bCG4+>UE6xxPxLJ3ov*kk(;oW;RDA zK2*#}$feHDM3j{*EzoXh`=Rs7N&iGOo6%2j07hTulm?fPsfU7Gta5%z(ok}4Wbnia zDGENXu6aLD9Ci{%*sCag<8=Mm{n~!P^1Bo+R{0R7=Zwfl0U36~HBG!(D-YuJNK(?N zTAXsz5CeQU>}^s8VlT(3=RybLYZbYrhWxzZ=LZ?6SJAJxx2_5637WR}L53_x+%bmu$9Y=lJf-}y zsRh(gO>fJnnZF%TILmSA9nJEtx|xebubMUXP}JjY=L?Su%8p%pDvT=)*pqEq;TPD9 zm(ll1pt$?=$@SuKs2%ZyRJsI=HrKThD*>w#HrR5;s2VZ&sc}g9(O;#sYN{TZPs2j}E)8 z>>tsXH8)@QJLkw*oY_>edDSgYCSu(dt!;Yc${>{vAqcBh3kukv z8>CT-ke^*fE26!GbYf5f8K~giqU@@mbH^-c7OP#BYEr;q+jbrM^4o)p75!4GnHg2s zHKslBb~eTZ4I3vBc}U9nFnmKe{}tp}oF=yKRX7F9WNWpPsp#fh*rkQyUcxA1=RoK>wRd`QNt(tEDG zdRv3}^M#Q8)9l5j_UfftP+hM4&a#;ZIdT+Q$4<4o`A*mJ{3}CtoT3%lIp@H&MSYTt z$Udp6n!H8mOmXEvNg6da^2X>TH7c|Ad0l$j`{(2gSgy2e5v{?zeUo5aaP~-$9(C+7 zRoFJ*)$%8T&!`rix+^NI4T^1SqsE9|mRq)*GauO|@a5)LskRTXQbE zuL&(L%&&^C$KN^=d9N4srF6KRBoz>{O3vpFh=!$92mP9=ql+UyyKYA7(p*?sLQ>~^ ztb3aC9B=(5M1OE&5Dx3-k% z2KzfP^l?NTME5VGKruY4`tc2VB&Pq@cF|UoNg1ll5FOkG- z4JT^dHc+dSv(sh_*)8;ikb6Cr>wv(4=$20oYa_z@85Q6e3aD=BONhS*f||HbYV~~< zEevkV=#QP2vTNK#p0u?>@114X3nmypn_RY^aZx`MC>2D^Q=1JsIz8m?wxVu>IUM+n zR#Vm;3Prgfbb`d@*E01p%`sn{5S#&cxaL&)`zLn3PIGc09^x=f9Q6&riw>2*T3;02 zfCoJvg*|Q-$s-~nPk)$~cT6Z8!@sOvev{m*mtZ)+MYS}#Iqc=U_~3KRnm&{Ud+%vV z6R(mML64TET|4x+vy>#YFUWmP?pWP8_X?F%x9n?xzX{cFu|j$TyTI9EhDzsV!)RaT zDgqMwt?4+8G|EHjCD;@+frOC(`SghvD6<56?v-=*t==*S~@uXRxX>8LOYr zZVdh9`Qe(G5M;M11Gt0d`H56Pgj@)*Q5hbd5v=35QrN(Cu+r!w*tck_p7%bi;Mn^q zxm9wh)=JG3{48W~B9xpF{FZGu8B{I3k2yAU)nf)ZEQM}_M?+&L>W8{ygl#=yt2$K& zC386e%;1!L#LT6L*T<^i?%g+aS18kEY43S*N#K`@K=Wk9iPe$dx{=V(ff*-V#{HU` zv(89bmv&u+zBJn*>*9jqiol&E;BujO@^|KZ<-O2VD&DH;eh%q8+9)F=46X;K8<-=T zvbSS{f3+)QZ7TuQ=BfvudMq*EKAx`y6m-l3^yccnmshTj2exoaku$e9ZIfjU-DW=6uAeWT4$slHzpq*4FjD9 zDA*O1wd*h650&icZiI5hoj3Z*J?7zTO)Yi2@ELD z)j0iE_fv#1xzGb-Ruy>@4kEq;S)zO5V_PURBO`Wjc z3wXd7PcvJ9gY+lWdmV~WgJ|*k$Wq|m2T)>E=2ke0s3gJ`RIo2v0}n{Eym8yn?AXka zw_jwr`p6}sOw0U4)py?o21@R~LvhbB%Ur?Z&P3tyI0#U$%vMYhhZV*k$IIu4Ra0V} zk6~O!FIkMrB-Q$p&PnTpe-D~K{|^?%QTU)fEjzFYWX3E3qJHetO;UPAINGJXv_E%g z3$$Xq+{fj%4qC2aLvd7f!#k}}^RbT#>%=iH4kHnv)0Nj;8IR9Zbfk=FnY$F9-^=htcp!0J`wCaqz#_#=eemGntT=H-it$ zqMHCsw4)7x+vo9@loBmLn0I^E8PWD5SW50r&TpSo}Kue&z?`vGt(_5bbFt1Ba{SS}5ku<4Coi|HM@Cg^VKQ@*vR_s+h66i;;Daf75?vJiKq4-(9{4{~>@LJ*@3Lmv?A*_?FPd z{0w`Y@VPLYFj=1Ky`NPNr5@rC|2QkSb9{J;gWJ!Vz;D^K<83yvdcM4K%Jl02K63Ol zdQz&)e)y}b{IICRrYAPa7}Px4NbOzZAAX0q;CnZ5yM@nOzKE6?^&h`ML(P?om7nC> zrd!+%xd6YB1R8~{S$>uRx&F?BD;u%-@{WfhhZE+GUZ`@JFZT_iacJdY{U&_l4fe zQ$F7H=2@1uX?FA_DfWUT(NX0mKRa$PW=kOFbAQLO0G)#~)c`GI zk@8&Rm9_b{ex>8n9O*KBweQ4nL(6n?VWH;fLEfL!o*R7gi(1;EEs3~R)63`AP5M$B zGSlPaYq1;{0zcw+Z*%W`mjEx-qM}1BHErr3oy5LvQO0~5Ju>t)jJUnMbr%&av_a;2 zjsJejc34r7FpZ-Fl6<1~X;1e;=d|+4N-i*wGT%hUGqa+b2?g$TMDTN>6cHy;^tt>( z*7);2>x>RkG^zdfqk^`1L6BsO^IrB4gMA#Ry#SZAzg=Ov^U*>ouKTUC@MsnntNc@O zJg~?s0jdd1>icOsfw3zdNtmc~POt_jjel#$&t+|YcVqdmatvd&P%1v5@UsoL$oEzf zMyOWW^U&IF7gYJA_w*|t7mk$Rd?4q0lO2hlCb$#*eW6ayVgD` zC+IA7P{n=@swL7XR;pjX-Uq!LLzN$g2~}U1%=Q;SZpLbATC_oq|cJm$uV& zk*AZ0ecPnI-Nyfhd+$`dzFJCghi@8*%L;ZMxO4x`<|{S+=1(vWG(b8J^$15>OwS}? z@(cKtnVPk|l2GG1f-9d?zq;efhRZ%fJ}&YjPu6DHy7pWCE&dL6lTuuoeb~{gAf3X~ zcWSMDhAz0sdjPh1SMM^oD6ZrQCSRSuy)@;mUC4;Y+f22cfv?1yexC|c2PGx+y{>7g zd6tAZ-#mSf#os(I+nZHath`aY1!#E`lytxE^=~aT*OD+TwbS>I>sm%gd1-UppH*+Y zvj6F?5_UxzAor%}LBIlUwU{PrfcA^>nbVbhS=&nq8-RTt&T9Pg^qt=?`?S8Dc5-NG zYXR2r1k)1Lw{79u{p4@gr?i|$aDhczFYFe|wFbUU=}UU3cUib7E>IJ6wVHn!hdo(w zQELNYKt*wOEp6vpY7RWXe7P1-%i_;6iwbElFNqC5U#;=aZ85#A3DP_&sDtS{=j9^b z63&OcN28c;(JV8Ecj?TJK2ZWG?|z>ee1h>5FY;|`sR2x&@M4Rpn<*a#T+RbJ2!FX` z(4)N3bEi!rzQV#MUto#RB=ied~A7#dXuS zei1-F{s~?+yoDYr+?szEXjnh};*`tygO-}@*8|KB6p8(HtNgE(TpPhXvi|84U=F>l z(=VJ{#)W}~R|Cv6fQI}dJ}|$@vRXYB{;|H%8Hu#{=^bEhKt@K2gIXTl-Q1g*gi-#v$QRR6^Qy&E0@$zz@buN+ zWr;JEQf9a6n%+XAo(8=FW~u*u$^vK>-S=8#?25LFeAL_Nd++(J>Ef+TinAvURHuka zoJLmm)6X52u)|*qDEy(4^ItsFOa*=<*4|$Joj1YqaWtEeeBfSUSNL%}0-c37otTi8RTl?+L3_c=cF)1XF z|591N3BZv9MRD3kLA_T*+5quL0dCG;%=)E|fj3d!*pjrlx1hzezkWLPjdZFK&_f+` z^$DL*T_7=HxuQ|I97O5Xqk}NZu8yVpkBc=1?sS0I850abR zsJs2LkA1`Ri=37=^_H5~M?vmNMPiru9PM~n!}jNs_CX)CX}4yE@V>yxcLD3r1c`lX zu303Oz~3C+t-~4No*d|sB}r`9oBvrKFtC}Hwx4|S)qxcO-VVF~19%4Nn-%z3g=`S$ zZ`$}q`m&GAwSbSCUBzc`VH_&&%YRG`TMs}`)fO%?ib2!ZZR#dpMIq-D5VaX z*jW^p%cs0Mvk_9iso#43Zli$x1%saKM#z*?O-cLo3y+qzy}mv!F7o!*13vomMY<0L zw>G`2Fr7$dAeLu4_gMSPXo4mV6!`+uPU<@+nS}96;$NE8Gz7R?%|+5bU?cFTLil`Rr>t)}{|)6?pLuE2hs zfj<0vcjzMEM%3!i+)OA(W#g-pjgmm~^0w*e35Im){l0U(z=HmM!UHIT_!$E7D+lB! ztY7{4M!@fVeRMrtp&~IrM(6k-tvs!oCd-vA&*Fgdw;oLwQ6nT{@?R_z71jCrNB}fz zf;Igjj?ZnLN@Z;pSp6ipI_OyTt?{$Y?~Py0mB;1sMbl14j|47Hi0hXEi_u=dqnUUi$(}sl@4a5NPMMbT=NSD; zULWe&5w-$%Xp{vx2uxYv{D%TSSVc*lFsMQ5?Is1GI7Emwv8}Z&rnLqLlKI+spoxuq zvBnKgP&cod(()QSTBTxJde3p?v7_~;`l41B)CBS&1%z`drm9BbuqoFKSGLR8_#6YJn1lTd@h-06V?3EY!S-T``NLN+ ztb`^@tnAtNhN*xpmxsWReG5OySOXo$uNNbjtt3_}#f*rG#P|8%2ncXm(>)!Syp-QR zmog;;C}*Ohljy)?FeJ8qJZkeDb^Ss>gl&y(v_*OJynN{OCLbW6iP%FMI;ZuIfj}B- zYGeA{2>5u8UvH3Gn-0%HS@jWOp(iiLrg&EelIxfaviK-|3qp64#4BPMyh654so`|toz6h9^u8Au2SoMR+@cA|dTb7gIDt$qOdtdr)xTcHiJrcd`&7@Twa)}d1zwmbxPB#gv> z3diTjb)m*~X^5uh^rE8CHNVi}iSq?tbes7_=aRC(abNmofhd7ZDh@o`@J^;o7*GS- zKe*o>)j^HuY)K>lhs}mluC7sQzWYQJ*t8t~ro#zoW;40_XO*O%n9%Ka1q7RQQUEvc z<%q+Ex|n@^lr^9jtD7?IKg)-4{b|(#zi@FH28$m~lA)|{?guWq$pXF4T41*ujVAy! zpq(4+X-^tY0DY!7X4g{M3Sl@Om4o{MpiT~6D5!`l9P;nAC)4xLA?P|V2hGxP&Y~{e z!%D2FK*xmU*Kt~s-i#l7&7edHuI&nz{nxJA0?fcSMSzV-@onrf@|N78YU8$z$6s^C zyg7#}$ByOg>DP_rR?~ZI1@TW0c5l1`$B7jC0eIDNJ(=RJseqk#r%!uMtS#69c!m;n z&9C1ATIX3g`{E`aTwkF9cyvo=yL$X+n3UI6C`}AnPk0Z;ZM+uZjr39U~@`tc7 zx9i6rZoZS8pbFSw;1O(jc6U)xI?ozhgB7sa;eaVd^Fh6GHR0|~oy*g+FOeFUmmB;AH5nbsR-1wf43F*WETtNAP@^FBye$u z!)R7^?$6^G`0$6TG!l-phz}RZ-q}80L1NE-PVEz>tvI1qH>ubG{~HJg>MF!@ zmbQRC;7QF7V3s78;;76m`wuBy%T^PpoNrPJJ=y0b3Z6U8NOxqWk8Yq3ASewo;Y$w9cOWBDRK~;vVd?gFI2V|{-u%BHI3{q@u(`Rnz>09!x0cG zq)2@9DB$(LOV0kjulNlF{JPFZ=o;5yQcv%)wExRC+ROp2vs4%mP|wrlZOwfgdr=vf=;vnbU`+2u^w zO-IOiT5$pJ{lr0jZ}PlVSC3lKY40U+=yRP=c4xEHpe2ediAp23v&zpknEQSGkj826 zhKY%_bG-qN+k(F)*D!90MERpOJiX7e7f7VMjo$ZZOJO0C<|h}ZJclQYCfIIiRK#kh z00w=Z-=IrxZ1~Kejf9^3bfniFJ~ST|U_e~*AWG~DS?s9Ey!6Tve3~|nqFbT;zxt{3 z9I=B4*1m*vuon30{?}e3E`7J2D23eRE}>^n2p&}o_6z^#4V_vS!L6}B&Y4TNxD(!Z zWy6&bHo)R&^JfAK@R)%9PeeR`&=-*gzqI2~{fdbWK@*=XJn!Au@N}WVz`c(r*hC6% zvV=7@5v7oD(i>4H6*lGosPrQ6SaZT!=vcQsz|wz3?aA4ECwd2Us;C_TdR@TAE?45Y zaQ{^IDB#*F3`}HBWY;&xN~M31#97hc!z-KbB&hJvKmqGEI_=rV1FP2vvx=Um(_Ole z_*R~|r16_ewv7OT7K`@WaAi~F;zwh#=BI=--S@x=&JiqWmBi(w;WFo|m#Y9Zyi9x~UPeR8W(7*w&jS-axtq<}%wYBYMZfp)B&}_%FZUtRw`2Ylo#JmxjGF}#i46l>%0W`Jt8Rf@xkZwHim@+ebk z9#h)J`)x@1S1}x;{=-C1f5|>)9*g3qa1Fb3&Qjp)`t9*00Sj+EXDg_{tM!O;LMNqY z>VP9aWvV&!JWDy%LTyX!+>J?otk+Ap}s0 zf0;pb)*f-8boYAuF-s_L&z2CjKo&(KR`LO#a>M=ZX!oySbAgyPD5<+p>n-xu9{MRO&VM7~s9!MHn}pHJpiN=W*$u#am9&?In0X^)G)~uxx#59f^lBpU39* zmka}zi>m0c_4$_pW^KL$M6Chcq@v5yELJ<0HUAblNbm-pl*2Zdi|4hw<1wX36pi_I zs+Cp$9iE(d@Z|6fp0s)Nx6SHUC}2paF;|q!3i0~1O`Z$@UF#9ij=uTMl>w*O+3MBV zx#Zr;!XV)`dTXxkrWb*G1@^3dOOveO2_`Uu>%@%OQ2*%+E107mvWWI z@bkMU54R0>lxNGQa%<4X5}zZokW;p#y)4NEmkm$x- z763}{_y!9{fv?jCjz{r438;LY%1ObfB=bVocwX{vz|-GP7zQr_%6j(!irK=C3{R8= zl=n4sUMm#%(D@ADmzvN9v5QkKu2H=7r)?$M-{nJ3on22jXbHJ@C%U&W)463+wm>Qvzt^_f=g`#$J1LUflo&ja1?i1{rKa}cd!yFgiA!jB2C!_FbHq6Or>o6Cyn9{% zJG|yG`oj)ybxR631dcwgNsBk46(cAMO&gEH{spUc(q)6Uo3rexK#a%jl?Dw1theKc z7SF8K$EIq5xe-||!skf38CV6&sN`8kA9rzt{gWwx!~>B8(eTUD<(?QZ*L@iXq4fdOQe}-L z0}upUlL5r}7c2aZVSjJB)Yxv*UQB~GazrYTYfCGp{Yxv>-%#lKhAU6M^#-IUY43Lt zU84zCt!7Ya4&`0|kM@4&Ek&3!@G7g*|1}bz-xyiwFiBQ-6tKMn1~RgJo#6V-Zk3y648Gd@Xo{hDEWzU=3I%u+(l;1;`r2{YS=OLMsjg#B9QE`~?l$;|QQILhu$L0cZ2?(y0JV zd?(d>2}G%F74S9)1EBu``M**X$5X46C^eD4w^RK&@OwJ&Eqx@f<=b9>g||1-!}@+q zM0qJzeMo2{7Y`_;;UXIHH@9!sYUn&IX)iv!fL(TKZ;*8S=0ivugTIoDQownAi-Xoz zUh+Qy6oCMpW85!Rj|V(oH*4ICFJS-%HCMhriGTl?AoTKLB%W_HTb2X<@Kx=9=k0`B zJ9R9O9lm|yuqZi5EV9Pep}QhCnvjvAmmrSLgzHtQgL4 zsmQVOBM(7i-GLl%wnFW7N=D%Na?)A>LB0j3MgG7y@_>@gvgkf|_i^Ciji~()G-cu4 z#^ZnOB?Blp#8DJtd;VS&6j;=mb&KLjHaLtx2C8U=e-ClwTe7=WX93C&CP4k?TlwKC z3b4CqKD&D=LpftKc%^{IzO8_j2xU*p7bAcGp9*a-*C{~-lp@xuoGU;@T$}Im?45u= zq!elPw}FZb4dZ)jL6P%kKCQ`1r~mZZ{Z(A z04hhQ2upNC1^Ni5UjRQ=gLqRG8aKZA<6HR&|E&Tj@vQ=w3V8pz3LsFxiin`5MoAq7 zs%QHe%vUH2hk3$#8D!e=Pcf}C;(IYInFo;kJ6L)N)M-(*BM)M!sn3De*%Yo7IzP3e zPBinXNGoPwe`DAGr@ilfYcl!z29>p-po@YUFuE4}t`wC{D3%2maY5<5O9Z6}B(xBe zso zoHOV1nKPd7)|C_<27F%#lQik;KtQpf6X8u;(kiz}A$!hGS6{mdMlYSC%m@Xe->B#; zND5y5Az|#Ptpc;x6DE15zy!3fo&a4igY2TJ%41SAozVn?uDsmrYT0IUTgu$QZxUI_ ziK^M@+yx_5!DM0-a|BM>6#<0mDG;g@pMAvxoQVh7UOUP!NrTyS^OU&*Am2N)D&L!8 z(*8ByTM`KxBL3CTfkHS}_VM;kmzNV&Zb*6eAQyoiK%32xAT5mm7#=`6n3LT<1}6_3 z1-+lq!s%WcZ=&58#GHF~4XlK~NFLZ^1kF@Us$2#!88cdg9M0I%y_Wxw;caCu#Ymr9r5!1(pvzNDm5?LuoX$&O16|??U&e`*pVpgGlVY{ zB#4e<%7U*V5SN4!D+6th@0|Jh?@|V01XL-V=%+`3e{IG>L6orxD}JzEx=f$2$NG&Q@sMizL5$( zDyZWBtrFIW@a8S|CI)i@DkK7C)*4gh_US3Vnhe?`C)jMdz9JI8!ZAkyA`&EYvQ7Gi zD{2mUOTw8_2T~!hcfpL%tvQk;oQa~y^M~$&4nG!vV8UH}j4D8KKzaK=AKpMGse|&M z0M?QgrTmhL%xUTFMQ3m%Cn zGCw207H!@yeuv(`R$oSlaxFhvnMeUpH<%mb7%|YYx``WNA!r5LJNOO0WTbW)f{IP= zwvc{1v212f*LzYxq3Y@JsB?7$zIKqnXs*Lzi&x;r3``KH{Q3A@lm0oO7&R=!MVQDO zLh|0vH`yVnso`{O%*pt#$fxY*FM*hEN<32*iQixnxT0!p+n< zfB_eE?FxBW}i*3w>wBV8}sf491LOu8IKo>eLc|-a`1;9dR89ruQ2-ZVDC> zIaUTKx{WW<#E&b#kvhp{r=$kbwSDTYU(V?>vd+M492;%}XfH57;Npny?pOe8AATCf z%5%(lXGJP9$wy+WQc)k?{6AV9;<;J?K-qL)Lq2|~v|n*usX-YG3Z`PM5OeP*PRTaP zUJ-)%CN`@#O@o%s>og;mL->9tBanL(s@a+;sWf(FIYS@t5#lp|YwFXwrq!F#XJns= zIl8*OU^PeuCP0h>(zh}&VF=VINK;EdgpM^%`-R9mKZ3QOH&5C%?)wt2V#as3I#Vu zA(#zB-J}mBq9HEtn_7G#mI;Echxd$U+4%njh*Ka#xoK<|5RFQK#qnA-Gbbz**EAs~ zn;i|BlAp1WYXr0uLQA0|MnDNsz}_~1BTU{1sUF~|Aw&JD4i8-V|ESHOQ69styt-?37M{*RbapHMbNJp-zaK+S>>p2ar=YGl^j z0*}Zdv=o4Ex(R701|Eo_+gh4Pz@K>y?DHyJ7YTlpD8i6KG5r3}*?d+Cn$BSCg6698 zfZ|pk0-be35bIylM@YzMjB+Id|9T7#l=$OUVG8_x3sqzvtY zQsPCEZUT-Jc3ROfY1N{5j_)v8_E(1XK#8Ephav}5Zvk_1oxqL(Fe1h#k3d)r=~#wTegoJ8P%PS?hY3dvv|Dw&u-4+-DH z*xT(Mh*O|O;&)q7r%H)SGZ0a4e4LCH7l8mu6)-63iG5mdA$hD%!BbxKyHNvbc&_y~ z5lA}$y-%T19Wbc?HBP_@gA1NaGJfC)>K;;D)63GEvmjU0CiDODtSCStyHhIcGdnfqp8jko$z}J0ZDG=sUie$fJn?KtJh5%jgVC zZh!|Bzg60SMwmJvgH=%p^f%93p$)E9Xzabx4O{tyy0GS!vEIfnZ^9otS~#I*6tUdPR!$P@sZxPu%3uSdGolRu zuTlmpHTxa;BSG!KsW$9x$8`dvV}O5sh>bkNL*0mP)MPL&)Yxn3`BJDY1hlI{rKc;R zGe*eXbj!e~g1vxHqd>?iT=H#=l?G8V&Jo9!zr^cL;css>Y;G}#h-hlLk<7ntE2LZk zfa1{6^@(nXmZvnAnEO7c%@0O?cQ==z8G`7>r}p3{%j1fi?q%>iyTU z3Tb_B?AlCvZN6z1SP)iVLHL0_REj57-jcz;(%?VSk7?P z6fc5+No$@`h|;oA#opcy1|p6e^Rdo&e&(C#u}=UG6lCa2er`0i`zFX zg)U~WAm|P%BEXG`7SMeGT@#s!5j(zKCa$l~(Nq48fXMdkC`IpKu++~DY)GQ(CE;ol zAU8DaJ}@y|*ofPW|ri0XiviU0)dqk;Y-fB_8@!}gn<1a(NGlDBJc z@?)dAap$SEy;QQiwDmIAa;cl;+Xa}{)#HHDtpXE=ODG)>aIj4dr>?|X_k_}fC<5Dz zmhoQxxLso}m9Rf?Kd+WTSLE>w$v)mxbUk2U10tD;Ck?RHWzqys@^`+IG4w>!}K4AyqEGlM0KMi2S)9|t^hH$-`KUzJN+eJot? zv~l#idlJ0I9JOulAzxJj@(5weYqklY^5iaZ`%b<|GxhM9WmFT{c$wQoHeW_HlkJ(l zT@y$^w-i&Tlx~@FZ~*-~zTUoG@FcGwvC8)Vs(WnIgyXY-)5mKQkNMgV@U{V`Re0?$ zx0X~NRA8Q+7m`* z;-ZYfq=2<5Gk9Iz^i4=MgKjl2>_Kf>!iD8}E$dn=b*18=5S}8uzAVHOUIo}Wfb#!l zO5ytn0}8Y|Suk^*-7=z2Z8s#u-!^@P&h7hB!63UyJIGw`vy}#bV_W@)K}$DgC;dN`&+O#8`1%H~9v512uhf3m)E~w{(T?XFqV_BA3FIrsG5O z2}@Yc?3i~W8TJEQ`q)Tr%!QMXBcJh~r4#q^NAL1USZqrIwa!2DhBlbju>jNO2EkJ*$(rYKg*D&ZV z1Ai7tucM4G<`WM%kwa<=`@}nBT{#|fb>0`HBU8UHy z_XgM48Y(lndcK!h@R!|GOgfk|hXmcc3%VTx6&wLj+Y+;h$k>KrX5!f`P3WK%CdYonMy3n5Nog(PT%2}x< z{ZEigHsvH(Zr0rzVea|KBi_`?+P&v}l3KP*aQ>+?R<0?1^x#`BJzp(z2$5y=&Wrj; z>yE(x(lTz(e!pmuyg0YW9OOkIyOScA* zcDH-c2MexD#a^-7A|zsF>{{kozI zeIuxS)=s*y)^*9N#&#~vxFNezk_7T zXK!9A8r)6f^tQWBTt-jb;&o(kXUvx>V4kfo;s!W>{F+XptggI-uOvMwojD`kuuxFE zDSaU1uq=-GxBmphbc@UYrA$;8I4N51T8;Q=TF$m1PTCW#dBTv6CO6ty!m|Ai+OQ2j zmB-7zqOYmx0z!eXwtDIO?2pWO&p|)OtxEU+=Mazi*q*gL@hU?v4`wl{PPi=g zYD;q{3N;J;FVBvivJ~lQ7xzSP}nQ12H8{+_CZ&?>jP45J!=s? zq{Y2V6>XOKF{1Vmuer6@P4p*fs~4Hd4s9dmONW}mbh8-ta0S}Dz61XkPN{@r&Ad_W zGg6x%t6ICdy-rY0@ z<@6kKc>50h;O&y=)?KbkWVKLcu8S z$BX(q_-RPkOJslKcu-j7r?fM(k?6XFP~{WvP64rgI$xQupGe&39C^k6-P7|r7j_1g z#1ix?QX}VP{loZ`*oJhzm;Z3}f3mI2bxy7T8DHx}9JH^wRrT^u|KZQqN~i-B3*=1v zs2nY3QZi@52uZ#%+Xt>Vy=OwvZwt*K9PNWVTqYJJpBlF|2L>;J~5%M3rRvsjTHqY_3#5}l!rL(iaZdy}W6M(TDj zj+e(2qG8PvOs8o_7H5VXRx$TnyqG=+jix?|{+OPEn?0`?z;1P&MB8ak=kdp9OXeO< z?8%#H9chY9_(b3qv@~$hp`n@uX!MnheklH^Ye&TgR-TeHI9x1|H`XxfN{&Gp72exs z-kx>MofVHM(?WeH*BzXKRcFu5Wn1CE^h{TM6d$SfJVx(Dz}AVm=g5IP zdhOn+q$jB0{)d}4J$o1$iqc#v9lGcs*Ad8M=HZDtoI<%HT*P6|#6;WS(TIKvp zlGvYR7b3$nxhQx{Va}`Hukd?!8RKHZT0fLyQP>3aUv-IxtV)Bmkcr2&q z@_4mg=%nRH{t==rXXY_7TS_Sv=YNBYDV=w3G?@}lXPH*)AHaWDGge?4hoMQRm+DuR zia(#0k&r%ViqnyOcJ_t$1*(3I6j5t>iZ`lEHf>Fx^XTn@MNFb6RIs!c z-P}D5+@7o<{VT0YVXVMcx*v%WxwRKtJ41iZ?r*<-IbK#gTu(jY7&-){sXFG!De!Ev zP|2l{mKsyu_x0bew-3X5gp3`Kjwz@bfbpl%Rm!sLF=7dq%b=M^S`T6u{&5e+Og4AT z`iiaZSXP1P!hnyf5;LoU-=XV9K#yEyXP^)Et?FhD86H_+nQK@ zGZ%p!?pIVEf)+uy&l-y-^~A5;FCGS_^o|Oxm+U*8<9jKNw24W=MqU&}cqye$y_|^G zh!>}f>y^tpPW6Bl8~TAc)_*YIJ<_9yiA3%0n8UytI{zpvYJE+Pk&%p<<{FF^x^~#f z5Gg9_?JqI2cA3>2d!}7`FOW3(c7pkF$Npk+8u<@Zvt^6?xf@T2Jrg)_kQ{n`G@_5X*_S#y zNjTHMi$; zHZySeHsh65M+pO4YZp~GHk)zoSag2tvm#^WSd3+0AwRD;q?Y97!=am$s_Iw76wK_B zJrSN6tyo(98Az~vx3vyN^C-K?%ujD9WUtO_mYk7EgqkpqxeSua0&O_w^^oN9UR$}i1%ydB#P$mC zgwvlf*0I=M5(`N3=~nGAd7i{l70un4MiO3mLk~H0uk6ltUDG1FQd!HIdY7PWDr)$6 z8ikGDn73$T3AUoJAg`lJN^*L0`ep>Kk00|HN|w#e0{5U64EdLGIOn*W zf(r}_%uG@VH?1k6X6}~%l>wjeOjTp+P-b1Hk8IUY0X>mKgE3K}SG2$7d;~czj{=S3 zU5V>R<8jh_E1v`_NIP7TPJEObX3guIS^5C(#QQ{8=!P_y#woK{6?|a1>~E);z}Zb* zO$NgU&3BlhYv$?Lc!X}RyE`5NA`mCF{Vhr&CTf8xarZlt=RBF4opg=Qz)@=HaB|^b zCyr_DImiK5jOfbRIGs;9)aKFO0{oXzbJyFMyULXpj|LsObD)Li<=lI$*I*o`Z#@kv zxixp?_c%Yw*}glA_C@D@c}Tuz-6cx#d4s7S5sF8o)=ImZ6uhzOoKvEg92R2$MhFI&!POVRFAlR#}kN ziv8G^8l#I2MQf^?l~{(VwP2F>jq*bjo>B0OvkDlCEcYbRMfK zzo%bDqEIc$X+Wd*pjBaIel65{0I9mooC{w{G^l;QfsQ*ERr4DY3ceUHAW#q}_ll z_Rpi$m3?eYAL^CNOz@m{ZwU)mckRI2pu_1w*0HS}sL6bT z1X54&({egrTrbqJH zhGhR!kwc%BKB;X0+*a`=^sH)Vx0E2M21x7vaU5VEE8L_G~#e ze9q6~0R@cEfUW(R#6Ysx0rv zFBX4v!T+RX80+c;u499YLugLL+Jze_M`OAgX&23TPi+M4_DIubhh3P!8ryPa^rN{;9>i`> z0>t#=?uDjp8mAdSCVr8F=xMF!p7`82tkUvTO;wI)j;8(Pxp7W-Sh2f#PG#7Oh252` zFyAu@+5hk+hw3L^@j~aFpw6&a&ei-yT}rlG^wOoH&QkKQg*u}D;w-A_;43CK9TdFY zOs)$al&O3A9U%aFC>yOFZ=d_e(@FUSB`*F8Jrqs|5Q~CjSc*;;>RF?=i^ZH~xEJiX6S* zaH(*X%~#Xh`e1(w6q+3^Qumz-AOJ(^{?O;5ixsPRbN%SVgrI%<10_Xven*u7(LyBS zb1JX(%RdDPe_97}@X*bMB70Y_ToIeD_{iLH_Jq0VZ=GfHV?($h^_>YKb(*Uo7f?X* zg!#`OzOnLwIDX3Gr#k#phgJ8<{#1vb>hMz?fJgr68~#^?3Sans9Vvf_L|xmNFs&c_ zG79F-K6U**MdZ>h)6VyuvUZ-}1`Y7{?s)d&M^Cg$+rC~^eo@EhjB((l%&!X58uE8a z+St3})Rn8y(-;Tj6x$zOB&=L8uShl0w!3$_H3{5%1^(;^w}VPm;Db( CzhF}U literal 127488 zcmeFa2RNKtyEi^U)F4G~lOQ2FA)-zY1Y7jpqZ35$CLxF(Jp>cI*XX_XUK73dHtJxE z|Fiem*=O(X`u^8<-uJxkd%p7>T#vPkXO^|@x!1k!^1JWn>ig9^=(eyq5zKoE23WMVL|pH~ldVRAg?Gi*UJ7_*uSr95_Ah4 z4fycT?|>kngLuZ2QZ8J^lz0EK>xBJ1_z-*rs(@bR>I&$#?bD$QuRtezSD-R;d~0`V zA_Wl+6~Z|70G(>QGh?=eiNd`1MA#O5)fZnI}{rOer3o#OVHHXX^YkcJ2cz zCdTC}5KQw}58ve#D^B}p=$>Zd4Bf3ZGr~g~>&)-=vb0*oD}Cvk%r50t1`ceWjrBRo z4}?UQghIq@t!zyNMC>t&era&ylg}k=oT%->F0HAk9BrP?!}x4d?rRo6_5j{K>vnX! z5xEMv%d$%g*Pf8W!*dI>Zcd_^e^No5p4ck#Bk68$o*PpZpEWAVd+mX^lNt?1;4EX| zoK87~VE7)X-;;zB`M54fn4!kxnjfjm2&m$U zo2*E>TbkMweigZG=^byMV3^P`R~}Dd-*-S>s0g{Y*qIGZ&9pcY@4{?RIjaxe$t6>V zA>@xRb`1cE-eOD%1pQ|(=I`|w&|WBN1|WLA%Z@u~W7t{P6=S>o{GA#+$QH-&vJojr(*LE9;174`XVBLirnukr;** zv+v3mbo#?ln4)W8LQ~T~x|5pT-QzDk z0{ix_Kp~Fo1nPC^2_#{Zokow0UfVsV{%qT3Y{_5NUX>%0rNd&$UD+9YI~7F0C71qK zqYjIj-Od--GSs{+duP<8SD*_(pnc_$$#ZGYs~F!~2vSa@3tfF7PBxEupCH;db@SVL zd>X%RiNv9JV($HiuSBTk!!!$>?M`FA&AcQM9Oe!t{Yt;QbeJ*QT)yT`lkJ)7!8-n^ zp~`hx>x2BjcjIT57O}z6Y5Ntvs~_)KvpU`7$7){obb*8yxXtNRm+v2se++N7Dlob$ zKrGNiwr(G`K%LVbzE~&rvc#QO#%F;29SR!N}7wY~v3tv!c#R!E=9hre$e%B0*TdgfF90 zJ;uEw-0iZUzHP|aDu4Usvx^@LBjt}})Sz;WbeW3|{8fKWHYga{oL({@ zw}Xz1ui6T6+FW~HI+XNGye>EHP89WW!ZBZm{iOedb=v;6=7Ui1H%ZoR)gKg7d95P;lU2n3s2TDnRq|JL@?UY)12b{t*Si6hbuA38W#P3fyq1O6vhe>E zS(s8VB;MT7rmS+tZ6(UG=aVT@6ewpZc1rpvnliw6&>q~vPzLuJr_?&VqnyjQ?7hs%M1QPl;_+* z>d$sc31xihe?7)EJ^n}YhYhf|)=x&SKyjW6$5d*_i`ePzQpHM6J}IJHg0I08)(cyj zx@%8l1voAL;%;7D*wXFTr}DW1;b1~Gel-LK@$A2^rv&2ubg?^YP-m6vvD_uL8mYGOTW({Cdl6%WV=%e6HN+;_WT|-!m{Ld5dP;~fcJ&}BKC}$5J@k8 z1scFYw*C9c2NWbn8J&<~mqxwNM9&^%=>K+oshdOsUJpW14_AdM&BH9fm45FJ3DpUX=9Nw%s@#&chSxsT9mQeBhkU%^#MixUeGDq?sA zibohtR0qvka_XPB5yw?SR4&@1KVf1nxOD_@G#ZSSGi+D(XWp^|lpBjcIDsZ$Lqo z-f8iM=3i3+AHP!&w`^L_yQA~`+tfOLT2Wh&Xg4n1Mt=6+0T%DtP_hI3o_49*8HzGDU$qmeNs zy`my%cTvFI%qZTR(n69V#zgoXfz;O*$||uVg{h;EZ?oMCHqLJ@_yRtBse8J_mqks% zYvjnZ`L??S11u)JL@7>eF(unjS5aA6R#KHjX*?qs{-U>-!1XmI(Ob833VoH|G>*Kk zj5+blwk#>*w=s(4ox2o|?$p>fzR^!2T(&3!5h;lOO&2je7!C(tES-%y{?Y3((P0m9 zOJoBABKO;`KrU*mI;bg_T`s+^+(@sJ)6PuEo9oNpN}D5RyQ$Ehp2GPJ`Vw1vedC`J~tn8 z%*7hdm@dSL=MMkqgm-X*?>>QdZQ+uhA8V=JovBS#+q}1%(tWgyjd!yt!8wZGR0m;j-X zX9_A0Ap%8y#m56IY;;FA7?u9Hrq^L0$)(2TOG2 zatoTah^-D`!um+rFN&4;Z!R?P+U|x5eK)1~%q{a(GMA7F(IE>DjZ4?&Nhr1)Wb5hr zU{T!7`Q0b7bBQiEegNL^0kVKUy7LA#b23n>#R`ZhhMuVi3C_y3OwDbt``C_q{``<2 z8;8z3o-aXRkCA0lYmX>bo0h|(?j-EHE#`VyW1;Q}J=}LHkN<4lv1$>LSPSsRaCObj zd76EN7ZIv`xRyQNv!Wk07U>+hVs%hna@sXBF{qY`l-rMq%yLQRp*C$JndFzn1t39a zadYH7x3MQ{k2lnX*(+iunji!Vandi*ITbl1r0*a%W7`pHdNjG+g)2g7UJ63kJBk|Uqs&KW*)$CoaynYrdE9fFNyI78re=>V5(Or>wS zdF;jVji&_D=M)(pl-UTqMU&_>@oai~Sr$9ZIeL*sj8BvXOZR=+Eyb1s7&_kGa8nl# zyV%&ugzF)r2hguT6Q$sDN(I^Sr7fGSU83RT*z_w9`yu6gKA%3KC--S~!DX)6yAt0E znt@VpANFVgB`Vw5+HY+q@t&{%wn$mPn^Q__YU(Q7KlH|~C&B`1avMaD#XFUXdb)iN^X*`w^TTwd$X@;RhY>N z&*cy~nx7?{`U&1-t4fp0-{eoAkO;x~Zk?&qHZ`P1C!FZE|3uXh;%`fc-&5$~tm~s) zQUSdE)2GiyAKjwpVP3Xc!Vgp{5%>#LtVE@^XE-V+g}02`UP@mxc7KFrryxDaJ1d?IL2aI{KmUFU0AFI<9pcK+mXLnF1yp-`!@g41R2d0hzs>PC$o)r3I$p^ zQG+5FVu!?uu>TgcC(}#F(jw%v<1lOUXD>UZw_1kB@%+#FVL(`{MoB4*_%fW zBrg`8uAqp|ecjm`_0B#+cBXy>XN0JgAa4TBJs;Fe!%4m~_-Tbu3;+=URJq^0PiTZc zU2`9Pdh60oC|HNYiRpg2@vhr{VT!`PG9BW-=zj@oj6s*RYtJ)LADDv*MpOWKV|Ta- zJHK7P76ZPVg=KE`Pp_dQfY)0ajy?k_t!9$#>+X{^*?PZ|epA0yQbMYau8Pc)N02b5 zls+Zh;a%auR7@?_yu!V*k5gyy``wHi298y`@pBzDawS}5nmSFD2Jav4jwzo`rtKoz zT+%Nu$1|Zk$J>28y}P${htz0HFPaVOadyN@cw^HvAA0>0Z62bCk|JJb`z$zCm?P|-ct$}6 z>&!O2IbFECpIs~XOM>A$Oi+GLSg_m9@~rW0bn;A2kxOc2sj0g9En}`TEO?z-dih%&u`tWb-Q-<=sr@gpajabsyBgpVmDscBkT;maVXBPP;z| zBEyKB8ZR+Rb}zU5se3FE zRpoETto`!hrNSL=4#k(#ct*$D;&xH;rQdl~`PFM}b48Xk1XXaSI-qnU4%i+kxI1(h zYh|U*6Boq%YY-kY4M@X?Xc_RA&I|$LDy6<@u z%dZ{2s4~U!gl&BpJwrldNl1h^dl3=<2=f4k$n@SARi`)h^s2T*ckL6XikAf8+Qvje z&KNtz!wAC;{sCXwq}kpkp{Og+mKEF5A+6z7un#PGtxjVcMZ3j69?Jc}yG3>c8aVaJ zr(5#;z~zLjKq8Q)#@p%p4!2Ff(3PN8cT~XulPYr0|i%+&)u?oT6jM%#>cqO5WUU5&*2DUvB&@5WF-WlrY^D$OFnB z`tRMiH$Ca_LZ$Qx=rY;d9_~tR_8+|0#vU-O{?~IgPIQt5ejJvpDa0Qy#IQPoi>4z>G z>mg;XKvEw8`J27=PxmlLVIe&vQ3#YH^&dU1AF(n^x|uQsK#Yj9VV1Z@P3hmeBsxfN z+>xLP3fa)yAYW*Vz`FuH!Md!0?r6euPW0Y29CARXDo|quORM8+C4jC+^wWj(U_(jc z@>k5Od{Q(IY&LQqRhrZyUd@=2X7GOrdiXzzGQPg=zei#yR(E{c=;PFB&2Nk5PEp!4 zZ(9V93M9>s3UU=k;?GizKRuQg>D*F=76ESIq>VF*rIw|~Oq{`q@c>J&AHIpDRHIhS zXL43F->rFgPUbF`PAQje4b=yBr_jg&_sKJ80GqNEuvY`e0eiw+JIv_*9e{{5ykxiFA zK!kN5^R)1g;{+!C6ec~5YK%p>XD*xlY|mE+R|&VNI#8aFI%u4K>A6Dbx$4>pM#JF@ zgii8j3($?@nrzo(`!C|QwD)q8bV~^T!~CCnDv`vORyufujmZ1$V;OSMxjPOAEN!y9 zGKG#PKif-3P;1tkAO4r7LWq+ z!3T2mTZptH_s>`%XYjKMq0nou!?h>nzs{2)%r9#Kr^Bn#0=M?`v7gMWokpV0@-~@- z^Ciy|#=pI>Q#~oU=mYQ64D2yn9!mak1LJuactzwgKNL8?hA)1kURJnX0%`g6W~ixJ z)B$vkCs7=%nu^SVloih%p6j`DL(BIdwXoAZ@RT!bR_%E3_JLW8t;G(Y)F=g>nxXKTqH`0GODp_YaXYe)4bW>=u! z2Z#JVT~h_gu`B>%OStpK8HtLWy(YyqDNg(T6I@{~QQ;!c4xQ)u@^2diI*GUfon5fk z#xJgFW%G7lgc*ES=2p;bXJYD{&sKD#U*mN;FDiVExC5Z*fW&L(E6`-wQrWiN4ah14 zz~KNdseX=GE50_bu1&H3I#Ud*O<6(Q%;-1!Uip*Q!(Mn3E@_Meb|8O*%Z(aK8{hbz zKC;Tn(m+~?g943rd+$RVLP%b}S>etnRT$sB#qXKh+jIJHL2Xm+M)|QG$l_c$@@c9g z4=noj&2akH)1M$Up{TNLP3uVWw2dA^V{r7{U?a~(Xsw*RHaK)GNNhikA9r@ZQKynK$lzCBEIQcCyJ6aYE zaA)EM*5EMg6NlI?U_TtT=D{b`2vU+Uh+De-IZ-kTBjFU8Wzm?}uNk;rT6jy+F*Q+3 zaddGcPv*?FM#9LP@`vQo8HKE23ywirTdw(el%}swLyVC!XGuk}lGxzAssR8k^be_^ z@@q`)Z>XYj4fOjhp&Ow7b^EWO0UBHd2!N6Q>Jp_U_2H3aml)T)P?FN?2|g3n^lP~k zjw@8!-wSb=yE83ggB^i05f!R!JTYo`rsk`9>SJ151{Y87Pv-E$4ia8`HsceF;*pw@ z5K|qa>01=iU)RL%PoceA`uwKoTO4{cSbr_4!l-qb)&@KKLoV3|T?}7OXuNCWC|nNk zExKjd<=%&U<6sv|AZZ%tuJe1ob*6@{P*df8A3z1Z&rVqh5xneMCtcv_n}MVK}pCIhv+b=X>dAuX7Tg=Hd?A^3E0wW{>b zJ+kKQnVmpK1507uMGiVp`9U^KOL7naC%rW7?JPQKd!sj)o>66NJqox&=bS#;`oY!DXI!K@ozrX77vNXOVT)?E_gOBlmvjex`%0!d zvme&3(M4ck(XXvyW#@T<+uZ!ZOdm!_S~;3N=&Gm=b2FVFC$c~>HH>YimT+oLY9vpy z?$90#!W6m8V!j6RGp4);qNe{CAmE>CN)=N7RXZ#7ySD28i(=^4o`iq%BnUI51Tu>O z66jx-O7+)0@V{CppT@5(l>a*I{Ms@ilK-=X@;3t6e?~3;)|U7$MtZH&U&;dwQ%s)# zN0PVq@2Ch2Atp;m9mB$<#l7K-Zx_Q};PJ@sW)-=}==i5&KK@Y(@&6;^fQbR9P4cJTEP3gi{w5U}&S#)o z5tf4va=wSt&nx)by2=!vEVrY*73!Mz;rVMj_tZ;YGgSgZaH>n)af@GtKb8-Pj!gSGME=S;ydmZl)DD73^s zKsIJj{Vg)F%Sg`vg4>sIDC;L3RB)9Mxw=o&*xS2QQ_BP8v>thmGab}~deWf6Ki>9t zsxp5mEpFNIm%<4T@&^n7eU$l z8FnF=!3#pmwwwIEUvAUK^EjMNzJeYx`M2uw^Saf|__hV}YKroOb@8dbCY&SSr#d>x z6pDBxP;NUZzB1;QZ)t1P%SIwn`uO695QHXrj~^9HQjt9ULpqf$PV91m%I9n%v?<># z{sqi@*rCV2j;e!oJ7){$SVf-8t89aY6z+VI>APEocNxI*$>v#BqdulcN5f(A7G}+j zVKu>Dya@;u#F?pkwMJJX(lRaA9bp8JW&83gP=OtE{ZVN>ZFB69`qroSex{{gs1wPS z&UlxZc^5j6p-ywh{c@KWN&l<~;bTYitFcIi~yjL*9c+W}3a0xhc1afYrb)1zAC*g>DXKD$`)2R*n$2a`5DC7g(UifpO4$QAT?o6UhVHjnnOgG z#5=)$OjD(W8)x?{+h;TUY8KRa!ah2yl6=cAc88Xh(@pcj!-IoO3zxJ{k{4=ZOhd<8 z)WhUQ;|0i@1Gwm$-~R~W=8Yr52jxBuT^!AtG!Po?Rbj+n5rfi zrkEc#p20cxpd!_?p|cW&H)3UNSlj>?%CG<0@|onuX4=us0}#^om*HA|?m`MJ^(|6M zD|)*QdV$wmtWDD#Xw+!OLd>MCxBHVGjT**j_3_FzQ6&`99b%|iErNy+K~0}t*}GCS zKN`^9)ou7_k>gg`z$qh>;;B+#jr7FN_YYXr@{4O2>Z{?>3@2a46i^Qj@5ncGaitFt zw2qWn?Jnx1k*xs&{6_N*1L^z|D48cf(iRZ195P8v0eB9+b`c5XVTF9DY_+n-Bo|nP zZy1dVDy8M&bzQL3+rtpp9miQ^`CbDT9A@!Egaq-F4kc2kk_CUKlaq6Y4ec=^*I(mZ zw+4qCgF^6>LQ!M1uJX4rzD6$_785^BT%?>~*Y(t$d3_=F?He6>(e5T^!V-K(gImE? zgQ91VatzLJvT;XfF5YOeal4;0*z_6tYz!rLf3 z8)}Q3+y$z3e)y5KJP?&kZo}=Q?yp#Unsqq$V?)A*cbnt=6T-!)_pm|2-U(vq-u||@ zpeIxjucWx6{Fg1JwoX-NX{v|rK9eo$^_F~{=|V?MV2h+HGgc7i;67rt+iE%8aW$u6 ze-PXeAVP;l^TAEk?IpasB$X&!%dRBhwx>8(AckWQ980-ZQO-C+TjvEwtdf?A0>T>- z{aU!f^JxW5p)EVinJV`_7XgmaCl1aI>YJtt;>VR*GL)Q%m;=Z?Y7cR&$)MQp%G+Jq z4*gI~ExmiXlTY)8+N@t?k;zrb_0;f94S-#a`(6*pr2CJrwg)U)Ns8RO?FUxrOeMg<(PHb+l=lqe|kJF|~?wK=}tPrmS;N%c_JqCU zj0Db;b3<>OwfR99yy zIjec=KCiz9R?z9oZ*`9OG~H}Z)JaX4MMqnfh#R$-{?Mq`d?Hiv6^FB>ohD;S$SeK1KVMxu3Avnlf<}Atc;PZEuCq()fTGH?Z^0Gzl?uN<+oLg7}_>| zRmUG;8#HDu9LSg#BKqIdnz->#n)jcql%J;ZKfAp~g8VJWjt{)~O6 z&3j$(Hjie=kXcoY0FFpc^b`3@A?!Af!+i6J+1qfbMyA1@HLaJD1XZ1o!HL}HgF~6K z{N!bm>FHvW)~1^-Y(G`B;=TCNw2o|8T*bEb1=_%>>{C78cq88O(|D`V{jOVG46%<1 zKHMM>lAMUISeOf-5H1QJUiNO@-&a&R05F5b)m~WO5GQ?aE&`DPw`jQXig|fxyt>Cj zb=`PbFUTs9l2JHqKw7wW%8lNRujtu@DmD9S?GtXk z6DMrBk7KOTnx0%Ph33p=QmqwYmM7uc@eRYcv_6ltH<)FQ`2@LB=m8>aSzhn_Om5l2 zA1E?*hHe2l-iI7fG{E&x;y}87VH8Rt%L8~+-KVFiS0Fv0B7{hODr)}n7a(hgj(FWo zoVd?rpv@McCv%rJFE(;TKltd>aGSaJXa~KGMtIKC-p#A8=7x;_1DxGE&vy{GtEnIC z{Yw+745v2=gXLl0KDefTKc!%9-eVP%+uuCFA=x{<3yx}EN%nA5RSJa?CKu}OpOg`P z(x<}fgA&_tBGpu?Emb|`+#Oc)%QBL zLpwP9PN=tnth|IY)ic43L{a8)F-wnGkNOV747_a-uS+w%t!akSs+72L!=y96rIS7i zb1ZOgpRAkpUvLlFdL*8xK`g^I&(-Kqpwu)=0%!0;%Hrz29LVIhCTrm|t1t}C?pO<4 zLt16HlstB%%>BXEA8L&w?q1YYTG4Z0eeSfF6G{TBP*{^SyYvXi56a%As-kc!lzWni zFfO!+Um4GSl%1ls^&fMg3>#ts?1O=LS=*02(&yvkanVWsiDJNAS5NLQdG-{OiI_6P`sQ;8c!A9zVB*gJ8chG6XcX=m<9aB*825Ih$VNX}w&5+X`GQ+vxb zcPq5j`Fso-7$35mnD$*LWTovJ_wo}F<_B~(Vm{hEco3^?<5d16Hky{H4rS*fwR`)` z=X>(|qrYsmvgD6SVB&u>oM@~V zkrlJD$=o4KX{nii8l+0u_v&Pk{Xu_Mrl~f(AE#h#@!?EgtdF&6X=Ti;u}fnF17iz| zK;UG?UF9t%*ZvzC4k6yQ@$=AZbFS9SixpRuu7|I9^&`*5%Q$HIr23__Y@RT=`j^?t`#8^a9>+7L z)E*A;S!pet_~vZ_U}m+){N1hOclkIiaq37Yiwf^meOKHpN@}=n+j{>bG6CGxiQcE$R1BbjXGcb2LB>Kg=JnodPFehnF;R-Zf z_-5Eqz|&BsVwCmO>o{JJo6j`(txH}pnTCS2TD7Na(f7&;>!+o$Vy(jT(9T2X5%4Iw z=fu0?F!&nQrM=#Y?majiO7aU(jShMaDhM=OECfHM(4u6e;ao|M5F?9v@BNBP>x?kp zjd=Y+n!8{-Vs1H%mr(DATCTCvY=GOh@vK5vnGRN~E_PMuLb{6>)x);DOmkuXWF+=# zomOKKlg&MI{FB6#^Y6Z2%o_FGo(lb1X_X1V`-*1x?rOlCe+N^vP}EWnU2aOC*x+Ji z=u!m2w{Q*t74=(s6;2l~^}+ZDy1srA+ESSc!ZcTJi(mG*J=O7HYswm^YW-;_XdP>~ zcB@U(G;n+2(6NhcvM+=70nhhn`+km^&}Rb{K`m~kLF{>}P{J;!@~ubo2W+ikPFM<+ zror~Qlg&!!WmU0-$a_sL-3^)z%w_dp1lFLkS}Y=Pq%|Jea)DQ6vo}A=O6@z|Q~oge z^L#{p|FgZ>LJt=B&Hij-cVkaS*0xqQdbPNoVzG8E*ZHvV+A{-+w9U2|2E*Y{PX3&z zRFkm*(UjCAqw91L%weu6U zr}$Bi1*f~y>v%|~GPY0H2^9n$LIckeQ3>&S-)#!4oa1<1@0M7%Pn%-JHw`-i=la$; zYrTJJ&)ZWmd!5;y$J0c`jkB)kGzsUbMNQHU(OPM+d1EConTkW+lwQF0sM?1uVRSfK z0J??U%UDWb*M}9+aU;{q=!oD6x&68J3bhI7_v*A&QzqHke!C88br5ld~5J7s*Y{S2u~4!Ua7 zI0=4qD4oT3`w@@5vYKShE}$c&U=?BRIudFR+}}}ZLecMQgD0IT8e2cuM2t(^gwA5z z3cmE*8uGweEQddhT3j=$FMnV>$w5UE66R%*Yrywl=K-5pq$K%((hub@me7(w#)2oM z|7=C{|EBS;gXjMkB;`LdaBRAH{s>m^L3IkXhx1N7pgLo1{p#*yGqR!V zLAew(N2r_bd(rhMLCcUnZU*a3*mHrvQxg3gjgPpPQVi)H$4l-t0#_h1Jvn`q1ru~P z2Pe=C$tB(TyQRfa!+8ZYNo-sn-iD~AZ0{9sHQctLm}{IEelfSDVt7l4{2e6SmE8xa z1opLIWBc-o)K2<6(b?3~TR?Ul&Q`~YZf40&Pm>2QxOOU-b5?pqqf^90|MhwRL-% zyoxv&Mv>Kq^X%J_z1g`n4Pc><2*2+@*CxRi^UC)(dOTQ(w1V=J^n)3viD>GlHXs9m`VV@Kks;nle>@|I;xLmExc{D5av~DBB_o5#rHM}0 z{4Vx&<$myeh-JI&=w<(<1+2{-hZU(ny*f9e-CPOTVaI^7_t%j&#hyD=EYwKRg`#sS z|#Jl#Xv6u?yomE*_BP(Pc|B6js}?@2gdT<9o=ZdcH4xZ`~{&ZhRqu zG@LR!J}!VQ5Vt@LEEFmp6E!uS7ec!0(01~cB%Hl(s*tmEf^I~N<+HtMEjM}OZP3z= z38?>dwxexI(WKJ1OOC6iFM6U^NKL3d$8w#SdCE$nU!X=R#*Cg^P3e>6o1v%k6D@{R z>1mmyaP6c@pwdsn0i6yIQF#eut8nVrI;g6c$oHL!2*T3s^zMlWW7YsBW=!}MX#I8N z+lCJR9054A5thmq<^$Vuv^h1r0{zGx%k4rDR0oBq3bnYt?4<7OfrhxXSpiA>wK6Mb zWGfB3LuV)hdEH5IsG(3=3#Tl3h<`Cz_Id&Lq_d=@RPK`|AKh3uSQXW~AJWYHTC3!~T=K=-S4g5Ql=#TUG;hlCc1Z5qr+n0jZs1lZQx81s zBRyRCoHf6pw;5yl>v7yZ_T^3Yq`ub-uPR&2TK zyrzd@Z$j@Ob0ZLO4yVFART2zuu*0*C$OaA{)#XZz7!Gq=$_fiwd2^(Nu2mslS$23m zHXqh>DVm8ER++Zwo}JeZ8}xFFxm%JO95tsCVjHEyLZn~K1Kk=u!6#^EQ9`oR{PqquEMG_nZGtmQFL`Oc!zaZ2)RQpsHM8;_RPFZf22XIM;Nff zJ69Wy_ZW@}MG$~$O3O2M=>X`hV#I56xPXmruXlbLd9L;qDGO97qu4`LO9HiP0=L?r z4Pq;22`+M{I7q-C|AY|B6n~oS38nROcNwE&K**Rf9DwuPE?sS!2C8yBU;oOFqq&is zR*Mgz?48>^it1)l1BZauurlrsatS!M91Wj?qHmt=D`%e=SJ@`|aHV17z(7@0w>H z#fhL!K~TYo>WOzQbgFU$F7>>1y$Xrrha{+5rx4Mb?gu5+SQ5ALxUl&!luZJ4$9T&^ zxMIPthiA`|?BQw-i(fbougun0bS6fIpgouN;7`WA$Q$7kz4!$==nf5FPq zRCxpxcKee_P-rvE`nq^%dq-csxC1u}eM9%bpZ{vQznX8AZ>Cz|)Q`>S>xdPl&xaXo z0aZ;q*E$kye0FJ)vjrWCLfTtVPk0}Thl~8c3HOY1C0>JIuR6e4N1r20G$SO3{2+-= zu)AfI+;g{NO;_0`3ngO$lAP97A-~unCOG$fUOCk@jK;~ugnv{%kFbM2hwwA%K6_mC z=7icQJ7R0X9^H9UnoCNU~73p`Ad%cvOAEiGres4i@Up7p2fx zF~DtO5j201J$Q`Wg1@+YS71J?KGj0q2>Qzk8O3c-F=QiPEhOL6fIQ zahEV3>xcUjVZ+1AQN1DNY@YpS$i_lvfD^LQ%KRE^avO2;J1IX>)9V5<5W*B(Z}Cdm zxHQ}~m`d5Q!=<5dyZ+(k+>iRZF+^`=@^6bMA5lk6hdX|91-dtJ&X3ppfuaA3)sxOS*;ubYw zRTd_dCSN8lUAQdFi)Fc59?eypsnu>yCYFa~$ay*}&;>{>UoL$owAZDapVDw_pw{eE zdzLWW?FkZSNWxl@&~tp1mNZUdY_6P6TnB9OZ7+@){z>zgcb`J{=-fYnz`YX!3vwUt zAoN~7l}ZhmN#hc1F#A!Nlwc)uPGir{^K8OtSyVu`n5h5$qbW(M7Rub?Sqz#Zus+&F;eYDf0&OsfEg zdMj>mBS$gw6-h&4f4Wc3W_!AHtYvpw^{=jl;opJ||3mUmuJb(puYcS!x{h;uSAsQ$ zNv+8hXhF5kQb5YU*2eDZ3T{;haDp`=w?wUvregaqN!-`D*c&f~LhB zsWKqvR&IHz`_s^9M`2QgcD3WO^JFW0T8e!7(+iyE?sw(%G60uoAhSiBuX#tbh8x*P zkC~`;|IV4ef{(7&BGlJT&h_D{FN<`^-9D?wQ>8%FY)_%EHQjV^`@vS_R5r6GDfKjo zE(d)yoLGLsbQ0b(40V2Y05vk;BQDnbS)1c5_P*??f8p9_8RcmFs#!PDFBOUeGBlZC z(z|B)IZHXpK_L+hMuswIuM(mPrgVPvA*NbRz32yH+WTJ`M*UJ^V)EcE+7Z@xt_xxF zX+7!4?tFdBtVF#0roPGjTZRemb*zX33wdOII6QT6;jf>{4qt>2{6NIYSTfgUV>#Fr z&W6-UBGjUL_Sf2~g?>azM@hS}luyto8P3rg9`fuss77%*G8U+B);!@yFsaOOmwFB@ z<<3K?ET4<#%ym(blaunAh#cdl1*23ph%wGnQ?y-Ng5ermV#nFF5!O0k zmA4_kr77N@$j20!G1HQh^c96!IHW#W$RDZXo@$})qU4iNCReqHzrWXZt^)Vuwqb6b z*3$mi54nq7D7&v4m$TtUDVf)8TpOVp9+%Gb6dPn*=TZl2ReoI_0|@d3Xf~2*F09rT>tk||^ze%?n^jC# z+)qZ(E{=B{r<~t<{O~`OGe%-<#UCrhP+~8!QdQH4$+sC2rRIcg zZ!$Bq3e%LpAH|P52TBmEFYDD&x&@A>)K+%29ZK3yL>WG-?9zGCmI>sa=F@EVq`i8y zT{0!NTrYGW0l8srh;Uupu~?u~tK!Uouj*)@^V;PSb3ZcGj8t%hIN{;$_Glq)FJ4N7qT5QNK<0oDu` ztA!YoSj*dQHLB)m-agSy7I*C7%ztRbK9}8gtNeq`afRi&Ylk-yEeB!WiU)PIHG*Tu zapD!fNpQ<5$ZV%z9r!57C!A;oryu6xFqgt>LH`b3(Hd2K(mI+buCg2Y(ZIYpGOgT{ zd>)KFhtRO`JsKbO{F*WM;PVoBzg_Bh4L@Vv=2)~wRgaUobdr~r`qx5AF1b%DN$ z*~D4(pP880DhOe&FA8o2=@uEzz zmI(bhwkBA!%!kviB6 zPv1N25o}hYo5EOw)7IHY46|~xvQgr3(=v7_jfL5H_UWah)U%~Knp^##FxYpNt;9ULtZ#13iTFiDT3J$FhpSs>jqyG!&3s)zEjJW6X2{-ofNXb_d*Qd_kCoB zRn^MxVwy6EL|22bwC(5WEPc~e^NkfJbx+rggYu=Lv7}F(%BnvhS%&K87A5M#?TI6- z!Fx$q#}Hq4D|=nc{X-Y$mhcv?_l}&0rY8(3C?Lf_Mq|m+U9&Gg#_uyOO?#l6bFN^; z5_o#ptK#BEjE$8u>TjcP+T$taYK`wQmcbkE!86pQ2gcRA_ru}vs# z=t|~zu#D%i^`5hA*;14nUarM5p%~ENSo?)WDJgzT>4ZM7;bhV(I|J35E|V@wiw_Sj zlQs`FpQ#(vUW(a;?-B<NndD+M0DH2dq23#&dl@IqKF& z)RvTwvg%ImV#Je@8QF&GGUp+LSI4hF>RRAAk<5~BY+kmMA&nPHby6})vA>pLcvx3^ zYM)0k*u~v6 zK{mfQb??VO&Uffk1W}V^zS(^<8xxbRBJQTPZ}bE_*5h~$3%K}l#xfnVHfL$8E4)`- ztygDCh@bUqQ4tPT810*UF%@*u3=7D&D#JwQz{bnDH%Quv4sT=TsD5_Ntb5qv7M)aE z{p1DBM(3Sv2(!=7`y-$vgqG}l9~WiY$gZG*B~6+O2R7MPo88!ctfhf6Jo&m%{zNJp zmLMv#0lH5)Ib2dgAmThh&TQD7E0Dq2bDiCG`-7c!XVVd?3CRo>IXRzJlEFzt;N&W5 z(!rIrHs#%pzB===w*(=$J3(aD3$972t70WU1WLVH- z5%s*ya&&*D_nQK?IXVBhEX0s0}8tUH5P(tZDDhX=TBPjQmW<&o!_P#nU zs1;9{=Dsvp4sS+55g%T#6si-k=;Ed}VCpOr{+2%1d&60wS=TRVBto+UW3&TMt9CL1#Mbd|EA0g;@rm+sG*(|{@ zf{3A5jD1JO9{uG4m3a;IP7UeZ^e1X*M8oj4@4C9Kbd)Y0ilXj)a^V*6p^%(j(ttOVG=}#Xs%eqU}Rb9 zNpATHC8*iwh?Dt-$?3$D`Je-Mzj)u%{j*QBS9+bcy%D=O6x~?QbVNapVTcMI!I+!p zgj+$5a!y!c*BLkYkjQyw`eYCn@q;tILwr z9BgHD9nuQ;{bwwe>KYzdTp$nhtrhg49&Np^8V`f?_8BPY9iOmCQSj5>kAg(Vw154gDBh6eFgTq2YJq^NGC^-;*^DoTa^El|WV6lP@kHN~(It3Hp@{@N*dW4+Hkvn%7sg zv*_E;C+aiZO(us%@oezh^TyJB@58ul7)^bL>W^4jP{Uj-*?077Jl`GC&9Zvr>E&u? z7YJwhg<(p(w?!Ji|GW(uziD#e<%e$G%&AAe6gy_~YZIo{v(HN5K|k)TVR)t4&PSoo zlzo9|F{{qIIr>tIjeAbC)baF89t#f9*`9PFiHe5zf{ zQ@P^?3W8u7Izoe~7oC zhraPbG?(<_riY4k6)?u0U93?)mqS8o8QSVt?+*{vI|=MbdoCT#?f@vjiWk3!#`yoy z*v-YFPPr~lMuRrT9zE07iVic~7DkS`3DxjSQP^iiU!I7FSjvA>c}VA;e9%hdYG?0= zgJip*exk>PwZ~%B7p?of&3k2!VOu{OiQ;4D_c{eLeTR=saoWe&*}W`KPBE@IY&Xk&|u*n1}OUO9k>A&T4uUP`U^P)d%E80 z>m{s(zvo(E!$B#>Fd&n%y>jsio)#1qlzCM$d%51m*j#sC=pSkidH2@P%EIR=;l3R> zf&@~`iKo(JZJIOQ!{OJ2EX$ z;TZ!fJcg;pjt5`7*SS1Aal-9&%+nhCQIova;tFkINoM9F;}Xo1FJ4&^`rplLLgFjx z`s?RnSRGPdkWZ-Jtb^~dMYC96&GSzXYfEl(w4Uo84<0XIlF^IzzG}ls6le#^VZGNN zb|vaSAF+3o@IOXAeu{XAXkK@YFO{)_hh@<b+Y+TESN&Rpqj&BAELauy!E?)Awwya`m?Yy4 zQ@HJ(cQ>(O51$WW&h%B;@DGX_EoIzduDg&AiO>+e`ykL(z9P4HL;QMvOEa?NT5o0< zqe+teC+Qt2HBR-m@dIX$EQ) z>f=gzrL`);dO;xuu5YB~gy7T^Rra^E4o0ZuUoR*MtLME#>nS9*OOU#$5ZRu@lUJp( zROIBNv0H{G@c;ut<9_U$;4!p!cqS@Z>vsCtfpuuxf%;U@$+M{^+%f~zdAoNl)dKSF zj5Ozju*aOpiAT*Kkp`wTJg#gxl3YX=Hh4E-*2ZnhlJcdU@wD2dfaM@{!F6cJeU>L< zkWD`e%dm2FVrTs`WCp%M(Cs4u@H1@nJWeJG9me6KiiKRS(sI-dvhX#oh|o?fMgX)U zV8EZsMaFhzqIl(JkQxt03UzX;Iy5EdNAa|45%WqDJZLi|`1w zY=0}ymMPt$i}&z6cGgc-KoLk(%a3TTjvQqTCoZzGDfy zxP!>VYKL}<5!thc-AAHzTXpr`o@;+lIpk|3I=P5Hm(5nxu_(~P+I9YEQJ?P3(;jek zvF>g72d^&*$!LYVz%oS0o#aTbyUg;)FXL&za!&DLCYyYkLXWym+4&c@?YS3hdf<^5 z9xkhU!9yCYjCb5=B2MrJb(1rw97_WPm~jNj2GHb*##-DnY+0_I$}W{ip2vIYdXo@r>^H?S~G~zNJ}*_-&Tj8kP^E16JCt z$cm~}wDs>d;GZs@OAuc(7+ni;7Dwx0I}5SH+JN&x{U$!PeG+lJEXfnAtOx3~1IOE( zElVkta^3=6&xf>uWW|#?&i% z&*#Nh2Y`vrB=XB8{5dXR{HKY|(68;k^_)S1I7``8du*tk*Cgt~J>vMN5vJ^M@w+kQ zNf5uc$;Kz$#=oJ7KZeQUrY5g~;R+i=MSdN!l_!?5xyy@f-^O!wfIYJ&0u0_ABHi%=)V0C#5NG`_g$!6Z$F8ui#5l>rkFy z^<<|o4&oKi(Z+piDy|9d#x^1FzZwMos@eKCi_HDaB6avxAo_FZu7W7*+%t#9Z3^lX zDRNJW^x#jHr*KE+?^=;*P4xC;9xg6W@uOPezG>n9LI15P*I0)Aea^0leWLrOL|^Q% zc&n6KZ=1ywbVKv`RUMk3mx%)7=3%54*1AO1Umh$tPerff8@L@= z>sqwFw?%~CrcNI$2yU6Oejd1^6Uq6e@4-TRMWL8?sg((-b@(wKk1$D#kcPt#4LU95 zotIjjrz9>>FiZ)F3gDkOp_k!b z-A0VatOt$`=TVTaLq(=tYUGXpCQ?k~{%XYQZl(V=py@S4}AJG9@h z#R#zp4=!}?jF$+flt-tg2%XS)jhM@OBi9tRBG{JhZ&)VuKb}c!H zoff?DEHJq6#XziDu441h`N9}BEy(2=Qf=Ao@dFEi6$ zP1`e0O_UCW#IVk_y_xAX+YKnbcDOLtX++GQ9hPgb%uyaG+x4Jd;k>a~EcFz`hiduD- zpf2c?-}|7_lz+E%cpo2K>lhEqgIj0nNt=khSdF*&+>TY$SL~dT3!h*X^g4X-I6mt; z?1dRvwtscdwj4rq{hM* z??Ub^#U(=cH*LJF+0SP_hDV?QE80V8Mw|;-N2vhn@x?DsJtX_ps`Y=>SStl zT~AP3lZFZU=nU}c8`NXh<^UFXJ9?mebrKMHoMpt@Sr8}q5syLl%L#3>cM5+o6m~HQ z{Sy9bCJmjSR(75`A#2Zv>)!joGVo6ly%06;Wfci{* z_0!^EiLXhaTt$V~5IgDpCZJo6Pyv1OzZ-DC9VB3FBh#^AZBj{%J9Z&HQ$a){LsTu^ zrBKB?PZY;Ilgz+-+j}+EHqz~Nxfjxz3B&X~a&mW3O`A7DoJwh3W z%WWYo3LRnpGQhM=;WaxcbT27BWMf(=Hg3&m`0}|vKNg<_Nxd{!|9p8zPaStw4j0RF z0(-1&&5)K^f5Q$AZ^emACe*#WRH_6~KKt)4579VVjluSp56gsLi)+x690M@4TOJ2$ zx*SDh*LFGmYUqu;fQQdI(mZN)ReW*_3)bARWVD``?Z*%^olWZyfqx~|EdstgW!RLO z6_BogRjS4Iyn>HTu0^cU0d?{F|2Iq*aRA($2GV~iZw!P_rQmh2u0l^V24N186yG2Q z6y72l6-Tj{OxSGd2lyDvhPG)hrS=w16JI^MW^){5>t;`D58E}MSp`T?1RyBemzVIz z-E<^?KHALd5HrGf1UA2jsYeF@9`YX-%|%osfM{!6QUhj9ZNY#Ul>~cFGSyv)buKSN zfRyS@7Y$fZwuUW`7Oq3(hFX52Fy#dF15EK79^`E z?0c;EeNa91k4wV;_<{C3q)v(fbDm4n;EBMsyUSnB$8Nf$_@*D2Ef&A=Sml58Qw`z% z{0_3eb_b$K)!%pmHNVY)ieGQw+w&UnH|DkC)z78?;HiCWVddt2{$BsnGwauS{fqSq z-}%tgEWI1Ii!<}jLSS_&b6QYfXs?M8pZ=7w8-vR=8=;z~f&uoZeI@i&+H(rih{;97 z-E5nO$64R#+Xx+_54nGQ@kp_MxeI71GzL5N;{Yzn4I_meIzpN&kK_F%)QA%_;SE)6 z0tLDh<+n9SO8>_zb!~2=^q*-6{C_GNBY`_K7Yfb&zFxGuq$8O@{~jqd=to$>L(WWy zmr+a@n}F4J?F#tnb_4Cj9>fAj%!~%*es0Dt_QFRmA$EdVKV)f$4j|JS4c)Y7d|J>A z$w^=U)jH&}D4qajcCX{hZ>0(->UgU`QfeD_zr{aa5Pkyo+1(A`C;Xet0KrezUpA<` zg${LyMPal@*m@@2w<9--C2@^Ebzk8%nZ@1#Pj5!}`oI4SL%5;b(5vAt`$@m&MKZGJ z#o_(Sw$%CdPumO}{yX0{6$;539mw)Yp6DpZMi9t0vfdqZR=qzQ)E9@IX0{4!r1H?l#WuHSC4R7W3t|BV+O2aRhAFDp2Ru`tD>NWi$_xkaVQk2QlGJ zdXLSN$i?g1-)bkv?t5fwG|WjJecJl@3v=b5l@T9;(OsgJSk`>d_M%?7>ALqJbCPJC zBfFbUwWQKI6FLXaMz-&6 z`>^!lQ1#RJmrg!Hnz&mL&g_-McqPVnCQ8OKGe{Pfoe*Ifml?mpXtzja?!=1fP}CG2 zd^cFEvooihilN`4g$~1}!hvfWtQ)pp+OewZ;=Mh)*Y=qr2%yo_%z8Qe6*rV*g8TW< zT=vH@@8xmF!|CC-#c!na>SLMmo)on>PKMq)og3=HuvGi<`QrE)RefB0jhgsUT#;F1 zTZx4o`-mSYx%IST*OkMeUUyc->-{RnI5qr>lQi|IALe(M?!4q>DN-laEXz-=+VzUO zV2(>KzL;d!rOxgZ!>Z;6qur0s-lS%-tt1uD4r59=-5i~3@`%QX3?bdb!)~DCBbys+ ztq@L+qdFnC4Wv-;Jk$$%7L@&FC~ZYqoMFa}()XEziyU(T*g#FFH?Cy&>%qa!%V z&WRE$o24KkC><+J?%xum6Dl)l zE|faEQBv&gpQ!&|t33ICqPKVAq{w#4r2pym{I>+dd0-#xk``r>fk5eEQ@v3kD7p?Ep&P}PtFC%4+xI_rydZ>lg)%;f zB|d&Z z1}z^uC;gPJdo#xlyB#zu6C>anU(DuT0YwNoP1bIJz;=5?H9Nn0CIR8wuHoE}6UXd5 zo;CEFt+pv!7reZik#DV{x)kY4=M?rzb+7J$gv%nrv1&slR{a?9Dz&ia>X7O@rMf_g zx9@5NN>MnuB`Bmq#UP~I2Py94IT5VMQ9C{JVA*@m<5|~B zdiS6f{KX`O5Y?UeEjbOKCJE7SQ|=|6oq0H>jV{e%SG#J=iRGjQ4D;Ht^qQ0W^y3wE zywvjKmsHN!5clski%rX7O;HbPY-zl?s9vf-?EBE<5S>Aol_E4VPh&H#NrGk~FIb0; zl@3e_)RNS-hzS;QE)I{5iCWkn%eHth5!lz+Uklr=o`Bm*JwDow-kWybc*atCEbF>J zlDf?S%4O^!6wZalm?XJSAZ?*;eWzuOh#IV<$~;tHrDcc>nQLduP_ zMGtc97fn}FACEM;`?z_^e{As0rKnb{Zb#a1yI;p0v_MwZrLw{s&K#Kudn0mhbTQKv z3Odu3>Lczht$9wy1rV&j=>hc=wXYd1-D=!bm0>u_Ux_Et6rTfej@oqerrG*blk-<6 z2asjn53i_>vUDUP4C^EvWx74RU2jtB$ZYH#lkh=orQ+E=1i>QmT-nPP_xl$P)3k(_ zJCI%o*3wQhRI)B_c07>Q9o^w)n!^>7^}tiRe(_-g%)rL|#dL;1dM711kkZ9dkB@GZ zNBRQtyuE);vY<+GpG-@%;9b)9Zf?8aicq>1Q(IhniMT~-7lYUCBM1Z?n6GFUGqvh^jo>ExB4<o6lB~W(AR!8i8FQqiy@U+2j z$(1a1TX!|}E{PF|DmJ6SRn{m$n{a-=42|)QjC|#@9+VCADcPx0svEXX zryd2rs;A}Jo$c>1=z^oWYT|}5tX+?-CvN54@EK2R#!`&yU!S7r(z#|jW~x48D%THZ z6(r;?Kxq{ub{9+DSP~1m;D63&LP!Wf@Tv(dc{P$=&;I@Wf|Cbc1qm|jI&5+i*^SlB zlsn~R8O?RBu*Fe8*V50rShuM7Mz3n)_Mr#Dp-D?6;BKEu-Yljto|(Rh-9H}GRoCR> z)L0 z!e}GKee{?m8BXl86CA=L&Qq61(%mzPlT5I6ZXaaUz1*?CJ7`4GzmJ1*9pX)4dRX$@ zl22_a>ogL-GDewBLAFlpAr>oREW_`K9xGLy*{U*Z{)DW8&? z$l`-b8OLqZ;%EGNEGOBSDued=@ZyCGmH1(SoN~Q@AhFeAF+$|$)jHQFiofisl>3m7 z1&=jRh%ifgo@V=CkzJI*P~TXvb}!WPM38y31vk^Gv)sGnP!_k?QiCcB!IVGu+th@& z_=!;kr&NfXVmvtp5qrw-@Q?)^o-;bAeXJKYom~zPOM=SGXloM^se{;x{L)))y(XO! zeB-{CW9?rg$^$Rz*;c)he|kqACN+q^xe4f|zd>Bx5W&9*0gT`vuA4#`qv!z-yC;fDE@#Kc^BBK z`?+&B@TkB4A~(7Xe(P^jvOh&`&ozZ+2HlpKBLy6OT_i;Q_h?Dfo}}3(sI^Ur<}IRw zD~PR)F5pkPfUHtcQe%3p%4+J~d-_Z2LSh%=X!pC}x6lVvq-#$Upb|2)wd71y1srP_ zj@kEHqgkU#jW>cj9~rGbB_k>O>I#^Woa@k<4dBq`4ds2%-Uc55wbemw*fo{F_R;^{ z09&*SM63UUCjH0BzdwncG@1xRV!sa?`{>)zm{iTI?!Ug#X%=J^)~|rF)CV)AvBlX~ zceU6r2hvQXF-UvFy@l5D;j?h0b1Dcvec7@H9v8Hj1e!x{RY-4T9FT3>zxZqEK3&(h zY<;j&LJb(_?0K^3#piA->Gtc;y@l@6XqrgCFf9XH{#R9(7s7DI@kR|?6Vuj|) zZ^MQDJE8%5EIn_A)7PkYYWHeTW!c$tCUPf@`?@Kv+7J>{z!RAb*d5$Tx~BoHoi0QT%AL-SoY_1>0_~ex2Y+H*<8%jUu4g*{@=kvm)%;KB%R_NyBEvsN z%5F&8ZFG9zx4Hi`b~AppvVy4I_5V?}nhlkj6f$kuCR$k1=(r1S})4@G`cxgYFm0yqTvp74{9dD8#tvxCkwIuV!QSHS9$+S}3eT z&j7qf4Iem)q`=_wz|gx8$T*TD=IlrSkZjtQ@Yw+nO7Wk4mcCiJJ9)(ol;io!{COW5 zOKx$S%f8B-i;0VbmcJEj*t7VW9Y5*dciS63$<~n2*DsZ2#u5K!^g2gSbTArTfmE+e zJ^Ehvh;o%{A?4)D#mu64m~Z(UNptz+!`PSdJ7Ur7*??wky@o{xU|9oJ)U^&^r(O3EfZ>c3&To^ zJvxt+&NBvG)2M=Kd1)u}vhQ)jRw;8EP-{LL-HaA`z6nt=o%L2b+rmxoNto3TBF z*<4&su(HF1#1L*0p|J0^*l8L)Rcj|Z3>({DhL$p{eH#51c>DG_3OYHcN62d`)$Q9( zr(Rubyn~mIMs?RP zAxUX}Izwv4Jj%M#^)E`%a!Tmz{AguX|$9E|EBsA?W5kowp8o%;|L=jl8Ai z@>EYmpPtibs1Tni$u!sM(VM7eC-awM*b6)+cBb4(2o8?B0cZJOFjK|O*xn$YSN6z_ z*XURR+hYcT^a)d28sfg7RACy#%Tc3sNCULrBd8(TgY--v5)}v$SJM;jWzp6nEO2vSaSg+4!ZmBU*!|H?mKTj)H87+x z%n$EW&Kcfg3y+lz?d+IAv!mRcCs&=5kM?dyvX>BXN2-5N2zh+&T0k5FHY_4VZyA+y z|Drng2;MvcfyhCJZFVSP!?Ia_-y&_VL-)`!H_XgfE-$q=Q=KoNnHKJ`qfLkJ;>$yF zc=sM<@nU3|^5jiiSc~naKzO62gehjj7CpU>MyR)3TN9jQY`vT4dPifS7Gro{hI~z> zgTu(O{w$j~TXg!F^wewJ{RD4U9Jrs@ws+Ul7qYSKdnGHWCD&d=vgbv3@4iwx>S(Pa zMK@ZbapoywKy1S1zFAin-;t(O=LfW%f^%KcrKPPzeh+szxJa<=-OW2;&RbJvrn(5* zIoT&eO@qV6|XHp625reH(soKUvW!pVjKnG8LF#e=g2w}x8MxcZ$b8AN1rsMx{ZL)oixX zUZP^1GGN)^(YfD})y61z1lZL&hJo9Y)bPu?zbzYt9jgWNYac_pVW1`%$4V< zFhO!gIIJBl{xrNx@p1D5YyZ05w%ukd6{8Hg3uJJQ86J{7hSYfWd2xp7SFxh?RqsM? zGi~>(yqrCLMzVCf1KX2rFx-`3hZ|1=q(bVi1Qj-^!=^)Gn;#{4OEB8%_$34pw~JG< zI_=Dwxv0rHRAtfodg#)XB7Yr%_zI=reoW)(sTTjdu%S$G={769V+&ob0^B&dX8RdO zX=g>Il$?|7X!1xE&VR`V{>y!(KKQ5Wc}dQ1)Q z1va8Vze#q_BpKc1L^M3}jJBcBcRO0*$JJC}dZTW_itqb+d^L2HbwwYzlWaYFDqbG?r|f zR31MH5bp0cmya>ot#YgY7L&mWIl?`bT{{Q`jXIVR2a%J~F{dcnb~$a;)8 zq!q1o4HsFJ^}%rEz;JLmN*?fK@fe+=b|vLyVDV zVoYHafIzB{5Ad=Lbw}qS7yyFYK|bffj;G}doZ2!SuOdm%zG}J{r|OLkk|^VEGm-TA z2csqIM?T^5{A}*do=E&{AsI@kKf_Skd`=va!!E$%pmw4C6zu%ojOj0q-{k+>>sH9E z`XIVEFfqAhpi63xp1jtJz9WhEf-G83?m*xMyeTP{Y3HFmEaoIUr*6K|^Jw<%cu?bF z27iP%Po`F@y0&)h(SxHpZSu}fap8V^WhT!Qnz|RVJ^0V(^OHtAE=M~*x_*9^%$#dc z5F3VH#j)1_MYa^fz2>@Gk)DglwIEO&pFWvnQp@hjpr6V4+%@)?groArUe9sIgK7pY za4f206xy=0nVzymmt6~{5Hb~TCY9{zAz-anO8Yul$D7r@JDc95!oIrf_TWyF>0nha zGS|J3dXWF{g;J8Le3Z7g3CD={rK85LGLDLDtG+mFc3(FIp*EtI;$+N5dm4G}#0rvU zJR#~s?J*S|=#j|r9bBLr@-hGV=#&XAJ_f;p^HB*(9aqzy!!<4#u~jWmJ7;EVa@(+z#|0lfcz9Z| zDmH3lc`9-^^}z(&0q6bg>OC%odytA#_*XD5M7#*_=y;e!g#TZR~L+J+5ot z!nh`}yG62^E&iooX?ym1rVP`4$H`k#ej<>d17-Y_)9H>y580 zoYJx!``5%?nf38C(1pEeQd}gY&0O{p9OS-2{SvUOVC=zFA&>gn(K z{0CME#Yu@_X>D3A3!V7R@Md8)rR|HBihPZ!NR+}_lp`o{{d%mtVHL8>RdpNF-fQJ_ zN+Wz}Nw>J5(6M2ckWkCxKmkj5>e3XK@{{(X^rboXZn&L(PJ4QRK4&S8F!W;#af_ea zX4U+W&dcr5-<9F=qqS`!Hx#WtE}N)zZwq0SlA0k7wEIFfYLaP=Y`kp*PK6kTB_TR4 z!2{!nai7EH_{ZUkvutCXtM!{Upt)>H$FuDV9AsMK`921hJ$3p{oRDLFXxEU&a~nWI zx>d0~IRF}B0?<&FPjxxbq7$NOIWju4&HJqI>kPK1S8iS4s|aefZ|6X>U>vXr%f7W$ zFnF&B82}q^u7s@R!17bsC#Y44s&~g_gEliUM2wlS{vMcoRF0kbqixt3Ezzr4M7v`U3rlwq+%q84yv*t z5m?a!CMc{N24@WS<-(_!kYjBeI7@XhqN7_IMVXyEwpm4g@q41sMp_4HZqxH>{@*Q_ zp9&IjE1r^qDB8mVO_K#6Z3U9jFuhwNa`!`{~VntLvP)S8c> zLn(2EZV*A(H@Q6UpuqdxaDG67lvyQA6pN6Jqq%Kjt6pu>OWrCtX)&T+xXsWS&K5A{ zM0C0R&69y*40&hW^z}Y{I|6CaI^NY#+LT6LEH};qxliWXNtA*1Md`AW1EH8<Ag9=RP*e;5Trob4=Kt}LT}ggh2mU@iA@@Zw!`Q^5nQXDMXZ z^e+c|D~jC@;KL+m&n$db)Z!;ZEF^kZ^iwh+=KihldB`@ao^07{&Xsz=mcp=Mi)zug zAyYblN(6NPdTh3_7-Bs9T%d&il`wl z4j{x6=Uu97slAG$zYZ-&laQP@p}1@AGGMa9v-E-(MD@G{Rp>v4MkG5w&FPd+RBV!{$=xPdU6|!;HqWr z;ChDUlE=k@{?c4IwO8$f4O>faXJ>4}GA0AjCEVm${3xz~VK`h0%I=4Gdd%0gEvDue zRi3^%gwahnj-82{S(qGi=7emyRH4g+;?K}*=8w?VK-#Y{n8;e$=v3Sh6zZ&W%aoNI z-gv&vqlfKtZ!4${V4knS>{rXm?#+&(y-{))yR%@{5r4A0 zb0_rHUVIlk^ApH6QRp+#GBDdl+|(`5QU4D@d>|olZ`pX9W!wU{PxJ2&`=<(foaAgJ zDSe8RA<*LByEF2tkQCmQ4ZAz-3LcNBz=~Y??1bGPEpy{!nwb$L6*ri%itfHThsVcfu%pZ3<=RQ+y@!!H*@J6sUX@3;BZ=KnD_w_< zsu~WE+ zYFjCt-{h!dmvYHt&*}HKyUk@4BpKF%Wt?d41c~LmNWYUnkbWFmAREN;&Bm0b+1AS! zKWSl#>^&33Ab;G{n`D9rvd<5&yIFL^E$`K#+qswBh5|w_^yPOV8d~AYv#Jeusx0LcD*g%AJXn3H1ef&_u0N31q!mH?c*!Wpep3QnZ5cseQ8ss{P#}; z<}Z4O-#X7fj^{6f<&;fchiE}lkgdo?Hn){~gHFh>8N=hX*GQI1H2Qftrc_fRWqN?k!XKOCWc56s&0soiw+tN(W(I& z$tPuJo{`SMj#&Ex)--3w)G^ub7sqh$U7Trs5NsFQ2VQjp9VR@QaC~urwyac07%bR_ zaF8MPGU_3s19+#b-Jfnke^N@Tk)La+mSkyCi;CIT)U^lFRK%!ulgNt-dqYSLT1VlK zbNW2&?9?*c_e-4ANRZmjJs86~Ic0?{CRazA9?t%-e*iL$P zEuo=4Po2uwZMheH)XZDdnMXKOw3nQUo%kW^7(!%qw%63H{YLIeeh52qTQki2rKKrU zho<&X^==t`GurBsY|^IuVM|_Cr;IU@Z7UwjRrcF@v_b9X1UqL_%1b$D5rpu&@iBN`xhC5=?UlxTL`eR%)Nwz{9{-Ft{`9yunVj~#aMT^RJ0Nk9jJf@p zNc$Hn509Ip=RuKtx#gCUsc`y>BC1G8)h7x9tJ5FX&`f&^-hBgI#h-1{Ub$EX5>Mlb z5q%*3yb_s^d&+O=(RSG+x1IgixiWWbWiw)o;>J4UGL9w18PJ~1al8ch`Jm`RIVXS$ zxe*oBNL|eKF6>4)yQ%{u)Lj{eFAzQk#70XH->pNQ3`iCo(A1JxnB=y4ayR2B!A=2^ zgM%)-+Bmi~+Uc-0dIyr6?P(-)1m}cqZHC`zkSFEmBAg)q*ESPa@AA(luF@ z8*$2z2L9#V8LPe%4YGauDKR9x!3x->4?>8)d*Gh_xeROdzY|C z3_Q#(IVy27hNqoD4_)tdf5h;fTwSbE*ISZjD$BQo4`>-WYfh(>wa5vk5ABl@^1L#Q zAW>T~g3CT>ei+>RqUWG1|6!6=RIG&vr54$&Sk!>+tvls7v2>p=gIVWv98e-FTwq%XpkCk+I?-~mWhAHc zjwD{fZW4tt@dsqa-q4^R%j{r7WyhShMjgj7vW9oH8aq33YL-jath0LmWgUL1Vg8Q7 z`6w$~+MLPt3$Ta0QM-g2bf$_%-t9OATN1*VL8;o{?N#fl!Q1=LOzeRI>1AeGC~YeW z8H~B>3~+t9WLWu%Jnq;QyaeWdjrq}ap*(^FF#0lwZBYL|J1YcN7vXA7oDLX0oP~d> z*q@rG+deK;Zt28wAc+;zv^CHXcVqwnMK9N(M-kZ84VdVYO+Dyg0O>~H4>p0%N<_=BBxm<>L#VA7?VA$^ zR8v&%qQUwIbPfYalGm|kTzEb_I1>6s#PHP?_~bMPX18jK)jXp+5EkSBRn|5`j9o2@+S<##8_hDB#L54MzWL1R z{gK&{I0uq;EsGL02MNz?4Y9!;nUa%kgXhmMdc2s&3TrkslVgzw-L{{qN1j^dLDWg1 zWWAx)Cm*-O-K&D>>4ClcsFLNl*fB^h*dM8#41yp~AD@D+|VE0DG12^gKt2;8VEW2M~;IBw2uaCYi<1B8{^L2;05R za^oO*?!Uto!nBpwzINKH(coROH+dzVp226yq5pUUGau?WC=j&PxV@a%Hsd^!Cnf5} zF-C>8dYgfk*=eC_qmDQHF#%b&9IAea^X%`H?iycVGQ^O~<0Hw&h4-$+8l|pUtG$yp zd{WVE>3?T*i{JZeIF6Jd!lx+=ws3(C_F$~@2k#GGpTl9msZZrkvTCyJ#zgP zXOL;q9V(0-ZsS%^N%6Jh+v5^|tBp+3`g*mCt*p{7jro-vez!Llb!{OIze$2vc?&$y zkLKUTCmYEN_#ERTY!Sr$>k2;EauGJkPIlx2k$^J53Dp7#N@Lr~bpEu}A4Wz{Vh_y;eHv5NKA{JnJaGI3&O!!A2(S!5?d(Gg#SUn` z0Zj3O>~L&p0#J>jT5%-2V2%nTA{xa&jXE0O$Iruw4+gx5RNOsawbP{mV@b9U?bs8C zc!^9#ycOfXP=eq%^%l%1N{nYIuVr#UA6>=Pqu+9J>yTdAjdxK^D7ou zQ70yB$pbJ?wmZK5YzL&j6@tTNZbrx2(?xC$De?C%3n`x_b$$yb`>}OSYsTf03#1@< zbDX5)l%ATTKX`**y^nv#BPjG!uR#%erM*tksz1?vw!rqH7ftGwhDL9&{s`qNVe`GC zSTr>>rI*EqC6D!hkCrxIRsh1=GFyXQ-mn07xCp4R0ey2jWguX62H1zrM6lUhGysBy zrV%sGfaJ>n%M-Raso)MO|^HyIBg8$vn2(9&Cn7WS%U zR2exRom2@A>=M6mS&_1x=V*2?R0HRc25;C?1LTs^3Sd_dz)}!OE;DX~58JlDKhRTP zeaTu86$2gc(HaogDT8egh?Oc$Mr`XP%{hzZZw+BP2{tMYd`rZ!?0Ux3>_W+B?ld8F zm6+xGguvzn`jzjS>bh;F82!}m|G-Mn^n%BEj&QyZ<&#yW{=V3EB{Bf*RaAqqFQ3b- z+q4l!5;Zv!I%pmyplrI11}d-T8*CLHz$5uPybDuein9Twhk-Y`tGVVe%&z3Ius+RQSfdQ ze4aq`F_xvoy9i%g6I_SpU&yTuH!az%L!-X2%eC2QV{s(m!VPdVG6cKI?G%Gp9b&@F z8BW+UOfHG>1+gM8$PHG49d26r)4_g4)_Q>HO^dk6mn*g!D#fQ^&>v~@Wm|9 zs&uoB_@(bG!LSIrT$JxT5ScWkEUj)ozGBiZ9?Ne@klU!Q(s0t1w1(|BH5ZV_%atb< z)BCm%_^9nZ!>M!77w`(SNZSs&F@@{1M=R{PogNu{u9Pc-&Pba{)I?XR8Fb&* zq>v7shq0~EF>jOlmZta~EA~BxO(9C6jeU{ON-VNxIWBNt;#$A=6nwzc)*b7_-LSUE z1WNOY(7hA21=u9X9z;X(Jz$vvMK(AFR`K&elGTM4_)KdoS`%`(K{kk(eJThfrf{S# z1_eL^p9Cxg{vG&e%XwhnDk6}UI8&jjUSOs7Bk|EF{n+IRIqWfjg3=uX){_hu76b5; z>PGy01ubw{LiZ}z{CF~=FAcs@^yfSJ6l2el{`5*#L4$BQgBYfk;1^4Tyn~YG;wpTJ zMd*v^gs+KEqCQQDF1!8-Ja73;_evRISBu`x_lBZ-mSxXndEDwvX*WN+PMX-z(-ikW zlFZ#KIrsOK-vI#RcjQrg#q|5jAAGBS|M>aS-OVq}&tMtv{{h-;+scaC;+X&XP~XOj zQM#y<8n;8tK08Iq>0BR6w}7(C@bxV*kpA-hef^yu;-%t--jRqdPD;h!E$8fZX#Jjy zCDM(n{+!C+somF=Lp@yf9xxv*x3@Y+T(iXUhehNt0J+E@Z&-kgil>j7UWn7iG(wUU}XwZeWRma<( zD|ZmlokmH% ze+qtEQlSW>)`Ti46dZF{PWfWtjISk>Zh5~#VQnXz+9JQL!MU|nTMaVzXKDR@?%J8?H2mi(&V+E+#3s ziV$!6to_l?CC+}lkDYh-nDcuF?x{TN#>yjb0RKVUF1;Td8Tjb+ojKzh>35omDeZy( z(z7vEkslv}_;{sXv4}s=fkU7*DxP4%b!VRAd{m_Eq}4`#xqtxy{J-yDz;8*Q?{^Im zJ@>jaKcEP%2FYqZMi7J&h?0J96^;Kv41RPp`=e>~o8`7M@PkpxgK@=H<3iG$GH0b$ z=iIIFkiO;5ZT`QQQ35;+&*g-{G4LdCL{tZHYNaZ=YjNP21Y7YSML>D^e*nN~LX z$N!#Fxn&dPGj8o4$B_|o2KGDkjf&tfiVaN%v?nS^WO#oW^Z%(}@ke2|@p()69GOu? za^hi^?UySLhd+3KfF4Q4`shl~?}Y>Yxc|?3e0=qHel||x6)7lVMS8teqxd>@n+rsE za!AotIKXP;z1SUIj@>&<(l zZRYn+u}zMP>{k$12-xh8yb2y&mi*O~3nZI)Xnq#PzvQ zIm61RNGy)0+)&#&cwcS=pl6;+L7;rv<|Bm6l`4~45DsJEBVlro>CG7gG zNuh@?&blrmqJ>mRSqEePu)#i?oj)6R-24K$eOn!Y@kigDoE#)V@R0JlVk-lgKt!ex z%_&-vq^)uLHvi(!zX#oAu1zk!J(RYP3|*Wy&V|gULRLtm0(Z-A-vW)}FEz}!tBn2% z?0nftKY!yjQ2#E*zQ~s5BvoQwbSj&|V~e2Yi*dVBQ^Pe(hmdLZzTXbbeZHKS|SYV@OO-vjpe`_w>??V)s`mjcrl!CE}%V3 zi4dU0yq&b|ucMnk8_V1LoHxD$0Vz9_DI*DN#TzvwtF3tl81-Fpo%rZNafC?Q ztem5ri2lwhPJWVZH95wN)3#qaX8(M0aI;mvy;2*e2$Ax%eQTV8_n*5-eSTvo2+i^lCq4H2&Y()yN4k@1y`<{$#PE38rb-uH9Sym`@n?38HlftN>Ze{Pf zo}Jv#xA6jrR~fCYz^*)})6lK47HQm$D6yClA<_2~JrFQ{zLKvVRuPltZA%KZoma}) zXxIG%5Qv{_L42HtEIUEd&coMtL=vXRia$sjPtwC7zhKM{J1dtOd0aoP$dZrF!-Xk6 zem}|Jb_0>qRQOwYUNZV*_AyS|A^yO?1ct%*6HlC)0xTrF>+H_XxVuw!SDo4Y z^2aN=$t~x;=iKu?&v~B%(SJ^ZEC%uAxIzLXlC5@-ByMx7hd>D-H`_9^>ozn0Q84Wv z=NZk>3YVgQkpsk$bx6QvqoK?;9vzuA5F;agW)PxJC-F!?5{RkAv%gh|_x1J@$b#RX zsWfiJgn&$yLt8YvLqPY=p7z>QWWpHW zF2GlVqz&;j2|CbKAfvgG1qlXciu8zYYT-plo6emCDLi9F$UGsPwJR=5I!K)P`NDqm z{1v+;6}>3Go}e_ym-4S(-uf zbgLA9E{a)59|AJFQ#?-pf+c@u)qQ`jTl)7vsUgGZkj#9b7kwkRt^pRb zCi~AC;{NVX^ouw6*%&|Eq!`lfhexVZtT~f((2l0ky7wB9J96Q34|kg4f1SDhF%Er= zEkRTC7gN&T;X>c~w|}sU{wwbM8{hpyU)W#6QNHCd2u}E|9qB(gKEVlp83X!-4Eb9+ z%eOi) zB?t&XKs=t6Z0wGtvQ?{+>)wF!#rys^HKh#$hZJA0Qs4?gZc3z?36MO&`BWk{#MWd= ziWHt6pwSYK1)IMvD<7(C3rib%VM%l@ZBTjM+UR{;$VVK-1Qk|Qm_Q=G z?78~8-b|pLl(I=|_E@H4s=+n2&D;#4RURY3_#ErO>YBt%QtyRl#z8Hs&Y zvcKtX&)@+%|JVntiUJ;49d6BWmNfb}sL(7?12kikmpMVok2~a}X=k%qe^W_=-a#PP zg8Z(dqrC671|pU<9=YgRtfLSp+J#lbowv;KO`P8YS!?*@?re+)&-o`LMCWVu#XsX2 zF#>=AMbj@}NN2V~m=@eAdi^xW2bB2^(aY(sRq@FJ{7lQp=A4o5zWzEnT|_m!=eA{a zWz<`^|KZ0nNcSv8E}|%`wjI4ukIyC3jRoH61-hX&hxczcrf8o$-dqve{E>u4Kz8P9>m>i$a3gSUe~S@6#)_Y| zx{>;@H%%k3fn$LFlTNjJRY(v1s^A>HEOGZ}vBHFL#0r@Y)yJ*W@&TM}2G-?zPjTh@ zyg1txVuz#Ia&Hf8J-DI3^{!u_s~Jo-WMk3QQ4vjKXZtj{{Su~AaT7EkjB!Rv0$O{X zH#=`{X+|8?w6#tMV^{rNQn2SO8`fqReg z#bM>ruQ!Zw3pR51j21nM4^%mDGuiJBWx`bXQlR6>pe8CBy`iv}?yZ-F;dl7Uz>h@J zbGhiJ1o5yA6oWACU||+roM6H-0=roQw$7KMYr*1uo_F(=+YM# zdQ1Dt;@CML!JPi<`^#0Cm5clQBpScpE04?Hvbd(iRij^85*t;>GgGzB@9EK`4+*i;_vM{Y`Z%Vh6iTc-BC2f8~y#i>2 zx3942lP%@L{zF#W<#&&c^dRxVi!lMPI*uw`I3qV9kqD!A> zobFK5Ud-z*HY4i!WPE9P8n28BK@`De#xMIpqR4!}BG8VoVUKO!>Pguc$>10*%JcgI z<$bsEY|-K&oXhzEoaLqL##hm#WQ>E2M-7%i_;Y!sRrwR~Rj)lAuEuLUMOPeeU5Gx1 z*A``qqor799D@bY(L>GkQ4e%fcILXLgFi-=CfUo{JGU!vNxAe?yFc=zYS4*GFHN^h zZ?g5cJ(TK6rryJne`x@jkG036G?z!?y=>aLmF%C^&0MGh`xE=oX6vgcU3V1@FRYw3 z^9*W|fCMb)`0C8d)toq2%fGZnw6&I}^L7FfAh>w{tZzl1Qf-W78`0XQ1azq1Hx zgt0B=u4`Y)V&y?9hsMU-P~%T)T+0A z$0_PgvpDME&ymSc`AV`&K`4YcU+yQ}^t4p%~tl5R7tn1mz<7WCsT znb{JHvs-JvWvz{-r}W^KbiAt221Ub9lsvq$n~dZWInpE{T)|%Z=6+O5Ib)^|pgH$w zGZ2D;?H-b#dwtL7qVc?WqC@T6ydS%j{!)OOk=}Kql7L4;PKUdX3zE*zI}y7KhkRJh z$jO+KVJWRf3NRs?;QZH~_Dh%Dq-sm^Z(Y#BAKwGLf%MnRz25_!F$@aKKh=ZZ*X2S) z8_ixeSi3^z&bVu1nA^T{a{EbaePxlY9u_|C3%=%kVpG>ye#qQbuYs4k*E}+@LDW=+ z3k&ttR4FJC7Z=D{SV;?`w>jG6xOGF87z))fsjEKGhN>%^RXW7{HXGdjB(&gA)5Yn6 z2}qa=La^ud+8!t@0qm`)!Qs68mN%*P^c3|A=%WJ~I!|tHKOULj5ezB@a=XC)qDs!6 z=jf|lp*^E@V?O#tuQUu(45RT=R3z_92P10P7?zDw$5oV)3ST42tYQ~b&jz_gfrZ$2 zN16loOL}=kj%MqI4IB?jru`_fS#Q0Toz{nA&g!yuIU*u|GB;~hoDW^X6tYpG*sY4S!P6S06y>Bn;1 ztM@KMF1dGgtQ?DK(bSS*!M05_A=t`yocH-!_>>#T6W_TP*kV@Kx{>EGW`gsaOI*i* zS8Q@5i+7Is9y8hlwa=zvpN+5j<`@`?Hsx=ap0Ku%8*CICKE?1@X(8m^Wo-U;XD2**598g$TgVmd zrn*JXshd@u~tA1;EgfSlC| z^28b@r~(|;PZps(>{&}+tu_*yZ%m%y!0ZR)G($C{r!vRmk5BU0I`LQwdBgC~-m{?Mzcr$y6PC@{g5nK!v9jsa$A5N=o zyuAi{q&EI|RUyeUwprqIz;T5>LFG5>?T+%5X|SYu>COYUtiUu<=g}0h{2aE=RkA5L z!WV@qyOP)=6SHZjI&wwR@6{YjH))jk0ZLl?l@_i6n$%Y}VjWf~(!~H(x5ye> zOlEy7{YAzDyRV}n#Pkk+4(q#oMZH{sa}98pp|-~jCv#p)u;lE|BxXvTpkP+(lRBAY zFj7Y!%UW4=Tg&fLv=TAYjJC;#zcOKB+&!Bn?Frl($-kk=aNExu?{=*Jm{Fut#S;^E zbDR)HCvx0`$ar`*_JBA_D4dfy;UcxL|E?PCh^H;uB&bRq%AaeVF!~8GsjEnXkS68y zW9)E-5EbW$_ncl=c)+$`FZINo@l4*ktJ1;=es`0w<<<@vo;eK_xuLeSxYv)T&fNXT zC>FK!hPDWc^tIb!%nv9*PQUI}u;D%Q%M-0HR~miqMHrtW5??#w9IW-ZCD3Q$EufVq z4Yl)%1r}r+yS)dR+k&l;WGA3JrVPb_=f&ZnyeuBDh?Bn2Goy25Ww5yuEyVCRDJrtl zwzJA8MQORRvBIx%eCw79<8V%Gry|2-GS|z<9;r}Yz7MY8Eqd3k?KMDf`U34#B1^v6 z;|3+9ndIVRnK0=BM#z*r+TdT})))AE1k|=r>E|izUY3c{;K@ghlL)R4ltSq!3-~cCuK7J z$#rHT^a7lm?WW_o&e8kOXs%ECCc&#@hgk|gy>~dkA`;<{1;V>I_xm8y6ckO~ zEiCysjqh~r-vc>}S`W}m7{1<;AJyY<=M+1yG2|3Er(NaY?U03ItEeu1kW!j3N_Hab z36HkmQwA1FX6OH9WSpO?NOH38xJN=yg;Is>%8gy$SM(Byw-E%GF*|#YJ$P=b%8wu zgwpRfX)KyGN08?QB`Zy+NUwHV@%+#;4=p9rWb<{Jkr+%A1E`8qR z4e{-lItLECzDlA&GH2}I?zAJXZ;AJUEgZ~yI6;Dos;t>wpvT|mJGI-m2g;qdc7TqW zBwIGh$n+GWNc86p@^!OiG8(`HbfLn>+2I{6W2#aGmw1c+HM3N*7kZYM|Tps{S)H{n>@Mt(otG{w_S}S40h$m1r|~mEgdF7lZmE=w}P6%hfuS8mWah+NxSRQ z!K7~%T5l@!`Wr?xdgVLh`W#EQ5#W@c60uD;y~`;{^O(vh1~)fA`L}l_%yP@mB(lJ+ z1-rA}A(rbgix@XC_lm7Ec7xEs2J~vyc=!f#S`Z{Da$ab~iSw5|wXZ_&nCF{jY#2XW zA;GBh)?#XK!4*rpo}c4U+!UT1P*@HChORU0WL`uIfA*NFVUR%*!hhM9H~JLWM#nk~#eoGi(#FlPWV3XE=yE8za$;sS!Yec23^4jzBpZ687Xppuz|tc zL7pTzR~fSuU?s8#3TER*E|BdUzCAAQeDKYo{25hm3Z1Z16%Q?XW`KnGH~>I?b2fxf z81t)qdH(s*WkeCU`C-Hn(F75A=*zCDOs+QaO_$27wS4)MfYdT+?TGUjRKR+?Wix`M zS7Zkw(cgJl_{~(x{Qe7S#CJVeb22c9=N>s&Vap9fI&S z&$r!(?T|-3Bcj~L=`_pNo%PawWeQ(HTRo~-pH$ep-ffR{3}8 zKvN6$L}V#|tE0KXPX?(C+$=<{wN6rTTo?;#@i}-?E$JL>K9Ks7&xx7PmBiblyDZ{e zn1eG`uPt(B$Xs)lU`wfH%HSIg+o4n&jjnvLSU9!|(O!039%~oFG-ea|M=_rsae%=L zQ-+7x|yT(57s#`6`w+Y!oG?q_f@+q!w;DMB! zd#rv(D?90yp8s4I%S8A=(92CceZK|v?XCP4%XmTVyH$7_M|Yu}@{!)~zL{S2KAF1i zym4a%mvJ>ltYwRSAbk9}fy#`-hd0;LuF;$szO=v1Q(DM&fIBs}Aw6URVvX;D@7SYC zyi}%df2skCNf*Th=^<~gkMK^Y)R!06hmV`7agZF@J%xB| zS8ZJUK(HxBAy6u4M~6(deu zvF$hyqZBsIm*Xsk%c2ZkGa49#ystUNwco7@X!$>u0=chxo;K}-%^u4Hp@8uz`07a79(-(F2m^VWxno|Dl$ zJ<_?D$;y*>tcow!Gf3xMCh`@cn>FYBv#MUza&wL)>V%+ls47iN%nD2O94ar6KD2lz zSGR6EJ>ZG4UHkHrF!2M!&XSK0m}A8o#OY9!$1e0QglZ^8DTtW%1t0DhQDUXEEy_@2 zOmVvu$a_I1FzUlr1UzyQDjtM5fo1b5-aX39s-haB+PYeH=>BHits73E{*ijy8Hrhg zIHTq1jojH$8<*meC>M!ndnsF|lV*#Dpd>PT0c+;eQzy%4U(3gI1djD(R`?`HC ze#i1w7lcp==45B0{U}3a;OGhuyzEJ+;&a8JR*pQ0i8;etjfwPb%71ipS z$G7*ihcT?GZ-OZOKAoF0ejum4?ocjZWunSv{WD-klblrp1ugQK5HF z4b8t!lL&hzKz{zX9?LByRtcg+o0gtrY$EKoNVV1n>kWT*)vSX$@llA8SMq4dN7B#% z1N0n>S~a5{&sY}7_dbL2?U^m}D@Sx6DQHN`lr#Z7jq2YI2f9n8eS-3WF z_FSt6hfIS=Wg1;c2#m;#OguQ$OK9nVdW4TxVh=5R>~n~dFlmf9GCNS}rycg;N!{tV zu$r?6S@tVV^yvsaFlIdvnTtQ`O))?B*@N0FO2;5BEypF2>2@eC{K*)(9;b0 z3k@}#mL*UMT0Yf->xU+)EXBi5RKVG2O7B`3pEz}vp-#Sw*q#I4U&+KH>NGs&2lk{} zY8cRs>nw4MWvZU?pgm7Y_GY@Ai*cbzyoHUprcd;_12xutLBQ?djeV4koGq6UyjmDI zh~4rpq&)8*y=dgq#f>P&rlY;Nkf#?!M$RJDOKtBzx;0R?WfUP(t815ry$PoB6jMFg za26iPF6empwa|I49?)SQ$x|IV5$m zz+!v!N*6UR2n0sJqHVBd<4x^M(>P8Up5BA$FYlGpn71&_J70Li8VCYMLr>D`_U|f{ z#>_a@&`L?a^o(lI>5o&*ALLR=kBn;9st+)IQAkvlOQA8;xPhQWMm3=KWevW%Xk}?y zDn}M*xHcC)k*PnMl~%0k!Q7WQbm@&~JUN;w;yApgq>@5d`E`gopJF)cgE|(_!uy1C z+XJTPF4EGHu}ewUmFW~U&JZbko;(ve(R*liCOIc(|LAlgc0a0l+vCGpu5)ug1uybw zWLsV?N8&h`j}6py1FB_0C3X6hl^+*|bt>I*(qL4jf@D=49ySl<;dE5cU(@G78JilY zu)rFtkEvrw?oAdT7x3NA7Gd=c+GA{9HxFp{RY|`z#IZ+Q3>c=V8#+rj zOPlO|NQtqI7+4!ie3b8WBZ3Wk514sT)@;0@d-Rq(``*=JH#RfN10&CZ$1FkO(Dhkf zyYvtQT*yZ}UCnmvAqDTYFwI*F_Gd&y#S3+lEPmS4qcqE?2pnUUO2!9$ykn=g=`{8Es<+>(qvni7Q+`d_MsB%tC>PnWUH?Vqq**yG&5<}d^n}(#w`C+*xXN5 zVrE_7qt+`hpwxFDIv@`vXmsIh@q9exU*o224|lr#9WQu&KG`guzO27F)9OZudqnFg zE$4BSJ&@}QH&&Qf=S&&%A(C-p}kVoo>Nor-`82#91nnX>z|(?}8Bs-d?rK{Q$o zmRsIUcsO1rq8Qr_59eTB1kVY_H|{El=E^DSL~u(Rb#az1$A(prWf_6Vo*SvCmf6)0 zj^E+?^dfqa@+@-#I#=KGbmJ&5hBc#TLZLK}{2~({DW$w@CnXu^Sm=w#P9f_|Jh}Wk zME!Wn1RuTS&V00;@YNv2;vJO+Ns)W)muvYV&?jCR4&l_00`!W={^sr8Q>_j!UUo2T zP?X={gpLhlsw?rCw0jJ-PROu%p2TsOItk3bmC5w1%XH%mAUbzV%{VP;X#o(B))sfx z0N(D8>i=G(=o|b^pzHsGF#KC#`aj(NZKTfsfYQ(^nQy_8*JgK?9t#@TY88Yak50t3I=)F6WVW_G=K#|L!sV zAycJ5i>qUfep(UBot@kz?eOk!yn*0NsG0H=KLR76z=OMiYBr;E5N_jC)vc=I=y;hG#p4B)!<163 zWP=U8Liye)!g6o0tIyaP}J%y1Skj+u7`}XViIxNvnHF%Knckgzpow*kx2$mnc|E3 zfnGxDJBeEfK>rTXrBI*|+<*qq>mvvURMgz}XojuMWy4Snc+tf*^&d__wVnXSZASo2 zHQSfJyb1*z2SOM)nvn_~560wsq(o#XY(0g&TCbVz^_#|9-iOsoPC&NSdux^e43z21 ztMjg=@x2IXZ`ifW1#T;K6S6ABfu}|>0!>Bo0I~>1h3)>k>$4r1Nu8y}TY%@J9k$lN zKo!HY->?z*TEwc*9_acUtSf7`3)cl$>}PifbUOL@;FFv9pJ#GfzXPf)|?Sx}TU5N<7TwxirX4{j~E zE~e%DJ8miqI+D3MqbjzFFC-s$dMQqW+*-2pZ>5BMO1>zfo#J&q{FVV&{c1+c2SI;1 z8l9Gee>dOyZ~L8(HgLOvfDqBwksbOy$1!eN=D-8Jk2jX~Kqa;35xd43uLgzhw7cBlDbk>i)Ox4%Zt6ZZY@*ox4(;rk_vzbBz2 zeCH2;9wbd#>T*`*-*fguJ8!ZdG6mTK+3$gNbgJB|AC4*DTJ}H-s6CJkl4*EDf;;8z z-M`Kfh<}_u*c|(DMiKSi3NA;Z75d{;gRuR&O5^|E?Mdk1@rRQ;zeWBK_J3ajjzG5k zoZKS(|F@;$|JQ9qgnr4LQhfHi&JH(D`-4}(dmuP$H^*4=3pCc6!rq&eMqTztJa|7_ zv}odJ)H1$=?33p8-A@pzIi-0KgOm_n`R|$P5zh5>`c3^e*pWa;5QvTM-aF>k2!TI( zAAykg35+HH^8{NE2nhlqK_Fqi7y|@C;tvPF_Yp{lKh-lJ5E5UtZRr1ZF#mty{t1Kx zfsptUtho6zkuC&6;(r}rC*0q^MzRnHiT?(WoN$fbj}Rdc5(GkmKuG*-3=jwj0tina zVtyI}1VZ8qF#5aJvJi-XKc(#ngv6f;3@4Bi-*I{KFAzZlLV`d@0KGE)7l?$fbq@qW zfTJov=B&t! zbm?niOlU~B-FfRX*r%B4192}3>8I7^q8I_NBvU)QfLtJ!_5S_Z!4Ucs^I$H|4*f&$1m5okxUwT*0BzN|j z--qZy;?_OCR3F#{;OjlQMOce1Ip(_Gc{v0}A0~X51|bi;HT@{lk+Ya(I`F*Q*_JSk zIb{opX8LP>=fiKLU{USPpth5RP{4Zx{t1B)a0l-gmGuZ`IUR&lIf2P8?25P((dxI3{a-1{(5l zDe<21cCE!o^>*O!UVE{{Vfo2k^=oXHS^bTn_hpF2Wp9Th%t~zOSF0u^fa|q`rVxH% z?-!eUvy+_yw^w(U<@4rOlVN6u8x!$^SguZ?I`>_d4kliKrzDYt7H%Bocw33qt;Y`U zeHY!xqr!b#52IoegGf4jS@xbWcWmp=*_RS(VfGxX)s>|A*zaN|tQbFX)p^m)aXq1` z1FM_BR<%?+X{n$_n*CAkxPvM*KD;i)={qkcB-o6{{_v#;ClyIWcPvC>`UPt(L??>B zR!iSHoOd@AB^Zsu)oDM(J;J&V*m~jVi}x$NzPcYFt2)L94MhhNqFJTX0X;FsAMJKK zR`x{l@OLjpfa8MAh~r$BVA(nlAfu@`fktQ7fyIjqkU}&jzs_IlnlJI7_lPv)uUF z1^Qx_-t@`4aeTdzsLuRx(}Dc)^UN8Sbjmg6VzCJpGw&Pk??)V(G%{p8{O%lMKDKZs zwGpuu$7b;S@K-Uz&7Rac-#dvge)YpcI~BC;xJQMZKbuj>`0m=OQ=8kA$uKtX8p&82 zmHTIS6Dblg*5<<46zqJJAhhZaB&0ju zRIBH&UB*lm!M-hXTxh4EwSC!DP3SdAp-cm7=oXC*IjDPNS#wDqz!e3QPiMRHY;)0# zi@wi?T8$rOMLpsd4p~2F)F9`w$xdQKupdqu=VMtgkYtq`S1L^xvSf zuiTS%?TTM7e8b;@X<&AqMb35}{XlFeCe?Nw{IHDrC@sE1m^Ek;u&S}Ek=D(xOFG-T z>VC3cO{NI;&YFu4=GHi{>hUvR)<*iN&`l{vZj8`}cGWbXdee z$Guak<}#|J5>dUg5zYt6Jt*M>b37@mK1ywKTkpG&r^f+lN_1x-IYjDEpBnmSh{YA@ z6s*Wc>~JOu{jjtr>T}0Jv1;P0El%D?qCbQnEt|ernEXsDCVrSjTH$G1+4cgQ=iSxI zU3-zuUL3aT6x~FtzKeK>gCp!Eoh~y z0k~6NZGWt+K^JCC0g z>y=3m{jY5i2@W)ahI?bl52=+Oe>zj6PD_}&C_t^OeX9uw($nvWmU-cX{F6RgKeW7z zv3rLjudiqQkqzeh)MhiPMx=wQy$2mBtZztFb2WZ`HD+*$QQuQD_8-}_ej~lUQ{-qWRYfv}GUIWw`cu(Kq(pd*#$l?+ zO334i@6~g=?<4Bml0HzK?N9P@$l6;?sv4@^d%Nm}v`^w>e%Vq+Ps+#5s;QLGWJ=<0 z%=N2mHFxiUH&-qSimuwVd^oJl*w~+(ivC#cz3;IRY#RO1g_S^@zPVlQz34NoueDO( z8v=bsBS8?|XxOk9ql$)E((L!`wQLlQdmijW6hqvuS0@^z`8Bd^#!$6asXwPCz-)#l zye6u*BG4LJMu|c1Q2)!o22D*b)n?Ucq_?N&HIjNmeR>z1q%0kwV)lDDD1{Zwa0fDR zor5GYO=DPLxa!!mafA=TQKBiIJ=I|^sP5p*?|#o3vK{?-!?NPq<3g&wR__fhb;N~W z&?vSGJrxH5t{E4@DFKJ%;l>N;zSAo`6ia%flQXrK&6=7(( zok)n}_Q$;$L2sqA*Mt}V9>W1ks&U_m9jkJQW07$Fn33c)a(}byL5u7eknw;FKn%uo z#(EHKU_sc%&#pDGiS0~yOC!iYLKFaVEi4dp860OJ3Lh~W$Ttdso%o_k(lkO5QEJ)A zx-sh5$+-iS+@Jq!F;9S4pVHG+!^)#s|yzg#AU)$?C+tcDM88P z$CLa|`jk*KV-1i!o9s)YEML#p7A+`7#UQX;&6L2E`0^VwqWPNArrXu(gY~xA>4?Yb zQEF{k9L{^AqQ#kGfr)JGXI4aWDhMHEJVCv}<|gGf65gC8L61)z3q2wfyd#5e`^Jr! zY4gLgr+aF!nlmcM&y4Ku;!lLWnM?8>XLk60A&aO$nDM7(5?^hV-`Fy59k7ZdK0-IkQ^t>L=6?s*Um@ZizatJ|s#p+4a^z*$w=f_feW-soAfv*GOuY{J7Gi48F(#R`HLMM_|ym_w{OVv%b52Wyx=!MT+>)O5f9xcUQa_T*QN~-Q_lh` zgGVc>Q$>?M0@xD6&$}D<-C{|7Bl*0Fd~oRNK<(g(@dSl@`~f`1&U5*er+QMkc?FB# zER)1b!d8zBG_k;Z1l#U>Zkx!ijV3UbkAF)Nl+i7L+c3G5YrU_^+ry#j>ULM(l&N?_ zOP{+3;Z;^%dJ}Z_Ru0j1@j$lLz7tWKGInGHUp8I}VR9~@rg$q6gg+;HKQCv%&j{5o@yEC*)mZ}^aP70-6zGZ(X4RT9=X=h7pY}qrr2o1 zrSY3;7Z=Y01k<`meRevVvhFvIGiTHtPrh@2{SJ`p(CvI_L`r0ILRVzGj&I&B=k~(d zjy{9;XaQ03uQ3CGF|7#WtiKuxeDlVal2_HIoM}>m*@ab;9NMmwo%5>M9H}917-++q z>bd%gQ`eynJx&xQolYE3jjJgJh`*hvvZ;@=Kh5Cn1 z&)4Mn{)|dWyYGTA&_)w&pVxxwf9+i0EUQ_nNl+&k&f|PmwiD#tmTQ*>zVbg9D#E{% zFmmx8luCB+-Hvwn*eiS6MvlDy`o&nTYpMILKi)8e`%YpLBs!w_q3!jw(zXL=1L#NC z37O(6+eyBaLK(i#_)<<~(S?6@pyP#7^uIZ;*|9uUHMl%pBN3L>5hoTG#3UA5wNgoi zIr&OzSwH_~((Qz$0~QJvRZf{}`d2x1KY^0R58CPu-n-7AAmK-E7QgU(b^G;|JPZt) zZ;h-G&JZSQZt5Y@+`-F?>7l~r?}*i{CttY%;SjWWMFs7&FTFYp_S7}cRkLS#h^ zzyCq%J?w<=sXwAB83-eb27%nzA)rvE|Jje){a|n-(a!dc4j~mWmot}M8Ck3!cBNUk z6X5avR8Nx(jCGOY{#UZ9ue4Op_Bgar2PLic@o5g~Y|OP+n`3P5if3zGfS-Evnk2@L zCom&+QLN&f?Gh!h3z?r_N?!YOpSNH5Z@mj_l<;{2lRXm(2EAhKh|cLyfj;YBe573*h|SzI1J4Obrzi}!kMMfDo*Q7#CbJR8Erm*})>PJLvXBd}5(VdA^`u}##J&vKnDzl8kM6LfRh%uz$Qu6mN3o``|xdiXTvmauVwGEu*?33 zJMZ4!U5%LexLUij5TB*wsqldplHlroXUK>xsZ0EqKPNu&a*44-#m0)qO_9}9$=>y0 zqMWQvJk(2Xo?Y} zpY_B^IS*tiLO=8<7-0& zTRy(O=BHofoTp#->c=w^zp*&cZqUb|hIPk$AaRU|?i90J-p4MfCIVFPrT z-I%pq9nrb?<=W5W^C-p+tqolkP%%h2VmEgly*b^~X_b*+5yhwf%&y|cnPTgH-eQL-g<|uzs8p6d z*8JLi>mI2ew`H01M@etKNOK#EIJTSChPH#6bQ~I`n4Xe8WsmIuf3Uf zm#prUF3YDBaaiy|H_mq8LKZ)mZGFc7dV3@I776E_dsl2TDBw8i&=iyJwUyK2d@ht6 zuz7@*C8mc{)YDXuL%;6JU)mj_dB~Qbx?s9oC_(&uVuMv?;o8g!Y2&PADee=0rXfF5r%lyjN>kAbk zaBcE_c_d}OPIuAs7z;AE%c7$5%Ba;!-O*v#C+8ylayx@t&hdv*2fymyIqi9q&j5&? zjb~@f#b|caeYIvbC4g0V_H*r}Y;pjHW3u943l)wH^pMkJ^4-M8+1@%&##_0KR ze9X|D<>838I)<25I-^8?P7aMiKMvl@E&Tvo4WDlxiFhn!mFPMfW0o{kG`k6;7rV{GC0XU#GR@(T1bl)vL!$GXP94%U%CuzpxMvHFz>cPVqSk^KSm;p;9Z#PrhgRFsMNijN-iZ zx~SV+=H0rW*V_9PqaD^&_ufvTbG4E|lZwqwgV)is^(F2-$9D8)9BdLxV zu7oY{pPqRj_C5!TRBXF7-{`_$HImFTPd6leI3+zBl~c7^D{(~vO9Y$5mcaDG zC^@woljqw%P7{QZ_CD-=A(wguQv)4SYYtFbbcU!JB;JvpVou4QGa>`;J`I2li$j3j z`@^;CiI{c!XXS-c%SIIa+!RXce7c!VQLTwtMPaI#Rg?x}VP;+lHM3KfY;*tV<3z55 z0(;gFKL}yxF9uNwR^QsA^w8=qvSK98=t48q7K5jiEsdRGR5~*dGmEduS8pozAyM2We&s1NgBf7-&VmfpA)>_s}M~VD#p?W+c45bFu zgt1ix^i9yQTt2}T5+k~_2DbQaNb=MyJZCJ;JzURG>k~yP6#WVd@%VnwV00kA4&#-a zYNh8A8=-lyicA{eF6@aES^0d`U$RWL<+`?=hwIhL`1gS;f)CNH=niTQ2t%E@69n!Tc13ROxew-V4jq?!Tx(#aqClc27S8L#g(7t z?#`fcl8z4MJoCruwvj;3uv^1*`gDMa%I!dTt`%yQ2rjPCyYZbu=tvz08(svx+vqpG zq{4NIgs+-${nT4*BePCMFPuRH$qgsBK4(TUkuMN3CqnqdLjpq9talI|Jl}C;FsDuipKXB8$1yW(qB*7-7l6>@`G(-)ny?(yC1warocIo_({dAPzx&3rz zKMK$H8gM;^P2*0};CqeL8wsnz0c>6`uXCaAn!npmd_tOFcU_oIHh-gU zGV*ZWZ(2F?9P#ar`vk#oBiN<7-eXZK>5j`75`xgB!o;2<$;~1bNG@-WdIjAYd-?9$ znZWu)4;>5o{E?EZHUx5t>k0?ySc#>cQTUxpH7=%v2Wo;b z-9mYN>D$ZEpVn;|_if%N=j({^tLV=g7VlKgNyc|4fPT+|1mapMhirEY=(^Ml!69l4;&eEy1g2hLeebmJ_v}vnodWn(MwT zi%Z!)c8Eb=2(Z|mHj{pXI1Y8ap~|3 z%|uQcW>_VZR{rp3YFFY)FcYX+E}FhTS3cq8j%b1GWU#o+r?hxNG>*;5;wqx_Swmpt zrOZ>^)h5Dy7W+z<%K7`tZe0z(ItKF^G6~>(rZZckb$c^6C*NZRJX_Cp_(GFy<}({H z6SMpC{jl`Svm`%`DS<7RyhWem-Zv8e5HNzZe_VPS^x-(xlbUj-(P#M#t1E=gG|I#0 z%Xo|i$*-`Hj7&%gLzz(8Q_aq1{h-d^o;%qs)@GeEwFfhTT(pj-Ufqg_1Rp6at3>fF ze0W`*$v;+>hl{|S$Ucccy~befO;fUeOPyApcqXycr=@lBi+;|}U!8_od3&?$U*_2% zB#`}6RT-_DZR{{pdw$GIj~-OD@1`=PpVH{PhsvWvx9Ch-+t9C^G7EhDB3Chg@pUem)EBf3N?hS zp>;J&FLKXhQUa`pf?wS=Z|et8P{iWAVBNtw_T%FEh)q6c-G+0A$lynNZvt+1$>_+< znh|?{qus2c(MDX&h_kQJ&a;#T#Lo2h5T%2Laf|ag2mG>1JtL(y9ROBH$lfT(HL9!C zk`wt=Cz6Q-frkI42>@!N+KP1m^1*(zPMN9u83^8Ib&BQGi2&z4DEE3w4hp6IEw4K> z%1u<|5z_dQ-tavgYW^d@riURA14At+u#l}{V%b@#T9tC_*MY2b8KF_Kd_;TlSc!|17x6c z^-gsJ)Hz5yNYZ5+fNxlx0wGT)4Cy$ zK8vO{P^@@KGs*o_RqctP(+6=Qr2^?eW7tjK9=pEigM&F~F|qNNrt2d?4_!#L$R_DS z2bbzHHKLA@wvJQ!uh0iHx5-(YG{m@)JP<_(u3b`Y-!CYBL@o^={j~$$P%+$W%c?`3 z`HRH$Rnw6Q2UAN`!twR?_f_U4uFeB^EZ z?d9Q%k-TxCG7Uf5`U`A_3QSU;vsQ*+gE&T8V^MdBI>aQ8b)}8nsSqx2@AZfzj_Wyd zNtk+il=~5JCk|&<#qXN$sXD{Wt4OplAYtCc5A+E7^O<^p-C8_JhwX)!Xzb}cXc8Iw zVz)FD>5lFju){rGayTb`Sb3d35Uj3idthSsJ4voJSnI7+E}_GonS(F243Qw!^tm_z z{NW_I^rPCC?3u1x_@r>5el5!(j&3@PmioOk_q`{dJ`?pV!#9H<3G&63s1k$uqE89B z`C~-QBOn0xN`x)gSFE2AHh8_<+muG=m%*66XoCGALsM&Z~xXkC|n{gXc} zUki;DU&_@cXA}RTd*Wzz6+qa+_s07fOG~R%pEC4LrbQc4QlglQ5;^!%Ohjw%Q@ldbRypP(`&)Tw8_@*Hu-8}T|m{YG>3$2Td(ZIHCn*2%gZX2*P9bE#v z==x{ZrBZr^o%YCw&8WmDov^*6L8t8U@utP+OV6ZjGIq>6^V%~1_79+D?w0NAz10tq zFwK4wr9*C7j^1~T10Gij$rCl8Rc)Va zG1uz7`DC@p_|Bhbb7L*M?@_grvo5a2uro=zPNIWU7Ck_AyF)7dP|WUSR+ljA(o3IA zbW{24^iw+b@6hv_UJCuwy1++PbFmPe?lhFW#}rf6e)eIsfwMlp;Kl4bqIt}j?10)Z z#&E#A9yQQ~2kQL;3z2AYcW%8|67O8W@i3{^h%f@+$4s>?va|0B#&Ifcy2fW(0{t2j zO|HtLpG_Sg{J`4RFg=#GlBbUW*sHtEI<&-2!yOm~sawLlZxt`Dq^mQnJb$IyW)PjM z$5VTsCp@a_MMHF;2?=`vY9hff2XihJk3vnR-TF~IE6q{Fvb#(5Gn~%UvQ)O|Yk_hl zZQ&sAS>;DQxziu6L$1jd`F($szZGhPqTlC3h>#t};#M640jkP*jLOmGTe9ya`Z+`8mSH*ASIZc(-8f4Ud_r{`X-(_sE4|j`ZS=ra_%}!sS0Pti( zbjC4Vq>uE8e1&(+Ih$+QE)luD25LoB%q(Uk_iTRB606g- za7#7*RYj?Tx>2KU`oiN0te98;XvoChnaLIqD+%b*&sGt@aB%Az9qkWxv;zA+Hu*IJ zF_mtad1K;_m;9X>aJNV7Ut?I3#TCmP%X&xD`&hhJYj$Fjh|87+_qt5!ILK)4|B_jM z8WuTUyl-`bB5K@bcg$r1PFjRd<1F?E5^BRT7KDS!+==mFhAP5x*Xfl>qHW+&@o+_$ zH#gD@8i67EHl`CeQOBvM&+gRteTO#f$B+-o<=qI!k_qSSs{u(bOpZ z`-FgBt3#Xo@gjf})wAnAU-3`LuQKvjYw!$D+o{yPyKli6wQwg zi7lUL`W94FIV?w{owiktoHe|5>)CEvPC0dkGg0@Jt0OxTIXs7p%v)l_s3DBJkSC15 z9`g;8BFi7kHbj<7V6`d0@03KB{&um$2M_yW`shpBem1-(aHzCpnws`&coN;Pbm9?V zbIbCVg=vq@(e$zXl~S^4q`ieGE6-I}bX;|8WVB3;QnBRd3)Yc?;|CmKfi@+ln+nk% zTSHlQ?nL?MiCp|=&(Zrdho($U(h(zh%GzrIaVHi*2-FDx#;C_2(g-S}G=`7m>?vJF z9TXSoQtS%Qbo&%^<@izokTGEYe_(@Flt6a{%RxA&in!_G<9_d7`o~G--$`=q8-8+& z{Y&;(oUnk~KtGWB#|%V?GIlx|OgVXDWrP1t!{5A_AM~%NU1jjx==KbTpjNCj$3{hy zN}Fzhfcn-Snb#K6bBbv!xYk;%bm`s$H}Ty}+SC0J4fs0?OQR~=k%e>KDNys7 z-y97i(zwj3SUgbzNfeTLPeOF>u2ye^2XNCw@Rbe{cP5hd%O+iFHxiU3J)Xdq%P<2c zKxU`@p3UJKex9DV@1;|KFkiZ9)w4>TP3>faO6%yf^-<`A@o zLdD|5nRAYj;jui25uBT5j^)LWJ)N)7IZv(^_RPqZn&@cYDv{c_Q!>uKL)y)25E3xX zuCLo)%#kon55aCe3MBz6ZI%G}=nMV2IuGDW16uz9`I~_VD_QIXyTE>@O$rwHYnX+SfsPHpzE`!fnt2~IlR1#zcpqN+F%H(om60&C(I?)DI zbYUa4p6=^Z2XtUhT<#+b9o@z9Ui;R^KeQ>1pgz`)perQA6;DEw?}Q+7(l3{t4}gly z;)mCtG~z1Z)oxph*ejrB%0DEIO`Wy|)(=_s!#uIHmwZQh;Ly_25f45)Ov}fkO*d1+ zqx=thc}Di~-V|9HCGf-Xr4M^pF$pHP!=F%B8T~y-Z2TkLtDD4u$&xj16%DxLWxAZf zqoo%C3Xf)b!jF1uM~7d&|7{eO;egSy8Jy6#>yBbO%0+!DV|`zKsE~Z>M(Ve2?=zIA z|IjAz8fTQCQDQhKu|AUR=-xBysX_;dyxzKJ?Sf5_!msY=ds@0d`Op7$6s0zu`DOYE zW@fe{CZE;(?3B^K;JWGYhb~E993U+Z>)Kn6z(S0T-E~zWwcz^3oct#mO z=2D!Lts;Z4UPdY@Yo@xIKL0mQGy^zSJYGiM+dm+*tB~){?1d-6MUPgq13It_G#_ab zYH!9|cFXMObGDaUbKV$(CMlOb$u(e7`+Mk;8Rgx_b`ZNNd>K}+AXDwY`6@wa*_S)3 zp3e7sOW-Lvkt^V&YoR&M%5{FLfVUY{F&(b5|J`J!6_rq!T=om62-m&KiPXDH;Uh@F zRDRg%;q;^OVJ-WI5mg`hlNes-KLwo?2wV7wSi#I~Q=F!~LM_d$4NnsV#{;2}>!SY9_xS@3XRaw=2Yh-x<|c7|#uw)U57x5tj-#QkSY1KyY^ z_F9d0>=28fmNn%+O`DQlF8G_p^FzGxn}HLo&J~O9t6J_6CM%X>t`%)5=faCdw0g<{ z|5$@*(PQXHXA>;r9TbutwXZYgb}@oW!$z_%DK8fAl}(jDo2=V0SMbi8AKKC7fVmdU zRPC%o)WrDeCEhDxloGzvD=F#0c5eWPw~=E;=48%fs&n1#WHe>Hrg zCnt;;jw3#=p*J(nQ5RoN8T38f9LYpRqx9+>U}VLlGX%6er5 zoMY5^U3w}i%SsL}lF4uVw_&N%&nV9?tIpGjBe9Y^t_Vg&v@Iq!S|TB&(F(IH?ChF*#j2vN{i^PO zBqS(36;A&<@w6g1opLE8(`CD98IG&8lSgNZgQ5)Jw{8P0d2rCtQ9{`L1Zm`YE21l4 z_b6KY9)3?~1el?_dikQHiM--0L~(LmLj^7-PMQgAJ+69MnPcy`l3xOi^mGr_M5>#;TG7*1hg**W1kzkvoa0{q_YP--c+*vID+-_sIq z%yjzHK*9IFRe@&| ziG*}j6Em^d?rA{eI>{aXx8>h#M#+ErWIyyKafLOoHZ6mYgswZ>e;6ogH~NvUZbTNJ z!j+ced8wuh&T+f9WWSwSY&10T_B@lfm$q1cAwtuf=RXH*LCHrC0c9h&K*gj_@}%mG z>^5LaDcf<+{yY!_L2v#JF})8li0+6E`#R$G>kvt9Az-6MvM?iuLH0j#2C#7jb7}|n zRY#9MST7<_n2dRK>tCgFHDvashuxa`!CrsPg81Uel8k5aI$2V zX&9=LuF1Q6_K%cAgCOG>|79PqzBHIH=%>vyUZ&6f+cui%Q+Iz-EU3Q^&7KoHvb3Dy z_m2t9!M9!hm=+fSu~;}r%kx$|1;7?yQnn`61t-J!n(I1hHn@Z4Bxli)pU(?md3UIu zYI5Axyqr65Ynn3o-0}XWkmIcNc#34VGS%P<(&7s!sLId^P+5@bX~zc`CX0-ZQShG) zGQBH9h4nzqDegR7^PhmK%5}Yk>tf;4QfqpkG$}+8O(w*S`PokY#aPY~EowUi6z;yL z-nocC0lQu7!GaKIneA;_$<6b}FaN=N@bqFo#7G3#AQE%mUcODrA+*v5xBmGC7&4NN zT}MA^>FsOC$1hfzPP7_b@79*oL4MNY#(*dd(@(RwWT z=6L=ah|)1WNhd>u7Gc|b!!5-8uG8z5PY_^f$K28ntl^@TxddxAH4zj2KrM(4I7$?% z-}a#y*q)s1+AcpG27dLk5MeGzZOt|~#mJ#2?wh9KS4plShFOC8xoMxerq)U^*V!+g zVs5Rei-uVZl>eUJ76KE9>( zB!cV~a|^0wN7*M$NU!o?CeeUUSu4*yd_DEuy_s0dQ?3$KwN{{ZWk=e{dswoWbf0LM z-`*Yy3!+7kg;Im{w&_x~nZ~9Cx;32?-iO|t1rPq_hU5Nz8vPh;TJZuR-Jr$Xd3QrN?o1(`5yLDbNFl8t2>vG65 z(jgRfg-3dek?FoL)c5Q?N0ux^t$cF~7zE;e_1DDz>Q4Eo7)E+yVL-3;jio&Us*e0QBt7My2UJxa_~4 z@XROXta(Gv|6AJpVXbSpe~Tl$dE-Bx{9CfTWC?jiJ5PgvfCmEI{!_NXBee7=?ou|s zSeAcI+kbjqDk{Y>f^Z9P`@fAM!g$N447L{kPr>m9jhlbXouDh37m{#ZzE$P2dK0)s zu)iJ@H?9Ymoz+#>y_skUN&7-}NZJ=_l7XR++{Zl3XrM~`eHU2FizC$S2Kz!?|yPADJXu|L|6<4yu~_HhuD{g=Ja11EO=5eWZpBakNk z-w45Mc6#w08E`Afz}ne)T+Y{cO!TA9>>_kF2GuW}DjNP*d@jBL5x?u=EGjuOM7=kd!5S+Wld0F|fC|r_Oau_juDtulJO9Xe{|@Qd>^hM-UDzklGVvO5{ha5y`Ye!?=Wg~H9!FPzD;-;Q z^~{b%y0;gti^a*#T(r(V&g}qiyrRc%Q+%f=)lWF-OHm3*(4+?rP7S*102beH_r>HZ z*9pRS^x4b6?Nhndw&=Qp8THj;Z#a(U`M{;yWWKwOA`eLE1hhp4I02ANkFyylS%%~x zpjRhJsSZf}BoFy#V@`UJ+v}LPILj!zmFm?3&*A&U#>lx2?#$}F9C3TfXB(PA_I9r+ zqOu(V1VnHW_5sDMd>`Ye1aR4ViR0FY`v#)ag%O*rUcga&NYO^45CTyzK+Y7wWQPFV`C%BF`CaGkT|;? z*(OYv!)m#_7^%BRX*`ZiyG60xo2AH>}*{gB}5Pjmh2sU2Z%x3MU%SL*}t*}c3w zDhb=_Y@1C+<*3zHP`F5EbaWu{i@<^>C*eRhZ$+)4k|o0{!mymU8j!MeO$i|#ac)V% znSZ|IolC9;Zpi#0`jagBru^KipR4CgUB0BTC7+bgnFzkbRg&vTK62?Wc-Wmy{OvhG zd$~(Bc7&0X=haAw*@MyT&SE>?q&!XXT(|u#t7KT(i>PR197-u4cevy5bjyaNKDsiA z(G9i}LXUNy=Dj*?tvSS^$CO&UwlF!h$~UG}kVOq)DrFfQsem38zo0$4GUr47t?xvt zfvz=qF;ZoA0h>c_Ihw8OS%sWmd12BcQCk<`|I4L)oX{m}wkzu`H*Hvsx58Jrwne4m zWM^o1VRBJO9K7vIW$9G>-Y%TwQ@Q$j9`32rms7Y>->Tqh&l`i4_>~BF-)v2pbw5Aj zpiSE$JXa_f$@FEQI29SkOuIZcy1fu;HC7goV@F&X2<}BV`74g}^EV0Dydb9j8m!wG z&+%sRhGxZlPJIE@N^l+PEtbAYtGJBN$HxFR@MkE*vAaOBes88Ft7=!XyEPQVAP8EK@jRjVocN@FE8RFaG(7q?lpPYvAi{#?RjZ44{FA5<*yP;iB&w=pThd3HOv z(Avv#P6^LfvvBc%g{Q(aO=-2l(AB_g>$zZlF|+#Kp{&el+RXl^vI#a){>DoUbLs+(4l5sSfJDp z_2rIBIZ-#R9a@%M#lO|6K6tw(KecJAjPI3e~7z~dS%x7)^TZ=w0Al344M2|C7+1I4SHP{w&nF3N$+>gf#SR~ysX{p z)NCc_3+>?0p|}sY4af2~@$R=2UlrkCh9;Qb?hTOFR%VyRb7p+5+pK+wwJuRzQ5c;dGf*t7y_ zXfN!m9<8+vNZy3ty_)G)HS~2(V61sZE)atRhEI%9*W6IcGf|3XDK8zqNuIV{G0>P; znKDKgO+IB{Svh{7onAb}AyMXC@Enz~_i}1~oc!$S{0gEZ0^ONhDY3Oc7>#VHvBSa5 zoA$&SEubscmIiEaeU)2{)G%5l#oipn@YlHK68eLAMrH0_5(QG+Tn5sLvAs*&V~jIK z<@h@XM=)wzC@!j{a&1Zax#aiq&h5GQwY&m5DCffg_`pnXK5lz=g+Lmy^w&hdS+&n5 zb_VWk`TMES_kzj|HsNCZq($sMGt+;Ww*I$h41L;y;=PN8@sQR(ph&gjwLR&haO~6O zexP_n839I-Qntor*BbM_cKCSp@b@C%I1~q|{DUil3w25na))c&UCP5#J)=cIDU7~kCbPY(n$$&Z z-K^(XTDw(x;Sl8>1-+=coH;n9lGMZRI~{_ObDG`UTtNG^ECs!WncM~k(BUj2*R{cq z4`=000T3RzsQdExdi8lZ%5$U(-N03HEpb%7V~h{@Z}FO;JT>a`5r$%4iyR|4gDA) zK)N96MrFt&zgE(*K@cd&qBBwQqNFcS69bl(p8@IaJnS?C?i_(YUC*8IBj(#HW5Asv z;OnK=U`WJyKGIqobhX2=Q!?O>$7ILjXt3*|Zz8oM{uuPk|8d6q{|r9<|KW#bMTo6W adq)!2)tXP{m?i<*04d3Nwy zndL5w3n(r)jVL84H7*#qq==-53J8b@9ITo5ojL#OoZt07=Xd`9_d4fz;ib>>@I2r9 zdq4O0UOxBzz0)pux^4Jk#}`^!S{u%uIeAe_>oXlKtxt6SzFu=?-O->QG=DyUUv&Fc z3*TorqdE9A@`T3;Ev-7tm#ddQ(;R<(<4h1-OG~G7?e`NJwBkE0Ew}H^o;=~35GEL1 zzyKu(8BD!B`5w#k{>nG^#oy%q^ki2_<3p#rn{V#kZTNI#kB7yB(-o&STjcyiE@dC? zI z&;@LMgE;wYo^*9tkX^-XBRS^U!YaOj{r$tW+khLBuMxGuecdrLR4=1(E?e$WPPPrRW zgxv<58SRfJw`9i#?$J5+g{e+}zkkoO^Lahh1WIrif}11C#;tz0%|#`|eE!2XfB)y_ ziTFnHIr2en&BNABvLC2B|4jFZjXozPfBF*#G(%N>13U8`5SmqP^6^rjd(UMDoF6?7;&;}C*r>S$n@(^v+Y0qaKmvBm3legW#=<5LzPCX zjl5hwKIi904?B0khEM$T$Mc^qq>uXksMj$>OEq$KWnWYhik4G#2 zpk8)6F}d;M5$fCjk>RcdD4aYt(hbF@|8@8ARu=F24}sH>?@kB=dr**Hap_*yji#TUUk>;Qxn0OSIilG^NAobA+f%xbN zbNDL*&1|j+^-{g^TwCLFLiyd@`nP{)L!tMd=n8D3PMqhab-MN3A8c-zLoeX#`RxjN zu|C`6pclwEAb-$uv4TBn_xBw6xq7Z;f_iDD=XdT$pZ?SEe=t)Jg)9I3 zN`Cm*+N)^SynhCm`%0-zr|1IxPNNlzRrRtvY`Ux3QJM5gLY?5oPBWxBqy-CY^~^IN zsgBwA&Zq@anuthVeJua z8o}^@iF$Oj^B*HBg8RO|mFuT2JS<=E|LsOgG(vvzZAX%Z5}9tUmF8v#`0PoCSiVh! z%=ay9a?#53p}t*5`J5sf-w`{;U7|S!A()5y zp=YVpJrn|PMt0zXw6mW{5s7A;Ngv(3V*oM%d4qwikKAao`l6P*EaOMqiZ3!twb6F# z%Nzo^1Z1H^L$S-Rp83xAgmd%5xkFyYmR^*L<1Tp#^82wjZmeo|#Me>K-(UH_dZmY9 zn57po>egFg652s1O6aembG-%ZxKPCVUlaO|dm2)fa=i5xS~?Q(?#Wh*;9LtTb|>9Yv=&-JfU6rO!S(VQg7m59awNvE?TnF z|1g>K63t}5SDqEMs=Mn0UjFK3e2`{xETj(*S8u59xb-l#Bg72Ng-z5Q>!~$y@bB}J z_PSi0T;yhR-wh_wpA$frcL{zY$IfA(=C}S4Z<(yT^fpQIJXXZ@ji*n3>mZM8%0=mB zU?OtAf!QBfGj~w^)UCz`nI3z%4}>bPYMiBdWRK+KyGATrQjfLbJkNGMr$9%8;_Z|c zRIyu+V!dR3HBZ>0pUMOr&H2iFq3A#pcpT*9Ja)*_Fii2IXfol;HJQ02)T~hGshXng zNZZkdCd1X%F9((EO}wo4A(^QPjiRf1bpG2IecHE^H?FQNP0BtUD|P?#dN5Pbb6?Ea z;Da?wU8?>Smv}`IlHzHYJYmLpI)URtzs_=!1+4f_JdQbEql~IU`@Mbn{92n;25QNI z7Rs$M(8z4MUeW z#mc^q6%3}V1b{+xMo?A21h8);x66mSXrf?ZWw~HzQgabHRMD;6(1htcTbmw9w`P*B zg5+~4Jo-J_p-pDoA8ERIjO3Hy+aDS&p$3hPUbt;kNNT>p9=MRvyFCY{&`ymi3=~d}4l*#!4e86~sa`IaSlwD38+$I&HpQpcW+8TKOu>>3EKrT{ zd@_?*>ex!3pFoGz6P%7p%=&hFjn^totU${{`LSg?v5GpMWX{xM75w3W{xnyFAjGXl zHU5ThQCF&zQexlptsWd-X+7CiVdi;Zk%5gvz$nX077dYo*zFJ7GpHfDuFU|rJ#dm&}Y<%*`hJOEEk(cttLfg zG&K-1XXe&X4I{6PiJp5ITSTI7)U_=RUyC7_Fcc>%7y(%G)SKoqCo-#C|9kCum!Q7t z%ziie_s>`7+67b8)vquz7j3#DA%(ncpElFMtCTz*60=HoTd%Ns2nRXZyL%2XI5{7U zUSgsBszMpj4mC7JEIb~=6$N?p)b2A-D`#TQs4IK4b#hG)zU<_wFb@(H^wPS{(*|CK zEHI;Je`?G&CG>TYdX_BoUt6iaJgG!sDWXTtF#G#OEAwyaZ!sj?;RZ0b4?VL|&zmjH z0l65X>95mTPfUUQhQ?Re4ZH^K)GS&rwz6oY|HO9F2G*{<($f$cY0T1URgjqi(Wak8 z@n6%#$L+B>-akMVlm)3I>O=xiVPa`jBeQ5Zim#NMaNnbo3U6sLOpJQC$FMtuN3V0) zwscFD-ANs-Ko-fqK5iK#?6p$tG z>((AYd8FL<-L!t{ZmY~JsI27M*$>Xk79W9lDTnqYcUk2e!4R0zpUtXH9NpLUa8Hl5 z=XweIi(xUDj1hqoW8)glez4>Wl>+*mwc9{`lVK)y_fF7Ibr)HyFV7pEPIAC9X z8WJ+O2z01|TG{UTmDNoQo4oPxp#cWg=3*LZnpIkluEp*))iLOHQTfeIH0`EwMr0jJ zCew3fx@c(aNFwig;yhd(V(@WNkjd9`o~9S&AdXR7i;f+X{z&Zf-nOZT;x9` zo@6hwVTzu*NXrfA&*Clod)Q`?^pa}XL;eCFC@dkU;~w&IH`H*<+fx-8wQzA+I%@Ba z{S7@Gi*i^RejiX0kbKC2;>}L&$u= zqR$%+b<~vyLV^6fI&2E;JxELuQ7^OYbZCOaqLuE6<$!W8C9gs=^)%{bR8h$GJvvD_ z_J#@aHzMJ<@f=%;tyj2s+QDO^X1-TgfyXC^Q319JF)xTi(-&pL9JuOZ$y zFmpy*&w%7))vU6d5t&Lw8y_MFBSR*{aszt$a1+GeRnw{x52B}oVEZ6%tGDz zxg*ye=n#-K3mqMey}4?qTxt>-Te_(c7p5m)UY7GExGUEb%L|Jhdvq28&?!c%7wA>+UHJ(?&neBn+*fh^Kiq`sX+<%wF_u}URZKTk)g`sIZ>GTAV762 zX#%=D3Xfm9wbD&%kQIc`-;}S1)M&1+<1Q~k?MZue#x3)VJHj8mYZY3Vs=Fr=Il+Z_ z+1wu^8M*MSKD#cQ{q=bfUYY5t7IdMw!^G=jW+(az_Cf7M7$}|p}ig7x=qZ}x! zw0d|X=T{=NWEt()_p*~d?&2NOad~0?_x5+I0Y4KA+tls|0lNFMJGKw(+kjY#`qEq3 z_ix5+Mu_}1w>E|Q<#?6x_799dq47JRL2Yq$MX1HB*h|YU69?tu=k*KOw;pDndpf9> zdT+UdKmRra-_Fg7T}@hHV$F4VQ$uL%!COj#cfE*ZmHFVD*|@IS zr*1{f>Eyl~{50q;lc9O1203g+QVPJ!#mO3AO7O%w0Cm1XdjzW+7G4t7Z<#bP(<_&Q zo{iVZP3_=Afh1F%&09SQ%7vFn=!N5&Wevt3!M|x6aV#K!=l7S#PF{$xECwlXRjKgA z3MWxJ5&A=n6U=@apK~k6fX2J>=0M4P#Lc|2^1v z9Qq#x8-Kq4e_0^*M?BeBjl%A`8}ZQv`S*)PwO;>!8vOp-4fjNr)~R+30-)G058H^u zvK(g|I=StzK+ae=AS*T@X#Bh`VxKcgvD#SFCg|r*p)&)_l2?Yy=~x#0@3|O!zjM;i zm40TIm`iaMFh|XoY7{dx(sYku7*9CdZ5<6_OeW16WbF9T3RIevD+?%A)V5>|y>BNT z&a!Bpuj~AZ@YaLUu{hTF^;#UPRCiximL#`Rzgb%tiB~S`r?Su>GC}129Nw*0z>aeq zI;%bxB|6cj9#Xq)Vq>Y74=pe^Y+;FU>ZNza$M`n(inz?;6b(A?MoqmexIp#vIuC{GSWNY+2K z!z@Z3_Jc5v*zFt1W>c9mUB; zD6u{AisVQ;G}w?Gm=JHgHy;ZUfD_y=N9(`5IS9v*J)#?CK0Tj2E*jjXpIQor)`rzI zBr3BAsk{I}|I6euDl;>eqYy48rZeHqo~AuE8J{U_n~hKN>T@?1A-!4`kX&cH0FG+2 zv`g&0QnH36#h;u!)SjOaV_wR&*$^QzI0wYo)qkJB(YW?TpUXI`_U1&ir))7zQl@Cd zeln@OJ_o$2Wu1TgCBs+}xoPfNC%?<&oc4V}Q(9yn>~G&m9D(P5at{kBjrCyQ^qI*m zcqab&8|~TQJ5od_9h}x`pH*N_s=EQ)!C--}9+jpc7KJ?C!qNEXrWwX5m@1VAYojD) zc8To{$&I(N3@8|UD5$QsDh=-9fNzG19F1ye;AY1yOT$x$q6C7$T^`!QQ6@y4W?n=t z>uf(8tKhh_t{rQx3-p>GJ|KH(kmFd)S58Q(U{RTx~>UPqx7Zc7jK#{#@81O=koudc|VFJ>C(kdsM+8}i6dgvGc>PSK+5>6-6oOoN;WJ+#I%(a zjKc@Ql0hNuWw!s6Cilr3tEWrQo5Y1(iyI;(r=J?UV{M3>ynN8K7oyHh zT3YY@T_2o~%@t_@N6WO;H*}dOG%>SGkyLf$V1VLyi?qb?qKPqMAQd==+`ycfm8R*I zW;33Z#(}`Xm2*s1A~WtiTCii_h-iu+A%<29m@!5Utwi;+Vw(e>wWd$xfg7 z=J@JY#6-`2#Z`Pm)TE=y3w{tcu2g+iHWi*duEkJZcMj_utb1JkTM})U73ZMD+WscO zX5+`-B)v{Rl`_q@e6gkrczbkc(0%$=qBpMj9iQD8e^)lCy>s+Zn^TRQrEG?w8*{NF zv)Sv->NlD!3hC7HB?lPDc$@%cf46)&woOTSu5?cRIl868D;m1O@IGjAGqrJ4sq6Ce zE7rD}@UzQBgzk5)GmD+qElwOc_sh)N0WK439io@Iu9r!~CAlfPB1efR&fl2}-;rk| zXZty6CiRk)`~$H4*C}P$TEhSb>m%o=HCW6I?7qI4<)TbH#+SzII-Pwe>PQDpN0CpN zPi3|$3M={#ZCUc^w<#4=^WsI$y$eCrvKyf$mBlX=$@HZD#6Ko$x_1p(zTFjY_Lon# zRqoMe>$9|SH~$m}Ld*nneb6_U)uCi%OL4+RDVZC@ykUs;+F4HA;$+H7YzixZvI5)4xwXT-rZg_IO9H+LyeFw} zFB-RBFR6st_J-;c@AC(`4HeQrW}#S8QS5{Xc8Qzyjq4LR45$*?uQ}z?^l3Q7NDa#= zjfH!ro5>6pfuB|OBIn*FDk2BtGlQ_VQVBQ~?5sfYEG}A;Z=9R;J>?@Q}GL173o=Ps8%g z0Vxptw8cK1ag=N{cA;H{PZlq_nyg%qzjt#9Hb$=sMSPGt5g49)HJ_G%JtfA`yZGfp zcZQU~zDe^|l(N0^>8O}8PXm*t*{eiO`ar6JF7ab38jEatvy-FRD@SW=PdB|es?7rM zSIrADf#t9)W!QD$>VA<^wFlk@-`{(W7wBEcJ~ai`V?;XTt~wTM%{&0|8V0b7?S22o zQ2$beYZ{3?kC>~Y-Qlx;q)v3NIU#2Z!-6J%mi{ahK+#Fl26;U*m(6mS8%$kZoMJ7c z=B&LV*6Za0)`Q_YvsG@-C$*nrK<5Xw!=_>&s>htSjse zEF_p`-P%nHD-HKi+M5)5^FvrG81dt~FZb8-3m%Nz$%TUuo`uaLHH6L}F2eU%u#9{^ z@HA&BZVLh_w~u0*pqAcDBw|e<{hjEd(!er7(b2c=#pWjPBKVKf-Mo`7&bJUTe&zAC zy{_(tLlo5VTmvK5Ag1~9@@=PusmoZkI~$cL-|<^wO*X?v0HlJS&9t8rC?r_+hx>~uD{vW76W42#6TEvgJl#!l`IwT+j3!F z+VZg9{IPL)->i^F9%v(mny`4{U*dY=TD%=S8D3c0sVy`M0F**n@?lDy2D-NpOrvp+ zHZH|xHA8o6Lfj?y#?f)dJ+Nl$kL$kB%a9}}{BhRC`Qgk2P4{?En?o0um9TXDLel#* zQLpE~9oEG{Sg769a4O4hbhvzK#Rfv##*KxA3ZPTZEt)oWSvU2v6XAd_i4#(e(;&+S zba32uU%EtGH`FJ-dbF6`+h{^#$Ip3O&Cd=x8oCui`3kUf@$pgH)BTJi!s+gZreYy_5En@~sd;dzsj##V`Er7Y%!K zB<~W%y>_}L57FYn@eJ8b!c0K}ONE5=###>CVT69ot}0fr97f%X3lc_8$(P;v!W~~X z-RC{thv`*Tc>1*394wc=5+IQG>`k9Yph%8EEiduD{88ai0fYB!#<8(X$tpu|9l4H> zYzz^Syya}$Jc=TYv#~;WiXBN_77mYg&$I6a>7^$6iwM?yW`Um@!&jq7c{gxeNL`p} z^ZO$En{R_dW#m_DTJG3hScM&LIVZ{4ehPiUXZ!dnX9JC%`1co}IELbD3c0Fw_0auF z3d_eCty!!r< zRz*0xGQ+vcjcwi0%Fg6@G#c_$g8`u8io!0^<{jlUeRfO%+@iXudsaZ?cMsg@6bZW= zYKzOwv)m$_p-)y|VHSNQir*OaVMGo^ysg%jG7?|AP!{j3Q`}T0Y|hZTK2M_+$>cj; zlbnnP)8Knyb3`y*Y(9Hlz_Z>5;}9xY=#+q?r%UC|LCS9Hu5jk+dv?KCuB_UlX+f?p2q3c*%`2k1 zmLkQXMq@=9RwqZPkVa&vPf%)Q2E9A{C5=mR#x8$yy|pIKepO=h|&nXv|8giu_8$&Wo1&H@H} zCK)({+|aQqqSrF~@=4Y0%R4q^25@SW>+bekZ)FK~8!enB_z21U!2QVNuMw6C8%{+T z+}BSHX5!#!pF1mO>t*23mnWR92K6$=%DImg`)vh2M&3!DhST*Bl%<)*Ja>7XF`S*A zw2N#1o5o3COdP z%m}-945BkF9;KIotb}>->_U2hZ zkjf5wx2o#H!F_PN_Pb$RvTdlHKYJf6>rI-mL8oFP~*}B>598-PTRncMbqF- zWSKy2*cF~3u*r0TP_1 zP0MUcZJ=cjDyoPA7|0!RmI7@rZpYaC)%cs~oH0KdKb}45l%DkxE|4S?ImXx0$l`iN18*OLKt<#PYH~JA_lJbtL=&2k zO(;A#fGtdX!RKcZW*mN_MU#!9alx^N+(w!8}2oIzZ5<79q!sV4TQP6fA@5B)`K(bwu31x1C@;g6%wuC(DE$n(d#iVhHL6D;-R6IMU+_5}cVgB^l&_!)qlIbs> zyzY3gQgFC80y((lQ}cFr6%>I8la^`vc)d-sCom zi)csNCL{KXIFxynKJRFjM;n$m2C1ebM|F_6;c2vTr4ZR_!pH->B6~plJI`Pv!7avn zq-Iqnyl$jnR422dNgkO=E5~8aXSz)$M(>WfrTC&9(1<4;y$aFKv$kO3Lm4T%vh1?F zeL;K}kX4S`kwWT140|RH53IX3u9u1h_hw(~n~b|k1n=4x??>bHFkJxzqPRAe6+_1L z4qVUX2m2Vz+I56!oU~&U)zjLVuESp#6^>wRt^?2o-r~zUp6B> zOOZy9r_n$RCy%vK!%62OJO}d#o1+Q83x>74Jo52!?Qz1f%5AP^kH2Zi&h-ODgdejE zue17*;tds=k^KC5B3HrpXgcN&QVGEDO zZNv+y%#t2%xKMZ<*OL?mR=%GAp(-*IV(lMxrh^nJE*H#(s4ss{k{f~@q&S83px;|$1}p3Uo<>wB=-MJBLMoT2udju zf0RI>?7^ldWClVCr{b@Wdk2CrJ(b8@3?m9YX&YN?uboRH-A{IxaI~R6&S%e>4vFjW z=Og`yoR4ECs>d0VTH%arP zpI-~o-IUB>G9I;6N@`WUmP{hDRM_!dzEu9ai*IQ)>m}JUR&g|JbS^e%0ot;Cy0Cc4 zAS`UF1)m*{tsYLUZKEC*HjAwBO;(tIn}$&#sOoXn2sl(Mkn3BRO+F&lkbsubN=TT0 z(BPY<#sM6ir;_wDJ&M+O8)4W3xM^GCri$@Ef0vvAVM`CsQBM&NWKzD8#SDI`ob}oy z1Gh2|U-M2yL8?7ttiX;^jT?KUuXQM{=wLP1i2*|f$G4Xuz}5QKLq_l7u_jNq-Kf)?HeGXWBw?wtpWQ}b>Ep8en!2yR*RG@j@Ip@~>LGJ_5S77FKE7jlyp z+HWmyL~3GcQJNR>$=0+^`_zJ_1j-X6-q(s^+7*u+w}tc@wnjgV6)Jrp6}7zs`KS8H zeYCAiF}SpHA+S$b0YnO-S_!TDhcbOZs42vaV>?p1XQV`r&kZnVUBCiqW>zONFUtm0 zE-$El$xG)yoMMq#`JFaz{vHio1syZqXBZ`C@fTcuSJpGSW5~O$wfAL zv@#UNMYX&b*=9RX)>GvfV@hZp4eHj6Uo;(Ghx%QTrl=+RC(5D9nfx=Kw=K)oAAtnC zUESbn!C0}=*=&ty4#Uy!YOwk`P0e9)HbWh}e`ULiA-}w*qHQ)U#@s{KCfGk{^tP{+ zsr;nf7qZ&Bb~>{1#b!~ao1ii@FFjm8FJOShusbO60h?j^X%E;N2h()j5u?5gR0Y(6 zA8C%GO~E(uc3Pfi1DeX?8%G6+$+fn!2Q*GNgSL&{g~$*;tEcz_!xW97aiX}=Se7xS zoa7NPm`l^^3eVyl!uYUJpbmTL8An{Q6y4ZRV_m+r=k!i*Aj^1(0XT3h!v{D1oq=uIWB&A@Eb)=1>^3DWKKI zBYP#s2$VM)@IIHo?|00dKkUhwvgywZ(*Hq|D}Q40Ap~=VRcKqNy0k;tpQpJxFD`?E zwmyH7sjY?gRSo)F+5{`Uvgy6ti}8Ir_9qU47;S)dam&PJu7todwAjhGC=p)ypx-I^5y-J-0BvQ6y#xM5ro>9! zpLT>pDE0PZnTAQ3SsoG&GWv8a^g7?AFJ%{CI4YF+aLTr^NYMjlQMp%9MtuhvrK~qC zj7;j`9p!|E7aTgxo2`0ZiqT<;SFaW7rNFYTVvmZtIo)%0A!L8j-1wM+vG3>rYK( zY47ksps(Gf6NT?f+N3v(yKLPd6&A$pOi9UpX-T}Fo%8@X*@JC;nhXR6C=BGc@3Zes zK4MQp+8uB|vTi6~*eF4zRSwi3yK6-r2<9P<6!ZzMvM?OTG3maBS1-*A@ad%RdvUCu zE)+5RVVOfOFlLB3Q4_0PsYLiCgaEQwd{04zy8wB#*A=*>w{nPMtBLL2Fj(X$eFa4i zW++@uCo#kNY{&K!#r=a%cl!db$c`z0<=46TH|`++mRav@Gp^&Q}VV)tVMGA zl@KBvbgaCv(np_gjJTO#m8NJQIsD{g2`)$tq}b-3VGnu&{Fy#e6rw5nr&f3tmV3f52U${)~r?^Nn9M*;H}m(hB_VOr20Z+nZMB!1N~6y#87h{>WlHO z2mO=Z_xg@^`17D>a#+9i)-+uf8P{xONei#nz}}12R3X*Ye7u>HSmZXyY*_=rl*3Jm zd$eb`g7CqB?aCz>HI^KjIV$KD?+ea?t7oeTo&&vC*k5F7^1BebUUIIVXShVliVKhG zyewayEE?+b*+MI=$J%)-?Fn0WuTe9Rh6EOD9FgH~0E%fHy5-YqoZ1)Cp8!0ac2vd* z?^7T*TnjPA%u>k@?4>Ls7qLb1ObR#9!+6tIj@SH8fv(Fb86>VUs$Y<#I;rr(O96Na zvc@vc#r(gGxt39T{e!Jw%he!dRI{qoM0w4gdGLrAWxdh;Ar$r%j|JaR86C45D|6?f z!X!teg9D7Kvjfx?Vd7ziTWzmoB@L1k6fu|ziUh2GlME{769w=Gv>H)WfI@p?PQP^6 z^mc7Ivn;GM*?CqHov6u~Z|s;L{5$namkt)7Y2v~?EMrzGP z6Q?fyp5@@Lz%8vy%0Z#hy8RDKfSk-T^p@w%HOW;TFBOgRpx!j)y&+ETxrkWNMJG!L#;$Y!yar3e8H0d$ zVjhM)IUH`Vg=bBmaYRnn&~s*agv_1BdC7@mz5!fkfS}rW&v*YVeDvhEb6zfmc!O$4 zw!Kvyp}9Sc#(^cidJ;d`LX~!*r~+766hD_89OyIFSbazw#3?a3!Vlln~iV8x5Sv`rpMwmDpQyEEDqzGm+9Lb zp4o^Ou%M1%#L^3eWq?MnxTEV$Ftb$*;FvEh+@HK$ARf7gaUBVBSCK(7eR`7NB;rCd z?hFKsAH-e{62Ak}1?Qqe(!N#RD`PDtTsg)*p_>m$bJal4g*?~mZ8}F~Fje8S*-#qN znCQ|YC}3q0N)pDN4rv77Xj&YYHVb32qWNx+DvJer_}2oTe0!_lAtEqoG{iVEIs}ra z;>0U`*(;R-Es`5@!@Y4=cS6VV%211p(8M6{+6>u@NyRDS5Tve8QXNzvb`GF>Ftmo> z{;nJL2W?fu9~M3CZ@8b0dVEEs@Z5dykwlmes`E7_-ui1*{ zz2ppgpBjwb4}U-X0BS2-ulGCNd8g&TV9Y;MWoPF5fmUPtj|>JJd!R@YbaPpOU{u1K zzdjeK$yeu$2JON5^(W$y6pAiFOH(`be4rD2ZnoH57= zn%^@7xM^#Y)SoaJ5AQ&Lj6u;I*K5dG6~M7Al`ckw>@3Js3t)>T9kYYBFN^_SAo+n@ zh#_p*B~V5&yz`&iG|fN~x4jfcdoWWpXupIr@(AM!S>7SBRo4iYm-^wA!ZSb& zger&ul`vC>26MY>Ad`p7xGW+cae*rAw!SwDcFsDlSn>&fO|sWzpVci))CygqUdDDJ zL$CKuws66}Mv3|S=cqsdk_X-;YshzmTp4xR`hd4PYSs;M@M;Mq)Zb+^uls_+?eny_ zJ$SJBj_g@P62#!P;QshlQA-#)(I|Mtg%7Y|jy1S1>`mU1rv8~P`&*iz8aWp-;$?1% zFgh0Sd%Eq>YX6Z7tNxFf#l9IOdmg`kdzM~BsIN21s~gVycKl!lva`*4~k8T6^ zGF@#J4>t`7i)nc~g_?4Bhd8XI8mU-H*2hc=qWPA&tg|@Kg!-o zrc`9nuYHAWa`GCRP9C|_GX*^oaWLCuV+h1EitWy}S+72t>Bi~dEGAeOLm$#ldC)A0 zLaO)*i||4b+W|x3v$>8$Rn(RCvIt?^6X|5!M$KE5tu_}8*4B|Glo%sEA?#e!i5R}kSmuEHx820X3flV zHOVqbMN!XrK>|lrW2+6}f}Mtt7Uuz$K5`_b$UT$xE+SM-xdr(yMk~4~tU)f>jyBlU z;YO9+50W9sQsX^4Qb0VS>{HC%p@8DyrlXSn0jqfIA5e5YPqMt>xas=BYdHG*^6L(0 z&IQP9VM^lAMYO8Bgd*Ig2qZ8fPHL3(`=rSWH~SZRP5XOK>Xx!hpvJ<>O@gbX5zce7 z<+83Rk6NgtJeG{_qS#Op8ny_;>qyMdumK#jXk|ua6Xra?s~-a;z0VuIQ)$urJuOEA zQ>>7Rc?U@#GpjmxCpO=R$7h>tWx-6O#wyb9*mS)qE%8j#4Xo5R(z;R);!#} z3voY~U6swI7i6{D8Y?rz_&D^s({u!uqd}JvJ;X!)roWMzLm}iaO}@*kF*JhXAe0Vc znQ(_Uu}GsR=dh$YhP}oJ!Pjsi3d!Q45~(}~@UP4SwS_7ZK#-6v7`F%~ZGua4Bp_GMrt%3}fjr}bWB|MlM0rCJwf{R7``RgYaN z6C5n;xLm;vQg@f^+NaCp!?%AZu>5xNwcTEIbx>R7`^eQBHMoPNA`iQ~W_(1>VCDgJ zdeO`@kFq(~ZVhd`S(;&Yfmj|NYMJE$4D$xN;1vmA2p6^z7;Esvv0$AEILvMkZZaD} zU)fIQS<8V`#9jJtB!-~U2y&|x3##sIpAs$)YgSP#%$S3$tRZa9V*(%n#mLc$A`*>L z3GoRca_dfN@Ir@IoPl^yfN+`UqOoya6XI+dmjlNI6>Uw2R9lz^hNH>O7EFyucsV7EpkBCxAt34J$(s>@=NVZav~=n39V!j_q)?#BGH#czf1TTl9P>58%AzN47|K ztuVRDbFSQ0#=UOWYq+_3#GjYEX#LJ05C@!o6;m`rS*194V|FANclrpZr z8dhm5O5iB2u@#^&TA$zldla#>&`1B zGE+8Tw=w)$q11xqdfn3=k_))IvMz=>u+X-${&vBmB|1y`iTDus+Z#tTh2`tk%16sh zbtK;(w{v)W{@6|plIV)okJzMvC$-+4$O--UyTo&%eB_}LO2rq=m<#Mf8845 zZd#|^SVFjy@*}GF%aw~d=^9a*picDkoFm`VY{Qid2GGq2Z5NX@jZePf&|Fm3f9C9o zo5p=!2LhfI<-`VPnqa^J|7e1Nb{F;9jhp9LfsC(0BsmNw2wnV_J{!}$(y)=SigJ0G2mn2Gy194QKtX+>+qY2hW-KnUL8yhj9IEp`pPRaYqzuxt-X-5 Nr#w&MzrFOse*x4|CE)-7 literal 0 HcmV?d00001 diff --git a/docs/img/pai_token_profile.jpg b/docs/img/pai_token_profile.jpg deleted file mode 100644 index 52d68bb7b571dc71ca51b62dfe722daa90048bcc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 55722 zcmeFZ1ymegvoG2cJa~e;26uN09yGx%!3TE+2oebH9^Bmm3~otq5AN>nZjb->-E-bM zw(dFKyZ7An*2~PQo;BS)Jzcebd+)0H)$XV1r)2;`PD)k^0D(ZjkLNe=GzUlki16?T z@NkF-2na|>h%ZsFP*ISPQSe{AM#my1ASEFtAR;28Vx}RZV5B4>qUE7uWM$*z5sKEeUfM8%=z{0@6!NNYD?fHBhfW?G+Mb0J$ z|61h}0);&m`pV+D+$5b4K4!%e)ad7eQ38>%D(9+R!a&hzU^6^W&m6Vc} zk(GP@K}}slQ%l>(*u>P#9Bkp}K_mo6dV#08yBCD_$?_pJ0~|Uzo4+F zxT+djQ(ITx(DbJXZPgv?EK>L>iXvPFTFqj z%-_}eXU+bVUYO5%y?}*=M?k(hcx?V#r|)-W&so!&~xL#U;-k*)h%td&&xmZ z#|D2e@CO5bFz|N_81;T%!@mb#z%G6Eqy;T{--(ow{I!vg1QfQOfa=?kzir1km0|xJ z{C{PHJ?2dq+dWk^NZaJgN>Mm0m474H&UodUV>X80>vcTp!oPB`2OwTAN4c*yw{fm<*Xy580M+5qW0T|)kQI^yuEQHR#x^l4B@^1w$QNLno~GwM>@yY z+N85B=F*Pc0y_Kbu!1*3f7rw5;6xSQGYU$Xb)b!y#Ic@A(iH+zPr!ODMCY;P%@gq3 zxBxsGQMbZ^Hv{p|95?tdRzq!DxzJaiz(e}0yUljMlb!{$C;GG5@jwiJF4&BNgY5^v zm_#*dfMrmz-hFA@94VdMe8_O1w2IbDsL<3jcF$%+j%p7Zj+I!>@BgVs>1--LBaCZ7 zZuSXycf9D^Pf}F4{*WzJuhY?DfKaqI^?PK*EpXd~QzO;lk>CV3;JoOfg{N~iZDS(W z6r9;&ecV^l545#+d3QJ$eKuR=JMzMO0&I512F)0EkOKn5OTRV-fCA79a~I;{*a0Tz`>UTfZ35760#`6(b(_(_FxO&eP5Fyo`m$_*@72&jP}GA1yHGF5O!n=x;IL?R z13KDnD`$$DjxDCSOYP*)*Rsy)b$H^S`33Knd)q*E2<_jc%34smUm(q6<6KL zjG#RAP%hUslcB$9XVTa8D#zaSld*DiZJZ+E-Q_FG+JK9{fkIs==(BXewUMIEf@Cgc zb32buXwg!4;CsuL(Lg%!dz4*RdT^%jN%S@6=)fD*B-)XyWG@ib!qjX*kVM6a&rC)y=R1iPa*T8#N`FI;#S=WZ*iL zM1N|iOv2BxV!hGzySY5l56v@gG!3-8v}W%k)C}&$EJr3)b&tKspEuXGo~*Mh9kgyz zt`UzKYzg=+2K$kSq@Xa*B>UoCMZ^7;$n2|geV70e! zn)~Lx>?m~eq|3k1a-_+;`t4-33|?XQX{Rh@0n-i4j=q1{wX5t2cC(#Dt)*Rb?fFfT zmQpnxte3NeC-=~v(jDAHqwZ|{OQVb{pD$wqRkBxtzYq5ppMY-(iSnj)?DqoKj&K>X z8xYkw{4*V8a+%x1PL>Wr24aEb^MuBa6rHedh$=$=iezjTRK zrQL8JZ%irA%LX+h$7;%$po8Mm@}_r=7^Y&UQPT8cct5hWOI|FlCu`h)*>Brh6nTKv zz+D!(A^L3l1Q_%`yB->DShBEmSVk5U{wk@r4IPj3&0mip6g#i-kDO&8k* zms8OS^6Bc__}m}OWj%`AiPCdcmJqHB_%|Z56EwapJY*T!Q8){i3VAw!u1ZKzxyq!q zjvso*&Fas<#H{*qw3=PzF23v!@n3b$(}PMT{(mG$x`u6F?fX!K=~?!vY{>pDNax| zOZxphrY}R^zSX_f@$BK|V!OPjtwj7lbm`lhATOU+Xl&_-ya(bgHV4X=J^Mz|Tq*mdW2Dwth|V>xC4taP_Bk zT!Q3^9-U*CQMqr2yGWSQcl$OG%^DU^VmkV&P>=dz_;&f-eJ;D7Je)O1!aZ~6MVUh<)^TlJ))QK(RoOpeVoJqWTB-cd66Hd~5?eL!g@^AP)O# z_nVSJBj-nRVVvZ>RqmDobPS8*mMS_P6Z|I&( zyY51gIv=$mXtq?QTV-siOq%C%%HNCH;x^Zdv}L@E3va^6Lyal2X$n|-_K_oBL0pdH zPf8!jmC&Rsz*Es*get=XI+j84s^|ru<|Am87Q&x^d{j%8JuxR#cTdu8m)n<+Fw0xL zbCc|(e63vnA2e`ZL>3+--Tmzr33MzMEcnXqVUs1BWpoZ)aeA?`Zh?}pE5Rv8g?;9N zG)Tt&Ua=l`(a8%>F0qHfw74$>iZZ4cmlq!#&ibZ_byJHY z#iriW-eZ4KzCoDWRbAbqgkMJ`(NSwg#@y5iq^n`cUZ+ zIBoR^ru@t-!OY>;O}W?@32UAVU8I~4)GC{Oh*-UNYp3)2jKY3u%wJ_#kOa>1vbpRQ zzantx2`KVgQCT&3W#L6pWq_eKJJabRq0rn*cJmE3l4hN_m&6weWjL^GC&hUG>o&cJ zxxmZ~DPb1e%TD)GFgEs{%1Agcr2SaEqS*YlE%=a$Gs&4jqz;qQrTeO9E&~PkI?zx! z{x`&1Ve+%|+VHhJmnXLZ_w4Lz)V$_-83-kn^Wz)AFXQeXBntA|>{a*n3Kk_+&~F%8 z8J_^xC%{xYW#zEam1!xIdYq-4CSY_s?zP+yrLqCJ0|bw_g_QU^Oea3B`A) zSF;@ELy}Z(3>rH=K4Rmx9}pZ0RN!6cPmYpyfm}K5J-UaYS?GE-LmE;b!U)gKX?vNX{b~Hw|^S}UgSMdHE)=ue~Oea4f z4S!Q&k`&BwC6{N zi<&c?c1VHyPe2P3^a<$a!dD|HD3^5h#&S}S&_0f1+nGGD=fp*3c-sakR7FE@9F}vZ zO4)t_dhC}bu$vV=d92t566E|W-%aF!TXU*UUkQueh{Gu5gSiKDiQp`5i=c3N^k@}~ zAAJg;&5b(}m~qhDDq3rEplc;>n-F||JoVn-TKWQ6K64~nNA@*7RXZKE27b5xLXoop z(vsy83a2G6*3Y6D36#oi+G$t%<)UGUVXxIhX|KIQEEfZIzI)=;X{V|2#4e4a;#|}B z2>0I^;a%W{gsQ_lKH#=8w@kL8&hKFkQkQ2DGOEhjY#@HzMS^u;N4($dNM}1S8A`pi zfN(!>Nr!cq35pWp&Eti6tciq66hg$$B(DV=b0n}5T~FEjDRiLm_hXzo$5nKM8G}U( z8^!f#pRUzXDf^=kn-oiX(smgp5+wC?u$R`?EHuLYu6gGn4>f}T91U&nFJ;z8*7>z* z#!=rk!*ZULM@X(a>2-Pne5GeXA8VPekKTF8|4PhbIXn;^Mu{b@?yVaH5w6>+33WPn zqdtXC{PrZHs|8&f8uJ8*k29`zC)rz5$)+cnka_bj zH^V#>kOVDaOnescw8o1tjyd=p)MIsw#;PwrO-mP@|>&d#WoC?}|o&15J&P@iD3XXUIi@$3( z!+F>5R;2fS6sxzU=-l4SC)PGSyNv!39PR1!q3!ID(ynx`JV)=1nK>5f`RcH>-^N~k zEiShXyZR`{){k`+2J=h1)F>5L)z#AYBFqtWkTdLFhwijPyK^@F_{$Jl?c&Vd?y^i9DXpx5Q z_2yq1L~A=*);t04)1eKGlblASi}#Wd-D=VAzRyTPqB}{iRAL1)D%O(Q99It{G_N_F zOOS(^F7Wg|#hBxD$s_2C5wjx>3GF7t5}Hq{?6`NgzxQwhrt8!%!4wMHT9rPYr(QcKc=`&c`bz6p@6AHdLHZB*CJ!g^6Vvi>*1vEcs!M}cRSB{%aq zKn9(7mtq^1bUuDp3jt`e|Hyx*4e0;!t3UibvHODmf0pkK+2l^1G>`Kc3d<4V!)C%t zZu~x0-OzV0ay)V5FUWuEdH!9NEiDfop4v{7_{UbrZeey&Q%rn>#7}+f*5i%K2BG!Z z!@4VBD{&k{(SFhtGRHHtr1I7fI+TAz=g$UvaGJ$`L(d{|ImY3UU9)9~=C^!2g#q zpfay32DZ7v<&Z{U3m>Km!~WnN1Hl4OV~sl{aK&(INP*(x)-l{MEV!C#E$g5lc>`-Q?mt?P39&iVxZ;krA z`MY(+MPZ+#w-|Gfd0|<3*%8y|>LjGV|8YN>*Tgo9h#=wbUI#rx=r_iL#nnyoWnZ?$ z7>$?3Lb8G{pL5e++uB9W*}$aNgrrXZwkRQ|8t;W3zi}<323ZSdJK9Fwgu;-2e`~Y5 zMmO_C2jcb4v=Y)Oe^(r;&JJsjmD7Aa z2qDaMu|i1&93W-<0Bvz1e|%orLSE_Ve)7o-nO^SEM;^>5R$k)W=T`aK%p)^UnD@WNT=K_96d@}VEY9h5ltEjXODO!N@DVB~e3<^D zB<_EDzp}3WtQM5P(HRAmBWU$i4^~ficDZ}Tr}w}APRfZhAB4|9aWM&_(mw@47IAqO z>dHA5^Z$#s`wwiL*#ZrD>{=aiUO+j4+6al^aS1)UxYe)+MST$XQ>AOE6oqEn^D+vv zs&5%@?+UgxurpRtQWB~rn$^%UxGsiDTtm%`$Ys9>%baTZ
    7l`i3PZ3eG0)N`I& zU8m15DHq-)f_!l$zZPvMA9MQ88IxcVY{ITZJH##G z;F9?AgPN&bWE0Pe2eiiaj=Hwl%Ujc$9PSjM(aKuqdabpn!hU8^ypMcc0?y?T&36)^ z&w0cqBj$|KJ^q@`YJ4LOT?J0%v|N)=6gsjsVz&yhyx-~1e4{Gfe;Swn4*mEK{1()6 zo$)AbfbaxNb#_07aGBn9Om%BEjdAK>*WmBREA$KX{Rq-D`S1$KDALMf!uIuZ&TMb_ z3Anwke*#9Nsy{2at`uIBBxz_ij|z$r_%B&SCLOc;>r z>_g7y8dgu&8x)e(Kbj$gPj9A{$Cdq7skT4kAAGO#ICxxoyEJn@Oh0!(qIho>w&o(D z`++|(SoPPi65C9_vv(or(vtU^jbojCKM{~gqB%uecHRAMT$oT_bJp$DAQYY1(tyrS zRNSZYlr_`w;vhMrqgIUic(T&@O%_fHT#kgJ(AS76v4#&N#Hj~s2*(8sj>1dFs~S~D zPXLP3tXa@u{=}5*Bxk~yt#MdC@H#jrNrUkF4>T!hST&auwTF#^0c{-L+whvw0Q_?8 zolmN3NAxeyR{71oq+g`0^ayf!uOfStai~vR=!9x6#At0Lm}jG~N^Ro7Ze{YV&1N5W z2QdmVfO!HhY7-`Jk(0tcWx3qRaU`g}gJ}||{4$KyR?*>!?2T?Cx!y`$kJ*cxR3h`v zp&VavqK9%^uvk&uUYoSwkLtoN- zy-uhp6%rN(60!TltQE#{cMAAN{9WRbp3l_f*ChSo7?!NFHK+3moA{htac7i9n~08G z0#g+(6|Bcf5zSoL*Va~C#42Qs5vvX(-{#lY*@!{JsLVOu=)?CzLq|H1vg1xE5~f83 z`j49oan>b7-}=bk+2J|y?Xrc)e12$g^a08ufQT6Ea9+O~wqS!8jre9T~ zt1dhNvx9WCIW*4|OH=)~GhbC3^)sf1Zx>$ALb}z*&pkZfzdLT8;vU9uLu&fK*F8W9 zYjV=lI~T({sM&h*Cdp%EkVRb{tF^YN$(MI66)<%1YziW~?{mD4s0m-mAlxBP^5ZPA zJQ60{-v2FaL2l~yXb{)^C17D6GR#|C1+LRJ^Ks&>Fk;i~Fa0Udp>$hyb~twdOon@0 z>gaiSPh~Nzw8h>>q-dTPH`)eyS6V}hY|GcHaWSYI{F&$a4*2ktfnGz;N_87&PYVG7 z7RvS*1y4bb&94jD-8PUuXxgygN0N96a}k_S40cX2zbC*t-1IvCa#94h+jdo;sYwsD z3{-cT8^fEIVn0uu^Rox<2Mx9mZrT%|nts%`6vM;0^h)1?)k1wuhINk{?rj3QF7xC( zd#((KP-d9-VDx({(bDb}fndK|0nH4`72_0>YfI=&AYHYsVh-QIXv#8P>FSAXa6p;~ zby|}A`~kF*4tFtLbpja$08nLoQb_D<%>Pz7zt>6>)2`_H z!SdVTs1ceXC|VSc=mK`Np&!{Y=XmK?a(oRm(O@P~-kA|Itf>osqBdgPW=N<+sY9Q3 zOi*RzuI#mV^wH5#Y%eAivtX1G6FHd`Oa+(2!dz=dS^VIFW5*LPJ=>a8Q1-RX!2+2j zfxPPpb9k4I`e$)R&ahTyWGdaW1}U7Mf4rvP`uVha^FVZr0n zw}+<(<%uG1#pKJMfL`h2f(OUvocM1X|HhlgtVaH#<$z6ufK{pl8a(4@2?lgK%b-}X zk?xLbrhr(Zbz<`Q&P%6Rok5jUBm8dn!@aoJN4b`^hcCv@Q&Yd}XZfug_SXK##_Y{3 z?uz4NDZ-N*lA*NEWT5Z`= zZomMpG(x;wq`__LR>zSKo`X+7wsttfqnuKZ_ulB(6JVCtwsZ?Kc4&|Uml1C3OfHlc z8EEEdIMKH-D{ogAz?bk~06UNdfM$fz%KL=m_CYv(FJT7SA* zchcLxZQbc4@xfE)3E0lkw9cE~6UI52@J~*tk5!M$_Vew0Yh8EJB|_jSta0BqKv{A# zclHEO+TWWM>gc>F@Lp*O)x+=G+$hKBC#VDjI{0AH-Kg9(bP>rg9MI)XYrK&xTzrQ- z9Xm$n z&8>{04`0EZA<-is;8Sgp+!Sq|*i@TYI1?Vd)D_DsTs#fUb<0sY;uQ7z$VO3}q0}_) zg#o5BLC>&qq(IWfGC9Q>=-=1g?jgks)T>xuotw~w`?mX`)8RL{XzyK=LvEvGlypm$!!0P57N zDt98d9q)J8_vH&pp<++@t^^lt;mBg0pVSlOdHiov*oj?VcpaiCO0jhd?FEczNQ^r5 zKwDh(rp{Xw$MM!{bb2RrV0{dc+v#?>r^E`UXLfEi{Fq7&(*@Kf49!QEG09*BKL0pH z7uv~%UWnVeu0LZ&Uj*AgtW0Zqj{qzAGtYogO+79hElb$GU7J&skjzziS{!@)$CuH* zh|5#dYZ=$q$K4MBnU4y3&P-Dm8;?frB6|ZXpCvQ5ParwJM!IvSa~&4mQK{965+Q!` zk>cB|&2EK`+$-2-isg%e;nvXwImBNu^yI>;W}>+)Z*ggKES+*gq#s*1pL-~iDr9zH zN;XilPSwUKc=AMaw}&^{8;j}YrR?0Zr^M24XK|#sRPqD_u*jFMb?jxgTW?}RdEO7U zy&zAeUn5H3JLFe?leVUyp);~F?i7jA>kx*X(t2V9_=AoHa2q+#x1J`e*g1J7y~JUO$1=Q;7B;nubb*We**ljkPZrxcAzSfeiio3c=mrMdV^Uv6k`#HA?IpN2M)M?MYAt??;z*5* zLu;}Vc^RoNnqL{(DGV2~D*Zeb41SGm)k=_@oe>_s-)ZXPy?Ahc0`#Jwu}TFy4_>AR zZJ}J&Qd})#f^0%v3F}t%F*_77Z-gPu;2I zoCO|vt+W$?>PW-E=8HU|OwfKI{=4DoiI3rAo`Q32NzJJB$_N8!yp&-d_$MA}{Z6>Y zK8&w(UvlgySj1u(wwlLrkA3;ssalm>JNq2OE+Q>5-|gI_oGi^{5DXSPkLw+E@&%=v zXW<+2(bf5Oh{FcIs;4@W6LQaFGa+sl$h2uWcOMeLFZ%lIK7SWuFUZ2Q5T-IHr!5C09`;z?SunNnq2 zWl+`?KdaxCHD1DyT@|GkAV%ED^kKAZ9n<6q&_8)Ti=u2oRtx%0Z}ijufk^bUqsfqO3ZPz~ifW`CVwa1eAH>RShJ!Y3k_%nr#vJd{&ce<|Y(Noq$f>6br zK5-dmq^nTuow*E)9okG5L&UP2+Dds%3`YAd%5dqA@IYxKJa|f5=_}_R;dXP$H|5~h zc2>k47qAf*Et0rP1C0Mj5cI$0H>&J`s-^6x(wFAm4Y$caLP+TT?M0j<|JTq`|Jv>N^Zk!q|6t(%00#O}IvNFW-z>_nzhbbq zV!(dCt%-s@RE{b>zQhz2=b4prreN#n+ zN$wdbp}dVrY%h-3SC5GK!@Y>4Ow>Mx*b(P65qW4cZsH4W%b6hMya;x_AFykS<<}9r z$V&)%^Ez@>9osLf&P7-=QEwUW1!Z%78P&$z;E2A+h&wh(D-RbfRy6~~)F^ZbLcI+0Y z%v^yueGtP)59uNg%+jzo#MuzMj1YpRh3)q28U7%Eh@uWWN~#(F5*@XIXI zWLZ8l*n5L1nfF{x=c+m}JKolwDB-yVUnyak z@R-b3{eh#)>5+{EXitG4H~;wT;CXnVbbhA`rXP!z@ zqZG|ciVy&hviEextTq!ELh~YId=pd)1kbyPGCqkCV5t!jtG260h?zN1&9#4tH@Sb# z%T02n7tazdXG)7EjVT&xl=L9)~tXp%*Uptv^p^Pr=)*;IUQ4;qwhsn)um~I`(-n&c)xLn;Gz0d^N-OUh7L4*TG8|nX6P*2a0 z)H-a*dFSe8J{-2ji(h0>p7%^{H4WeXRa=0RiRLiep-3fL&Qu|v$xUsmiG8$sf z6OiCsl(U!r=w@YGh8j?SIDc72X4m7+&ketE)s5h%`>@Vne|*kz zEcL-xcAcxXUi`}VYS2T{=%tIZ?WL^?S z6QbMqdc)2()*ZWM)ry5Y^Fv!|%s`0%HKJ;m^G9dyiqDfc zVANLqQ^yURB0lQ2X3s!Xa~Q>tpT}RvhBHg%hp9$`_2hdxgMVIdQIDT4TqjKS;p-5K5d99KtM#(vn7`QEl8r%yZDu8r@muB=WiTMR zIt!~#Qe2g^Uk|Zo;fq1+aQkfCg+b0SOWev=N^uFNZeW7To=3{P z>uAyXW+KZ|N!gcIuqD{mP8p>_@Ix&RkIO;nPPLv;_YB1n*1p$g_E=wO`#n)&ChXp8 z4%LQaoa&ZjSrb~C;B~{!sAD(bfV`849DBIzSuMGfp}sMJt`2so*0u3vWdtQjwK1&> zd8n)5!0#+ZMOgt!?f+&cguXn_J=rD2-sOQp{D*sb`&lOP2Mw0z8nkT|>ah#H+ypH{ zox-p%S>Ry<|6HiV=19G=@|&cyT_s;w4-ZUuPDG2yOQ1>8Y}(S@CXYc9>_j4oPy3<7 zXbJ@$9D5xoR|{6x`Cf*VuEX3#^HI{}-gs}smRB=BU_w!H6sAe^A_5jB2KPW0Va)G>FNSI1}&-EkvYEx zynv?}T!?(PM(PhZovVMgwsD?$v8YQJNyb*o`1yOo-47%DdkcoPS74mKO`lXs&4poi zUQJ@1fW?Sf8crzq)osO$9&3mcOpiL#0u<>>p%eGf}k znfcpd)o8(8rvlo}hQqd&X9AzZ@p-|X*~5pW`^+ccqPXQKtUR-pT?_IA9Mpm0nA$E` zBfVsX2Q*+p&K*|?wV13}ti8RBoJFF{He6mup&<_Opt$UP5GH#rGK=@Tc&_Y}V!|lG z<-$c>y7IokRE&$GrX4nUIC_-XiHZqPQl$e|J^>{LnFSi@?o!T2L7djyYx`HTE&ip` zs2jr$LDNFViWZ#-IYvcqO};nRHHH!4`l{LA`02`nlf8;g991HfD{Gs=IeU`SSFxjD z>Q6!NnX!svPQxMEZ@q{@7~BZ(y`fn(fD~nSpO!SR=_5oC@@w*5t6)1(Wd^F@T6bE0 zUS0XMpU&V2^EY}C)&fW8Btco;ew3IS@kESmdKct#lm8m!1`W~W78a;YQhCJLL(fCH zc6|Tp1{bM8!UxaYFYJ`QHe*#!MA=WJLHu}j`rUEAJvTO9#Uj1qZ}(JX_)d-1mbx)3 znwF;6{<<$HF~XxRh0lvjkY?q>EPKdV(>FX>nqpk7$^ z^Ceiil~Nf;E5TZGf=0CG+daUj6HP|}#;>HNDe0xees1QKYfl+e9v^+-4DFwEAeh7o zp-ps(X;AT3nV&3sY%|F-?=RZ*GD)>UILrfj>(6!R_-sEV1LN5Z;R4;fF5y&VG1fU1es~ z*1mSK=jR3)TP7GDcF9j)5cUWb;>Yff?MtnZr8<|o<(-uYGEWWZ&rIWMOGFLVZOkjR zm03n2Zc{vu1~Yr9Tkz79r$Y>imV{dSkG)LI*0$lT`b%+W)5L|K(;Lsrqgj`&idKfu za4Hg{(IZUcP=RQHLVN*St=PL2qUb5|dKHiH?t=M1gSk?GC zN}?U>{w`|wUpg%%VOd4LOAvIw3WF8%fdh;n3X>u)XPct1C1R95)lV`a_8-rl1g$Eu zP0iz$hZvylyw(x3rOw~Z98PF8vX6~j%IGQ|G_tlgoKzKWGc}nuGKe!8i--da?HNnv znQJI@uqq+TEG^*e>F`gCV{G#{3KmnE<1v^Wy<%nCORk^%E>tn2PYvPTpMEy6q9rr> z%k6}-(vj&m7kq?i-(b#Zut3;((<6e^re{U$L29bxS7|*fl3F9@J4|E=yZeD{jW=7i zhqtts*xXq;)4OH(E&jP|W~NjoU0z>5xTKSgJqJpc$NW7YAStc!?v*6){@C_DG}?Pg zR^*&gmh?Wd+I!09xj?GKu&-}PsTSw5^2Yv!j9$m7qLMaT`1dx;71CRI>5asBuS3T$ zr%V*fwi^KcE3q_!aY0t+_bC7JwL$Ss$%l94s_xoa7T4W)R{T_kXUwidaYSum zc-s`&r%}Mydk^01nqDe9iF}|oI&D7`apmjcS4I%wbZsS)@1J`Qx0ps1E?6njPq9@# zx$R95^Qe~5kxm>BLfm1Xr6j_FFV8?R?^+(Rifiy($2H8oJs1*^YhwzBPN`q9OD2__ zZE(xXMf-Kj0bQfnVb^s!-!j=*;<74x;M^Y7P=he{yr{$Bv5L77iX>w7VAa)+^^_cm zJWMl4jBIFJ;1P)1URr=Ji|ifwJA;`;UvsaW0jvTi-FbjLw>DEpD`s7@@8O{pookhw zQapvPLnaEQo-um%w+W6u1zau-zBp z)t=XW_~wR--YRodi|if&xn~o_(@swTXX%5_cF4;PO&T<3v7@5tNg7};OI#MUPCAcf z?KeZ6?zoSh>r`=;x}c0ss4R*Z!!=N~L_D;CA<;`lMS496*~$rQxP!+2?;DCTpUx?h zfq|2E-Y3jD-&FLfd5L}zeSY)-g-o(U*Y2(};_-)d*FqbsGLw_plywbR zCXl}|zWPVEwFa2vc~APzy zWuF43x<)$ho2_v;^=n#a&`Xt z_G(~m3)%RzdTI%~+Bt7VzZ1n3;}9ci1fx?6@tkb45-kH?#&Db5&;--giO~L;mK7$r0i!|r^Ns@&sUn*mQ-0*9PqvIW zwX0N#NAX4tZOdB2d+%pv=5{WNtSvDjuns?1I8+~F_ifM%46hR^BdonNti>5k z_MywV+CssGC>ZGS3q&F58bv2mri#gn(%(bhW@UNt@bGlkBJ&iy^JaE~g58j!$op1i zWJHuD0ml*(f$Q3zG~@cWt1~3BIv*v&WqyU{WKm72GJmw?u{z11U46%UBitH~8jxa^ z|JJa&?zwq&h+;2jVr7!m!VLGB8o#*6X2hOtMYbPjB84w}58+ej5a&b0Y_JcY$wzlj zF60g`Xi00y(V_EQ@OzsiLPsocc0F+m(AQ=ojWPege}ntuxtKe-S`u)}hD5$=JiZDz zFU+mKGj_$X*uIR(gp=FaG(o(;!L_o<0GqW){$LH=NnZI9RPBSKR4jt?+M&d zb}pTB&3Z5AdY|+G$)q>5xZanjq0fG%kQ4TooNf2`^h*083-@N%)BYDfvIUf=Xw_-% z2GeWW{zdSQpPZk^`C7urgrL{c4oP(DWe9(@hlgvyPO|FGqg9pM6mD$lpg7^MQ8*AL z5*$hO=&S#pzlE=Nt$?8 zu(+LbhYwxgq{5V^IpPr@jZ3ZNKebddl>91Q4ExE&pCpYI2G;G72F3Lor(LIA!B)f1 zi%0C?Rj}aOU@&v2J(RkoX0*h}CqxQiMfKS!A$<1whl8Jhwo~sDwH&l~BluLSR|{T! zj^E}J5CpFV0h8;@;MKAHs7 zZ$vZpH5;SD(1I*Y18*L`fBmuT?0x6yV{b%TXC2<{R!CEUd81d%C?PG1lqig;l-1g< zf_N+8$mG$1co!S3^;&G5aFb)?s1GU%bwqf9)-VOt0pc7XD~ck!oxmDb$!PWL&-JO% zhZz(VQKN9j9z!>g1UPfdXg^V+)}oaMoH_aQ#|5uJMswqcRXR=L1!*q_OVwZI<(w_- z)-vMEX=<^9U9M}_j!Qc#-79*p=)RVG%!{uBh6bJu*QoeB**O->F3Q?ccaYC6Ty@A& zf-_O}`um&iZkz_WI)s%;!Htx{`tI}*2N4e?5=!6w6smPBoyl=yG}kMIWQBT*dGv8d zDwuN{>Z@5e1DO{s!!k0t%oImTIWsu+mGPPfY#!FQ?Z=9Qp`f&)=NM}(`w3{^IW0@P zqnR^26O62}7S`Xl?f&fANma`|rZ3@2<(3Y-wXMHz*z1;)<-F6fU2TO(=>E!6eU}c} zI}pX^kaw&Y7RoCPSW*fect6zUrOID?i~e2(k~}IPg7%~KnMT$ta!lW1k~_U~S*1OvVrvEh0A@=0sYJtqodt9CpJ*_#GGJ zhMNTE*vHwa1r~CGI_GkD)URkM?rd5${A~oezZQM|_pK?$9=OgO_I5PN(RMS{(ix7DO>5G~-*@!&j(3{S zvSLb9_V~+#%nWNV^fGwogazu`&C#i9s+wf!3`_*SnYv?sm?QK{TPVJ`sr4oVOBsG8 zbY zd8=uE8Fgqt{1&n$JU)yb>U}N4HFo-tb3hs~@69+_6A)Uk6LDl_Md`YdT4_o-Mir~+ z?iFiPY{B2t>?2!sd_W!&eHyXsC;Dpo!1mQ%nQbIxpBTA*z&oNxAxEvHGm0J3iwn}n zFs>)tuPey%AT{6wHk-6{i2A{nm_I)`|W z#V++0`0uB$O588ANYe5Z*dIMvQz8jxbiIuJe@T%S3b44!I?4p<3Lve|O-9FGJ}=rte0!9wX8PZG>E#wPtj; z4XbAz4@KHSyG=pT8tTj)zWfe~_6MT{Z6nYJg*A_Ud&=%BO*U1bD#TSE4lnQPcNKzd zZ4as353cr#$AQsL0OGz5+vcGy)U5nI_3*09xvEZnvRU)QZX_u5P{fI7jhH@uD&Rb{ zKcGIn_g&;wNE7o>AuNn29xEIP$q7w#WEH;2sr{9^W2Duj{{ob587J6C*|&!k?+8lt z;5cZ_9%C8v3O zLBGL{XH@6QYp7S*mG!}l7bZ{mAA1IhRmSezHh4UYhmIr%3)FS3+I^?AQtBdhU(GI; zb&^)#@$u1lM03G{9ozRv0Ybz~;2clD?me^Rx|!Y3gpLS_l!xlblj6B?_sGgTKaP{q z$~bnsQbz`K5_`4l-lH_JFr`_~4)tYmyz2+yymcLA5`fEguW|WUN z&08Y+VXP!+sp!}(F1@7^)uQ?crB~>q8X}dFgZx%-@pqG=tbEsSh39%XXf91{GVr{E z(M$3u%j6qDRJRCXgvMq9xr%qWecRyv+HU}`O-iXi6)bs9T3GqE`b=N8!&^D*3!CBm z?q#-RI6FU+^cfYW2Yz-!c-_jp=V$EIp5x64xgfYNkICk6+@U! z<5_raxNj(^NZxjtoZc)8U7xSJ_5;~l-jakXN6zTY(8y>IN7?ojr#jyUa8tTO`%G8X z{d$nlfkj_ba@c|~CI(27cy81Z&*`!7uQhI<JaR<)t6D&T-d~OHq=VI%m>7|#5S8`P|LmqAKK+ML*n3@3`t}f`T3EE_T@#$y!gh7g0d}U z`48@zoO)Km&!~8ruT@t^=p>A}mwb~a1`c9uH(@W1*A!KZ6gBk}MNb~Gp!>8vS57n3 zbD+Jti^m=8w-n$&1*^73?DI(s&w5j2wQ*`l; z1TR$mf2;-Hzj6@eF)6iFwGxgW@uAduB|41K9wM`l^SNW@EC)wZYxLf)s@=ii^CMZb zU?p9KSYw7QhDwI%i}{u3Xweh{m!9oZ;worPM`=6%BJliw zQhwp@z5Y#D`)^zPZHxcfK7jNTV0vD3Q@VGnzMX^Sh1jcZ7FX>k=lPTU*~_M7wH1no z11$dRsja;q6KXf78W(ZMMvv%s?Iz{;y5+e+SOL?fJJ4 z{D0O5WXz=5I#E>-rDTN0n9K5U?4msK5`6n~sJK!&%j~1M>Qn%=+`aXJ_qww+(0z~O zkb{=HL;SG7P0yrKY#Ui8|7=A7Z9#!1gTyJuaWO{EtxMfb`H?AhC$^*chEdyeF^0_o zInpIIcS2DyEeI>^~ zmcT}-REwMYW3|?vk3aqAwBOt@dxW79t#nL*m+@0lu{qMF#nMA>rb4oJ>Jt($cQPwFVEF~>y_W`gj%N4 zeOE(Zg4svq|M+oG*RM~g?Ki+%`2ojYM4%BGDP!Q7@Ln>q?rU?FzxQNhf9UW&k%noF zAV8C&+^?r9_%~p0ljZ>p3Rb>BF+k|&AnN3!iU6%YI)7&2(sBG`z_;IJYl>&Y3Kk5d zR-)MEv_Ny5Q9lOh=5GSil9i^bI4&q6gj}b-Nw&4b+6x4kjCmR;uSrtijbINN%Q48C zFd@YT;ts0a)ZDacrnG+2Z|4~nbK22ENSP7Hi!u|u`dms@h{?EcAQB^Z^s&C21|gqL zu87}|wgT@J8Q?KXNqRJxM4AwrYF6(-Nyeui_wAkgVSbBqK_5r2S|2Bz$2`D82@sR7 zROn2?F1gO?$5`hYWY=GH{IXZt`(V8@Ib6I#ahd3@FO3zuAp0bp`IrlstAn#*zr{hGenL%zbp>jW6wdzhU055RL8GB*vQER#V z5VICjiC@h`i4AzSK)uYIgA@Gydzp4904vT1NS}7mKK1&1Pm%AC2?}>+cvgU)5hqKd zF==^*35b-6)^z3wE%%z^9LZ_b;KG-L?4#vbR*=T@tE=SE~!}*X?kj(Cx}ReX)cld_JT$cH}xARun$n zJExaJZOSc4=i3rPCG9fm*ZZ~HZ^p{+{DmYHOc=&m+F#u(J_(NG^LuS0COnQ?zff;I z9(4z#?*R%*B=T<=B#9Z@aW5IM&?h z3%qg$DpD7Z0<`Hi#VkN4#P*VZryb3JsYeb zB;i{!%>fR^{Wda?H538;f0ZAbZ&!)`mBkk~w}gJmb-R#D-p@J?IUwp!rfSs6YYdc| zaNUSqZJa8yvS%=CCcGbt`D~;qO%qb@3F@R%gs`A-j|H1tXIUb@ksT#Nc zgPBH0CW}5~*h|$I+5Pm#YeYFioFVGPapJye<)qx3l5=FzioJHqt4Atub9Bh0va<14k}T{)-1Wm(R(ky27THUq#K>;^IMlaZhP>kd9=|d33ct6 zMwl@azuS&+!j~M!Dj6w3baQ&QSfa5&R;&OVDME0!r}*;6F*lo`XOBMGcH=0GPz5)m z>R@wJv{j`gZiI{rrKZKdE5PHAIH6X^=!WHob4?W(<)85;UGZP3-Xu-*a9dF1J9eBA zVNa)9_=s{ZO4gA|%~o>fi=9k}0$-;SzFNaP+CSF->$+)onmpaynvXbrZiRHPOxRYsKI}s#&Fy1(1P$VC!5nbaE zMl1)DHf5J?Gb8I@hT0pSv2R9e^>uI0a^80lC}T08)aH>SL^S zQ)HE?9zBHXYDqHp47#DtjVO6<@nyTd)|DD{9Vl|MX(SLj^syrLkTe7$auA-Jc@ zy`%FvS3ffhoiR-e>NTS@BP8o5LpGrNvrz{x4Y%fR|@ zWJ0LYP`QDh9H+^{;uOwSX~#9{-hXEfqQRX-fCA_>dm|TovxmvXG{UNS#m>^ zT{mYY97-;S01ovv`FXtIl{Xj#zaU~_Ru{c~7Gg1xyv1p!Hl-Wd3Vlq-Uw^31`e;h? z?Vs#IE+foXJ6L%5en_yHx+A7AVEUR43VGJDmlx)O$S1`-EHB)R%FyJQ{8?j9qSNWL z6@A(Fhs)`WS)G0|(GvJ78~E-OH-<0yFewz^qAkE;ze{cve@Or&-3BGjCqSB^sR^c` z8$h~KKJ$jP*NLSzFl);`Lhg2^RJI73g!d+ifM1X=iL;}q@?d>oI56DjPOHG6uR8?IA|p!@W`9C&PC zdJOm@z5IW$1Wu@`4ZFCRsJ985c(9^)NnN>sU?89llCTUp@Hz+CRhT%=iic3`>a3qD z0FN&rX)ZOVRd!SD5@%DgLpK*7eOWzz@&e+g=2ryD_^z|F^nnvuT+2eMe>iqFcJ_V; z%=>OsVig-H&oQ6A3%}mw<_c8B#nT({@v^WeZ0VdHx0Psp)^I@|jSCfHMs`bKNiy+A zQ~VgAUYpd<+SM3kw;?F&Qq48YH7vARI%kbwj&UwY@zuc>=SvBG4pNX&;>oIN3;Ry; zXJL*n`N$n67HVKz|6u4{>=g|M>{(W{$ZGK;;ORQ>mc{*u=68{VEVp&k0htaZ2I;zC zuj?CMByltmIzJU1$zAp5EvewDG>~$0pw2GM~dGD&vUK9hgXx z(N!xYYXlH3U!6tcf&p&EtIr+nI>%B24!I5~{1MYTjl^WC1*T4C_(L3Ds@=CJr|wmr zs21Tbd>`HR20G|=yx%U)(-EDexkU<5xd#YQpCh$Mn=-Z#!l{PyDLG&Z0;$NE6%{QS zQvxUK7N*S;#id{EcwcNXY0;b8w==ZJpW1=u4cVa}#a@mHy`_F6m6SkXv#$ ze&q|hb6%u1a*~kN`2uMtL?NByGU?Lqyz0P@Y49fI*do&Po>k!yU*fjN@CGrB-MI74zP^->q_G$HVmgzr`JC^ScPpYy z$TNg6*~z8hg~{H>2qDzplpjlzA2r)GE+>9kpmQMVeoe1J1c;9?d31+ckN_*e zI7nnUIJMzFkF%PWXpa}jgAgMKva1JIrKCLJN(lWgC+#F^UXvA&a;HwDQ&BciM`7B} z;?E^GQ^U#(=|x`Ywy|6o1GM=RRpv}$d8%$ zSDu)#jZ*}O2_V%AtjL$!RC~s(m{*lZQM^;k=wsu!%6CKI&d$xjDOYHD93QviXl~;; zz$<7Hu(1Dy!?OgM!OEgfS2|t_EVLrt@d7Q*b$;wxn_}UBNM+qCzjSi34{3V7GaOu>@SU=e_fj8Ggcm z@_fB_5a=UqlL@5l(3@-ol3y^l~V+iBDQA+q;jIL=O4KcNS|6l^?y zg&GS^zKr>#rh50IM`|HoZ3<=q5*!0Nz<dqU*Wv!)+Kh%{0h$y*qSS(FCf#?4w>@%?e@ zS~&ix0oK|&ZExf$dX6vkJd!iggRX`N?0}%F6orAbcBqMK?(_L`S`NJLSReXj6lDc96l6$7p?1lV_Smoo5kO6;MT}r-lEnOW5}?lMnng zY4y}{(CSHN`&T;3lW5jr#2FC}?>sF!AV6_r_O1D?}-Qs36RWpm#t)>^7K z`>sEXvqMqN)ms@54*2ok;0xp+F17dnkO4}si{Ofade-j!9|Sm4UXS^96Fugf=-Uw@ z^)4ZXt$@S z{K|Cdn_e}wT@k9lm$3KD$mob)IW`-SRJ;+3BIR(uUcb)li(!cez>K0azzHrB39}ZZ*V7*hu z)3-X_kuFYS4N5H5>cywsi<9+xpJkIUdavl=^|=D#5t}3bGYMm$dm=iuGASIe1!hvl z-#%hY*jR_8ng*r5FRwlOnx@tKsjv*S)ynoz78m7vtJFO@-zXw>hFhGWNig=G^r2OL zWKX2wvs+*h71yzQB^an*V2%-bYG72@T!c<1xQ*?apn@*+iDmbZ*-mMct1fuH?<_o# zfJk@q$_HAQ*y6);--mycH9e1y zynK=yf)KO+R-tb#pc_Y`P853RPt$fmuc<2yFuC?oi@TgUsC?AjlQ}I`U#6lssdQdl z`Z#TN3@;^~(Nu7Y9hIF5*<@|X^XC$kvT~Nr$~7TL7mG~4&z%~LLlS@UrQ=txN{hi9t7N^6qTn8obx<9MCm^u98z>oU?yJ$mo3&_Cfz!%TvX$HzKL?)J7hg34Io0@SLBQ8l;b{n&hj8vx}Ae7 zoOv&0Q?M;;pPM(2_*;a_*sER3$pZnO)6w<3PYfb7!1{~>y0!e?-xS$U6E7fkcq-38$rf5XG-A;cH2abI8QIi$@!~to>ONQ4yU%iK(XbCmZy(ao`zbB zgi3eSe@}KLc%|0sMRuNpWrry+GXrn(oipqUd~7WC>)gyi*ccRoy9TFkZ=xz`eBv!5 zm2>*UQ0?d7T_zUT6||wND&<+t8s|too;up~M>vay3;IWp$liD2TBJtaTcsm~cH6`w z7WrY75^Jl@_Kadz0)qEwsJfH9*I-t+0Qc`QRQib+6xA*WQVmp+QY@=2-(UgztyT+~ znajE^>1+YAC^Z_U&va3hsCQ6Qa80p&WAykhxi_x{IAnf^&d5sxoUdR(&@jf@+KJSn zC=2aM`q?+#<-tXRB*fPQ1QN8$xaEwP1jc_ zNu76lxPoK@;GNba4KVT?c$Dt*+bi(9#RJ(4%(~u5)Q`0F>)EpuR^xugnL^xm{X*=` zL?^+V?hf0;f~;d4SrRu2i*TIW(T=_uBI(i*$9yXHtkb@8Ye*kBg_wnQYi5POVNVZ? z>c%Q*Awe(0K7D4v!U?WIBrjXwxI}>Kz@AfjIv$g<rTb+pUjKe-i2T(BU4!-^DFdzma4L9o7*Ua*10EHU4GA5Lz3}M3sIUn;B7I{ zAt}s@I&9U35EQH(#tD<}l{3lKHh^2FOw+_l#YR%e7H~FxY(-~B0aV5ZtjV*=%kzYF z2JpfOc4bol8`VBQhx=Zb?1QiDZ$Lh_`?(6bfS2%gD|5U`vnH`7C)*e2(oYnxww*@F z`Rr^CVAzxJck!H2i3XOV0fgBpmEPZ33-bI$ANTHv6+qmIugubv!CjjRjV{sN1dAik zLfc&u$DJ?S;fE)Yz|*;uB*lV&`MQ7334XOYL{{NZI!ySZ?@npTOz)rTbBpn)n}cDH z{zS{!BQilVT9>rl?HnAbPw9{NEnhCRvhY1klLQw&1*`#L=lSw|egj_eAV&F6M2P~` zK~<8l-r?qn7UBTRZ~J8>BFX+wy^#NdbUsmvfr95v4p_f`Zi9W*ap$8Ow1_ppB6xuc zhq6_GG0nBMCV`uCnwHpTbGSvD*n!u~6~Jq|6-MJ8`SpzK^HjgW2H|w z|G4(Oa2j0u{wmD)?JGT!j*?++uTqH>9fb7sSDG6h4utv9`!?)m`U+Aj(ZJZp_&*2U zF)4AYIXmd?w+%8qtBnW`*U|NOlvSs>1>Zh%qXhuar4;mvwYv~fE`3ImzXAJ1B#*(? zE!&1Dp3@ZD3DS8sY%d$7N#*P%f=))!wrwE9?&Y2{5q20bSBwc1iaA+H-Ab{NrrP>R z_Mw)83IR?olpfwsld%(lT>vk$;Z!rbwMBa>lg_ltU=DiSbZz#kFC&C|k(6;<4b;Lj zZ&#vlC6x*I9%8~BF3PuQxvH~n?BCGQs0gmR4!M(fyG1=TK$~l!jo)0OEfXidpcqGp zi4Q1+^w-ou!7-J+FEJ!&nURMs$|JI|cj1>|=A_~+wYpfY+NL7(_pKLp*KLwM4k4pj zJKxD<(nbegs18Pup@uYyrtKBg)HgH+Z`x=vWIg%7-E36vwO5}dy8>iJPLJqmC5cxc z+0vW6!m!VY!wk(&BO@&Dc^M2uFF`E&0leIU>5o~oZCvv8wr>mT zpKPuK@7OL|A~Ti7%tOywPH^C47aX`Y+`A-6Fb%V{q$}lQn|JicHiv-wVp68178JNH zg+widXH?|~M;|l4T{r*1PNaK5aR2zypmw_r7BI&Sui%pku3~$USR`yqyC_ zC$_H28N*wmI_n)ycl4Net0RL9LDDi}Z|d0MLG#-23cErv0}59eiBO_1|Ksmz|J81n zf3Nqq?f>?H|1Et0Zyg6sQ2-%j;&M?uCW*E18?f+T`H!vDxeBs?#Fd23KQ_LEfT59x zk?E*En`~(S&p%Gd|9j$}7W&&be|yf~==hsD{{NL8d5M`4e%wx!ennHVR3M{_VS@j&^)!jxj%hgL)UcG>xy(0YTAi| z0$P|>HrufHRj%n-5{)1P zZNY-Nab)Eq@#*GP=aCIox*hu`G)I6)CJgK1mrED$UqHzDh1P~}#}TH)jT`l5!NrV& zqw&mgL?f8(xGdcmI3ys=HQ9S1SK@+P(x_TX-H3A_$pdBtO9R1pVPx~-_W;4XBbA0p zQOpwX^n?>;iYYTEX*rH1a%B{>yDHKiu$t=PhML*5px0mXE|6`|lW1fx?Oxd8GQb+I zT$UvDQi7Hcj^>`Um*G)z_*z|O$`v)%Z>%t?@`FXLksvxkGTPyhAVqE|&7hoK?A?I1 zd+gqf;61DL)3{ZP&B(YoX0zTNw31!#pF5@w)vXQsg{cRtSY_y?ypqMoYIP30ZNl9s-`mRH; zst0jL0{`we0b`6uaG;ZyQ|YzldlKg)fn*!XK>(S4PqcV|KcC+UE5y-YS$Xo%h~{*O z!dT^zK4wEFNhhi@*W$8VUUc^6F8{kP#=iB>m)+r^xgTW!$o{O^bu?xtUrMng*qrAp z_VaT-uSrA!?B*7IYo{zunF8JYJcR9>E{LT0?zDNe!K#!?@1z4?sWt^h3-#>DhHx;o zje-vCO_zBN7iN(KjfJ0WvA3;fEdK_yFg=DyQYm&cWff133Kl5N?>;jQ%txUeviJ*b7ogLJ8a< zlwXg4Pk#)#z!brYJwfLqwbs}c5In8W#4EpL{Z}wTSmws!;OVZiOL0%iDkGsXUsLSY z(J%E#HATBByGg7bPhj2hq~8Iag8PcncA(iMYllyE0-0=9EptMu!(7=0HUdAt_qeP` zp$x!;$feDOxiQJj5-W(m@W)|Vsxc@1_@rXwy9*efOqn<|U57XBg=!-daw_lgqZhb=$W3qLD%Dt&8LqN6R`YYJPeE+M6m zQSz!FtpZ~!t2KY+3$4HTl+r_0`c*qsRz{>z=(1e6$ZzG_3B=`H>iE9hL3tk&GugU!HpA>oaLY znb8sS**^jJ#AhYeY2n#u@~i8>t-W>OvM=zZxd-XhW`9v4g}W_oJY3c>rWSul8hC=9 zwUUn`B}OE*c9w(P{fAh=@-HlBXU)5>bW&hM;1D8Z-ELI4CwVY1&Ur8gp$!t5#GXCz z+yU}ozn}jQWZGapkXh3m(YV+GIJ9Byk~n+%8&LhtZ@O%G$cH0}m{rg?PRouvic5s9 zZzEcWiPA}uBJE3o!(4k!3uG4QH$XnZe*f0ly4*A1d9J?ocPTpdChi1h!Wwk+*fd2E z^_IuU_z#mSLG~zLY@H>Qdi2(E*YUOR{uSX|q2^WH!^qY0)?|YBcFT_A@WSd>h+=YI zE%f8_g&_1nTi>o;x?Rh(y^oD2>`R$8uHPlU=ifw;8mFF9^<5#LsKg8!-wQ*@e(YLj zrItyR>+&9>>sP9-eg702V$3SsefT({|JXnI8n|N?I)Jsx8P&caM+uY&WAr5f##UHp zwbj4u`q3_~N$b_@x+@K^mtVCJ{MFEaw;ZYbSb*LTM%2&pguN>Ju4Ab z*$F@c{2)Y9+EWXdcyA}&QIAO)|4iH%x26Xq>rtUIr`_Sik@ET8OZDZU2fYOV=fbed z@RZG(=3?YFe5l36#3|Q0UgpRG3zfl4ZOLGR)D~wFsp=DGetDs#GSGYQFvZu=JSiou!)~gm1|$T-pd0Y%yjod zlaA;02ua+TLTd#?aYZ5sfgEdzIn&t%vx{SEDmGL%F0T)#)0h~o&0gl9j9(}y8vhiT zf93J>F&|ZKw?YJe;bZ?=vsivd&h?0DJ-DGMNoaqyA^w1;J}BGg^?ueh;0w|0=)YF! zP^n};hA48;b%{hYqfb|#SHqLq#A{<~b-Z3;C}5F!_1SuRtDr&h{cuuEuhfH?8z@xFQsVv5TeyD% zKGsdJe;M;){)YcO>MHCUxv}Emt#C|e#XdSGJO-?Mu^o7}ncSJ?o4?~RGh!d*3>hq^ z!%Cz)KS#(IUrrQGRcPV|5NP1!!hS_g3g|@gCj%jam8T1h8XAB#Wcn=cXXa4|>30R1 z+biV|8VEE>#2%^%$PM=`TQQuW9f4)AM3|r^-keOq5cwO`nYsZfG!2i7Omy1TT z4*#6CQvOw}L8r>qp3W5%0<&vcSb%k>?nP@|0z7yDLn7*s$pb|Mjzc5S%?bQHIwyK6 z#Z|zyb#3{!h8Ao@KnR)pi93cq20uZ`y!gr=^ASk&9^6% zeH9kBgsD3)_fjx^zs3Q>_h~N~NUq@QlRUq$gUY<#r(hraJ-y(N*Nr40d0G9_BYn?I zS$PD~xrVjY9eL*Yz(iAytGV4~bC{kvG^LOeG9%)Z0UB0ae-cFq3*7dB+I~jF*j^QjWzhWPjI^XoCa9Cyas(J~| z8owNtZwOuJs24g%XG)u`R6E_=T?0Rr&nVmou!_sN3!V}8P5fUd?9dj8vQ zm-5BCB-~O_n@f1VOiew?`MswKFA`=rZvuNq(w|q(yg_r6I?__LAr@8+zsQCRnw*J( z{AGqRII6|LVsQ$NZbKUF^mvQX5c*ihK_a;YS=&*x)|d+Tp1b+xq+A)OzQP|3{g^Sc z+7-K{8e@lB?A1c(NIG#UwnwvtT#gnp6ln!JdAS z7;;XCG&4U4B_31q4~j+h_eR8NqB=@#I(B*pE)9uZYjS4EYnn9B4ECEu^)0GqUx`nq` zm&wxsCXiRJU%)GTu(ocdT4aIH3DJUBh9oN`x?sYEqv!^r?17o3Mh9> z6Q04`n(`ya7u60%dytMy54l0IDI)>ZvcwuVU`RALlU}_&P;29*fg@ zXBc6AL{^Z?(SmV$Wj14(pRCjZ0;9;9c6?%Qs0_qJK87#5ZX1w+pmu`07^0j0eD;+l zbr7Ghw~10?gf(LR0FR1!KYZOGFQSUeWQen3r_3f*?nsW1=u|q&WNv->!FZR8_@(82 zO}*VC9zt1O{3685#m%YQYI&aDyc;eQ_^x+V<Ml8FeSfF94YbNbGRgEoLhze)@0C5H&%3?9x@m<5l_4=(9fVc3vT_>YdTE z_8e@Hz3qMW`wJ!cz>*>w@k5SX(oNvqHuUZ|w*C40>>;J{H`%#pA z9h={fePP(by+Kp4kX}rN+v#Q$ExNd*qfkrUOSpHeCo=R0Q+)Kg_q6L5X2_X|LOT1K zK2xJ|_CX4j_)!bS^fkH|%tabk#BxZG$`qfCo!^JYf$?dD)FKwLZy(C>PzXA|BM1!E zclz}cU=dxiX;p@i1xK{8VIiT5OOWu|y;QNErDwqSfXCppnyrHjI_;y;$LotzQAUSZ zze3v6_tdQ(i#-V69E189b`Ce2Okw;NNwa1PU+YiNkcNG3ftk+K)HT8$VFeHun3NHS z&b!ezdveC`-x0x3|J4ZF+!{71{2}S21w44FM}W^h$|bnhEp0W!wW`e&qBlklf!@7q zU!NyFPH$CRKyc8+Bn%*vuhS;m-E5r8o26ZdyUi|$O7&VmooAk9WNtRV+KiulHl#+o zi}SB>4Q-z}FWBe=Lrb-FmKo~#4^t9;R%)_dNgidZ|1o45VdyR`UtWA`;SF)0j@jVN z!#iSK%fbbyI(U_WhAv~Bgj-gnHs|#!K6xPbuujm9qQ8}Pp0G)d_XG~x$ovL`MGatl zg$p`m_K@}*XAf71*EqS^iyLj)A5(Cdi}zSU_5SG0)w zlREC`NDpV11E5=`ffmQXd2f~7e&*U411z0Xb>i96cKSc}a!FGHy_(ilOjd(Vs;-`y zWYOjX;5@4~U=e%bYXb}yDRmfU_1rKsJ=LiHLA@1~4@K}aCZ-hh-%#{bs^M+eEt|ZT zgzGk-w~CDKYGT_IF6wbCMKgZ7c+%K3WrGW7CsYI%YjwtCeR0O5nH)t+zs{TN9k8B|$F{j#G}XmdoBiXK{%g(rAHA6`io9nML)# zC4*>V)_-bl^`i9d$u!b-TQPtawEg!r^xH!MgMavCVqm2y!e^kvZL|H9I7;1O;Ky1O zLQlXxd}Opz=jcss=CFPqMNBCr`RM-`_2u7FUADyW#zkdaR?@91KC54{np@1FUw(GB zy%{-q9zVj@`0#c0=$5MOG?Cv@sZBu)vd_U;0k4xe=dhi>NP}0_2eQ#Lc z89m=2z+>oHd;sb1YsKBaURyqkSw*MIvT<7XX~Wd9J~qsJ??a~ea{)s%pC#aJ<aB zPz|T5M%Bwhj|4L{^-n&u@-~dtW1Jg(hZlB;T_-a+(E)NF6JMvV+99l^++j3f`)sRe zC)yzU3jIhW4bxKWUcm}GL>vOEuWS4mV-a7=x*u*UwRWc_Rc$Z20efH>2`Owm@e2kZ+GR#D4?dvWp-mW;qk;yAS1$Mn^!Yq7q+i66As2^+3VcP*-3Mso$rUqF*SUT z%;0e*bY0EN+4VX>T@Lzn!fHdTNvy~YUR|h&m+S~uf9WQ=PF;_*ouJ9^tHs>>xMHVV zX~?UfkD+BDj$dnkor|eb6P+){rfSz))4#G=nrA@lveX4O{B$89RKzYyas-(Y(G&Bg zCBm!hd6%E=LLqTp9(X#2Yi_PxL2$%wo4+tFI5RIO3v|mv@|(RZ)@6w1t62r8$8s zn4X5m-erK&p9kS~1R~mi3WJJ+w(45Pn;jTyV;p1kwe^;J9Q8?m)gtxB&V*n5(<|&h zn|J=lV=SyG#JkhF9=+Nsxjnh0rItB3O5+1669>~n_hJ^F#y4u~^5h{@5l?v1#-6`tU@e4S`< zL(mEY96w8mXSt3QcSi1P@NH!wuxn4}X9N#`9K6JjBQ*+7ziGMyya)tW8rre-P)Mn9j9tjF+Yk{!V*JI{L?i?FKtUz`P*aaf;{#UH_As*+^x z$vU9P3aE|_{2ks#B97ZR% zY9;YSDG0`rO;9lgtC>!y}i`Zn0+n`}2pN|vo^kT(JwSgmg&Kg!N-G~Ri( z(y_UsA>%>$d~XKKmPXY{jhX%-^l4@0Q$boZXJgKbFcn*pMYs$_bz zMv6&S@hP262YhS(7*p305LS83A^C>Bq?nxQ&>!N3NQI6y-u5NjAlde`t6r8j;$=Px zm1N(p^-?$uj~_yRo{6r5Uz7Yx1XauA7roqt#06Z`FsfGs z-Rqt(g*BR69vzI1LmD4b6s6>HIfl%6#7Wu~oag`RYVcZp9R`>cfOa!5+%nc_U z__ymj!&@MQss<#IV^TY=AIUY4)-a5BqsKcIGFp=#?(*Js>;8ue^SF zTiGa9;o**HVue!9bC_ysL6<|}tpRmU3kn9mE-uyu!S!AiTy9E6#g2Uy1tJVIqr@_& zn+itIa&6f%Q%K+RQhkq(lv}H^4q#m-(4`Hwqh{?f5WPN3M85kKIU|KY#P23~M?F*~ zji7w_tZBtIKyhD^FE(Ppr94@2lryz9Rlf~fxvyfhB>PFCw~nVwhCU5J@V6AjhcCCZ zeG!qyo@#vh$zRYpUx+@o`%B^*cIt=EFG{phxLkPSxEU|jiBViKG(RFd3&M&Fwy#w6 zcD~8v*`zQg`w0ve&t*45}2{s=R@LEQ=*?d=WaeQ!R)u z2Cw(-E=IO8?q*&|U`{$3MwNXtYtZ6JRP4b%71jL8#K``~&HJ##Xa0hF(+E}oDFm22 zlUBCtognY9@j{M$>MC&rXe>A~UeUu}rm1qq{t1=zAs+X!+9Yb>WMsVFG6$=aX*xL5 ze8&Q*$wSqN%rB9o5y6p(S<|Ukl|+?w!f1db5jfJn`F3k+{0G6lr#tQ-%2#L=w{)Fm zbv__6V8%IMN5E2Igv3_f6a#WVtPyt{$2_2eBzPW)CJ;1*;4qLCY%7|>o_C=4Fi%mO z-XiiCH)V@x=0D)hU0@Hb`A((UDfP=xthq!SB$`JO<2x2{e{dh67YBMboUFvtqX^kHlT zq6+)?q4Du7OL-5t5kU4cEKmLkr`b}@ujp?+lL<(@JAU(N-UIYm)I5uWU4;z-u>lLm zV*ckK);om*-4D?0{XMG^-9g4^f$xI7Vo#~_5OIy;l{3Yc-X!H~3;{yMpQe6Zd65_* z$u;V0jahPGdO|ppJ-cil&PhC(o*+eou$Xf8nNgJnn94Y~`G}M&%6{Ni?W6v&m4&FM z+~J7wxopjZt*vpm=hTS{U4{|0Jmqctp00)pn&Q-D4eDeVPBc_y><Ezjzb-?WerH zOKd5JInf8H)I$7+$=)BxGenID^3bXT9yZ4v%|2&#d@o6>=X@RM@N!EAFtb?z_W6>S ze^lo7v*SUMxhEICW&U>>-Xryg^{4elQXi?gh(;PS_@LE zR64tImmLzx8~ndKJ~?{sN|28=M-|pIM4{=gnW^snp_t*=@#?0EQaBjzY%EKDUAqYx zc$pFH?RUGQDbpD9lLVOldEhR=B6*XBH>!_p=Rlu|5Xs;tQ-~5cTnao+2qfEVs$rosIf@&`#2Za>xfL0o8<(gUKSDclL251ToO?y<(c8 zpKxvbbtf7RMfhzU&zroRpbLx-Z>zF$DBCe#BGKBE)uv;gaNNnZ9zA{WJhfAPiwJHC z7r@`iejkS*lupjLH7W1!Me(x=W>jT5THs)yGg(J_^>|l@^WhR2xi}Q|)cf>z+_F1N zI<;dl464$b&AS|^2+*kSS|UI1biWN6oVIc>c3{Vk7{Se`386a-%*t zV0kQC64cP)abMIwqq~5aOZlu$As8Ce+%W1&Qge3btB9y=wFUwsG^?T7i@B#3P;OK< zp6^nkroatCq(BGk@x?_OScU?!ERtLf4vjI6JiSvK!Y}T9Lj!F5pz?Jhft88~%M-u| zU8z_Nupo*dDjy+06HKOMihuY@@)-bts7+CoAk%2pGzm<@va-V1yBbU_^&vN*-u^!1 z(eN3SNxl*{sy9=1bPyS>PiM$A=DVHI!oUMUq(q`_@IxkV+-&Sp3sVMz$UUoNMtrx|1qs%5ZSO_`#-~Fd4_whjUD3Jsj`nu{%rU z?3MZD){tb)=`b?~^R96T#DuleM2Hee_xwK%$k$=T%BIQci?tNxjUQ^+{M>v27hQ

    E8kbR_FQo}A8mf-kj;DjW{zX{>3C?{e2Hr?y*zx_T@KGUfE> z6fDuCT<7RnJTervj`zhrHek6v*c2k@q$@RzhQtfYyq*xBk5Pt>8()XHrXl}5;LeX3 zP}RKJ1P7b#;;*vpc){;W$u^}D z&p$7AyO_7;>&_>08MZW7-I6`<#5&KW?6}uUU_T=|eA|_+6-%~Gc53mQyD?6>^(R|J z;veUi!C6+Wm*Rk})ab)2JVlr|7tUyW#{ZtFg54C@o|cq~DQA1SWV+k7%NO)SR%?44 z|Hov&s^9izosLOq(=&U$K;@0G2R=khytH-20WEd$`BbYr; z$lLPM8s$$aU)}xe4IE7O2RNta=i7XCl{-GCZ8A^6@$0h24iXz;e)Jtu-8S93>pw%n zp*!wH8{f%plHO|_vgyiKU8Q@@B8K-5TAiE7erk@K$o!Z((fz+CV;%Qdm0hk| zo8H`Nl#IObd#VJ}^uyo6#q8vpu067ytdX@)NheV-KSkYhkoB?VAx-Gb#FjS$ZIQ&(Kul4qR9!>9uX`1UC*d@M5&{6_bzN6WjTrW|!OK+MWB$HXMEGeZj)wG{57W z>&pYZL^Y2tTl58Fity5~!pxJ|PyaKp={9uU zIKy1Q+~;?QoyNvJmulP-AI;Uemfgk@^-Ao6V%apIiiE|JLat{5cUf&)y7=g#ABzL7 zy!)g1cki-Su4%lYi~Mc(%s3d^d+ze}zNz~kmR#1;d$M!WhvJNd!rV6B0;5m-e8mty zTRsNV(yrX!WfB`uG40iP-)%G3)HqF4y>oG)Hly3&&4QPU3eQ)){1f`{+BF-UTe;mj z#s`ADWBy+FI3Ywn&oGu3c)f@FW8cUGvC9?KQC16Lmj2$ulKx@ct1gBICf6>kf4{Kp z{== zN5bs*-~SnwN`pz@{RwH;|NUop=>{f2cPnfK-emVH5KMwDQOE`90u#XN6-Kp=2GeMg z7|l7OrNn57NT}jyIPmGG+>GVGVK}l*)fm+`8g8R$U^ESkrh(BkFq#I4P6PG-Zvp@T CkE@OV From f8627a2f48783570a98c550035dfa4fd80ba197c Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Tue, 19 May 2020 14:15:57 +0800 Subject: [PATCH 05/14] Reduce trial numbers for PAI IT (#2451) --- test/config/training_service.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/config/training_service.yml b/test/config/training_service.yml index 65f2676960..5ceb8b2c6e 100644 --- a/test/config/training_service.yml +++ b/test/config/training_service.yml @@ -78,12 +78,15 @@ paiYarn: pai: nniManagerIp: maxExecDuration: 15m + # PAI has job submission limitation, set maxTrialNum=1 to control trial job numbers for PAI + maxTrialNum: 1 + trialConcurrency: 1 paiConfig: host: userName: trainingServicePlatform: pai trial: - gpuNum: 1 + gpuNum: 1 cpuNum: 1 image: memoryMB: 8192 From 4c2b4c807e2aed4c8163bc0d63e67a259fd7ccbb Mon Sep 17 00:00:00 2001 From: Jimmy Yao Date: Mon, 18 May 2020 23:22:11 -0700 Subject: [PATCH 06/14] typo --- src/sdk/pynni/nni/ppo_tuner/distri.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/pynni/nni/ppo_tuner/distri.py b/src/sdk/pynni/nni/ppo_tuner/distri.py index 0c1626730d..9af2e1add9 100644 --- a/src/sdk/pynni/nni/ppo_tuner/distri.py +++ b/src/sdk/pynni/nni/ppo_tuner/distri.py @@ -61,7 +61,7 @@ def sample_placeholder(self, prepend_shape, name=None): class CategoricalPd(Pd): """ - Categorical prossibility distribution + Categorical probability distribution """ def __init__(self, logits, mask_npinf, nsteps, size, is_act_model): self.logits = logits From bd7edf36a22813c89281adfd8aae29ee59851448 Mon Sep 17 00:00:00 2001 From: chicm-ms <38930155+chicm-ms@users.noreply.github.com> Date: Tue, 19 May 2020 14:36:53 +0800 Subject: [PATCH 07/14] fix speedup issue (#2447) --- .../compression/speedup/torch/compressor.py | 2 ++ src/sdk/pynni/tests/test_model_speedup.py | 32 +++++++++++++++---- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/sdk/pynni/nni/compression/speedup/torch/compressor.py b/src/sdk/pynni/nni/compression/speedup/torch/compressor.py index 835c6f269a..084d5b8ea4 100644 --- a/src/sdk/pynni/nni/compression/speedup/torch/compressor.py +++ b/src/sdk/pynni/nni/compression/speedup/torch/compressor.py @@ -163,9 +163,11 @@ def speedup_model(self): first, do mask/shape inference, second, replace modules """ + training = self.bound_model.training _logger.info("start to speed up the model") _logger.info("infer module masks...") self.infer_modules_masks() _logger.info("replace compressed modules...") self.replace_compressed_modules() + self.bound_model.train(training) _logger.info("speedup done") diff --git a/src/sdk/pynni/tests/test_model_speedup.py b/src/sdk/pynni/tests/test_model_speedup.py index 6b9417268b..31cb5712ce 100644 --- a/src/sdk/pynni/tests/test_model_speedup.py +++ b/src/sdk/pynni/tests/test_model_speedup.py @@ -10,9 +10,11 @@ from torchvision.models.resnet import resnet18 from unittest import TestCase, main -from nni.compression.torch import L1FilterPruner +from nni.compression.torch import L1FilterPruner, apply_compression_results from nni.compression.speedup.torch import ModelSpeedup +torch.manual_seed(0) + class BackboneModel1(nn.Module): def __init__(self): super().__init__() @@ -58,7 +60,10 @@ def forward(self, x): x = self.fc3(x) return x +dummy_input = torch.randn(2, 1, 28, 28) SPARSITY = 0.5 +MODEL_FILE, MASK_FILE = './11_model.pth', './l1_mask.pth' + def prune_model_l1(model): config_list = [{ 'sparsity': SPARSITY, @@ -66,14 +71,14 @@ def prune_model_l1(model): }] pruner = L1FilterPruner(model, config_list) pruner.compress() - pruner.export_model(model_path='./11_model.pth', mask_path='./l1_mask.pth') + pruner.export_model(model_path=MODEL_FILE, mask_path=MASK_FILE) class SpeedupTestCase(TestCase): def test_speedup_vgg16(self): prune_model_l1(vgg16()) model = vgg16() model.train() - ms = ModelSpeedup(model, torch.randn(2, 3, 32, 32), './l1_mask.pth') + ms = ModelSpeedup(model, torch.randn(2, 3, 32, 32), MASK_FILE) ms.speedup_model() orig_model = vgg16() @@ -88,20 +93,33 @@ def test_speedup_vgg16(self): def test_speedup_bigmodel(self): prune_model_l1(BigModel()) model = BigModel() + apply_compression_results(model, MASK_FILE, 'cpu') + model.eval() + mask_out = model(dummy_input) + model.train() - ms = ModelSpeedup(model, torch.randn(2, 1, 28, 28), './l1_mask.pth') + ms = ModelSpeedup(model, dummy_input, MASK_FILE) ms.speedup_model() + assert model.training + + model.eval() + speedup_out = model(dummy_input) + if not torch.allclose(mask_out, speedup_out, atol=1e-07): + print('input:', dummy_input.size(), torch.abs(dummy_input).sum((2,3))) + print('mask_out:', mask_out) + print('speedup_out:', speedup_out) + raise RuntimeError('model speedup inference result is incorrect!') orig_model = BigModel() - assert model.training + assert model.backbone2.conv1.out_channels == int(orig_model.backbone2.conv1.out_channels * SPARSITY) assert model.backbone2.conv2.in_channels == int(orig_model.backbone2.conv2.in_channels * SPARSITY) assert model.backbone2.conv2.out_channels == int(orig_model.backbone2.conv2.out_channels * SPARSITY) assert model.backbone2.fc1.in_features == int(orig_model.backbone2.fc1.in_features * SPARSITY) def tearDown(self): - os.remove('./11_model.pth') - os.remove('./l1_mask.pth') + os.remove(MODEL_FILE) + os.remove(MASK_FILE) if __name__ == '__main__': main() From d7456c168231f35bd510ace6256570946ea55d83 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Tue, 19 May 2020 14:49:53 +0800 Subject: [PATCH 08/14] Refactor code storage logic for trial (#2403) --- .../frameworkcontrollerTrainingService.ts | 58 +++++++++---------- .../kubeflow/kubeflowTrainingService.ts | 36 ++++++------ .../kubernetes/kubernetesData.ts | 4 +- .../kubernetes/kubernetesTrainingService.ts | 55 +++++++++--------- .../local/localTrainingService.ts | 39 ++++++++----- .../training_service/pai/paiK8S/paiK8SData.ts | 7 +-- .../pai/paiK8S/paiK8STrainingService.ts | 21 +++++-- .../remote_machine/remoteMachineData.ts | 11 ++-- .../remoteMachineTrainingService.ts | 32 +++++++--- .../remote_machine/shellExecutor.ts | 5 +- test/nni_test/nnitest/run_tests.py | 1 + 11 files changed, 149 insertions(+), 120 deletions(-) diff --git a/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts b/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts index 87e8ad6c7a..7f57178cdb 100644 --- a/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts +++ b/src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts @@ -3,6 +3,7 @@ 'use strict'; +import * as assert from 'assert'; import * as cpp from 'child-process-promise'; import * as fs from 'fs'; import * as path from 'path'; @@ -72,6 +73,11 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple this.kubernetesRestServerPort = restServer.clusterRestServerPort; } + // wait upload of code Dir to finish + if (this.copyExpCodeDirPromise !== undefined) { + await this.copyExpCodeDirPromise; + } + const trialJobId: string = uniqueString(5); // Set trial's NFS working folder const trialWorkingFolder: string = path.join(this.CONTAINER_MOUNT_PATH, 'nni', getExperimentId(), trialJobId); @@ -81,8 +87,8 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple this.generateContainerPort(); await this.prepareRunScript(trialLocalTempFolder, trialJobId, trialWorkingFolder, form); - //upload code files - const trialJobOutputUrl: string = await this.uploadCodeFiles(trialJobId, trialLocalTempFolder); + //wait upload of script files to finish + const trialJobOutputUrl: string = await this.uploadFolder(trialLocalTempFolder, `nni/${getExperimentId()}/${trialJobId}`); let initStatus: TrialJobStatus = 'WAITING'; if (!trialJobOutputUrl) { initStatus = 'FAILED'; @@ -151,6 +157,8 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple // Validate to make sure codeDir doesn't have too many files try { await validateCodeDir(this.fcTrialConfig.codeDir); + //upload codeDir to storage + this.copyExpCodeDirPromise = this.uploadFolder(this.fcTrialConfig.codeDir, `nni/${getExperimentId()}/nni-code`); } catch (error) { this.log.error(error); @@ -171,41 +179,31 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple } /** - * upload code files to nfs or azureStroage - * @param trialJobId - * @param trialLocalTempFolder - * return: trialJobOutputUrl + * upload local folder to nfs or azureStroage */ - private async uploadCodeFiles(trialJobId: string, trialLocalTempFolder: string): Promise { + private async uploadFolder(srcDirectory: string, destDirectory: string): Promise { if (this.fcClusterConfig === undefined) { throw new Error('Kubeflow Cluster config is not initialized'); } - if (this.fcTrialConfig === undefined) { - throw new Error('Kubeflow trial config is not initialized'); - } - - let trialJobOutputUrl: string = ''; + assert(this.fcClusterConfig.storage === undefined + || this.fcClusterConfig.storage === 'azureStorage' + || this.fcClusterConfig.storage === 'nfs'); - if (this.fcClusterConfig.storageType === 'azureStorage') { - const azureFrameworkControllerClusterConfig: FrameworkControllerClusterConfigAzure = - this.fcClusterConfig; - trialJobOutputUrl = await this.uploadFilesToAzureStorage(trialJobId, trialLocalTempFolder, this.fcTrialConfig.codeDir, - azureFrameworkControllerClusterConfig.uploadRetryCount); - } else if (this.fcClusterConfig.storageType === 'nfs') { - const nfsFrameworkControllerClusterConfig: FrameworkControllerClusterConfigNFS = - this.fcClusterConfig; - // Creat work dir for current trial in NFS directory - await cpp.exec(`mkdir -p ${this.trialLocalNFSTempFolder}/nni/${getExperimentId()}/${trialJobId}`); - // Copy code files from local dir to NFS mounted dir - await cpp.exec(`cp -r ${trialLocalTempFolder}/* ${this.trialLocalNFSTempFolder}/nni/${getExperimentId()}/${trialJobId}/.`); - // Copy codeDir to NFS mounted dir - await cpp.exec(`cp -r ${this.fcTrialConfig.codeDir}/* ${this.trialLocalNFSTempFolder}/nni/${getExperimentId()}/${trialJobId}/.`); - const nfsConfig: NFSConfig = nfsFrameworkControllerClusterConfig.nfs; - trialJobOutputUrl = `nfs://${nfsConfig.server}:${path.join(nfsConfig.path, 'nni', getExperimentId(), trialJobId, 'output')}`; + if (this.fcClusterConfig.storage === 'azureStorage') { + if (this.azureStorageClient === undefined) { + throw new Error('azureStorageClient is not initialized'); + } + const fcClusterConfigAzure: FrameworkControllerClusterConfigAzure = this.fcClusterConfig; + return await this.uploadFolderToAzureStorage(srcDirectory, destDirectory, fcClusterConfigAzure.uploadRetryCount); + } else if (this.fcClusterConfig.storage === 'nfs' || this.fcClusterConfig.storage === undefined) { + await cpp.exec(`mkdir -p ${this.trialLocalNFSTempFolder}/${destDirectory}`); + await cpp.exec(`cp -r ${srcDirectory}/* ${this.trialLocalNFSTempFolder}/${destDirectory}/.`); + const fcClusterConfigNFS: FrameworkControllerClusterConfigNFS = this.fcClusterConfig; + const nfsConfig: NFSConfig = fcClusterConfigNFS.nfs; + return `nfs://${nfsConfig.server}:${destDirectory}`; } - - return Promise.resolve(trialJobOutputUrl); + return ''; } /** diff --git a/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts b/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts index 98c84a30b0..8a082e949f 100644 --- a/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts +++ b/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts @@ -74,14 +74,20 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber const restServer: KubeflowJobRestServer = component.get(KubeflowJobRestServer); this.kubernetesRestServerPort = restServer.clusterRestServerPort; } + + // upload code Dir to storage + if (this.copyExpCodeDirPromise !== undefined) { + await this.copyExpCodeDirPromise; + } + const trialJobId: string = uniqueString(5); const trialWorkingFolder: string = path.join(this.CONTAINER_MOUNT_PATH, 'nni', getExperimentId(), trialJobId); const kubeflowJobName: string = `nni-exp-${this.experimentId}-trial-${trialJobId}`.toLowerCase(); const trialLocalTempFolder: string = path.join(getExperimentRootDir(), 'trials-local', trialJobId); //prepare the runscript await this.prepareRunScript(trialLocalTempFolder, trialJobId, trialWorkingFolder, form); - //upload files to sotrage - const trialJobOutputUrl: string = await this.uploadCodeFiles(trialJobId, trialLocalTempFolder); + //upload script files to sotrage + const trialJobOutputUrl: string = await this.uploadFolder(trialLocalTempFolder, `nni/${getExperimentId()}/${trialJobId}`); let initStatus: TrialJobStatus = 'WAITING'; if (!trialJobOutputUrl) { initStatus = 'FAILED'; @@ -152,6 +158,8 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber // Validate to make sure codeDir doesn't have too many files try { await validateCodeDir(this.kubeflowTrialConfig.codeDir); + //upload codeDir to storage + this.copyExpCodeDirPromise = this.uploadFolder(this.kubeflowTrialConfig.codeDir, `nni/${getExperimentId()}/nni-code`); } catch (error) { this.log.error(error); @@ -172,12 +180,9 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber } /** - * upload code files to nfs or azureStroage - * @param trialJobId - * @param trialLocalTempFolder - * return: trialJobOutputUrl + * upload local folder to nfs or azureStroage */ - private async uploadCodeFiles(trialJobId: string, trialLocalTempFolder: string): Promise { + private async uploadFolder(srcDirectory: string, destDirectory: string): Promise { if (this.kubeflowClusterConfig === undefined) { throw new Error('Kubeflow Cluster config is not initialized'); } @@ -186,8 +191,6 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber throw new Error('Kubeflow Trial config is not initialized'); } - let trialJobOutputUrl: string = ''; - assert(this.kubeflowClusterConfig.storage === undefined || this.kubeflowClusterConfig.storage === 'azureStorage' || this.kubeflowClusterConfig.storage === 'nfs'); @@ -197,20 +200,15 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber throw new Error('azureStorageClient is not initialized'); } const azureKubeflowClusterConfig: KubeflowClusterConfigAzure = this.kubeflowClusterConfig; - trialJobOutputUrl = await this.uploadFilesToAzureStorage(trialJobId, trialLocalTempFolder, this.kubeflowTrialConfig.codeDir, azureKubeflowClusterConfig.uploadRetryCount); + return await this.uploadFolderToAzureStorage(srcDirectory, destDirectory, azureKubeflowClusterConfig.uploadRetryCount); } else if (this.kubeflowClusterConfig.storage === 'nfs' || this.kubeflowClusterConfig.storage === undefined) { + await cpp.exec(`mkdir -p ${this.trialLocalNFSTempFolder}/${destDirectory}`); + await cpp.exec(`cp -r ${srcDirectory}/* ${this.trialLocalNFSTempFolder}/${destDirectory}/.`); const nfsKubeflowClusterConfig: KubeflowClusterConfigNFS = this.kubeflowClusterConfig; - // Creat work dir for current trial in NFS directory - await cpp.exec(`mkdir -p ${this.trialLocalNFSTempFolder}/nni/${getExperimentId()}/${trialJobId}`); - // Copy script files from local dir to NFS mounted dir - await cpp.exec(`cp -r ${trialLocalTempFolder}/* ${this.trialLocalNFSTempFolder}/nni/${getExperimentId()}/${trialJobId}/.`); - // Copy codeDir to NFS mounted dir - await cpp.exec(`cp -r ${this.kubeflowTrialConfig.codeDir}/* ${this.trialLocalNFSTempFolder}/nni/${getExperimentId()}/${trialJobId}/.`); const nfsConfig: NFSConfig = nfsKubeflowClusterConfig.nfs; - trialJobOutputUrl = `nfs://${nfsConfig.server}:${path.join(nfsConfig.path, 'nni', getExperimentId(), trialJobId, 'output')}`; + return `nfs://${nfsConfig.server}:${destDirectory}`; } - - return Promise.resolve(trialJobOutputUrl); + return ''; } private async prepareRunScript(trialLocalTempFolder: string, trialJobId: string, trialWorkingFolder: string, diff --git a/src/nni_manager/training_service/kubernetes/kubernetesData.ts b/src/nni_manager/training_service/kubernetes/kubernetesData.ts index 3d3cd3a5b8..7e0729e24c 100644 --- a/src/nni_manager/training_service/kubernetes/kubernetesData.ts +++ b/src/nni_manager/training_service/kubernetes/kubernetesData.ts @@ -39,7 +39,7 @@ export class KubernetesTrialJobDetail implements TrialJobDetail { export const kubernetesScriptFormat: string = `#!/bin/bash export NNI_PLATFORM={0} -export NNI_SYS_DIR=$PWD/nni/{1} +export NNI_SYS_DIR={1} export NNI_OUTPUT_DIR={2} export MULTI_PHASE=false export NNI_TRIAL_JOB_ID={3} @@ -49,7 +49,7 @@ export NNI_TRIAL_SEQ_ID={6} {7} mkdir -p $NNI_SYS_DIR mkdir -p $NNI_OUTPUT_DIR -cp -rT $NNI_CODE_DIR $NNI_SYS_DIR +cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR cd $NNI_SYS_DIR sh install_nni.sh python3 -m nni_trial_tool.trial_keeper --trial_command '{8}' --nnimanager_ip {9} --nnimanager_port {10} \ diff --git a/src/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts b/src/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts index 56870fac97..f21ac9ad69 100644 --- a/src/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts +++ b/src/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts @@ -49,6 +49,8 @@ abstract class KubernetesTrainingService { protected kubernetesClusterConfig?: KubernetesClusterConfig; protected versionCheck: boolean = true; protected logCollection: string; + protected copyExpCodeDirPromise?: Promise; + protected expContainerCodeFolder: string; constructor() { this.log = getLogger(); @@ -57,6 +59,7 @@ abstract class KubernetesTrainingService { this.trialLocalNFSTempFolder = path.join(getExperimentRootDir(), 'trials-nfs-tmp'); this.experimentId = getExperimentId(); this.CONTAINER_MOUNT_PATH = '/tmp/mount'; + this.expContainerCodeFolder = path.join(this.CONTAINER_MOUNT_PATH, 'nni', this.experimentId, 'nni-code'); this.genericK8sClient = new GeneralK8sClient(); this.logCollection = 'none'; } @@ -272,11 +275,11 @@ abstract class KubernetesTrainingService { const runScript: string = String.Format( kubernetesScriptFormat, platform, - trialJobId, + trialWorkingFolder, path.join(trialWorkingFolder, 'output', `${roleName}_output`), trialJobId, getExperimentId(), - trialWorkingFolder, + this.expContainerCodeFolder, trialSequenceId, nvidiaScript, command, @@ -329,51 +332,45 @@ abstract class KubernetesTrainingService { ); return registrySecretName; } - - protected async uploadFilesToAzureStorage(trialJobId: string, trialLocalTempFolder: string, codeDir: string, uploadRetryCount: number | undefined): Promise { + + /** + * upload local directory to azureStorage + * @param srcDirectory the source directory of local folder + * @param destDirectory the target directory in azure + * @param uploadRetryCount the retry time when upload failed + */ + protected async uploadFolderToAzureStorage(srcDirectory: string, destDirectory: string, uploadRetryCount: number | undefined): Promise { if (this.azureStorageClient === undefined) { throw new Error('azureStorageClient is not initialized'); } - let trialJobOutputUrl: string = ''; let retryCount: number = 1; if(uploadRetryCount) { retryCount = uploadRetryCount; } - let resultUploadNNIScript: boolean = false; - let resultUploadCodeFile: boolean = false; + let uploadSuccess: boolean = false; + let folderUriInAzure = ''; try { do { - //upload local files, including scripts for running the trial and configuration (e.g., hyperparameters) for the trial, to azure storage - if(!resultUploadNNIScript) { - resultUploadNNIScript = await AzureStorageClientUtility.uploadDirectory(this.azureStorageClient, - `nni/${getExperimentId()}/${trialJobId}`, this.azureStorageShare, - `${trialLocalTempFolder}`); - } - //upload code files to azure storage - if(!resultUploadCodeFile) { - resultUploadCodeFile = await AzureStorageClientUtility.uploadDirectory(this.azureStorageClient, - `nni/${getExperimentId()}/${trialJobId}`, this.azureStorageShare, - `${codeDir}`); - } - if (resultUploadNNIScript && resultUploadCodeFile) { - trialJobOutputUrl = `https://${this.azureStorageAccountName}.file.core.windows.net/${this.azureStorageShare}` + - `/${path.join('nni', getExperimentId(), trialJobId, 'output')}`; - break; - } else { + uploadSuccess = await AzureStorageClientUtility.uploadDirectory( + this.azureStorageClient, + `${destDirectory}`, + this.azureStorageShare, + `${srcDirectory}`); + if (!uploadSuccess) { //wait for 5 seconds to re-upload files await delay(5000); this.log.info('Upload failed, Retry: upload files to azure-storage'); + } else { + folderUriInAzure = `https://${this.azureStorageAccountName}.file.core.windows.net/${this.azureStorageShare}/${destDirectory}`; + break; } } while (retryCount-- >= 0) } catch (error) { this.log.error(error); //return a empty url when got error - return Promise.resolve(""); - } - if(!trialJobOutputUrl) { - this.log.info(`Retry-count is used up, upload files to azureStorage for trial ${trialJobId} failed!`); + return Promise.resolve(''); } - return Promise.resolve(trialJobOutputUrl); + return Promise.resolve(folderUriInAzure); } } diff --git a/src/nni_manager/training_service/local/localTrainingService.ts b/src/nni_manager/training_service/local/localTrainingService.ts index fed15a3ff7..27bd42c385 100644 --- a/src/nni_manager/training_service/local/localTrainingService.ts +++ b/src/nni_manager/training_service/local/localTrainingService.ts @@ -361,21 +361,25 @@ class LocalTrainingService implements TrainingService { trialJobDetail: TrialJobDetail, resource: { gpuIndices: number[] }, gpuNum: number | undefined): { key: string; value: string }[] { - const envVariables: { key: string; value: string }[] = [ - { key: 'NNI_PLATFORM', value: 'local' }, - { key: 'NNI_EXP_ID', value: this.experimentId }, - { key: 'NNI_SYS_DIR', value: trialJobDetail.workingDirectory }, - { key: 'NNI_TRIAL_JOB_ID', value: trialJobDetail.id }, - { key: 'NNI_OUTPUT_DIR', value: trialJobDetail.workingDirectory }, - { key: 'NNI_TRIAL_SEQ_ID', value: trialJobDetail.form.sequenceId.toString() }, - { key: 'MULTI_PHASE', value: this.isMultiPhase.toString() } - ]; - if (gpuNum !== undefined) { - envVariables.push({ - key: 'CUDA_VISIBLE_DEVICES', - value: this.gpuScheduler === undefined ? '-1' : resource.gpuIndices.join(',') - }); - } + if (this.localTrialConfig === undefined) { + throw new Error('localTrialConfig is not initialized!'); + } + const envVariables: { key: string; value: string }[] = [ + { key: 'NNI_PLATFORM', value: 'local' }, + { key: 'NNI_EXP_ID', value: this.experimentId }, + { key: 'NNI_SYS_DIR', value: trialJobDetail.workingDirectory }, + { key: 'NNI_TRIAL_JOB_ID', value: trialJobDetail.id }, + { key: 'NNI_OUTPUT_DIR', value: trialJobDetail.workingDirectory }, + { key: 'NNI_TRIAL_SEQ_ID', value: trialJobDetail.form.sequenceId.toString() }, + { key: 'MULTI_PHASE', value: this.isMultiPhase.toString() }, + { key: 'NNI_CODE_DIR', value: this.localTrialConfig.codeDir} + ]; + if (gpuNum !== undefined) { + envVariables.push({ + key: 'CUDA_VISIBLE_DEVICES', + value: this.gpuScheduler === undefined ? '-1' : resource.gpuIndices.join(',') + }); + } return envVariables; } @@ -473,12 +477,16 @@ class LocalTrainingService implements TrainingService { private getScript(localTrialConfig: TrialConfig, workingDirectory: string): string[] { const script: string[] = []; if (process.platform === 'win32') { + script.push(`Copy-Item $env:NNI_CODE_DIR\\* -Destination $env:NNI_SYS_DIR -Recurse`); + script.push(`cd $env:NNI_SYS_DIR`); script.push( `cmd.exe /c ${localTrialConfig.command} 2>"${path.join(workingDirectory, 'stderr')}"`, `$NOW_DATE = [int64](([datetime]::UtcNow)-(get-date "1/1/1970")).TotalSeconds`, `$NOW_DATE = "$NOW_DATE" + (Get-Date -Format fff).ToString()`, `Write $LASTEXITCODE " " $NOW_DATE | Out-File "${path.join(workingDirectory, '.nni', 'state')}" -NoNewline -encoding utf8`); } else { + script.push(`cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR`); + script.push(`cd $NNI_SYS_DIR`); script.push(`eval ${localTrialConfig.command} 2>"${path.join(workingDirectory, 'stderr')}"`); if (process.platform === 'darwin') { // https://superuser.com/questions/599072/how-to-get-bash-execution-time-in-milliseconds-under-mac-os-x @@ -506,7 +514,6 @@ class LocalTrainingService implements TrainingService { if (process.platform !== 'win32') { runScriptContent.push('#!/bin/bash'); } - runScriptContent.push(`cd '${this.localTrialConfig.codeDir}'`); for (const variable of variables) { runScriptContent.push(setEnvironmentVariable(variable)); } diff --git a/src/nni_manager/training_service/pai/paiK8S/paiK8SData.ts b/src/nni_manager/training_service/pai/paiK8S/paiK8SData.ts index a1733f99cd..f75511d826 100644 --- a/src/nni_manager/training_service/pai/paiK8S/paiK8SData.ts +++ b/src/nni_manager/training_service/pai/paiK8S/paiK8SData.ts @@ -31,7 +31,6 @@ fi`; export const PAI_K8S_TRIAL_COMMAND_FORMAT: string = `export NNI_PLATFORM=pai NNI_SYS_DIR={0} NNI_OUTPUT_DIR={1} NNI_TRIAL_JOB_ID={2} NNI_EXP_ID={3} NNI_TRIAL_SEQ_ID={4} MULTI_PHASE={5} \ -&& ls $NNI_SYS_DIR \ -&& cd $NNI_SYS_DIR && sh install_nni.sh \ -&& python3 -m nni_trial_tool.trial_keeper --trial_command '{6}' --nnimanager_ip '{7}' --nnimanager_port '{8}' \ ---nni_manager_version '{9}' --log_collection '{10}'`; +&& NNI_CODE_DIR={6} && cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR && cd $NNI_SYS_DIR && sh install_nni.sh \ +&& python3 -m nni_trial_tool.trial_keeper --trial_command '{7}' --nnimanager_ip '{8}' --nnimanager_port '{9}' \ +--nni_manager_version '{10}' --log_collection '{11}'`; diff --git a/src/nni_manager/training_service/pai/paiK8S/paiK8STrainingService.ts b/src/nni_manager/training_service/pai/paiK8S/paiK8STrainingService.ts index 48737ead35..f9be655711 100644 --- a/src/nni_manager/training_service/pai/paiK8S/paiK8STrainingService.ts +++ b/src/nni_manager/training_service/pai/paiK8S/paiK8STrainingService.ts @@ -53,6 +53,7 @@ const yaml = require('js-yaml'); @component.Singleton class PAIK8STrainingService extends PAITrainingService { protected paiTrialConfig: NNIPAIK8STrialConfig | undefined; + private copyExpCodeDirPromise?: Promise; private paiJobConfig: undefined; private nniVersion: string | undefined; constructor() { @@ -78,7 +79,7 @@ class PAIK8STrainingService extends PAITrainingService { } break; - case TrialConfigMetadataKey.TRIAL_CONFIG: + case TrialConfigMetadataKey.TRIAL_CONFIG: { if (this.paiClusterConfig === undefined) { this.log.error('pai cluster config is not initialized'); break; @@ -86,10 +87,15 @@ class PAIK8STrainingService extends PAITrainingService { this.paiTrialConfig = JSON.parse(value); // Validate to make sure codeDir doesn't have too many files await validateCodeDir(this.paiTrialConfig.codeDir); + const nniManagerNFSExpCodeDir = path.join(this.paiTrialConfig.nniManagerNFSMountPath, this.experimentId, 'nni-code'); + await execMkdir(nniManagerNFSExpCodeDir); + //Copy codeDir files to local working folder + this.copyExpCodeDirPromise = execCopydir(this.paiTrialConfig.codeDir, nniManagerNFSExpCodeDir); if (this.paiTrialConfig.paiConfigPath) { this.paiJobConfig = yaml.safeLoad(fs.readFileSync(this.paiTrialConfig.paiConfigPath, 'utf8')); } break; + } case TrialConfigMetadataKey.VERSION_CHECK: this.versionCheck = (value === 'true' || value === 'True'); this.nniVersion = this.versionCheck ? await getVersion() : ''; @@ -152,6 +158,7 @@ class PAIK8STrainingService extends PAITrainingService { if (this.paiTrialConfig === undefined) { throw new Error('trial config is not initialized'); } + const containerNFSExpCodeDir = `${this.paiTrialConfig.containerNFSMountPath}/${this.experimentId}/'nni-code`; const containerWorkingDir: string = `${this.paiTrialConfig.containerNFSMountPath}/${this.experimentId}/${trialJobDetail.id}`; const nniManagerIp: string = this.nniManagerIpConfig ? this.nniManagerIpConfig.nniManagerIp : getIPV4Address(); const nniPaiTrialCommand: string = String.Format( @@ -162,6 +169,7 @@ class PAIK8STrainingService extends PAITrainingService { this.experimentId, trialJobDetail.form.sequenceId, this.isMultiPhase, + containerNFSExpCodeDir, command, nniManagerIp, this.paiRestServerPort, @@ -264,15 +272,18 @@ class PAIK8STrainingService extends PAITrainingService { throw new Error('paiJobRestServer is not initialized'); } + // Make sure experiment code files is copied from local to NFS + if (this.copyExpCodeDirPromise !== undefined) { + await this.copyExpCodeDirPromise; + } + this.paiRestServerPort = this.paiJobRestServer.clusterRestServerPort; // Step 1. Prepare PAI job configuration //create trial local working folder locally. await execMkdir(trialJobDetail.logPath); - - const runScriptContent: string = CONTAINER_INSTALL_NNI_SHELL_FORMAT; // Write NNI installation file to local files - await fs.promises.writeFile(path.join(trialJobDetail.logPath, 'install_nni.sh'), runScriptContent, { encoding: 'utf8' }); + await fs.promises.writeFile(path.join(trialJobDetail.logPath, 'install_nni.sh'), CONTAINER_INSTALL_NNI_SHELL_FORMAT, { encoding: 'utf8' }); // Write file content ( parameter.cfg ) to local working folders if (trialJobDetail.form !== undefined) { @@ -284,7 +295,7 @@ class PAIK8STrainingService extends PAITrainingService { //Generate Job Configuration in yaml format const paiJobConfig = this.generateJobConfigInYamlFormat(trialJobDetail); this.log.debug(paiJobConfig); - // Step 3. Submit PAI job via Rest call + // Step 2. Submit PAI job via Rest call // Refer https://github.com/Microsoft/pai/blob/master/docs/rest-server/API.md for more detail about PAI Rest API const submitJobRequest: request.Options = { uri: `${this.protocol}://${this.paiClusterConfig.host}/rest-server/api/v2/jobs`, diff --git a/src/nni_manager/training_service/remote_machine/remoteMachineData.ts b/src/nni_manager/training_service/remote_machine/remoteMachineData.ts index 5f5f6455b7..2ace908132 100644 --- a/src/nni_manager/training_service/remote_machine/remoteMachineData.ts +++ b/src/nni_manager/training_service/remote_machine/remoteMachineData.ts @@ -179,13 +179,14 @@ export enum ScheduleResultType { export const REMOTEMACHINE_TRIAL_COMMAND_FORMAT: string = `#!/bin/bash export NNI_PLATFORM=remote NNI_SYS_DIR={0} NNI_OUTPUT_DIR={1} NNI_TRIAL_JOB_ID={2} NNI_EXP_ID={3} \ -NNI_TRIAL_SEQ_ID={4} export MULTI_PHASE={5} +NNI_TRIAL_SEQ_ID={4} MULTI_PHASE={5} NNI_CODE_DIR={6} +cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR cd $NNI_SYS_DIR sh install_nni.sh -echo $$ >{6} -python3 -m nni_trial_tool.trial_keeper --trial_command '{7}' --nnimanager_ip '{8}' --nnimanager_port '{9}' \ ---nni_manager_version '{10}' --log_collection '{11}' 1>$NNI_OUTPUT_DIR/trialkeeper_stdout 2>$NNI_OUTPUT_DIR/trialkeeper_stderr -echo $? \`date +%s%3N\` >{12}`; +echo $$ >{7} +python3 -m nni_trial_tool.trial_keeper --trial_command '{8}' --nnimanager_ip '{9}' --nnimanager_port '{10}' \ +--nni_manager_version '{11}' --log_collection '{12}' 1>$NNI_OUTPUT_DIR/trialkeeper_stdout 2>$NNI_OUTPUT_DIR/trialkeeper_stderr +echo $? \`date +%s%3N\` >{13}`; export const HOST_JOB_SHELL_FORMAT: string = `#!/bin/bash diff --git a/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts b/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts index 157da50d0c..5af10230f5 100644 --- a/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts +++ b/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts @@ -26,7 +26,7 @@ import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData'; import { GPUSummary } from '../common/gpuData'; import { TrialConfig } from '../common/trialConfig'; import { TrialConfigMetadataKey } from '../common/trialConfigMetadataKey'; -import { execCopydir, execMkdir, validateCodeDir, getGpuMetricsCollectorBashScriptContent } from '../common/util'; +import { execMkdir, validateCodeDir, getGpuMetricsCollectorBashScriptContent } from '../common/util'; import { GPUScheduler } from './gpuScheduler'; import { REMOTEMACHINE_TRIAL_COMMAND_FORMAT, RemoteMachineMeta, @@ -42,11 +42,13 @@ import { ShellExecutor } from 'training_service/remote_machine/shellExecutor'; @component.Singleton class RemoteMachineTrainingService implements TrainingService { private readonly machineExecutorManagerMap: Map; //machine excutor map + private readonly machineCopyExpCodeDirPromiseMap: Map>; private readonly trialExecutorMap: Map; //trial excutor map private readonly trialJobsMap: Map; private readonly MAX_TRIAL_NUMBER_PER_EXECUTOR: number = 5; // every excutor has a max trial concurrency number private readonly expRootDir: string; private readonly remoteExpRootDir: string; + private readonly remoteExpCodeDir: string; private trialConfig: TrialConfig | undefined; private gpuScheduler?: GPUScheduler; private readonly jobQueue: string[]; @@ -68,9 +70,11 @@ class RemoteMachineTrainingService implements TrainingService { this.trialJobsMap = new Map(); this.trialExecutorMap = new Map(); this.machineExecutorManagerMap = new Map(); + this.machineCopyExpCodeDirPromiseMap = new Map>(); this.jobQueue = []; this.expRootDir = getExperimentRootDir(); this.remoteExpRootDir = this.getRemoteExperimentRootDir(); + this.remoteExpCodeDir = unixPathJoin(this.remoteExpRootDir, 'nni-code'); this.timer = timer; this.log = getLogger(); this.trialSequenceId = -1; @@ -320,9 +324,20 @@ class RemoteMachineTrainingService implements TrainingService { throw new Error(`codeDir ${remoteMachineTrailConfig.codeDir} is not a directory`); } - // Validate to make sure codeDir doesn't have too many files try { + // Validate to make sure codeDir doesn't have too many files await validateCodeDir(remoteMachineTrailConfig.codeDir); + // Copy codeDir to remote machine + for (const [rmMeta, executorManager] of this.machineExecutorManagerMap.entries()) { + const executor: ShellExecutor = await executorManager.getAvailableExecutor(); + if (executor !== undefined) { + this.machineCopyExpCodeDirPromiseMap.set( + rmMeta, + executor.copyDirectoryToRemote(remoteMachineTrailConfig.codeDir, this.remoteExpCodeDir, this.remoteOS) + ); + } + } + } catch (error) { this.log.error(error); @@ -480,6 +495,10 @@ class RemoteMachineTrainingService implements TrainingService { const trialWorkingFolder: string = unixPathJoin(this.remoteExpRootDir, 'trials', trialJobId); trialJobDetail.rmMeta = rmScheduleInfo.rmMeta; + const copyExpCodeDirPromise = this.machineCopyExpCodeDirPromiseMap.get(trialJobDetail.rmMeta); + if (copyExpCodeDirPromise !== undefined) { + await copyExpCodeDirPromise; + } await this.allocateExecutorForTrial(trialJobDetail); await this.launchTrialOnScheduledMachine( @@ -554,6 +573,7 @@ class RemoteMachineTrainingService implements TrainingService { getExperimentId(), trialJobDetail.form.sequenceId.toString(), this.isMultiPhase, + this.remoteExpCodeDir, unixPathJoin(trialWorkingFolder, '.nni', 'jobpid'), command, nniManagerIp, @@ -565,12 +585,8 @@ class RemoteMachineTrainingService implements TrainingService { //create tmp trial working folder locally. await execMkdir(path.join(trialLocalTempFolder, '.nni')); - - //create tmp trial working folder locally. - await execCopydir(this.trialConfig.codeDir, trialLocalTempFolder); - const installScriptContent: string = CONTAINER_INSTALL_NNI_SHELL_FORMAT; - // Write NNI installation file to local tmp files - await fs.promises.writeFile(path.join(trialLocalTempFolder, 'install_nni.sh'), installScriptContent, { encoding: 'utf8' }); + // Write install_nni.sh + await fs.promises.writeFile(path.join(trialLocalTempFolder, 'install_nni.sh'), CONTAINER_INSTALL_NNI_SHELL_FORMAT, { encoding: 'utf8' }); // Write file content ( run.sh and parameter.cfg ) to local tmp files await fs.promises.writeFile(path.join(trialLocalTempFolder, 'run.sh'), runScriptTrialContent, { encoding: 'utf8' }); await this.writeParameterFile(trialJobId, form.hyperParameters); diff --git a/src/nni_manager/training_service/remote_machine/shellExecutor.ts b/src/nni_manager/training_service/remote_machine/shellExecutor.ts index 4b60bd963d..d35f94a239 100644 --- a/src/nni_manager/training_service/remote_machine/shellExecutor.ts +++ b/src/nni_manager/training_service/remote_machine/shellExecutor.ts @@ -183,13 +183,14 @@ class ShellExecutor { * Copy files and directories in local directory recursively to remote directory * @param localDirectory local diretory * @param remoteDirectory remote directory - * @param sshClient SSH client + * @param remoteOS the OS of remote machine */ public async copyDirectoryToRemote(localDirectory: string, remoteDirectory: string, remoteOS: string): Promise { const tmpSuffix: string = uniqueString(5); const localTarPath: string = path.join(os.tmpdir(), `nni_tmp_local_${tmpSuffix}.tar.gz`); const remoteTarPath: string = unixPathJoin(getRemoteTmpDir(remoteOS), `nni_tmp_remote_${tmpSuffix}.tar.gz`); - + // Create remote directory + await this.createFolder(remoteDirectory); // Compress files in local directory to experiment root directory await tarAdd(localTarPath, localDirectory); // Copy the compressed file to remoteDirectory and delete it diff --git a/test/nni_test/nnitest/run_tests.py b/test/nni_test/nnitest/run_tests.py index 2b8c4c2e29..96334176bf 100644 --- a/test/nni_test/nnitest/run_tests.py +++ b/test/nni_test/nnitest/run_tests.py @@ -168,6 +168,7 @@ def launch_test(config_file, training_service, test_case_config): trial_stats = get_trial_stats(TRIAL_JOBS_URL) print(json.dumps(trial_stats, indent=4), flush=True) if status != 'DONE' or trial_stats['SUCCEEDED'] + trial_stats['EARLY_STOPPED'] < max_trial_num: + print_experiment_log(experiment_id=experiment_id) print_trial_job_log(training_service, TRIAL_JOBS_URL) raise AssertionError('Failed to finish in maxExecDuration') From 063efd967a9e11aca194ba8dd75c1636296c4712 Mon Sep 17 00:00:00 2001 From: liuzhe-lz <40699903+liuzhe-lz@users.noreply.github.com> Date: Tue, 19 May 2020 16:18:26 +0800 Subject: [PATCH 09/14] Add Python SDK version string (#2418) --- Makefile | 1 + deployment/pypi/Makefile | 1 + deployment/pypi/install.ps1 | 2 ++ src/sdk/pynni/nni/__init__.py | 2 ++ 4 files changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 829c68bd3f..20696461a4 100644 --- a/Makefile +++ b/Makefile @@ -167,6 +167,7 @@ install-dependencies: $(NNI_NODE_TARBALL) $(NNI_YARN_TARBALL) .PHONY: install-python-modules install-python-modules: #$(_INFO) Installing Python SDK $(_END) + sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' src/sdk/pynni/nni/__init__.py sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' setup.py && $(PIP_INSTALL) $(PIP_MODE) . .PHONY: dev-install-python-modules diff --git a/deployment/pypi/Makefile b/deployment/pypi/Makefile index c430b9195e..a9559f7f1e 100644 --- a/deployment/pypi/Makefile +++ b/deployment/pypi/Makefile @@ -47,6 +47,7 @@ build: cp $(CWD)../../src/nni_manager/package.json $(CWD)nni sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' $(CWD)nni/package.json cd $(CWD)nni && $(NNI_YARN) --prod + sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' $(CWD)../../src/sdk/pynni/nni/__init__.py cd $(CWD) && sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' setup.py && python3 setup.py bdist_wheel -p $(WHEEL_SPEC) cd $(CWD) diff --git a/deployment/pypi/install.ps1 b/deployment/pypi/install.ps1 index d8012de094..7332a37489 100644 --- a/deployment/pypi/install.ps1 +++ b/deployment/pypi/install.ps1 @@ -60,6 +60,8 @@ Copy-Item $CWD\..\..\src\nni_manager\package.json $CWD\nni (Get-Content $CWD\nni\package.json).replace($NNI_VERSION_TEMPLATE, $NNI_VERSION_VALUE) | Set-Content $CWD\nni\package.json cd $CWD\nni yarn --prod +cd $CWD\..\..\src\sdk\pynni\nni +(Get-Content __init__.py).replace($NNI_VERSION_TEMPLATE, $NNI_VERSION_VALUE) | Set-Content __init__.py cd $CWD (Get-Content setup.py).replace($NNI_VERSION_TEMPLATE, $NNI_VERSION_VALUE) | Set-Content setup.py python setup.py bdist_wheel -p $WHEEL_SPEC diff --git a/src/sdk/pynni/nni/__init__.py b/src/sdk/pynni/nni/__init__.py index a8cd78bbf7..c7236adc1c 100644 --- a/src/sdk/pynni/nni/__init__.py +++ b/src/sdk/pynni/nni/__init__.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +__version__ = '999.0.0-developing' + from .env_vars import dispatcher_env_vars if dispatcher_env_vars.SDK_PROCESS != 'dispatcher': From e83a6ea7108e45ce29d000b0122fcabcabe207ff Mon Sep 17 00:00:00 2001 From: Chi Song <27178119+squirrelsc@users.noreply.github.com> Date: Tue, 19 May 2020 17:43:26 +0800 Subject: [PATCH 10/14] rename remote IT for clear (#2456) --- ...es-it-remote.yml => pipelines-it-remote-linux-to-linux.yml} | 2 +- ...te-windows.yml => pipelines-it-remote-windows-to-linux.yml} | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) rename test/pipelines/{pipelines-it-remote.yml => pipelines-it-remote-linux-to-linux.yml} (98%) rename test/pipelines/{pipelines-it-remote-windows.yml => pipelines-it-remote-windows-to-linux.yml} (96%) diff --git a/test/pipelines/pipelines-it-remote.yml b/test/pipelines/pipelines-it-remote-linux-to-linux.yml similarity index 98% rename from test/pipelines/pipelines-it-remote.yml rename to test/pipelines/pipelines-it-remote-linux-to-linux.yml index 4eab1cf650..78b3c9edaa 100644 --- a/test/pipelines/pipelines-it-remote.yml +++ b/test/pipelines/pipelines-it-remote-linux-to-linux.yml @@ -1,5 +1,5 @@ jobs: -- job: 'integration_test_remote' +- job: 'integration_test_remote_linux_to_linux' timeoutInMinutes: 120 steps: diff --git a/test/pipelines/pipelines-it-remote-windows.yml b/test/pipelines/pipelines-it-remote-windows-to-linux.yml similarity index 96% rename from test/pipelines/pipelines-it-remote-windows.yml rename to test/pipelines/pipelines-it-remote-windows-to-linux.yml index 63bcfb5c41..36a98a9819 100644 --- a/test/pipelines/pipelines-it-remote-windows.yml +++ b/test/pipelines/pipelines-it-remote-windows-to-linux.yml @@ -1,5 +1,5 @@ jobs: -- job: 'integration_test_remote_windows' +- job: 'integration_test_remote_windows_to_linux' timeoutInMinutes: 120 steps: @@ -23,6 +23,7 @@ jobs: sshEndpoint: $(end_point) runOptions: inline inline: cd /tmp/nnitest/$(Build.BuildId)/nni-remote/deployment/pypi;make build + failOnStdErr: false continueOnError: true displayName: 'build nni bdsit_wheel' - task: SSH@0 From 76c819c00acb2952436da611c574030aa0beedd4 Mon Sep 17 00:00:00 2001 From: liuzhe-lz <40699903+liuzhe-lz@users.noreply.github.com> Date: Tue, 19 May 2020 19:33:13 +0800 Subject: [PATCH 11/14] Merge dev-nas-tf to master (#2459) --- examples/nas/enas-tf/datasets.py | 12 ++ examples/nas/enas-tf/macro.py | 142 ++++++++++++++ examples/nas/enas-tf/micro.py | 176 ++++++++++++++++++ examples/nas/enas-tf/search.py | 35 ++++ examples/nas/enas-tf/utils.py | 19 ++ examples/nas/naive-tf/train.py | 87 +++++++++ pylintrc | 4 +- src/sdk/__init__.py | 0 src/sdk/pynni/__init__.py | 0 .../pynni/nni/nas/tensorflow/base_mutator.py | 73 ++++++++ .../pynni/nni/nas/tensorflow/enas/__init__.py | 5 + .../pynni/nni/nas/tensorflow/enas/mutator.py | 160 ++++++++++++++++ .../pynni/nni/nas/tensorflow/enas/trainer.py | 159 ++++++++++++++++ src/sdk/pynni/nni/nas/tensorflow/mutables.py | 136 ++++++++++++++ src/sdk/pynni/nni/nas/tensorflow/mutator.py | 77 ++++++++ src/sdk/pynni/nni/nas/tensorflow/utils.py | 93 +++++++++ 16 files changed, 1177 insertions(+), 1 deletion(-) create mode 100644 examples/nas/enas-tf/datasets.py create mode 100644 examples/nas/enas-tf/macro.py create mode 100644 examples/nas/enas-tf/micro.py create mode 100644 examples/nas/enas-tf/search.py create mode 100644 examples/nas/enas-tf/utils.py create mode 100644 examples/nas/naive-tf/train.py delete mode 100644 src/sdk/__init__.py delete mode 100644 src/sdk/pynni/__init__.py create mode 100644 src/sdk/pynni/nni/nas/tensorflow/base_mutator.py create mode 100644 src/sdk/pynni/nni/nas/tensorflow/enas/__init__.py create mode 100644 src/sdk/pynni/nni/nas/tensorflow/enas/mutator.py create mode 100644 src/sdk/pynni/nni/nas/tensorflow/enas/trainer.py create mode 100644 src/sdk/pynni/nni/nas/tensorflow/mutables.py create mode 100644 src/sdk/pynni/nni/nas/tensorflow/mutator.py create mode 100644 src/sdk/pynni/nni/nas/tensorflow/utils.py diff --git a/examples/nas/enas-tf/datasets.py b/examples/nas/enas-tf/datasets.py new file mode 100644 index 0000000000..2c5e44902b --- /dev/null +++ b/examples/nas/enas-tf/datasets.py @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import tensorflow as tf +from tensorflow.data import Dataset + +def get_dataset(): + (x_train, y_train), (x_valid, y_valid) = tf.keras.datasets.cifar10.load_data() + x_train, x_valid = x_train / 255.0, x_valid / 255.0 + train_set = (x_train, y_train) + valid_set = (x_valid, y_valid) + return train_set, valid_set diff --git a/examples/nas/enas-tf/macro.py b/examples/nas/enas-tf/macro.py new file mode 100644 index 0000000000..f0d73c2e69 --- /dev/null +++ b/examples/nas/enas-tf/macro.py @@ -0,0 +1,142 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import tensorflow as tf +from tensorflow.keras import Model, Sequential +from tensorflow.keras.layers import ( + AveragePooling2D, + BatchNormalization, + Conv2D, + Dense, + Dropout, + GlobalAveragePooling2D, + MaxPool2D, + ReLU, + SeparableConv2D, +) + +from nni.nas.tensorflow.mutables import InputChoice, LayerChoice, MutableScope + + +def build_conv(filters, kernel_size, name=None): + return Sequential([ + Conv2D(filters, kernel_size=1, use_bias=False), + BatchNormalization(trainable=False), + ReLU(), + Conv2D(filters, kernel_size, padding='same'), + BatchNormalization(trainable=False), + ReLU(), + ], name) + +def build_separable_conv(filters, kernel_size, name=None): + return Sequential([ + Conv2D(filters, kernel_size=1, use_bias=False), + BatchNormalization(trainable=False), + ReLU(), + SeparableConv2D(filters, kernel_size, padding='same', use_bias=False), + Conv2D(filters, kernel_size=1, use_bias=False), + BatchNormalization(trainable=False), + ReLU(), + ], name) + +def build_avg_pool(filters, name=None): + return Sequential([ + Conv2D(filters, kernel_size=1, use_bias=False), + BatchNormalization(trainable=False), + ReLU(), + AveragePooling2D(pool_size=3, strides=1, padding='same'), + BatchNormalization(trainable=False), + ], name) + +def build_max_pool(filters, name=None): + return Sequential([ + Conv2D(filters, kernel_size=1, use_bias=False), + BatchNormalization(trainable=False), + ReLU(), + MaxPool2D(pool_size=3, strides=1, padding='same'), + BatchNormalization(trainable=False), + ], name) + + +class FactorizedReduce(Model): + def __init__(self, filters): + super().__init__() + self.conv1 = Conv2D(filters // 2, kernel_size=1, strides=2, use_bias=False) + self.conv2 = Conv2D(filters // 2, kernel_size=1, strides=2, use_bias=False) + self.bn = BatchNormalization(trainable=False) + + def call(self, x): + out1 = self.conv1(x) + out2 = self.conv2(x[:, 1:, 1:, :]) + out = tf.concat([out1, out2], axis=3) + out = self.bn(out) + return out + + +class ENASLayer(MutableScope): + def __init__(self, key, prev_labels, filters): + super().__init__(key) + self.mutable = LayerChoice([ + build_conv(filters, 3, 'conv3'), + build_separable_conv(filters, 3, 'sepconv3'), + build_conv(filters, 5, 'conv5'), + build_separable_conv(filters, 5, 'sepconv5'), + build_avg_pool(filters, 'avgpool'), + build_max_pool(filters, 'maxpool'), + ]) + if len(prev_labels) > 0: + self.skipconnect = InputChoice(choose_from=prev_labels, n_chosen=None) + else: + self.skipconnect = None + self.batch_norm = BatchNormalization(trainable=False) + + def call(self, prev_layers): + out = self.mutable(prev_layers[-1]) + if self.skipconnect is not None: + connection = self.skipconnect(prev_layers[:-1]) + if connection is not None: + out += connection + return self.batch_norm(out) + + +class GeneralNetwork(Model): + def __init__(self, num_layers=12, filters=24, num_classes=10, dropout_rate=0.0): + super().__init__() + self.num_layers = num_layers + + self.stem = Sequential([ + Conv2D(filters, kernel_size=3, padding='same', use_bias=False), + BatchNormalization() + ]) + + labels = ['layer_{}'.format(i) for i in range(num_layers)] + self.enas_layers = [] + for i in range(num_layers): + layer = ENASLayer(labels[i], labels[:i], filters) + self.enas_layers.append(layer) + + pool_num = 2 + self.pool_distance = num_layers // (pool_num + 1) + self.pool_layers = [FactorizedReduce(filters) for _ in range(pool_num)] + + self.gap = GlobalAveragePooling2D() + self.dropout = Dropout(dropout_rate) + self.dense = Dense(num_classes) + + def call(self, x): + cur = self.stem(x) + prev_outputs = [cur] + + for i, layer in enumerate(self.enas_layers): + if i > 0 and i % self.pool_distance == 0: + pool = self.pool_layers[i // self.pool_distance - 1] + prev_outputs = [pool(tensor) for tensor in prev_outputs] + cur = prev_outputs[-1] + + cur = layer(prev_outputs) + prev_outputs.append(cur) + + cur = self.gap(cur) + cur = self.dropout(cur) + logits = self.dense(cur) + return logits diff --git a/examples/nas/enas-tf/micro.py b/examples/nas/enas-tf/micro.py new file mode 100644 index 0000000000..8c52f4b441 --- /dev/null +++ b/examples/nas/enas-tf/micro.py @@ -0,0 +1,176 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import tensorflow as tf +from tensorflow.keras import Model, Sequential +from tensorflow.keras.layers import ( + AveragePooling2D, + BatchNormalization, + Conv2D, + Dense, + Dropout, + GlobalAveragePooling2D, + MaxPool2D, + ReLU, + SeparableConv2D, +) + +from nni.nas.tensorflow.mutables import InputChoice, LayerChoice, MutableScope + + +def build_conv_1x1(filters, name=None): + return Sequential([ + Conv2D(filters, kernel_size=1, use_bias=False), + BatchNormalization(trainable=False), + ReLU(), + ], name) + +def build_sep_conv(filters, kernel_size, name=None): + return Sequential([ + ReLU(), + SeparableConv2D(filters, kernel_size, padding='same'), + BatchNormalization(trainable=True), + ], name) + + +class FactorizedReduce(Model): + def __init__(self, filters): + super().__init__() + self.conv1 = Conv2D(filters // 2, kernel_size=1, strides=2, use_bias=False) + self.conv2 = Conv2D(filters // 2, kernel_size=1, strides=2, use_bias=False) + self.bn = BatchNormalization(trainable=False) + + def call(self, x): + out1 = self.conv1(x) + out2 = self.conv2(x[:, 1:, 1:, :]) + out = tf.concat([out1, out2], axis=3) + out = self.bn(out) + return out + + +class ReductionLayer(Model): + def __init__(self, filters): + super().__init__() + self.reduce0 = FactorizedReduce(filters) + self.reduce1 = FactorizedReduce(filters) + + def call(self, prevprev, prev): + return self.reduce0(prevprev), self.reduce1(prev) + + +class Calibration(Model): + def __init__(self, filters): + super().__init__() + self.filters = filters + self.process = None + + def build(self, shape): + assert len(shape) == 4 # batch_size, width, height, filters + if shape[3] != self.filters: + self.process = build_conv_1x1(self.filters) + + def call(self, x): + if self.process is None: + return x + return self.process(x) + + +class Cell(Model): + def __init__(self, cell_name, prev_labels, filters): + super().__init__() + self.input_choice = InputChoice(choose_from=prev_labels, n_chosen=1, return_mask=True, key=cell_name + '_input') + self.op_choice = LayerChoice([ + build_sep_conv(filters, 3), + build_sep_conv(filters, 5), + AveragePooling2D(pool_size=3, strides=1, padding='same'), + MaxPool2D(pool_size=3, strides=1, padding='same'), + Sequential(), # Identity + ], key=cell_name + '_op') + + def call(self, prev_layers): + chosen_input, chosen_mask = self.input_choice(prev_layers) + cell_out = self.op_choice(chosen_input) + return cell_out, chosen_mask + + +class Node(MutableScope): + def __init__(self, node_name, prev_node_names, filters): + super().__init__(node_name) + self.cell_x = Cell(node_name + '_x', prev_node_names, filters) + self.cell_y = Cell(node_name + '_y', prev_node_names, filters) + + def call(self, prev_layers): + out_x, mask_x = self.cell_x(prev_layers) + out_y, mask_y = self.cell_y(prev_layers) + return out_x + out_y, mask_x | mask_y + + +class ENASLayer(Model): + def __init__(self, num_nodes, filters, reduction): + super().__init__() + self.preproc0 = Calibration(filters) + self.preproc1 = Calibration(filters) + + self.nodes = [] + node_labels = [InputChoice.NO_KEY, InputChoice.NO_KEY] + name_prefix = 'reduce' if reduction else 'normal' + for i in range(num_nodes): + node_labels.append('{}_node_{}'.format(name_prefix, i)) + self.nodes.append(Node(node_labels[-1], node_labels[:-1], filters)) + + self.conv_ops = [Conv2D(filters, kernel_size=1, padding='same', use_bias=False) for _ in range(num_nodes + 2)] + self.bn = BatchNormalization(trainable=False) + + def call(self, prevprev, prev): + prev_nodes_out = [self.preproc0(prevprev), self.preproc1(prev)] + nodes_used_mask = tf.zeros(len(self.nodes) + 2, dtype=tf.bool) + for i, node in enumerate(self.nodes): + node_out, mask = node(prev_nodes_out) + nodes_used_mask |= tf.pad(mask, [[0, nodes_used_mask.shape[0] - mask.shape[0]]]) + prev_nodes_out.append(node_out) + + outputs = [] + for used, out, conv in zip(nodes_used_mask.numpy(), prev_nodes_out, self.conv_ops): + if not used: + outputs.append(conv(out)) + out = tf.add_n(outputs) + return prev, self.bn(out) + + +class MicroNetwork(Model): + def __init__(self, num_layers=6, num_nodes=5, out_channels=20, num_classes=10, dropout_rate=0.1): + super().__init__() + self.num_layers = num_layers + self.stem = Sequential([ + Conv2D(out_channels * 3, kernel_size=3, padding='same', use_bias=False), + BatchNormalization(), + ]) + + pool_distance = num_layers // 3 + pool_layer_indices = [pool_distance, 2 * pool_distance + 1] + + self.enas_layers = [] + + filters = out_channels + for i in range(num_layers + 2): + if i in pool_layer_indices: + reduction = True + filters *= 2 + self.enas_layers.append(ReductionLayer(filters)) + else: + reduction = False + self.enas_layers.append(ENASLayer(num_nodes, filters, reduction)) + + self.gap = GlobalAveragePooling2D() + self.dropout = Dropout(dropout_rate) + self.dense = Dense(num_classes) + + def call(self, x): + prev = cur = self.stem(x) + for layer in self.enas_layers: + prev, cur = layer(prev, cur) + cur = tf.keras.activations.relu(cur) + cur = self.gap(cur) + cur = self.dropout(cur) + logits = self.dense(cur) + return logits diff --git a/examples/nas/enas-tf/search.py b/examples/nas/enas-tf/search.py new file mode 100644 index 0000000000..b68daf62f3 --- /dev/null +++ b/examples/nas/enas-tf/search.py @@ -0,0 +1,35 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +from tensorflow.keras.losses import Reduction, SparseCategoricalCrossentropy +from tensorflow.keras.optimizers import SGD + +from nni.nas.tensorflow import enas + +import datasets +from macro import GeneralNetwork +from micro import MicroNetwork +from utils import accuracy, accuracy_metrics + + +# TODO: argparse + + +dataset_train, dataset_valid = datasets.get_dataset() +#model = GeneralNetwork() +model = MicroNetwork() + +loss = SparseCategoricalCrossentropy(from_logits=True, reduction=Reduction.NONE) +optimizer = SGD(learning_rate=0.05, momentum=0.9) + +trainer = enas.EnasTrainer(model, + loss=loss, + metrics=accuracy_metrics, + reward_function=accuracy, + optimizer=optimizer, + batch_size=64, + num_epochs=310, + dataset_train=dataset_train, + dataset_valid=dataset_valid) +trainer.train() diff --git a/examples/nas/enas-tf/utils.py b/examples/nas/enas-tf/utils.py new file mode 100644 index 0000000000..dc924a96f3 --- /dev/null +++ b/examples/nas/enas-tf/utils.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import tensorflow as tf + + +def accuracy_metrics(y_true, logits): + return {'enas_acc': accuracy(y_true, logits)} + +def accuracy(y_true, logits): + # y_true: shape=(batch_size) or (batch_size,1), type=integer + # logits: shape=(batch_size, num_of_classes), type=float + # returns float + batch_size = y_true.shape[0] + y_true = tf.squeeze(y_true) + y_pred = tf.math.argmax(logits, axis=1) + y_pred = tf.cast(y_pred, y_true.dtype) + equal = tf.cast(y_pred == y_true, tf.int32) + return tf.math.reduce_sum(equal).numpy() / batch_size diff --git a/examples/nas/naive-tf/train.py b/examples/nas/naive-tf/train.py new file mode 100644 index 0000000000..a4e56a39f8 --- /dev/null +++ b/examples/nas/naive-tf/train.py @@ -0,0 +1,87 @@ +import tensorflow as tf +from tensorflow.keras import Model +from tensorflow.keras.layers import (AveragePooling2D, BatchNormalization, Conv2D, Dense, MaxPool2D) +from tensorflow.keras.losses import Reduction, SparseCategoricalCrossentropy +from tensorflow.keras.optimizers import SGD + +from nni.nas.tensorflow.mutables import LayerChoice, InputChoice +from nni.nas.tensorflow.enas import EnasTrainer + +tf.get_logger().setLevel('ERROR') + + +class Net(Model): + def __init__(self): + super().__init__() + self.conv1 = LayerChoice([ + Conv2D(6, 3, padding='same', activation='relu'), + Conv2D(6, 5, padding='same', activation='relu'), + ]) + self.pool = MaxPool2D(2) + self.conv2 = LayerChoice([ + Conv2D(16, 3, padding='same', activation='relu'), + Conv2D(16, 5, padding='same', activation='relu'), + ]) + self.conv3 = Conv2D(16, 1) + + self.skipconnect = InputChoice(n_candidates=1) + self.bn = BatchNormalization() + + self.gap = AveragePooling2D(2) + self.fc1 = Dense(120, activation='relu') + self.fc2 = Dense(84, activation='relu') + self.fc3 = Dense(10) + + def call(self, x): + bs = x.shape[0] + + t = self.conv1(x) + x = self.pool(t) + x0 = self.conv2(x) + x1 = self.conv3(x0) + + x0 = self.skipconnect([x0]) + if x0 is not None: + x1 += x0 + x = self.pool(self.bn(x1)) + + x = self.gap(x) + x = tf.reshape(x, [bs, -1]) + x = self.fc1(x) + x = self.fc2(x) + x = self.fc3(x) + return x + + +def accuracy(output, target): + bs = target.shape[0] + predicted = tf.cast(tf.argmax(output, 1), target.dtype) + target = tf.reshape(target, [-1]) + return sum(tf.cast(predicted == target, tf.float32)) / bs + + +if __name__ == '__main__': + cifar10 = tf.keras.datasets.cifar10 + (x_train, y_train), (x_test, y_test) = cifar10.load_data() + x_train, x_test = x_train / 255.0, x_test / 255.0 + split = int(len(x_train) * 0.9) + dataset_train = tf.data.Dataset.from_tensor_slices((x_train[:split], y_train[:split])).batch(64) + dataset_valid = tf.data.Dataset.from_tensor_slices((x_train[split:], y_train[split:])).batch(64) + dataset_test = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(64) + + net = Net() + trainer = EnasTrainer( + net, + loss=SparseCategoricalCrossentropy(reduction=Reduction.SUM), + metrics=accuracy, + reward_function=accuracy, + optimizer=SGD(learning_rate=0.001, momentum=0.9), + batch_size=64, + num_epochs=2, + dataset_train=dataset_train, + dataset_valid=dataset_valid, + dataset_test=dataset_test + ) + + trainer.train() + #trainer.export('checkpoint') diff --git a/pylintrc b/pylintrc index a5a924ee7c..e23cacfb12 100644 --- a/pylintrc +++ b/pylintrc @@ -45,4 +45,6 @@ enable= unused-wildcard-import, ignore-patterns=test* # List of members which are set dynamically and missed by pylint inference -generated-members=numpy.*,torch.* +generated-members=numpy.*,torch.*,tensorflow.* + +ignored-modules=tensorflow diff --git a/src/sdk/__init__.py b/src/sdk/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sdk/pynni/__init__.py b/src/sdk/pynni/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sdk/pynni/nni/nas/tensorflow/base_mutator.py b/src/sdk/pynni/nni/nas/tensorflow/base_mutator.py new file mode 100644 index 0000000000..860680f199 --- /dev/null +++ b/src/sdk/pynni/nni/nas/tensorflow/base_mutator.py @@ -0,0 +1,73 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from tensorflow.keras import Model + +from .mutables import Mutable, MutableScope, InputChoice +from .utils import StructuredMutableTreeNode + + +class BaseMutator(Model): + def __init__(self, model): + super().__init__() + self.__dict__['model'] = model + self._structured_mutables = self._parse_search_space(self.model) + + def _parse_search_space(self, module, root=None, prefix='', memo=None, nested_detection=None): + if memo is None: + memo = set() + if root is None: + root = StructuredMutableTreeNode(None) + if module not in memo: + memo.add(module) + if isinstance(module, Mutable): + if nested_detection is not None: + raise RuntimeError('Cannot have nested search space. Error at {} in {}' + .format(module, nested_detection)) + module.name = prefix + module.set_mutator(self) + root = root.add_child(module) + if not isinstance(module, MutableScope): + nested_detection = module + if isinstance(module, InputChoice): + for k in module.choose_from: + if k != InputChoice.NO_KEY and k not in [m.key for m in memo if isinstance(m, Mutable)]: + raise RuntimeError('"{}" required by "{}" not found in keys that appeared before, and is not NO_KEY.' + .format(k, module.key)) + for submodule in module.layers: + if not isinstance(submodule, Model): + continue + submodule_prefix = prefix + ('.' if prefix else '') + submodule.name + self._parse_search_space(submodule, root, submodule_prefix, memo=memo, nested_detection=nested_detection) + return root + + @property + def mutables(self): + return self._structured_mutables + + def undedup_mutables(self): + return self._structured_mutables.traverse(deduplicate=False) + + def call(self, *inputs): + raise RuntimeError('Call is undefined for mutators.') + + def __setattr__(self, name, value): + if name == 'model': + raise AttributeError("Attribute `model` can be set at most once, and you shouldn't use `self.model = model` to " + "include your network, as it will include all parameters in model into the mutator.") + return super().__setattr__(name, value) + + def enter_mutable_scope(self, mutable_scope): + pass + + def exit_mutable_scope(self, mutable_scope): + pass + + def on_forward_layer_choice(self, mutable, *inputs): + raise NotImplementedError + + def on_forward_input_choice(self, mutable, tensor_list): + raise NotImplementedError + + def export(self): + raise NotImplementedError diff --git a/src/sdk/pynni/nni/nas/tensorflow/enas/__init__.py b/src/sdk/pynni/nni/nas/tensorflow/enas/__init__.py new file mode 100644 index 0000000000..d3372836eb --- /dev/null +++ b/src/sdk/pynni/nni/nas/tensorflow/enas/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import EnasMutator +from .trainer import EnasTrainer diff --git a/src/sdk/pynni/nni/nas/tensorflow/enas/mutator.py b/src/sdk/pynni/nni/nas/tensorflow/enas/mutator.py new file mode 100644 index 0000000000..de43195fa2 --- /dev/null +++ b/src/sdk/pynni/nni/nas/tensorflow/enas/mutator.py @@ -0,0 +1,160 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import tensorflow as tf +from tensorflow.keras.layers import Dense, Embedding, LSTMCell, RNN +from tensorflow.keras.losses import SparseCategoricalCrossentropy, Reduction + +from nni.nas.tensorflow.mutator import Mutator +from nni.nas.tensorflow.mutables import LayerChoice, InputChoice, MutableScope + + +class EnasMutator(Mutator): + def __init__(self, model, + lstm_size=64, + lstm_num_layers=1, + tanh_constant=1.5, + cell_exit_extra_step=False, + skip_target=0.4, + temperature=None, + branch_bias=0.25, + entropy_reduction='sum'): + super().__init__(model) + self.tanh_constant = tanh_constant + self.temperature = temperature + self.cell_exit_extra_step = cell_exit_extra_step + + cells = [LSTMCell(units=lstm_size, use_bias=False) for _ in range(lstm_num_layers)] + self.lstm = RNN(cells, stateful=True) + self.g_emb = tf.random.normal((1, 1, lstm_size)) * 0.1 + self.skip_targets = tf.constant([1.0 - skip_target, skip_target]) + + self.max_layer_choice = 0 + self.bias_dict = {} + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + if self.max_layer_choice == 0: + self.max_layer_choice = len(mutable) + assert self.max_layer_choice == len(mutable), \ + "ENAS mutator requires all layer choice have the same number of candidates." + if 'reduce' in mutable.key: + bias = [] + for choice in mutable.choices: + if 'conv' in str(type(choice)).lower(): + bias.append(branch_bias) + else: + bias.append(-branch_bias) + self.bias_dict[mutable.key] = tf.constant(bias) + + # exposed for trainer + self.sample_log_prob = 0 + self.sample_entropy = 0 + self.sample_skip_penalty = 0 + + # internal nn layers + self.embedding = Embedding(self.max_layer_choice + 1, lstm_size) + self.soft = Dense(self.max_layer_choice, use_bias=False) + self.attn_anchor = Dense(lstm_size, use_bias=False) + self.attn_query = Dense(lstm_size, use_bias=False) + self.v_attn = Dense(1, use_bias=False) + assert entropy_reduction in ['sum', 'mean'], 'Entropy reduction must be one of sum and mean.' + self.entropy_reduction = tf.reduce_sum if entropy_reduction == 'sum' else tf.reduce_mean + self.cross_entropy_loss = SparseCategoricalCrossentropy(from_logits=True, reduction=Reduction.NONE) + + self._first_sample = True + + def sample_search(self): + self._initialize() + self._sample(self.mutables) + self._first_sample = False + return self._choices + + def sample_final(self): + return self.sample_search() + + def _sample(self, tree): + mutable = tree.mutable + if isinstance(mutable, LayerChoice) and mutable.key not in self._choices: + self._choices[mutable.key] = self._sample_layer_choice(mutable) + elif isinstance(mutable, InputChoice) and mutable.key not in self._choices: + self._choices[mutable.key] = self._sample_input_choice(mutable) + for child in tree.children: + self._sample(child) + if self.cell_exit_extra_step and isinstance(mutable, MutableScope) and mutable.key not in self._anchors_hid: + self._anchors_hid[mutable.key] = self.lstm(self._inputs, 1) + + def _initialize(self): + self._choices = {} + self._anchors_hid = {} + self._inputs = self.g_emb + # seems the `input_shape` parameter of RNN does not work + # workaround it by omitting `reset_states` for first run + if not self._first_sample: + self.lstm.reset_states() + self.sample_log_prob = 0 + self.sample_entropy = 0 + self.sample_skip_penalty = 0 + + def _sample_layer_choice(self, mutable): + logit = self.soft(self.lstm(self._inputs)) + if self.temperature is not None: + logit /= self.temperature + if self.tanh_constant is not None: + logit = self.tanh_constant * tf.tanh(logit) + if mutable.key in self.bias_dict: + logit += self.bias_dict[mutable.key] + softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1)) + branch_id = tf.reshape(tf.random.categorical(softmax_logit, num_samples=1), [1]) + log_prob = self.cross_entropy_loss(branch_id, logit) + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = log_prob * tf.math.exp(-log_prob) + self.sample_entropy += self.entropy_reduction(entropy) + self._inputs = tf.reshape(self.embedding(branch_id), [1, 1, -1]) + mask = tf.one_hot(branch_id, self.max_layer_choice) + return tf.cast(tf.reshape(mask, [-1]), tf.bool) + + def _sample_input_choice(self, mutable): + query, anchors = [], [] + for label in mutable.choose_from: + if label not in self._anchors_hid: + self._anchors_hid[label] = self.lstm(self._inputs) + query.append(self.attn_anchor(self._anchors_hid[label])) + anchors.append(self._anchors_hid[label]) + query = tf.concat(query, axis=0) + query = tf.tanh(query + self.attn_query(anchors[-1])) + query = self.v_attn(query) + + if self.temperature is not None: + query /= self.temperature + if self.tanh_constant is not None: + query = self.tanh_constant * tf.tanh(query) + + if mutable.n_chosen is None: + logit = tf.concat([-query, query], axis=1) + softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1)) + skip = tf.reshape(tf.random.categorical(softmax_logit, num_samples=1), [-1]) + skip_prob = tf.math.sigmoid(logit) + kl = tf.reduce_sum(skip_prob * tf.math.log(skip_prob / self.skip_targets)) + self.sample_skip_penalty += kl + log_prob = self.cross_entropy_loss(skip, logit) + + skip = tf.cast(skip, tf.float32) + inputs = tf.tensordot(skip, tf.concat(anchors, 0), 1) / (1. + tf.reduce_sum(skip)) + self._inputs = tf.reshape(inputs, [1, 1, -1]) + + else: + assert mutable.n_chosen == 1, "Input choice must select exactly one or any in ENAS." + logit = tf.reshape(query, [1, -1]) + softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1)) + index = tf.reshape(tf.random.categorical(softmax_logit, num_samples=1), [-1]) + skip = tf.reshape(tf.one_hot(index, mutable.n_candidates), [-1]) + # when the size is 1, tf does not accept tensor here, complaining the shape is wrong + # but using a numpy array seems fine + log_prob = self.cross_entropy_loss(logit, query.numpy()) + self._inputs = tf.reshape(anchors[index.numpy()[0]], [1, 1, -1]) + + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = log_prob * tf.exp(-log_prob) + self.sample_entropy += self.entropy_reduction(entropy) + assert len(skip) == mutable.n_candidates, (skip, mutable.n_candidates, mutable.n_chosen) + return tf.cast(skip, tf.bool) diff --git a/src/sdk/pynni/nni/nas/tensorflow/enas/trainer.py b/src/sdk/pynni/nni/nas/tensorflow/enas/trainer.py new file mode 100644 index 0000000000..2d0d3cdb5a --- /dev/null +++ b/src/sdk/pynni/nni/nas/tensorflow/enas/trainer.py @@ -0,0 +1,159 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import tensorflow as tf +from tensorflow.data import Dataset +from tensorflow.keras.optimizers import Adam + +from nni.nas.tensorflow.utils import AverageMeterGroup, fill_zero_grads + +from .mutator import EnasMutator + +logger = logging.getLogger(__name__) + + +log_frequency = 100 +entropy_weight = 0.0001 +skip_weight = 0.8 +baseline_decay = 0.999 +child_steps = 500 +mutator_lr = 0.00035 +mutator_steps = 50 +mutator_steps_aggregate = 20 +aux_weight = 0.4 +test_arc_per_epoch = 1 + + +class EnasTrainer: + def __init__(self, model, loss, metrics, reward_function, optimizer, batch_size, num_epochs, + dataset_train, dataset_valid): + self.model = model + self.loss = loss + self.metrics = metrics + self.reward_function = reward_function + self.optimizer = optimizer + self.batch_size = batch_size + self.num_epochs = num_epochs + + x, y = dataset_train + split = int(len(x) * 0.9) + self.train_set = Dataset.from_tensor_slices((x[:split], y[:split])) + self.valid_set = Dataset.from_tensor_slices((x[split:], y[split:])) + self.test_set = Dataset.from_tensor_slices(dataset_valid) + + self.mutator = EnasMutator(model) + self.mutator_optim = Adam(learning_rate=mutator_lr) + + self.baseline = 0. + + + def train(self, validate=True): + for epoch in range(self.num_epochs): + logger.info("Epoch %d Training", epoch + 1) + self.train_one_epoch(epoch) + logger.info("Epoch %d Validating", epoch + 1) + self.validate_one_epoch(epoch) + + def validate(self): + self.validate_one_epoch(-1) + + + def train_one_epoch(self, epoch): + train_loader, valid_loader = self._create_train_loader() + + # Sample model and train + meters = AverageMeterGroup() + + for step in range(1, child_steps + 1): + x, y = next(train_loader) + self.mutator.reset() + + with tf.GradientTape() as tape: + logits = self.model(x, training=True) + if isinstance(logits, tuple): + logits, aux_logits = logits + aux_loss = self.loss(aux_logits, y) + else: + aux_loss = 0. + metrics = self.metrics(y, logits) + loss = self.loss(y, logits) + aux_weight * aux_loss + + grads = tape.gradient(loss, self.model.trainable_weights) + grads = fill_zero_grads(grads, self.model.trainable_weights) + grads, _ = tf.clip_by_global_norm(grads, 5.0) + self.optimizer.apply_gradients(zip(grads, self.model.trainable_weights)) + + metrics['loss'] = tf.reduce_mean(loss).numpy() + meters.update(metrics) + + if log_frequency and step % log_frequency == 0: + logger.info("Model Epoch [%d/%d] Step [%d/%d] %s", epoch + 1, + self.num_epochs, step, child_steps, meters) + + # Train sampler (mutator) + meters = AverageMeterGroup() + for mutator_step in range(1, mutator_steps + 1): + grads_list = [] + for step in range(1, mutator_steps_aggregate + 1): + with tf.GradientTape() as tape: + x, y = next(valid_loader) + self.mutator.reset() + + logits = self.model(x, training=False) + metrics = self.metrics(y, logits) + reward = self.reward_function(y, logits) + entropy_weight * self.mutator.sample_entropy + self.baseline = self.baseline * baseline_decay + reward * (1 - baseline_decay) + loss = self.mutator.sample_log_prob * (reward - self.baseline) + loss += skip_weight * self.mutator.sample_skip_penalty + + meters.update({ + 'reward': reward, + 'loss': tf.reduce_mean(loss).numpy(), + 'ent': self.mutator.sample_entropy.numpy(), + 'log_prob': self.mutator.sample_log_prob.numpy(), + 'baseline': self.baseline, + 'skip': self.mutator.sample_skip_penalty, + }) + + cur_step = step + (mutator_step - 1) * mutator_steps_aggregate + if log_frequency and cur_step % log_frequency == 0: + logger.info("RL Epoch [%d/%d] Step [%d/%d] [%d/%d] %s", epoch + 1, self.num_epochs, + mutator_step, mutator_steps, step, mutator_steps_aggregate, + meters) + + grads = tape.gradient(loss, self.mutator.trainable_weights) + grads = fill_zero_grads(grads, self.mutator.trainable_weights) + grads_list.append(grads) + total_grads = [tf.math.add_n(weight_grads) for weight_grads in zip(*grads_list)] + total_grads, _ = tf.clip_by_global_norm(total_grads, 5.0) + self.mutator_optim.apply_gradients(zip(total_grads, self.mutator.trainable_weights)) + + def validate_one_epoch(self, epoch): + test_loader = self._create_validate_loader() + + for arc_id in range(test_arc_per_epoch): + meters = AverageMeterGroup() + for x, y in test_loader: + self.mutator.reset() + logits = self.model(x) + if isinstance(logits, tuple): + logits, _ = logits + metrics = self.metrics(logits, y) + loss = self.loss(y, logits) + metrics['loss'] = tf.reduce_mean(loss).numpy() + meters.update(metrics) + + logger.info("Test Epoch [%d/%d] Arc [%d/%d] Summary %s", + epoch + 1, self.num_epochs, arc_id + 1, test_arc_per_epoch, + meters.summary()) + + + def _create_train_loader(self): + train_set = self.train_set.shuffle(1000000).batch(self.batch_size) + test_set = self.test_set.shuffle(1000000).batch(self.batch_size) + return iter(train_set), iter(test_set) + + def _create_validate_loader(self): + return iter(self.test_set.shuffle(1000000).batch(self.batch_size)) diff --git a/src/sdk/pynni/nni/nas/tensorflow/mutables.py b/src/sdk/pynni/nni/nas/tensorflow/mutables.py new file mode 100644 index 0000000000..1665112732 --- /dev/null +++ b/src/sdk/pynni/nni/nas/tensorflow/mutables.py @@ -0,0 +1,136 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +from tensorflow.keras import Model + +from .utils import global_mutable_counting + + +_logger = logging.getLogger(__name__) + + +class Mutable(Model): + def __init__(self, key=None): + super().__init__() + if key is None: + self._key = '{}_{}'.format(type(self).__name__, global_mutable_counting()) + elif isinstance(key, str): + self._key = key + else: + self._key = str(key) + _logger.warning('Key "%s" is not string, converted to string.', key) + self.init_hook = None + self.forward_hook = None + + def __deepcopy__(self, memodict=None): + raise NotImplementedError("Deep copy doesn't work for mutables.") + + def __call__(self, *args, **kwargs): + self._check_built() + return super().__call__(*args, **kwargs) + + def set_mutator(self, mutator): + if 'mutator' in self.__dict__: + raise RuntimeError('`set_mutator is called more than once. ' + 'Did you parse the search space multiple times? ' + 'Or did you apply multiple fixed architectures?') + self.__dict__['mutator'] = mutator + + def call(self, *inputs): + raise NotImplementedError('Method `call` of Mutable must be overridden') + + @property + def key(self): + return self._key + + @property + def name(self): + return self._name if hasattr(self, '_name') else self._key + + @name.setter + def name(self, name): + self._name = name + + def _check_built(self): + if not hasattr(self, 'mutator'): + raise ValueError( + "Mutator not set for {}. You might have forgotten to initialize and apply your mutator. " + "Or did you initialize a mutable on the fly in forward pass? Move to `__init__` " + "so that trainer can locate all your mutables. See NNI docs for more details.".format(self)) + + def __repr__(self): + return '{} ({})'.format(self.name, self.key) + + +class MutableScope(Mutable): + def __call__(self, *args, **kwargs): + try: + self._check_built() + self.mutator.enter_mutable_scope(self) + return super().__call__(*args, **kwargs) + finally: + self.mutator.exit_mutable_scope(self) + + +class LayerChoice(Mutable): + def __init__(self, op_candidates, reduction='sum', return_mask=False, key=None): + super().__init__(key=key) + self.length = len(op_candidates) + self.choices = op_candidates + self.reduction = reduction + self.return_mask = return_mask + self._built = False + + def call(self, *inputs): + if not self._built: + for op in self.choices: + if len(inputs) > 1: # FIXME: not tested + op.build([inp.shape for inp in inputs]) + elif len(inputs) == 1: + op.build(inputs[0].shape) + self._built = True + out, mask = self.mutator.on_forward_layer_choice(self, *inputs) + if self.return_mask: + return out, mask + return out + + def __len__(self): + return len(self.choices) + + +class InputChoice(Mutable): + NO_KEY = '' + + def __init__(self, n_candidates=None, choose_from=None, n_chosen=None, reduction='sum', return_mask=False, key=None): + super().__init__(key=key) + assert n_candidates is not None or choose_from is not None, \ + 'At least one of `n_candidates` and `choose_from` must be not None.' + if choose_from is not None and n_candidates is None: + n_candidates = len(choose_from) + elif choose_from is None and n_candidates is not None: + choose_from = [self.NO_KEY] * n_candidates + assert n_candidates == len(choose_from), 'Number of candidates must be equal to the length of `choose_from`.' + assert n_candidates > 0, 'Number of candidates must be greater than 0.' + assert n_chosen is None or 0 <= n_chosen <= n_candidates, \ + 'Expected selected number must be None or no more than number of candidates.' + + self.n_candidates = n_candidates + self.choose_from = choose_from.copy() + self.n_chosen = n_chosen + self.reduction = reduction + self.return_mask = return_mask + + def call(self, optional_inputs): + optional_input_list = optional_inputs + if isinstance(optional_inputs, dict): + optional_input_list = [optional_inputs[tag] for tag in self.choose_from] + assert isinstance(optional_input_list, list), \ + 'Optional input list must be a list, not a {}.'.format(type(optional_input_list)) + assert len(optional_inputs) == self.n_candidates, \ + 'Length of the input list must be equal to number of candidates.' + out, mask = self.mutator.on_forward_input_choice(self, optional_input_list) + if self.return_mask: + return out, mask + return out diff --git a/src/sdk/pynni/nni/nas/tensorflow/mutator.py b/src/sdk/pynni/nni/nas/tensorflow/mutator.py new file mode 100644 index 0000000000..20c57f9405 --- /dev/null +++ b/src/sdk/pynni/nni/nas/tensorflow/mutator.py @@ -0,0 +1,77 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import tensorflow as tf + +from .base_mutator import BaseMutator + + +_logger = logging.getLogger(__name__) + + +class Mutator(BaseMutator): + def __init__(self, model): + super().__init__(model) + self._cache = {} + + def sample_search(self): + raise NotImplementedError('Method `sample_search` must be overridden') + + def sample_final(self): + raise NotImplementedError('Method `sample_final` must be overriden for exporting') + + def reset(self): + self._cache = self.sample_search() + + def export(self): + return self.sample_final() + + # TODO: status + # TODO: graph + + def on_forward_layer_choice(self, mutable, *inputs): + mask = self._get_decision(mutable) + assert len(mask) == len(mutable), \ + 'Invalid mask, expected {} to be of length {}.'.format(mask, len(mutable)) + out = self._select_with_mask(lambda choice: choice(*inputs), mutable.choices, mask) + return self._tensor_reduction(mutable.reduction, out), mask + + def on_forward_input_choice(self, mutable, tensor_list): + mask = self._get_decision(mutable) + assert len(mask) == mutable.n_candidates, \ + 'Invalid mask, expected {} to be of length {}.'.format(mask, mutable.n_candidates) + out = self._select_with_mask(lambda tensor: tensor, tensor_list, mask) + return self._tensor_reduction(mutable.reduction, out), mask + + def _select_with_mask(self, map_fn, candidates, mask): + if mask.dtype.is_bool: + out = [map_fn(cand) for cand, m in zip(candidates, mask) if m] + elif mask.dtype.is_floating: + out = [map_fn(cand) * m for cand, m in zip(candidates, mask) if m] + else: + raise ValueError('Unrecognized mask, dtype is {}'.format(mask.dtype.name)) + return out + + def _tensor_reduction(self, reduction_type, tensor_list): + if reduction_type == 'none': + return tensor_list + if not tensor_list: + return None + if len(tensor_list) == 1: + return tensor_list[0] + if reduction_type == 'sum': + return sum(tensor_list) + if reduction_type == 'mean': + return sum(tensor_list) / len(tensor_list) + if reduction_type == 'concat': + return tf.concat(tensor_list, axis=0) + raise ValueError('Unrecognized reduction policy: "{}'.format(reduction_type)) + + def _get_decision(self, mutable): + if mutable.key not in self._cache: + raise ValueError('"{}" not found in decision cache.'.format(mutable.key)) + result = self._cache[mutable.key] + _logger.debug('Decision %s: %s', mutable.key, result) + return result diff --git a/src/sdk/pynni/nni/nas/tensorflow/utils.py b/src/sdk/pynni/nni/nas/tensorflow/utils.py new file mode 100644 index 0000000000..0cfc6e815d --- /dev/null +++ b/src/sdk/pynni/nni/nas/tensorflow/utils.py @@ -0,0 +1,93 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import tensorflow as tf + +_counter = 0 + +def global_mutable_counting(): + global _counter + _counter += 1 + return _counter + + +class AverageMeter: + def __init__(self, name): + self.name = name + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val): + self.val = val + self.sum += val + self.count += 1 + self.avg = self.sum / self.count + + def __str__(self): + return '{name} {val:4f} ({avg:4f})'.format(**self.__dict__) + + def summary(self): + return '{name}: {avg:4f}'.format(**self.__dict__) + + +class AverageMeterGroup: + def __init__(self): + self.meters = {} + + def update(self, data): + for k, v in data.items(): + if k not in self.meters: + self.meters[k] = AverageMeter(k) + self.meters[k].update(v) + + def __str__(self): + return ' '.join(str(v) for v in self.meters.values()) + + def summary(self): + return ' '.join(v.summary() for v in self.meters.values()) + + +class StructuredMutableTreeNode: + def __init__(self, mutable): + self.mutable = mutable + self.children = [] + + def add_child(self, mutable): + self.children.append(StructuredMutableTreeNode(mutable)) + return self.children[-1] + + def type(self): + return type(self.mutable) + + def __iter__(self): + return self.traverse() + + def traverse(self, order="pre", deduplicate=True, memo=None): + if memo is None: + memo = set() + assert order in ["pre", "post"] + if order == "pre": + if self.mutable is not None: + if not deduplicate or self.mutable.key not in memo: + memo.add(self.mutable.key) + yield self.mutable + for child in self.children: + for m in child.traverse(order=order, deduplicate=deduplicate, memo=memo): + yield m + if order == "post": + if self.mutable is not None: + if not deduplicate or self.mutable.key not in memo: + memo.add(self.mutable.key) + yield self.mutable + + +def fill_zero_grads(grads, weights): + ret = [] + for grad, weight in zip(grads, weights): + if grad is not None: + ret.append(grad) + else: + ret.append(tf.zeros_like(weight)) + return ret From 1180599a34e7aac274688604b98a40c9e440de88 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Tue, 19 May 2020 20:40:04 +0800 Subject: [PATCH 12/14] Fix pai trainingservice (#2462) --- .../training_service/pai/paiK8S/paiK8STrainingService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nni_manager/training_service/pai/paiK8S/paiK8STrainingService.ts b/src/nni_manager/training_service/pai/paiK8S/paiK8STrainingService.ts index f9be655711..a7a02e5cc7 100644 --- a/src/nni_manager/training_service/pai/paiK8S/paiK8STrainingService.ts +++ b/src/nni_manager/training_service/pai/paiK8S/paiK8STrainingService.ts @@ -158,7 +158,7 @@ class PAIK8STrainingService extends PAITrainingService { if (this.paiTrialConfig === undefined) { throw new Error('trial config is not initialized'); } - const containerNFSExpCodeDir = `${this.paiTrialConfig.containerNFSMountPath}/${this.experimentId}/'nni-code`; + const containerNFSExpCodeDir = `${this.paiTrialConfig.containerNFSMountPath}/${this.experimentId}/nni-code`; const containerWorkingDir: string = `${this.paiTrialConfig.containerNFSMountPath}/${this.experimentId}/${trialJobDetail.id}`; const nniManagerIp: string = this.nniManagerIpConfig ? this.nniManagerIpConfig.nniManagerIp : getIPV4Address(); const nniPaiTrialCommand: string = String.Format( From 69cae211e2790a14f5a0c667b4ac92daaad5ea00 Mon Sep 17 00:00:00 2001 From: Chi Song <27178119+squirrelsc@users.noreply.github.com> Date: Wed, 20 May 2020 00:06:59 +0800 Subject: [PATCH 13/14] Support Windows as remote node. (#2431) --- .../TrainingService/RemoteMachineMode.md | 46 +++- docs/en_US/Tutorial/InstallationWin.md | 66 +++-- .../mnist-batch-tune-keras/mnist-keras.py | 6 +- examples/trials/mnist-keras/mnist-keras.py | 6 +- .../FashionMNIST/FashionMNIST_keras.py | 6 +- .../network_morphism/cifar10/cifar10_keras.py | 8 +- install.ps1 | 5 +- src/nni_manager/common/utils.ts | 63 +++-- src/nni_manager/core/nnimanager.ts | 10 +- .../training_service/common/util.ts | 7 +- .../remote_machine/extends/linuxCommands.ts | 44 +++- .../remote_machine/extends/windowsCommands.ts | 124 ++++++++++ .../remote_machine/osCommands.ts | 13 +- .../remote_machine/remoteMachineData.ts | 103 ++++---- .../remoteMachineTrainingService.ts | 234 ++++++++---------- .../remote_machine/shellExecutor.ts | 214 ++++++++++++---- .../remote_machine/test/linuxCommands.test.ts | 5 - .../remote_machine/test/shellExecutor.test.ts | 55 ++-- .../test/windowsCommands.test.ts | 102 ++++++++ test/config/examples/mnist-annotation.yml | 1 - test/config/examples/mnist-keras.yml | 1 - .../examples/mnist-nested-search-space.yml | 1 - test/config/examples/mnist-pytorch.yml | 1 - test/config/examples/mnist-tfv1.yml | 1 - test/config/integration_tests.yml | 13 +- test/nni_test/nnitest/naive_test.py | 11 +- test/nni_test/nnitest/run_tests.py | 14 +- test/nni_test/nnitest/utils.py | 13 +- .../pipelines-it-remote-linux-to-windows.yml | 48 ++++ tools/nni_cmd/launcher.py | 4 +- tools/nni_cmd/nnictl_utils.py | 2 +- tools/nni_trial_tool/constants.py | 2 - tools/nni_trial_tool/trial_keeper.py | 34 ++- 33 files changed, 873 insertions(+), 390 deletions(-) create mode 100644 src/nni_manager/training_service/remote_machine/extends/windowsCommands.ts create mode 100644 src/nni_manager/training_service/remote_machine/test/windowsCommands.test.ts create mode 100644 test/pipelines/pipelines-it-remote-linux-to-windows.yml diff --git a/docs/en_US/TrainingService/RemoteMachineMode.md b/docs/en_US/TrainingService/RemoteMachineMode.md index fb3aeaca7f..bb8c9d5d67 100644 --- a/docs/en_US/TrainingService/RemoteMachineMode.md +++ b/docs/en_US/TrainingService/RemoteMachineMode.md @@ -2,18 +2,54 @@ NNI can run one experiment on multiple remote machines through SSH, called `remote` mode. It's like a lightweight training platform. In this mode, NNI can be started from your computer, and dispatch trials to remote machines in parallel. -## Remote machine requirements +The OS of remote machines supports `Linux`, `Windows 10`, and `Windows Server 2019`. -* It only supports Linux as remote machines, and [linux part in system specification](../Tutorial/InstallationLinux.md) is same as NNI local mode. +## Requirements -* Follow [installation](../Tutorial/InstallationLinux.md) to install NNI on each machine. - -* Make sure remote machines meet environment requirements of your trial code. If the default environment does not meet the requirements, the setup script can be added into `command` field of NNI config. +* Make sure the default environment of remote machines meets requirements of your trial code. If the default environment does not meet the requirements, the setup script can be added into `command` field of NNI config. * Make sure remote machines can be accessed through SSH from the machine which runs `nnictl` command. It supports both password and key authentication of SSH. For advanced usages, please refer to [machineList part of configuration](../Tutorial/ExperimentConfig.md). * Make sure the NNI version on each machine is consistent. +* Make sure the command of Trial is compatible with remote OSes, if you want to use remote Linux and Windows together. For example, the default python 3.x executable called `python3` on Linux, and `python` on Windows. + +### Linux + +* Follow [installation](../Tutorial/InstallationLinux.md) to install NNI on the remote machine. + +### Windows + +* Follow [installation](../Tutorial/InstallationWin.md) to install NNI on the remote machine. + +* Install and start `OpenSSH Server`. + + 1. Open `Settings` app on Windows. + + 2. Click `Apps`, then click `Optional features`. + + 3. Click `Add a feature`, search and select `OpenSSH Server`, and then click `Install`. + + 4. Once it's installed, run below command to start and set to automatic start. + + ```bat + sc config sshd start=auto + net start sshd + ``` + +* Make sure remote account is administrator, so that it can stop running trials. + +* Make sure there is no welcome message more than default, since it causes ssh2 failed in NodeJs. For example, if you're using Data Science VM on Azure, it needs to remove extra echo commands in `C:\dsvm\tools\setup\welcome.bat`. + + The output like below is ok, when opening a new command window. + + ```text + Microsoft Windows [Version 10.0.17763.1192] + (c) 2018 Microsoft Corporation. All rights reserved. + + (py37_default) C:\Users\AzureUser> + ``` + ## Run an experiment e.g. there are three machines, which can be logged in with username and password. diff --git a/docs/en_US/Tutorial/InstallationWin.md b/docs/en_US/Tutorial/InstallationWin.md index a9f3deb3e4..012188a1b4 100644 --- a/docs/en_US/Tutorial/InstallationWin.md +++ b/docs/en_US/Tutorial/InstallationWin.md @@ -1,46 +1,56 @@ # Install on Windows -## Installation +## Prerequires -Anaconda or Miniconda is highly recommended to manage multiple Python environments. +* Python 3.5 (or above) 64-bit. [Anaconda](https://www.anaconda.com/products/individual) or [Miniconda](https://docs.conda.io/en/latest/miniconda.html) is highly recommended to manage multiple Python environments on Windows. -### Install NNI through pip +* If it's a newly installed Python environment, it needs to install [Microsoft C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/) to support build NNI dependencies like `scikit-learn`. - Prerequisites: `python 64-bit >= 3.5` + ```bat + pip install cython wheel + ``` - ```bash - python -m pip install --upgrade nni - ``` +* git for verifying installation. -### Install NNI through source code +## Install NNI - If you are interested in special or the latest code versions, you can install NNI through source code. +In most cases, you can install and upgrade NNI from pip package. It's easy and fast. - Prerequisites: `python 64-bit >=3.5`, `git`, `PowerShell`. +If you are interested in special or the latest code versions, you can install NNI through source code. - ```bash - git clone -b v1.5 https://github.com/Microsoft/nni.git - cd nni - powershell -ExecutionPolicy Bypass -file install.ps1 - ``` +If you want to contribute to NNI, refer to [setup development environment](SetupNniDeveloperEnvironment.md). + +* From pip package + + ```bat + python -m pip install --upgrade nni + ``` + +* From source code + + ```bat + git clone -b v1.5 https://github.com/Microsoft/nni.git + cd nni + powershell -ExecutionPolicy Bypass -file install.ps1 + ``` ## Verify installation The following example is built on TensorFlow 1.x. Make sure **TensorFlow 1.x is used** when running it. -* Download the examples via clone the source code. +* Clone examples within source code. - ```bash - git clone -b v1.5 https://github.com/Microsoft/nni.git - ``` + ```bat + git clone -b v1.5 https://github.com/Microsoft/nni.git + ``` * Run the MNIST example. - ```bash - nnictl create --config nni\examples\trials\mnist-tfv1\config_windows.yml - ``` + ```bat + nnictl create --config nni\examples\trials\mnist-tfv1\config_windows.yml + ``` - Note: for other examples you need to change trial command `python3` to `python` in each example YAML, if python3 is called through `python` on your machine. + Note: If you are familiar with other frameworks, you can choose corresponding example under `examples\trials`. It needs to change trial command `python3` to `python` in each example YAML, since default installation has `python.exe`, not `python3.exe` executable. * Wait for the message `INFO: Successfully started experiment!` in the command line. This message indicates that your experiment has been successfully started. You can explore the experiment using the `Web UI url`. @@ -112,18 +122,20 @@ If there is a stderr file, please check it. Two possible cases are: * forgetting to install experiment dependencies such as TensorFlow, Keras and so on. ### Fail to use BOHB on Windows + Make sure a C++ 14.0 compiler is installed when trying to run `nnictl package install --name=BOHB` to install the dependencies. ### Not supported tuner on Windows + SMAC is not supported currently; for the specific reason refer to this [GitHub issue](https://github.com/automl/SMAC3/issues/483). -### Use a Windows server as a remote worker -Currently, you can't. +### Use Windows as a remote worker -Note: +Refer to [Remote Machine mode](../TrainingService/RemoteMachineMode.md). -* If an error like `Segmentation fault` is encountered, please refer to the [FAQ](FAQ.md) +### Segmentation fault (core dumped) when installing +Refer to [FAQ](FAQ.md). ## Further reading diff --git a/examples/trials/mnist-batch-tune-keras/mnist-keras.py b/examples/trials/mnist-batch-tune-keras/mnist-keras.py index 9df8f32a35..40aa9f33e4 100644 --- a/examples/trials/mnist-batch-tune-keras/mnist-keras.py +++ b/examples/trials/mnist-batch-tune-keras/mnist-keras.py @@ -84,7 +84,11 @@ def on_epoch_end(self, epoch, logs={}): Run on end of each epoch ''' LOG.debug(logs) - nni.report_intermediate_result(logs["val_acc"]) + # TensorFlow 2.0 API reference claims the key is `val_acc`, but in fact it's `val_accuracy` + if 'val_acc' in logs: + nni.report_intermediate_result(logs['val_acc']) + else: + nni.report_intermediate_result(logs['val_accuracy']) def train(args, params): ''' diff --git a/examples/trials/mnist-keras/mnist-keras.py b/examples/trials/mnist-keras/mnist-keras.py index 2d7dac0004..794b7deb2a 100644 --- a/examples/trials/mnist-keras/mnist-keras.py +++ b/examples/trials/mnist-keras/mnist-keras.py @@ -86,7 +86,11 @@ def on_epoch_end(self, epoch, logs={}): Run on end of each epoch ''' LOG.debug(logs) - nni.report_intermediate_result(logs["val_acc"]) + # TensorFlow 2.0 API reference claims the key is `val_acc`, but in fact it's `val_accuracy` + if 'val_acc' in logs: + nni.report_intermediate_result(logs['val_acc']) + else: + nni.report_intermediate_result(logs['val_accuracy']) def train(args, params): ''' diff --git a/examples/trials/network_morphism/FashionMNIST/FashionMNIST_keras.py b/examples/trials/network_morphism/FashionMNIST/FashionMNIST_keras.py index 7d69a0241a..c357e2c4b8 100644 --- a/examples/trials/network_morphism/FashionMNIST/FashionMNIST_keras.py +++ b/examples/trials/network_morphism/FashionMNIST/FashionMNIST_keras.py @@ -152,7 +152,11 @@ def on_epoch_end(self, epoch, logs=None): if logs is None: logs = dict() logger.debug(logs) - nni.report_intermediate_result(logs["val_accuracy"]) + # TensorFlow 2.0 API reference claims the key is `val_acc`, but in fact it's `val_accuracy` + if 'val_acc' in logs: + nni.report_intermediate_result(logs['val_acc']) + else: + nni.report_intermediate_result(logs['val_accuracy']) # Training diff --git a/examples/trials/network_morphism/cifar10/cifar10_keras.py b/examples/trials/network_morphism/cifar10/cifar10_keras.py index ef371b811b..91f9879e4f 100644 --- a/examples/trials/network_morphism/cifar10/cifar10_keras.py +++ b/examples/trials/network_morphism/cifar10/cifar10_keras.py @@ -152,9 +152,11 @@ def on_epoch_end(self, epoch, logs=None): if logs is None: logs = dict() logger.debug(logs) - # accuracy key for keras 2.2.2: val_acc - # for keras 2.3.1: val_accuracy - nni.report_intermediate_result(logs["val_accuracy"]) + # TensorFlow 2.0 API reference claims the key is `val_acc`, but in fact it's `val_accuracy` + if 'val_acc' in logs: + nni.report_intermediate_result(logs['val_acc']) + else: + nni.report_intermediate_result(logs['val_accuracy']) # Training diff --git a/install.ps1 b/install.ps1 index 9f03454036..33cae44cc9 100644 --- a/install.ps1 +++ b/install.ps1 @@ -148,12 +148,13 @@ cmd /c $NNI_YARN cmd /c $NNI_YARN build Copy-Item config -Destination .\dist\ -Recurse -Force # Building WebUI +# office-ui-fabric-react need longer time. the 180000 is in ms, mean 180 seconds, longer than default 30 seconds. cd ..\webui -cmd /c $NNI_YARN +cmd /c $NNI_YARN --network-timeout 180000 cmd /c $NNI_YARN build # Building NasUI cd ..\nasui -cmd /c $NNI_YARN +cmd /c $NNI_YARN --network-timeout 180000 cmd /c $NNI_YARN build cd ..\.. diff --git a/src/nni_manager/common/utils.ts b/src/nni_manager/common/utils.ts index a79484dd3a..413d2ee220 100644 --- a/src/nni_manager/common/utils.ts +++ b/src/nni_manager/common/utils.ts @@ -22,7 +22,7 @@ import { HyperParameters, TrainingService, TrialJobStatus } from './trainingServ function getExperimentRootDir(): string { return getExperimentStartupInfo() - .getLogDir(); + .getLogDir(); } function getLogDir(): string { @@ -31,7 +31,7 @@ function getLogDir(): string { function getLogLevel(): string { return getExperimentStartupInfo() - .getLogLevel(); + .getLogLevel(); } function getDefaultDatabaseDir(): string { @@ -113,11 +113,16 @@ function uniqueString(len: number): string { return String.fromCharCode(...codes); } +function randomInt(max: number): number { + return Math.floor(Math.random() * max); +} + function randomSelect(a: T[]): T { assert(a !== undefined); return a[Math.floor(Math.random() * a.length)]; } + function parseArg(names: string[]): string { if (process.argv.length >= 4) { for (let i: number = 2; i < process.argv.length - 1; i++) { @@ -132,7 +137,7 @@ function parseArg(names: string[]): string { function getCmdPy(): string { let cmd = 'python3'; - if(process.platform === 'win32'){ + if (process.platform === 'win32') { cmd = 'python'; } return cmd; @@ -160,7 +165,7 @@ function generateParamFileName(hyperParameters: HyperParameters): string { assert(hyperParameters.index >= 0); let paramFileName: string; - if(hyperParameters.index == 0) { + if (hyperParameters.index == 0) { paramFileName = 'parameter.cfg'; } else { paramFileName = `parameter_${hyperParameters.index}.cfg` @@ -211,9 +216,9 @@ function getIPV4Address(): string { return cachedipv4Address; } - if(os.networkInterfaces().eth0) { - for(const item of os.networkInterfaces().eth0) { - if(item.family === 'IPv4') { + if (os.networkInterfaces().eth0) { + for (const item of os.networkInterfaces().eth0) { + if (item.family === 'IPv4') { cachedipv4Address = item.address; return cachedipv4Address; } @@ -225,14 +230,6 @@ function getIPV4Address(): string { throw Error('getIPV4Address() failed because no valid IPv4 address found.') } -function getRemoteTmpDir(osType: string): string { - if (osType == 'linux') { - return '/tmp'; - } else { - throw Error(`remote OS ${osType} not supported`); - } -} - /** * Get the status of canceled jobs according to the hint isEarlyStopped */ @@ -245,7 +242,7 @@ function getJobCancelStatus(isEarlyStopped: boolean): TrialJobStatus { * @param directory directory name */ function countFilesRecursively(directory: string): Promise { - if(!fs.existsSync(directory)) { + if (!fs.existsSync(directory)) { throw Error(`Direcotory ${directory} doesn't exist`); } @@ -261,13 +258,13 @@ function countFilesRecursively(directory: string): Promise { let fileCount: number = -1; let cmd: string; - if(process.platform === "win32") { + if (process.platform === "win32") { cmd = `powershell "Get-ChildItem -Path ${directory} -Recurse -File | Measure-Object | %{$_.Count}"` } else { cmd = `find ${directory} -type f | wc -l`; } cpp.exec(cmd).then((result) => { - if(result.stdout && parseInt(result.stdout)) { + if (result.stdout && parseInt(result.stdout)) { fileCount = parseInt(result.stdout); } deferred.resolve(fileCount); @@ -280,20 +277,20 @@ function countFilesRecursively(directory: string): Promise { function validateFileName(fileName: string): boolean { const pattern: string = '^[a-z0-9A-Z._-]+$'; const validateResult = fileName.match(pattern); - if(validateResult) { + if (validateResult) { return true; } return false; } async function validateFileNameRecursively(directory: string): Promise { - if(!fs.existsSync(directory)) { + if (!fs.existsSync(directory)) { throw Error(`Direcotory ${directory} doesn't exist`); } const fileNameArray: string[] = fs.readdirSync(directory); let result = true; - for(const name of fileNameArray){ + for (const name of fileNameArray) { const fullFilePath: string = path.join(directory, name); try { // validate file names and directory names @@ -301,14 +298,14 @@ async function validateFileNameRecursively(directory: string): Promise if (fs.lstatSync(fullFilePath).isDirectory()) { result = result && await validateFileNameRecursively(fullFilePath); } - if(!result) { + if (!result) { return Promise.reject(new Error(`file name in ${fullFilePath} is not valid!`)); } - } catch(error) { + } catch (error) { return Promise.reject(error); } } - return Promise.resolve(result); + return Promise.resolve(result); } /** @@ -316,9 +313,9 @@ async function validateFileNameRecursively(directory: string): Promise */ async function getVersion(): Promise { const deferred: Deferred = new Deferred(); - import(path.join(__dirname, '..', 'package.json')).then((pkg)=>{ + import(path.join(__dirname, '..', 'package.json')).then((pkg) => { deferred.resolve(pkg.version); - }).catch((error)=>{ + }).catch((error) => { deferred.reject(error); }); return deferred.promise; @@ -331,9 +328,9 @@ function getTunerProc(command: string, stdio: StdioOptions, newCwd: string, newE let cmd: string = command; let arg: string[] = []; let newShell: boolean = true; - if(process.platform === "win32"){ + if (process.platform === "win32") { cmd = command.split(" ", 1)[0]; - arg = command.substr(cmd.length+1).split(" "); + arg = command.substr(cmd.length + 1).split(" "); newShell = false; } const tunerProc: ChildProcess = spawn(cmd, arg, { @@ -383,7 +380,7 @@ async function killPid(pid: any): Promise { if (process.platform === "win32") { await cpp.exec(`cmd.exe /c taskkill /PID ${pid} /F`); } - else{ + else { await cpp.exec(`kill -9 ${pid}`); } } catch (error) { @@ -397,7 +394,7 @@ function getNewLine(): string { if (process.platform === "win32") { return "\r\n"; } - else{ + else { return "\n"; } } @@ -412,6 +409,8 @@ function unixPathJoin(...paths: any[]): string { return dir; } -export {countFilesRecursively, validateFileNameRecursively, getRemoteTmpDir, generateParamFileName, getMsgDispatcherCommand, getCheckpointDir, +export { + countFilesRecursively, validateFileNameRecursively, generateParamFileName, getMsgDispatcherCommand, getCheckpointDir, getLogDir, getExperimentRootDir, getJobCancelStatus, getDefaultDatabaseDir, getIPV4Address, unixPathJoin, - mkDirP, mkDirPSync, delay, prepareUnitTest, parseArg, cleanupUnitTest, uniqueString, randomSelect, getLogLevel, getVersion, getCmdPy, getTunerProc, isAlive, killPid, getNewLine }; + mkDirP, mkDirPSync, delay, prepareUnitTest, parseArg, cleanupUnitTest, uniqueString, randomInt, randomSelect, getLogLevel, getVersion, getCmdPy, getTunerProc, isAlive, killPid, getNewLine +}; diff --git a/src/nni_manager/core/nnimanager.ts b/src/nni_manager/core/nnimanager.ts index b276168be1..5b86815ee0 100644 --- a/src/nni_manager/core/nnimanager.ts +++ b/src/nni_manager/core/nnimanager.ts @@ -266,7 +266,7 @@ class NNIManager implements Manager { const delay1: Promise<{}> = new Promise((resolve: Function, reject: Function): void => { timeoutId = setTimeout( () => { reject(new Error('TrainingService setClusterMetadata timeout. Please check your config file.')); }, - 10000); + 30000); }); await Promise.race([delay1, this.trainingService.setClusterMetadata(key, value)]).finally(() => { clearTimeout(timeoutId); @@ -368,7 +368,7 @@ class NNIManager implements Manager { CUDA_VISIBLE_DEVICES: this.getGpuEnvvarValue() }; const newEnv = Object.assign({}, process.env, nniEnv); - const tunerProc: ChildProcess = getTunerProc(command,stdio,newCwd,newEnv); + const tunerProc: ChildProcess = getTunerProc(command, stdio, newCwd, newEnv); this.dispatcherPid = tunerProc.pid; this.dispatcher = createDispatcherInterface(tunerProc); @@ -436,7 +436,9 @@ class NNIManager implements Manager { } await killPid(this.dispatcherPid); const trialJobList: TrialJobDetail[] = await this.trainingService.listTrialJobs(); - // TO DO: to promise all + + // DON'T try to make it in parallel, the training service may not handle it well. + // If there is performance concern, consider to support batch cancellation on training service. for (const trialJob of trialJobList) { if (trialJob.status === 'RUNNING' || trialJob.status === 'WAITING') { @@ -444,7 +446,7 @@ class NNIManager implements Manager { this.log.info(`cancelTrialJob: ${trialJob.id}`); await this.trainingService.cancelTrialJob(trialJob.id); } catch (error) { - // pid does not exist, do nothing here + this.log.debug(`ignorable error on canceling trial ${trialJob.id}. ${error}`); } } } diff --git a/src/nni_manager/training_service/common/util.ts b/src/nni_manager/training_service/common/util.ts index 9328f01e61..dd7e368ac6 100644 --- a/src/nni_manager/training_service/common/util.ts +++ b/src/nni_manager/training_service/common/util.ts @@ -174,10 +174,11 @@ export async function tarAdd(tarPath: string, sourcePath: string): Promise script.push( `import os`, `import tarfile`, - String.Format(`tar = tarfile.open("{0}","w:gz")\r\nfor root,dir,files in os.walk("{1}"):`, tarFilePath, sourceFilePath), + String.Format(`tar = tarfile.open("{0}","w:gz")\r\nroot="{1}"\r\nfor file_path,dir,files in os.walk(root):`, tarFilePath, sourceFilePath), ` for file in files:`, - ` fullpath = os.path.join(root,file)`, - ` tar.add(fullpath, arcname=file)`, + ` full_path = os.path.join(file_path, file)`, + ` file = os.path.relpath(full_path, root)`, + ` tar.add(full_path, arcname=file)`, `tar.close()`); await fs.promises.writeFile(path.join(os.tmpdir(), 'tar.py'), script.join(getNewLine()), { encoding: 'utf8', mode: 0o777 }); const tarScript: string = path.join(os.tmpdir(), 'tar.py'); diff --git a/src/nni_manager/training_service/remote_machine/extends/linuxCommands.ts b/src/nni_manager/training_service/remote_machine/extends/linuxCommands.ts index 29bb688d17..a2db2d7515 100644 --- a/src/nni_manager/training_service/remote_machine/extends/linuxCommands.ts +++ b/src/nni_manager/training_service/remote_machine/extends/linuxCommands.ts @@ -7,6 +7,36 @@ import { OsCommands } from "../osCommands"; import { RemoteCommandResult } from "../remoteMachineData"; class LinuxCommands extends OsCommands { + + public getScriptExt(): string { + return "sh"; + } + + public generateStartScript(workingDirectory: string, trialJobId: string, experimentId: string, + trialSequenceId: string, isMultiPhase: boolean, jobIdFileName: string, + command: string, nniManagerAddress: string, nniManagerPort: number, + nniManagerVersion: string, logCollection: string, exitCodeFile: string, + codeDir: string, cudaVisibleSetting: string): string { + + return `#!/bin/bash + export NNI_PLATFORM=remote NNI_SYS_DIR=${workingDirectory} NNI_OUTPUT_DIR=${workingDirectory} NNI_TRIAL_JOB_ID=${trialJobId} \ + NNI_EXP_ID=${experimentId} NNI_TRIAL_SEQ_ID=${trialSequenceId} NNI_CODE_DIR=${codeDir} + export MULTI_PHASE=${isMultiPhase} + + cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR + cd $NNI_SYS_DIR + sh install_nni.sh + python3 -m nni_trial_tool.trial_keeper --trial_command '${cudaVisibleSetting} ${command}' --nnimanager_ip '${nniManagerAddress}' \ + --nnimanager_port '${nniManagerPort}' --nni_manager_version '${nniManagerVersion}' \ + --job_id_file ${jobIdFileName} \ + --log_collection '${logCollection}' 1>$NNI_OUTPUT_DIR/trialkeeper_stdout 2>$NNI_OUTPUT_DIR/trialkeeper_stderr + echo $? \`date +%s%3N\` >${exitCodeFile}`; + } + + public generateGpuStatsScript(scriptFolder: string): string { + return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni_gpu_tool.gpu_metrics_collector`; + } + public createFolder(folderName: string, sharedFolder: boolean = false): string { let command; if (sharedFolder) { @@ -64,7 +94,19 @@ class LinuxCommands extends OsCommands { } public killChildProcesses(pidFileName: string): string { - const command = `pkill -P \`cat '${pidFileName}'\``; + // prevent trialkeeper to be killed, so it can save exit code. + const command = `list_descendants () + { + local children=$(ps -o pid= --ppid "$1") + + for pid in $children + do + list_descendants "$pid" + done + + echo "$children" + } + kill $(list_descendants \`cat '${pidFileName}'\`)` return command; } diff --git a/src/nni_manager/training_service/remote_machine/extends/windowsCommands.ts b/src/nni_manager/training_service/remote_machine/extends/windowsCommands.ts new file mode 100644 index 0000000000..c47d017168 --- /dev/null +++ b/src/nni_manager/training_service/remote_machine/extends/windowsCommands.ts @@ -0,0 +1,124 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +'use strict'; + +import { OsCommands } from "../osCommands"; +import { RemoteCommandResult } from "../remoteMachineData"; + +class WindowsCommands extends OsCommands { + + protected pathSpliter: string = '\\'; + + public getScriptExt(): string { + return "cmd"; + } + public generateStartScript(workingDirectory: string, trialJobId: string, experimentId: string, + trialSequenceId: string, isMultiPhase: boolean, jobIdFileName: string, + command: string, nniManagerAddress: string, nniManagerPort: number, + nniManagerVersion: string, logCollection: string, exitCodeFile: string, + codeDir: string, cudaVisibleSetting: string): string { + return `echo off + set NNI_PLATFORM=remote + set NNI_SYS_DIR=${workingDirectory} + set NNI_OUTPUT_DIR=${workingDirectory} + set NNI_TRIAL_JOB_ID=${trialJobId} + set NNI_EXP_ID=${experimentId} + set NNI_TRIAL_SEQ_ID=${trialSequenceId} + set MULTI_PHASE=${isMultiPhase} + set NNI_CODE_DIR=${codeDir} + ${cudaVisibleSetting !== "" ? "set " + cudaVisibleSetting : ""} + + robocopy /s %NNI_CODE_DIR%/. %NNI_SYS_DIR% + cd %NNI_SYS_DIR% + python -c "import nni" 2>nul + if not %ERRORLEVEL% EQU 0 ( + echo installing NNI as exit code of "import nni" is %ERRORLEVEL% + python -m pip install --user --upgrade nni + ) + + echo starting script + python -m nni_trial_tool.trial_keeper --trial_command "${command}" --nnimanager_ip "${nniManagerAddress}" --nnimanager_port "${nniManagerPort}" --nni_manager_version "${nniManagerVersion}" --log_collection "${logCollection}" --job_id_file ${jobIdFileName} 1>%NNI_OUTPUT_DIR%/trialkeeper_stdout 2>%NNI_OUTPUT_DIR%/trialkeeper_stderr + + echo save exit code(%ERRORLEVEL%) and time + echo|set /p="%ERRORLEVEL% " > ${exitCodeFile} + powershell -command "Write (((New-TimeSpan -Start (Get-Date "01/01/1970") -End (Get-Date).ToUniversalTime()).TotalMilliseconds).ToString("0")) | Out-file ${exitCodeFile} -Append -NoNewline -encoding utf8"`; + } + + public generateGpuStatsScript(scriptFolder: string): string { + return `powershell -command $env:METRIC_OUTPUT_DIR='${scriptFolder}';$app = Start-Process -FilePath python -NoNewWindow -passthru -ArgumentList '-m nni_gpu_tool.gpu_metrics_collector' -RedirectStandardOutput ${scriptFolder}\\scriptstdout -RedirectStandardError ${scriptFolder}\\scriptstderr;Write $PID ^| Out-File ${scriptFolder}\\pid -NoNewline -encoding utf8;wait-process $app.ID`; + } + + public createFolder(folderName: string, sharedFolder: boolean = false): string { + let command; + if (sharedFolder) { + command = `mkdir "${folderName}"\r\nICACLS "${folderName}" /grant "Users":F`; + } else { + command = `mkdir "${folderName}"`; + } + return command; + } + + public allowPermission(isRecursive: boolean = false, ...folders: string[]): string { + let commands: string = ""; + + folders.forEach(folder => { + commands += `ICACLS "${folder}" /grant "Users":F${isRecursive ? " /T" : ""}\r\n` + }); + return commands; + } + + public removeFolder(folderName: string, isRecursive: boolean = false, isForce: boolean = true): string { + let flags = ''; + if (isForce || isRecursive) { + flags = `${isRecursive ? ' /s' : ''}${isForce ? ' /q' : ''}`; + } + + const command = `rmdir${flags} "${folderName}"`; + return command; + } + + public removeFiles(folderName: string, filePattern: string): string { + const files = this.joinPath(folderName, filePattern); + const command = `del "${files}"`; + return command; + } + + public readLastLines(fileName: string, lineCount: number = 1): string { + const command = `powershell.exe Get-Content "${fileName}" -Tail ${lineCount}`; + return command; + } + + public isProcessAliveCommand(pidFileName: string): string { + const command = `powershell.exe Get-Process -Id (get-content "${pidFileName}") -ErrorAction SilentlyContinue`; + return command; + } + + public isProcessAliveProcessOutput(commandResult: RemoteCommandResult): boolean { + let result = true; + if (commandResult.exitCode !== 0) { + result = false; + } + return result; + } + + public killChildProcesses(pidFileName: string): string { + const command = `powershell "$ppid=(type ${pidFileName}); function Kill-Tree {Param([int]$subppid);` + + `Get-CimInstance Win32_Process | Where-Object { $_.ParentProcessId -eq $subppid } | ForEach-Object { Kill-Tree $_.ProcessId }; ` + + `if ($subppid -ne $ppid){Stop-Process -Id $subppid}}` + + `kill-tree $ppid"`; + return command; + } + + public extractFile(tarFileName: string, targetFolder: string): string { + const command = `tar -xf "${tarFileName}" -C "${targetFolder}"`; + return command; + } + + public executeScript(script: string, _isFile: boolean): string { + const command = `${script}`; + return command; + } +} + +export { WindowsCommands }; diff --git a/src/nni_manager/training_service/remote_machine/osCommands.ts b/src/nni_manager/training_service/remote_machine/osCommands.ts index 804d964586..7f9144e435 100644 --- a/src/nni_manager/training_service/remote_machine/osCommands.ts +++ b/src/nni_manager/training_service/remote_machine/osCommands.ts @@ -8,8 +8,16 @@ import { RemoteCommandResult } from "./remoteMachineData"; abstract class OsCommands { protected pathSpliter: string = '/'; - protected multiplePathSpliter: RegExp = new RegExp(`\\${this.pathSpliter}{2,}`); + protected multiplePathSpliter: RegExp = new RegExp(`[\\\\/]{2,}`); + protected normalizePath: RegExp = new RegExp(`[\\\\/]`); + public abstract getScriptExt(): string; + public abstract generateStartScript(workingDirectory: string, trialJobId: string, experimentId: string, + trialSequenceId: string, isMultiPhase: boolean, jobIdFileName: string, + command: string, nniManagerAddress: string, nniManagerPort: number, + nniManagerVersion: string, logCollection: string, exitCodeFile: string, + codeDir: string, cudaVisibleSetting: string): string; + public abstract generateGpuStatsScript(scriptFolder: string): string; public abstract createFolder(folderName: string, sharedFolder: boolean): string; public abstract allowPermission(isRecursive: boolean, ...folders: string[]): string; public abstract removeFolder(folderName: string, isRecursive: boolean, isForce: boolean): string; @@ -26,6 +34,9 @@ abstract class OsCommands { if (dir === '') { dir = '.'; } else { + // normalize + dir = dir.replace(this.normalizePath, this.pathSpliter); + // reduce duplicate ones dir = dir.replace(this.multiplePathSpliter, this.pathSpliter); } return dir; diff --git a/src/nni_manager/training_service/remote_machine/remoteMachineData.ts b/src/nni_manager/training_service/remote_machine/remoteMachineData.ts index 2ace908132..28d49762f4 100644 --- a/src/nni_manager/training_service/remote_machine/remoteMachineData.ts +++ b/src/nni_manager/training_service/remote_machine/remoteMachineData.ts @@ -85,78 +85,82 @@ export class RemoteMachineTrialJobDetail implements TrialJobDetail { * The remote machine executor manager */ export class ExecutorManager { - private readonly executorArray: ShellExecutor[]; - private readonly maxTrialNumberPerConnection: number; + private readonly executorMap: Map = new Map(); private readonly rmMeta: RemoteMachineMeta; - constructor(executorArray: ShellExecutor[], maxTrialNumberPerConnection: number, rmMeta: RemoteMachineMeta) { + + private executors: ShellExecutor[] = []; + + constructor(rmMeta: RemoteMachineMeta) { this.rmMeta = rmMeta; - this.executorArray = executorArray; - this.maxTrialNumberPerConnection = maxTrialNumberPerConnection; } - /** - * find a available executor, if no executor available, return a new one - */ - public async getAvailableExecutor(): Promise { - for (const index of this.executorArray.keys()) { - const connectionNumber: number = this.executorArray[index].getUsedConnectionNumber; - if (connectionNumber < this.maxTrialNumberPerConnection) { - this.executorArray[index].addUsedConnectionNumber(); + public async getExecutor(id: string): Promise { + let isFound = false; + let executor: ShellExecutor | undefined; - return this.executorArray[index]; + // already assigned + if (this.executorMap.has(id)) { + executor = this.executorMap.get(id); + if (executor === undefined) { + throw new Error("executor shouldn't be undefined before return!"); } + return executor; } - //init a new executor if could not get an available one - return await this.initNewShellExecutor(); - } + for (const candidateExecutor of this.executors) { + if (candidateExecutor.addUsage()) { + isFound = true; + executor = candidateExecutor; + break; + } + } + // init a new executor if no free one. + if (!isFound) { + executor = await this.createShellExecutor(); + } - /** - * add a new executor to executorArray - * @param executor ShellExecutor - */ - public addNewShellExecutor(executor: ShellExecutor): void { - this.executorArray.push(executor); - } + if (executor === undefined) { + throw new Error("executor shouldn't be undefined before set!"); + } + this.executorMap.set(id, executor); - /** - * first executor instance is used for gpu collector and host job - */ - public getFirstExecutor(): ShellExecutor { - return this.executorArray[0]; + return executor; } /** * close all of executor */ - public closeAllExecutor(): void { - for (const executor of this.executorArray) { + public releaseAllExecutor(): void { + this.executorMap.clear(); + for (const executor of this.executors) { executor.close(); } + this.executors = []; } /** * retrieve resource, minus a number for given executor * @param executor executor */ - public releaseConnection(executor: ShellExecutor | undefined): void { + public releaseExecutor(id: string): void { + const executor = this.executorMap.get(id); if (executor === undefined) { - throw new Error(`could not release a undefined executor`); - } - for (const index of this.executorArray.keys()) { - if (this.executorArray[index] === executor) { - this.executorArray[index].minusUsedConnectionNumber(); - break; - } + throw new Error(`executor for ${id} is not found`); } + executor.releaseUsage(); + this.executorMap.delete(id); } /** * Create a new connection executor and initialize it */ - private async initNewShellExecutor(): Promise { + private async createShellExecutor(): Promise { const executor = new ShellExecutor(); await executor.initialize(this.rmMeta); + if (!executor.addUsage()) { + throw new Error("failed to add usage on new created Executor! It's a wired bug!"); + } + this.executors.push(executor); return executor; } } @@ -175,22 +179,3 @@ export enum ScheduleResultType { // Cannot match requirement even if all GPU are a REQUIRE_EXCEED_TOTAL } - -export const REMOTEMACHINE_TRIAL_COMMAND_FORMAT: string = - `#!/bin/bash -export NNI_PLATFORM=remote NNI_SYS_DIR={0} NNI_OUTPUT_DIR={1} NNI_TRIAL_JOB_ID={2} NNI_EXP_ID={3} \ -NNI_TRIAL_SEQ_ID={4} MULTI_PHASE={5} NNI_CODE_DIR={6} -cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR -cd $NNI_SYS_DIR -sh install_nni.sh -echo $$ >{7} -python3 -m nni_trial_tool.trial_keeper --trial_command '{8}' --nnimanager_ip '{9}' --nnimanager_port '{10}' \ ---nni_manager_version '{11}' --log_collection '{12}' 1>$NNI_OUTPUT_DIR/trialkeeper_stdout 2>$NNI_OUTPUT_DIR/trialkeeper_stderr -echo $? \`date +%s%3N\` >{13}`; - -export const HOST_JOB_SHELL_FORMAT: string = - `#!/bin/bash -cd {0} -echo $$ >{1} -eval {2} >stdout 2>stderr -echo $? \`date +%s%3N\` >{3}`; diff --git a/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts b/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts index 5af10230f5..138fa6e0ae 100644 --- a/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts +++ b/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts @@ -8,7 +8,6 @@ import { EventEmitter } from 'events'; import * as fs from 'fs'; import * as path from 'path'; import { Deferred } from 'ts-deferred'; -import { String } from 'typescript-string-operations'; import * as component from '../../common/component'; import { NNIError, NNIErrorNames } from '../../common/errors'; import { getExperimentId } from '../../common/experimentStartupInfo'; @@ -19,17 +18,17 @@ import { TrialJobDetail, TrialJobMetric } from '../../common/trainingService'; import { - delay, generateParamFileName, getExperimentRootDir, getIPV4Address, getJobCancelStatus, getRemoteTmpDir, - getVersion, uniqueString, unixPathJoin + delay, generateParamFileName, getExperimentRootDir, getIPV4Address, getJobCancelStatus, + getVersion, uniqueString } from '../../common/utils'; import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData'; import { GPUSummary } from '../common/gpuData'; import { TrialConfig } from '../common/trialConfig'; import { TrialConfigMetadataKey } from '../common/trialConfigMetadataKey'; -import { execMkdir, validateCodeDir, getGpuMetricsCollectorBashScriptContent } from '../common/util'; +import { execMkdir, validateCodeDir } from '../common/util'; import { GPUScheduler } from './gpuScheduler'; import { - REMOTEMACHINE_TRIAL_COMMAND_FORMAT, RemoteMachineMeta, + RemoteMachineMeta, RemoteMachineScheduleInfo, RemoteMachineScheduleResult, RemoteMachineTrialJobDetail, ScheduleResultType, ExecutorManager } from './remoteMachineData'; @@ -41,14 +40,12 @@ import { ShellExecutor } from 'training_service/remote_machine/shellExecutor'; */ @component.Singleton class RemoteMachineTrainingService implements TrainingService { + private readonly initExecutorId = "initConnection"; private readonly machineExecutorManagerMap: Map; //machine excutor map private readonly machineCopyExpCodeDirPromiseMap: Map>; - private readonly trialExecutorMap: Map; //trial excutor map + private readonly trialExecutorManagerMap: Map; //trial excutor map private readonly trialJobsMap: Map; - private readonly MAX_TRIAL_NUMBER_PER_EXECUTOR: number = 5; // every excutor has a max trial concurrency number private readonly expRootDir: string; - private readonly remoteExpRootDir: string; - private readonly remoteExpCodeDir: string; private trialConfig: TrialConfig | undefined; private gpuScheduler?: GPUScheduler; private readonly jobQueue: string[]; @@ -57,27 +54,21 @@ class RemoteMachineTrainingService implements TrainingService { private readonly metricsEmitter: EventEmitter; private readonly log: Logger; private isMultiPhase: boolean = false; - private trialSequenceId: number; private remoteRestServerPort?: number; - private readonly remoteOS: string; private nniManagerIpConfig?: NNIManagerIpConfig; private versionCheck: boolean = true; private logCollection: string; constructor(@component.Inject timer: ObservableTimer) { - this.remoteOS = 'linux'; this.metricsEmitter = new EventEmitter(); this.trialJobsMap = new Map(); - this.trialExecutorMap = new Map(); - this.machineExecutorManagerMap = new Map(); + this.trialExecutorManagerMap = new Map(); this.machineCopyExpCodeDirPromiseMap = new Map>(); + this.machineExecutorManagerMap = new Map(); this.jobQueue = []; this.expRootDir = getExperimentRootDir(); - this.remoteExpRootDir = this.getRemoteExperimentRootDir(); - this.remoteExpCodeDir = unixPathJoin(this.remoteExpRootDir, 'nni-code'); this.timer = timer; this.log = getLogger(); - this.trialSequenceId = -1; this.logCollection = 'none'; this.log.info('Construct remote machine training service.'); } @@ -110,14 +101,14 @@ class RemoteMachineTrainingService implements TrainingService { } await delay(3000); } - this.log.info('Remote machine training service exit.'); + this.log.info('RemoteMachineTrainingService run loop exited.'); } /** * give trial an executor * @param trial remote machine trial job detail */ - public async allocateExecutorForTrial(trial: RemoteMachineTrialJobDetail): Promise { + public allocateExecutorManagerForTrial(trial: RemoteMachineTrialJobDetail): void { if (trial.rmMeta === undefined) { throw new Error(`rmMeta not set in trial ${trial.id}`); } @@ -125,23 +116,23 @@ class RemoteMachineTrainingService implements TrainingService { if (executorManager === undefined) { throw new Error(`executorManager not initialized`); } - const shellExecutor: ShellExecutor = await executorManager.getAvailableExecutor(); - this.trialExecutorMap.set(trial.id, shellExecutor); + this.trialExecutorManagerMap.set(trial.id, executorManager); } /** * If a trial is finished, release the connection resource * @param trial remote machine trial job detail */ - public releaseTrialExecutor(trial: RemoteMachineTrialJobDetail): void { + public releaseTrialResource(trial: RemoteMachineTrialJobDetail): void { if (trial.rmMeta === undefined) { throw new Error(`rmMeta not set in trial ${trial.id}`); } - const executorManager: ExecutorManager | undefined = this.machineExecutorManagerMap.get(trial.rmMeta); + const executorManager = this.trialExecutorManagerMap.get(trial.id); if (executorManager === undefined) { - throw new Error(`executorManager not initialized`); + throw new Error(`ExecutorManager is not assigned for trial ${trial.id}`); } - executorManager.releaseConnection(this.trialExecutorMap.get(trial.id)); + // Note, it still keep reference in trialExecutorManagerMap, as there may be following requests from nni manager. + executorManager.releaseExecutor(trial.id); } /** @@ -174,10 +165,7 @@ class RemoteMachineTrainingService implements TrainingService { if (trialJob.rmMeta === undefined) { throw new Error(`rmMeta not set for submitted job ${trialJobId}`); } - const executor: ShellExecutor | undefined = this.trialExecutorMap.get(trialJob.id); - if (executor === undefined) { - throw new Error(`Invalid job id: ${trialJobId}, cannot find executor`); - } + const executor = await this.getExecutor(trialJob.id); return this.updateTrialJobStatus(trialJob, executor); } else { @@ -212,13 +200,12 @@ class RemoteMachineTrainingService implements TrainingService { // Generate trial job id(random) const trialJobId: string = uniqueString(5); - const trialWorkingFolder: string = unixPathJoin(this.remoteExpRootDir, 'trials', trialJobId); const trialJobDetail: RemoteMachineTrialJobDetail = new RemoteMachineTrialJobDetail( trialJobId, 'WAITING', Date.now(), - trialWorkingFolder, + "unset", form ); this.jobQueue.push(trialJobId); @@ -268,26 +255,23 @@ class RemoteMachineTrainingService implements TrainingService { // Get executor where the job is running if (trialJob.rmMeta !== undefined) { // If the trial job is already scheduled, check its status and kill the trial process in remote machine - const executor: ShellExecutor | undefined = this.trialExecutorMap.get(trialJob.id); - if (executor === undefined) { - throw new Error(`Invalid job id ${trialJobId}, cannot find executor`); - } + const executor = await this.getExecutor(trialJob.id); if (trialJob.status === 'UNKNOWN') { - this.releaseTrialExecutor(trialJob); trialJob.status = 'USER_CANCELED'; + this.releaseTrialResource(trialJob); return } - const jobpidPath: string = this.getJobPidPath(trialJob.id); + const jobpidPath: string = this.getJobPidPath(executor, trialJob.id); try { // Mark the toEarlyStop tag here trialJob.isEarlyStopped = isEarlyStopped; await executor.killChildProcesses(jobpidPath); - this.releaseTrialExecutor(trialJob); + this.releaseTrialResource(trialJob); } catch (error) { // Not handle the error since pkill failed will not impact trial job's current status - this.log.error(`remoteTrainingService.cancelTrialJob: ${error.message}`); + this.log.error(`remoteTrainingService.cancelTrialJob: ${error}`); } } else { // Job is not scheduled yet, set status to 'USER_CANCELLED' directly @@ -329,15 +313,15 @@ class RemoteMachineTrainingService implements TrainingService { await validateCodeDir(remoteMachineTrailConfig.codeDir); // Copy codeDir to remote machine for (const [rmMeta, executorManager] of this.machineExecutorManagerMap.entries()) { - const executor: ShellExecutor = await executorManager.getAvailableExecutor(); + const executor: ShellExecutor = await executorManager.getExecutor(this.initExecutorId); if (executor !== undefined) { this.machineCopyExpCodeDirPromiseMap.set( rmMeta, - executor.copyDirectoryToRemote(remoteMachineTrailConfig.codeDir, this.remoteExpCodeDir, this.remoteOS) - ); + executor.copyDirectoryToRemote(remoteMachineTrailConfig.codeDir, executor.getRemoteCodePath(getExperimentId())) + ); } } - + } catch (error) { this.log.error(error); @@ -376,7 +360,15 @@ class RemoteMachineTrainingService implements TrainingService { public async cleanUp(): Promise { this.log.info('Stopping remote machine training service...'); this.stopping = true; - await Promise.race([delay(10000), this.cleanupConnections()]); + await this.cleanupConnections(); + } + + private async getExecutor(trialId: string): Promise { + const executorManager = this.trialExecutorManagerMap.get(trialId); + if (executorManager === undefined) { + throw new Error(`ExecutorManager is not assigned for trial ${trialId}`); + } + return await executorManager.getExecutor(trialId); } /** @@ -397,21 +389,19 @@ class RemoteMachineTrainingService implements TrainingService { */ private async cleanupConnections(): Promise { try { - for (const [rmMeta, executorManager] of this.machineExecutorManagerMap.entries()) { - const jobpidPath: string = unixPathJoin(this.getRemoteScriptsPath(rmMeta.username), 'pid'); - const executor: ShellExecutor | undefined = executorManager.getFirstExecutor(); + for (const executorManager of this.machineExecutorManagerMap.values()) { + const executor = await executorManager.getExecutor(this.initExecutorId); if (executor !== undefined) { - await executor.killChildProcesses(jobpidPath); - await executor.removeFolder(this.getRemoteScriptsPath(rmMeta.username)); + this.log.info(`killing gpu metric collector on ${executor.name}`); + const gpuJobPidPath: string = executor.joinPath(executor.getRemoteScriptsPath(getExperimentId()), 'pid'); + await executor.killChildProcesses(gpuJobPidPath); } - executorManager.closeAllExecutor(); + executorManager.releaseAllExecutor(); } } catch (error) { //ignore error, this function is called to cleanup remote connections when experiment is stopping - this.log.error(`Cleanup connection exception, error is ${error.message}`); + this.log.error(`Cleanup connection exception, error is ${error}`); } - - return Promise.resolve(); } private async setupConnections(machineList: string): Promise { @@ -423,10 +413,14 @@ class RemoteMachineTrainingService implements TrainingService { rmMetaList.forEach(async (rmMeta: RemoteMachineMeta) => { rmMeta.occupiedGpuIndexMap = new Map(); - const executorManager: ExecutorManager = new ExecutorManager([], this.MAX_TRIAL_NUMBER_PER_EXECUTOR, rmMeta); - const executor: ShellExecutor = await executorManager.getAvailableExecutor(); + const executorManager: ExecutorManager = new ExecutorManager(rmMeta); + this.log.info(`connecting to ${rmMeta.username}@${rmMeta.ip}:${rmMeta.port}`); + const executor: ShellExecutor = await executorManager.getExecutor(this.initExecutorId); + this.log.debug(`reached ${executor.name}`); this.machineExecutorManagerMap.set(rmMeta, executorManager); + this.log.debug(`initializing ${executor.name}`); await this.initRemoteMachineOnConnected(rmMeta, executor); + this.log.info(`connected to ${executor.name}`); if (++connectedRMNum === rmMetaList.length) { deferred.resolve(); } @@ -437,27 +431,36 @@ class RemoteMachineTrainingService implements TrainingService { private async initRemoteMachineOnConnected(rmMeta: RemoteMachineMeta, executor: ShellExecutor): Promise { // Create root working directory after executor is ready - const nniRootDir: string = unixPathJoin(getRemoteTmpDir(this.remoteOS), 'nni'); - await executor.createFolder(this.remoteExpRootDir); + const nniRootDir: string = executor.joinPath(executor.getTempPath(), 'nni'); + await executor.createFolder(executor.getRemoteExperimentRootDir(getExperimentId())); // the directory to store temp scripts in remote machine - const remoteGpuScriptCollectorDir: string = this.getRemoteScriptsPath(rmMeta.username); + const remoteGpuScriptCollectorDir: string = executor.getRemoteScriptsPath(getExperimentId()); + + // clean up previous result. await executor.createFolder(remoteGpuScriptCollectorDir, true); await executor.allowPermission(false, nniRootDir, `${nniRootDir}/*`, `${nniRootDir}/scripts/*`); //Begin to execute gpu_metrics_collection scripts - const script = getGpuMetricsCollectorBashScriptContent(remoteGpuScriptCollectorDir); + const script = executor.generateGpuStatsScript(getExperimentId()); executor.executeScript(script, false, true); + // the timer is trigger in 1 second, it causes multiple runs on server. + // So reduce it's freqeunce, only allow one of it run. + const collectingCount: boolean[] = []; const disposable: Rx.IDisposable = this.timer.subscribe( async () => { - const cmdresult = await executor.readLastLines(unixPathJoin(remoteGpuScriptCollectorDir, 'gpu_metrics')); - if (cmdresult !== "") { - rmMeta.gpuSummary = JSON.parse(cmdresult); - if (rmMeta.gpuSummary.gpuCount === 0) { - this.log.warning(`No GPU found on remote machine ${rmMeta.ip}`); - this.timer.unsubscribe(disposable); + if (collectingCount.length == 0) { + collectingCount.push(true); + const cmdresult = await executor.readLastLines(executor.joinPath(remoteGpuScriptCollectorDir, 'gpu_metrics')); + if (cmdresult !== "") { + rmMeta.gpuSummary = JSON.parse(cmdresult); + if (rmMeta.gpuSummary.gpuCount === 0) { + this.log.warning(`No GPU found on remote machine ${rmMeta.ip}`); + this.timer.unsubscribe(disposable); + } } + collectingCount.pop(); } } ); @@ -492,7 +495,6 @@ class RemoteMachineTrainingService implements TrainingService { } else if (rmScheduleResult.resultType === ScheduleResultType.SUCCEED && rmScheduleResult.scheduleInfo !== undefined) { const rmScheduleInfo: RemoteMachineScheduleInfo = rmScheduleResult.scheduleInfo; - const trialWorkingFolder: string = unixPathJoin(this.remoteExpRootDir, 'trials', trialJobId); trialJobDetail.rmMeta = rmScheduleInfo.rmMeta; const copyExpCodeDirPromise = this.machineCopyExpCodeDirPromiseMap.get(trialJobDetail.rmMeta); @@ -500,12 +502,16 @@ class RemoteMachineTrainingService implements TrainingService { await copyExpCodeDirPromise; } - await this.allocateExecutorForTrial(trialJobDetail); + this.allocateExecutorManagerForTrial(trialJobDetail); + const executor = await this.getExecutor(trialJobDetail.id); + + trialJobDetail.workingDirectory = executor.joinPath(executor.getRemoteExperimentRootDir(getExperimentId()), 'trials', trialJobDetail.id); + await this.launchTrialOnScheduledMachine( - trialJobId, trialWorkingFolder, trialJobDetail.form, rmScheduleInfo); + trialJobId, trialJobDetail.form, rmScheduleInfo); trialJobDetail.status = 'RUNNING'; - trialJobDetail.url = `file://${rmScheduleInfo.rmMeta.ip}:${trialWorkingFolder}`; + trialJobDetail.url = `file://${rmScheduleInfo.rmMeta.ip}:${trialJobDetail.workingDirectory}`; trialJobDetail.startTime = Date.now(); this.trialJobsMap.set(trialJobId, trialJobDetail); @@ -520,19 +526,13 @@ class RemoteMachineTrainingService implements TrainingService { return deferred.promise; } - private async launchTrialOnScheduledMachine(trialJobId: string, trialWorkingFolder: string, form: TrialJobApplicationForm, + private async launchTrialOnScheduledMachine(trialJobId: string, form: TrialJobApplicationForm, rmScheduleInfo: RemoteMachineScheduleInfo): Promise { if (this.trialConfig === undefined) { throw new Error('trial config is not initialized'); } const cudaVisibleDevice: string = rmScheduleInfo.cudaVisibleDevice; - const executor: ShellExecutor | undefined = this.trialExecutorMap.get(trialJobId); - if (executor === undefined) { - assert(false, 'ShellExecutor is undefined.'); - - // for lint - return; - } + const executor = await this.getExecutor(trialJobId); const trialJobDetail: RemoteMachineTrialJobDetail | undefined = this.trialJobsMap.get(trialJobId); if (trialJobDetail === undefined) { throw new Error(`Can not get trial job detail for job: ${trialJobId}`); @@ -540,23 +540,22 @@ class RemoteMachineTrainingService implements TrainingService { const trialLocalTempFolder: string = path.join(this.expRootDir, 'trials-local', trialJobId); - await executor.createFolder(trialWorkingFolder); - await executor.createFolder(unixPathJoin(trialWorkingFolder, '.nni')); + await executor.createFolder(executor.joinPath(trialJobDetail.workingDirectory, '.nni')); // RemoteMachineRunShellFormat is the run shell format string, // See definition in remoteMachineData.ts - let command: string; + let cudaVisible: string; // Set CUDA_VISIBLE_DEVICES environment variable based on cudaVisibleDevice // If no valid cudaVisibleDevice is defined, set CUDA_VISIBLE_DEVICES to empty string to hide GPU device // If gpuNum is undefined, will not set CUDA_VISIBLE_DEVICES in script if (this.trialConfig.gpuNum === undefined) { - command = this.trialConfig.command; + cudaVisible = "" } else { if (typeof cudaVisibleDevice === 'string' && cudaVisibleDevice.length > 0) { - command = `CUDA_VISIBLE_DEVICES=${cudaVisibleDevice} ${this.trialConfig.command}`; + cudaVisible = `CUDA_VISIBLE_DEVICES=${cudaVisibleDevice}`; } else { - command = `CUDA_VISIBLE_DEVICES=" " ${this.trialConfig.command}`; + cudaVisible = `CUDA_VISIBLE_DEVICES=" "`; } } const nniManagerIp: string = this.nniManagerIpConfig ? this.nniManagerIpConfig.nniManagerIp : getIPV4Address(); @@ -565,50 +564,36 @@ class RemoteMachineTrainingService implements TrainingService { this.remoteRestServerPort = restServer.clusterRestServerPort; } const version: string = this.versionCheck ? await getVersion() : ''; - const runScriptTrialContent: string = String.Format( - REMOTEMACHINE_TRIAL_COMMAND_FORMAT, - trialWorkingFolder, - trialWorkingFolder, + const runScriptTrialContent: string = executor.generateStartScript( + trialJobDetail.workingDirectory, trialJobId, getExperimentId(), trialJobDetail.form.sequenceId.toString(), this.isMultiPhase, - this.remoteExpCodeDir, - unixPathJoin(trialWorkingFolder, '.nni', 'jobpid'), - command, + this.trialConfig.command, nniManagerIp, this.remoteRestServerPort, version, - this.logCollection, - unixPathJoin(trialWorkingFolder, '.nni', 'code') - ); + this.logCollection, cudaVisible); //create tmp trial working folder locally. await execMkdir(path.join(trialLocalTempFolder, '.nni')); - // Write install_nni.sh - await fs.promises.writeFile(path.join(trialLocalTempFolder, 'install_nni.sh'), CONTAINER_INSTALL_NNI_SHELL_FORMAT, { encoding: 'utf8' }); + + // Write install_nni.sh, it's not used in Windows platform. + await fs.promises.writeFile(path.join(trialLocalTempFolder, executor.getScriptName("install_nni")), CONTAINER_INSTALL_NNI_SHELL_FORMAT, { encoding: 'utf8' }); // Write file content ( run.sh and parameter.cfg ) to local tmp files - await fs.promises.writeFile(path.join(trialLocalTempFolder, 'run.sh'), runScriptTrialContent, { encoding: 'utf8' }); + await fs.promises.writeFile(path.join(trialLocalTempFolder, executor.getScriptName("run")), runScriptTrialContent, { encoding: 'utf8' }); await this.writeParameterFile(trialJobId, form.hyperParameters); // Copy files in codeDir to remote working directory - await executor.copyDirectoryToRemote(trialLocalTempFolder, trialWorkingFolder, this.remoteOS); + await executor.copyDirectoryToRemote(trialLocalTempFolder, trialJobDetail.workingDirectory); // Execute command in remote machine - executor.executeScript(unixPathJoin(trialWorkingFolder, 'run.sh'), true, true); - } - - private getRmMetaByHost(host: string): RemoteMachineMeta { - for (const rmMeta of this.machineExecutorManagerMap.keys()) { - if (rmMeta.ip === host) { - return rmMeta; - } - } - throw new Error(`Host not found: ${host}`); + executor.executeScript(executor.joinPath(trialJobDetail.workingDirectory, executor.getScriptName("run")), true, true); } private async updateTrialJobStatus(trialJob: RemoteMachineTrialJobDetail, executor: ShellExecutor): Promise { const deferred: Deferred = new Deferred(); - const jobpidPath: string = this.getJobPidPath(trialJob.id); - const trialReturnCodeFilePath: string = unixPathJoin(this.remoteExpRootDir, 'trials', trialJob.id, '.nni', 'code'); + const jobpidPath: string = this.getJobPidPath(executor, trialJob.id); + const trialReturnCodeFilePath: string = executor.joinPath(executor.getRemoteExperimentRootDir(getExperimentId()), 'trials', trialJob.id, '.nni', 'code'); /* eslint-disable require-atomic-updates */ try { const isAlive = await executor.isProcessAlive(jobpidPath); @@ -617,7 +602,7 @@ class RemoteMachineTrainingService implements TrainingService { const trialReturnCode: string = await executor.getRemoteFileContent(trialReturnCodeFilePath); this.log.debug(`trailjob ${trialJob.id} return code: ${trialReturnCode}`); const match: RegExpMatchArray | null = trialReturnCode.trim() - .match(/^(\d+)\s+(\d+)$/); + .match(/^-?(\d+)\s+(\d+)$/); if (match !== null) { const { 1: code, 2: timestamp } = match; // Update trial job's status based on result code @@ -632,13 +617,13 @@ class RemoteMachineTrainingService implements TrainingService { } } trialJob.endTime = parseInt(timestamp, 10); - this.releaseTrialExecutor(trialJob); + this.releaseTrialResource(trialJob); } this.log.debug(`trailJob status update: ${trialJob.id}, ${trialJob.status}`); } deferred.resolve(trialJob); } catch (error) { - this.log.error(`Update job status exception, error is ${error.message}`); + this.log.debug(`(Ignorable mostly)Update job status exception, error is ${error.message}`); if (error instanceof NNIError && error.name === NNIErrorNames.NOT_FOUND) { deferred.resolve(trialJob); } else { @@ -650,45 +635,30 @@ class RemoteMachineTrainingService implements TrainingService { return deferred.promise; } - private getRemoteScriptsPath(userName: string): string { - return unixPathJoin(getRemoteTmpDir(this.remoteOS), userName, 'nni', 'scripts'); - } - - private getHostJobRemoteDir(jobId: string): string { - return unixPathJoin(this.remoteExpRootDir, 'hostjobs', jobId); - } - - private getRemoteExperimentRootDir(): string { - return unixPathJoin(getRemoteTmpDir(this.remoteOS), 'nni', 'experiments', getExperimentId()); - } - public get MetricsEmitter(): EventEmitter { return this.metricsEmitter; } - private getJobPidPath(jobId: string): string { + private getJobPidPath(executor: ShellExecutor, jobId: string): string { const trialJobDetail: RemoteMachineTrialJobDetail | undefined = this.trialJobsMap.get(jobId); if (trialJobDetail === undefined) { throw new NNIError(NNIErrorNames.INVALID_JOB_DETAIL, `Invalid job detail information for trial job ${jobId}`); } - return unixPathJoin(trialJobDetail.workingDirectory, '.nni', 'jobpid'); + return executor.joinPath(trialJobDetail.workingDirectory, '.nni', 'jobpid'); } private async writeParameterFile(trialJobId: string, hyperParameters: HyperParameters): Promise { - const executor: ShellExecutor | undefined = this.trialExecutorMap.get(trialJobId); - if (executor === undefined) { - throw new Error('ShellExecutor is undefined.'); - } + const executor = await this.getExecutor(trialJobId); - const trialWorkingFolder: string = unixPathJoin(this.remoteExpRootDir, 'trials', trialJobId); + const trialWorkingFolder: string = executor.joinPath(executor.getRemoteExperimentRootDir(getExperimentId()), 'trials', trialJobId); const trialLocalTempFolder: string = path.join(this.expRootDir, 'trials-local', trialJobId); const fileName: string = generateParamFileName(hyperParameters); const localFilepath: string = path.join(trialLocalTempFolder, fileName); await fs.promises.writeFile(localFilepath, hyperParameters.value, { encoding: 'utf8' }); - await executor.copyFileToRemote(localFilepath, unixPathJoin(trialWorkingFolder, fileName)); + await executor.copyFileToRemote(localFilepath, executor.joinPath(trialWorkingFolder, fileName)); } } diff --git a/src/nni_manager/training_service/remote_machine/shellExecutor.ts b/src/nni_manager/training_service/remote_machine/shellExecutor.ts index d35f94a239..deb9d26b9f 100644 --- a/src/nni_manager/training_service/remote_machine/shellExecutor.ts +++ b/src/nni_manager/training_service/remote_machine/shellExecutor.ts @@ -4,27 +4,39 @@ 'use strict'; import * as assert from 'assert'; +import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; -import * as fs from 'fs'; -import { Client, ClientChannel, SFTPWrapper, ConnectConfig } from 'ssh2'; -import { Deferred } from "ts-deferred"; -import { RemoteCommandResult, RemoteMachineMeta } from "./remoteMachineData"; +import { Client, ClientChannel, ConnectConfig, SFTPWrapper } from 'ssh2'; import * as stream from 'stream'; -import { OsCommands } from "./osCommands"; -import { LinuxCommands } from "./extends/linuxCommands"; +import { Deferred } from "ts-deferred"; import { getLogger, Logger } from '../../common/log'; -import { NNIError, NNIErrorNames } from '../../common/errors'; +import { uniqueString, randomInt } from '../../common/utils'; import { execRemove, tarAdd } from '../common/util'; -import { getRemoteTmpDir, uniqueString, unixPathJoin } from '../../common/utils'; +import { LinuxCommands } from "./extends/linuxCommands"; +import { WindowsCommands } from './extends/windowsCommands'; +import { OsCommands } from "./osCommands"; +import { RemoteCommandResult, RemoteMachineMeta } from "./remoteMachineData"; +import { NNIError, NNIErrorNames } from '../../common/errors'; class ShellExecutor { - private sshClient: Client = new Client(); - private osCommands: OsCommands | undefined; - private usedConnectionNumber: number = 0; //count the connection number of every client + public name: string = ""; - protected pathSpliter: string = '/'; - protected multiplePathSpliter: RegExp = new RegExp(`\\${this.pathSpliter}{2,}`); + private readonly lineBreaker = new RegExp(`[\r\n]+`); + private readonly maxUsageCount = 5; + + private osCommands: OsCommands | undefined; + private usedCount: number = 0; //count the connection number of every client + private readonly sshClient: Client; + private readonly log: Logger; + private tempPath: string = ""; + private isWindows: boolean = false; + private channelDefaultOutputs: string[] = []; + + constructor() { + this.log = getLogger(); + this.sshClient = new Client(); + } public async initialize(rmMeta: RemoteMachineMeta): Promise { const deferred: Deferred = new Deferred(); @@ -33,8 +45,9 @@ class ShellExecutor { host: rmMeta.ip, port: rmMeta.port, username: rmMeta.username, - tryKeyboard: true + tryKeyboard: true, }; + this.name = `${rmMeta.username}@${rmMeta.ip}:${rmMeta.port}`; if (rmMeta.passwd !== undefined) { connectConfig.password = rmMeta.passwd; } else if (rmMeta.sshKeyPath !== undefined) { @@ -49,20 +62,42 @@ class ShellExecutor { } else { deferred.reject(new Error(`No valid passwd or sshKeyPath is configed.`)); } + this.sshClient.on('ready', async () => { // check OS type: windows or else const result = await this.execute("ver"); if (result.exitCode == 0 && result.stdout.search("Windows") > -1) { - // not implement Windows commands yet. - throw new Error("not implement Windows commands yet."); + this.osCommands = new WindowsCommands(); + this.isWindows = true; + + // detect default output and trying to remove it under windows. + // Anaconda has this kind of output. + let defaultResult = await this.execute(""); + if (defaultResult.stdout !== "") { + deferred.reject(new Error(`The windows remote node shouldn't output welcome message, below content should be removed from the command window! \n` + + `${defaultResult.stdout}`)); + } + defaultResult = await this.execute("powershell -command \"\""); + if (defaultResult.stdout !== "") { + this.channelDefaultOutputs.push(defaultResult.stdout); + } + this.log.debug(`set channelDefaultOutput to "${this.channelDefaultOutputs}"`); + + // parse temp folder to expand possible environment variables. + const commandResult = await this.execute("echo %TEMP%"); + this.tempPath = commandResult.stdout.replace(this.lineBreaker, ""); } else { this.osCommands = new LinuxCommands(); + // it's not stable to get tmp path by Linux command, like "echo /tmp" or "ld -d /tmp". + // Sometime it returns empty back, so hard code tmp path here. + this.tempPath = "/tmp"; } + deferred.resolve(); }).on('error', (err: Error) => { // SSH connection error, reject with error message deferred.reject(new Error(err.message)); - }).on("keyboard-interactive", (name, instructions, lang, prompts, finish) => { + }).on("keyboard-interactive", (_name, _instructions, _lang, _prompts, finish) => { finish([rmMeta.passwd]); }).connect(connectConfig); @@ -73,43 +108,108 @@ class ShellExecutor { this.sshClient.end(); } - public get getUsedConnectionNumber(): number { - return this.usedConnectionNumber; + public addUsage(): boolean { + let isAddedSuccess = false; + if (this.usedCount < this.maxUsageCount) { + this.usedCount++; + isAddedSuccess = true; + } + return isAddedSuccess; + } + + public releaseUsage(): boolean { + let canBeReleased = false; + if (this.usedCount > 0) { + this.usedCount--; + } + if (this.usedCount == 0) { + canBeReleased = true; + } + return canBeReleased; + } + + public getScriptName(mainName: string): string { + if (this.osCommands === undefined) { + throw new Error("osCommands must be initialized!"); + } + return `${mainName}.${this.osCommands.getScriptExt()}`; + } + + public generateStartScript(workingDirectory: string, trialJobId: string, experimentId: string, + trialSequenceId: string, isMultiPhase: boolean, + command: string, nniManagerAddress: string, nniManagerPort: number, + nniManagerVersion: string, logCollection: string, cudaVisibleSetting: string): string { + if (this.osCommands === undefined) { + throw new Error("osCommands must be initialized!"); + } + const jobIdFileName = this.joinPath(workingDirectory, '.nni', 'jobpid'); + const exitCodeFile = this.joinPath(workingDirectory, '.nni', 'code'); + const codeDir = this.getRemoteCodePath(experimentId); + + return this.osCommands.generateStartScript(workingDirectory, trialJobId, experimentId, + trialSequenceId, isMultiPhase, jobIdFileName, command, + nniManagerAddress, nniManagerPort, nniManagerVersion, + logCollection, exitCodeFile, codeDir, cudaVisibleSetting); + } + + public generateGpuStatsScript(experimentId: string): string { + if (this.osCommands === undefined) { + throw new Error("osCommands must be initialized!"); + } + return this.osCommands.generateGpuStatsScript(this.getRemoteScriptsPath(experimentId)); + } + + public getTempPath(): string { + if (this.tempPath === "") { + throw new Error("tempPath must be initialized!"); + } + return this.tempPath; + } + + public getRemoteScriptsPath(experimentId: string): string { + return this.joinPath(this.getRemoteExperimentRootDir(experimentId), 'scripts'); + } + + public getRemoteCodePath(experimentId: string): string { + return this.joinPath(this.getRemoteExperimentRootDir(experimentId), 'nni-code'); } - public addUsedConnectionNumber(): void { - this.usedConnectionNumber += 1; + public getRemoteExperimentRootDir(experimentId: string): string { + return this.joinPath(this.tempPath, 'nni', 'experiments', experimentId); } - public minusUsedConnectionNumber(): void { - this.usedConnectionNumber -= 1; + public joinPath(...paths: string[]): string { + if (!this.osCommands) { + throw new Error("osCommands must be initialized!"); + } + return this.osCommands.joinPath(...paths); } public async createFolder(folderName: string, sharedFolder: boolean = false): Promise { const commandText = this.osCommands && this.osCommands.createFolder(folderName, sharedFolder); const commandResult = await this.execute(commandText); - const result = commandResult.exitCode >= 0; + const result = commandResult.exitCode == 0; return result; } public async allowPermission(isRecursive: boolean = false, ...folders: string[]): Promise { const commandText = this.osCommands && this.osCommands.allowPermission(isRecursive, ...folders); const commandResult = await this.execute(commandText); - const result = commandResult.exitCode >= 0; + const result = commandResult.exitCode == 0; return result; } public async removeFolder(folderName: string, isRecursive: boolean = false, isForce: boolean = true): Promise { const commandText = this.osCommands && this.osCommands.removeFolder(folderName, isRecursive, isForce); const commandResult = await this.execute(commandText); - const result = commandResult.exitCode >= 0; + const result = commandResult.exitCode == 0; return result; } public async removeFiles(folderOrFileName: string, filePattern: string = ""): Promise { const commandText = this.osCommands && this.osCommands.removeFiles(folderOrFileName, filePattern); const commandResult = await this.execute(commandText); - const result = commandResult.exitCode >= 0; + const result = commandResult.exitCode == 0; return result; } @@ -142,10 +242,10 @@ class ShellExecutor { return commandResult.exitCode == 0; } - public async executeScript(script: string, isFile: boolean, isInteractive: boolean = false): Promise { + public async executeScript(script: string, isFile: boolean = false, isInteractive: boolean = false): Promise { const commandText = this.osCommands && this.osCommands.executeScript(script, isFile); const commandResult = await this.execute(commandText, undefined, isInteractive); - return commandResult.exitCode == 0; + return commandResult; } /** @@ -154,13 +254,13 @@ class ShellExecutor { * @param remoteFilePath the target path in remote machine */ public async copyFileToRemote(localFilePath: string, remoteFilePath: string): Promise { - const log: Logger = getLogger(); - log.debug(`copyFileToRemote: localFilePath: ${localFilePath}, remoteFilePath: ${remoteFilePath}`); + const commandIndex = randomInt(10000); + this.log.debug(`copyFileToRemote(${commandIndex}): localFilePath: ${localFilePath}, remoteFilePath: ${remoteFilePath}`); const deferred: Deferred = new Deferred(); this.sshClient.sftp((err: Error, sftp: SFTPWrapper) => { if (err !== undefined && err !== null) { - log.error(`copyFileToRemote: ${err.message}, ${localFilePath}, ${remoteFilePath}`); + this.log.error(`copyFileToRemote(${commandIndex}): ${err}`); deferred.reject(err); return; @@ -169,6 +269,7 @@ class ShellExecutor { sftp.fastPut(localFilePath, remoteFilePath, (fastPutErr: Error) => { sftp.end(); if (fastPutErr !== undefined && fastPutErr !== null) { + this.log.error(`copyFileToRemote(${commandIndex}) fastPutErr: ${fastPutErr}, ${localFilePath}, ${remoteFilePath}`); deferred.reject(fastPutErr); } else { deferred.resolve(true); @@ -183,12 +284,15 @@ class ShellExecutor { * Copy files and directories in local directory recursively to remote directory * @param localDirectory local diretory * @param remoteDirectory remote directory - * @param remoteOS the OS of remote machine */ - public async copyDirectoryToRemote(localDirectory: string, remoteDirectory: string, remoteOS: string): Promise { + public async copyDirectoryToRemote(localDirectory: string, remoteDirectory: string): Promise { const tmpSuffix: string = uniqueString(5); const localTarPath: string = path.join(os.tmpdir(), `nni_tmp_local_${tmpSuffix}.tar.gz`); - const remoteTarPath: string = unixPathJoin(getRemoteTmpDir(remoteOS), `nni_tmp_remote_${tmpSuffix}.tar.gz`); + if (!this.osCommands) { + throw new Error("osCommands must be initialized!"); + } + const remoteTarPath: string = this.osCommands.joinPath(this.tempPath, `nni_tmp_remote_${tmpSuffix}.tar.gz`); + // Create remote directory await this.createFolder(remoteDirectory); // Compress files in local directory to experiment root directory @@ -202,12 +306,13 @@ class ShellExecutor { } public async getRemoteFileContent(filePath: string): Promise { + const commandIndex = randomInt(10000); + this.log.debug(`getRemoteFileContent(${commandIndex}): filePath: ${filePath}`); const deferred: Deferred = new Deferred(); this.sshClient.sftp((err: Error, sftp: SFTPWrapper) => { if (err !== undefined && err !== null) { - getLogger() - .error(`getRemoteFileContent: ${err.message}`); - deferred.reject(new Error(`SFTP error: ${err.message}`)); + this.log.error(`getRemoteFileContent(${commandIndex}) sftp: ${err}`); + deferred.reject(new Error(`SFTP error: ${err}`)); return; } @@ -228,8 +333,7 @@ class ShellExecutor { deferred.resolve(dataBuffer); }); } catch (error) { - getLogger() - .error(`getRemoteFileContent: ${error.message}`); + this.log.error(`getRemoteFileContent(${commandIndex}): ${error.message}`); sftp.end(); deferred.reject(new Error(`SFTP error: ${error.message}`)); } @@ -239,16 +343,20 @@ class ShellExecutor { } private async execute(command: string | undefined, processOutput: ((input: RemoteCommandResult) => RemoteCommandResult) | undefined = undefined, useShell: boolean = false): Promise { - const log: Logger = getLogger(); - log.debug(`remoteExeCommand: command: [${command}]`); const deferred: Deferred = new Deferred(); let stdout: string = ''; let stderr: string = ''; let exitCode: number; + const commandIndex = randomInt(10000); + this.log.debug(`remoteExeCommand(${commandIndex}): [${command}]`); + + // Windows always uses shell, and it needs to disable to get it works. + useShell = useShell && !this.isWindows; + const callback = (err: Error, channel: ClientChannel): void => { if (err !== undefined && err !== null) { - log.error(`remoteExeCommand: ${err.message}`); + this.log.error(`remoteExeCommand(${commandIndex}): ${err.message}`); deferred.reject(err); return; } @@ -258,7 +366,23 @@ class ShellExecutor { }); channel.on('exit', (code: any) => { exitCode = code; - log.debug(`remoteExeCommand exit(${exitCode})\nstdout: ${stdout}\nstderr: ${stderr}`); + + // remove default output to get stdout correct. + if (this.channelDefaultOutputs.length > 0) { + let modifiedStdout = stdout; + this.channelDefaultOutputs.forEach(defaultOutput => { + if (modifiedStdout.startsWith(defaultOutput)) { + if (modifiedStdout.length > defaultOutput.length) { + modifiedStdout = modifiedStdout.substr(defaultOutput.length); + } else if (modifiedStdout.length === defaultOutput.length) { + modifiedStdout = ""; + } + } + }); + stdout = modifiedStdout; + } + + this.log.debug(`remoteExeCommand(${commandIndex}) exit(${exitCode})\nstdout: ${stdout}\nstderr: ${stderr}`); let result = { stdout: stdout, stderr: stderr, @@ -270,7 +394,7 @@ class ShellExecutor { } deferred.resolve(result); }); - channel.stderr.on('data', function (data) { + channel.stderr.on('data', function (data: any) { stderr += data; }); diff --git a/src/nni_manager/training_service/remote_machine/test/linuxCommands.test.ts b/src/nni_manager/training_service/remote_machine/test/linuxCommands.test.ts index ffe89cbc4f..ee7d1904a9 100644 --- a/src/nni_manager/training_service/remote_machine/test/linuxCommands.test.ts +++ b/src/nni_manager/training_service/remote_machine/test/linuxCommands.test.ts @@ -8,7 +8,6 @@ import * as chaiAsPromised from 'chai-as-promised'; import * as component from '../../../common/component'; import { cleanupUnitTest, prepareUnitTest } from '../../../common/utils'; import { LinuxCommands } from '../extends/linuxCommands'; -// import { TrialConfigMetadataKey } from '../trialConfigMetadataKey'; describe('Unit Test for linuxCommands', () => { @@ -88,10 +87,6 @@ describe('Unit Test for linuxCommands', () => { )).to.equal(false); }) - it('killChildProcesses', async () => { - chai.expect(linuxCommands.killChildProcesses("test")).to.equal("pkill -P `cat 'test'`"); - }) - it('extractFile', async () => { chai.expect(linuxCommands.extractFile("test.tar", "testfolder")).to.equal("tar -oxzf 'test.tar' -C 'testfolder'"); }) diff --git a/src/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts b/src/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts index fb8b6bbf2b..4e9d9ffb68 100644 --- a/src/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts +++ b/src/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts @@ -8,29 +8,29 @@ import * as fs from 'fs'; import * as chai from 'chai'; import * as chaiAsPromised from 'chai-as-promised'; -import { Client } from 'ssh2'; import { ShellExecutor } from '../shellExecutor'; import { prepareUnitTest, cleanupUnitTest } from '../../../common/utils'; -const LOCALFILE: string = '/tmp/localSshclientUTData'; -const REMOTEFILE: string = '/tmp/remoteSshclientUTData'; -const REMOTEFOLDER: string = '/tmp/remoteSshclientUTFolder'; +const LOCALFILE: string = 'localSshUTData'; +const REMOTEFILE: string = 'remoteSshUTData'; +const REMOTEFOLDER: string = 'remoteSshUTFolder'; async function copyFile(executor: ShellExecutor): Promise { - await executor.copyFileToRemote(LOCALFILE, REMOTEFILE); + const remoteFullName = executor.joinPath(executor.getTempPath(), REMOTEFILE); + await executor.copyFileToRemote(LOCALFILE, remoteFullName); } async function copyFileToRemoteLoop(executor: ShellExecutor): Promise { - for (let i: number = 0; i < 10; i++) { - // console.log(i); - await executor.copyFileToRemote(LOCALFILE, REMOTEFILE); + const remoteFullName = executor.joinPath(executor.getTempPath(), REMOTEFILE); + for (let i: number = 0; i < 3; i++) { + await executor.copyFileToRemote(LOCALFILE, remoteFullName); } } async function getRemoteFileContentLoop(executor: ShellExecutor): Promise { - for (let i: number = 0; i < 10; i++) { - // console.log(i); - await executor.getRemoteFileContent(REMOTEFILE); + const remoteFullName = executor.joinPath(executor.getTempPath(), REMOTEFILE); + for (let i: number = 0; i < 3; i++) { + await executor.getRemoteFileContent(remoteFullName); } } @@ -41,14 +41,16 @@ describe('ShellExecutor test', () => { rmMeta = JSON.parse(fs.readFileSync('../../.vscode/rminfo.json', 'utf8')); console.log(rmMeta); } catch (err) { - console.log(`Please configure rminfo.json to enable remote machine test.${err}`); + console.log(`Please configure rminfo.json to enable remote machine test. ${err}`); skip = true; } before(async () => { chai.should(); chai.use(chaiAsPromised); - await cpp.exec(`echo '1234' > ${LOCALFILE}`); + if (!fs.existsSync(LOCALFILE)){ + await cpp.exec(`echo '1234' > ${LOCALFILE}`); + } prepareUnitTest(); }); @@ -61,26 +63,27 @@ describe('ShellExecutor test', () => { if (skip) { return; } - const shellExecutor: ShellExecutor = new ShellExecutor(); - await shellExecutor.initialize(rmMeta); - let result = await shellExecutor.createFolder(REMOTEFOLDER, false); + const executor: ShellExecutor = new ShellExecutor(); + await executor.initialize(rmMeta); + const remoteFullPath = executor.joinPath(executor.getTempPath(), REMOTEFOLDER); + let result = await executor.createFolder(remoteFullPath, false); chai.expect(result).eq(true); - result = await shellExecutor.removeFolder(REMOTEFOLDER); + const commandResult = await executor.executeScript("dir"); + chai.expect(commandResult.exitCode).eq(0); + result = await executor.removeFolder(remoteFullPath); chai.expect(result).eq(true); + await executor.close(); }); it('Test ShellExecutor', async () => { if (skip) { return; } - const shellExecutor: ShellExecutor = new ShellExecutor(); - await shellExecutor.initialize(rmMeta); - await copyFile(shellExecutor); - await Promise.all([ - copyFileToRemoteLoop(shellExecutor), - copyFileToRemoteLoop(shellExecutor), - copyFileToRemoteLoop(shellExecutor), - getRemoteFileContentLoop(shellExecutor) - ]); + const executor: ShellExecutor = new ShellExecutor(); + await executor.initialize(rmMeta); + await copyFile(executor); + await copyFileToRemoteLoop(executor); + await getRemoteFileContentLoop(executor); + await executor.close(); }); }); diff --git a/src/nni_manager/training_service/remote_machine/test/windowsCommands.test.ts b/src/nni_manager/training_service/remote_machine/test/windowsCommands.test.ts new file mode 100644 index 0000000000..2f2408697a --- /dev/null +++ b/src/nni_manager/training_service/remote_machine/test/windowsCommands.test.ts @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +'use strict'; + +import * as chai from 'chai'; +import * as chaiAsPromised from 'chai-as-promised'; +import * as component from '../../../common/component'; +import { cleanupUnitTest, prepareUnitTest } from '../../../common/utils'; +import { WindowsCommands } from '../extends/windowsCommands'; + + +describe('Unit Test for Windows Commands', () => { + + let windowsCommands: WindowsCommands + + before(() => { + chai.should(); + chai.use(chaiAsPromised); + prepareUnitTest(); + }); + + after(() => { + cleanupUnitTest(); + }); + + beforeEach(() => { + windowsCommands = component.get(WindowsCommands); + }); + + afterEach(() => { + }); + + it('joinPath', async () => { + chai.expect(windowsCommands.joinPath("/root/", "\\first")).to.equal("\\root\\first"); + chai.expect(windowsCommands.joinPath("root/", "first")).to.equal("root\\first"); + chai.expect(windowsCommands.joinPath("\\root/", "\\first")).to.equal("\\root\\first"); + chai.expect(windowsCommands.joinPath("\\root\\", "\\first")).to.equal("\\root\\first"); + chai.expect(windowsCommands.joinPath("\\root", "first")).to.equal("\\root\\first"); + chai.expect(windowsCommands.joinPath("\\root\\", "first")).to.equal("\\root\\first"); + chai.expect(windowsCommands.joinPath("root\\", "first")).to.equal("root\\first"); + chai.expect(windowsCommands.joinPath("root\\")).to.equal("root\\"); + chai.expect(windowsCommands.joinPath("root")).to.equal("root"); + chai.expect(windowsCommands.joinPath(".\\root")).to.equal(".\\root"); + chai.expect(windowsCommands.joinPath("")).to.equal("."); + chai.expect(windowsCommands.joinPath("..")).to.equal(".."); + }) + + it('createFolder', async () => { + chai.expect(windowsCommands.createFolder("test")).to.equal("mkdir \"test\""); + chai.expect(windowsCommands.createFolder("test", true)).to.equal("mkdir \"test\"\r\nICACLS \"test\" /grant \"Users\":F"); + }) + + it('allowPermission', async () => { + chai.expect(windowsCommands.allowPermission(true, "test", "test1")).to.equal("ICACLS \"test\" /grant \"Users\":F /T\r\nICACLS \"test1\" /grant \"Users\":F /T\r\n"); + chai.expect(windowsCommands.allowPermission(false, "test")).to.equal("ICACLS \"test\" /grant \"Users\":F\r\n"); + }) + + it('removeFolder', async () => { + chai.expect(windowsCommands.removeFolder("test")).to.equal("rmdir /q \"test\""); + chai.expect(windowsCommands.removeFolder("test", true)).to.equal("rmdir /s /q \"test\""); + chai.expect(windowsCommands.removeFolder("test", true, false)).to.equal("rmdir /s \"test\""); + chai.expect(windowsCommands.removeFolder("test", false, false)).to.equal("rmdir \"test\""); + chai.expect(windowsCommands.removeFolder("test", true, true)).to.equal("rmdir /s /q \"test\""); + }) + + it('removeFiles', async () => { + chai.expect(windowsCommands.removeFiles("test", "*.sh")).to.equal("del \"test\\*.sh\""); + chai.expect(windowsCommands.removeFiles("test", "")).to.equal("del \"test\""); + }) + + it('readLastLines', async () => { + chai.expect(windowsCommands.readLastLines("test", 3)).to.equal("powershell.exe Get-Content \"test\" -Tail 3"); + }) + + it('isProcessAlive', async () => { + chai.expect(windowsCommands.isProcessAliveCommand("test")).to.equal("powershell.exe Get-Process -Id (get-content \"test\") -ErrorAction SilentlyContinue"); + chai.expect(windowsCommands.isProcessAliveProcessOutput( + { + exitCode: 0, + stdout: "", + stderr: "" + } + )).to.equal(true); + chai.expect(windowsCommands.isProcessAliveProcessOutput( + { + exitCode: 10, + stdout: "", + stderr: "" + } + )).to.equal(false); + }) + + it('extractFile', async () => { + chai.expect(windowsCommands.extractFile("test.tar", "testfolder")).to.equal("tar -xf \"test.tar\" -C \"testfolder\""); + }) + + it('executeScript', async () => { + chai.expect(windowsCommands.executeScript("test.sh", true)).to.equal("test.sh"); + chai.expect(windowsCommands.executeScript("test script'\"", false)).to.equal("test script'\""); + }) +}); diff --git a/test/config/examples/mnist-annotation.yml b/test/config/examples/mnist-annotation.yml index 17d28684e6..330400570d 100644 --- a/test/config/examples/mnist-annotation.yml +++ b/test/config/examples/mnist-annotation.yml @@ -13,7 +13,6 @@ assessor: trial: codeDir: ../../../examples/trials/mnist-annotation command: python3 mnist.py --batch_num 10 - gpuNum: 0 useAnnotation: true multiPhase: false diff --git a/test/config/examples/mnist-keras.yml b/test/config/examples/mnist-keras.yml index 3743be301e..6bb9e0e999 100644 --- a/test/config/examples/mnist-keras.yml +++ b/test/config/examples/mnist-keras.yml @@ -14,7 +14,6 @@ assessor: trial: codeDir: ../../../examples/trials/mnist-keras command: python3 mnist-keras.py --num_train 200 --epochs 1 - gpuNum: 0 useAnnotation: false multiPhase: false diff --git a/test/config/examples/mnist-nested-search-space.yml b/test/config/examples/mnist-nested-search-space.yml index 9be7c8ef7e..89e51a180a 100644 --- a/test/config/examples/mnist-nested-search-space.yml +++ b/test/config/examples/mnist-nested-search-space.yml @@ -15,7 +15,6 @@ assessor: trial: codeDir: ../../../examples/trials/mnist-nested-search-space command: python3 mnist.py --batch_num 10 - gpuNum: 0 useAnnotation: false multiPhase: false diff --git a/test/config/examples/mnist-pytorch.yml b/test/config/examples/mnist-pytorch.yml index c62f0579d4..570d9de81f 100644 --- a/test/config/examples/mnist-pytorch.yml +++ b/test/config/examples/mnist-pytorch.yml @@ -14,7 +14,6 @@ assessor: trial: codeDir: ../../../examples/trials/mnist-pytorch command: python3 mnist.py --epochs 1 --batch_num 10 - gpuNum: 0 useAnnotation: false multiPhase: false diff --git a/test/config/examples/mnist-tfv1.yml b/test/config/examples/mnist-tfv1.yml index f66e288efc..f8393918ad 100644 --- a/test/config/examples/mnist-tfv1.yml +++ b/test/config/examples/mnist-tfv1.yml @@ -14,7 +14,6 @@ assessor: trial: codeDir: ../../../examples/trials/mnist-tfv1 command: python3 mnist.py --batch_num 10 - gpuNum: 0 useAnnotation: false multiPhase: false diff --git a/test/config/integration_tests.yml b/test/config/integration_tests.yml index b89e44285b..c6c5b44fa3 100644 --- a/test/config/integration_tests.yml +++ b/test/config/integration_tests.yml @@ -1,6 +1,6 @@ defaultTestCaseConfig: - launchCommand: nnictl create --config $configFile + launchCommand: nnictl create --config $configFile --debug stopCommand: nnictl stop experimentStatusCheck: True platform: linux darwin win32 @@ -22,7 +22,7 @@ testCases: validator: # launch command, default launch command is 'nnictl create --config $configFile' - launchCommand: nnictl create --config $configFile + launchCommand: nnictl create --config $configFile --debug # stop command, default stop command is 'nnictl stop', empty means no stop command stopCommand: nnictl stop @@ -38,15 +38,24 @@ testCases: - name: mnist-tfv1 configFile: test/config/examples/mnist-tfv1.yml + config: + maxTrialNum: 1 + trialConcurrency: 1 - name: mnist-keras configFile: test/config/examples/mnist-keras.yml + config: + maxTrialNum: 2 + trialConcurrency: 1 - name: mnist-pytorch configFile: test/config/examples/mnist-pytorch.yml - name: mnist-annotation configFile: test/config/examples/mnist-annotation.yml + config: + maxTrialNum: 1 + trialConcurrency: 1 - name: cifar10-pytorch configFile: test/config/examples/cifar10-pytorch.yml diff --git a/test/nni_test/nnitest/naive_test.py b/test/nni_test/nnitest/naive_test.py index cedd8b4ad4..b998686960 100644 --- a/test/nni_test/nnitest/naive_test.py +++ b/test/nni_test/nnitest/naive_test.py @@ -10,7 +10,7 @@ import time import traceback -from utils import is_experiment_done, get_experiment_id, get_nni_log_path, read_last_line, remove_files, setup_experiment, detect_port, snooze +from utils import is_experiment_done, get_experiment_id, get_nni_log_path, read_last_line, remove_files, setup_experiment, detect_port, wait_for_port_available from utils import GREEN, RED, CLEAR, EXPERIMENT_URL NNI_SOURCE_DIR = '..' @@ -71,7 +71,7 @@ def naive_test(args): assert assessor_result == expected, 'Bad assessor result' subprocess.run(['nnictl', 'stop']) - snooze() + wait_for_port_available(8080, 10) def stop_experiment_test(args): config_file = args.config @@ -86,19 +86,20 @@ def stop_experiment_test(args): experiment_id = get_experiment_id(EXPERIMENT_URL) proc = subprocess.run(['nnictl', 'stop', experiment_id]) assert proc.returncode == 0, '`nnictl stop %s` failed with code %d' % (experiment_id, proc.returncode) - snooze() + wait_for_port_available(8080, 10) assert not detect_port(8080), '`nnictl stop %s` failed to stop experiments' % experiment_id # test cmd `nnictl stop --port` proc = subprocess.run(['nnictl', 'stop', '--port', '8990']) assert proc.returncode == 0, '`nnictl stop %s` failed with code %d' % (experiment_id, proc.returncode) - snooze() + wait_for_port_available(8990, 10) assert not detect_port(8990), '`nnictl stop %s` failed to stop experiments' % experiment_id # test cmd `nnictl stop --all` proc = subprocess.run(['nnictl', 'stop', '--all']) assert proc.returncode == 0, '`nnictl stop --all` failed with code %d' % proc.returncode - snooze() + wait_for_port_available(8888, 10) + wait_for_port_available(8989, 10) assert not detect_port(8888) and not detect_port(8989), '`nnictl stop --all` failed to stop experiments' diff --git a/test/nni_test/nnitest/run_tests.py b/test/nni_test/nnitest/run_tests.py index 96334176bf..d817eb4465 100644 --- a/test/nni_test/nnitest/run_tests.py +++ b/test/nni_test/nnitest/run_tests.py @@ -15,7 +15,7 @@ from utils import get_experiment_status, get_yml_content, dump_yml_content, get_experiment_id, \ parse_max_duration_time, get_trial_stats, deep_update, print_trial_job_log, get_failed_trial_jobs, \ get_experiment_dir, print_experiment_log -from utils import GREEN, RED, CLEAR, STATUS_URL, TRIAL_JOBS_URL, EXPERIMENT_URL, REST_ENDPOINT, detect_port +from utils import GREEN, RED, CLEAR, STATUS_URL, TRIAL_JOBS_URL, EXPERIMENT_URL, REST_ENDPOINT, wait_for_port_available import validators it_variables = {} @@ -157,7 +157,7 @@ def launch_test(config_file, training_service, test_case_config): if num_failed > 0: print('failed jobs: ', num_failed) break - time.sleep(3) + time.sleep(1) except: print_experiment_log(experiment_id=experiment_id) raise @@ -189,16 +189,6 @@ def case_included(name, cases): return True return False -def wait_for_port_available(port, timeout): - begin_time = time.time() - while True: - if not detect_port(port): - return - if time.time() - begin_time > timeout: - msg = 'port {} is not available in {} seconds.'.format(port, timeout) - raise RuntimeError(msg) - time.sleep(5) - def match_platform(test_case_config): return sys.platform in test_case_config['platform'].split(' ') diff --git a/test/nni_test/nnitest/utils.py b/test/nni_test/nnitest/utils.py index e362b0de07..81031500c1 100644 --- a/test/nni_test/nnitest/utils.py +++ b/test/nni_test/nnitest/utils.py @@ -168,6 +168,13 @@ def detect_port(port): except: return False -def snooze(): - '''Sleep to make sure previous stopped exp has enough time to exit''' - time.sleep(6) + +def wait_for_port_available(port, timeout): + begin_time = time.time() + while True: + if not detect_port(port): + return + if time.time() - begin_time > timeout: + msg = 'port {} is not available in {} seconds.'.format(port, timeout) + raise RuntimeError(msg) + time.sleep(1) diff --git a/test/pipelines/pipelines-it-remote-linux-to-windows.yml b/test/pipelines/pipelines-it-remote-linux-to-windows.yml new file mode 100644 index 0000000000..613db986fc --- /dev/null +++ b/test/pipelines/pipelines-it-remote-linux-to-windows.yml @@ -0,0 +1,48 @@ +jobs: + - job: "integration_test_remote_linux_to_windows" + timeoutInMinutes: 120 + steps: + - script: make clean + displayName: "clean nni source code" + - task: CopyFilesOverSSH@0 + inputs: + sshEndpoint: $(end_point) + contents: | + ** + !**/dist/** + !**/node_modules/** + targetFolder: /tmp/nnitest/$(Build.BuildId) + overwrite: true + displayName: "Copy all files to remote machine" + timeoutInMinutes: 10 + - task: SSH@0 + inputs: + sshEndpoint: $(end_point) + runOptions: commands + commands: cd "\tmp\nnitest\$(Build.BuildId)" && powershell.exe -command "conda activate l2w | .\uninstall.ps1 | .\install.ps1" + failOnStdErr: false + displayName: "install on remote windows" + - script: python3 -m pip install --upgrade pip setuptools --user + displayName: "Install python tools" + - script: make easy-install + displayName: "Install nni via source code" + - script: | + sudo apt-get install swig -y + PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC + PATH=$HOME/.local/bin:$PATH nnictl package install --name=BOHB + displayName: "Install dependencies for integration tests in remote mode" + - script: | + set -e + cd test + python3 nni_test/nnitest/generate_ts_config.py --ts remote --remote_user $(remote_user) --remote_host $(remote_host) \ + --remote_port $(remote_port) --remote_pwd $(remote_pwd) --nni_manager_ip $(nni_manager_ip) + cat config/training_service.yml + PATH=$HOME/.local/bin:$PATH python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote + displayName: "integration test" + - task: SSH@0 + inputs: + sshEndpoint: $(end_point) + runOptions: commands + commands: rmdir /s /q "\\?\c:\tmp\nnitest\$(Build.BuildId)" + condition: always() + displayName: "clean up on remote server" diff --git a/tools/nni_cmd/launcher.py b/tools/nni_cmd/launcher.py index 00067352ea..6ea6eb348f 100644 --- a/tools/nni_cmd/launcher.py +++ b/tools/nni_cmd/launcher.py @@ -139,7 +139,9 @@ def set_remote_config(experiment_config, port, config_file_name): for i in range(len(request_data['machine_list'])): if isinstance(request_data['machine_list'][i].get('gpuIndices'), int): request_data['machine_list'][i]['gpuIndices'] = str(request_data['machine_list'][i].get('gpuIndices')) - response = rest_put(cluster_metadata_url(port), json.dumps(request_data), REST_TIME_OUT) + # It needs to connect all remote machines, the time out of connection is 30 seconds. + # So timeout of this place should be longer. + response = rest_put(cluster_metadata_url(port), json.dumps(request_data), 60, True) err_message = '' if not response or not check_response(response): if response is not None: diff --git a/tools/nni_cmd/nnictl_utils.py b/tools/nni_cmd/nnictl_utils.py index df3922a801..7bccc1085a 100644 --- a/tools/nni_cmd/nnictl_utils.py +++ b/tools/nni_cmd/nnictl_utils.py @@ -227,7 +227,7 @@ def stop_experiment(args): experiment_config = Experiments() experiment_dict = experiment_config.get_all_experiments() for experiment_id in experiment_id_list: - print_normal('Stoping experiment %s' % experiment_id) + print_normal('Stopping experiment %s' % experiment_id) nni_config = Config(experiment_dict[experiment_id]['fileName']) rest_pid = nni_config.get_config('restServerPid') if rest_pid: diff --git a/tools/nni_trial_tool/constants.py b/tools/nni_trial_tool/constants.py index 69d0449036..bef4013370 100644 --- a/tools/nni_trial_tool/constants.py +++ b/tools/nni_trial_tool/constants.py @@ -7,8 +7,6 @@ BASE_URL = 'http://{}' -HOME_DIR = os.path.join(os.environ['HOME'], 'nni') - LOG_DIR = os.environ['NNI_OUTPUT_DIR'] NNI_PLATFORM = os.environ['NNI_PLATFORM'] diff --git a/tools/nni_trial_tool/trial_keeper.py b/tools/nni_trial_tool/trial_keeper.py index 6da3a7bfc2..10ee7af211 100644 --- a/tools/nni_trial_tool/trial_keeper.py +++ b/tools/nni_trial_tool/trial_keeper.py @@ -2,23 +2,27 @@ # Licensed under the MIT license. import argparse -import os -from subprocess import Popen -import time +import ctypes +import json import logging -import shlex +import os import re +import shlex import sys -import json import threading -from pyhdfs import HdfsClient +import time +from subprocess import Popen + import pkg_resources -from .rest_utils import rest_post, rest_get -from .url_utils import gen_send_version_url, gen_parameter_meta_url +from pyhdfs import HdfsClient -from .constants import LOG_DIR, NNI_PLATFORM, MULTI_PHASE, NNI_TRIAL_JOB_ID, NNI_SYS_DIR, NNI_EXP_ID -from .hdfsClientUtility import copyDirectoryToHdfs, copyHdfsDirectoryToLocal, copyHdfsFileToLocal -from .log_utils import LogType, nni_log, RemoteLogger, StdOutputType +from .constants import (LOG_DIR, MULTI_PHASE, NNI_EXP_ID, NNI_PLATFORM, + NNI_SYS_DIR, NNI_TRIAL_JOB_ID) +from .hdfsClientUtility import (copyDirectoryToHdfs, copyHdfsDirectoryToLocal, + copyHdfsFileToLocal) +from .log_utils import LogType, RemoteLogger, StdOutputType, nni_log +from .rest_utils import rest_get, rest_post +from .url_utils import gen_parameter_meta_url, gen_send_version_url logger = logging.getLogger('trial_keeper') regular = re.compile('v?(?P[0-9](\.[0-9]){0,1}).*') @@ -80,6 +84,10 @@ def main_loop(args): if hdfs_client is not None: copyHdfsDirectoryToLocal(args.nni_hdfs_exp_dir, os.getcwd(), hdfs_client) + if args.job_id_file: + with open(args.job_id_file, 'w') as job_file: + job_file.write("%d" % os.getpid()) + # Notice: We don't appoint env, which means subprocess wil inherit current environment and that is expected behavior log_pipe_stdout = trial_syslogger_stdout.get_pipelog_reader() process = Popen(args.trial_command, shell=True, stdout=log_pipe_stdout, stderr=log_pipe_stdout) @@ -91,6 +99,9 @@ def main_loop(args): retCode = process.poll() # child worker process exits and all stdout data is read if retCode is not None and log_pipe_stdout.set_process_exit() and log_pipe_stdout.is_read_completed == True: + # In Windows, the retCode -1 is 4294967295. It's larger than c_long, and raise OverflowError. + # So covert it to int32. + retCode = ctypes.c_long(retCode).value nni_log(LogType.Info, 'subprocess terminated. Exit code is {}. Quit'.format(retCode)) if hdfs_output_dir is not None: # Copy local directory to hdfs for OpenPAI @@ -218,6 +229,7 @@ def run(self): PARSER.add_argument('--webhdfs_path', type=str, help='the webhdfs path used in webhdfs URL') PARSER.add_argument('--nni_manager_version', type=str, help='the nni version transmitted from nniManager') PARSER.add_argument('--log_collection', type=str, help='set the way to collect log in trialkeeper') + PARSER.add_argument('--job_id_file', type=str, help='set job id file for operating and monitoring job.') args, unknown = PARSER.parse_known_args() if args.trial_command is None: exit(1) From 41d02d84e6b055383ebdeb145cf305833e3df5bd Mon Sep 17 00:00:00 2001 From: JSong-Jia <60089275+JSong-Jia@users.noreply.github.com> Date: Wed, 20 May 2020 10:42:02 +0800 Subject: [PATCH 14/14] Update README.md (#2465) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b67efc1ce..27a33c5f18 100644 --- a/README.md +++ b/README.md @@ -342,7 +342,7 @@ With authors' permission, we listed a set of NNI usage examples and relevant art Join IM discussion groups: |Gitter||WeChat| |----|----|----| -|![image](https://user-images.githubusercontent.com/39592018/80665738-e0574a80-8acc-11ea-91bc-0836dc4cbf89.png)| OR |![image](https://github.com/JSong-Jia/NNI-user-group/blob/master/user%20group%20code_0512.jpg)| +|![image](https://user-images.githubusercontent.com/39592018/80665738-e0574a80-8acc-11ea-91bc-0836dc4cbf89.png)| OR |![image](https://github.com/JSong-Jia/NNI-user-group/blob/master/user%20group%20code_0512.png)| ## Related Projects