From abbf679ee676928bc6649c819046056a8fda6bef Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Mon, 1 Nov 2021 15:08:56 -0700 Subject: [PATCH] Updating Deploy Scripts to Properly Validate Config JSON (#1410) * Creating InstanceConfig Attributes for NSG Refactor (#1331) * Updating instance_config * Updating attribute names. * Updating list factory. * Updating config attributes. Co-authored-by: nharper285 * NSG deployment on a creation of new debug/repro proxy. (#1340) Co-authored-by: stas * Build fix (#1374) * Bump reqwest from 0.11.4 to 0.11.5 in /src/proxy-manager (#1336) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.4 to 0.11.5. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.4...v0.11.5) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump reqwest from 0.11.4 to 0.11.5 in /src/agent (#1335) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.4 to 0.11.5. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.4...v0.11.5) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Work around for newly-upgraded pip breaking pip-licenses (#1346) * Work around for newly upgrdaded pip breaking pip-licenses (can be reverted once https://github.com/raimon49/pip-licenses/issues/113 is fixed) * Update .github/workflows/ci.yml Co-authored-by: Joe Ranweiler Co-authored-by: stas Co-authored-by: Joe Ranweiler * Bump iced-x86 from 1.14.0 to 1.15.0 in /src/agent (#1337) Bumps [iced-x86](https://github.com/icedland/iced) from 1.14.0 to 1.15.0. - [Release notes](https://github.com/icedland/iced/releases) - [Commits](https://github.com/icedland/iced/compare/v1.14.0...v1.15.0) --- updated-dependencies: - dependency-name: iced-x86 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * revert pip breaking pip-licenses workaround (#1348) Co-authored-by: stas * Bump thiserror from 1.0.29 to 1.0.30 in /src/proxy-manager (#1341) Bumps [thiserror](https://github.com/dtolnay/thiserror) from 1.0.29 to 1.0.30. - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/1.0.29...1.0.30) --- updated-dependencies: - dependency-name: thiserror dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump thiserror from 1.0.29 to 1.0.30 in /src/agent (#1342) Bumps [thiserror](https://github.com/dtolnay/thiserror) from 1.0.29 to 1.0.30. - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/1.0.29...1.0.30) --- updated-dependencies: - dependency-name: thiserror dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump strum from 0.21.0 to 0.22.0 in /src/agent (#1343) Bumps [strum](https://github.com/Peternator7/strum) from 0.21.0 to 0.22.0. - [Release notes](https://github.com/Peternator7/strum/releases) - [Changelog](https://github.com/Peternator7/strum/blob/master/CHANGELOG.md) - [Commits](https://github.com/Peternator7/strum/commits) --- updated-dependencies: - dependency-name: strum dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump azure cli to 2.27.2 (#1355) * Bump azure cli to 2.27.2 * fixing up add-corpus-storage-account script Co-authored-by: stas * Bump azure-identity to 1.6.1 (#1356) Co-authored-by: stas * Bump strum_macros from 0.21.1 to 0.22.0 in /src/agent (#1344) Bumps [strum_macros](https://github.com/Peternator7/strum) from 0.21.1 to 0.22.0. - [Release notes](https://github.com/Peternator7/strum/releases) - [Changelog](https://github.com/Peternator7/strum/blob/master/CHANGELOG.md) - [Commits](https://github.com/Peternator7/strum/commits) --- updated-dependencies: - dependency-name: strum_macros dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc Greisen * Bump sysinfo from 0.20.4 to 0.20.5 in /src/agent (#1353) Bumps [sysinfo](https://github.com/GuillaumeGomez/sysinfo) from 0.20.4 to 0.20.5. - [Release notes](https://github.com/GuillaumeGomez/sysinfo/releases) - [Changelog](https://github.com/GuillaumeGomez/sysinfo/blob/master/CHANGELOG.md) - [Commits](https://github.com/GuillaumeGomez/sysinfo/commits) --- updated-dependencies: - dependency-name: sysinfo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Release 3.2.0 (#1361) * Release 3.2.0 * Added python dependencies * Update CHANGELOG.md Co-authored-by: Cheick Keita * Update CHANGELOG.md Co-authored-by: Joe Ranweiler * Update grammar * Update CHANGELOG.md Co-authored-by: Joe Ranweiler Co-authored-by: Cheick Keita Co-authored-by: Joe Ranweiler * Temporarily ignore non-actionable `cargo audit` errors (#1365) * Azure DevOps notifications not appearing (#1370) Co-authored-by: stas * Bump procfs from 0.10.1 to 0.11.0 in /src/agent (#1360) Bumps [procfs](https://github.com/eminence/procfs) from 0.10.1 to 0.11.0. - [Release notes](https://github.com/eminence/procfs/releases) - [Commits](https://github.com/eminence/procfs/compare/v0.10.1...v0.11.0) --- updated-dependencies: - dependency-name: procfs dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc Greisen * Bump structopt from 0.3.23 to 0.3.25 in /src/agent (#1364) Bumps [structopt](https://github.com/TeXitoi/structopt) from 0.3.23 to 0.3.25. - [Release notes](https://github.com/TeXitoi/structopt/releases) - [Changelog](https://github.com/TeXitoi/structopt/blob/master/CHANGELOG.md) - [Commits](https://github.com/TeXitoi/structopt/compare/v0.3.23...v0.3.25) --- updated-dependencies: - dependency-name: structopt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump reqwest from 0.11.5 to 0.11.6 in /src/proxy-manager (#1367) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.5 to 0.11.6. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.5...v0.11.6) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: stas Co-authored-by: Joe Ranweiler Co-authored-by: Marc Greisen Co-authored-by: Cheick Keita Co-authored-by: Joe Ranweiler * Delete NSG if no resources associated with it (#1358) Co-authored-by: stas * Update NSGs after changes to instance config (#1385) * Refactor set_admins into configuration.py and update deployment params with nsg_config * Fixing arguments. * Param takes in network config json * Fixing Client in deploy * removing import * Adding onefuzztypes to reqs.txt * Reverting to single list * Removing imports. * Retriggering build * Setting specific pip version for local testing. * Removing imports? * More imports. * Fixing formatting. * Updating how to parse nsg param. * Removing old logging statements. * Fixing types. * REmoving bad log * Removing local pip version. * Removing comments * fixing * Formatting * Fixing .split() * Adding NSG rule checks and type. * Formatting. * Formatting. * Removing imports. * Fixing formatting. * Testing formatting. * Retrigger? * New InstanceConfigClient class. * Retrigger. * Cherry picked commit. * Reformatting. * Actually fixing formatting. * Fixing table_service call. * Fixing return statement and nsg_rule pass. * Full config. * Removing commented out code. * Fixing logic. * Adding wildcard check. * Code for updating NSGs when instance_config updated. * Updating argument to set_allowed_rules * Updating model to no longer be optional. * Fixing args for set_allowed_rules * trying to fix calls to get_nsg * Updating calls to nsg lib * Fixing imports. * Updating calls to set_allowed and creating constructor for NSGConfig type. * Removing constructor and manually setting default ip * Fixing models. * Hopefully fixing docs. * Fix set_allowed call * Adding error handling for update config. * Changing to error check. * Fixing error call. * Fixing imports. * Adding empty() function on request. * Removing empty function. # Conflicts: # src/pytypes/onefuzztypes/models.py * Fixing files for update. * Fixing nsg.py. * Fixing imports. * removing commented code. Co-authored-by: nharper285 * Aligning feature branch with main (#1389) * Bump reqwest from 0.11.4 to 0.11.5 in /src/proxy-manager (#1336) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.4 to 0.11.5. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.4...v0.11.5) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump reqwest from 0.11.4 to 0.11.5 in /src/agent (#1335) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.4 to 0.11.5. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.4...v0.11.5) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Work around for newly-upgraded pip breaking pip-licenses (#1346) * Work around for newly upgrdaded pip breaking pip-licenses (can be reverted once https://github.com/raimon49/pip-licenses/issues/113 is fixed) * Update .github/workflows/ci.yml Co-authored-by: Joe Ranweiler Co-authored-by: stas Co-authored-by: Joe Ranweiler * Bump iced-x86 from 1.14.0 to 1.15.0 in /src/agent (#1337) Bumps [iced-x86](https://github.com/icedland/iced) from 1.14.0 to 1.15.0. - [Release notes](https://github.com/icedland/iced/releases) - [Commits](https://github.com/icedland/iced/compare/v1.14.0...v1.15.0) --- updated-dependencies: - dependency-name: iced-x86 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * revert pip breaking pip-licenses workaround (#1348) Co-authored-by: stas * Bump thiserror from 1.0.29 to 1.0.30 in /src/proxy-manager (#1341) Bumps [thiserror](https://github.com/dtolnay/thiserror) from 1.0.29 to 1.0.30. - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/1.0.29...1.0.30) --- updated-dependencies: - dependency-name: thiserror dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump thiserror from 1.0.29 to 1.0.30 in /src/agent (#1342) Bumps [thiserror](https://github.com/dtolnay/thiserror) from 1.0.29 to 1.0.30. - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/1.0.29...1.0.30) --- updated-dependencies: - dependency-name: thiserror dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump strum from 0.21.0 to 0.22.0 in /src/agent (#1343) Bumps [strum](https://github.com/Peternator7/strum) from 0.21.0 to 0.22.0. - [Release notes](https://github.com/Peternator7/strum/releases) - [Changelog](https://github.com/Peternator7/strum/blob/master/CHANGELOG.md) - [Commits](https://github.com/Peternator7/strum/commits) --- updated-dependencies: - dependency-name: strum dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump azure cli to 2.27.2 (#1355) * Bump azure cli to 2.27.2 * fixing up add-corpus-storage-account script Co-authored-by: stas * Bump azure-identity to 1.6.1 (#1356) Co-authored-by: stas * Bump strum_macros from 0.21.1 to 0.22.0 in /src/agent (#1344) Bumps [strum_macros](https://github.com/Peternator7/strum) from 0.21.1 to 0.22.0. - [Release notes](https://github.com/Peternator7/strum/releases) - [Changelog](https://github.com/Peternator7/strum/blob/master/CHANGELOG.md) - [Commits](https://github.com/Peternator7/strum/commits) --- updated-dependencies: - dependency-name: strum_macros dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc Greisen * Bump sysinfo from 0.20.4 to 0.20.5 in /src/agent (#1353) Bumps [sysinfo](https://github.com/GuillaumeGomez/sysinfo) from 0.20.4 to 0.20.5. - [Release notes](https://github.com/GuillaumeGomez/sysinfo/releases) - [Changelog](https://github.com/GuillaumeGomez/sysinfo/blob/master/CHANGELOG.md) - [Commits](https://github.com/GuillaumeGomez/sysinfo/commits) --- updated-dependencies: - dependency-name: sysinfo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Release 3.2.0 (#1361) * Release 3.2.0 * Added python dependencies * Update CHANGELOG.md Co-authored-by: Cheick Keita * Update CHANGELOG.md Co-authored-by: Joe Ranweiler * Update grammar * Update CHANGELOG.md Co-authored-by: Joe Ranweiler Co-authored-by: Cheick Keita Co-authored-by: Joe Ranweiler * Temporarily ignore non-actionable `cargo audit` errors (#1365) * Azure DevOps notifications not appearing (#1370) Co-authored-by: stas * Bump procfs from 0.10.1 to 0.11.0 in /src/agent (#1360) Bumps [procfs](https://github.com/eminence/procfs) from 0.10.1 to 0.11.0. - [Release notes](https://github.com/eminence/procfs/releases) - [Commits](https://github.com/eminence/procfs/compare/v0.10.1...v0.11.0) --- updated-dependencies: - dependency-name: procfs dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc Greisen * Bump structopt from 0.3.23 to 0.3.25 in /src/agent (#1364) Bumps [structopt](https://github.com/TeXitoi/structopt) from 0.3.23 to 0.3.25. - [Release notes](https://github.com/TeXitoi/structopt/releases) - [Changelog](https://github.com/TeXitoi/structopt/blob/master/CHANGELOG.md) - [Commits](https://github.com/TeXitoi/structopt/compare/v0.3.23...v0.3.25) --- updated-dependencies: - dependency-name: structopt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump reqwest from 0.11.5 to 0.11.6 in /src/proxy-manager (#1367) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.5 to 0.11.6. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.5...v0.11.6) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump reqwest from 0.11.5 to 0.11.6 in /src/agent (#1368) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.5 to 0.11.6. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.5...v0.11.6) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump crossterm from 0.21.0 to 0.22.1 in /src/agent (#1369) Bumps [crossterm](https://github.com/crossterm-rs/crossterm) from 0.21.0 to 0.22.1. - [Release notes](https://github.com/crossterm-rs/crossterm/releases) - [Changelog](https://github.com/crossterm-rs/crossterm/blob/master/CHANGELOG.md) - [Commits](https://github.com/crossterm-rs/crossterm/commits) --- updated-dependencies: - dependency-name: crossterm dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc Greisen * Fix validation of `target_exe` blob name (#1371) * NSG Updated After CLI Update to Instance_Config (#1375) * Creating InstanceConfig Attributes for NSG Refactor (#1331) * Updating instance_config * Updating attribute names. * Updating list factory. * Updating config attributes. Co-authored-by: nharper285 * NSG deployment on a creation of new debug/repro proxy. (#1340) Co-authored-by: stas * Code for updating NSGs when instance_config updated. * Updating argument to set_allowed_rules * Temporarily ignore non-actionable `cargo audit` errors (#1365) * Updating model to no longer be optional. * Fixing args for set_allowed_rules * trying to fix calls to get_nsg * Updating calls to nsg lib * Fixing imports. * Updating calls to set_allowed and creating constructor for NSGConfig type. * Removing constructor and manually setting default ip * Fixing models. * Hopefully fixing docs. * Fix set_allowed call * Adding error handling for update config. * Changing to error check. * Fixing error call. * Fixing imports. * Updating instanceconfig retrieval. * Fixing imports. * Adding empty() function on request. * Fixing name of function. * Removing empty function. Co-authored-by: nharper285 Co-authored-by: Stas Co-authored-by: stas Co-authored-by: Joe Ranweiler * Revert "NSG Updated After CLI Update to Instance_Config (#1375)" (#1384) This reverts commit 357bc4fcadc0bae9886e53ba1a9a316365e41094. * Bump backtrace from 0.3.61 to 0.3.62 in /src/agent (#1382) Bumps [backtrace](https://github.com/rust-lang/backtrace-rs) from 0.3.61 to 0.3.62. - [Release notes](https://github.com/rust-lang/backtrace-rs/releases) - [Commits](https://github.com/rust-lang/backtrace-rs/compare/0.3.61...0.3.62) --- updated-dependencies: - dependency-name: backtrace dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc Greisen * Set compiler env vars to effect Win10 SDK downgrade (#1388) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: stas Co-authored-by: Joe Ranweiler Co-authored-by: Marc Greisen Co-authored-by: Cheick Keita Co-authored-by: Joe Ranweiler Co-authored-by: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Co-authored-by: nharper285 * Creating InstanceConfig Attributes for NSG Refactor (#1331) * Updating instance_config * Updating attribute names. * Updating list factory. * Updating config attributes. Co-authored-by: nharper285 * NSG deployment on a creation of new debug/repro proxy. (#1340) Co-authored-by: stas * Delete NSG if no resources associated with it (#1358) Co-authored-by: stas * Update NSGs after changes to instance config (#1385) * Refactor set_admins into configuration.py and update deployment params with nsg_config * Fixing arguments. * Param takes in network config json * Fixing Client in deploy * removing import * Adding onefuzztypes to reqs.txt * Reverting to single list * Removing imports. * Retriggering build * Setting specific pip version for local testing. * Removing imports? * More imports. * Fixing formatting. * Updating how to parse nsg param. * Removing old logging statements. * Fixing types. * REmoving bad log * Removing local pip version. * Removing comments * fixing * Formatting * Fixing .split() * Adding NSG rule checks and type. * Formatting. * Formatting. * Removing imports. * Fixing formatting. * Testing formatting. * Retrigger? * New InstanceConfigClient class. * Retrigger. * Cherry picked commit. * Reformatting. * Actually fixing formatting. * Fixing table_service call. * Fixing return statement and nsg_rule pass. * Full config. * Removing commented out code. * Fixing logic. * Adding wildcard check. * Code for updating NSGs when instance_config updated. * Updating argument to set_allowed_rules * Updating model to no longer be optional. * Fixing args for set_allowed_rules * trying to fix calls to get_nsg * Updating calls to nsg lib * Fixing imports. * Updating calls to set_allowed and creating constructor for NSGConfig type. * Removing constructor and manually setting default ip * Fixing models. * Hopefully fixing docs. * Fix set_allowed call * Adding error handling for update config. * Changing to error check. * Fixing error call. * Fixing imports. * Adding empty() function on request. * Removing empty function. # Conflicts: # src/pytypes/onefuzztypes/models.py * Fixing files for update. * Fixing nsg.py. * Fixing imports. * removing commented code. Co-authored-by: nharper285 * associate subnets with NSG (#1393) * associate subnets with NSG change NSG rule protocol to ANY * subnet wait * Improve NSG update logic Co-authored-by: stas * Update configuration.py to check for 'block all' configuration. (#1394) * Creating InstanceConfig Attributes for NSG Refactor (#1331) * Updating instance_config * Updating attribute names. * Updating list factory. * Updating config attributes. Co-authored-by: nharper285 * NSG deployment on a creation of new debug/repro proxy. (#1340) Co-authored-by: stas * Build fix (#1374) * Bump reqwest from 0.11.4 to 0.11.5 in /src/proxy-manager (#1336) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.4 to 0.11.5. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.4...v0.11.5) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump reqwest from 0.11.4 to 0.11.5 in /src/agent (#1335) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.4 to 0.11.5. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.4...v0.11.5) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Work around for newly-upgraded pip breaking pip-licenses (#1346) * Work around for newly upgrdaded pip breaking pip-licenses (can be reverted once https://github.com/raimon49/pip-licenses/issues/113 is fixed) * Update .github/workflows/ci.yml Co-authored-by: Joe Ranweiler Co-authored-by: stas Co-authored-by: Joe Ranweiler * Bump iced-x86 from 1.14.0 to 1.15.0 in /src/agent (#1337) Bumps [iced-x86](https://github.com/icedland/iced) from 1.14.0 to 1.15.0. - [Release notes](https://github.com/icedland/iced/releases) - [Commits](https://github.com/icedland/iced/compare/v1.14.0...v1.15.0) --- updated-dependencies: - dependency-name: iced-x86 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * revert pip breaking pip-licenses workaround (#1348) Co-authored-by: stas * Bump thiserror from 1.0.29 to 1.0.30 in /src/proxy-manager (#1341) Bumps [thiserror](https://github.com/dtolnay/thiserror) from 1.0.29 to 1.0.30. - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/1.0.29...1.0.30) --- updated-dependencies: - dependency-name: thiserror dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump thiserror from 1.0.29 to 1.0.30 in /src/agent (#1342) Bumps [thiserror](https://github.com/dtolnay/thiserror) from 1.0.29 to 1.0.30. - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/1.0.29...1.0.30) --- updated-dependencies: - dependency-name: thiserror dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump strum from 0.21.0 to 0.22.0 in /src/agent (#1343) Bumps [strum](https://github.com/Peternator7/strum) from 0.21.0 to 0.22.0. - [Release notes](https://github.com/Peternator7/strum/releases) - [Changelog](https://github.com/Peternator7/strum/blob/master/CHANGELOG.md) - [Commits](https://github.com/Peternator7/strum/commits) --- updated-dependencies: - dependency-name: strum dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump azure cli to 2.27.2 (#1355) * Bump azure cli to 2.27.2 * fixing up add-corpus-storage-account script Co-authored-by: stas * Bump azure-identity to 1.6.1 (#1356) Co-authored-by: stas * Bump strum_macros from 0.21.1 to 0.22.0 in /src/agent (#1344) Bumps [strum_macros](https://github.com/Peternator7/strum) from 0.21.1 to 0.22.0. - [Release notes](https://github.com/Peternator7/strum/releases) - [Changelog](https://github.com/Peternator7/strum/blob/master/CHANGELOG.md) - [Commits](https://github.com/Peternator7/strum/commits) --- updated-dependencies: - dependency-name: strum_macros dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc Greisen * Bump sysinfo from 0.20.4 to 0.20.5 in /src/agent (#1353) Bumps [sysinfo](https://github.com/GuillaumeGomez/sysinfo) from 0.20.4 to 0.20.5. - [Release notes](https://github.com/GuillaumeGomez/sysinfo/releases) - [Changelog](https://github.com/GuillaumeGomez/sysinfo/blob/master/CHANGELOG.md) - [Commits](https://github.com/GuillaumeGomez/sysinfo/commits) --- updated-dependencies: - dependency-name: sysinfo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Release 3.2.0 (#1361) * Release 3.2.0 * Added python dependencies * Update CHANGELOG.md Co-authored-by: Cheick Keita * Update CHANGELOG.md Co-authored-by: Joe Ranweiler * Update grammar * Update CHANGELOG.md Co-authored-by: Joe Ranweiler Co-authored-by: Cheick Keita Co-authored-by: Joe Ranweiler * Temporarily ignore non-actionable `cargo audit` errors (#1365) * Azure DevOps notifications not appearing (#1370) Co-authored-by: stas * Bump procfs from 0.10.1 to 0.11.0 in /src/agent (#1360) Bumps [procfs](https://github.com/eminence/procfs) from 0.10.1 to 0.11.0. - [Release notes](https://github.com/eminence/procfs/releases) - [Commits](https://github.com/eminence/procfs/compare/v0.10.1...v0.11.0) --- updated-dependencies: - dependency-name: procfs dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc Greisen * Bump structopt from 0.3.23 to 0.3.25 in /src/agent (#1364) Bumps [structopt](https://github.com/TeXitoi/structopt) from 0.3.23 to 0.3.25. - [Release notes](https://github.com/TeXitoi/structopt/releases) - [Changelog](https://github.com/TeXitoi/structopt/blob/master/CHANGELOG.md) - [Commits](https://github.com/TeXitoi/structopt/compare/v0.3.23...v0.3.25) --- updated-dependencies: - dependency-name: structopt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump reqwest from 0.11.5 to 0.11.6 in /src/proxy-manager (#1367) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.5 to 0.11.6. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.5...v0.11.6) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: stas Co-authored-by: Joe Ranweiler Co-authored-by: Marc Greisen Co-authored-by: Cheick Keita Co-authored-by: Joe Ranweiler * Delete NSG if no resources associated with it (#1358) Co-authored-by: stas * Update NSGs after changes to instance config (#1385) * Refactor set_admins into configuration.py and update deployment params with nsg_config * Fixing arguments. * Param takes in network config json * Fixing Client in deploy * removing import * Adding onefuzztypes to reqs.txt * Reverting to single list * Removing imports. * Retriggering build * Setting specific pip version for local testing. * Removing imports? * More imports. * Fixing formatting. * Updating how to parse nsg param. * Removing old logging statements. * Fixing types. * REmoving bad log * Removing local pip version. * Removing comments * fixing * Formatting * Fixing .split() * Adding NSG rule checks and type. * Formatting. * Formatting. * Removing imports. * Fixing formatting. * Testing formatting. * Retrigger? * New InstanceConfigClient class. * Retrigger. * Cherry picked commit. * Reformatting. * Actually fixing formatting. * Fixing table_service call. * Fixing return statement and nsg_rule pass. * Full config. * Removing commented out code. * Fixing logic. * Adding wildcard check. * Code for updating NSGs when instance_config updated. * Updating argument to set_allowed_rules * Updating model to no longer be optional. * Fixing args for set_allowed_rules * trying to fix calls to get_nsg * Updating calls to nsg lib * Fixing imports. * Updating calls to set_allowed and creating constructor for NSGConfig type. * Removing constructor and manually setting default ip * Fixing models. * Hopefully fixing docs. * Fix set_allowed call * Adding error handling for update config. * Changing to error check. * Fixing error call. * Fixing imports. * Adding empty() function on request. * Removing empty function. # Conflicts: # src/pytypes/onefuzztypes/models.py * Fixing files for update. * Fixing nsg.py. * Fixing imports. * removing commented code. Co-authored-by: nharper285 * Aligning feature branch with main (#1389) * Bump reqwest from 0.11.4 to 0.11.5 in /src/proxy-manager (#1336) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.4 to 0.11.5. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.4...v0.11.5) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump reqwest from 0.11.4 to 0.11.5 in /src/agent (#1335) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.4 to 0.11.5. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.4...v0.11.5) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Work around for newly-upgraded pip breaking pip-licenses (#1346) * Work around for newly upgrdaded pip breaking pip-licenses (can be reverted once https://github.com/raimon49/pip-licenses/issues/113 is fixed) * Update .github/workflows/ci.yml Co-authored-by: Joe Ranweiler Co-authored-by: stas Co-authored-by: Joe Ranweiler * Bump iced-x86 from 1.14.0 to 1.15.0 in /src/agent (#1337) Bumps [iced-x86](https://github.com/icedland/iced) from 1.14.0 to 1.15.0. - [Release notes](https://github.com/icedland/iced/releases) - [Commits](https://github.com/icedland/iced/compare/v1.14.0...v1.15.0) --- updated-dependencies: - dependency-name: iced-x86 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * revert pip breaking pip-licenses workaround (#1348) Co-authored-by: stas * Bump thiserror from 1.0.29 to 1.0.30 in /src/proxy-manager (#1341) Bumps [thiserror](https://github.com/dtolnay/thiserror) from 1.0.29 to 1.0.30. - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/1.0.29...1.0.30) --- updated-dependencies: - dependency-name: thiserror dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump thiserror from 1.0.29 to 1.0.30 in /src/agent (#1342) Bumps [thiserror](https://github.com/dtolnay/thiserror) from 1.0.29 to 1.0.30. - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/1.0.29...1.0.30) --- updated-dependencies: - dependency-name: thiserror dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump strum from 0.21.0 to 0.22.0 in /src/agent (#1343) Bumps [strum](https://github.com/Peternator7/strum) from 0.21.0 to 0.22.0. - [Release notes](https://github.com/Peternator7/strum/releases) - [Changelog](https://github.com/Peternator7/strum/blob/master/CHANGELOG.md) - [Commits](https://github.com/Peternator7/strum/commits) --- updated-dependencies: - dependency-name: strum dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump azure cli to 2.27.2 (#1355) * Bump azure cli to 2.27.2 * fixing up add-corpus-storage-account script Co-authored-by: stas * Bump azure-identity to 1.6.1 (#1356) Co-authored-by: stas * Bump strum_macros from 0.21.1 to 0.22.0 in /src/agent (#1344) Bumps [strum_macros](https://github.com/Peternator7/strum) from 0.21.1 to 0.22.0. - [Release notes](https://github.com/Peternator7/strum/releases) - [Changelog](https://github.com/Peternator7/strum/blob/master/CHANGELOG.md) - [Commits](https://github.com/Peternator7/strum/commits) --- updated-dependencies: - dependency-name: strum_macros dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc Greisen * Bump sysinfo from 0.20.4 to 0.20.5 in /src/agent (#1353) Bumps [sysinfo](https://github.com/GuillaumeGomez/sysinfo) from 0.20.4 to 0.20.5. - [Release notes](https://github.com/GuillaumeGomez/sysinfo/releases) - [Changelog](https://github.com/GuillaumeGomez/sysinfo/blob/master/CHANGELOG.md) - [Commits](https://github.com/GuillaumeGomez/sysinfo/commits) --- updated-dependencies: - dependency-name: sysinfo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Release 3.2.0 (#1361) * Release 3.2.0 * Added python dependencies * Update CHANGELOG.md Co-authored-by: Cheick Keita * Update CHANGELOG.md Co-authored-by: Joe Ranweiler * Update grammar * Update CHANGELOG.md Co-authored-by: Joe Ranweiler Co-authored-by: Cheick Keita Co-authored-by: Joe Ranweiler * Temporarily ignore non-actionable `cargo audit` errors (#1365) * Azure DevOps notifications not appearing (#1370) Co-authored-by: stas * Bump procfs from 0.10.1 to 0.11.0 in /src/agent (#1360) Bumps [procfs](https://github.com/eminence/procfs) from 0.10.1 to 0.11.0. - [Release notes](https://github.com/eminence/procfs/releases) - [Commits](https://github.com/eminence/procfs/compare/v0.10.1...v0.11.0) --- updated-dependencies: - dependency-name: procfs dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc Greisen * Bump structopt from 0.3.23 to 0.3.25 in /src/agent (#1364) Bumps [structopt](https://github.com/TeXitoi/structopt) from 0.3.23 to 0.3.25. - [Release notes](https://github.com/TeXitoi/structopt/releases) - [Changelog](https://github.com/TeXitoi/structopt/blob/master/CHANGELOG.md) - [Commits](https://github.com/TeXitoi/structopt/compare/v0.3.23...v0.3.25) --- updated-dependencies: - dependency-name: structopt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump reqwest from 0.11.5 to 0.11.6 in /src/proxy-manager (#1367) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.5 to 0.11.6. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.5...v0.11.6) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump reqwest from 0.11.5 to 0.11.6 in /src/agent (#1368) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.5 to 0.11.6. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.5...v0.11.6) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump crossterm from 0.21.0 to 0.22.1 in /src/agent (#1369) Bumps [crossterm](https://github.com/crossterm-rs/crossterm) from 0.21.0 to 0.22.1. - [Release notes](https://github.com/crossterm-rs/crossterm/releases) - [Changelog](https://github.com/crossterm-rs/crossterm/blob/master/CHANGELOG.md) - [Commits](https://github.com/crossterm-rs/crossterm/commits) --- updated-dependencies: - dependency-name: crossterm dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc Greisen * Fix validation of `target_exe` blob name (#1371) * NSG Updated After CLI Update to Instance_Config (#1375) * Creating InstanceConfig Attributes for NSG Refactor (#1331) * Updating instance_config * Updating attribute names. * Updating list factory. * Updating config attributes. Co-authored-by: nharper285 * NSG deployment on a creation of new debug/repro proxy. (#1340) Co-authored-by: stas * Code for updating NSGs when instance_config updated. * Updating argument to set_allowed_rules * Temporarily ignore non-actionable `cargo audit` errors (#1365) * Updating model to no longer be optional. * Fixing args for set_allowed_rules * trying to fix calls to get_nsg * Updating calls to nsg lib * Fixing imports. * Updating calls to set_allowed and creating constructor for NSGConfig type. * Removing constructor and manually setting default ip * Fixing models. * Hopefully fixing docs. * Fix set_allowed call * Adding error handling for update config. * Changing to error check. * Fixing error call. * Fixing imports. * Updating instanceconfig retrieval. * Fixing imports. * Adding empty() function on request. * Fixing name of function. * Removing empty function. Co-authored-by: nharper285 Co-authored-by: Stas Co-authored-by: stas Co-authored-by: Joe Ranweiler * Revert "NSG Updated After CLI Update to Instance_Config (#1375)" (#1384) This reverts commit 357bc4fcadc0bae9886e53ba1a9a316365e41094. * Bump backtrace from 0.3.61 to 0.3.62 in /src/agent (#1382) Bumps [backtrace](https://github.com/rust-lang/backtrace-rs) from 0.3.61 to 0.3.62. - [Release notes](https://github.com/rust-lang/backtrace-rs/releases) - [Commits](https://github.com/rust-lang/backtrace-rs/compare/0.3.61...0.3.62) --- updated-dependencies: - dependency-name: backtrace dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc Greisen * Set compiler env vars to effect Win10 SDK downgrade (#1388) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: stas Co-authored-by: Joe Ranweiler Co-authored-by: Marc Greisen Co-authored-by: Cheick Keita Co-authored-by: Joe Ranweiler Co-authored-by: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Co-authored-by: nharper285 * Updating configuration.py to check for 'block all' config. * Fixing error message. * associate subnets with NSG (#1393) * associate subnets with NSG change NSG rule protocol to ANY * subnet wait * Improve NSG update logic Co-authored-by: stas Co-authored-by: nharper285 Co-authored-by: Stas Co-authored-by: stas Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Ranweiler Co-authored-by: Marc Greisen Co-authored-by: Cheick Keita Co-authored-by: Joe Ranweiler * Add functional tests to validate NSG feature (#1395) * Tests Newly created scaleset proxy are behind NSG #1325 Apply existing rules to newly created scaleset in a new region #1326 * addressing PR comments * added README.md Co-authored-by: stas * Updating Threat Model with NSG * Updating deploy.py to check json. * Updating configuration.py to validate json. * Fixing other files. * Updated with new class that validates json. * Fixing logic. * Fixing logic. * Fixing typo. * Bug fixing. * Fixing formatting. * Updating deployment scripts w/ comments. * Adding extra check to parse_json * adding tests. * Working through import issues. * Functional Unit Tests and updated configuration script. * Updating test_deploy_config * Restructing deployment directory to support testing. * Removing files. * Adding .gitignore * Lint errors. * Updates to deploy tests and scripts. Co-authored-by: nharper285 Co-authored-by: Stas Co-authored-by: stas Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Ranweiler Co-authored-by: Marc Greisen Co-authored-by: Cheick Keita Co-authored-by: Joe Ranweiler --- src/deployment/.gitignore | 73 +++++++++++++++ src/deployment/deploy.py | 21 ++--- src/deployment/deploylib/__init__.py | 0 .../{ => deploylib}/configuration.py | 74 +++++++++++++-- .../{ => deploylib}/data_migration.py | 0 .../{ => deploylib}/registration.py | 0 src/deployment/{ => deploylib}/set_admins.py | 2 +- src/deployment/deploylib/tests/README.md | 7 ++ src/deployment/deploylib/tests/__init__.py | 4 + .../deploylib/tests/test_deploy_config.py | 90 +++++++++++++++++++ 10 files changed, 252 insertions(+), 19 deletions(-) create mode 100644 src/deployment/.gitignore create mode 100644 src/deployment/deploylib/__init__.py rename src/deployment/{ => deploylib}/configuration.py (65%) rename src/deployment/{ => deploylib}/data_migration.py (100%) rename src/deployment/{ => deploylib}/registration.py (100%) rename src/deployment/{ => deploylib}/set_admins.py (97%) create mode 100644 src/deployment/deploylib/tests/README.md create mode 100644 src/deployment/deploylib/tests/__init__.py create mode 100644 src/deployment/deploylib/tests/test_deploy_config.py diff --git a/src/deployment/.gitignore b/src/deployment/.gitignore new file mode 100644 index 0000000000..21ef600e0c --- /dev/null +++ b/src/deployment/.gitignore @@ -0,0 +1,73 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# pyenv +.python-version + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# mypy +.mypy_cache/ + +test-cov.xml +test-output.xml \ No newline at end of file diff --git a/src/deployment/deploy.py b/src/deployment/deploy.py index 6b40bfeba8..251d27a362 100644 --- a/src/deployment/deploy.py +++ b/src/deployment/deploy.py @@ -47,15 +47,16 @@ ) from msrest.serialization import TZ_UTC -from configuration import ( +from deploylib.configuration import ( InstanceConfigClient, + NetworkSecurityConfig, parse_rules, update_admins, update_allowed_aad_tenants, update_nsg, ) -from data_migration import migrate -from registration import ( +from deploylib.data_migration import migrate +from deploylib.registration import ( GraphQueryError, OnefuzzAppRole, add_application_password, @@ -624,18 +625,18 @@ def set_instance_config(self) -> None: with open(self.nsg_config, "r") as template_handle: config_template = json.load(template_handle) - if ( - not config_template["proxy_nsg_config"] - and not config_template["proxy_nsg_config"]["allowed_ips"] - and not config_template["proxy_nsg_config"]["allowed_service_tags"] - ): + try: + config = NetworkSecurityConfig(config_template) + rules = parse_rules(config) + except Exception as ex: + logging.info( + "An Exception was encountered while parsing nsg_config file: %s", ex + ) raise Exception( "proxy_nsg_config and sub-values were not properly included in config." + "Please submit a configuration resembling" + " { 'proxy_nsg_config': { 'allowed_ips': [], 'allowed_service_tags': [] } }" ) - proxy_config = config_template["proxy_nsg_config"] - rules = parse_rules(proxy_config) update_nsg(config_client, rules) diff --git a/src/deployment/deploylib/__init__.py b/src/deployment/deploylib/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/deployment/configuration.py b/src/deployment/deploylib/configuration.py similarity index 65% rename from src/deployment/configuration.py rename to src/deployment/deploylib/configuration.py index 43861c7b4a..cdeecd0767 100644 --- a/src/deployment/configuration.py +++ b/src/deployment/deploylib/configuration.py @@ -6,7 +6,7 @@ import ipaddress import json import logging -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional from uuid import UUID from azure.cosmosdb.table.tableservice import TableService @@ -49,6 +49,67 @@ def create_if_missing(self, table_service: TableService) -> None: self.enable_storage_client_logging() +class NetworkSecurityConfig: + allowed_ips: List[str] + allowed_service_tags: List[str] + + def __init__(self, config: Any): + self.parse_nsg_json(config) + + def parse_nsg_json(self, config: Any) -> None: + if not isinstance(config, Dict): + raise Exception( + "Configuration is not a Dictionary. Please provide Valid Config." + ) + if len(config.keys()) == 0: + raise Exception( + "Empty Configuration File Provided. Please Provide Valid Config." + ) + if None in config.keys() or "proxy_nsg_config" not in config.keys(): + raise Exception( + "proxy_nsg_config not provided as valid key. Please Provide Valid Config." + ) + proxy_config = config["proxy_nsg_config"] + if not isinstance(proxy_config, Dict): + raise Exception( + "Inner Configuration is not a Dictionary. Please provide Valid Config." + ) + if len(proxy_config.keys()) == 0: + raise Exception( + "Empty Inner Configuration File Provided. Please Provide Valid Config." + ) + if ( + None in proxy_config.keys() + or "allowed_ips" not in proxy_config.keys() + or "allowed_service_tags" not in proxy_config.keys() + ): + raise Exception( + "allowed_ips and allowed_service_tags not provided. Please Provide Valid Config." + ) + if not isinstance(proxy_config["allowed_ips"], List) or not isinstance( + proxy_config["allowed_service_tags"], List + ): + raise Exception( + "allowed_ips and allowed_service_tags are not a list. Please Provide Valid Config." + ) + + if ( + len(proxy_config["allowed_ips"]) != 0 + and not all(isinstance(x, str) for x in proxy_config["allowed_ips"]) + ) or ( + len(proxy_config["allowed_ips"]) != 0 + and not all( + isinstance(x, str) for x in proxy_config["allowed_service_tags"] + ) + ): + raise Exception( + "allowed_ips and allowed_service_tags are not a list of strings. Please Provide Valid Config." + ) + + self.allowed_ips = proxy_config["allowed_ips"] + self.allowed_service_tags = proxy_config["allowed_service_tags"] + + class NsgRule: rule: str @@ -117,9 +178,10 @@ def update_admins(config_client: InstanceConfigClient, admins: List[UUID]) -> No ) -def parse_rules(proxy_config: Dict[str, str]) -> List[NsgRule]: - allowed_ips = proxy_config["allowed_ips"] - allowed_service_tags = proxy_config["allowed_service_tags"] +def parse_rules(proxy_config: NetworkSecurityConfig) -> List[NsgRule]: + + allowed_ips = proxy_config.allowed_ips + allowed_service_tags = proxy_config.allowed_service_tags nsg_rules = [] if "*" in allowed_ips: @@ -166,7 +228,3 @@ def update_nsg( "proxy_nsg_config": json.dumps(nsg_config), }, ) - - -if __name__ == "__main__": - pass diff --git a/src/deployment/data_migration.py b/src/deployment/deploylib/data_migration.py similarity index 100% rename from src/deployment/data_migration.py rename to src/deployment/deploylib/data_migration.py diff --git a/src/deployment/registration.py b/src/deployment/deploylib/registration.py similarity index 100% rename from src/deployment/registration.py rename to src/deployment/deploylib/registration.py diff --git a/src/deployment/set_admins.py b/src/deployment/deploylib/set_admins.py similarity index 97% rename from src/deployment/set_admins.py rename to src/deployment/deploylib/set_admins.py index 20eea27d49..92e67d92bc 100644 --- a/src/deployment/set_admins.py +++ b/src/deployment/deploylib/set_admins.py @@ -10,7 +10,7 @@ from azure.cosmosdb.table.tableservice import TableService from azure.mgmt.storage import StorageManagementClient -from configuration import ( +from deploylib.configuration import ( InstanceConfigClient, update_admins, update_allowed_aad_tenants, diff --git a/src/deployment/deploylib/tests/README.md b/src/deployment/deploylib/tests/README.md new file mode 100644 index 0000000000..2867e6cb73 --- /dev/null +++ b/src/deployment/deploylib/tests/README.md @@ -0,0 +1,7 @@ +# Deployment tests. + +- `test_deploy_config.py` + + Requires the deployment packages and scripts from OneFuzz. + + `test_deploy_config.py` validates that the Deploy script takes properly formatted json configuration.. diff --git a/src/deployment/deploylib/tests/__init__.py b/src/deployment/deploylib/tests/__init__.py new file mode 100644 index 0000000000..d2effe12b0 --- /dev/null +++ b/src/deployment/deploylib/tests/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python +# +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. diff --git a/src/deployment/deploylib/tests/test_deploy_config.py b/src/deployment/deploylib/tests/test_deploy_config.py new file mode 100644 index 0000000000..1bc816802e --- /dev/null +++ b/src/deployment/deploylib/tests/test_deploy_config.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import unittest +from typing import Any, List + +from deploylib.configuration import NetworkSecurityConfig + + +class TestNetworkSecurityConfig: + allowed_ips: List[str] + allowed_service_tags: List[str] + + +class DeployTests(unittest.TestCase): + def test_config(self) -> None: + ## Test Invalid Configs + # Test Dictionary + invalid_config: Any = "" + with self.assertRaises(Exception): + NetworkSecurityConfig(invalid_config) + # Test Empty Dic + invalid_config = {} + with self.assertRaises(Exception): + NetworkSecurityConfig(invalid_config) + # Test Invalid Outer Keys + invalid_config = {"": ""} + with self.assertRaises(Exception): + NetworkSecurityConfig(invalid_config) + # Test Inner Dictionary + invalid_config = {"proxy_nsg_config": ""} + with self.assertRaises(Exception): + NetworkSecurityConfig(invalid_config) + # Test Inner Keys + invalid_config = {"proxy_nsg_config": {}} + with self.assertRaises(Exception): + NetworkSecurityConfig(invalid_config) + # Test Inner Keys + invalid_config = {"proxy_nsg_config": {"allowed_ips": ""}} + with self.assertRaises(Exception): + NetworkSecurityConfig(invalid_config) + # Test Inner Dict Values (lists) + invalid_config = { + "proxy_nsg_config": {"allowed_ips": [], "allowed_service_tags": ""} + } + with self.assertRaises(Exception): + NetworkSecurityConfig(invalid_config) + # Test List Values + invalid_config = { + "proxy_nsg_config": { + "allowed_ips": [1, 2], + "allowed_service_tags": ["10.0.0.0"], + } + } + with self.assertRaises(Exception): + NetworkSecurityConfig(invalid_config) + + ## Test Valid Configs + # Test Empty Lists + valid_config: Any = { + "proxy_nsg_config": {"allowed_ips": [], "allowed_service_tags": []} + } + NetworkSecurityConfig(valid_config) + # Test Wild Card Lists + valid_config = { + "proxy_nsg_config": {"allowed_ips": ["*"], "allowed_service_tags": []} + } + NetworkSecurityConfig(valid_config) + # Test IPs Lists + valid_config = { + "proxy_nsg_config": { + "allowed_ips": ["10.0.0.1", "10.0.0.2"], + "allowed_service_tags": [], + } + } + NetworkSecurityConfig(valid_config) + # Test Tags Lists + valid_config = { + "proxy_nsg_config": { + "allowed_ips": ["10.0.0.1", "10.0.0.2"], + "allowed_service_tags": ["Internet"], + } + } + NetworkSecurityConfig(valid_config) + + +if __name__ == "__main__": + unittest.main()