From 22f16399c4bc775625563d0241c2839b93fe7377 Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Sun, 5 Apr 2020 02:05:45 +0400 Subject: [PATCH 01/19] modify cwd prior to compiling a project --- brownie/project/main.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/brownie/project/main.py b/brownie/project/main.py index d924c32cc..bf19ad1e5 100644 --- a/brownie/project/main.py +++ b/brownie/project/main.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 import json +import os import shutil import sys import zipfile @@ -67,16 +68,22 @@ def _compile(self, contract_sources: Dict, compiler_config: Dict, silent: bool) if self._path is not None: allow_paths = self._path.joinpath("contracts").as_posix() compiler_config.setdefault("solc", {}) - build_json = compiler.compile_and_format( - contract_sources, - solc_version=compiler_config["solc"].get("version", None), - optimize=compiler_config["solc"].get("optimize", None), - runs=compiler_config["solc"].get("runs", None), - evm_version=compiler_config["evm_version"], - silent=silent, - allow_paths=allow_paths, - interface_sources=self._sources.get_interface_sources(), - ) + cwd = os.getcwd() + if self._path is not None: + os.chdir(self._path) + try: + build_json = compiler.compile_and_format( + contract_sources, + solc_version=compiler_config["solc"].get("version", None), + optimize=compiler_config["solc"].get("optimize", None), + runs=compiler_config["solc"].get("runs", None), + evm_version=compiler_config["evm_version"], + silent=silent, + allow_paths=allow_paths, + interface_sources=self._sources.get_interface_sources(), + ) + finally: + os.chdir(cwd) for data in build_json.values(): if self._path is not None: path = self._path.joinpath(f"build/contracts/{data['contractName']}.json") From 68b01ad139ad825c0f11b7079a443c3a31a5db63 Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Sun, 5 Apr 2020 02:30:42 +0400 Subject: [PATCH 02/19] feat: brownie pm cli tool --- brownie/_cli/accounts.py | 2 +- brownie/_cli/pm.py | 162 +++++++++++++++++++++++++++++++++++++++ brownie/exceptions.py | 4 + 3 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 brownie/_cli/pm.py diff --git a/brownie/_cli/accounts.py b/brownie/_cli/accounts.py index d2fcdab00..1d2781c03 100644 --- a/brownie/_cli/accounts.py +++ b/brownie/_cli/accounts.py @@ -41,7 +41,7 @@ def main(): try: fn(*args[""]) except TypeError: - print(f"Invalid arguments for command '{args['']}'. Try brownie ethpm --help") + print(f"Invalid arguments for command '{args['']}'. Try brownie accounts --help") return diff --git a/brownie/_cli/pm.py b/brownie/_cli/pm.py new file mode 100644 index 000000000..19fe24c82 --- /dev/null +++ b/brownie/_cli/pm.py @@ -0,0 +1,162 @@ +#!/usr/bin/python3 + +import shutil +import sys +import zipfile +from io import BytesIO +from pathlib import Path +from urllib.parse import urlparse + +import requests +from tqdm import tqdm + +from brownie import project +from brownie._config import _get_data_folder +from brownie.exceptions import InvalidPackage +from brownie.utils import color, notify +from brownie.utils.docopt import docopt + +__doc__ = """Usage: brownie pm [ ...] [options] + +Commands: + list List available accounts + install [version] Install a new package + export [path] Copy an installed package into a new folder + delete Delete an installed package + +Options: + --help -h Display this message + +TODO +""" + + +def main(): + args = docopt(__doc__) + _get_data_folder().joinpath("packages").mkdir(exist_ok=True) + try: + fn = getattr(sys.modules[__name__], f"_{args['']}") + except AttributeError: + print("Invalid command. Try brownie pm --help") + return + try: + fn(*args[""]) + except TypeError: + print(f"Invalid arguments for command '{args['']}'. Try brownie pm --help") + return + + +def _list(): + org_names = [] + for path in _get_data_folder().joinpath("packages").iterdir(): + if not path.is_dir(): + continue + if not list(path.iterdir()): + path.unlink() + continue + org_names.append(path) + + if not org_names: + print("No packages are currently installed.") + else: + print("The following packages are currently installed:") + + for org_path in org_names: + packages = list(org_path.iterdir()) + print(f"\n{color('bright blue')}{org_path.name}{color}") + for path in packages: + u = "\u2514" if path == packages[-1] else "\u251c" + try: + name, version = path.name.rsplit("@", maxsplit=1) + except ValueError: + continue + print( + f" {color('bright black')}{u}\u2500{color}{org_path.name}/" + f"{color('bright white')}{name}{color}@{color('bright white')}{version}{color}" + ) + + +def _export(id_, path_str="."): + source_path = _get_data_folder().joinpath(f"packages/{id_}") + if not source_path.exists(): + raise FileNotFoundError(f"Package '{color('bright blue')}{id_}{color}' is not installed") + dest_path = Path(path_str) + if dest_path.exists(): + if not dest_path.is_dir(): + raise FileExistsError(f"Destination path already exists") + dest_path = dest_path.joinpath(id_) + shutil.copytree(source_path, dest_path) + notify( + "SUCCESS", f"Package '{color('bright blue')}{id_}{color}' has been exported to {dest_path}" + ) + + +def _delete(id_): + source_path = _get_data_folder().joinpath(f"packages/{id_}") + if not source_path.exists(): + raise FileNotFoundError(f"Package '{color('bright blue')}{id_}{color}' is not installed") + shutil.rmtree(source_path) + notify("SUCCESS", f"Package '{color('bright blue')}{id_}{color}' has been deleted") + + +def _install(uri): + if urlparse(uri).scheme in ("erc1319", "ethpm", "ipfs"): + # TODO + return + _install_from_github(uri) + + +def _install_from_github(package_id): + try: + path, version = package_id.split("@") + org, repo = path.split("/") + except ValueError: + raise ValueError( + "Invalid package ID. Must be given as [ORG]/[REPO]@[VERSION]" + "\ne.g. openzeppelin/openzeppelin-contracts@v2.5.0" + ) from None + + data = requests.get(f"https://api.github.com/repos/{org}/{repo}/releases?per_page=100").json() + org, repo = data[0]["html_url"].split("/")[3:5] + releases = [i["tag_name"] for i in data] + if version not in releases: + raise ValueError( + "Invalid version for this package. Available versions are:\n" + ", ".join(releases) + ) from None + + install_path = _get_data_folder().joinpath(f"packages/{org}") + install_path.mkdir(exist_ok=True) + install_path = install_path.joinpath(f"{repo}@{version}") + if install_path.exists(): + raise FileExistsError("Package is aleady installed") + + download_url = next(i["zipball_url"] for i in data if i["tag_name"] == version) + response = requests.get(download_url, stream=True) + total_size = int(response.headers.get("content-length", 0)) + progress_bar = tqdm(total=total_size, unit="iB", unit_scale=True) + content = bytes() + + for data in response.iter_content(1024, decode_unicode=True): + progress_bar.update(len(data)) + content += data + progress_bar.close() + + existing = list(install_path.parent.iterdir()) + with zipfile.ZipFile(BytesIO(content)) as zf: + zf.extractall(str(install_path.parent)) + installed = next(i for i in install_path.parent.iterdir() if i not in existing) + shutil.move(installed, install_path) + + try: + if not install_path.joinpath("contracts").exists(): + raise Exception + project.new(install_path) + project.load(install_path) + except Exception: + shutil.rmtree(install_path) + raise InvalidPackage(f"{package_id} cannot be interpreted as a Brownie project") + + notify( + "SUCCESS", + f"Package '{color('bright blue')}{org}/{repo}@{version}{color}' has been installed", + ) diff --git a/brownie/exceptions.py b/brownie/exceptions.py index 9a4733bfb..c9a2cb381 100644 --- a/brownie/exceptions.py +++ b/brownie/exceptions.py @@ -119,3 +119,7 @@ class InvalidManifest(Exception): class UnsupportedLanguage(Exception): pass + + +class InvalidPackage(Exception): + pass From 1668f7bee200ff0bfe5c176d4aae9fff6146fedd Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Sun, 5 Apr 2020 14:06:56 +0400 Subject: [PATCH 03/19] update erc1319 to ethpm --- brownie/_cli/ethpm.py | 4 ++-- brownie/project/ethpm.py | 6 ++++-- docs/api-project.rst | 2 +- docs/ethpm.rst | 14 +++++++------- tests/cli/test_cli_ethpm.py | 2 +- tests/project/ethpm/test_get_manifest.py | 6 +++--- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/brownie/_cli/ethpm.py b/brownie/_cli/ethpm.py index 3d8d71701..3b1bad7be 100644 --- a/brownie/_cli/ethpm.py +++ b/brownie/_cli/ethpm.py @@ -141,7 +141,7 @@ def _release(project_path, registry_address, sender): ) if tx.status == 1: notify("SUCCESS", f"{name} has been released!") - print(f"\nURI: {color('bright magenta')}erc1319://{registry_address}:1/{name}{color}") + print(f"\nURI: {color('bright magenta')}ethpm://{registry_address}:1/{name}{color}") return except Exception: pass @@ -161,7 +161,7 @@ def _all(project_path): if not package_list: path.unlink() continue - print(f"{color('bright magenta')}erc1319://{path.name}{color}") + print(f"{color('bright magenta')}ethpm://{path.name}{color}") for package_path in package_list: u = "\u2514" if package_path == package_list[-1] else "\u251c" versions = sorted(package_path.glob("*.json")) diff --git a/brownie/project/ethpm.py b/brownie/project/ethpm.py index 18288f727..b8ae6498e 100644 --- a/brownie/project/ethpm.py +++ b/brownie/project/ethpm.py @@ -22,7 +22,9 @@ from . import compiler -URI_REGEX = r"""^(?:erc1319://|)([^/:\s]*)(?::[0-9]+|)/([a-z][a-z0-9_-]{0,255})@([^\s:/'";]*)$""" +URI_REGEX = ( + r"""^(?:erc1319://|ethpm://|)([^/:\s]*)(?::[0-9]+|)/([a-z][a-z0-9_-]{0,255})@([^\s:/'";]*)$""" +) REGISTRY_ABI = [ { @@ -297,7 +299,7 @@ def install_package(project_path: Path, uri: str, replace_existing: bool = False Args: project_path: Path to the root folder of the project - uri: manifest URI, can be erc1319 or ipfs + uri: manifest URI, can be erc1319, ethpm or ipfs replace_existing: if True, existing files will be overwritten when installing the package diff --git a/docs/api-project.rst b/docs/api-project.rst index 556b44062..cefa3f8be 100644 --- a/docs/api-project.rst +++ b/docs/api-project.rst @@ -411,7 +411,7 @@ Module Methods Installs an ethPM package within the project. * ``project_path``: Path to the root folder of the project - * ``uri``: manifest URI, can be erc1319 or ipfs + * ``uri``: manifest URI, can be ethpm, erc1319 or ipfs * ``replace_existing``: if True, existing files will be overwritten when installing the package Returns the package name as a string. diff --git a/docs/ethpm.rst b/docs/ethpm.rst index e3be20597..84897d2d5 100644 --- a/docs/ethpm.rst +++ b/docs/ethpm.rst @@ -24,13 +24,13 @@ To obtain an ethPM package, you must know both the package name and the address :: - erc1319://[CONTRACT_ADDRESS]:[CHAIN_ID]/[PACKAGE_NAME]@[VERSION] + ethpm://[CONTRACT_ADDRESS]:[CHAIN_ID]/[PACKAGE_NAME]@[VERSION] For example, here is a registry URI for the popular OpenZeppelin `Math `_ package, served by the Snake Charmers `Zeppelin registry `_: :: - erc1319://zeppelin.snakecharmers.eth:1/math@1.0.0 + ethpm://zeppelin.snakecharmers.eth:1/math@1.0.0 Working with ethPM Packages =========================== @@ -73,10 +73,10 @@ Any packages that are installed from a registry are also saved locally. To view $ brownie ethpm all Brownie - Python development framework for Ethereum - erc1319://erc20.snakecharmers.eth + ethpm://erc20.snakecharmers.eth └─dai-dai@1.0.0 - erc1319://zeppelin.snakecharmers.eth + ethpm://zeppelin.snakecharmers.eth ├─access@1.0.0 ├─gns@1.0.0 └─math@1.0.0 @@ -270,7 +270,7 @@ Once the package is successfully released, Brownie provides you with a registry SUCCESS: nftoken@1.0.1 has been released! - URI: erc1319://erc20.snakecharmers.eth:1/nftoken@1.0.1 + URI: ethpm://erc20.snakecharmers.eth:1/nftoken@1.0.1 .. _ethpm-deployments: @@ -282,7 +282,7 @@ You can load an entire package as a :func:`Project >> from brownie.project import from_ethpm - >>> maker = from_ethpm("erc1319://erc20.snakecharmers.eth:1/dai-dai@1.0.0") + >>> maker = from_ethpm("ethpm://erc20.snakecharmers.eth:1/dai-dai@1.0.0") >>> maker >>> maker.dict() @@ -296,7 +296,7 @@ Or, create a :func:`Contract ` object >>> from brownie import network, Contract >>> network.connect('mainnet') - >>> ds = Contract("DSToken", manifest_uri="erc1319://erc20.snakecharmers.eth:1/dai-dai@1.0.0") + >>> ds = Contract("DSToken", manifest_uri="ethpm://erc20.snakecharmers.eth:1/dai-dai@1.0.0") >>> ds diff --git a/tests/cli/test_cli_ethpm.py b/tests/cli/test_cli_ethpm.py index dc5cf1319..71c037248 100644 --- a/tests/cli/test_cli_ethpm.py +++ b/tests/cli/test_cli_ethpm.py @@ -13,7 +13,7 @@ "settings": {"deployment_networks": False, "include_dependencies": False}, } -ERC1319_URI = "erc1319://zeppelin.snakecharmers.eth:1/access@1.0.0" +ERC1319_URI = "ethpm://zeppelin.snakecharmers.eth:1/access@1.0.0" def test_all(np_path): diff --git a/tests/project/ethpm/test_get_manifest.py b/tests/project/ethpm/test_get_manifest.py index 0f56c5ddc..fb152bc3f 100644 --- a/tests/project/ethpm/test_get_manifest.py +++ b/tests/project/ethpm/test_get_manifest.py @@ -13,14 +13,14 @@ def test_get_manifest_from_ipfs(): path = _get_data_folder().joinpath("ethpm/zeppelin.snakecharmers.eth") if path.exists(): shutil.rmtree(path) - ethpm.get_manifest("erc1319://zeppelin.snakecharmers.eth:1/access@1.0.0") + ethpm.get_manifest("ethpm://zeppelin.snakecharmers.eth:1/access@1.0.0") assert _get_data_folder().joinpath("ethpm/zeppelin.snakecharmers.eth").exists() - ethpm.get_manifest("erc1319://zeppelin.snakecharmers.eth:1/access@1.0.0") + ethpm.get_manifest("ethpm://zeppelin.snakecharmers.eth:1/access@1.0.0") assert _get_data_folder().joinpath("ethpm/zeppelin.snakecharmers.eth").exists() def test_meta_brownie(): - manifest = ethpm.get_manifest("erc1319://zeppelin.snakecharmers.eth:1/access@1.0.0") + manifest = ethpm.get_manifest("ethpm://zeppelin.snakecharmers.eth:1/access@1.0.0") assert manifest["meta_brownie"] == { "registry_address": "zeppelin.snakecharmers.eth", "manifest_uri": "ipfs://QmWqn5uYx9LvV4aqj2qZ5FiFZykmS3LGdLpod7XLjxPVYr", From 3e582304f8301f4708d6e46037b379726c3f796b Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Sun, 5 Apr 2020 16:41:10 +0400 Subject: [PATCH 04/19] allow ethpm package installation via brownie pm --- brownie/_cli/pm.py | 98 +++++++++++++++++++++++++++++++--------------- 1 file changed, 66 insertions(+), 32 deletions(-) diff --git a/brownie/_cli/pm.py b/brownie/_cli/pm.py index 19fe24c82..1c7572ce5 100644 --- a/brownie/_cli/pm.py +++ b/brownie/_cli/pm.py @@ -13,6 +13,7 @@ from brownie import project from brownie._config import _get_data_folder from brownie.exceptions import InvalidPackage +from brownie.project.ethpm import get_manifest, install_package from brownie.utils import color, notify from brownie.utils.docopt import docopt @@ -27,7 +28,9 @@ Options: --help -h Display this message -TODO +Package manager for ethPM or Github packages which can be imported into your own +projects. See https://eth-brownie.readthedocs.io/en/stable/pm.html for more +information. """ @@ -63,58 +66,73 @@ def _list(): for org_path in org_names: packages = list(org_path.iterdir()) - print(f"\n{color('bright blue')}{org_path.name}{color}") + print(f"\n{color('bright magenta')}{org_path.name}{color}") for path in packages: u = "\u2514" if path == packages[-1] else "\u251c" try: name, version = path.name.rsplit("@", maxsplit=1) except ValueError: continue - print( - f" {color('bright black')}{u}\u2500{color}{org_path.name}/" - f"{color('bright white')}{name}{color}@{color('bright white')}{version}{color}" - ) + print(f" {color('bright black')}{u}\u2500{_format_pkg(org_path.name, name, version)}") -def _export(id_, path_str="."): - source_path = _get_data_folder().joinpath(f"packages/{id_}") +def _export(package_id, path_str="."): + org, repo, version = _split_id(package_id) + source_path = _get_data_folder().joinpath(f"packages/{org}/{repo}@{version}") if not source_path.exists(): - raise FileNotFoundError(f"Package '{color('bright blue')}{id_}{color}' is not installed") + raise FileNotFoundError(f"Package '{_format_pkg(org, repo, version)}' is not installed") dest_path = Path(path_str) if dest_path.exists(): if not dest_path.is_dir(): raise FileExistsError(f"Destination path already exists") - dest_path = dest_path.joinpath(id_) + dest_path = dest_path.joinpath(package_id) shutil.copytree(source_path, dest_path) - notify( - "SUCCESS", f"Package '{color('bright blue')}{id_}{color}' has been exported to {dest_path}" - ) + notify("SUCCESS", f"Package '{_format_pkg(org, repo, version)}' was exported to {dest_path}") -def _delete(id_): - source_path = _get_data_folder().joinpath(f"packages/{id_}") +def _delete(package_id): + org, repo, version = _split_id(package_id) + source_path = _get_data_folder().joinpath(f"packages/{org}/{repo}@{version}") if not source_path.exists(): - raise FileNotFoundError(f"Package '{color('bright blue')}{id_}{color}' is not installed") + raise FileNotFoundError(f"Package '{_format_pkg(org, repo, version)}' is not installed") shutil.rmtree(source_path) - notify("SUCCESS", f"Package '{color('bright blue')}{id_}{color}' has been deleted") + notify("SUCCESS", f"Package '{_format_pkg(org, repo, version)}' has been deleted") def _install(uri): - if urlparse(uri).scheme in ("erc1319", "ethpm", "ipfs"): - # TODO - return - _install_from_github(uri) + if urlparse(uri).scheme in ("erc1319", "ethpm"): + org, repo, version = _install_from_ethpm(uri) + else: + org, repo, version = _install_from_github(uri) + notify("SUCCESS", f"Package '{_format_pkg(org, repo, version)}' has been installed") + + +def _install_from_ethpm(uri): + manifest = get_manifest(uri) + org = manifest["meta_brownie"]["registry_address"] + repo = manifest["package_name"] + version = manifest["version"] + + install_path = _get_data_folder().joinpath(f"packages/{org}") + install_path.mkdir(exist_ok=True) + install_path = install_path.joinpath(f"{repo}@{version}") + if install_path.exists(): + raise FileExistsError("Package is aleady installed") -def _install_from_github(package_id): try: - path, version = package_id.split("@") - org, repo = path.split("/") - except ValueError: - raise ValueError( - "Invalid package ID. Must be given as [ORG]/[REPO]@[VERSION]" - "\ne.g. openzeppelin/openzeppelin-contracts@v2.5.0" - ) from None + project.new(install_path) + install_package(install_path, uri) + project.load(install_path) + except Exception as e: + shutil.rmtree(install_path) + raise e + + return org, repo, version + + +def _install_from_github(package_id): + org, repo, version = _split_id(package_id) data = requests.get(f"https://api.github.com/repos/{org}/{repo}/releases?per_page=100").json() org, repo = data[0]["html_url"].split("/")[3:5] @@ -156,7 +174,23 @@ def _install_from_github(package_id): shutil.rmtree(install_path) raise InvalidPackage(f"{package_id} cannot be interpreted as a Brownie project") - notify( - "SUCCESS", - f"Package '{color('bright blue')}{org}/{repo}@{version}{color}' has been installed", + return org, repo, version + + +def _split_id(package_id): + try: + path, version = package_id.split("@") + org, repo = path.split("/") + return org, repo, version + except ValueError: + raise ValueError( + "Invalid package ID. Must be given as [ORG]/[REPO]@[VERSION]" + f"\ne.g. {_format_pkg('openzeppelin', 'openzeppelin-contracts', 'v2.5.0')}" + ) from None + + +def _format_pkg(org, repo, version): + return ( + f"{color('blue')}{org}/{color('bright blue')}{repo}" + f"{color('blue')}@{color('bright blue')}{version}{color}" ) From 21fc00f228b0c48ca4733ac75ab32649ee1b4e3a Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Sun, 5 Apr 2020 18:18:20 +0400 Subject: [PATCH 05/19] remove ethpm from main cli menu (temporary deprecation) --- brownie/_cli/__main__.py | 4 ++-- brownie/_cli/pm.py | 7 ++++--- brownie/project/main.py | 5 ----- tests/project/main/test_main_project.py | 1 - 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/brownie/_cli/__main__.py b/brownie/_cli/__main__.py index f8632efdf..46068aa74 100644 --- a/brownie/_cli/__main__.py +++ b/brownie/_cli/__main__.py @@ -17,8 +17,8 @@ Commands: init Initialize a new brownie project bake Initialize from a brownie-mix template - ethpm Commands related to the ethPM package manager - compile Compiles the contract source files + pm Install and manage external packages + compile Compile the contract source files console Load the console test Run test cases in the tests/ folder run Run a script in the scripts/ folder diff --git a/brownie/_cli/pm.py b/brownie/_cli/pm.py index 1c7572ce5..e9cc44af8 100644 --- a/brownie/_cli/pm.py +++ b/brownie/_cli/pm.py @@ -28,9 +28,10 @@ Options: --help -h Display this message -Package manager for ethPM or Github packages which can be imported into your own -projects. See https://eth-brownie.readthedocs.io/en/stable/pm.html for more -information. +Manager for ethPM or Github packages. Installed packages can be added +as dependencies and imported into your own projects. + +See https://eth-brownie.readthedocs.io/en/stable/pm.html for more information. """ diff --git a/brownie/project/main.py b/brownie/project/main.py index bf19ad1e5..e1640171f 100644 --- a/brownie/project/main.py +++ b/brownie/project/main.py @@ -382,11 +382,6 @@ def new(project_path_str: str = ".", ignore_subfolder: bool = False) -> str: BROWNIE_FOLDER.joinpath("data/brownie-config.yaml"), project_path.joinpath("brownie-config.yaml"), ) - if not project_path.joinpath("ethpm-config.yaml").exists(): - shutil.copy( - BROWNIE_FOLDER.joinpath("data/ethpm-config.yaml"), - project_path.joinpath("ethpm-config.yaml"), - ) _add_to_sys_path(project_path) return str(project_path) diff --git a/tests/project/main/test_main_project.py b/tests/project/main/test_main_project.py index ebc580d89..cc3663b51 100644 --- a/tests/project/main/test_main_project.py +++ b/tests/project/main/test_main_project.py @@ -33,7 +33,6 @@ def test_check_for_project(project, newproject): def test_new(tmp_path, project): assert str(tmp_path) == project.new(tmp_path) assert tmp_path.joinpath("brownie-config.yaml").exists() - assert tmp_path.joinpath("ethpm-config.yaml").exists() assert tmp_path.joinpath(".gitattributes").exists() assert tmp_path.joinpath(".gitignore").exists() From 2e0ab37922ca229f663796442ea57c3e4a8ee98c Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Sun, 5 Apr 2020 20:53:48 +0400 Subject: [PATCH 06/19] doc: add package manager documentation --- brownie/_cli/pm.py | 7 +- docs/api-network.rst | 10 +- docs/api-project.rst | 10 +- docs/package-manager.rst | 164 +++++++++++++++++++++++++++++++++ docs/tests-pytest-fixtures.rst | 15 ++- docs/tests-pytest-intro.rst | 19 ++++ docs/toctree.rst | 2 +- 7 files changed, 212 insertions(+), 15 deletions(-) create mode 100644 docs/package-manager.rst diff --git a/brownie/_cli/pm.py b/brownie/_cli/pm.py index e9cc44af8..e1dd91634 100644 --- a/brownie/_cli/pm.py +++ b/brownie/_cli/pm.py @@ -28,10 +28,11 @@ Options: --help -h Display this message -Manager for ethPM or Github packages. Installed packages can be added -as dependencies and imported into your own projects. +Manager for packages installed from ethPM and Github. Installed packages can +be added as dependencies and imported into your own projects. -See https://eth-brownie.readthedocs.io/en/stable/pm.html for more information. +See https://eth-brownie.readthedocs.io/en/stable/package-manager.html for +more information on how to install and use packages. """ diff --git a/docs/api-network.rst b/docs/api-network.rst index 732e80913..504f6cb90 100644 --- a/docs/api-network.rst +++ b/docs/api-network.rst @@ -11,7 +11,7 @@ The ``network`` package holds classes for interacting with the Ethereum blockcha The ``main`` module contains methods for conncting to or disconnecting from the network. All of these methods are available directly from ``brownie.network``. -.. py:method:: main.connect(network: str = None, launch_rpc: bool = True) -> None +.. py:method:: main.connect(network = None, launch_rpc = True) Connects to the network. Network settings are retrieved from ``brownie-config.yaml`` @@ -25,7 +25,7 @@ The ``main`` module contains methods for conncting to or disconnecting from the >>> from brownie import network >>> network.connect('development') -.. py:method:: main.disconnect(kill_rpc: bool = True) -> None +.. py:method:: main.disconnect(kill_rpc = True) Disconnects from the network. @@ -46,7 +46,7 @@ The ``main`` module contains methods for conncting to or disconnecting from the >>> network.is_connected() True -.. py:method:: main.show_active() -> Optional[str] +.. py:method:: main.show_active() Returns the name of the network that is currently active, or ``None`` if not connected. @@ -56,7 +56,7 @@ The ``main`` module contains methods for conncting to or disconnecting from the >>> network.show_active() 'development' -.. py:method:: main.gas_limit(*args: Tuple[Union[int, str, bool, None]]) -> Union[int, bool] +.. py:method:: main.gas_limit(*args) Gets and optionally sets the default gas limit. @@ -76,7 +76,7 @@ The ``main`` module contains methods for conncting to or disconnecting from the >>> network.gas_limit("auto") False -.. py:method:: main.gas_price(*args: Tuple[Union[int, str, bool, None]]) -> Union[int, bool] +.. py:method:: main.gas_price(*args) Gets and optionally sets the default gas price. diff --git a/docs/api-project.rst b/docs/api-project.rst index cefa3f8be..8f3b3f954 100644 --- a/docs/api-project.rst +++ b/docs/api-project.rst @@ -25,17 +25,17 @@ Project Project Methods *************** -.. py:classmethod:: Project.load() -> None +.. py:classmethod:: Project.load() Collects project source files, compiles new or updated contracts, instantiates :func:`ContractContainer ` objects, and populates the namespace. Projects are typically loaded via :func:`project.load `, but if you have a :func:`Project ` object that was previously closed you can reload it using this method. -.. py:classmethod:: Project.load_config() -> None +.. py:classmethod:: Project.load_config() Updates the configuration settings from the ``brownie-config.yaml`` file within this project's root folder. -.. py:classmethod:: Project.close(raises: bool = True) -> None +.. py:classmethod:: Project.close(raises = True) Removes this object and the related :func:`ContractContainer ` objects from the namespace. @@ -69,7 +69,7 @@ TempProject Module Methods -------------- -.. py:method:: main.check_for_project(path: Union[str, 'Path']) -> Optional[Path] +.. py:method:: main.check_for_project(path) Checks for an existing Brownie project within a folder and it's parent folders, and returns the base path to the project as a ``Path`` object. Returns ``None`` if no project is found. @@ -83,7 +83,7 @@ Module Methods >>> project.check_for_project('.') PosixPath('/my_projects/token') -.. py:method:: main.get_loaded_projects() -> List +.. py:method:: main.get_loaded_projects() Returns a list of currently loaded :func:`Project ` objects. diff --git a/docs/package-manager.rst b/docs/package-manager.rst new file mode 100644 index 000000000..740719616 --- /dev/null +++ b/docs/package-manager.rst @@ -0,0 +1,164 @@ +.. _package-manager: + +======================= +Brownie Package Manager +======================= + +Brownie allows you to install other projects as packages. Some benefits of packages include: + +* Easily importing and building upon code ideas written by others +* Reducing duplicated code between projects +* Writing unit tests that verify interactions between your project and another project + +The Brownie package manager is available from the commandline: + +.. code-block:: bash + + $ brownie pm + +Installing a Package +==================== + +Brownie supports package installation from ethPM and Github. + +Installing from Github +---------------------- + +The easiest way to install a package is from a Github repository. Brownie considers a Github repository to be a package if meets the following criteria: + + * The repository must have one or more tagged versions. + * The repository must include a ``contracts/`` folder containing one or more Solidity or Vyper source files. + +A repository does not have to implement Brownie in order to function as a package. Many popular projects using frameworks such as Truffle or Embark can be added as Brownie packages. + +To install a package from Github you must use a package ID. A package ID is comprised of the name of an organization, a repository, and a version tag. Package IDs are not not case sensitive. + +.. code-block:: bash + + [ORGANIZATION]/[REPOSITORY]@[VERSION] + +Examples +******** + +To install `OpenZeppelin contracts `_ version ``2.5.0``: + +.. code-block:: bash + + $ brownie pm install OpenZeppelin/openzeppelin-contracts@v2.5.0 + +To install `AragonOS `_ version ``4.0.0``: + +.. code-block:: bash + + $ brownie pm install aragon/aragonos@v4.0.0 + +Installing from ethPM +--------------------- + +The `Ethereum Package Manager `_ (ethPM) is a decentralized package manager used to distribute EVM smart contracts and projects. + +At its core, an ethPM package is a JSON object containing the ABI, source code, bytecode, deployment data and any other information that combines together to compose the smart contract idea. The `ethPM specification `_ defines a schema to store all of this data in a structured JSON format, enabling quick and efficient transportation of smart contract ideas between tools and frameworks which support the specification. + +To obtain an ethPM package, you must know both the package name and the address of the registry where it is available. This information is communicated through a `registry URI `_. Registry URIs use the following format: + +:: + + ethpm://[CONTRACT_ADDRESS]:[CHAIN_ID]/[PACKAGE_NAME]@[VERSION] + +The Snake Charmers maintain an `ethPM registry explorer `_ where you can obtain registry URIs. + +Examples +******** + +To install OpenZeppelin's `Math `_ package, served from the Snake Charmers `Zeppelin registry `_: + +.. code-block:: bash + + $ brownie pm install ethpm://zeppelin.snakecharmers.eth:1/math@1.0.0 + + +To install v2 of the `Compound Protocol `_, served from the Snake Charmers `DeFi registry `_: + + +.. code-block:: bash + + $ brownie pm install ethpm://defi.snakecharmers.eth:1/compound@1.1.0 + +Working with Packages +===================== + +Viewing Installed Packages +-------------------------- + +Use ``brownie pm list`` to view currently installed packages. After installing all of the examples given above, the output looks something like this: + +.. code-block:: bash + + $ brownie pm list + Brownie - Python development framework for Ethereum + + The following packages are currently installed: + + OpenZeppelin + └─OpenZeppelin/openzeppelin-contracts@v2.5.0 + + aragon + └─aragon/aragonOS@v4.0.0 + + zeppelin.snakecharmers.eth + └─zeppelin.snakecharmers.eth/access@1.0.0 + + defi.snakecharmers.eth + └─defi.snakecharmers.eth/compound@1.1.0 + +Exporting a Package +------------------- + +Use ``brownie pm export`` to copy the contents of a package into another folder. This is useful for exploring the filestructure of a package, or when you wish to build a project on top of an existing package. + +To copy the Aragon package to the current folder: + +.. code-block:: bash + + $ brownie pm export aragon/aragonOS@v4.0.0 + +Using Packages in your Project +============================== + +Importing Sources from a Package +-------------------------------- + +You can import sources from an installed package in the same way that you would a source within your project. The root path is based on the name of the package and can be obtained via ``brownie pm list``. + +For example, to import ``SafeMath`` from OpenZeppelin contracts: + +.. code-block:: solidity + + import "OpenZeppelin/openzeppelin-contracts@v2.5.0/contracts/math/SafeMath.sol"; + +Using Packages in Tests +----------------------- + +The ``pm`` fixture provides access to installed packages during testing. It returns a :func:`Project ` object when called with a project ID: + +.. code-block:: python + + def test_with_compound_token(pm): + compound = pm('defi.snakecharmers.eth/compound@1.1.0').CToken + +See the :ref:`unit test documentation` for more detailed information. + +.. _package-manager-deps: + +Declaring Project Dependencies +------------------------------ + +Project dependencies are declared by adding a ``dependencies`` field to ``brownie-config.yaml``: + +.. code-block:: yaml + + dependencies: + - aragon/aragonOS@v4.0.0 + - defi.snakecharmers.eth/compound@1.1.0 + +Brownie attempts to install any listed dependencies prior to compiling a project. This is useful when your project may be used outside of your local environment. diff --git a/docs/tests-pytest-fixtures.rst b/docs/tests-pytest-fixtures.rst index 3f7e5f514..7e5a8bada 100644 --- a/docs/tests-pytest-fixtures.rst +++ b/docs/tests-pytest-fixtures.rst @@ -10,7 +10,7 @@ Brownie provides :ref:`fixtures ` to allow you to interact Session Fixtures ================ -These fixtures provide quick access to Brownie objects that are frequently used during testing. If you are unfamiliar with these objects, you may wish to read the documentation liested under "Core Functionality" in the table of contents. +These fixtures provide quick access to Brownie objects that are frequently used during testing. If you are unfamiliar with these objects, you may wish to read the documentation listed under "Core Functionality" in the table of contents. .. _test-fixtures-accounts: @@ -45,6 +45,19 @@ These fixtures provide quick access to Brownie objects that are frequently used accounts[0].transfer(accounts[1], "10 ether") assert len(history) == 1 +.. py:attribute:: pm + + Callable fixture that provides access to :func:`Project ` objects, used for testing against installed packages. + + .. code-block:: python + :linenos: + + @pytest.fixture(scope="module") + def compound(pm, accounts): + ctoken = pm('defi.snakecharmers.eth/compound@1.1.0').CToken + yield ctoken.deploy({'from': accounts[0]}) + + .. py:attribute:: rpc Yields an :func:`Rpc ` object, used for interacting with the local test chain. diff --git a/docs/tests-pytest-intro.rst b/docs/tests-pytest-intro.rst index ae016c375..d3afddc0e 100644 --- a/docs/tests-pytest-intro.rst +++ b/docs/tests-pytest-intro.rst @@ -301,6 +301,25 @@ You can achieve a similar effect with the ``@given`` decorator to automatically This technique is known as `property-based testing`. To learn more, read :ref:`hypothesis`. +.. _pytest-other-projects: + +Testing against Other Projects +============================== + +The ``pm`` fixture provides access to packages that have been installed with the :ref:`Brownie package manager`. Using this fixture, you can write test cases that verify interactions between your project and another project. + +``pm`` is a function that accepts a project ID as an argument and returns a :func:`Project ` object. This way you can deploy contracts from the package and deliver them as fixtures to be used in your tests: + +.. code-block:: python + :linenos: + + @pytest.fixture(scope="module") + def compound(pm, accounts): + ctoken = pm('defi.snakecharmers.eth/compound@1.1.0').CToken + yield ctoken.deploy({'from': accounts[0]}) + +Be sure to add required testing packages to your project :ref:`dependency list`. + Running Tests ============= diff --git a/docs/toctree.rst b/docs/toctree.rst index d09149541..5279e4587 100644 --- a/docs/toctree.rst +++ b/docs/toctree.rst @@ -18,6 +18,7 @@ Brownie structure.rst compile.rst interaction.rst + package-manager.rst gui.rst @@ -51,7 +52,6 @@ Brownie deploy.rst nonlocal-networks.rst accounts.rst - ethpm.rst .. toctree:: From 85256059b63afd25f734e6a3de5feca0675ef775 Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Sun, 5 Apr 2020 22:13:07 +0400 Subject: [PATCH 07/19] feat: add pm fixture --- brownie/test/fixtures.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/brownie/test/fixtures.py b/brownie/test/fixtures.py index f265388fa..5212f26e3 100644 --- a/brownie/test/fixtures.py +++ b/brownie/test/fixtures.py @@ -5,7 +5,7 @@ import pytest import brownie -from brownie._config import ARGV +from brownie._config import ARGV, _get_data_folder from .stateful import _BrownieStateMachine, state_machine @@ -75,6 +75,25 @@ def web3(self): """Yields an instantiated Web3 object, connected to the active network.""" yield brownie.web3 + @pytest.fixture(scope="session") + def pm(self): + """ + Yields a function for accessing installed packages. + """ + _open_projects = {} + + def package_loader(project_id): + if project_id not in _open_projects: + path = _get_data_folder().joinpath(f"packages/{project_id}") + _open_projects[project_id] = brownie.project.load(path, project_id) + + return _open_projects[project_id] + + yield package_loader + + for project in _open_projects.values(): + project.close(raises=False) + @pytest.fixture def no_call_coverage(self): """ From 1e4068b2047bafdf2d4e1de71b818e508134c528 Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Sun, 5 Apr 2020 23:28:42 +0400 Subject: [PATCH 08/19] feat: include path remappings to allow packages imports --- brownie/project/compiler/__init__.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/brownie/project/compiler/__init__.py b/brownie/project/compiler/__init__.py index 255dba89e..601480c13 100644 --- a/brownie/project/compiler/__init__.py +++ b/brownie/project/compiler/__init__.py @@ -9,6 +9,7 @@ from eth_utils import remove_0x_prefix from semantic_version import Version +from brownie._config import _get_data_folder from brownie.exceptions import UnsupportedLanguage from brownie.project import sources from brownie.project.compiler.solidity import ( # NOQA: F401 @@ -21,6 +22,11 @@ from . import solidity, vyper +remappings = [] +for path in _get_data_folder().joinpath("packages").iterdir(): + remappings.append(f"{path.name}={path.as_posix()}") + + STANDARD_JSON: Dict = { "language": None, "sources": {}, @@ -29,7 +35,7 @@ "*": {"*": ["abi", "evm.bytecode", "evm.deployedBytecode"], "": ["ast"]} }, "evmVersion": None, - "remappings": [], + "remappings": remappings, }, } EVM_SOLC_VERSIONS = [ @@ -183,8 +189,15 @@ def compile_from_input_json( if input_json["language"] == "Vyper": return vyper.compile_from_input_json(input_json, silent, allow_paths) + if input_json["language"] == "Solidity": + if allow_paths: + allow_paths = f"{allow_paths},{ _get_data_folder().joinpath('packages').as_posix()}" + else: + allow_paths = _get_data_folder().joinpath("packages").as_posix() + return solidity.compile_from_input_json(input_json, silent, allow_paths) + raise UnsupportedLanguage(f"{input_json['language']}") From 3c60350e9d3a697d58f02953b8e42c5af4645ddf Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Sun, 5 Apr 2020 23:56:50 +0400 Subject: [PATCH 09/19] do not include leading v in package versions --- brownie/_cli/pm.py | 5 ++--- docs/package-manager.rst | 14 +++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/brownie/_cli/pm.py b/brownie/_cli/pm.py index e1dd91634..e17be5105 100644 --- a/brownie/_cli/pm.py +++ b/brownie/_cli/pm.py @@ -38,7 +38,6 @@ def main(): args = docopt(__doc__) - _get_data_folder().joinpath("packages").mkdir(exist_ok=True) try: fn = getattr(sys.modules[__name__], f"_{args['']}") except AttributeError: @@ -138,7 +137,7 @@ def _install_from_github(package_id): data = requests.get(f"https://api.github.com/repos/{org}/{repo}/releases?per_page=100").json() org, repo = data[0]["html_url"].split("/")[3:5] - releases = [i["tag_name"] for i in data] + releases = [i["tag_name"].lstrip("v") for i in data] if version not in releases: raise ValueError( "Invalid version for this package. Available versions are:\n" + ", ".join(releases) @@ -150,7 +149,7 @@ def _install_from_github(package_id): if install_path.exists(): raise FileExistsError("Package is aleady installed") - download_url = next(i["zipball_url"] for i in data if i["tag_name"] == version) + download_url = next(i["zipball_url"] for i in data if i["tag_name"].lstrip("v") == version) response = requests.get(download_url, stream=True) total_size = int(response.headers.get("content-length", 0)) progress_bar = tqdm(total=total_size, unit="iB", unit_scale=True) diff --git a/docs/package-manager.rst b/docs/package-manager.rst index 740719616..b3aa7f59d 100644 --- a/docs/package-manager.rst +++ b/docs/package-manager.rst @@ -44,13 +44,13 @@ To install `OpenZeppelin contracts `_ version ``4.0.0``: .. code-block:: bash - $ brownie pm install aragon/aragonos@v4.0.0 + $ brownie pm install aragon/aragonos@4.0.0 Installing from ethPM --------------------- @@ -100,10 +100,10 @@ Use ``brownie pm list`` to view currently installed packages. After installing a The following packages are currently installed: OpenZeppelin - └─OpenZeppelin/openzeppelin-contracts@v2.5.0 + └─OpenZeppelin/openzeppelin-contracts@2.5.0 aragon - └─aragon/aragonOS@v4.0.0 + └─aragon/aragonOS@4.0.0 zeppelin.snakecharmers.eth └─zeppelin.snakecharmers.eth/access@1.0.0 @@ -120,7 +120,7 @@ To copy the Aragon package to the current folder: .. code-block:: bash - $ brownie pm export aragon/aragonOS@v4.0.0 + $ brownie pm export aragon/aragonOS@4.0.0 Using Packages in your Project ============================== @@ -134,7 +134,7 @@ For example, to import ``SafeMath`` from OpenZeppelin contracts: .. code-block:: solidity - import "OpenZeppelin/openzeppelin-contracts@v2.5.0/contracts/math/SafeMath.sol"; + import "OpenZeppelin/openzeppelin-contracts@2.5.0/contracts/math/SafeMath.sol"; Using Packages in Tests ----------------------- @@ -158,7 +158,7 @@ Project dependencies are declared by adding a ``dependencies`` field to ``browni .. code-block:: yaml dependencies: - - aragon/aragonOS@v4.0.0 + - aragon/aragonOS@4.0.0 - defi.snakecharmers.eth/compound@1.1.0 Brownie attempts to install any listed dependencies prior to compiling a project. This is useful when your project may be used outside of your local environment. From 4c8a399bdb5b8f2359c46441b328c00f94889ca2 Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Sun, 5 Apr 2020 23:58:08 +0400 Subject: [PATCH 10/19] create brownie data subfolders on load --- brownie/_cli/accounts.py | 1 - brownie/_config.py | 16 +++++++++++++--- tests/conftest.py | 1 + 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/brownie/_cli/accounts.py b/brownie/_cli/accounts.py index 1d2781c03..b50504396 100644 --- a/brownie/_cli/accounts.py +++ b/brownie/_cli/accounts.py @@ -32,7 +32,6 @@ def main(): args = docopt(__doc__) - _get_data_folder().joinpath("accounts").mkdir(exist_ok=True) try: fn = getattr(sys.modules[__name__], f"_{args['']}") except AttributeError: diff --git a/brownie/_config.py b/brownie/_config.py index a23f0fe9e..136222119 100644 --- a/brownie/_config.py +++ b/brownie/_config.py @@ -15,6 +15,8 @@ BROWNIE_FOLDER = Path(__file__).parent DATA_FOLDER = Path.home().joinpath(".brownie") +DATA_SUBFOLDERS = ("accounts", "ethpm", "packages") + REPLACE = ["active_network", "networks"] EVM_EQUIVALENTS = {"atlantis": "byzantium", "agharta": "petersburg"} @@ -88,9 +90,6 @@ def _load_default_config() -> "ConfigDict": # Loads the default configuration settings from brownie/data/config.yaml base_config = BROWNIE_FOLDER.joinpath("data/brownie-config.yaml") - if not DATA_FOLDER.exists(): - DATA_FOLDER.mkdir() - config = _Singleton("Config", (ConfigDict,), {})(_load_config(base_config)) # type: ignore config["active_network"] = {"name": None} _modify_hypothesis_settings(config) @@ -166,6 +165,17 @@ def _get_data_folder() -> Path: return DATA_FOLDER +def _make_data_folders(data_folder: Path) -> None: + # create data folder structure + data_folder.mkdir(exist_ok=True) + for folder in DATA_SUBFOLDERS: + data_folder.joinpath(folder).mkdir(exist_ok=True) + + +# create data folders +_make_data_folders(DATA_FOLDER) + + # create argv object ARGV = _Singleton("Argv", (defaultdict,), {})(lambda: None) # type: ignore diff --git a/tests/conftest.py b/tests/conftest.py index d11e05635..2de6c37b8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -121,6 +121,7 @@ def xdist_id(worker_id): @pytest.fixture(scope="session", autouse=True) def _base_config(tmp_path_factory, xdist_id): brownie._config.DATA_FOLDER = tmp_path_factory.mktemp(f"data-{xdist_id}") + brownie._config._make_data_folders(brownie._config.DATA_FOLDER) with brownie._config.BROWNIE_FOLDER.joinpath("data/brownie-config.yaml").open() as fp: config = yaml.safe_load(fp) if xdist_id: From ca95dec8eac38e615b137be28bc7846b5eaf5c07 Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Mon, 6 Apr 2020 01:13:13 +0400 Subject: [PATCH 11/19] move package installation logic to project.main, install project dependencies on load --- brownie/_cli/pm.py | 103 +++------------------ brownie/_config.py | 5 ++ brownie/project/compiler/__init__.py | 9 +- brownie/project/main.py | 130 +++++++++++++++++++++++---- 4 files changed, 131 insertions(+), 116 deletions(-) diff --git a/brownie/_cli/pm.py b/brownie/_cli/pm.py index e17be5105..7e186c91a 100644 --- a/brownie/_cli/pm.py +++ b/brownie/_cli/pm.py @@ -2,18 +2,10 @@ import shutil import sys -import zipfile -from io import BytesIO from pathlib import Path -from urllib.parse import urlparse - -import requests -from tqdm import tqdm from brownie import project from brownie._config import _get_data_folder -from brownie.exceptions import InvalidPackage -from brownie.project.ethpm import get_manifest, install_package from brownie.utils import color, notify from brownie.utils.docopt import docopt @@ -22,7 +14,7 @@ Commands: list List available accounts install [version] Install a new package - export [path] Copy an installed package into a new folder + clone [path] Make a copy of an installed package delete Delete an installed package Options: @@ -55,10 +47,10 @@ def _list(): for path in _get_data_folder().joinpath("packages").iterdir(): if not path.is_dir(): continue - if not list(path.iterdir()): - path.unlink() - continue - org_names.append(path) + elif not list(i for i in path.iterdir() if i.is_dir() and "@" in i.name): + shutil.rmtree(path) + else: + org_names.append(path) if not org_names: print("No packages are currently installed.") @@ -70,14 +62,11 @@ def _list(): print(f"\n{color('bright magenta')}{org_path.name}{color}") for path in packages: u = "\u2514" if path == packages[-1] else "\u251c" - try: - name, version = path.name.rsplit("@", maxsplit=1) - except ValueError: - continue + name, version = path.name.rsplit("@", maxsplit=1) print(f" {color('bright black')}{u}\u2500{_format_pkg(org_path.name, name, version)}") -def _export(package_id, path_str="."): +def _clone(package_id, path_str="."): org, repo, version = _split_id(package_id) source_path = _get_data_folder().joinpath(f"packages/{org}/{repo}@{version}") if not source_path.exists(): @@ -88,7 +77,7 @@ def _export(package_id, path_str="."): raise FileExistsError(f"Destination path already exists") dest_path = dest_path.joinpath(package_id) shutil.copytree(source_path, dest_path) - notify("SUCCESS", f"Package '{_format_pkg(org, repo, version)}' was exported to {dest_path}") + notify("SUCCESS", f"Package '{_format_pkg(org, repo, version)}' was cloned at {dest_path}") def _delete(package_id): @@ -101,81 +90,9 @@ def _delete(package_id): def _install(uri): - if urlparse(uri).scheme in ("erc1319", "ethpm"): - org, repo, version = _install_from_ethpm(uri) - else: - org, repo, version = _install_from_github(uri) - - notify("SUCCESS", f"Package '{_format_pkg(org, repo, version)}' has been installed") - - -def _install_from_ethpm(uri): - manifest = get_manifest(uri) - org = manifest["meta_brownie"]["registry_address"] - repo = manifest["package_name"] - version = manifest["version"] - - install_path = _get_data_folder().joinpath(f"packages/{org}") - install_path.mkdir(exist_ok=True) - install_path = install_path.joinpath(f"{repo}@{version}") - if install_path.exists(): - raise FileExistsError("Package is aleady installed") - - try: - project.new(install_path) - install_package(install_path, uri) - project.load(install_path) - except Exception as e: - shutil.rmtree(install_path) - raise e - - return org, repo, version - - -def _install_from_github(package_id): + package_id = project.main.install_package(uri) org, repo, version = _split_id(package_id) - - data = requests.get(f"https://api.github.com/repos/{org}/{repo}/releases?per_page=100").json() - org, repo = data[0]["html_url"].split("/")[3:5] - releases = [i["tag_name"].lstrip("v") for i in data] - if version not in releases: - raise ValueError( - "Invalid version for this package. Available versions are:\n" + ", ".join(releases) - ) from None - - install_path = _get_data_folder().joinpath(f"packages/{org}") - install_path.mkdir(exist_ok=True) - install_path = install_path.joinpath(f"{repo}@{version}") - if install_path.exists(): - raise FileExistsError("Package is aleady installed") - - download_url = next(i["zipball_url"] for i in data if i["tag_name"].lstrip("v") == version) - response = requests.get(download_url, stream=True) - total_size = int(response.headers.get("content-length", 0)) - progress_bar = tqdm(total=total_size, unit="iB", unit_scale=True) - content = bytes() - - for data in response.iter_content(1024, decode_unicode=True): - progress_bar.update(len(data)) - content += data - progress_bar.close() - - existing = list(install_path.parent.iterdir()) - with zipfile.ZipFile(BytesIO(content)) as zf: - zf.extractall(str(install_path.parent)) - installed = next(i for i in install_path.parent.iterdir() if i not in existing) - shutil.move(installed, install_path) - - try: - if not install_path.joinpath("contracts").exists(): - raise Exception - project.new(install_path) - project.load(install_path) - except Exception: - shutil.rmtree(install_path) - raise InvalidPackage(f"{package_id} cannot be interpreted as a Brownie project") - - return org, repo, version + notify("SUCCESS", f"Package '{_format_pkg(org, repo, version)}' has been installed") def _split_id(package_id): diff --git a/brownie/_config.py b/brownie/_config.py index 136222119..5b37a0ff6 100644 --- a/brownie/_config.py +++ b/brownie/_config.py @@ -115,6 +115,11 @@ def _load_project_compiler_config(project_path: Optional[Path]) -> Dict: return compiler_data +def _load_project_dependencies(project_path: Path) -> Dict: + compiler_data = _load_config(project_path.joinpath("brownie-config")) + return compiler_data.get("dependencies", []) + + def _modify_network_config(network: str = None) -> Dict: """Modifies the 'active_network' configuration settings""" CONFIG._unlock() diff --git a/brownie/project/compiler/__init__.py b/brownie/project/compiler/__init__.py index 601480c13..aafd4ea61 100644 --- a/brownie/project/compiler/__init__.py +++ b/brownie/project/compiler/__init__.py @@ -22,11 +22,6 @@ from . import solidity, vyper -remappings = [] -for path in _get_data_folder().joinpath("packages").iterdir(): - remappings.append(f"{path.name}={path.as_posix()}") - - STANDARD_JSON: Dict = { "language": None, "sources": {}, @@ -35,7 +30,7 @@ "*": {"*": ["abi", "evm.bytecode", "evm.deployedBytecode"], "": ["ast"]} }, "evmVersion": None, - "remappings": remappings, + "remappings": [], }, } EVM_SOLC_VERSIONS = [ @@ -161,6 +156,8 @@ def generate_input_json( input_json["settings"]["evmVersion"] = evm_version if language == "Solidity": input_json["settings"]["optimizer"] = {"enabled": optimize, "runs": runs if optimize else 0} + for path in _get_data_folder().joinpath("packages").iterdir(): + input_json["settings"]["remappings"].append(f"{path.name}={path.as_posix()}") input_json["sources"] = _sources_dict(contract_sources, language) if interface_sources: diff --git a/brownie/project/main.py b/brownie/project/main.py index e1640171f..5935ed160 100644 --- a/brownie/project/main.py +++ b/brownie/project/main.py @@ -9,6 +9,7 @@ from io import BytesIO from pathlib import Path from typing import Any, Dict, Iterator, KeysView, List, Optional, Set, Tuple, Union +from urllib.parse import urlparse import requests from semantic_version import Version @@ -17,15 +18,17 @@ from brownie._config import ( BROWNIE_FOLDER, CONFIG, + _get_data_folder, _get_project_config_path, _load_project_compiler_config, _load_project_config, + _load_project_dependencies, ) -from brownie.exceptions import ProjectAlreadyLoaded, ProjectNotFound +from brownie.exceptions import InvalidPackage, ProjectAlreadyLoaded, ProjectNotFound from brownie.network import web3 from brownie.network.contract import Contract, ContractContainer, ProjectContract from brownie.network.state import _add_contract, _remove_contract -from brownie.project import compiler +from brownie.project import compiler, ethpm from brownie.project.build import BUILD_KEYS, Build from brownie.project.ethpm import get_deployment_addresses, get_manifest from brownie.project.sources import Sources, get_pragma_spec @@ -64,13 +67,15 @@ class _ProjectBase: _build: Build def _compile(self, contract_sources: Dict, compiler_config: Dict, silent: bool) -> None: - allow_paths = None - if self._path is not None: - allow_paths = self._path.joinpath("contracts").as_posix() compiler_config.setdefault("solc", {}) + + allow_paths = None cwd = os.getcwd() if self._path is not None: + _install_dependencies(self._path) + allow_paths = self._path.joinpath("contracts").as_posix() os.chdir(self._path) + try: build_json = compiler.compile_and_format( contract_sources, @@ -84,6 +89,7 @@ def _compile(self, contract_sources: Dict, compiler_config: Dict, silent: bool) ) finally: os.chdir(cwd) + for data in build_json.values(): if self._path is not None: path = self._path.joinpath(f"build/contracts/{data['contractName']}.json") @@ -407,18 +413,7 @@ def from_brownie_mix( raise FileExistsError(f"Folder already exists - {project_path}") print(f"Downloading from {url}...") - response = requests.get(url, stream=True) - total_size = int(response.headers.get("content-length", 0)) - progress_bar = tqdm(total=total_size, unit="iB", unit_scale=True) - content = bytes() - - for data in response.iter_content(1024, decode_unicode=True): - progress_bar.update(len(data)) - content += data - progress_bar.close() - - with zipfile.ZipFile(BytesIO(content)) as zf: - zf.extractall(str(project_path.parent)) + _stream_download(url, str(project_path.parent)) project_path.parent.joinpath(project_name + "-mix-master").rename(project_path) _create_folders(project_path) _create_gitfiles(project_path) @@ -513,6 +508,92 @@ def load(project_path: Union[Path, str, None] = None, name: Optional[str] = None return Project(name, project_path) +def _install_dependencies(path: Path) -> None: + for package_id in _load_project_dependencies(path): + try: + install_package(package_id) + except FileExistsError: + pass + + +def install_package(package_id: str) -> str: + if urlparse(package_id).scheme in ("erc1319", "ethpm"): + return _install_from_ethpm(package_id) + else: + return _install_from_github(package_id) + + +def _install_from_ethpm(uri: str) -> str: + manifest = get_manifest(uri) + org = manifest["meta_brownie"]["registry_address"] + repo = manifest["package_name"] + version = manifest["version"] + + install_path = _get_data_folder().joinpath(f"packages/{org}") + install_path.mkdir(exist_ok=True) + install_path = install_path.joinpath(f"{repo}@{version}") + if install_path.exists(): + raise FileExistsError("Package is aleady installed") + + try: + new(str(install_path)) + ethpm.install_package(install_path, uri) + project = load(install_path) + project.close() + except Exception as e: + shutil.rmtree(install_path) + raise e + + return f"{org}/{repo}@{version}" + + +def _install_from_github(package_id: str) -> str: + try: + path, version = package_id.split("@") + org, repo = path.split("/") + except ValueError: + raise ValueError( + "Invalid package ID. Must be given as [ORG]/[REPO]@[VERSION]" + "\ne.g. 'OpenZeppelin/openzeppelin-contracts@v2.5.0'" + ) from None + + install_path = _get_data_folder().joinpath(f"packages/{org}") + install_path.mkdir(exist_ok=True) + install_path = install_path.joinpath(f"{repo}@{version}") + if install_path.exists(): + raise FileExistsError("Package is aleady installed") + + data = requests.get(f"https://api.github.com/repos/{org}/{repo}/tags?per_page=100").json() + if not data: + raise ValueError("Github repository has no tags set") + org, repo = data[0]["zipball_url"].split("/")[3:5] + tags = [i["name"].lstrip("v") for i in data] + if version not in tags: + raise ValueError( + "Invalid version for this package. Available versions are:\n" + ", ".join(tags) + ) from None + + download_url = next(i["zipball_url"] for i in data if i["name"].lstrip("v") == version) + + existing = list(install_path.parent.iterdir()) + _stream_download(download_url, str(install_path.parent)) + + installed = next(i for i in install_path.parent.iterdir() if i not in existing) + shutil.move(installed, install_path) + + try: + if not install_path.joinpath("contracts").exists(): + raise Exception + new(str(install_path)) + project = load(install_path) + project.close() + except Exception: + shutil.rmtree(install_path) + raise InvalidPackage(f"{package_id} cannot be interpreted as a Brownie project") + + return f"{org}/{repo}@{version}" + + def _create_gitfiles(project_path: Path) -> None: gitignore = project_path.joinpath(".gitignore") if not gitignore.exists(): @@ -556,3 +637,18 @@ def _load_sources(project_path: Path, subfolder: str, allow_json: bool) -> Dict: path_str: str = path.relative_to(project_path).as_posix() contract_sources[path_str] = source return contract_sources + + +def _stream_download(download_url: str, target_path: str) -> None: + response = requests.get(download_url, stream=True) + total_size = int(response.headers.get("content-length", 0)) + progress_bar = tqdm(total=total_size, unit="iB", unit_scale=True) + content = bytes() + + for data in response.iter_content(1024, decode_unicode=True): + progress_bar.update(len(data)) + content += data + progress_bar.close() + + with zipfile.ZipFile(BytesIO(content)) as zf: + zf.extractall(target_path) From c90a858c815c0e28c3b1cf3f0dcfc9c4d25881ed Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Mon, 6 Apr 2020 19:45:13 +0400 Subject: [PATCH 12/19] test: add pm test cases --- tests/cli/test_cli_main.py | 4 + tests/cli/test_cli_pm.py | 85 ++++++++++++++++++ tests/conftest.py | 9 +- tests/project/packages/test_import.py | 35 ++++++++ tests/project/packages/test_install.py | 88 +++++++++++++++++++ .../project/packages/test_popular_packages.py | 10 +++ tests/project/test_brownie_mix.py | 2 +- tests/test/plugin/test_pm_fixture.py | 19 ++++ 8 files changed, 249 insertions(+), 3 deletions(-) create mode 100644 tests/cli/test_cli_pm.py create mode 100644 tests/project/packages/test_import.py create mode 100644 tests/project/packages/test_install.py create mode 100644 tests/project/packages/test_popular_packages.py create mode 100644 tests/test/plugin/test_pm_fixture.py diff --git a/tests/cli/test_cli_main.py b/tests/cli/test_cli_main.py index 2b52adb99..000bd2bd2 100644 --- a/tests/cli/test_cli_main.py +++ b/tests/cli/test_cli_main.py @@ -231,3 +231,7 @@ def test_no_args_shows_help(cli_tester, capfd): with pytest.raises(SystemExit): cli_tester.run_and_test_parameters() assert cli_main.__doc__ in capfd.readouterr()[0].strip() + + +def test_cli_pm(cli_tester): + cli_tester.run_and_test_parameters("pm list", None) diff --git a/tests/cli/test_cli_pm.py b/tests/cli/test_cli_pm.py new file mode 100644 index 000000000..58fd7aca1 --- /dev/null +++ b/tests/cli/test_cli_pm.py @@ -0,0 +1,85 @@ +#!/usr/bin/python3 + +import shutil + +import pytest + +from brownie._cli import pm as cli_pm +from brownie._config import _get_data_folder + + +@pytest.fixture(autouse=True) +def setup(): + yield + path = _get_data_folder().joinpath("packages") + shutil.rmtree(path) + path.mkdir() + + +def _mk_repo_path(*folder_names): + path = _get_data_folder().joinpath("packages") + + for name in folder_names: + path = path.joinpath(name) + path.mkdir(exist_ok=True) + + return path + + +def test_list_no_installed(capfd): + cli_pm._list() + assert "No packages are currently installed." in capfd.readouterr()[0] + + +def test_list_installed(capfd): + _mk_repo_path("testorg", "testrepo@1.0.0") + _mk_repo_path("testorg", "testrepo@1.0.1") + + cli_pm._list() + stdout = capfd.readouterr()[0] + assert "1.0.0" in stdout + assert "1.0.1" in stdout + + +def test_list_remove_spurious_files(capfd): + bad_path1 = _mk_repo_path("emptynothing") + bad_path2 = _mk_repo_path("bad-repo", "package-without-version") + with bad_path2.parent.joinpath("package-as-file@1.0.0").open("w") as fp: + fp.write("i'm a file!") + + cli_pm._list() + assert "No packages are currently installed." in capfd.readouterr()[0] + assert not bad_path1.exists() + assert not bad_path2.exists() + + +def test_clone(tmp_path): + _mk_repo_path("testorg", "testrepo@1.0.0") + cli_pm._clone("testorg/testrepo@1.0.0", tmp_path.as_posix()) + + assert tmp_path.joinpath("testorg").exists() + + +def test_clone_not_installed(tmp_path): + with pytest.raises(FileNotFoundError): + cli_pm._clone("testorg/testrepo@1.0.0", tmp_path.as_posix()) + + +def test_clone_already_exists(tmp_path): + _mk_repo_path("testorg", "testrepo@1.0.0") + cli_pm._clone("testorg/testrepo@1.0.0", tmp_path.as_posix()) + + with pytest.raises(FileExistsError): + cli_pm._clone("testorg/testrepo@1.0.0", tmp_path.as_posix()) + + +def test_delete(tmp_path): + path = _mk_repo_path("testorg", "testrepo@1.0.0") + cli_pm._delete("testorg/testrepo@1.0.0") + + assert not path.exists() + + +def test_delete_not_installed(tmp_path): + with pytest.raises(FileNotFoundError): + cli_pm._delete("testorg/testrepo@1.0.0") diff --git a/tests/conftest.py b/tests/conftest.py index 2de6c37b8..821bfb368 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,7 +22,7 @@ pytest_plugins = "pytester" -TARGET_OPTS = {"evm": "evmtester", "mixes": "browniemix", "plugin": "plugintester"} +TARGET_OPTS = {"evm": "evmtester", "pm": "package_test", "plugin": "plugintester"} def pytest_addoption(parser): @@ -72,7 +72,7 @@ def pytest_generate_tests(metafunc): metafunc.parametrize("evmtester", params, indirect=True) # parametrize the browniemix fixture - if "browniemix" in metafunc.fixturenames and target in ("all", "mixes"): + if "browniemix" in metafunc.fixturenames and target in ("all", "pm"): if os.getenv("GITHUB_TOKEN"): auth = b64encode(os.getenv("GITHUB_TOKEN").encode()).decode() headers = {"Authorization": "Basic {}".format(auth)} @@ -408,3 +408,8 @@ def ipfs_mock(monkeypatch): if ipfs_path.exists(): shutil.rmtree(ipfs_path) temp_path.rename(ipfs_path) + + +@pytest.fixture +def package_test(): + pass diff --git a/tests/project/packages/test_import.py b/tests/project/packages/test_import.py new file mode 100644 index 000000000..a87eb15fb --- /dev/null +++ b/tests/project/packages/test_import.py @@ -0,0 +1,35 @@ +import shutil + +import pytest + +import brownie +from brownie.exceptions import CompilerError +from brownie.project import compile_source +from brownie.project.main import install_package + + +@pytest.fixture(autouse=True) +def setup(): + yield + path = brownie._config._get_data_folder().joinpath("packages") + shutil.rmtree(path) + path.mkdir() + + +code = """ +pragma solidity ^0.5.0; + +import "brownie-mix/token-mix@1.0.0/contracts/Token.sol"; + +contract Foo is Token {} + """ + + +def test_import_from_package(): + install_package("brownie-mix/token-mix@1.0.0") + compile_source(code) + + +def test_import_fails_without_package_installed(): + with pytest.raises(CompilerError): + compile_source(code) diff --git a/tests/project/packages/test_install.py b/tests/project/packages/test_install.py new file mode 100644 index 000000000..0a42aa946 --- /dev/null +++ b/tests/project/packages/test_install.py @@ -0,0 +1,88 @@ +import shutil + +import pytest +import yaml + +from brownie._config import _get_data_folder +from brownie.exceptions import InvalidPackage +from brownie.project.main import install_package + + +@pytest.fixture(autouse=True) +def setup(package_test): + yield + path = _get_data_folder().joinpath("packages") + shutil.rmtree(path) + path.mkdir() + + +@pytest.fixture +def dependentproject(newproject): + with newproject._path.joinpath("brownie-config.yaml").open() as fp: + config = yaml.safe_load(fp) + config["dependencies"] = ["brownie-mix/token-mix@1.0.0"] + with newproject._path.joinpath("brownie-config.yaml").open("w") as fp: + yaml.dump(config, fp) + + yield newproject + + +def test_install_from_github(): + install_package("brownie-mix/token-mix@1.0.0") + + +def test_install_from_ethpm(ipfs_mock): + install_package("ethpm://zeppelin.snakecharmers.eth:1/access@1.0.0") + + +def test_github_already_installed(): + path = _get_data_folder().joinpath("packages/brownie-mix") + path.mkdir() + path.joinpath("token-mix@1.0.0").mkdir() + + with pytest.raises(FileExistsError): + install_package("brownie-mix/token-mix@1.0.0") + + +def test_ethpm_already_installed(): + path = _get_data_folder().joinpath("packages/zeppelin.snakecharmers.eth") + path.mkdir() + path.joinpath("access@1.0.0").mkdir() + + with pytest.raises(FileExistsError): + install_package("ethpm://zeppelin.snakecharmers.eth:1/access@1.0.0") + + +def test_unknown_version(): + with pytest.raises(ValueError): + install_package("brownie-mix/token-mix@1.0.1") + + +def test_bad_project_id_version(): + with pytest.raises(ValueError): + install_package("brownie-mix/token-mix") + + +def test_bad_project_id_repo_org(): + with pytest.raises(ValueError): + install_package("token-mix@1.0.0") + + +def test_valid_repo_not_a_project(): + with pytest.raises(InvalidPackage): + install_package("iamdefinitelyahuman/eth-event@0.2.2") + + assert not _get_data_folder().joinpath("packages/iamdefinitelyahuman/eth-event@0.2.2").exists() + + +def test_install_from_config_dependencies(dependentproject): + package_folder = _get_data_folder().joinpath("packages/brownie-mix/token-mix@1.0.0") + assert not package_folder.exists() + + dependentproject.load() + assert package_folder.exists() + + +def test_dependency_already_installed(dependentproject): + install_package("brownie-mix/token-mix@1.0.0") + dependentproject.load() diff --git a/tests/project/packages/test_popular_packages.py b/tests/project/packages/test_popular_packages.py new file mode 100644 index 000000000..7e196fc74 --- /dev/null +++ b/tests/project/packages/test_popular_packages.py @@ -0,0 +1,10 @@ +import pytest + +from brownie.project.main import install_package + +PACKAGES = ["OpenZeppelin/openzeppelin-contracts@2.5.0", "aragon/aragonOS@4.0.0"] + + +@pytest.mark.parametrize("package_id", PACKAGES) +def test_popular_packages(package_test, package_id): + install_package(package_id) diff --git a/tests/project/test_brownie_mix.py b/tests/project/test_brownie_mix.py index f74dd56ec..65665b617 100644 --- a/tests/project/test_brownie_mix.py +++ b/tests/project/test_brownie_mix.py @@ -9,7 +9,7 @@ # browniemix is parametrized with every mix repo from https://www.github.com/brownie-mix/ -def test_mixes(plugintesterbase, project, tmp_path, rpc, browniemix): +def test_mixes(plugintesterbase, project, tmp_path, rpc, browniemix, package_test): path = Path(project.from_brownie_mix(browniemix, tmp_path.joinpath("testmix"))) os.chdir(path) diff --git a/tests/test/plugin/test_pm_fixture.py b/tests/test/plugin/test_pm_fixture.py new file mode 100644 index 000000000..6113debc1 --- /dev/null +++ b/tests/test/plugin/test_pm_fixture.py @@ -0,0 +1,19 @@ +from brownie.project.main import install_package + +test_source = """ +import pytest + +@pytest.fixture +def token(pm, accounts): + Token = pm('brownie-mix/token-mix@1.0.0').Token + yield Token.deploy("Test", "TST", 18, 100000, {'from': accounts[0]}) + +def test_token(token, accounts): + token.transfer(accounts[1], 100, {'from': accounts[0]}) + """ + + +def test_pm_fixture(plugintester): + install_package("brownie-mix/token-mix@1.0.0") + result = plugintester.runpytest() + result.assert_outcomes(passed=1) From 91f8c8c24cc2704973d49a949020069c6fadee65 Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Tue, 7 Apr 2020 01:41:31 +0400 Subject: [PATCH 13/19] test: rename mixtest to pmtest --- .travis.yml | 4 ++-- tox.ini | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0f343d925..990da658c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ jobs: - sudo apt-get update - sudo apt-get install -y python3.6-dev npm solc script: tox -e lint,doctest,py36 - - name: "Standard Tests, Brownie Mix Tests - Python 3.7 on Bionic Linux" + - name: "Standard Tests, Package Tests - Python 3.7 on Bionic Linux" language: python python: 3.7 dist: bionic @@ -38,7 +38,7 @@ jobs: - sudo add-apt-repository -y ppa:deadsnakes/ppa - sudo apt-get update - sudo apt-get install -y python3.6-dev npm solc - script: tox -e py37,mixtest + script: tox -e py37,pmtest - name: "Standard Tests, Plugin Tests - Python 3.8 on Bionic Linux" language: python python: 3.8 diff --git a/tox.ini b/tox.ini index b6f4bd4db..5818ad6ef 100644 --- a/tox.ini +++ b/tox.ini @@ -3,26 +3,26 @@ envlist = lint doctest py{36,37,38} - {mix,evm,plugin}test + {pm,evm,plugin}test [testenv] passenv = GITHUB_TOKEN WEB3_INFURA_PROJECT_ID envdir = - py36,mixtest,lint,doctest: {toxinidir}/.tox/py36 + py36,pmtest,lint,doctest: {toxinidir}/.tox/py36 py37,evmtest: {toxinidir}/.tox/py37 py38,plugintest: {toxinidir}/.tox/py38 deps = - py{36,37,38},{mix,evm,plugin}test: coverage==4.5.4 - py{36,37,38},{mix,evm,plugin}test: pytest==5.4.1 - py{36,37,38},{mix,evm,plugin}test: pytest-cov==2.8.1 - py{36,37,38},{mix,evm,plugin}test: pytest-mock==2.0.0 - py{36,37,38},{mix,evm,plugin}test: pytest-xdist==1.31.0 + py{36,37,38},{pm,evm,plugin}test: coverage==4.5.4 + py{36,37,38},{pm,evm,plugin}test: pytest==5.4.1 + py{36,37,38},{pm,evm,plugin}test: pytest-cov==2.8.1 + py{36,37,38},{pm,evm,plugin}test: pytest-mock==2.0.0 + py{36,37,38},{pm,evm,plugin}test: pytest-xdist==1.31.0 commands = py{36,37,38}: python -m pytest tests/ --cov-append evmtest: python -m pytest tests/ --target evm --cov-append - mixtest: python -m pytest tests/ --target mixes --cov-append -n 0 + pmtest: python -m pytest tests/ --target pm --cov-append -n 0 plugintest: python -m pytest tests/ --target plugin --cov-append -n 0 [testenv:lint] From fa68c27ff2641a69b88524ada3932e6ccd6abd53 Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Tue, 7 Apr 2020 14:12:45 +0400 Subject: [PATCH 14/19] feat: use github api token for package tag queries --- brownie/project/main.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/brownie/project/main.py b/brownie/project/main.py index 5935ed160..87ba851c0 100644 --- a/brownie/project/main.py +++ b/brownie/project/main.py @@ -5,6 +5,7 @@ import shutil import sys import zipfile +from base64 import b64encode from hashlib import sha1 from io import BytesIO from pathlib import Path @@ -563,7 +564,27 @@ def _install_from_github(package_id: str) -> str: if install_path.exists(): raise FileExistsError("Package is aleady installed") - data = requests.get(f"https://api.github.com/repos/{org}/{repo}/tags?per_page=100").json() + headers: Dict = {} + if os.getenv("GITHUB_TOKEN"): + auth = b64encode(os.environ["GITHUB_TOKEN"].encode()).decode() + headers = {"Authorization": "Basic {}".format(auth)} + + response = requests.get( + f"https://api.github.com/repos/{org}/{repo}/tags?per_page=100", headers=headers + ) + if response.status_code != 200: + msg = "Status {} when getting package versions from Github: '{}'".format( + response.status_code, response.json()["message"] + ) + if response.status_code == 403: + msg += ( + "\n\nIf this issue persists, generate a Github API token and store" + " it as the environment variable `GITHUB_TOKEN`:\n" + "https://github.blog/2013-05-16-personal-api-tokens/" + ) + raise ConnectionError(msg) + + data = response.json() if not data: raise ValueError("Github repository has no tags set") org, repo = data[0]["zipball_url"].split("/")[3:5] From cadd05374d5c069be7bc27322c6cc6906a4d81e9 Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Tue, 7 Apr 2020 14:58:39 +0400 Subject: [PATCH 15/19] doc: add api docs and docstrings for install_package --- brownie/project/main.py | 13 +++++++++++++ docs/api-project.rst | 8 ++++++++ 2 files changed, 21 insertions(+) diff --git a/brownie/project/main.py b/brownie/project/main.py index 87ba851c0..05f54ec80 100644 --- a/brownie/project/main.py +++ b/brownie/project/main.py @@ -518,6 +518,19 @@ def _install_dependencies(path: Path) -> None: def install_package(package_id: str) -> str: + """ + Install a package. + + Arguments + --------- + package_id : str + Package ID or ethPM URI. + + Returns + ------- + str + ID of the installed package. + """ if urlparse(package_id).scheme in ("erc1319", "ethpm"): return _install_from_ethpm(package_id) else: diff --git a/docs/api-project.rst b/docs/api-project.rst index 8f3b3f954..2bed2b2ad 100644 --- a/docs/api-project.rst +++ b/docs/api-project.rst @@ -170,6 +170,14 @@ Module Methods >>> container.SimpleTest +.. py:method:: main.install_package(package_id) + + Install a package. + + See the :ref:`Brownie Package Manager ` documentation for more information on packages. + + * ``package_id``: Package identifier or ethPM URI + ``brownie.project.build`` ========================= From 47c65cb6c5d206369eb0c15c19b8511d661be046 Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Tue, 7 Apr 2020 18:15:54 +0400 Subject: [PATCH 16/19] feat: handle dependencies given as a string --- brownie/_config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/brownie/_config.py b/brownie/_config.py index 5b37a0ff6..3826d6e68 100644 --- a/brownie/_config.py +++ b/brownie/_config.py @@ -117,7 +117,10 @@ def _load_project_compiler_config(project_path: Optional[Path]) -> Dict: def _load_project_dependencies(project_path: Path) -> Dict: compiler_data = _load_config(project_path.joinpath("brownie-config")) - return compiler_data.get("dependencies", []) + dependencies = compiler_data.get("dependencies", []) + if isinstance(dependencies, str): + dependencies = [dependencies] + return dependencies def _modify_network_config(network: str = None) -> Dict: From cd5dc959dfa6fcd2e1d0f5757f1f30c8b3539b83 Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Tue, 7 Apr 2020 18:37:53 +0400 Subject: [PATCH 17/19] feat: allow path remappings for solidity compiler --- brownie/project/compiler/__init__.py | 51 +++++++++++++++++++++++----- brownie/project/main.py | 8 +++-- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/brownie/project/compiler/__init__.py b/brownie/project/compiler/__init__.py index aafd4ea61..59b90f95a 100644 --- a/brownie/project/compiler/__init__.py +++ b/brownie/project/compiler/__init__.py @@ -49,17 +49,21 @@ def compile_and_format( silent: bool = True, allow_paths: Optional[str] = None, interface_sources: Optional[Dict[str, str]] = None, + remappings: Optional[list] = None, ) -> Dict: """Compiles contracts and returns build data. Args: - contracts: a dictionary in the form of {'path': "source code"} + contract_sources: a dictionary in the form of {'path': "source code"} solc_version: solc version to compile with (use None to set via pragmas) optimize: enable solc optimizer runs: optimizer runs evm_version: evm version to compile for silent: verbose reporting allow_paths: compiler allowed filesystem import path + interface_sources: dictionary of interfaces as {'path': "source code"} + remappings: list of solidity path remappings + Returns: build data dict @@ -109,8 +113,9 @@ def compile_and_format( to_compile = {k: v for k, v in contract_sources.items() if k in path_list} input_json = generate_input_json( - to_compile, optimize, runs, evm_version, language, interfaces + to_compile, optimize, runs, evm_version, language, interfaces, remappings ) + output_json = compile_from_input_json(input_json, silent, allow_paths) output_json["contracts"] = { @@ -128,6 +133,7 @@ def generate_input_json( evm_version: Union[int, str, None] = None, language: str = "Solidity", interface_sources: Optional[Dict[str, str]] = None, + remappings: Optional[list] = None, ) -> Dict: """Formats contracts to the standard solc input json. @@ -138,6 +144,8 @@ def generate_input_json( runs: optimizer runs evm_version: evm version to compile for language: source language (Solidity or Vyper) + interface_sources: dictionary of interfaces as {'path': "source code"} + remappings: list of solidity path remappings Returns: dict """ @@ -156,8 +164,7 @@ def generate_input_json( input_json["settings"]["evmVersion"] = evm_version if language == "Solidity": input_json["settings"]["optimizer"] = {"enabled": optimize, "runs": runs if optimize else 0} - for path in _get_data_folder().joinpath("packages").iterdir(): - input_json["settings"]["remappings"].append(f"{path.name}={path.as_posix()}") + input_json["settings"]["remappings"] = _get_solc_remappings(remappings) input_json["sources"] = _sources_dict(contract_sources, language) if interface_sources: @@ -169,6 +176,36 @@ def generate_input_json( return input_json +def _get_solc_remappings(remappings: Optional[list]) -> list: + if remappings is None: + remap_dict: Dict = {} + elif isinstance(remappings, str): + remap_dict = dict([remappings.split("=")]) + else: + remap_dict = dict(i.split("=") for i in remappings) + + for path in _get_data_folder().joinpath("packages").iterdir(): + key = next((k for k, v in remap_dict.items() if v.startswith(path.name)), None) + if key: + remap_dict[key] = path.parent.joinpath(remap_dict[key]).as_posix() + else: + remap_dict[path.name] = path.as_posix() + + return [f"{k}={v}" for k, v in remap_dict.items()] + + +def _get_allow_paths(allow_paths: Optional[str], remappings: list) -> str: + # generate the final allow_paths field based on path remappings + path_list = [] if allow_paths is None else [allow_paths] + + remapping_paths = [i[i.index("=") + 1 :] for i in remappings] + data_path = _get_data_folder().joinpath("packages").as_posix() + remapping_paths = [i for i in remapping_paths if not i.startswith(data_path)] + + path_list = path_list + [data_path] + remapping_paths + return ",".join(path_list) + + def compile_from_input_json( input_json: Dict, silent: bool = True, allow_paths: Optional[str] = None ) -> Dict: @@ -188,11 +225,7 @@ def compile_from_input_json( return vyper.compile_from_input_json(input_json, silent, allow_paths) if input_json["language"] == "Solidity": - if allow_paths: - allow_paths = f"{allow_paths},{ _get_data_folder().joinpath('packages').as_posix()}" - else: - allow_paths = _get_data_folder().joinpath("packages").as_posix() - + allow_paths = _get_allow_paths(allow_paths, input_json["settings"]["remappings"]) return solidity.compile_from_input_json(input_json, silent, allow_paths) raise UnsupportedLanguage(f"{input_json['language']}") diff --git a/brownie/project/main.py b/brownie/project/main.py index 05f54ec80..10c4e122c 100644 --- a/brownie/project/main.py +++ b/brownie/project/main.py @@ -87,6 +87,7 @@ def _compile(self, contract_sources: Dict, compiler_config: Dict, silent: bool) silent=silent, allow_paths=allow_paths, interface_sources=self._sources.get_interface_sources(), + remappings=compiler_config["solc"].get("remappings", []), ) finally: os.chdir(cwd) @@ -240,7 +241,9 @@ def _compare_build_json(self, contract_name: str) -> bool: return True if build_json["language"] == "Solidity": # compare solc-specific compiler settings - if _compare_settings(config["solc"], build_json["compiler"]): + solc_config = config["solc"].copy() + solc_config["remappings"] = None + if _compare_settings(solc_config, build_json["compiler"]): return True # compare solc pragma against compiled version if Version(build_json["compiler"]["version"]) not in get_pragma_spec(source): @@ -653,7 +656,8 @@ def _add_to_sys_path(project_path: Path) -> None: def _compare_settings(left: Dict, right: Dict) -> bool: return next( - (True for k, v in left.items() if v and not isinstance(v, dict) and v != right[k]), False + (True for k, v in left.items() if v and not isinstance(v, dict) and v != right.get(k)), + False, ) From 64c546acb3e67a7275ef90fb79bd66e3469e9784 Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Tue, 7 Apr 2020 19:48:34 +0400 Subject: [PATCH 18/19] test: test case for remapping path of installed package --- tests/project/compiler/test_solidity.py | 9 ++++++--- tests/project/packages/test_import.py | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/tests/project/compiler/test_solidity.py b/tests/project/compiler/test_solidity.py index 7ed226e76..427a68070 100644 --- a/tests/project/compiler/test_solidity.py +++ b/tests/project/compiler/test_solidity.py @@ -102,17 +102,20 @@ def _test_compiler(a, **kwargs): assert kwargs["optimize_runs"] == 666 monkeypatch.setattr("solcx.compile_standard", _test_compiler) - input_json = {"language": "Solidity", "settings": {"optimizer": {"enabled": True, "runs": 666}}} + input_json = { + "language": "Solidity", + "settings": {"optimizer": {"enabled": True, "runs": 666}, "remappings": []}, + } compiler.compile_from_input_json(input_json) input_json = { "language": "Solidity", - "settings": {"optimizer": {"enabled": True, "runs": 31337}}, + "settings": {"optimizer": {"enabled": True, "runs": 31337}, "remappings": []}, } with pytest.raises(AssertionError): compiler.compile_from_input_json(input_json) input_json = { "language": "Solidity", - "settings": {"optimizer": {"enabled": False, "runs": 666}}, + "settings": {"optimizer": {"enabled": False, "runs": 666}, "remappings": []}, } with pytest.raises(AssertionError): compiler.compile_from_input_json(input_json) diff --git a/tests/project/packages/test_import.py b/tests/project/packages/test_import.py index a87eb15fb..7fe423402 100644 --- a/tests/project/packages/test_import.py +++ b/tests/project/packages/test_import.py @@ -1,6 +1,7 @@ import shutil import pytest +import yaml import brownie from brownie.exceptions import CompilerError @@ -33,3 +34,25 @@ def test_import_from_package(): def test_import_fails_without_package_installed(): with pytest.raises(CompilerError): compile_source(code) + + +def test_dependency_with_remapping(newproject): + with newproject._path.joinpath("brownie-config.yaml").open() as fp: + config = yaml.safe_load(fp) + config["dependencies"] = ["brownie-mix/token-mix@1.0.0"] + config["compiler"]["solc"]["remappings"] = ["token=brownie-mix/token-mix@1.0.0/contracts"] + with newproject._path.joinpath("brownie-config.yaml").open("w") as fp: + yaml.dump(config, fp) + + remapped_code = """ +pragma solidity ^0.5.0; + +import "token/Token.sol"; + +contract Foo is Token {} + """ + + with newproject._path.joinpath("contracts/Test.sol").open("w") as fp: + fp.write(remapped_code) + + newproject.load() From 46057565a0e356e479a15d5eb9fe62428011e6ac Mon Sep 17 00:00:00 2001 From: iamdefinitelyahuman Date: Tue, 7 Apr 2020 19:50:36 +0400 Subject: [PATCH 19/19] doc: add documentation for path remappings --- docs/compile.rst | 63 ++++++++++++++++++++++++++++++++++++---- docs/config.rst | 1 + docs/package-manager.rst | 4 +++ 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/docs/compile.rst b/docs/compile.rst index a3dd6af84..366645045 100644 --- a/docs/compile.rst +++ b/docs/compile.rst @@ -43,11 +43,12 @@ Settings for the compiler are found in ``brownie-config.yaml``: .. code-block:: yaml - evm_version: null - solc: - version: 0.6.0 - optimize: true - runs: 200 + compiler: + evm_version: null + solc: + version: 0.6.0 + optimize: true + runs: 200 Modifying any compiler settings will result in a full recompile of the project. @@ -84,7 +85,57 @@ Compiler optimization is enabled by default. Coverage evaluation was designed us See the `Solidity documentation `_ for more info on the ``solc`` optimizer. -.. _compile-json: +.. _compile-remap: + +Path Remappings +--------------- + +The Solidity compiler allows path remappings. Brownie exposes this functionality via the ``compiler.solc.remappings`` field in the configuration file: + +.. code-block:: yaml + + compiler: + solc: + remappings: + - zeppelin=/usr/local/lib/open-zeppelin/contracts/ + - github.com/ethereum/dapp-bin/=/usr/local/lib/dapp-bin/ + +Each value under ``remappings`` is a string in the format ``prefix=path``. A remapping instructs the compiler to search for a given prefix at a specific path. For example: + +:: + + github.com/ethereum/dapp-bin/=/usr/local/lib/dapp-bin/ + +This remapping instructs the compiler to search for anything starting with ``github.com/ethereum/dapp-bin/`` under ``/usr/local/lib/dapp-bin``. + +Brownie automatically ensures that all remapped paths are allowed. You do not have to declare ``allow_paths``. + +.. warning:: + + Brownie does not detect modifications to files that are imported from outside the root folder of your project. You must manually recompile your project when an external source file changes. + +.. _compile-remap-packages: + +Remapping Installed Packages +**************************** + +Remappings can be applied to installed packages. For example: + +:: + + oz=OpenZeppelin/openzeppelin-contracts@2.5.0/contracts + +With the ``OpenZeppelin/openzeppelin-contracts@2.5.0`` package installed, and the above remapping added to the configuration file, both of the following import statements point to the same location: + +:: + + import "OpenZeppelin/openzeppelin-contracts@2.5.0/contracts/math/SafeMath.sol"; + +:: + + import "oz/math/SafeMath.sol"; + + Installing the Compiler ======================= diff --git a/docs/config.rst b/docs/config.rst index 82549f527..cb4914d13 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -73,6 +73,7 @@ The following settings are available: * ``version``: The version of solc to use. Should be given as a string in the format ``0.x.x``. If set to ``null``, the version is set based on the contract pragma. Brownie supports solc versions ``>=0.4.22``. * ``optimize``: Set to ``true`` if you wish to enable compiler optimization. * ``runs``: The number of times the optimizer should run. + * ``remappings``: Optional field used to supply :ref:`path remappings `. Optional Settings ================= diff --git a/docs/package-manager.rst b/docs/package-manager.rst index b3aa7f59d..dd0583af2 100644 --- a/docs/package-manager.rst +++ b/docs/package-manager.rst @@ -136,6 +136,10 @@ For example, to import ``SafeMath`` from OpenZeppelin contracts: import "OpenZeppelin/openzeppelin-contracts@2.5.0/contracts/math/SafeMath.sol"; + +You can modify the import path with the ``remappings`` field in your project configuration file. See :ref:`Remapping Installed Packages ` for more information. + + Using Packages in Tests -----------------------