From 0f47890276bd3d4d90b55a132704d17a05d40ee6 Mon Sep 17 00:00:00 2001 From: Achilles Rasquinha Date: Mon, 28 Jan 2019 01:28:54 -0600 Subject: [PATCH] feat: Update project and automatic dependency management --- Makefile | 11 +++- README.md | 2 +- src/pipupgrade/__init__.py | 1 + src/pipupgrade/cli/parser.py | 15 ++++- src/pipupgrade/commands/__init__.py | 87 ++++++++++++++++++---------- src/pipupgrade/model/__init__.py | 2 + src/pipupgrade/model/project.py | 30 ++++++++++ src/pipupgrade/util/environ.py | 27 +++++++++ src/pipupgrade/util/system.py | 54 ++++++++++++++++- src/pipupgrade/util/types.py | 13 ++++- tests/pipupgrade/util/test_string.py | 12 +++- tests/pipupgrade/util/test_system.py | 43 +++++++++++++- tox.ini | 9 +-- 13 files changed, 263 insertions(+), 43 deletions(-) create mode 100644 src/pipupgrade/model/__init__.py create mode 100644 src/pipupgrade/model/project.py diff --git a/Makefile b/Makefile index bcbcc74..4f706c0 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ ifneq (${VERBOSE},true) endif ifneq (${PIPCACHEDIR},) - $(eval PIPCACHEDIR = --cache-dir $(PIPCACHEDIR)) + $(eval PIPCACHEDIR := --cache-dir $(PIPCACHEDIR)) endif $(call log,INFO,Building Requirements) @@ -143,7 +143,14 @@ docker-build: clean ## Build the Docker Image. docker-tox: clean ## Test using Docker Tox Image. $(call log,INFO,Testing the Docker Image) - @docker run --rm -v $(shell pwd):/app themattrix/tox + $(eval TMPDIR := /tmp/$(PROJECT)-$(shell date +"%Y_%m_%d_%H_%M_%S")) + + @mkdir $(TMPDIR) + @cp -R . $(TMPDIR) + + @docker run --rm -v $(TMPDIR):/app themattrix/tox + + @rm -rf $(TMPDIR) help: ## Show help and exit. @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) \ No newline at end of file diff --git a/README.md b/README.md index 97b6186..0e0a63b 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ * Updates system packages and local packages. * Updates packages mentioned within a `requirements.txt` file (Also pins upto-date versions if mentioned). * Detects semantic version to avoid updates that break changes. -* Python 2.7+ and Python 3.4+ compatible. Also pip 9+, pip 10+ and pip 18+ compatible. +* Python 2.7+ and Python 3.4+ compatible. Also pip 9+, pip 10+, pip 18+, pip19+ compatible. * Zero Dependencies! #### Installation diff --git a/src/pipupgrade/__init__.py b/src/pipupgrade/__init__.py index c1816ed..27b755f 100644 --- a/src/pipupgrade/__init__.py +++ b/src/pipupgrade/__init__.py @@ -1,5 +1,6 @@ # imports - module imports from pipupgrade.__attr__ import ( + __name__, __version__ ) from pipupgrade.__main__ import main \ No newline at end of file diff --git a/src/pipupgrade/cli/parser.py b/src/pipupgrade/cli/parser.py index be8f520..43e3497 100644 --- a/src/pipupgrade/cli/parser.py +++ b/src/pipupgrade/cli/parser.py @@ -2,12 +2,13 @@ import argparse # imports - module imports -from pipupgrade.__attr__ import ( +from pipupgrade.__attr__ import ( __name__, __version__, __description__, __command__ ) +from pipupgrade.util.environ import getenv def get_parser(): parser = argparse.ArgumentParser( @@ -43,6 +44,18 @@ def get_parser(): action = "append", help = "Path(s) to Project" ) + parser.add_argument("--git-username", + help = "Git Username", + default = getenv("GIT_USERNAME") + ) + parser.add_argument("--git-email", + help = "Git Email", + default = getenv("GIT_EMAIL") + ) + parser.add_argument("--pull-request", + action = "store_true", + help = "Perform a Pull Request" + ) parser.add_argument("-u", "--user", action = "store_true", help = "Install to the Python user install directory for environment \ diff --git a/src/pipupgrade/commands/__init__.py b/src/pipupgrade/commands/__init__.py index c6570c3..6760115 100644 --- a/src/pipupgrade/commands/__init__.py +++ b/src/pipupgrade/commands/__init__.py @@ -7,10 +7,12 @@ import glob # imports - module imports +from pipupgrade.model import Project from pipupgrade.commands.util import cli_format from pipupgrade.table import Table from pipupgrade.util.string import strip, pluralize -from pipupgrade.util.system import read, write +from pipupgrade.util.system import read, write, popen +from pipupgrade.util.environ import getenvvar from pipupgrade import _pip, request as req, cli, semver from pipupgrade.__attr__ import __name__ @@ -82,28 +84,51 @@ def _update_requirements(path, package): version = re.escape(package.current_version) ) lines = content.splitlines() + nlines = len(lines) with open(path, "w") as f: - for line in lines: + for i, line in enumerate(lines): if re.search(pattern, line, flags = re.IGNORECASE): line = line.replace( "==%s" % package.current_version, "==%s" % package.latest_version ) - f.write(line) - except Exception as e: - write(path, content) + f.write(line) -def _get_included_requirements(fname): - path = osp.realpath(fname) + if i < nlines - 1: + f.write("\n") + except Exception: + # In case we fucked up! + write(path, content, force = True) +def _get_included_requirements(filename): + path = osp.realpath(filename) + basepath = osp.dirname(path) + requirements = [ ] + with open(path) as f: + content = f.readlines() + + for line in content: + line = strip(line) + + if line.startswith("-r "): + filename = line.split("-r ")[1] + realpath = osp.join(basepath, filename) + requirements.append(realpath) + + requirements += _get_included_requirements(realpath) + + return requirements @cli.command def command( requirements = [ ], project = None, + pull_request = False, + git_username = None, + git_email = None, latest = False, self = False, user = False, @@ -124,21 +149,11 @@ def command( cli.echo("%s upto date." % cli_format(package, cli.CYAN)) else: if project: - for p in project: - projpath = osp.abspath(p) - requirements = requirements or [ ] - - # COLLECT ALL THE REQUIREMENTS FILES! - - # Detect Requirements Files - # Check requirements*.txt files in current directory. - for requirement in glob.glob(osp.join(projpath, "requirements*.txt")): - requirements.insert(0, requirement) + requirements = requirements or [ ] - # Check if requirements is a directory - if osp.isdir(osp.join(projpath, "requirements")): - for requirement in glob.glob(osp.join(projpath, "requirements", "*.txt")): - requirements.insert(0, requirement) + for i, p in enumerate(project): + project[i] = Project(osp.abspath(p)) + requirements += project[i].requirements if requirements: for requirement in requirements: @@ -148,14 +163,7 @@ def command( cli.echo(cli_format("{} not found.".format(path), cli.RED)) sys.exit(os.EX_NOINPUT) else: - filenames = _get_included_requirements(requirement) - - # with open(path) as f: - # content = f.readlines() - - # for line in content: - # if strip(line).startswith("-r "): - # # fname = + requirements += _get_included_requirements(requirement) for requirement in requirements: path = osp.realpath(requirement) @@ -241,4 +249,23 @@ def command( _pip.install(package.name, user = user, quiet = not verbose, no_cache_dir = True, upgrade = True) else: - cli.echo("%s upto date." % cli_format(stitle, cli.CYAN)) \ No newline at end of file + cli.echo("%s upto date." % cli_format(stitle, cli.CYAN)) + + if project and pull_request: + if not git_username: + raise ValueError('Git Username not found. Use --git-username or the environment variable "%s" to set value.' % getenvvar("GIT_USERNAME")) + if not git_email: + raise ValueError('Git Email not found. Use --git-email or the environment variable "%s" to set value.' % getenvvar("GIT_EMAIL")) + + for p in project: + popen("git config user.name %s" % git_username, cwd = p.path) + popen("git config user.email %s" % git_email, cwd = p.path) + + _, output, _ = popen("git status -s", output = True) + + if output: + # TODO: cross-check with "git add" ? + popen("git add %s" % " ".join(p.requirements), cwd = p.path) + popen("git commit -m 'fix(dependencies): Update dependencies to latest.'", cwd = p.path) + + popen("git push", cwd = p.path) \ No newline at end of file diff --git a/src/pipupgrade/model/__init__.py b/src/pipupgrade/model/__init__.py new file mode 100644 index 0000000..e8c6836 --- /dev/null +++ b/src/pipupgrade/model/__init__.py @@ -0,0 +1,2 @@ +# imports - module imports +from pipupgrade.model.project import Project \ No newline at end of file diff --git a/src/pipupgrade/model/project.py b/src/pipupgrade/model/project.py new file mode 100644 index 0000000..bc9f62b --- /dev/null +++ b/src/pipupgrade/model/project.py @@ -0,0 +1,30 @@ +# imports - standard imports +import os.path as osp +import glob + +class Project: + def __init__(self, path): + path = osp.realpath(path) + + if not osp.exists(path): + raise ValueError("Path %s does not exist." % path) + + self.path = path + self.requirements = self._get_requirements() + + def _get_requirements(self): + # COLLECT ALL THE REQUIREMENTS FILES! + path = self.path + requirements = [ ] + + # Detect Requirements Files + # Check requirements*.txt files in current directory. + for requirement in glob.glob(osp.join(path, "requirements*.txt")): + requirements.append(requirement) + + # Check if requirements is a directory + if osp.isdir(osp.join(path, "requirements")): + for requirement in glob.glob(osp.join(path, "requirements", "*.txt")): + requirements.append(requirement) + + return requirements \ No newline at end of file diff --git a/src/pipupgrade/util/environ.py b/src/pipupgrade/util/environ.py index 7d73a31..9f8098b 100644 --- a/src/pipupgrade/util/environ.py +++ b/src/pipupgrade/util/environ.py @@ -1,3 +1,30 @@ +# imports - standard imports +import os + +# imports - module imports +from pipupgrade.util.types import auto_typecast +import pipupgrade + +PREFIX = "%s" % pipupgrade.__name__.upper() + +def getenvvar(name, prefix = PREFIX, seperator = "_"): + if not prefix: + seperator = "" + + envvar = "%s%s%s" % (prefix, seperator, name) + return envvar + +def getenv(name, default = None, cast = True, prefix = PREFIX, seperator = "_", raise_err = False): + envvar = getenvvar(name, prefix = prefix, seperator = seperator) + + if not envvar in list(os.environ) and raise_err: + raise KeyError("Environment Variable %s not found." % envvar) + + value = os.getenv(envvar, default) + value = auto_typecast(value) if cast else value + + return value + def value_to_envval(value): """ Convert python types to environment values diff --git a/src/pipupgrade/util/system.py b/src/pipupgrade/util/system.py index c3956b0..b3467b4 100644 --- a/src/pipupgrade/util/system.py +++ b/src/pipupgrade/util/system.py @@ -1,5 +1,10 @@ # imports - standard imports -import os.path as osp +import os, os.path as osp +import subprocess as sp + +# imports - module imports +from pipupgrade.util.string import strip +from pipupgrade._compat import iteritems def read(fname): with open(fname) as f: @@ -10,4 +15,49 @@ def write(fname, data = None, force = False): if not osp.exists(fname) or force: with open(fname, "w") as f: if data: - f.write(data) \ No newline at end of file + f.write(data) + +def popen(*args, **kwargs): + output = kwargs.get("output", False) + directory = kwargs.get("cwd") + environment = kwargs.get("env") + shell = kwargs.get("shell", True) + raise_err = kwargs.get("raise_err", True) + + environ = os.environ.copy() + if environment: + environ.update(environment) + + for k, v in iteritems(environ): + environ[k] = str(v) + + command = " ".join([str(arg) for arg in args]) + + proc = sp.Popen(command, + stdin = sp.PIPE if output else None, + stdout = sp.PIPE if output else None, + stderr = sp.PIPE if output else None, + env = environ, + cwd = directory, + shell = shell + ) + + code = proc.wait() + + if code and raise_err: + raise sp.CalledProcessError(code, command) + + if output: + output, error = proc.communicate() + + if output: + output = output.decode("utf-8") + output = strip(output) + + if error: + error = error.decode("utf-8") + error = strip(error) + + return code, output, error + else: + return code diff --git a/src/pipupgrade/util/types.py b/src/pipupgrade/util/types.py index bc9fed8..97ec769 100644 --- a/src/pipupgrade/util/types.py +++ b/src/pipupgrade/util/types.py @@ -53,4 +53,15 @@ def get_function_arguments(fn): if not success: raise ValueError("Unknown Python Version {} for fetching functional arguments.".format(sys.version)) - return params \ No newline at end of file + return params + +def auto_typecast(value): + str_to_bool = lambda x: { "True": True, "False": False, "None": None}[x] + + for type_ in (str_to_bool, int, float): + try: + return type_(value) + except (KeyError, ValueError, TypeError): + pass + + return value \ No newline at end of file diff --git a/tests/pipupgrade/util/test_string.py b/tests/pipupgrade/util/test_string.py index 9970345..e77577f 100644 --- a/tests/pipupgrade/util/test_string.py +++ b/tests/pipupgrade/util/test_string.py @@ -1,7 +1,17 @@ # imports - module imports -from pipupgrade.util.string import strip_ansi, pluralize, kebab_case +from pipupgrade.util.string import strip, strip_ansi, pluralize, kebab_case from pipupgrade import cli +def test_strip(): + string = "foobar" + assert strip(string) == string + + string = "\n foobar\nfoobar \n " + assert strip(string) == "foobar\nfoobar" + + string = "\n\n\n" + assert strip(string) == "" + def test_strip_ansi(): assert strip_ansi(cli.format("foobar", cli.GREEN)) == "foobar" assert strip_ansi(cli.format("barfoo", cli.BOLD)) == "barfoo" diff --git a/tests/pipupgrade/util/test_system.py b/tests/pipupgrade/util/test_system.py index c73e0b5..a6c0ed9 100644 --- a/tests/pipupgrade/util/test_system.py +++ b/tests/pipupgrade/util/test_system.py @@ -1,4 +1,12 @@ -from pipupgrade.util.system import read, write +# imports - standard imports +import os, os.path as osp +import subprocess as sp + +# imports - test imports +import pytest + +# imports - module imports +from pipupgrade.util.system import read, write, popen def test_read(tmpdir): directory = tmpdir.mkdir("tmp") @@ -34,3 +42,36 @@ def test_write(tmpdir): write(path, next_, force = True) assert tempfile.read() == next_ + +def test_popen(tmpdir): + directory = tmpdir.mkdir("tmp") + dirpath = str(directory) + + string = "Hello, World!" + + code, out, err = popen("echo '%s'" % string, + output = True) + assert code == 0 + assert out == string + assert not err + + env = dict({ "FOOBAR": "foobar" }) + code, out, err = popen("echo $FOOBAR; echo $PATH", + output = True, env = env) + assert code == 0 + assert out == "%s\n%s" % (env["FOOBAR"], os.environ["PATH"]) + assert not err + + with pytest.raises(sp.CalledProcessError): + code = popen("exit 42") + + errstr = "foobar" + code, out, err = popen('python -c "raise Exception("%s")"' % errstr, + output = True, raise_err = False) + assert code == 1 + assert not out + assert errstr in err + + filename = "foobar.txt" + popen("touch %s" % filename, cwd = dirpath) + assert osp.exists(osp.join(dirpath, filename)) \ No newline at end of file diff --git a/tox.ini b/tox.ini index 5ec6d1a..adc9f8c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,9 @@ [tox] envlist = - py27-pip{9,10,18} - py34-pip{9,10,18} - py35-pip{9,10,18} - py36-pip{9,10,18} + py27-pip{9,10,18,19} + py34-pip{9,10,18,19} + py35-pip{9,10,18,19} + py36-pip{9,10,18,19} [testenv] deps = @@ -11,5 +11,6 @@ deps = pip9: pip < 10.0 pip10: pip >= 10.0 pip18: pip >= 18.0 + pip19: pip >= 19.0 commands = pytest {toxinidir}/tests --cov {envsitepackagesdir}/pipupgrade \ No newline at end of file