Depend on an unreleased version of AllenNLP (#33)

* Depend on an unreleased version of AllenNLP

* Change how we treat the allennlp dependency

* Cap'n Flake to the rescue!

* ALLENNLP_VERSION_OVERRIDE

* Fix regex

* Actually fix URLs

* fix release workflow

* fix

* update README local install instructions

* readme fixes

* Be explicit about master

* Workflow simplification

Co-authored-by: epwalsh <epwalsh10@gmail.com>

.github/workflows/main.yml

@@ -33,8 +33,7 @@ jobs:
     - name: Install requirements
       run: |
         pip install --upgrade pip setuptools wheel
-        pip install --upgrade git+https://github.com/allenai/allennlp.git@master
-        pip install --upgrade --upgrade-strategy eager -r <(grep -Ev '^allennlp$' requirements.txt)
+        pip install --upgrade --upgrade-strategy eager -r requirements.txt
         pip install --upgrade --upgrade-strategy eager -r dev-requirements.txt
 
     - name: Show pip freeze

.github/workflows/release.yml

@@ -20,6 +20,13 @@ jobs:
       with:
         python-version: ${{ matrix.python }}
 
+    - name: Set AllenNLP version override
+      run:
+        VERSION=$(./scripts/get_version.py current --minimal)
+        ALLENNLP_VERSION_OVERRIDE="allennlp==$VERSION"
+        echo "Setting ALLENNLP_VERSION_OVERRIDE to $ALLENNLP_VERSION_OVERRIDE"
+        echo "::set-env name=ALLENNLP_VERSION_OVERRIDE::$ALLENNLP_VERSION_OVERRIDE"
+
     - name: Install requirements
       run: |
         pip install --upgrade pip setuptools wheel

README.md

@@ -52,8 +52,11 @@ Once you have `allennlp` installed, run the following within the same Python env
 
 ```
 git clone https://github.com/allenai/allennlp-models.git
-EXCLUDE_ALLENNLP_IN_SETUP=true pip install -e .
+cd allennlp-models
+ALLENNLP_VERSION_OVERRIDE='allennlp' pip install -e .
 pip install -r dev-requirements.txt
 ```
 
+The `ALLENNLP_VERSION_OVERRIDE` flag ensures that your local install of `allennlp` won't be overridden. Otherwise `allennlp` will be installed from the latest commit to the master branch on GitHub.
+
 Both `allennlp` and `allennlp-models` are developed and tested side-by-side, so they should be kept up-to-date with each other. If you look at the GitHub Actions [workflow for `allennlp-models`](https://github.com/allenai/allennlp-models/actions), it's always tested against the master branch of `allennlp`. Similarly, `allennlp` is always tested against the master branch of `allennlp-models`.

requirements.txt

@@ -1,4 +1,5 @@
-allennlp
+git+https://github.com/allenai/allennlp.git@master
+
 # For RC models
 word2number>=1.1
 py-rouge==1.1

scripts/get_version.py

@@ -9,13 +9,16 @@
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument("version_type", choices=["stable", "latest", "current"])
+    parser.add_argument("--minimal", action="store_true", default=False)
     return parser.parse_args()
 
 
-def get_current_version() -> str:
+def get_current_version(minimal: bool = False) -> str:
     VERSION: Dict[str, str] = {}
     with open("allennlp_models/version.py", "r") as version_file:
         exec(version_file.read(), VERSION)
+    if minimal:
+        return VERSION["VERSION"]
     return "v" + VERSION["VERSION"]
 
 
@@ -32,11 +35,11 @@ def get_stable_version() -> str:
 def main() -> None:
     opts = parse_args()
     if opts.version_type == "stable":
-        print(get_stable_version())
+        print(get_stable_version(minimal=opts.minimal))
     elif opts.version_type == "latest":
-        print(get_latest_version())
+        print(get_latest_version(minimal=opts.minimal))
     elif opts.version_type == "current":
-        print(get_current_version())
+        print(get_current_version(minimal=opts.minimal))
     else:
         raise NotImplementedError
 

setup.py

@@ -24,13 +24,48 @@
 # Load requirements.txt with a special case for allennlp so we can handle
 # cross-library integration testing.
 with open("requirements.txt") as requirements_file:
-    install_requirements = filter(
-        lambda l: l and not l.startswith("#"), map(lambda l: l.strip(), requirements_file)
-    )
-    install_requirements = [r for r in install_requirements if r != "allennlp"]
-    if not os.environ.get("EXCLUDE_ALLENNLP_IN_SETUP"):
-        requirement = f"allennlp=={VERSION['VERSION']}"
-        install_requirements.append(requirement)
+    import re
+
+    def requirement_is_allennlp(req: str) -> bool:
+        if req == "allennlp":
+            return True
+        if re.match(r"^allennlp[>=<]", req):
+            return True
+        if re.match(r"^(git\+)?(https|ssh)://(git@)?github\.com/allenai/allennlp\.git", req):
+            return True
+        return False
+
+    def fix_url_dependencies(req: str) -> str:
+        """Pip and setuptools disagree about how URL dependencies should be handled."""
+        m = re.match(
+            r"^(git\+)?(https|ssh)://(git@)?github\.com/([\w-]+)/(?P<name>[\w-]+)\.git", req
+        )
+        if m is None:
+            return req
+        else:
+            return f"{m.group('name')} @ {req}"
+
+    install_requirements = []
+    allennlp_requirements = []
+    for line in requirements_file:
+        line = line.strip()
+        if line.startswith("#") or len(line) <= 0:
+            continue
+        if requirement_is_allennlp(line):
+            allennlp_requirements.append(line)
+        else:
+            install_requirements.append(line)
+
+    assert len(allennlp_requirements) == 1
+    allennlp_override = os.environ.get("ALLENNLP_VERSION_OVERRIDE")
+    if allennlp_override is not None:
+        if len(allennlp_override) > 0:
+            allennlp_requirements = [allennlp_override]
+        else:
+            allennlp_requirements = []
+
+    install_requirements.extend(allennlp_requirements)
+    install_requirements = [fix_url_dependencies(req) for req in install_requirements]
 
 # make pytest-runner a conditional requirement,
 # per: https://github.com/pytest-dev/pytest-runner#considerations