Track raw provider version, rather than rebuilding.

mach_nix/data/providers.py

@@ -8,7 +8,7 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from operator import itemgetter
-from typing import List, Optional, Tuple, Iterable
+from typing import Any, Iterable, List, Optional, Tuple
 
 import distlib.markers
 from pkg_resources import RequirementParseError
@@ -24,6 +24,7 @@
 class Candidate:
     name: str
     ver: Version
+    raw_version: str
     selected_extras: tuple
     provider_info: 'ProviderInfo'
     build: str = None
@@ -35,6 +36,7 @@ class ProviderInfo:
     wheel_fname: str = None  # only required for wheel
     url: str = None
     hash: str = None
+    data: Any = None  # Provider specific data
 
     def toDict(self):
         return dict(
@@ -113,16 +115,6 @@ def all_candidates_sorted(self, name, extras=None, build=None) -> Iterable[Candi
     def name(self):
         pass
 
-    def get_reqs_for_extras(self, pkg_name, pkg_ver, extras):
-        install_reqs_wo_extras, _ = self.get_pkg_reqs(pkg_name, pkg_ver)
-        install_reqs_w_extras, _ = self.get_pkg_reqs(pkg_name, pkg_ver, extras=extras)
-        if install_reqs_wo_extras is None:
-            install_reqs_wo_extras = []
-        if install_reqs_w_extras is None:
-            install_reqs_w_extras = []
-        install_reqs = set(install_reqs_w_extras) - set(install_reqs_wo_extras)
-        return list(install_reqs)
-
     def unify_key(self, key: str) -> str:
         return key.replace('_', '-').lower()
 
@@ -139,11 +131,6 @@ def get_pkg_reqs(self, candidate: Candidate) -> Tuple[Optional[List[Requirement]
     def all_candidates(self, name, extras=None, build=None) -> Iterable[Candidate]:
         pass
 
-    @abstractmethod
-    def deviated_version(self, pkg_name, normalized_version: Version, build):
-        # returns version like originally specified by package maintainer without normalization
-        pass
-
 
 class CombinedDependencyProvider(DependencyProviderBase):
     name = 'combined'
@@ -184,15 +171,8 @@ def allowed_providers_for_pkg(self, pkg_name):
         selected_providers = ((name, p) for name, p in self._all_providers.items() if name in provider_keys)
         return dict(sorted(selected_providers, key=lambda x: provider_keys.index(x[0])))
 
-    def get_provider(self, pkg_name, pkg_version, build) -> DependencyProviderBase:
-        for type, provider in self.allowed_providers_for_pkg(pkg_name).items():
-            if pkg_version in (c.ver for c in provider.all_candidates(pkg_name, build=build)):
-                return provider
-
     def get_pkg_reqs(self, c: Candidate) -> Tuple[Optional[List[Requirement]], Optional[List[Requirement]]]:
-        for provider in self.allowed_providers_for_pkg(c.name).values():
-            if c in provider.all_candidates(c.name, c.selected_extras, c.build):
-                return provider.get_pkg_reqs(c)
+        return c.provider_info.provider.get_pkg_reqs(c)
 
     def list_all_providers_for_pkg(self, pkg_name, extras, build):
         result = []
@@ -236,9 +216,6 @@ def all_candidates_sorted(self, pkg_name, extras=None, build=None) -> Iterable[C
     def all_candidates(self, name, extras=None, build=None) -> Iterable[Candidate]:
         return self.all_candidates_sorted(name, extras, build)
 
-    def deviated_version(self, pkg_name, pkg_version: Version, build):
-        self.get_provider(pkg_name, pkg_version, build).deviated_version(pkg_name, pkg_version, build)
-
 
 class NixpkgsDependencyProvider(DependencyProviderBase):
     name = 'nixpkgs'
@@ -256,20 +233,19 @@ def __init__(
         self.sdist_provider = sdist_provider
 
     def get_pkg_reqs(self, c: Candidate) -> Tuple[Optional[List[Requirement]], Optional[List[Requirement]]]:
-        if not self.nixpkgs.exists(c.name, c.ver):
-            raise Exception(f"Cannot find {c.name}:{c.ver} in nixpkgs")
         requirements = self.nixpkgs.get_requirements(c.name, c.ver)
         if requirements is not None:
             return list(parse_reqs(requirements)), None
-        install_reqs, setup_reqs = [], []
-        try:
-            return self.sdist_provider.get_pkg_reqs(c)
-        except PackageNotFound:
-            try:
-                # wheel provider only knows install deps
-                return self.wheel_provider.get_pkg_reqs(c)[0], None
-            except:
-                return None, None
+
+        for provider in (self.sdist_provider, self.wheel_provider):
+            candidates = [
+                candidate
+                for candidate in provider.all_candidates(c.name)
+                if candidate.ver == c.ver
+            ]
+            if len(candidates) > 0:
+                return provider.get_pkg_reqs(candidates[0])
+        return None, None
 
     def all_candidates(self, pkg_name, extras=None, build=None) -> Iterable[Candidate]:
         if build:
@@ -280,14 +256,11 @@ def all_candidates(self, pkg_name, extras=None, build=None) -> Iterable[Candidat
         return [Candidate(
             pkg_name,
             p.ver,
+            str(p.ver),
             extras,
             provider_info=ProviderInfo(self)
         ) for p in self.nixpkgs.get_all_candidates(name)]
 
-    def deviated_version(self, pkg_name, normalized_version: Version, build):
-        # not necessary for nixpkgs provider since source doesn't need to be fetched
-        return str(normalized_version)
-
 
 @dataclass
 class WheelRelease:
@@ -338,23 +311,21 @@ def all_candidates(self, pkg_name, extras=None, build=None) -> List[Candidate]:
         return [Candidate(
             w.name,
             parse_ver(w.ver),
+            w.ver,
             extras,
-            provider_info=ProviderInfo(provider=self, wheel_fname=w.fn)
+            provider_info=ProviderInfo(provider=self, wheel_fname=w.fn, data=w)
         ) for w in self._suitable_wheels(pkg_name)]
 
     def get_pkg_reqs(self, c: Candidate) -> Tuple[List[Requirement], List[Requirement]]:
         """
         Get requirements for package
         """
-        reqs_raw = self._choose_wheel(c.name, c.ver).requires_dist
+        reqs_raw = c.provider_info.data.requires_dist
         if reqs_raw is None:
             reqs_raw = []
         # handle extras by evaluationg markers
         install_reqs = list(filter_reqs_by_eval_marker(parse_reqs(reqs_raw), self.context_wheel, c.selected_extras))
-        return install_reqs, []
-
-    def deviated_version(self, pkg_name, pkg_version: Version, build):
-        return self._choose_wheel(pkg_name, pkg_version).ver
+        return install_reqs, None
 
     def _all_releases(self, pkg_name):
         name = self.unify_key(pkg_name)
@@ -460,20 +431,12 @@ def _get_candidates(self, name) -> dict:
                     parsed_py_requires = list(parse_reqs(f"python{specs}"))
                     if not filter_versions([self.py_ver.version], parsed_py_requires[0]):
                         continue
-                parsed_ver = parse_ver(ver)
-                candidates[parsed_ver] = pkg_data
+                candidates[ver] = pkg_data
         return candidates
 
-    def deviated_version(self, pkg_name, normalized_version: Version, build):
-        for raw_ver in self.data[normalize_name(pkg_name)].keys():
-            if parse_ver(raw_ver) == normalized_version:
-                return raw_ver
-        raise Exception(
-            f"Something went wrong while trying to find the deviated version for {pkg_name}:{normalized_version}")
-
-    def get_reqs_for_extras(self, pkg_name, pkg_ver: Version, extras):
-        name = self.unify_key(pkg_name)
-        pkg = self._get_candidates(name)[pkg_ver]
+    def _get_reqs_for_extras(self, pkg, extras):
+        if extras is None:
+            return []
         extras = set(extras)
         requirements = []
         if 'extras_require' in pkg:
@@ -491,9 +454,7 @@ def get_pkg_reqs(self, c: Candidate) -> Tuple[List[Requirement], List[Requiremen
         """
         Get requirements for package
         """
-        if c.ver not in self._get_candidates(c.name):
-            raise PackageNotFound(c.name, c.ver, self.name)
-        pkg = self._get_candidates(c.name)[c.ver]
+        pkg = c.provider_info.data
         requirements = dict(
             setup_requires=[],
             install_requires=[]
@@ -507,17 +468,18 @@ def get_pkg_reqs(self, c: Candidate) -> Tuple[List[Requirement], List[Requiremen
                 requirements[t] = list(filter_reqs_by_eval_marker(reqs, self.context))
         # even if no extras are selected we need to collect reqs for extras,
         # because some extras consist of only a marker which needs to be evaluated
-        requirements['install_requires'] += self.get_reqs_for_extras(c.name, c.ver, c.selected_extras)
+        requirements['install_requires'] += self._get_reqs_for_extras(pkg, c.selected_extras)
         return requirements['install_requires'], requirements['setup_requires']
 
     def all_candidates(self, pkg_name, extras=None, build=None) -> Iterable[Candidate]:
         if build:
             return []
         return [Candidate(
             pkg_name,
+            parse_ver(ver),
             ver,
             extras,
-            provider_info=ProviderInfo(self)
+            provider_info=ProviderInfo(self, data=pkg)
         ) for ver, pkg in self._get_candidates(pkg_name).items()]
 
 
@@ -598,62 +560,60 @@ def name(self):
         return f"conda/{self.channel}"
 
     def get_pkg_reqs(self, c: Candidate) -> Tuple[List[Requirement], List[Requirement]]:
-        name = normalize_name(c.name)
-        deviated_ver = self.deviated_version(name, c.ver, c.build)
-        candidate = self.pkgs[name][deviated_ver][c.build]
+        candidate = c.provider_info.data
         depends = list(filter(
             lambda d: d.split()[0] not in self.ignored_pkgs,
-           # lambda d: d.split()[0] not in self.ignored_pkgs and not d.startswith('_'),
+            # lambda d: d.split()[0] not in self.ignored_pkgs and not d.startswith('_'),
             candidate['depends']
             # always add optional dependencies to ensure constraints are applied
             + (candidate['constrains'] if 'constrains' in candidate else [])
         ))
-        return list(parse_reqs(depends)), []
+        return list(parse_reqs(depends)), None
 
     @cached()
+    def all_candidates_sorted(self, name, extras, build) -> Iterable[Candidate]:
+        candidates = self.all_candidates(name, extras, build)
+        candidates.sort(key=lambda c: (c.ver, c.provider_info.data['build_number']))
+        return candidates
+
+
     def all_candidates(self, pkg_name, extras=None, build=None) -> Iterable[Candidate]:
         pkg_name = normalize_name(pkg_name)
         if pkg_name not in self.pkgs:
             return []
         candidates = []
-        for ver in self.pkgs[pkg_name].keys():
-            for p in self.compatible_builds(pkg_name, parse_ver(ver), build):
-                if 'sha256' not in p:
-                    print(
-                        f"Ignoring conda package {p['name']}:{p['version']} from provider {self.channel} \n"
-                        "since it doesn't provide a sha256 sum.\n")
+        for p in self.compatible_builds(pkg_name, build):
+            if 'sha256' not in p:
+                print(
+                    f"Ignoring conda package {p['name']}:{p['version']} from provider {self.channel} \n"
+                    "since it doesn't provide a sha256 sum.\n")
+            else:
+                if self.channel in ('free', 'intel', 'main', 'r'):
+                    url = f"https://repo.anaconda.com/pkgs/{self.channel}/{p['subdir']}/{p['fname']}"
                 else:
-                    if self.channel in ('free', 'intel', 'main', 'r'):
-                        url = f"https://repo.anaconda.com/pkgs/{self.channel}/{p['subdir']}/{p['fname']}"
-                    else:
-                        url = f"https://anaconda.org/{self.channel}/{p['name']}/" \
-                              f"{p['version']}/download/{p['subdir']}/{p['fname']}"
-                    candidates.append(Candidate(
-                        p['name'],
-                        parse_ver(p['version']),
-                        selected_extras=tuple(),
-                        build=p['build'],
-                        provider_info=ProviderInfo(
-                            self,
-                            url=url,
-                            hash=p['sha256']
-                        )
-                    ))
-                    if 'collisions' in p:
-                        print(
-                            f"WARNING: Colliding conda package in channel '{self.channel}' "
-                            f"Ignoring {list(map(itemgetter(0), p['collisions']))} "
-                            f"from {list(map(itemgetter(1), p['collisions']))} "
-                            f"in favor of {p['name']} from '{p['subdir']}'")
+                    url = f"https://anaconda.org/{self.channel}/{p['name']}/" \
+                          f"{p['version']}/download/{p['subdir']}/{p['fname']}"
+                candidates.append(Candidate(
+                    p['name'],
+                    parse_ver(p['version']),
+                        p['version'],
+                    selected_extras=tuple(),
+                    build=p['build'],
+                    provider_info=ProviderInfo(
+                        self,
+                        url=url,
+                        hash=p['sha256'],
+                        data=p,
+                    )
+                ))
+                if 'collisions' in p:
+                    print(
+                        f"WARNING: Colliding conda package in channel '{self.channel}' "
+                        f"Ignoring {list(map(itemgetter(0), p['collisions']))} "
+                        f"from {list(map(itemgetter(1), p['collisions']))} "
+                        f"in favor of {p['name']} from '{p['subdir']}'")
         return candidates
 
-    def deviated_version(self, pkg_name, normalized_version: Version, build):
-        for builds in self.pkgs[pkg_name].values():
-            for p in builds.values():
-                if parse_ver(p['version']) == normalized_version:
-                    return p['version']
-        raise Exception(f"Cannot find deviated version for {pkg_name}:{normalized_version}")
-
     def python_ok(self, build):
         for dep in build['depends']:
             if dep == "pypy" or dep.startswith("pypy "):
@@ -665,19 +625,18 @@ def python_ok(self, build):
         return True
 
     @cached()
-    def compatible_builds(self, pkg_name, pkg_version: Version, build=None) -> list:
-        deviated_ver = self.deviated_version(pkg_name, pkg_version, build)
-        if build:
-            matched = set(fnmatch.filter(self.pkgs[pkg_name][deviated_ver], build))
-            pkgs = \
-                [p for p in self.pkgs[pkg_name][deviated_ver].values() if p['build'] in matched and self.python_ok(p)]
-            pkgs.sort(key=lambda p: p['build_number'], reverse=True)
-            return pkgs
-        compatible = []
-        for build in self.pkgs[pkg_name][deviated_ver].values():
-            # continue if python incompatible
-            if not self.python_ok(build):
-                continue
-            # python is compatible
-            compatible.append(build)
-        return compatible
+    def compatible_builds(self, pkg_name, build_pattern) -> list:
+        if build_pattern:
+            # fnmatch caches the regexes it builds.
+            def build_matches(build):
+                return fnmatch.fnmatch(build, build_pattern)
+        else:
+            def build_matches(build):
+                return True
+        return [
+            pkg_data
+            for ver, pkg_builds in self.pkgs[pkg_name].items()
+            for build, pkg_data in pkg_builds.items()
+            if self.python_ok(pkg_data)
+            and build_matches(build)
+        ]

mach_nix/deptree.py

@@ -28,7 +28,7 @@ def mark_removed_circular_dep(pkgs: Dict[str, ResolvedPkg], G: DiGraph, node, re
 
 
 def remove_dependecy(pkgs: Dict[str, ResolvedPkg], G: DiGraph, node_from, node_to):
-    if node_to in pkgs[node_from].build_inputs:
+    if pkgs[node_from].build_inputs is not None and node_to in pkgs[node_from].build_inputs:
         raise Exception(
             f"Fata error: cycle detected in setup requirements\n"
             f"Cannot fix automatically.\n{[node_from, node_to]}")
@@ -75,7 +75,7 @@ class Limiter:
 
         def name(self, node_name):
             if node_name in self.visited:
-                if indexed_pkgs[node_name].build_inputs + indexed_pkgs[node_name].prop_build_inputs == []:
+                if indexed_pkgs[node_name].build_inputs or indexed_pkgs[node_name].prop_build_inputs:
                     return make_name(indexed_pkgs[node_name], nixpkgs)
                 return f"{make_name(indexed_pkgs[node_name], nixpkgs)} -> ..."
             return make_name(indexed_pkgs[node_name], nixpkgs)