Implement missing METADATA parsing via pkginfo

internal_deps.bzl

@@ -96,6 +96,50 @@ def rules_python_internal_deps():
         ],
     )
 
+    maybe(
+        http_file,
+        name = "apispec_2_0_1_whl",
+        downloaded_file_path = "apispec-2.0.1-py2.py3-none-any.whl",
+        sha256 = "427293594699c77753b8fdc6d78d8dfe267c394df5186c236d57f6ef1af95027",
+        # From https://pypi.python.org/pypi/apispec
+        urls = [("https://pypi.python.org/packages/ea/c3/" +
+                 "1f60ae577a1f92a60bd254d3d8b7747f8ba02e5d5638bde79ea17ae45208/" +
+                 "apispec-2.0.1-py2.py3-none-any.whl")],
+    )
+
+    maybe(
+        http_file,
+        name = "gunicorn_19_9_0_whl",
+        downloaded_file_path = "gunicorn-19.9.0-py2.py3-none-any.whl",
+        sha256 = "aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471",
+        # From https://pypi.python.org/pypi/gunicorn
+        urls = [("https://pypi.python.org/packages/8c/da/" +
+                 "b8dd8deb741bff556db53902d4706774c8e1e67265f69528c14c003644e6/" +
+                 "gunicorn-19.9.0-py2.py3-none-any.whl")],
+    )
+
+    maybe(
+        http_file,
+        name = "pyOpenSSL_19_0_0_whl",
+        downloaded_file_path = "pyOpenSSL-19.0.0-py2.py3-none-any.whl",
+        sha256 = "c727930ad54b10fc157015014b666f2d8b41f70c0d03e83ab67624fd3dd5d1e6",
+        # From https://pypi.python.org/pypi/pyOpenSSL
+        urls = [("https://pypi.python.org/packages/01/c8/" +
+                 "ceb170d81bd3941cbeb9940fc6cc2ef2ca4288d0ca8929ea4db5905d904d/" +
+                 "pyOpenSSL-19.0.0-py2.py3-none-any.whl")],
+    )
+
+    maybe(
+        http_file,
+        name = "google_cloud_spanner_1_9_0_whl",
+        downloaded_file_path = "google_cloud_spanner-1.9.0-py2.py3-none-any.whl",
+        sha256 = "963d0afe48bef9f8c7fef3982aa69ea46c12703c1379f66f52f5d7ced47a0b42",
+        # From https://pypi.python.org/pypi/google-cloud-spanner
+        urls = [("https://pypi.python.org/packages/b1/df/" +
+                 "96686bc6abafacb579334f8c61be2f025f1be161d266893d17b47afd7685/" +
+                 "google_cloud_spanner-1.9.0-py2.py3-none-any.whl")],
+    )
+
     maybe(
         git_repository,
         name = "subpar",

packaging/BUILD

@@ -23,18 +23,23 @@ py_library(
     srcs = ["whl.py"],
     deps = [
         requirement("setuptools"),
+        requirement("pkginfo"),
     ],
 )
 
 py_test(
     name = "whl_test",
     srcs = ["whl_test.py"],
     data = [
+        "@apispec_2_0_1_whl//file",
         "@futures_2_2_0_whl//file",
         "@futures_3_1_1_whl//file",
         "@google_cloud_language_whl//file",
+        "@google_cloud_spanner_1_9_0_whl//file",
         "@grpc_whl//file",
+        "@gunicorn_19_9_0_whl//file",
         "@mock_whl//file",
+        "@pyOpenSSL_19_0_0_whl//file",
     ],
     deps = [
         ":whl",

packaging/whl.py

@@ -17,6 +17,7 @@
 import json
 import os
 import pkg_resources
+import pkginfo
 import re
 import stat
 import zipfile
@@ -81,9 +82,8 @@ def metadata(self):
           return json.loads(f.read().decode("utf-8"))
       except KeyError:
           pass
-      # fall back to METADATA file (https://www.python.org/dev/peps/pep-0427/)
-      with whl.open(self._dist_info() + '/METADATA') as f:
-        return self._parse_metadata(f.read().decode("utf-8"))
+      # fall back to parsing using pkginfo
+      return self._parse_with_pkginfo(pkginfo.Wheel(self.path()))
 
   def name(self):
     return self.metadata().get('name')
@@ -107,10 +107,12 @@ def dependencies(self, extra=None):
         # Match the requirements for the extra we're looking for.
         continue
       marker = requirement.get('environment')
-      if marker and not pkg_resources.evaluate_marker(marker):
-        # The current environment does not match the provided PEP 508 marker,
-        # so ignore this requirement.
-        continue
+      if marker:
+        parsed_marker = pkg_resources.extern.packaging.markers.Marker(marker)
+        if not parsed_marker.evaluate({'extra': extra}):
+          # The current environment does not match the provided PEP 508 marker,
+          # so ignore this requirement.
+          continue
       requires = requirement.get('requires', [])
       for entry in requires:
         # Strip off any trailing versioning data.
@@ -139,11 +141,31 @@ def expand(self, directory):
                 name = os.path.join(directory, name)
                 set_extracted_file_to_default_mode_plus_executable(name)
 
-  # _parse_metadata parses METADATA files according to https://www.python.org/dev/peps/pep-0314/
-  def _parse_metadata(self, content):
-    # TODO: handle fields other than just name
-    name_pattern = re.compile('Name: (.*)')
-    return { 'name': name_pattern.search(content).group(1) }
+  # _parse_with_pkginfo parses metadata from pkginfo.Wheel object.
+  def _parse_with_pkginfo(self, pkg):
+    metadata = { 'name' : pkg.name }
+    if len(pkg.provides_extras) > 0:
+      metadata['extras'] = sorted(set(pkg.provides_extras))
+    metadata['run_requires'] = []
+    for requirement_str in pkg.requires_dist:
+      requirement = pkg_resources.Requirement.parse(requirement_str)
+
+      r = {}
+      r['requires'] = [requirement.name]
+      r['requires'].extend([requirement.name + "[{0}]".format(extra)
+                            for extra in requirement.extras])
+
+      if requirement.marker:
+        r['environment'] = str(requirement.marker)
+
+        EXTRA_RE = re.compile("extra == ['\"](.*)['\"]")
+        extra_match = EXTRA_RE.search(str(requirement.marker))
+        if extra_match:
+          r['extra'] = extra_match.group(1)
+
+      metadata['run_requires'].append(r)
+
+    return metadata
 
 
 parser = argparse.ArgumentParser(

packaging/whl_test.py

@@ -56,6 +56,41 @@ def test_whl_with_METADATA_file(self):
     self.assertEqual(set(wheel.dependencies()), set())
     self.assertEqual('pypi__futures_2_2_0', wheel.repository_suffix())
 
+  def test_whl_with_METADATA_file_extras(self):
+    # Wheel which provides specific dependencies for extras.
+    td = TestData('apispec_2_0_1_whl/file/apispec-2.0.1-py2.py3-none-any.whl')
+    wheel = whl.Wheel(td)
+    self.assertEqual(wheel.extras(),
+        ['dev', 'lint', 'tests', 'validation', 'webframeworks-tests', 'yaml'])
+    self.assertEqual(set(wheel.dependencies()), set())
+    self.assertEqual(set(wheel.dependencies(extra='yaml')), set(['PyYAML']))
+
+  def test_whl_with_METADATA_file_depencendy_with_multiple_extras(self):
+    # Wheel with a dependency on a package with multiple extras specified.
+    td = TestData('google_cloud_spanner_1_9_0_whl/file/' +
+                  'google_cloud_spanner-1.9.0-py2.py3-none-any.whl')
+    wheel = whl.Wheel(td)
+    self.assertEqual(set(wheel.dependencies()),
+        set(['google-api-core', 'google-api-core[grpcgcp]',
+             'google-api-core[grpc]',
+             'grpc-google-iam-v1', 'google-cloud-core']))
+
+  def test_whl_with_METADATA_file_duplicate_extras(self):
+    # Wheel with the same extra mentioned several times in METADATA.
+    td = TestData('gunicorn_19_9_0_whl/file/' +
+                  'gunicorn-19.9.0-py2.py3-none-any.whl')
+    wheel = whl.Wheel(td)
+    # Ensure that extras() does not return any duplicates.
+    self.assertEqual(sorted(wheel.extras()),
+                     ['eventlet', 'gevent', 'gthread', 'tornado'])
+
+  def test_whl_with_METADATA_no_marker_in_dependencies(self):
+    # Wheel with simple dependencies without a marker.
+    td = TestData('pyOpenSSL_19_0_0_whl/file/' +
+                  'pyOpenSSL-19.0.0-py2.py3-none-any.whl')
+    wheel = whl.Wheel(td)
+    self.assertEqual(set(wheel.dependencies()), set(['cryptography','six']))
+
   @patch('platform.python_version', return_value='2.7.13')
   def test_mock_whl(self, *args):
     td = TestData('mock_whl/file/mock-2.0.0-py2.py3-none-any.whl')

python/requirements.txt

@@ -1,6 +1,7 @@
 pip==9.0.3
 setuptools==44.0.0
 wheel==0.30.0a0
+pkginfo==1.5.0.1
 
 # For tests
 mock==2.0.0