Add package_adder argument to assemble() #3034

    * Add test for add_to_package

Signed-off-by: Jono Yang <jyang@nexb.com>

src/packagedcode/about.py

@@ -107,7 +107,7 @@ def parse(cls, location):
         )
 
     @classmethod
-    def assemble(cls, package_data, resource, codebase):
+    def assemble(cls, package_data, resource, codebase, package_adder):
         """
         Yield a Package. Note that ABOUT files do not carry dependencies.
         """
@@ -122,8 +122,7 @@ def assemble(cls, package_data, resource, codebase):
 
             # NOTE: we do not attach files to the Package level. Instead we
             # update `for_package` in the file
-            resource.for_packages.append(package_uid)
-            resource.save(codebase)
+            package_adder(package_uid, resource, codebase)
 
             if not package.license_expression:
                 package.license_expression = cls.compute_normalized_license(package)
@@ -151,8 +150,7 @@ def assemble(cls, package_data, resource, codebase):
                         # path is found and processed: remove it, so we can
                         # check if we found all of them
                         del file_references_by_path[res.path]
-                        res.for_packages.append(package_uid)
-                        res.save(codebase)
+                        package_adder(package_uid, res, codebase)
 
                         yield res
 

src/packagedcode/alpine.py

@@ -64,7 +64,7 @@ def compute_normalized_license(cls, package):
         return detected
 
     @classmethod
-    def assemble(cls, package_data, resource, codebase):
+    def assemble(cls, package_data, resource, codebase, package_adder):
         # get the root resource of the rootfs
         levels_up = len('lib/apk/db/installed'.split('/'))
         root_resource = get_ancestor(
@@ -107,8 +107,7 @@ def assemble(cls, package_data, resource, codebase):
             # path is found and processed: remove it, so we can check if we
             # found all of them
             del file_references_by_path[res.path]
-            res.for_packages.append(package_uid)
-            res.save(codebase)
+            package_adder(package_uid, res, codebase)
             resources.append(res)
 
         # if we have left over file references, add these to extra data
@@ -139,11 +138,12 @@ def compute_normalized_license(cls, package):
         return detected
 
     @classmethod
-    def assign_package_to_resources(cls, package, resource, codebase):
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder):
         models.DatafileHandler.assign_package_to_parent_tree(
             package=package,
             resource=resource,
             codebase=codebase,
+            package_adder=package_adder,
         )
 
 

src/packagedcode/build.py

@@ -50,11 +50,12 @@ def parse(cls, location):
         )
 
     @classmethod
-    def assign_package_to_resources(cls, package, resource, codebase):
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder):
         models.DatafileHandler.assign_package_to_parent_tree(
             package=package,
             resource=resource,
             codebase=codebase,
+            package_adder=package_adder,
         )
 
 
@@ -75,7 +76,7 @@ class BaseStarlarkManifestHandler(models.DatafileHandler):
     """
 
     @classmethod
-    def assemble(cls, package_data, resource, codebase):
+    def assemble(cls, package_data, resource, codebase, package_adder):
         """
         Given a ``package_data`` PackageData found in the ``resource`` datafile
         of the ``codebase``, assemble package their files and dependencies
@@ -100,6 +101,7 @@ def assemble(cls, package_data, resource, codebase):
                 package=package,
                 resource=resource,
                 codebase=codebase,
+                package_adder=package_adder
             )
 
             yield package
@@ -175,14 +177,13 @@ def parse(cls, location):
             )
 
     @classmethod
-    def assign_package_to_resources(cls, package, resource, codebase, skip_name=None):
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder, skip_name=None):
         package_uid = package.package_uid
         if not package_uid:
             return
         parent = resource.parent(codebase)
         for res in walk_build(resource=parent, codebase=codebase, skip_name=skip_name):
-            res.for_packages.append(package_uid)
-            res.save(codebase)
+            package_adder(package_uid, res, codebase)
 
 
 def walk_build(resource, codebase, skip_name):
@@ -235,11 +236,12 @@ class BazelBuildHandler(BaseStarlarkManifestHandler):
     documentation_url = 'https://bazel.build/'
 
     @classmethod
-    def assign_package_to_resources(cls, package, resource, codebase, skip_name='BUILD'):
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder, skip_name='BUILD'):
         return super().assign_package_to_resources(
             package=package,
             resource=resource,
             codebase=codebase,
+            package_adder=package_adder,
             skip_name=skip_name,
         )
 
@@ -252,11 +254,12 @@ class BuckPackageHandler(BaseStarlarkManifestHandler):
     documentation_url = 'https://buck.build/'
 
     @classmethod
-    def assign_package_to_resources(cls, package, resource, codebase, skip_name='BUCK'):
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder, skip_name='BUCK'):
         return super().assign_package_to_resources(
             package=package,
             resource=resource,
             codebase=codebase,
+            package_adder=package_adder,
             skip_name=skip_name,
         )
 
@@ -377,9 +380,10 @@ def compute_normalized_license(cls, package):
             return models.compute_normalized_license(declared_license)
 
     @classmethod
-    def assign_package_to_resources(cls, package, resource, codebase):
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder):
         models.DatafileHandler.assign_package_to_parent_tree(
             package_=package,
             resource=resource,
             codebase=codebase,
+            package_adder=package_adder,
         )

src/packagedcode/build_gradle.py

@@ -65,11 +65,12 @@ def parse(cls, location):
 
     # TODO: handle complex cases of nested builds with many packages
     @classmethod
-    def assign_package_to_resources(cls, package, resource, codebase):
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder):
         models.DatafileHandler.assign_package_to_parent_tree(
             package=package,
             resource=resource,
             codebase=codebase,
+            package_adder=package_adder,
         )
 
 

src/packagedcode/cargo.py

@@ -85,14 +85,15 @@ def parse(cls, location):
         )
 
     @classmethod
-    def assemble(cls, package_data, resource, codebase):
+    def assemble(cls, package_data, resource, codebase, package_adder):
         """
         Assemble Cargo.toml and possible Cargo.lock datafiles
         """
         yield from cls.assemble_from_many_datafiles(
             datafile_name_patterns=('Cargo.toml', 'cargo.toml', 'Cargo.lock', 'cargo.lock'),
             directory=resource.parent(codebase),
             codebase=codebase,
+            package_adder=package_adder,
         )
 
 
@@ -144,14 +145,15 @@ def parse(cls, location):
         )
 
     @classmethod
-    def assemble(cls, package_data, resource, codebase):
+    def assemble(cls, package_data, resource, codebase, package_adder):
         """
         Assemble Cargo.toml and possible Cargo.lock datafiles
         """
         yield from cls.assemble_from_many_datafiles(
             datafile_name_patterns=('Cargo.toml', 'Cargo.lock',),
             directory=resource.parent(codebase),
             codebase=codebase,
+            package_adder=package_adder,
         )
 
 

src/packagedcode/chef.py

@@ -142,14 +142,15 @@ def format(self, tokens, outfile):
 class BaseChefMetadataHandler(models.DatafileHandler):
 
     @classmethod
-    def assemble(cls, package_data, resource, codebase):
+    def assemble(cls, package_data, resource, codebase, package_adder):
         """
         Assemble Package from Chef metadata.rb, then from metadata.json files.
         """
         yield from cls.assemble_from_many_datafiles(
             datafile_name_patterns=('metadata.rb', 'metadata.json',),
             directory=resource.parent(codebase),
             codebase=codebase,
+            package_adder=package_adder,
         )
 
 

src/packagedcode/cocoapods.py

@@ -114,7 +114,7 @@ def get_first_three_md5_hash_characters(podname):
 class BasePodHandler(models.DatafileHandler):
 
     @classmethod
-    def assemble(cls, package_data, resource, codebase):
+    def assemble(cls, package_data, resource, codebase, package_adder):
         """
         Assemble pod packages and dependencies and handle the specific cases where
         there are more than one podspec in the same directory.
@@ -147,6 +147,7 @@ def assemble(cls, package_data, resource, codebase):
                     datafile_name_patterns=datafile_name_patterns,
                     directory=parent,
                     codebase=codebase,
+                    package_adder=package_adder,
                 )
 
             elif has_multiple_podspec:
@@ -160,6 +161,7 @@ def assemble(cls, package_data, resource, codebase):
                     datafile_name_patterns=datafile_name_patterns,
                     directory=parent,
                     codebase=codebase,
+                    package_adder=package_adder,
                 )
 
                 for resource in sibling_podspecs:
@@ -170,7 +172,12 @@ def assemble(cls, package_data, resource, codebase):
                             package_data=package_data,
                             datafile_path=datafile_path,
                         )
-                        cls.assign_package_to_resources(package, resource, codebase)
+                        cls.assign_package_to_resources(
+                            package=package,
+                            resource=resource,
+                            codebase=codebase,
+                            package_adder=package_adder,
+                        )
                         yield package
                     yield resource
 
@@ -180,6 +187,7 @@ def assemble(cls, package_data, resource, codebase):
                     datafile_name_patterns=datafile_name_patterns,
                     directory=parent,
                     codebase=codebase,
+                    package_adder=package_adder,
                 )
 
 

src/packagedcode/conda.py

@@ -70,11 +70,12 @@ def get_conda_root(cls, resource, codebase):
         return resource.parent(codebase)
 
     @classmethod
-    def assign_package_to_resources(cls, package, resource, codebase):
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder):
         return models.DatafileHandler.assign_package_to_resources(
             package=package,
             resource=cls.get_conda_root(resource, codebase),
             codebase=codebase,
+            package_adder=package_adder,
         )
 
     @classmethod