Review changes

sno/fast_import.py

@@ -227,7 +227,7 @@ def fast_import_tables(
         cmd.append("--quiet")
 
     orig_commit = repo.head_commit
-    import_branches = []
+    import_refs = []
 
     if verbosity >= 1:
         click.echo("Starting git-fast-import...")
@@ -246,14 +246,12 @@ def fast_import_tables(
         for i in range(num_processes):
             if header is None:
                 # import onto a temp branch. then reset the head branch afterwards.
-                import_branch = f"refs/heads/{uuid.uuid4()}"
-                import_branches.append(import_branch)
+                import_ref = f"refs/sno-import/{uuid.uuid4()}"
+                import_refs.append(import_ref)
 
                 # may be None, if head is detached
                 orig_branch = repo.head_branch
-                generated_header = generate_header(
-                    repo, sources, message, import_branch
-                )
+                generated_header = generate_header(repo, sources, message, import_ref)
             else:
                 generated_header = header
                 # this won't work if num_processes > 1 because we'll try and write to
@@ -403,34 +401,40 @@ def _ids():
     if verbosity >= 1:
         click.echo(f"Closed in {(t3-t2):.0f}s")
 
-    if import_branches:
+    if import_refs:
         # we created temp branches for the import above.
         # each of the branches has _part_ of the import.
         # we have to merge the trees together to get a sensible commit.
         try:
-            trees = [repo.revparse_single(b).peel(pygit2.Tree) for b in import_branches]
-            builder = RichTreeBuilder(repo, trees[0])
-            for t in trees[1:]:
-                datasets = Datasets(t, SUPPORTED_DATASET_CLASS)
-                for ds in datasets:
-                    try:
-                        feature_tree = ds.feature_tree
-                    except KeyError:
-                        pass
-                    else:
-                        for subtree in feature_tree:
-                            builder.insert(
-                                f"{ds.path}/{ds.FEATURE_PATH}{subtree.name}", subtree
-                            )
-            new_tree = builder.flush()
+            trees = [repo.revparse_single(b).peel(pygit2.Tree) for b in import_refs]
+            if len(import_refs) > 1:
+                click.echo(f"Joining {len(import_refs)} parallel-imported trees...")
+                builder = RichTreeBuilder(repo, trees[0])
+                for t in trees[1:]:
+                    datasets = Datasets(t, SUPPORTED_DATASET_CLASS)
+                    for ds in datasets:
+                        try:
+                            feature_tree = ds.feature_tree
+                        except KeyError:
+                            pass
+                        else:
+                            for subtree in feature_tree:
+                                builder.insert(
+                                    f"{ds.path}/{ds.FEATURE_PATH}{subtree.name}",
+                                    subtree,
+                                )
+                new_tree = builder.flush()
+                t4 = time.monotonic()
+                click.echo(f"Joined trees in {(t4-t3):.0f}s")
+            else:
+                new_tree = trees[0]
+                t4 = time.monotonic()
             if not allow_empty:
                 if new_tree == orig_tree:
                     raise NotFound("No changes to commit", exit_code=NO_CHANGES)
 
             # use the existing commit details we already imported, but use the new tree
-            existing_commit = repo.revparse_single(import_branches[0]).peel(
-                pygit2.Commit
-            )
+            existing_commit = repo.revparse_single(import_refs[0]).peel(pygit2.Commit)
             repo.create_commit(
                 orig_branch or "HEAD",
                 existing_commit.author,
@@ -441,7 +445,7 @@ def _ids():
             )
         finally:
             # remove the import branches
-            for b in import_branches:
+            for b in import_refs:
                 repo.references.delete(b)
 
 

sno/utils.py

@@ -49,10 +49,9 @@ def get_num_available_cores():
     The result is a float which may or may not be a round number, and may be less than 1.
     """
     if platform.system() == "Linux":
-        quota_f = Path("/sys/fs/cgroup/cpu/cpu.cfs_quota_us")
         try:
-            quota = quota_f.read_text()
-            period = Path("/sys/fs/cgroup/cpu/cpu.cfs_period_us").read_text()
+            quota = float(Path("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").read_text())
+            period = float(Path("/sys/fs/cgroup/cpu/cpu.cfs_period_us").read_text())
         except FileNotFoundError:
             pass
         else:
@@ -62,7 +61,7 @@ def get_num_available_cores():
             else:
                 # note: this is a float, and may not be a round number
                 # (it's possible to allocate half-cores)
-                return float(quota / period)
+                return quota / period
     try:
         return float(len(os.sched_getaffinity(0)))
     except AttributeError: