Add support for pushing to http locations.

pyproject.toml

@@ -204,6 +204,7 @@ testpaths = ["tests"]
 
 [tool.mypy]
 allow_redefinition = true
+# check_untyped_defs = true
 
 # https://github.com/libgit2/pygit2/issues/709
 [[tool.mypy.overrides]]

src/pyorderly/outpack/location_driver.py

@@ -1,5 +1,6 @@
 from abc import abstractmethod
 from contextlib import AbstractContextManager
+from pathlib import Path
 from typing import Dict, List
 
 from pyorderly.outpack.metadata import MetadataCore, PacketFile, PacketLocation
@@ -23,3 +24,15 @@ def metadata(self, packet_ids: List[str]) -> Dict[str, str]: ...
     def fetch_file(
         self, packet: MetadataCore, file: PacketFile, dest: str
     ) -> None: ...
+
+    @abstractmethod
+    def list_unknown_packets(self, ids: List[str]) -> List[str]: ...
+
+    @abstractmethod
+    def list_unknown_files(self, hashes: List[str]) -> List[str]: ...
+
+    @abstractmethod
+    def push_file(self, src: Path, hash: str): ...
+
+    @abstractmethod
+    def push_metadata(self, src: Path, hash: str): ...

src/pyorderly/outpack/location_http.py

@@ -1,4 +1,5 @@
 import shutil
+from pathlib import Path
 from typing import Dict, List
 from urllib.parse import urljoin
 
@@ -78,3 +79,34 @@ def fetch_file(self, packet: MetadataCore, file: PacketFile, dest: str):
         response = self._client.get(f"file/{file.hash}", stream=True)
         with open(dest, "wb") as f:
             shutil.copyfileobj(response.raw, f)
+
+    @override
+    def list_unknown_packets(self, ids: List[str]) -> List[str]:
+        response = self._client.post(
+            "/packets/missing",
+            json={
+                "ids": ids,
+                "unpacked": True,
+            },
+        ).json()
+        return response["data"]
+
+    @override
+    def list_unknown_files(self, hashes: List[str]) -> List[str]:
+        response = self._client.post(
+            "/files/missing",
+            json={
+                "hashes": hashes,
+            },
+        ).json()
+        return response["data"]
+
+    @override
+    def push_file(self, src: Path, hash: str):
+        with open(src, "rb") as f:
+            self._client.post(f"file/{hash}", stream=True, data=f)
+
+    @override
+    def push_metadata(self, src: Path, hash: str):
+        with open(src, "rb") as f:
+            self._client.post(f"file/{hash}", stream=True, data=f)

src/pyorderly/outpack/location_path.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+from pathlib import Path
 from typing import Dict, List
 
 from typing_extensions import override
@@ -54,3 +55,19 @@ def fetch_file(self, _packet: MetadataCore, file: PacketFile, dest: str):
                 msg = f"Hash '{file.hash}' not found at location"
                 raise Exception(msg)
         shutil.copyfile(path, dest)
+
+    @override
+    def list_unknown_packets(self, ids: List[str]) -> List[str]:
+        raise NotImplementedError()
+
+    @override
+    def list_unknown_files(self, hashes: List[str]) -> List[str]:
+        raise NotImplementedError()
+
+    @override
+    def push_file(self, src: Path, hash: str):
+        raise NotImplementedError()
+
+    @override
+    def push_metadata(self, src: Path, hash: str):
+        raise NotImplementedError()

src/pyorderly/outpack/location_push.py

@@ -0,0 +1,64 @@
+from dataclasses import dataclass
+from typing import List, Union
+
+from pyorderly.outpack.location import _location_driver, location_resolve_valid
+from pyorderly.outpack.location_driver import LocationDriver
+from pyorderly.outpack.location_pull import _find_all_dependencies
+from pyorderly.outpack.root import OutpackRoot, find_file_by_hash, root_open
+from pyorderly.outpack.static import LOCATION_LOCAL
+from pyorderly.outpack.util import as_list
+
+
+@dataclass
+class LocationPushPlan:
+    packets: List[str]
+    files: List[str]
+
+
+def outpack_location_push(
+    ids: Union[str, List[str]],
+    location: str,
+    *,
+    root: Union[str, OutpackRoot, None] = None,
+    locate: bool = True,
+):
+    root = root_open(root, locate=locate)
+    (location_name,) = location_resolve_valid(
+        [location],
+        root,
+        include_local=False,
+        include_orphan=False,
+        allow_no_locations=False,
+    )
+
+    with _location_driver(location_name, root) as driver:
+        plan = location_build_push_plan(driver, as_list(ids), root)
+        for h in plan.files:
+            if root.files is not None:
+                path = root.files.filename(h)
+            else:
+                path = find_file_by_hash(root, h)
+                if path is None:
+                    msg = "Did not find suitable file, can't push this packet"
+                    raise Exception(msg)
+            driver.push_file(path, h)
+
+        packets = root.index.location(LOCATION_LOCAL)
+        for id in plan.packets:
+            path = root.path / ".outpack" / "metadata" / id
+            driver.push_metadata(path, packets[id].hash)
+
+
+def location_build_push_plan(
+    driver: LocationDriver, packet_ids: List[str], root: OutpackRoot
+) -> LocationPushPlan:
+    metadata = root.index.all_metadata()
+    all_packets = _find_all_dependencies(packet_ids, metadata)
+    missing_packets = driver.list_unknown_packets(all_packets)
+
+    all_files = list(
+        {f.hash for id in missing_packets for f in metadata[id].files}
+    )
+    missing_files = driver.list_unknown_files(all_files)
+
+    return LocationPushPlan(packets=missing_packets, files=missing_files)

src/pyorderly/outpack/location_ssh.py

@@ -1,7 +1,7 @@
 import base64
 import errno
 from contextlib import ExitStack
-from pathlib import PurePosixPath
+from pathlib import Path, PurePosixPath
 from typing import Dict, List
 from urllib.parse import urlsplit
 
@@ -139,6 +139,22 @@ def fetch_file(self, packet: MetadataCore, file: PacketFile, dest: str):
             else:
                 raise
 
+    @override
+    def list_unknown_packets(self, ids: List[str]) -> List[str]:
+        raise NotImplementedError()
+
+    @override
+    def list_unknown_files(self, hashes: List[str]) -> List[str]:
+        raise NotImplementedError()
+
+    @override
+    def push_file(self, src: Path, hash: str):
+        raise NotImplementedError()
+
+    @override
+    def push_metadata(self, src: Path, hash: str):
+        raise NotImplementedError()
+
     def _file_path(self, packet: MetadataCore, file: PacketFile):
         if self.config.core.use_file_store:
             dat = hash_parse(file.hash)

src/pyorderly/outpack/metadata.py

@@ -11,7 +11,7 @@
 @dataclass
 class PacketFile(DataClassJsonMixin):
     path: str
-    size: float
+    size: int
     hash: str
 
     @staticmethod

src/pyorderly/outpack/util.py

@@ -5,7 +5,7 @@
 from contextlib import contextmanager
 from itertools import filterfalse, tee
 from pathlib import Path, PurePath
-from typing import Dict, List, Optional, Union, overload
+from typing import Dict, List, Optional, TypeVar, Union
 
 
 def find_file_descend(filename, path):
@@ -60,12 +60,15 @@ def transient_working_directory(path):
             os.chdir(origin)
 
 
-def assert_file_exists(path, *, workdir=None, name="File"):
+def assert_file_exists(
+    path: Union[str, List[str]], *, workdir=None, name="File"
+):
     with transient_working_directory(workdir):
         if isinstance(path, list):
             missing = [str(p) for p in path if not os.path.exists(p)]
         else:
             missing = [] if os.path.exists(path) else [path]
+
     if len(missing):
         missing_str = ", ".join(missing)
         msg = f"{name} does not exist: {missing_str}"
@@ -192,19 +195,10 @@ def openable_temporary_file(*, mode: str = "w+b", dir: Optional[str] = None):
             pass
 
 
-@overload
-def as_posix_path(paths: str) -> str: ...
-
-
-@overload
-def as_posix_path(paths: List[str]) -> List[str]: ...
-
-
-@overload
-def as_posix_path(paths: Dict[str, str]) -> Dict[str, str]: ...
+Paths = TypeVar("Paths", str, List[str], Dict[str, str])
 
 
-def as_posix_path(paths):
+def as_posix_path(paths: Paths) -> Paths:
     """
     Convert a native path into a posix path.
 
@@ -217,3 +211,13 @@ def as_posix_path(paths):
         return [as_posix_path(v) for v in paths]
     else:
         return PurePath(paths).as_posix()
+
+
+T = TypeVar("T")
+
+
+def as_list(x: Union[T, List[T]]) -> List[T]:
+    if isinstance(x, list):
+        return x
+    else:
+        return [x]

tests/helpers/__init__.py

@@ -14,7 +14,7 @@
 from pyorderly.outpack.location import outpack_location_add_path
 from pyorderly.outpack.metadata import MetadataCore, PacketDepends
 from pyorderly.outpack.packet import Packet, insert_packet
-from pyorderly.outpack.root import root_open
+from pyorderly.outpack.root import OutpackRoot, root_open
 from pyorderly.outpack.schema import outpack_schema_version
 from pyorderly.outpack.util import openable_temporary_file
 from pyorderly.run import orderly_run
@@ -44,7 +44,9 @@ def create_packet(root, name, *, packet_id=None, parameters=None):
             insert_packet(root, Path(src), metadata)
 
 
-def create_random_packet(root, name="data", *, parameters=None, packet_id=None):
+def create_random_packet(
+    root, name="data", *, parameters=None, packet_id=None
+) -> str:
     d = [f"{random.random()}\n" for _ in range(10)]
 
     with create_packet(
@@ -76,7 +78,7 @@ def create_random_packet_chain(root, length, base=None):
     return ids
 
 
-def create_temporary_root(path, **kwargs):
+def create_temporary_root(path, **kwargs) -> OutpackRoot:
     outpack_init(path, **kwargs)
     return root_open(path, locate=False)
 

tests/helpers/outpack_server.py

@@ -8,6 +8,7 @@
 import pytest
 import requests
 
+from pyorderly.outpack.init import outpack_init
 from pyorderly.outpack.root import OutpackRoot
 
 
@@ -70,6 +71,13 @@ def start_outpack_server(root: Union[Path, OutpackRoot], port: int = 8080):
     else:
         root_path = str(root)
 
+    outpack_init(
+        root_path,
+        require_complete_tree=True,
+        use_file_store=True,
+        path_archive=None,
+    )
+
     args = [
         binary,
         "start-server",

tests/outpack/test_location_http.py

@@ -15,6 +15,7 @@
     outpack_location_pull_metadata,
     outpack_location_pull_packet,
 )
+from pyorderly.outpack.location_push import outpack_location_push
 from pyorderly.outpack.metadata import PacketFile, PacketLocation
 from pyorderly.outpack.static import LOCATION_LOCAL
 from pyorderly.outpack.util import read_string
@@ -195,3 +196,24 @@ def test_http_client_errors():
         client.get("/packit-error")
     with pytest.raises(HTTPError, match="400 Error: Custom error message"):
         client.get("/outpack-error")
+
+
+@pytest.mark.parametrize("use_file_store", [True, False])
+def test_can_push_packet(tmp_path, use_file_store) -> None:
+    root = create_temporary_root(
+        tmp_path / "root",
+        use_file_store=use_file_store,
+    )
+
+    ids = [create_random_packet(root) for _ in range(3)]
+
+    with start_outpack_server(tmp_path / "server") as url:
+        outpack_location_add(
+            "upstream",
+            "http",
+            {"url": url},
+            root=root,
+        )
+
+        outpack_location_push(ids[0], "upstream", root=root)
+        outpack_location_push(ids[1], "upstream", root=root)

tests/outpack/test_location_packit.py

@@ -1,10 +1,10 @@
-import pytest
 import re
+
+import pytest
 import responses
 from responses import matchers
 from responses.registries import OrderedRegistry
 
-from pyorderly.outpack.location_pull import outpack_location_pull_metadata
 from pyorderly.outpack.location import outpack_location_add
 from pyorderly.outpack.location_packit import (
     GITHUB_ACCESS_TOKEN_URL,
@@ -14,6 +14,7 @@
     outpack_location_packit,
     packit_authorisation,
 )
+from pyorderly.outpack.location_pull import outpack_location_pull_metadata
 
 from ..helpers import create_temporary_root