Merge pull request #555 from brichet/fix/repodata

Fixes issues and tests on indexes

quetz/main.py

@@ -1090,6 +1090,7 @@ def delete_package_version(
     filename: str,
     channel_name: str,
     package_name: str,
+    background_tasks: BackgroundTasks,
     dao: Dao = Depends(get_dao),
     db=Depends(get_db),
     auth: authorization.Rules = Depends(get_rules),
@@ -1115,6 +1116,10 @@ def delete_package_version(
 
     dao.update_channel_size(channel_name)
 
+    wrapped_bg_task = background_task_wrapper(indexing.update_indexes, logger)
+    # Background task to update indexes
+    background_tasks.add_task(wrapped_bg_task, dao, pkgstore, channel_name, [platform])
+
 
 @api_router.get(
     "/packages/search/", response_model=List[rest_models.PackageSearch], tags=["search"]
@@ -1300,6 +1305,10 @@ def post_file_to_package(
     handle_package_files(package.channel, files, dao, auth, force, package=package)
     dao.update_channel_size(package.channel_name)
 
+    wrapped_bg_task = background_task_wrapper(indexing.update_indexes, logger)
+    # Background task to update indexes
+    background_tasks.add_task(wrapped_bg_task, dao, pkgstore, package.channel_name)
+
 
 @api_router.post(
     "/channels/{channel_name}/upload/{filename}", status_code=201, tags=["upload"]

quetz/tests/api/conftest.py

@@ -3,22 +3,10 @@
 
 import pytest
 
-from quetz.config import Config
-from quetz.dao import Dao
-from quetz.db_models import Identity, PackageVersion, Profile, User
+from quetz.db_models import Identity, Profile, User
 from quetz.rest_models import Channel, Package
 
 
-@pytest.fixture
-def package_name():
-    return "my-package"
-
-
-@pytest.fixture
-def channel_name():
-    return "my-channel"
-
-
 @pytest.fixture
 def private_channel(dao, other_user):
 
@@ -43,7 +31,9 @@ def private_package(dao, other_user, private_channel):
 
 
 @pytest.fixture
-def private_package_version(dao, private_channel, private_package, other_user, config):
+def private_package_version(
+    dao, private_channel, private_package, other_user, config, package_name
+):
     package_format = "tarbz2"
     package_info = "{}"
     channel_name = private_channel.name
@@ -77,68 +67,6 @@ def private_package_version(dao, private_channel, private_package, other_user, c
     return version
 
 
-@pytest.fixture
-def make_package_version(
-    db,
-    user,
-    public_channel,
-    channel_name,
-    package_name,
-    public_package,
-    dao: Dao,
-    config: Config,
-):
-
-    pkgstore = config.get_package_store()
-
-    versions = []
-
-    def _make_package_version(filename, version_number, platform="linux-64"):
-        filename = Path(filename)
-        with open(filename, "rb") as fid:
-            pkgstore.add_file(fid.read(), channel_name, platform / filename)
-        package_format = "tarbz2"
-        package_info = "{}"
-        version = dao.create_version(
-            channel_name,
-            package_name,
-            package_format,
-            platform,
-            version_number,
-            0,
-            "",
-            str(filename),
-            package_info,
-            user.id,
-            size=11,
-        )
-
-        dao.update_package_channeldata(
-            channel_name,
-            package_name,
-            {'name': package_name, 'subdirs': [platform]},
-        )
-
-        dao.update_channel_size(channel_name)
-
-        versions.append(version)
-
-        return version
-
-    yield _make_package_version
-
-    for version in versions:
-        db.query(PackageVersion).filter(PackageVersion.id == version.id).delete()
-    db.commit()
-
-
-@pytest.fixture
-def package_version(db, make_package_version):
-    version = make_package_version("test-package-0.1-0.tar.bz2", "0.1")
-
-    return version
-
-
 @pytest.fixture()
 def other_user_without_profile(db):
     user = User(id=uuid.uuid4().bytes, username="other")
@@ -163,37 +91,6 @@ def other_user(other_user_without_profile, db):
     yield other_user_without_profile
 
 
-@pytest.fixture
-def channel_role():
-    return "owner"
-
-
-@pytest.fixture
-def package_role():
-    return "owner"
-
-
-@pytest.fixture
-def public_channel(dao: Dao, user, channel_role, channel_name):
-
-    channel_data = Channel(name=channel_name, private=False)
-    channel = dao.create_channel(channel_data, user.id, channel_role)
-
-    return channel
-
-
-@pytest.fixture
-def public_package(db, user, public_channel, dao, package_role, package_name):
-
-    package_data = Package(name=package_name)
-
-    package = dao.create_package(
-        public_channel.name, package_data, user.id, package_role
-    )
-
-    return package
-
-
 @pytest.fixture
 def pkgstore(config):
     return config.get_package_store()

quetz/tests/api/test_main_packages.py

@@ -1,4 +1,6 @@
+import json
 import os
+import time
 from pathlib import Path
 from typing import BinaryIO
 
@@ -10,7 +12,6 @@
 from quetz.config import Config
 from quetz.db_models import ChannelMember, Package, PackageMember, PackageVersion
 from quetz.errors import ValidationError
-from quetz.pkgstores import PackageStore
 from quetz.tasks.indexing import update_indexes
 
 
@@ -72,10 +73,22 @@ def test_delete_package_versions_with_package(
 
     update_indexes(dao, pkgstore, public_channel.name)
 
+    # Get package files
     package_filenames = [
         os.path.join(version.platform, version.filename)
         for version in public_package.package_versions  # type: ignore
     ]
+
+    # Get repodata content
+    package_dir = Path(pkgstore.channels_dir) / public_channel.name / 'linux-64'
+    with open(package_dir / 'repodata.json', 'r') as fd:
+        repodata = json.load(fd)
+
+    # Check that all packages are initially in repodata
+    for filename in package_filenames:
+        assert os.path.basename(filename) in repodata["packages"].keys()
+
+    # Get channel files
     init_files = sorted(pkgstore.list_files(public_channel.name))
 
     response = auth_client.delete(
@@ -95,9 +108,19 @@ def test_delete_package_versions_with_package(
 
     assert len(versions) == 0
 
+    # Check that repodata content has been updated
+    with open(package_dir / 'repodata.json', 'r') as fd:
+        repodata = json.load(fd)
+
+    assert repodata["info"] == repodata["info"]
+
+    # Remove package files from files list
+    # Check that packages have been removed from repodata
     for filename in package_filenames:
         init_files.remove(filename)
+        assert os.path.basename(filename) not in repodata["packages"]
 
+    # Check that the package tree files is the same except for package files
     files = sorted(pkgstore.list_files(public_channel.name))
 
     assert files == init_files
@@ -312,10 +335,6 @@ def test_upload_package_version_wrong_filename(
             files=files,
         )
 
-    with open(package_filename, "rb") as fid:
-        condainfo = CondaInfo(fid, package_filename)
-        condainfo._parse_conda()
-
     package_dir = Path(pkgstore.channels_dir) / public_channel.name / 'linux-64'
 
     assert response.status_code == 400
@@ -326,6 +345,78 @@ def test_upload_package_version_wrong_filename(
     assert not os.path.exists(package_dir)
 
 
+@pytest.mark.parametrize("package_name", ["test-package"])
+def test_upload_duplicate_package_version(
+    auth_client,
+    public_channel,
+    public_package,
+    package_name,
+    db,
+    config,
+    remove_package_versions,
+):
+    pkgstore = config.get_package_store()
+
+    package_filename = "test-package-0.1-0.tar.bz2"
+    package_filename_copy = "test-package-0.1-0_copy.tar.bz2"
+
+    with open(package_filename, "rb") as fid:
+        files = {"files": (package_filename, fid)}
+        response = auth_client.post(
+            f"/api/channels/{public_channel.name}/packages/"
+            f"{public_package.name}/files/",
+            files=files,
+        )
+
+    # Get repodata content
+    package_dir = Path(pkgstore.channels_dir) / public_channel.name / 'linux-64'
+    with open(package_dir / 'repodata.json', 'r') as fd:
+        repodata_init = json.load(fd)
+
+    # Try submitting the same package without 'force' flag
+    with open(package_filename, "rb") as fid:
+        files = {"files": (package_filename, fid)}
+        response = auth_client.post(
+            f"/api/channels/{public_channel.name}/packages/"
+            f"{public_package.name}/files/",
+            files=files,
+        )
+    assert response.status_code == 409
+    detail = response.json()['detail']
+    assert "Duplicate" in detail
+
+    # Change the archive to test force update
+    os.remove(package_filename)
+    os.rename(package_filename_copy, package_filename)
+
+    # Ensure the 'time_modified' value change in repodata.json
+    time.sleep(1)
+
+    # Submit the same package with 'force' flag
+    with open(package_filename, "rb") as fid:
+        files = {"files": (package_filename, fid)}
+        response = auth_client.post(
+            f"/api/channels/{public_channel.name}/packages/"
+            f"{public_package.name}/files/",
+            files=files,
+            data={"force": True},
+        )
+
+    assert response.status_code == 201
+
+    # Check that repodata content has been updated
+    with open(package_dir / 'repodata.json', 'r') as fd:
+        repodata = json.load(fd)
+
+    assert repodata_init["info"] == repodata["info"]
+    assert repodata_init["packages"].keys() == repodata["packages"].keys()
+    repodata_init_pkg = repodata_init["packages"][package_filename]
+    repodata_pkg = repodata["packages"][package_filename]
+    assert repodata_init_pkg["time_modified"] != repodata_pkg["time_modified"]
+    assert repodata_init_pkg["md5"] != repodata_pkg["md5"]
+    assert repodata_init_pkg["sha256"] != repodata_pkg["sha256"]
+
+
 @pytest.mark.parametrize("package_name", ["test-package"])
 def test_check_channel_size_limits(
     auth_client, public_channel, public_package, db, config
@@ -353,13 +444,22 @@ def test_check_channel_size_limits(
 
 
 def test_delete_package_version(
-    auth_client, public_channel, package_version, dao, pkgstore: PackageStore, db
+    auth_client, public_channel, package_version, dao, pkgstore, db
 ):
     assert public_channel.size > 0
     assert public_channel.size == package_version.size
 
     filename = "test-package-0.1-0.tar.bz2"
     platform = "linux-64"
+
+    update_indexes(dao, pkgstore, public_channel.name)
+
+    # Get repodata content and check that package is inside
+    package_dir = Path(pkgstore.channels_dir) / public_channel.name / 'linux-64'
+    with open(package_dir / 'repodata.json', 'r') as fd:
+        repodata = json.load(fd)
+    assert filename in repodata["packages"].keys()
+
     response = auth_client.delete(
         f"/api/channels/{public_channel.name}/"
         f"packages/{package_version.package_name}/versions/{platform}/{filename}"
@@ -381,6 +481,11 @@ def test_delete_package_version(
     db.refresh(public_channel)
     assert public_channel.size == 0
 
+    # Check that repodata content has been updated
+    with open(package_dir / 'repodata.json', 'r') as fd:
+        repodata = json.load(fd)
+    assert filename not in repodata["packages"].keys()
+
 
 def test_package_name_length_limit(auth_client, public_channel, db):