diff --git a/src/rez/cli/_entry_points.py b/src/rez/cli/_entry_points.py index 98d484970..06faff743 100644 --- a/src/rez/cli/_entry_points.py +++ b/src/rez/cli/_entry_points.py @@ -297,3 +297,10 @@ def run_rez_pkg_ignore(): check_production_install() from rez.cli._main import run return run("pkg-ignore") + + +@scriptname("rez-mv") +def run_rez_pkg_mv(): + check_production_install() + from rez.cli._main import run + return run("mv") diff --git a/src/rez/cli/_util.py b/src/rez/cli/_util.py index 58c99b9a9..a3d253a59 100644 --- a/src/rez/cli/_util.py +++ b/src/rez/cli/_util.py @@ -58,7 +58,8 @@ "yaml2py": {}, "bundle": {}, "benchmark": {}, - "pkg-ignore": {} + "pkg-ignore": {}, + "mv": {} } diff --git a/src/rez/cli/env.py b/src/rez/cli/env.py index 51fa8b0f9..0606cb9a8 100644 --- a/src/rez/cli/env.py +++ b/src/rez/cli/env.py @@ -9,6 +9,7 @@ def setup_parser(parser, completions=False): from rez.config import config from rez.system import system from rez.shells import get_shell_types + import os shells = get_shell_types() @@ -42,7 +43,7 @@ def setup_parser(parser, completions=False): help="create a build environment") parser.add_argument( "--paths", type=str, default=None, - help="set package search path") + help="set package search path (use %r separator)" % os.pathsep) parser.add_argument( "-t", "--time", type=str, help="ignore packages released after the given time. Supported formats " diff --git a/src/rez/cli/mv.py b/src/rez/cli/mv.py new file mode 100644 index 000000000..b4f3e7e44 --- /dev/null +++ b/src/rez/cli/mv.py @@ -0,0 +1,77 @@ +''' +Move a package from one repository to another. +''' +from __future__ import print_function + + +def setup_parser(parser, completions=False): + parser.add_argument( + "--dest-path", metavar="PATH", required=True, + help="package repository to move PKG to.") + parser.add_argument( + "-k", "--keep-timestamp", action="store_true", + help="keep timestamp of source package.") + parser.add_argument( + "-f", "--force", action="store_true", + help="move package even if it isn't relocatable (use at your own risk)") + pkg_action = parser.add_argument( + "PKG", + help="package to move") + parser.add_argument( + "PATH", nargs='?', + help="The repository containing the package. If not specified, this " + "command will present you with a list to select from.") + + if completions: + from rez.cli._complete_util import PackageCompleter + pkg_action.completer = PackageCompleter + + +def list_repos_containing_pkg(pkg_name, pkg_version): + from rez.config import config + from rez.package_repository import package_repository_manager + import sys + + # search for package in each searchpath + matching_repos = [] + + for path in config.packages_path: + repo = package_repository_manager.get_repository(path) + if repo.get_package(pkg_name, pkg_version): + matching_repos.append(repo) + + if matching_repos: + print("No action taken. Run again, and set PATH to one of:") + for repo in matching_repos: + print(str(repo)) + else: + print("Package not found.", file=sys.stderr) + sys.exit(1) + + +def command(opts, parser, extra_arg_groups=None): + from rez.vendor.version.requirement import VersionedObject + from rez.packages import get_package_from_repository + from rez.package_move import move_package + import sys + + obj = VersionedObject(opts.PKG) + + if opts.PATH is None: + list_repos_containing_pkg(obj.name, obj.version) + sys.exit(0) + + # find src pkg + src_pkg = get_package_from_repository(obj.name, obj.version, opts.PATH) + + if src_pkg is None: + print("Package not found.", file=sys.stderr) + sys.exit(1) + + move_package( + package=src_pkg, + dest_repository=opts.dest_path, + keep_timestamp=opts.keep_timestamp, + force=opts.force, + verbose=opts.verbose + ) diff --git a/src/rez/cli/pkg-ignore.py b/src/rez/cli/pkg-ignore.py index cdc84b5db..4a2c2b524 100644 --- a/src/rez/cli/pkg-ignore.py +++ b/src/rez/cli/pkg-ignore.py @@ -4,12 +4,13 @@ from __future__ import print_function -# TEMP COMMENT PLS REMOVE - def setup_parser(parser, completions=False): parser.add_argument( "-u", "--unignore", action="store_true", help="Unignore a package.") + parser.add_argument( + "-a", "--allow-missing", action="store_true", + help="Allow ignoring of packages that don't exist.") PKG_action = parser.add_argument( "PKG", type=str, help="The exact package to (un)ignore (eg 'foo-1.2.3').") @@ -23,6 +24,17 @@ def setup_parser(parser, completions=False): PKG_action.completer = PackageCompleter +def list_repos(): + from rez.config import config + from rez.package_repository import package_repository_manager + + print("No action taken. Run again, and set PATH to one of:") + + for path in config.packages_path: + repo = package_repository_manager.get_repository(path) + print(str(repo)) + + def list_repos_containing_pkg(pkg_name, pkg_version): from rez.config import config from rez.package_repository import package_repository_manager @@ -33,15 +45,8 @@ def list_repos_containing_pkg(pkg_name, pkg_version): for path in config.packages_path: repo = package_repository_manager.get_repository(path) - - fam = repo.get_package_family(pkg_name) - if fam is None: - continue - - for pkg in fam.iter_packages(): - if pkg.version == pkg_version: - matching_repos.append(repo) - break + if repo.get_package(pkg_name, pkg_version): + matching_repos.append(repo) if matching_repos: print("No action taken. Run again, and set PATH to one of:") @@ -60,7 +65,10 @@ def command(opts, parser, extra_arg_groups=None): obj = VersionedObject(opts.PKG) if opts.PATH is None: - list_repos_containing_pkg(obj.name, obj.version) + if opts.allow_missing: + list_repos() + else: + list_repos_containing_pkg(obj.name, obj.version) sys.exit(0) repo = package_repository_manager.get_repository(opts.PATH) @@ -68,7 +76,11 @@ def command(opts, parser, extra_arg_groups=None): if opts.unignore: i = repo.unignore_package(obj.name, obj.version) else: - i = repo.ignore_package(obj.name, obj.version) + i = repo.ignore_package( + obj.name, + obj.version, + allow_missing=opts.allow_missing + ) if i == 1: if opts.unignore: diff --git a/src/rez/exceptions.py b/src/rez/exceptions.py index 1b9b9f5b9..05734a4da 100644 --- a/src/rez/exceptions.py +++ b/src/rez/exceptions.py @@ -93,6 +93,11 @@ class PackageCopyError(RezError): pass +class PackageMoveError(RezError): + """There was a problem moving a package.""" + pass + + class ContextBundleError(RezError): """There was a problem bundling a context.""" pass diff --git a/src/rez/package_copy.py b/src/rez/package_copy.py index c0dd1a5b5..368b0026f 100644 --- a/src/rez/package_copy.py +++ b/src/rez/package_copy.py @@ -6,6 +6,7 @@ from rez.config import config from rez.exceptions import PackageCopyError from rez.package_repository import package_repository_manager +from rez.packages import Variant from rez.serialise import FileFormat from rez.utils import with_noop from rez.utils.base26 import create_unique_base26_symlink @@ -137,13 +138,15 @@ def finalize(): new_src_variants = [] for src_variant in src_variants: - existing_variant = dest_pkg_repo.install_variant( + existing_variant_resource = dest_pkg_repo.install_variant( src_variant.resource, overrides=overrides, dry_run=True ) - if existing_variant: + if existing_variant_resource: + existing_variant = Variant(existing_variant_resource) + if overwrite: if verbose: print_info("Source variant %s will overwrite %s", @@ -206,11 +209,13 @@ def finalize(): overrides_["timestamp"] = int(time.time()) # install the variant into the package definition - dest_variant = dest_pkg_repo.install_variant( + dest_variant_resource = dest_pkg_repo.install_variant( variant_resource=src_variant.resource, overrides=overrides_ ) + dest_variant = Variant(dest_variant_resource) + if verbose: print_info("Copied source variant %s to target variant %s", src_variant, dest_variant) diff --git a/src/rez/package_move.py b/src/rez/package_move.py new file mode 100644 index 000000000..20d336023 --- /dev/null +++ b/src/rez/package_move.py @@ -0,0 +1,90 @@ +from rez.exceptions import PackageMoveError +from rez.package_copy import copy_package +from rez.package_repository import package_repository_manager +from rez.utils.logging_ import print_info +from rez.vendor.six import six + + +basestring = six.string_types[0] + + +def move_package(package, dest_repository, keep_timestamp=False, force=False, + verbose=False): + """Move a package. + + Moving a package means copying the package to a destination repo, and + ignoring (ie hiding - not removing) the source package. The package must + not already exist in the destination repo. + + Args: + package (`Package`): Package to move. + dest_repository (`PackageRepository` or str): The package repository, or + a package repository path, to move the package into. + keep_timestamp (bool): By default, a newly copied package will get a + new timestamp (because that's when it was added to the target repo). + By setting this option to True, the original package's timestamp + is kept intact. + force (bool): Move the package regardless of its relocatable attribute. + Use at your own risk (there is no guarantee the resulting package + will be functional). + verbose (bool): Verbose mode. + + Returns: + `Package`: The newly created package in the destination repo. + """ + def _info(msg, *nargs): + if verbose: + print_info(msg, *nargs) + + # get dest repo + if isinstance(dest_repository, basestring): + repo_path = dest_repository + dest_pkg_repo = package_repository_manager.get_repository(repo_path) + else: + dest_pkg_repo = dest_repository + + # check that the package doesn't already exist in the dest repo + pkg = dest_pkg_repo.get_package(package.name, package.version) + if pkg: + raise PackageMoveError( + "Package already exists at destination: %s" + % pkg.uri + ) + + # move the pkg as atomically as possible: + # + # 1. Hide the dest package (even tho it doesn't exist yet) + # 2. Copy the package + # 3. Unhide the dest package + # 4. Hide the src package + # + + # 1. + dest_pkg_repo.ignore_package( + package.name, package.version, allow_missing=True) + _info("Ignored %s in %s ahead of time", package.qualified_name, dest_pkg_repo) + + try: + # 2. + result = copy_package( + package=package, + dest_repository=dest_pkg_repo, + force=force, + keep_timestamp=keep_timestamp, + verbose=verbose + ) + finally: + # 3. + dest_pkg_repo.unignore_package(package.name, package.version) + _info("Unignored %s in %s", package.qualified_name, dest_pkg_repo) + + # 4. + package.repository.ignore_package(package.name, package.version) + _info("Ignored %s", package.uri) + + # finish up + a_dest_variant = result["copied"][0][1] + dest_pkg = a_dest_variant.parent + + _info("Package %s moved to %s", package.uri, dest_pkg.uri) + return dest_pkg diff --git a/src/rez/package_repository.py b/src/rez/package_repository.py index 09da8c776..2c20a2e4f 100644 --- a/src/rez/package_repository.py +++ b/src/rez/package_repository.py @@ -170,6 +170,26 @@ def iter_variants(self, package_resource): """ raise NotImplementedError + def get_package(self, name, version): + """Get a package. + + Args: + name (str): Package name. + version (`Version`): Package version. + + Returns: + `PackageResource` or None: Matching package, or None if not found. + """ + fam = self.get_package_family(name) + if fam is None: + return None + + for pkg in fam.iter_packages(): + if pkg.version == version: + return pkg + + return None + def get_package_from_uri(self, uri): """Get a package given its URI. @@ -194,7 +214,7 @@ def get_variant_from_uri(self, uri): """ return None - def ignore_package(self, pkg_name, pkg_version): + def ignore_package(self, pkg_name, pkg_version, allow_missing=False): """Ignore the given package. Ignoring a package makes it invisible to further resolves. @@ -202,6 +222,10 @@ def ignore_package(self, pkg_name, pkg_version): Args: pkg_name (str): Package name pkg_version(`Version`): Package version + allow_missing (bool): if True, allow for ignoring a package that + does not exist. This is useful when you want to copy a package + to a repo and you don't want it visible until the copy is + completed. Returns: int: diff --git a/src/rez/packages.py b/src/rez/packages.py index e869f62b5..056e654d1 100644 --- a/src/rez/packages.py +++ b/src/rez/packages.py @@ -568,7 +568,7 @@ def iter_packages(name, range_=None, paths=None): def get_package(name, version, paths=None): - """Get an exact version of a package. + """Get a package by searching a list of repositories. Args: name (str): Name of the package, eg 'maya'. @@ -591,6 +591,25 @@ def get_package(name, version, paths=None): return None +def get_package_from_repository(name, version, path): + """Get a package from a repository. + + Args: + name (str): Name of the package, eg 'maya'. + version (Version or str): Version of the package, eg '1.0.0' + + Returns: + `Package` object, or None if the package was not found. + """ + repo = package_repository_manager.get_repository(path) + + package_resource = repo.get_package(name, version) + if package_resource is None: + return None + + return Package(package_resource) + + def get_package_from_handle(package_handle): """Create a package given its handle (or serialized dict equivalent) diff --git a/src/rez/tests/test_packages.py b/src/rez/tests/test_packages.py index 7fae16ef5..ff966e153 100644 --- a/src/rez/tests/test_packages.py +++ b/src/rez/tests/test_packages.py @@ -3,10 +3,10 @@ """ from rez.packages import iter_package_families, iter_packages, get_package, \ create_package, get_developer_package, get_variant_from_uri, \ - get_package_from_uri + get_package_from_uri, get_package_from_repository from rez.package_py_utils import expand_requirement from rez.package_resources import package_release_keys -from rez.package_repository import package_repository_manager +from rez.package_move import move_package from rez.tests.util import TestBase, TempdirMixin from rez.utils.formatting import PackageRequest from rez.utils.sourcecode import SourceCode @@ -443,16 +443,20 @@ def test_package_ignore(self): pkg_version = Version("2") # copy packages to a temp repo - repo_path = os.path.join(self.root, "tmp_packages") + repo_path = os.path.join(self.root, "tmp1_packages") shutil.copytree(self.solver_packages_path, repo_path) # check that a known package exists - pkg = get_package(pkg_name, pkg_version, paths=[repo_path]) + pkg = get_package_from_repository(pkg_name, pkg_version, repo_path) self.assertNotEqual(pkg, None) repo = pkg.repository - # ignore the package + # ignore a pkg that doesn't exist, but allow it + i = repo.ignore_package("i_dont_exist", Version("1"), allow_missing=True) + self.assertEqual(i, 1) + + # ignore an existing package i = repo.ignore_package(pkg_name, pkg_version) self.assertEqual(i, 1) @@ -461,8 +465,7 @@ def test_package_ignore(self): self.assertEqual(i, 0) # verify that we cannot see it - package_repository_manager.clear_caches() - pkg = get_package(pkg_name, pkg_version, paths=[repo_path]) + pkg = get_package_from_repository(pkg_name, pkg_version, repo_path) self.assertEqual(pkg, None) # unignore it @@ -474,10 +477,37 @@ def test_package_ignore(self): self.assertEqual(i, 0) # verify that we can see it again - package_repository_manager.clear_caches() - pkg = get_package(pkg_name, pkg_version, paths=[repo_path]) + pkg = get_package_from_repository(pkg_name, pkg_version, repo_path) self.assertNotEqual(pkg, None) + def test_package_move(self): + """Test package move.""" + pkg_name = "pydad" + pkg_version = Version("2") + + # copy packages to a temp repo + repo_path = os.path.join(self.root, "tmp2_packages") + shutil.copytree(self.solver_packages_path, repo_path) + + # create an empty temp repo + dest_repo_path = os.path.join(self.root, "tmp3_packages") + os.mkdir(dest_repo_path) + + # verify that source pkg exists + src_pkg = get_package_from_repository(pkg_name, pkg_version, repo_path) + self.assertNotEqual(src_pkg, None) + + # move it to dest repo + move_package(src_pkg, dest_repo_path) + + # verify it exists in dest repo + dest_pkg = get_package_from_repository(pkg_name, pkg_version, dest_repo_path) + self.assertNotEqual(dest_pkg, None) + + # verify it is not visible in source repo + src_pkg = get_package_from_repository(pkg_name, pkg_version, repo_path) + self.assertEqual(src_pkg, None) + class TestMemoryPackages(TestBase): def test_1_memory_variant_parent(self): diff --git a/src/rezplugins/package_repository/filesystem.py b/src/rezplugins/package_repository/filesystem.py index 34fc73d7b..47344baf7 100644 --- a/src/rezplugins/package_repository/filesystem.py +++ b/src/rezplugins/package_repository/filesystem.py @@ -536,6 +536,8 @@ def __init__(self, location, resource_pool): ) self._get_version_dirs = decorator2(self._get_version_dirs) + self._disable_pkg_ignore = False + def _uid(self): t = ["filesystem", self.location] if os.path.exists(self.location): @@ -604,17 +606,8 @@ def get_package_from_uri(self, uri): return None # find package - fam = self.get_package_family(pkg_name) - if fam is None: - return None - - ver = Version(pkg_ver_str) - - for package in fam.iter_packages(): - if package.version == ver: - return package - - return None + pkg_ver = Version(pkg_ver_str) + return self.get_package(pkg_name, pkg_ver) def get_variant_from_uri(self, uri): """ @@ -652,32 +645,39 @@ def get_variant_from_uri(self, uri): return None - def ignore_package(self, pkg_name, pkg_version): + def ignore_package(self, pkg_name, pkg_version, allow_missing=False): + fam_path = os.path.join(self.location, pkg_name) + # find family fam = self.get_package_family(pkg_name) if not fam: - return -1 + if allow_missing: + # we have to create the fam dir in order to create .ignore file + os.makedirs(fam_path) + else: + return -1 filename = self.ignore_prefix + str(pkg_version) - filepath = os.path.join(self.location, pkg_name, filename) + filepath = os.path.join(fam_path, filename) if os.path.exists(filepath): return 0 # find package - found = False - for pkg in fam.iter_packages(): - if pkg.version == pkg_version: - found = True - break + if not allow_missing: + found = False + for pkg in fam.iter_packages(): + if pkg.version == pkg_version: + found = True + break - if not found: - return -1 + if not found: + return -1 # create .ignore file with open(filepath, 'w'): pass - self._notify_changed_family(pkg_name) + self._on_changed(pkg_name) return 1 def unignore_package(self, pkg_name, pkg_version): @@ -692,7 +692,8 @@ def unignore_package(self, pkg_name, pkg_version): return 0 os.remove(filepath) - self._notify_changed_family(pkg_name) + + self._on_changed(pkg_name) return 1 def get_resource_from_handle(self, resource_handle, verify_repo=True): @@ -943,8 +944,11 @@ def _get_version_dirs__key(self, root): def _get_version_dirs(self, root): # Ignore a version if there is a .ignore file next to it def ignore_dir(name): - path = os.path.join(root, self.ignore_prefix + name) - return os.path.isfile(path) + if self._disable_pkg_ignore: + return False + else: + path = os.path.join(root, self.ignore_prefix + name) + return os.path.isfile(path) # simpler case if this test is on # @@ -1072,7 +1076,8 @@ def _create_family(self, name): path = os.path.join(self.location, name) if not os.path.exists(path): os.makedirs(path) - self.clear_caches() + + self._on_changed(name) return self.get_package_family(name) def _create_variant(self, variant, dry_run=False, overrides=None): @@ -1331,37 +1336,49 @@ def _remove_build_keys(obj): except: pass - self._notify_changed_family(variant_name) + self._on_changed(variant_name) - # load new variant + # load new variant. Note that we load it from a copy of this repo, with + # package ignore disabled. We do this so it's possible to install + # variants into a hidden (ignored) package. This is used by `move_package` + # in order to make the moved package visible only after all its variants + # have been copied over. + # new_variant = None - self.clear_caches() - family = self.get_package_family(variant_name) - if family: - for package in self.iter_packages(family): - if package.version == variant_version: - for variant_ in self.iter_variants(package): - if variant_.index == installed_variant_index: - new_variant = variant_ - break - elif new_variant: + repo_copy = self.__class__(self.location, self.pool) + repo_copy._disable_pkg_ignore = True + pkg = repo_copy.get_package(variant_name, variant_version) + + if pkg is not None: + for variant_ in self.iter_variants(pkg): + if variant_.index == installed_variant_index: + new_variant = variant_ break if not new_variant: raise RezSystemError("Internal failure - expected installed variant") + + # a bit hacky but it works. We need the variant to belong to the actual + # repo, not the temp copy we retrieved it from + # + new_variant._repository = self + return new_variant - def _notify_changed_family(self, pkg_name): - """ - This step is important. Whenever a package within a family is - changed/removed/added, we update the access time of the parent family - dir. We can then do far less filesystem stats to determine if a resolve - cache is stale. + def _on_changed(self, pkg_name): + """Called when a package is added/removed/changed. """ + + # update access time of family dir. This is done so that very few file + # stats are required to determine if a resolve cache entry is stale. + # family_path = os.path.join(self.location, pkg_name) os.utime(family_path, None) + # clear internal caches, otherwise change may not be visible + self.clear_caches() + def _delete_stale_build_tagfiles(self, family_path): now = time.time() diff --git a/wiki/pages/Managing-Packages.md b/wiki/pages/Managing-Packages.md index 71e3c3887..7f69c4276 100644 --- a/wiki/pages/Managing-Packages.md +++ b/wiki/pages/Managing-Packages.md @@ -85,8 +85,8 @@ Copying packages is enabled by default, however you're also able to specify whic packages are and are not _relocatable_, for much the same reasons as given [here](Managing-Packages#enabling-package-caching). -You can mark a package as non-relocatable by setting `relocatable = False` in its -package definition file. There are also config settings that affect relocatability +You can mark a package as non-relocatable by setting [relocatable](Package-Definition-Guide#relocatable) +to False in its package definition file. There are also config settings that affect relocatability in the event that relocatable is not defined in a package's definition. For example, see [default_relocatable](Configuring-Rez#default_relocatable), [default_relocatable_per_package](Configuring-Rez#default_relocatable_per_package) @@ -97,6 +97,44 @@ However, note that there is a `force` option that will override this - use at your own risk. +## Moving Packages + +Packages can be moved from one [package repository](Basic-Concepts#package-repositories) +to another. Be aware that moving a package does not actually delete the source +package however. Instead, the source package is hidden (ignored) - it is up to +you to delete it at some later date. + +You move a package like so: + +``` +>>> from rez.package_move import move_package +>>> from rez.packages import get_package_from_repository +>>> +>>> p = get_package_from_repository("python", "3.7.4", "/packages") +>>> p +Package(FileSystemPackageResource({'location': '/packages', 'name': 'python', 'repository_type': 'filesystem', 'version': '3.7.4'})) +>>> +>>> new_p = move_package(p, "/packages2") +>>> new_p +Package(FileSystemPackageResource({'location': '/packages2', 'name': 'python', 'repository_type': 'filesystem', 'version': '3.7.4'})) +>>> +>>> p = get_package_from_repository("python", "3.7.4", "/packages") +>>> p +None +``` + +Be aware that a non-relocatable package is also not movable (see +[here](Package-Definition-Guide#relocatable) for more details). Like package +copying, there is a `force` option to move it regardless. + +A typical reason you might want to move a package is to archive packages that are +no longer in use. In this scenario, you would move the package to some archival +package repository. In case an old runtime needs to be resurrected, you would add +this archival repository to the packages path before performing the resolve. Note +that you will probably want to use the `--keep-timestamp` option when doing this, +otherwise rez will think the package did not exist prior to its archival date. + + ## Package Caching Package caching is a feature that copies package payloads onto local disk in @@ -115,9 +153,9 @@ Package caching is not enabled by default. To enable it, you need to configure store the cache in. You also have granular control over whether an individual package will or will -not be cached. To make a package cachable, you can set `cachable = True` in its -package definition file. Reasons you may _not_ want to do this include packages -that are large, or that aren't relocatable because other compiled packages are +not be cached. To make a package cachable, you can set [cachable](Package-Definition-Guide#cachable) + to False in its package definition file. Reasons you may _not_ want to do this include +packages that are large, or that aren't relocatable because other compiled packages are linked to them in a way that doesn't support library relocation. There are also config settings that affect cachability in the event that `cachable`