Skip to content

Commit

Permalink
added test cases and rez env flag
Browse files Browse the repository at this point in the history
Signed-off-by: Ben Andersen <ben@isohedron.com.au>
  • Loading branch information
isohedronpipeline committed Mar 7, 2024
1 parent 98bd61f commit a8e829f
Show file tree
Hide file tree
Showing 5 changed files with 74 additions and 17 deletions.
7 changes: 6 additions & 1 deletion src/rez/cli/env.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,10 @@ def setup_parser(parser, completions=False):
parser.add_argument(
"--no-pkg-cache", action="store_true",
help="Disable package caching")
parser.add_argument(
"--pkg-cache-sync", action="store_true",
help="Disable asynchronous package caching. "
"Process will block until packages are cached.")
parser.add_argument(
"--pre-command", type=str, help=SUPPRESS)
PKG_action = parser.add_argument(
Expand Down Expand Up @@ -212,7 +216,8 @@ def command(opts, parser, extra_arg_groups=None):
caching=(not opts.no_cache),
suppress_passive=opts.no_passive,
print_stats=opts.stats,
package_caching=(not opts.no_pkg_cache)
package_caching=(not opts.no_pkg_cache),
package_cache_async=(not opts.pkg_cache_sync),
)

success = (context.status == ResolverStatus.solved)
Expand Down
12 changes: 4 additions & 8 deletions src/rez/package_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -366,7 +366,7 @@ def remove_variant(self, variant):

return self.VARIANT_REMOVED

def add_variants(self, variants, _async=True):
def add_variants(self, variants, package_cache_async=True):
"""Update the package cache by adding some or all of the given variants.
This method is called when a context is created or sourced. Variants
Expand Down Expand Up @@ -460,18 +460,14 @@ def add_variants(self, variants, _async=True):
else:
out_target = devnull

func = subprocess.Popen

# use subprocess.call blocks where subprocess.Popen doesn't
if not _async:
func = subprocess.call

func(
process = subprocess.Popen(
args,
stdout=out_target,
stderr=out_target,
**kwargs
)
if not package_cache_async:
process.wait()

except Exception as e:
print_warning(
Expand Down
10 changes: 8 additions & 2 deletions src/rez/resolved_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ def __init__(self, package_requests, verbosity=0, timestamp=None,
package_filter=None, package_orderers=None, max_fails=-1,
add_implicit_packages=True, time_limit=-1, callback=None,
package_load_callback=None, buf=None, suppress_passive=False,
print_stats=False, package_caching=None):
print_stats=False, package_caching=None, package_cache_async=None):
"""Perform a package resolve, and store the result.
Args:
Expand Down Expand Up @@ -205,6 +205,8 @@ def __init__(self, package_requests, verbosity=0, timestamp=None,
package_caching (bool|None): If True, apply package caching settings
as per the config. If None, enable as determined by config
setting :data:`package_cache_during_build`.
package_cache_async (Optional[bool]): If True, cache packages asynchronously.
If None, use the config setting :data:`package_cache_async`
"""
self.load_path = None

Expand Down Expand Up @@ -249,6 +251,10 @@ def __init__(self, package_requests, verbosity=0, timestamp=None,

self.package_caching = package_caching

if package_cache_async is None:
package_cache_async = config.package_cache_async
self.package_cache_async = package_cache_async

# patch settings
self.default_patch_lock = PatchLock.no_lock
self.patch_locks = {}
Expand Down Expand Up @@ -1846,7 +1852,7 @@ def _update_package_cache(self):
if pkgcache:
pkgcache.add_variants(
self.resolved_packages,
config.package_cache_async
self.package_cache_async
)

@classmethod
Expand Down
7 changes: 4 additions & 3 deletions src/rez/rezconfig.py
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,8 @@
package_cache_during_build = False

# Enable package caching to run asynchronously during a resolve.
package_cache_async = True
# If this is false, a resolve will block until all packages are cached.
package_cache_async = True

# Allow caching of local packages. You would only want to set this True for
# testing purposes.
Expand Down Expand Up @@ -316,7 +317,7 @@
# This is useful as Platform.os might show different
# values depending on the availability of ``lsb-release`` on the system.
# The map supports regular expression, e.g. to keep versions.
#
#
# .. note::
# The following examples are not necessarily recommendations.
#
Expand Down Expand Up @@ -1122,7 +1123,7 @@

# Enables/disables colorization globally.
#
# .. warning::
# .. warning::
# Turned off for Windows currently as there seems to be a problem with the colorama module.
#
# May also set to the string ``force``, which will make rez output color styling
Expand Down
55 changes: 52 additions & 3 deletions src/rez/tests/test_package_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,6 @@ def test_caching_on_resolve(self):

# Creating the context will asynchronously add variants to the cache
# in a separate proc.
#
c = ResolvedContext([
"timestamped-1.2.0",
"pyfoo-3.1.0" # won't cache, see earlier test
Expand All @@ -144,14 +143,64 @@ def test_caching_on_resolve(self):

# Retry 50 times with 0.1 sec interval, 5 secs is more than enough for
# the very small variant to be copied to cache.
#
cached_root = None
resolve_not_always_cached = False
for _ in range(50):
time.sleep(0.1)
cached_root = pkgcache.get_cached_root(variant)
if cached_root:
break

resolve_not_always_cached = True
time.sleep(0.1)

self.assertNotEqual(cached_root, None)

# Test that the package is not immediately cached, since it is asynchronous
# WARNING: This is dangerous since it does open the test to a race condition and
# will fail if the cache happens faster than the resolve.
self.assertNotEqual(resolve_not_always_cached, False)

expected_payload_file = os.path.join(cached_root, "stuff.txt")
self.assertTrue(os.path.exists(expected_payload_file))

# check that refs to root point to cache location in rex code
for ref in ("resolve.timestamped.root", "'{resolve.timestamped.root}'"):
proc = c.execute_rex_code(
code="info(%s)" % ref,
stdout=subprocess.PIPE,
universal_newlines=True
)

out, _ = proc.communicate()
root = out.strip()

self.assertEqual(
root, cached_root,
"Reference %r should resolve to %s, but resolves to %s"
% (ref, cached_root, root)
)

@install_dependent()
def test_caching_on_resolve_synchronous(self):
"""Test that cache is updated as expected on
resolved env using syncrhonous package caching."""
pkgcache = self._pkgcache()

with restore_os_environ():
# set config settings into env so rez-pkg-cache proc sees them
os.environ.update(self.get_settings_env())

# Creating the context will synchronously add variants to the cache
c = ResolvedContext([
"timestamped-1.2.0",
"pyfoo-3.1.0",
],
package_cache_async=False,
)

variant = c.get_resolved_package("timestamped")
# The first time we try to access it will be cached, because the cache is blocking
cached_root = pkgcache.get_cached_root(variant)
self.assertNotEqual(cached_root, None)

expected_payload_file = os.path.join(cached_root, "stuff.txt")
Expand Down

0 comments on commit a8e829f

Please sign in to comment.