Skip to content

Commit

Permalink
Keep nightly 20231010 for ExecuTorch alpha 0.1 for now (#4900)
Browse files Browse the repository at this point in the history
pytorch/executorch#1663. The indexing script
was migrated recently from builder to test-infra in
#4879. But there was a mistake
in which my [recent
change](pytorch/builder@6f3cb2b)
to keep nightly 20231010 for ExecuTorch wasn't copied over.

So, this PR copies the missing logic
pytorch/builder#1642. It also fixes some linter
issues flagged by test-infra lintrunner.
  • Loading branch information
huydhn authored Jan 22, 2024
1 parent 9178476 commit 6b27ab6
Showing 1 changed file with 14 additions and 7 deletions.
21 changes: 14 additions & 7 deletions s3_management/manage.py
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import functools
import time

from contextlib import suppress
from os import path, makedirs
from datetime import datetime
from collections import defaultdict
Expand Down Expand Up @@ -112,6 +113,12 @@
# How many packages should we keep of a specific package?
KEEP_THRESHOLD = 60

# TODO (huydhn): Clean this up once ExecuTorch has a new stable release that
# match PyTorch stable release cadence. This nightly version is currently
# referred to publicly in ExecuTorch alpha 0.1 release. So we want to keep
# nightly binaries around for now
KEEP_NIGHTLY_PACKAGES_FOR_EXECUTORCH = {datetime(2023, 10, 10, 0, 0)}

S3IndexType = TypeVar('S3IndexType', bound='S3Index')


Expand Down Expand Up @@ -139,11 +146,9 @@ def __lt__(self, other):
def extract_package_build_time(full_package_name: str) -> datetime:
result = search(PACKAGE_DATE_REGEX, full_package_name)
if result is not None:
try:
return datetime.strptime(result.group(2), "%Y%m%d")
except ValueError:
with suppress(ValueError):
# Ignore any value errors since they probably shouldn't be hidden anyways
pass
return datetime.strptime(result.group(2), "%Y%m%d")
return datetime.now()


Expand All @@ -160,7 +165,6 @@ def safe_parse_version(ver_str: str) -> Version:
return Version("0.0.0")



class S3Index:
def __init__(self: S3IndexType, objects: List[S3Object], prefix: str) -> None:
self.objects = objects
Expand Down Expand Up @@ -201,7 +205,10 @@ def nightly_packages_to_show(self: S3IndexType) -> List[S3Object]:
if package_name not in PACKAGE_ALLOW_LIST:
to_hide.add(obj)
continue
if packages[package_name] >= KEEP_THRESHOLD or between_bad_dates(package_build_time):
if package_build_time not in KEEP_NIGHTLY_PACKAGES_FOR_EXECUTORCH and (
packages[package_name] >= KEEP_THRESHOLD
or between_bad_dates(package_build_time)
):
to_hide.add(obj)
else:
packages[package_name] += 1
Expand Down Expand Up @@ -447,7 +454,7 @@ def sanitize_key(key: str) -> str:
checksum=None,
size=None) for key in obj_names], prefix)
if prefix == "whl/nightly":
rc.objects = rc.nightly_packages_to_show()
rc.objects = rc.nightly_packages_to_show()
if with_metadata:
rc.fetch_metadata()
return rc
Expand Down

0 comments on commit 6b27ab6

Please sign in to comment.