sagemathgh-37762: dist.yml: Download optional/experimental tarballs…

… for GitHub Release assets

    
<!-- ^ Please provide a concise and informative title. -->
<!-- ^ Don't put issue numbers in the title, do this in the PR
description below. -->
<!-- ^ For example, instead of "Fixes sagemath#12345" use "Introduce new method
to calculate 1 + 2". -->
<!-- v Describe your changes below in detail. -->
<!-- v Why is this change required? What problem does it solve? -->
<!-- v If this PR resolves an open issue, please link to it here. For
example, "Fixes sagemath#12345". -->

- Fixes sagemath#37752, at least for the non-"huge" tarballs (we skip the "huge"
tarballs here).
- More options for `sage -package download` (a subset of what `sage
-package list` supports)
- Also remove some zombie package files

Test run: https://github.com/mkoeppe/sage/actions/workflows/dist.yml

### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. -->

- [x] The title is concise and informative.
- [ ] The description explains in detail what this PR is about.
- [x] I have linked a relevant issue or discussion.
- [ ] I have created tests covering the changes.
- [ ] I have updated the documentation accordingly.

### ⌛ Dependencies

<!-- List all open PRs that this PR logically depends on. For example,
-->
<!-- - sagemath#12345: short description why this is a dependency -->
<!-- - sagemath#34567: ... -->

- Depends on sagemath#37099 (merged to resolve merge conflict)
    
URL: sagemath#37762
Reported by: Matthias Köppe
Reviewer(s): Kwankyu Lee, Matthias Köppe

.github/workflows/dist.yml

@@ -37,16 +37,37 @@ jobs:
           sudo DEBIAN_FRONTEND=noninteractive apt-get update
           sudo DEBIAN_FRONTEND=noninteractive apt-get install $(build/bin/sage-get-system-packages debian _bootstrap)
       - name: make dist (--disable-download-from-upstream-url)
+        id: make_dist
         run: |
           ./bootstrap -D && ./configure --disable-download-from-upstream-url && make dist
         env:
           MAKE: make -j8
+      - name: make download (--disable-download-from-upstream-url)
+        id: make_download
+        run: |
+          make -k download DOWNLOAD_PACKAGES=":all: --no-file huge"
+        env:
+          MAKE: make -j8
+      - name: Reconfigure with --enable-download-from-upstream-url
+        if: (success() || failure()) && (steps.make_dist.outcome != 'success' || steps.make_download.outcome != 'success')
+        run: |
+          ./configure
       - name: make dist (--enable-download-from-upstream-url)
-        if: failure()
+        if: (success() || failure()) && steps.make_dist.outcome != 'success'
         run: |
-          ./configure && make dist
+          make dist
         env:
           MAKE: make -j8
+      - name: make download (--enable-download-from-upstream-url)
+        if: (success() || failure()) && steps.make_download.outcome != 'success'
+        run: |
+          make -k download DOWNLOAD_PACKAGES=":all: --no-file huge --allow-upstream"
+        env:
+          MAKE: make -j8
+      - name: Remove what cannot be distributed
+        if: success() || failure()
+        run: |
+          rm -f upstream/*do-not-distribute*
       - uses: actions/upload-artifact@v4
         if: success() || failure()
         with:

Makefile

@@ -79,10 +79,11 @@ reconfigure:
 	fi
 
 # Preemptively download all source tarballs of normal packages.
+DOWNLOAD_PACKAGES=:all:
 download:
 	export SAGE_ROOT=$$(pwd) && \
 	export PATH=$$SAGE_ROOT/build/bin:$$PATH && \
-	sage-package download :all:
+	sage-package download $(DOWNLOAD_PACKAGES)
 
 dist: build/make/Makefile
 	./sage --sdist

build/sage_bootstrap/app.py

@@ -304,11 +304,14 @@ def download(self, package_name, allow_upstream=False):
         package.tarball.download(allow_upstream=allow_upstream)
         print(package.tarball.upstream_fqn)
 
-    def download_cls(self, package_name_or_class, allow_upstream=False, on_error='stop'):
+    def download_cls(self, *package_classes, **kwds):
         """
         Download a package or a class of packages
         """
-        pc = PackageClass(package_name_or_class, has_files=['checksums.ini'])
+        allow_upstream = kwds.pop('allow_upstream', False)
+        on_error = kwds.pop('on_error', 'stop')
+        has_files = list(kwds.pop('has_files', []))
+        pc = PackageClass(*package_classes, has_files=has_files + ['checksums.ini'], **kwds)
 
         def download_with_args(package):
             try:

build/sage_bootstrap/cmdline.py

@@ -270,18 +270,18 @@ def make_parser():
     parser_config = subparsers.add_parser(
         'config', epilog=epilog_config,
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        help='Print the configuration')
+        help='print the configuration')
 
     parser_list = subparsers.add_parser(
         'list', epilog=epilog_list,
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        help='Print a list of packages known to Sage')
+        help='print a list of packages known to Sage')
     parser_list.add_argument(
         'package_class', metavar='[package_name|:package_type:]',
         type=str, default=[':all-or-nothing:'], nargs='*',
         help=('package name or designator for all packages of a given type '
               '(one of :all:, :standard:, :optional:, and :experimental:); '
-              'default: :all: (or nothing when --include-dependencies or --exclude-dependencies is given'))
+              'default: :all: (or nothing when --include-dependencies or --exclude-dependencies is given)'))
     parser_list.add_argument(
         '--has-file', action='append', default=[], metavar='FILENAME', dest='has_files',
         help=('only include packages that have this file in their metadata directory '
@@ -303,7 +303,7 @@ def make_parser():
     parser_properties = subparsers.add_parser(
         'properties', epilog=epilog_properties,
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        help='Print properties of given packages')
+        help='print properties of given packages')
     parser_properties.add_argument(
         'package_class', metavar='[package_name|:package_type:]',
         type=str, nargs='+',
@@ -316,7 +316,7 @@ def make_parser():
     parser_dependencies = subparsers.add_parser(
         'dependencies', epilog=epilog_dependencies,
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        help='Print the list of packages that are dependencies of given packages')
+        help='print the list of packages that are dependencies of given packages')
     parser_dependencies.add_argument(
         'package_class', metavar='[package_name|:package_type:]',
         type=str, nargs='+',
@@ -341,74 +341,88 @@ def make_parser():
     parser_name = subparsers.add_parser(
         'name', epilog=epilog_name,
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        help='Find the package name given a tarball filename')
-    parser_name.add_argument('tarball_filename', type=str, help='Tarball filename')
+        help='find the package name given a tarball filename')
+    parser_name.add_argument('tarball_filename', type=str, help='tarball filename')
 
     parser_tarball = subparsers.add_parser(
         'tarball', epilog=epilog_tarball,
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        help='Find the tarball filename given a package name')
-    parser_tarball.add_argument('package_name', type=str, help='Package name')
+        help='find the tarball filename given a package name')
+    parser_tarball.add_argument('package_name', type=str, help='package name')
 
     parser_apropos = subparsers.add_parser(
         'apropos', epilog=epilog_apropos,
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        help='Find up to 5 package names that are close to the given name')
+        help='find up to 5 package names that are close to the given name')
     parser_apropos.add_argument(
         'incorrect_name', type=str,
-        help='Fuzzy name to search for')
+        help='fuzzy name to search for')
 
     parser_update = subparsers.add_parser(
         'update', epilog=epilog_update,
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        help='Update a package. This modifies the Sage sources.')
+        help='update a package, modifying the Sage sources')
     parser_update.add_argument(
-        'package_name', type=str, help='Package name')
+        'package_name', type=str, help='package name')
     parser_update.add_argument(
-        'new_version', type=str, help='New version')
+        'new_version', type=str, help='new version')
     parser_update.add_argument(
-        '--url', type=str, default=None, help='Download URL')
+        '--url', type=str, default=None, help='download URL')
     parser_update.add_argument(
         '--commit', action="store_true",
-        help='Whether to run "git commit"')
+        help='whether to run "git commit"')
 
     parser_update_latest = subparsers.add_parser(
         'update-latest', epilog=epilog_update_latest,
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        help='Update a package to the latest version. This modifies the Sage sources.')
+        help='update a package to the latest version, modifying the Sage sources')
     parser_update_latest.add_argument(
-        'package_name', type=str, help='Package name (:all: for all packages)')
+        'package_name', type=str, help='package name (:all: for all packages)')
     parser_update_latest.add_argument(
         '--commit', action="store_true",
-        help='Whether to run "git commit"')
+        help='whether to run "git commit"')
 
     parser_download = subparsers.add_parser(
         'download', epilog=epilog_download,
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        help='Download tarball')
+        help='download tarball')
     parser_download.add_argument(
-        'package_name', type=str, help='Package name or :type:')
+        'package_class', metavar='[package_name|:package_type:]',
+        type=str, nargs='+',
+        help=('package name or designator for all packages of a given type '
+              '(one of :all:, :standard:, :optional:, and :experimental:)'))
+    parser_download.add_argument(
+        '--has-file', action='append', default=[], metavar='FILENAME', dest='has_files',
+        help=('only include packages that have this file in their metadata directory '
+              '(examples: SPKG.rst, spkg-configure.m4, distros/debian.txt, spkg-install|spkg-install.in)'))
+    parser_download.add_argument(
+        '--no-file', action='append', default=[], metavar='FILENAME', dest='no_files',
+        help=('only include packages that do not have this file in their metadata directory '
+              '(examples: huge, patches, huge|has_nonfree_dependencies)'))
+    parser_download.add_argument(
+        '--exclude', nargs='*', action='append', default=[], metavar='PACKAGE_NAME',
+        help='exclude package from list')
     parser_download.add_argument(
         '--allow-upstream', action="store_true",
-        help='Whether to fall back to downloading from the upstream URL')
+        help='whether to fall back to downloading from the upstream URL')
     parser_download.add_argument(
         '--on-error', choices=['stop', 'warn'], default='stop',
-        help='What to do if the tarball cannot be downloaded')
+        help='what to do if the tarball cannot be downloaded')
     parser_download.add_argument(
         '--no-check-certificate', action='store_true',
-        help='Do not check SSL certificates for https connections')
+        help='do not check SSL certificates for https connections')
 
     parser_upload = subparsers.add_parser(
         'upload', epilog=epilog_upload,
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        help='Upload tarball to Sage mirrors')
+        help='upload tarball to Sage mirrors')
     parser_upload.add_argument(
-        'package_name', type=str, help='Package name or :type:')
+        'package_name', type=str, help='package name or :type:')
 
     parser_fix_checksum = subparsers.add_parser(
         'fix-checksum', epilog=epilog_fix_checksum,
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        help='Fix the checksum of normal packages.')
+        help='fix the checksum of normal packages')
     parser_fix_checksum.add_argument(
         'package_class', metavar='[package_name|:package_type:]',
         type=str, default=[':all:'], nargs='*',
@@ -419,39 +433,39 @@ def make_parser():
     parser_create = subparsers.add_parser(
         'create', epilog=epilog_create,
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        help='Create or overwrite package.')
+        help='create or overwrite a package')
     parser_create.add_argument(
         'package_name', default=None, type=str,
-        help='Package name.')
+        help='package name')
     parser_create.add_argument(
-        '--source', type=str, default=None, help='Package source (one of normal, wheel, script, pip); default depends on provided arguments')
+        '--source', type=str, default=None, help='package source (one of normal, wheel, script, pip); default depends on provided arguments')
     parser_create.add_argument(
-        '--version', type=str, default=None, help='Package version')
+        '--version', type=str, default=None, help='package version')
     parser_create.add_argument(
-        '--tarball', type=str, default=None, help='Tarball filename pattern, e.g. Foo-VERSION.tar.bz2')
+        '--tarball', type=str, default=None, help='tarball filename pattern, e.g. Foo-VERSION.tar.bz2')
     parser_create.add_argument(
-        '--type', type=str, default=None, help='Package type')
+        '--type', type=str, default=None, help='package type')
     parser_create.add_argument(
-        '--url', type=str, default=None, help='Download URL pattern, e.g. http://example.org/Foo-VERSION.tar.bz2')
+        '--url', type=str, default=None, help='download URL pattern, e.g. http://example.org/Foo-VERSION.tar.bz2')
     parser_create.add_argument(
-        '--description', type=str, default=None, help='Short description of the package (for SPKG.rst)')
+        '--description', type=str, default=None, help='short description of the package (for SPKG.rst)')
     parser_create.add_argument(
-        '--license', type=str, default=None, help='License of the package (for SPKG.rst)')
+        '--license', type=str, default=None, help='license of the package (for SPKG.rst)')
     parser_create.add_argument(
-        '--upstream-contact', type=str, default=None, help='Upstream contact (for SPKG.rst)')
+        '--upstream-contact', type=str, default=None, help='upstream contact (for SPKG.rst)')
     parser_create.add_argument(
         '--pypi', action="store_true",
-        help='Create a package for a Python package available on PyPI')
+        help='create a package for a Python package available on PyPI')
 
     parser_clean = subparsers.add_parser(
         'clean', epilog=epilog_clean,
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        help='Remove outdated source tarballs from the upstream/ directory')
+        help='remove outdated source tarballs from the upstream/ directory')
 
     parser_metrics = subparsers.add_parser(
         'metrics', epilog=epilog_metrics,
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        help='Print metrics of given packages')
+        help='print metrics of given packages')
     parser_metrics.add_argument(
         'package_class', metavar='[package_name|:package_type:]',
         type=str, nargs='*', default=[':all:'],
@@ -517,7 +531,9 @@ def run():
                 ssl._create_default_https_context = ssl._create_unverified_context
             except ImportError:
                 pass
-        app.download_cls(args.package_name,
+        app.download_cls(*args.package_class,
+                         has_files=args.has_files, no_files=args.no_files,
+                         exclude=args.exclude,
                          allow_upstream=args.allow_upstream,
                          on_error=args.on_error)
     elif args.subcommand == 'create':