diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index 8af0d7c2168..e727865bf44 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -37,16 +37,37 @@ jobs: sudo DEBIAN_FRONTEND=noninteractive apt-get update sudo DEBIAN_FRONTEND=noninteractive apt-get install $(build/bin/sage-get-system-packages debian _bootstrap) - name: make dist (--disable-download-from-upstream-url) + id: make_dist run: | ./bootstrap -D && ./configure --disable-download-from-upstream-url && make dist env: MAKE: make -j8 + - name: make download (--disable-download-from-upstream-url) + id: make_download + run: | + make -k download DOWNLOAD_PACKAGES=":all: --no-file huge" + env: + MAKE: make -j8 + - name: Reconfigure with --enable-download-from-upstream-url + if: (success() || failure()) && (steps.make_dist.outcome != 'success' || steps.make_download.outcome != 'success') + run: | + ./configure - name: make dist (--enable-download-from-upstream-url) - if: failure() + if: (success() || failure()) && steps.make_dist.outcome != 'success' run: | - ./configure && make dist + make dist env: MAKE: make -j8 + - name: make download (--enable-download-from-upstream-url) + if: (success() || failure()) && steps.make_download.outcome != 'success' + run: | + make -k download DOWNLOAD_PACKAGES=":all: --no-file huge --allow-upstream" + env: + MAKE: make -j8 + - name: Remove what cannot be distributed + if: success() || failure() + run: | + rm -f upstream/*do-not-distribute* - uses: actions/upload-artifact@v4 if: success() || failure() with: diff --git a/Makefile b/Makefile index 4662dcd14b9..8c9a4504b49 100644 --- a/Makefile +++ b/Makefile @@ -79,10 +79,11 @@ reconfigure: fi # Preemptively download all source tarballs of normal packages. +DOWNLOAD_PACKAGES=:all: download: export SAGE_ROOT=$$(pwd) && \ export PATH=$$SAGE_ROOT/build/bin:$$PATH && \ - sage-package download :all: + sage-package download $(DOWNLOAD_PACKAGES) dist: build/make/Makefile ./sage --sdist diff --git a/build/pkgs/deprecation/spkg-configure.m4 b/build/pkgs/deprecation/spkg-configure.m4 deleted file mode 100644 index 306d7c5cd1e..00000000000 --- a/build/pkgs/deprecation/spkg-configure.m4 +++ /dev/null @@ -1 +0,0 @@ -SAGE_SPKG_CONFIGURE([deprecation], [SAGE_PYTHON_PACKAGE_CHECK([deprecation])]) diff --git a/build/pkgs/pcre/distros/alpine.txt b/build/pkgs/pcre/distros/alpine.txt deleted file mode 100644 index a16e4119734..00000000000 --- a/build/pkgs/pcre/distros/alpine.txt +++ /dev/null @@ -1 +0,0 @@ -pcre-dev diff --git a/build/pkgs/pycygwin/spkg-configure.m4 b/build/pkgs/pycygwin/spkg-configure.m4 deleted file mode 100644 index 7876a693d7b..00000000000 --- a/build/pkgs/pycygwin/spkg-configure.m4 +++ /dev/null @@ -1 +0,0 @@ -SAGE_SPKG_CONFIGURE([pycygwin], [SAGE_PYTHON_PACKAGE_CHECK([pycygwin])]) diff --git a/build/pkgs/setuptools_scm_git_archive/distros/alpine.txt b/build/pkgs/setuptools_scm_git_archive/distros/alpine.txt deleted file mode 100644 index c08bc88e16c..00000000000 --- a/build/pkgs/setuptools_scm_git_archive/distros/alpine.txt +++ /dev/null @@ -1 +0,0 @@ -py3-setuptools-scm-git-archive diff --git a/build/pkgs/setuptools_scm_git_archive/distros/arch.txt b/build/pkgs/setuptools_scm_git_archive/distros/arch.txt deleted file mode 100644 index bc2a39f97a3..00000000000 --- a/build/pkgs/setuptools_scm_git_archive/distros/arch.txt +++ /dev/null @@ -1 +0,0 @@ -python-setuptools-scm-git-archive diff --git a/build/pkgs/setuptools_scm_git_archive/distros/debian.txt b/build/pkgs/setuptools_scm_git_archive/distros/debian.txt deleted file mode 100644 index 538474ff946..00000000000 --- a/build/pkgs/setuptools_scm_git_archive/distros/debian.txt +++ /dev/null @@ -1 +0,0 @@ -setuptools-scm-git-archive diff --git a/build/pkgs/setuptools_scm_git_archive/distros/fedora.txt b/build/pkgs/setuptools_scm_git_archive/distros/fedora.txt deleted file mode 100644 index ada37357769..00000000000 --- a/build/pkgs/setuptools_scm_git_archive/distros/fedora.txt +++ /dev/null @@ -1 +0,0 @@ -python-setuptools_scm_git_archive diff --git a/build/pkgs/setuptools_scm_git_archive/distros/freebsd.txt b/build/pkgs/setuptools_scm_git_archive/distros/freebsd.txt deleted file mode 100644 index 2ace76a2af0..00000000000 --- a/build/pkgs/setuptools_scm_git_archive/distros/freebsd.txt +++ /dev/null @@ -1 +0,0 @@ -devel/py-setuptools_scm_git_archive diff --git a/build/pkgs/setuptools_scm_git_archive/distros/gentoo.txt b/build/pkgs/setuptools_scm_git_archive/distros/gentoo.txt deleted file mode 100644 index fb7388e3dd7..00000000000 --- a/build/pkgs/setuptools_scm_git_archive/distros/gentoo.txt +++ /dev/null @@ -1 +0,0 @@ -dev-python/setuptools_scm_git_archive diff --git a/build/pkgs/setuptools_scm_git_archive/spkg-configure.m4 b/build/pkgs/setuptools_scm_git_archive/spkg-configure.m4 deleted file mode 100644 index 0da3db40d22..00000000000 --- a/build/pkgs/setuptools_scm_git_archive/spkg-configure.m4 +++ /dev/null @@ -1 +0,0 @@ -SAGE_SPKG_CONFIGURE([setuptools_scm_git_archive], [SAGE_PYTHON_PACKAGE_CHECK([setuptools_scm_git_archive])]) diff --git a/build/sage_bootstrap/app.py b/build/sage_bootstrap/app.py index 96bca3a6d8c..9de521c7430 100644 --- a/build/sage_bootstrap/app.py +++ b/build/sage_bootstrap/app.py @@ -304,11 +304,14 @@ def download(self, package_name, allow_upstream=False): package.tarball.download(allow_upstream=allow_upstream) print(package.tarball.upstream_fqn) - def download_cls(self, package_name_or_class, allow_upstream=False, on_error='stop'): + def download_cls(self, *package_classes, **kwds): """ Download a package or a class of packages """ - pc = PackageClass(package_name_or_class, has_files=['checksums.ini']) + allow_upstream = kwds.pop('allow_upstream', False) + on_error = kwds.pop('on_error', 'stop') + has_files = list(kwds.pop('has_files', [])) + pc = PackageClass(*package_classes, has_files=has_files + ['checksums.ini'], **kwds) def download_with_args(package): try: diff --git a/build/sage_bootstrap/cmdline.py b/build/sage_bootstrap/cmdline.py index 3ed185a9185..68c6b931e04 100644 --- a/build/sage_bootstrap/cmdline.py +++ b/build/sage_bootstrap/cmdline.py @@ -270,18 +270,18 @@ def make_parser(): parser_config = subparsers.add_parser( 'config', epilog=epilog_config, formatter_class=argparse.RawDescriptionHelpFormatter, - help='Print the configuration') + help='print the configuration') parser_list = subparsers.add_parser( 'list', epilog=epilog_list, formatter_class=argparse.RawDescriptionHelpFormatter, - help='Print a list of packages known to Sage') + help='print a list of packages known to Sage') parser_list.add_argument( 'package_class', metavar='[package_name|:package_type:]', type=str, default=[':all-or-nothing:'], nargs='*', help=('package name or designator for all packages of a given type ' '(one of :all:, :standard:, :optional:, and :experimental:); ' - 'default: :all: (or nothing when --include-dependencies or --exclude-dependencies is given')) + 'default: :all: (or nothing when --include-dependencies or --exclude-dependencies is given)')) parser_list.add_argument( '--has-file', action='append', default=[], metavar='FILENAME', dest='has_files', help=('only include packages that have this file in their metadata directory ' @@ -303,7 +303,7 @@ def make_parser(): parser_properties = subparsers.add_parser( 'properties', epilog=epilog_properties, formatter_class=argparse.RawDescriptionHelpFormatter, - help='Print properties of given packages') + help='print properties of given packages') parser_properties.add_argument( 'package_class', metavar='[package_name|:package_type:]', type=str, nargs='+', @@ -316,7 +316,7 @@ def make_parser(): parser_dependencies = subparsers.add_parser( 'dependencies', epilog=epilog_dependencies, formatter_class=argparse.RawDescriptionHelpFormatter, - help='Print the list of packages that are dependencies of given packages') + help='print the list of packages that are dependencies of given packages') parser_dependencies.add_argument( 'package_class', metavar='[package_name|:package_type:]', type=str, nargs='+', @@ -341,74 +341,88 @@ def make_parser(): parser_name = subparsers.add_parser( 'name', epilog=epilog_name, formatter_class=argparse.RawDescriptionHelpFormatter, - help='Find the package name given a tarball filename') - parser_name.add_argument('tarball_filename', type=str, help='Tarball filename') + help='find the package name given a tarball filename') + parser_name.add_argument('tarball_filename', type=str, help='tarball filename') parser_tarball = subparsers.add_parser( 'tarball', epilog=epilog_tarball, formatter_class=argparse.RawDescriptionHelpFormatter, - help='Find the tarball filename given a package name') - parser_tarball.add_argument('package_name', type=str, help='Package name') + help='find the tarball filename given a package name') + parser_tarball.add_argument('package_name', type=str, help='package name') parser_apropos = subparsers.add_parser( 'apropos', epilog=epilog_apropos, formatter_class=argparse.RawDescriptionHelpFormatter, - help='Find up to 5 package names that are close to the given name') + help='find up to 5 package names that are close to the given name') parser_apropos.add_argument( 'incorrect_name', type=str, - help='Fuzzy name to search for') + help='fuzzy name to search for') parser_update = subparsers.add_parser( 'update', epilog=epilog_update, formatter_class=argparse.RawDescriptionHelpFormatter, - help='Update a package. This modifies the Sage sources.') + help='update a package, modifying the Sage sources') parser_update.add_argument( - 'package_name', type=str, help='Package name') + 'package_name', type=str, help='package name') parser_update.add_argument( - 'new_version', type=str, help='New version') + 'new_version', type=str, help='new version') parser_update.add_argument( - '--url', type=str, default=None, help='Download URL') + '--url', type=str, default=None, help='download URL') parser_update.add_argument( '--commit', action="store_true", - help='Whether to run "git commit"') + help='whether to run "git commit"') parser_update_latest = subparsers.add_parser( 'update-latest', epilog=epilog_update_latest, formatter_class=argparse.RawDescriptionHelpFormatter, - help='Update a package to the latest version. This modifies the Sage sources.') + help='update a package to the latest version, modifying the Sage sources') parser_update_latest.add_argument( - 'package_name', type=str, help='Package name (:all: for all packages)') + 'package_name', type=str, help='package name (:all: for all packages)') parser_update_latest.add_argument( '--commit', action="store_true", - help='Whether to run "git commit"') + help='whether to run "git commit"') parser_download = subparsers.add_parser( 'download', epilog=epilog_download, formatter_class=argparse.RawDescriptionHelpFormatter, - help='Download tarball') + help='download tarball') parser_download.add_argument( - 'package_name', type=str, help='Package name or :type:') + 'package_class', metavar='[package_name|:package_type:]', + type=str, nargs='+', + help=('package name or designator for all packages of a given type ' + '(one of :all:, :standard:, :optional:, and :experimental:)')) + parser_download.add_argument( + '--has-file', action='append', default=[], metavar='FILENAME', dest='has_files', + help=('only include packages that have this file in their metadata directory ' + '(examples: SPKG.rst, spkg-configure.m4, distros/debian.txt, spkg-install|spkg-install.in)')) + parser_download.add_argument( + '--no-file', action='append', default=[], metavar='FILENAME', dest='no_files', + help=('only include packages that do not have this file in their metadata directory ' + '(examples: huge, patches, huge|has_nonfree_dependencies)')) + parser_download.add_argument( + '--exclude', nargs='*', action='append', default=[], metavar='PACKAGE_NAME', + help='exclude package from list') parser_download.add_argument( '--allow-upstream', action="store_true", - help='Whether to fall back to downloading from the upstream URL') + help='whether to fall back to downloading from the upstream URL') parser_download.add_argument( '--on-error', choices=['stop', 'warn'], default='stop', - help='What to do if the tarball cannot be downloaded') + help='what to do if the tarball cannot be downloaded') parser_download.add_argument( '--no-check-certificate', action='store_true', - help='Do not check SSL certificates for https connections') + help='do not check SSL certificates for https connections') parser_upload = subparsers.add_parser( 'upload', epilog=epilog_upload, formatter_class=argparse.RawDescriptionHelpFormatter, - help='Upload tarball to Sage mirrors') + help='upload tarball to Sage mirrors') parser_upload.add_argument( - 'package_name', type=str, help='Package name or :type:') + 'package_name', type=str, help='package name or :type:') parser_fix_checksum = subparsers.add_parser( 'fix-checksum', epilog=epilog_fix_checksum, formatter_class=argparse.RawDescriptionHelpFormatter, - help='Fix the checksum of normal packages.') + help='fix the checksum of normal packages') parser_fix_checksum.add_argument( 'package_class', metavar='[package_name|:package_type:]', type=str, default=[':all:'], nargs='*', @@ -419,39 +433,39 @@ def make_parser(): parser_create = subparsers.add_parser( 'create', epilog=epilog_create, formatter_class=argparse.RawDescriptionHelpFormatter, - help='Create or overwrite package.') + help='create or overwrite a package') parser_create.add_argument( 'package_name', default=None, type=str, - help='Package name.') + help='package name') parser_create.add_argument( - '--source', type=str, default=None, help='Package source (one of normal, wheel, script, pip); default depends on provided arguments') + '--source', type=str, default=None, help='package source (one of normal, wheel, script, pip); default depends on provided arguments') parser_create.add_argument( - '--version', type=str, default=None, help='Package version') + '--version', type=str, default=None, help='package version') parser_create.add_argument( - '--tarball', type=str, default=None, help='Tarball filename pattern, e.g. Foo-VERSION.tar.bz2') + '--tarball', type=str, default=None, help='tarball filename pattern, e.g. Foo-VERSION.tar.bz2') parser_create.add_argument( - '--type', type=str, default=None, help='Package type') + '--type', type=str, default=None, help='package type') parser_create.add_argument( - '--url', type=str, default=None, help='Download URL pattern, e.g. http://example.org/Foo-VERSION.tar.bz2') + '--url', type=str, default=None, help='download URL pattern, e.g. http://example.org/Foo-VERSION.tar.bz2') parser_create.add_argument( - '--description', type=str, default=None, help='Short description of the package (for SPKG.rst)') + '--description', type=str, default=None, help='short description of the package (for SPKG.rst)') parser_create.add_argument( - '--license', type=str, default=None, help='License of the package (for SPKG.rst)') + '--license', type=str, default=None, help='license of the package (for SPKG.rst)') parser_create.add_argument( - '--upstream-contact', type=str, default=None, help='Upstream contact (for SPKG.rst)') + '--upstream-contact', type=str, default=None, help='upstream contact (for SPKG.rst)') parser_create.add_argument( '--pypi', action="store_true", - help='Create a package for a Python package available on PyPI') + help='create a package for a Python package available on PyPI') parser_clean = subparsers.add_parser( 'clean', epilog=epilog_clean, formatter_class=argparse.RawDescriptionHelpFormatter, - help='Remove outdated source tarballs from the upstream/ directory') + help='remove outdated source tarballs from the upstream/ directory') parser_metrics = subparsers.add_parser( 'metrics', epilog=epilog_metrics, formatter_class=argparse.RawDescriptionHelpFormatter, - help='Print metrics of given packages') + help='print metrics of given packages') parser_metrics.add_argument( 'package_class', metavar='[package_name|:package_type:]', type=str, nargs='*', default=[':all:'], @@ -517,7 +531,9 @@ def run(): ssl._create_default_https_context = ssl._create_unverified_context except ImportError: pass - app.download_cls(args.package_name, + app.download_cls(*args.package_class, + has_files=args.has_files, no_files=args.no_files, + exclude=args.exclude, allow_upstream=args.allow_upstream, on_error=args.on_error) elif args.subcommand == 'create':