Merge remote-tracking branch 'origin/main' into feature/remove-mssql-…

…support-from-airflow-core

.dockerignore

@@ -36,8 +36,7 @@
 !licenses
 
 # Add those folders to the context so that they are available in the CI container
-!scripts/in_container
-!scripts/docker
+!scripts
 
 # Add tests and kubernetes_tests to context.
 !tests

.github/workflows/ci.yml

@@ -463,7 +463,7 @@ jobs:
       contents: read
     timeout-minutes: 70
     name: >
-      Verify CI Images & Generate constraints
+      Generate constraints with CI image verifying
       ${{needs.build-info.outputs.all-python-versions-list-as-string}}
     runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
     needs: [build-info, wait-for-ci-images]
@@ -605,10 +605,10 @@ jobs:
       - name: "Static checks: basic checks only"
         run: >
           breeze static-checks --show-diff-on-failure --color always --initialize-environment
-          --skip-image-check --commit-ref "${{ github.sha }}"
+          --skip-image-upgrade-check --commit-ref "${{ github.sha }}"
         env:
           VERBOSE: "false"
-          SKIP_IMAGE_PRE_COMMITS: "true"
+          SKIP_BREEZE_PRE_COMMITS: "true"
           SKIP: ${{ needs.build-info.outputs.skip-pre-commits }}
           COLUMNS: "250"
 
@@ -1620,7 +1620,7 @@ jobs:
 
   test-docker-compose-quick-start:
     timeout-minutes: 60
-    name: "Verify PROD image and docker-compose quick start"
+    name: "Docker-compose quick start with PROD image verifying"
     runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
     needs: [build-info, wait-for-prod-images]
     if: needs.build-info.outputs.prod-image-build == 'true'

.pre-commit-config.yaml

@@ -429,6 +429,7 @@ repos:
         language: python
         files: ^docs
         pass_filenames: false
+        additional_dependencies: ['rich>=12.4.4']
       - id: check-pydevd-left-in-code
         language: pygrep
         name: Check for pydevd debug statements accidentally left
@@ -934,7 +935,7 @@ repos:
           ^generated/provider_dependencies.json$
         require_serial: true
         pass_filenames: false
-        additional_dependencies: ['rich>=12.4.4', 'rich-click>=1.7.0', 'inputimeout', 'pyyaml', 'packaging', 'filelock']
+        additional_dependencies: ['rich>=12.4.4']
       - id: check-example-dags-urls
         name: Check that example dags url include provider versions
         entry: ./scripts/ci/pre_commit/pre_commit_update_example_dags_paths.py
@@ -1023,52 +1024,52 @@ repos:
         name: Run mypy for dev
         language: python
         entry: ./scripts/ci/pre_commit/pre_commit_mypy.py
-        files: ^dev/.*\.py$
+        files: ^dev/.*\.py$|^scripts/.*\.py$
         require_serial: true
-        additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml', 'jsonschema', 'filelock', 'markdown-it-py']
+        additional_dependencies: ['rich>=12.4.4']
       - id: mypy-core
         name: Run mypy for core
         language: python
         entry: ./scripts/ci/pre_commit/pre_commit_mypy.py --namespace-packages
         files: \.py$
-        exclude: ^.*/.*_vendor/|^airflow/migrations|^airflow/providers|^dev|^docs|^provider_packages|^tests/providers|^tests/system/providers|^tests/dags/test_imports.py
+        exclude: ^.*/.*_vendor/|^airflow/migrations|^airflow/providers|^dev|^scripts|^docs|^provider_packages|^tests/providers|^tests/system/providers|^tests/dags/test_imports.py
         require_serial: true
-        additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml', 'jsonschema', 'filelock', 'markdown-it-py']
+        additional_dependencies: ['rich>=12.4.4']
       - id: mypy-providers
         name: Run mypy for providers
         language: python
         entry: ./scripts/ci/pre_commit/pre_commit_mypy.py --namespace-packages
         files: ^airflow/providers/.*\.py$|^tests/providers/.*\.py$|^tests/system/providers/.*\.py$
         exclude: ^.*/.*_vendor/
         require_serial: true
-        additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml', 'jsonschema', 'filelock', 'markdown-it-py']
+        additional_dependencies: ['rich>=12.4.4']
       - id: mypy-docs
         name: Run mypy for /docs/ folder
         language: python
         entry: ./scripts/ci/pre_commit/pre_commit_mypy.py
         files: ^docs/.*\.py$
         exclude: ^docs/rtd-deprecation
         require_serial: true
-        additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml', 'jsonschema', 'filelock', 'markdown-it-py']
-      - id:   check-provider-yaml-valid
+        additional_dependencies: ['rich>=12.4.4']
+      - id: check-provider-yaml-valid
         name: Validate provider.yaml files
         entry: ./scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
         language: python
         files: ^airflow/providers/.*/provider\.yaml$
-        additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml', 'jsonschema', 'filelock', 'markdown-it-py']
+        additional_dependencies: ['rich>=12.4.4']
         require_serial: true
       - id: update-migration-references
         name: Update migration ref doc
         language: python
         entry: ./scripts/ci/pre_commit/pre_commit_migration_reference.py
         pass_filenames: false
         files: ^airflow/migrations/versions/.*\.py$|^docs/apache-airflow/migrations-ref\.rst$
-        additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml', 'jsonschema', 'filelock', 'markdown-it-py']
+        additional_dependencies: ['rich>=12.4.4']
       - id: update-er-diagram
         name: Update ER diagram
         language: python
         entry: ./scripts/ci/pre_commit/pre_commit_update_er_diagram.py
         pass_filenames: false
         files: ^airflow/migrations/versions/.*\.py$|^docs/apache-airflow/migrations-ref\.rst$
-        additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml', 'jsonschema', 'filelock', 'markdown-it-py']
+        additional_dependencies: ['rich>=12.4.4']
         ## ONLY ADD PRE-COMMITS HERE THAT REQUIRE CI IMAGE

BREEZE.rst

@@ -93,6 +93,7 @@ Here is an example configuration with more than 200GB disk space for Docker:
 - 5. In some cases you might make sure that "Allow the default Docker socket to
   be used" in "Advanced" tab of "Docker Desktop" settings is checked
 
+
 .. raw:: html
 
    <div align="center">
@@ -458,8 +459,13 @@ The choices you make are persisted in the ``./.build/`` cache directory so that
 them when you run the script. You can delete the ``.build/`` directory in case you want to restore the
 default settings.
 
+You can also run breeze with ``SKIP_SAVING_CHOICES`` to non-empty value and breeze invocation will not save
+used cache value to cache - this is useful when you run non-interactive scripts with ``breeze shell`` and
+want to - for example - force Python version used only for that execution without changing the Python version
+that user used last time.
+
 You can see which value of the parameters that can be stored persistently in cache marked with >VALUE<
-in the help of the commands.
+in the help of the commands (for example in output of ``breeze config --help``).
 
 Building the documentation
 --------------------------
@@ -1802,7 +1808,7 @@ check if there are any images that need regeneration.
   :alt: Breeze setup regenerate-command-images
 
 Breeze check-all-params-in-groups
-...................
+.................................
 
 When you add a breeze command or modify a parameter, you are also supposed to make sure that "rich groups"
 for the command is present and that all parameters are assigned to the right group so they can be
@@ -1813,6 +1819,17 @@ nicely presented in ``--help`` output. You can check that via ``check-all-params
   :width: 100%
   :alt: Breeze setup check-all-params-in-group
 
+Breeze synchronize-local-mounts
+...............................
+
+When you add volumes mounted to docker, they need to be added in ``docker_command_utils.py`` - so that they
+are added by plain ``docker`` command, but they also need to be synchronized with ``local.yml``. This can be
+done via ``synchronize-local-mounts`` command.
+
+.. image:: ./images/breeze/output_setup_synchronize-local-mounts.svg
+  :target: https://raw.githubusercontent.com/apache/airflow/main/images/breeze/output_setup_synchronize-local-mounts.svg
+  :width: 100%
+  :alt: Breeze setup synchronize-local-mounts
 
 CI tasks
 --------
@@ -2796,7 +2813,7 @@ make sure to follow these steps:
 * choose ``module`` to run and set it to ``airflow_breeze.breeze`` - this is the entrypoint of breeze
 * add parameters you want to run breeze with (for example ``ci-image build`` if you want to debug
   how breeze builds the CI image
-* set ``SKIP_UPGRADE_CHECK`` environment variable to ``true`` to bypass built-in upgrade check of breeze,
+* set ``SKIP_BREEZE_SELF_UPGRADE_CHECK`` environment variable to ``true`` to bypass built-in upgrade check of breeze,
   this will bypass the check we run in Breeze to see if there are new requirements to install for it
 
 See example configuration for PyCharm which has run/debug configuration for

CI.rst

@@ -338,7 +338,7 @@ This workflow is a regular workflow that performs all checks of Airflow code.
 +---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+
 | Build CI images                 | Builds images in-workflow (not in the ``build images``)  | -        | Yes      | Yes (1)   | Yes (4)           |
 +---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+
-| Verify CI/generate constraints  | Verify CI image and generate constraints for the build   | Yes (2)  | Yes (2)  | Yes (2)   | Yes (2)           |
+| Generate constraints/CI verify  | Generate constraints for the build and verify CI image   | Yes (2)  | Yes (2)  | Yes (2)   | Yes (2)           |
 +---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+
 | Build PROD images               | Builds images in-workflow (not in the ``build images``)  | -        | Yes      | Yes (1)   | Yes (4)           |
 +---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+
@@ -388,7 +388,7 @@ This workflow is a regular workflow that performs all checks of Airflow code.
 +---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+
 | Wait for PROD Images            | Waits for and verify PROD Images                         | Yes (2)  | Yes (2)  | Yes (2)   | Yes (2)           |
 +---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+
-| Verify PROD/test compose        | Verify PROD image and tests quick-start Docker Compose   | Yes      | Yes      | Yes       | Yes               |
+| Docker Compose test/PROD verify | Tests quick-start Docker Compose and verify PROD image   | Yes      | Yes      | Yes       | Yes               |
 +---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+
 | Tests Kubernetes                | Run Kubernetes test                                      | Yes      | Yes      | Yes       | -                 |
 +---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+
@@ -535,7 +535,7 @@ Depending whether the scripts are run locally via `Breeze <BREEZE.rst>`_ or whet
 are run in ``Build Images`` or ``Tests`` workflows they can take different values.
 
 You can use those variables when you try to reproduce the build locally (alternatively you can pass
-those via command line flags passed to ``breeze`` command.
+those via corresponding command line flags passed to ``breeze shell`` command.
 
 +-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
 | Variable                                | Local       | Build Images | CI         | Comment                                         |
@@ -582,6 +582,10 @@ those via command line flags passed to ``breeze`` command.
 |                                         |             |              |            |                                                 |
 |                                         |             |              |            | \* set to true in pre-commits                   |
 +-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
+| ``SKIP_IMAGE_UPGRADE_CHECK``            |   false\*   |    false\*   |   false\*  | Skip checking if image should be upgraded       |
+|                                         |             |              |            |                                                 |
+|                                         |             |              |            | \* set to true in pre-commits                   |
++-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
 | ``SKIP_PROVIDER_TESTS``                 |   false\*   |    false\*   |   false\*  | Skip running provider integration tests         |
 +-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
 | ``SKIP_SSH_SETUP``                      |   false\*   |    false\*   |   false\*  | Skip setting up SSH server for tests.           |

STATIC_CODE_CHECKS.rst

@@ -123,8 +123,8 @@ require Breeze Docker image to be built locally.
   ``export SKIP=ruff,mypy-core,``. You can also add this to your ``.bashrc`` or ``.zshrc`` if you
   do not want to set it manually every time you enter the terminal.
 
-  In case you do not have breeze image configured locally, you can also disable all checks that require
-  the image by setting ``SKIP_IMAGE_PRE_COMMITS`` to "true". This will mark the tests as "green" automatically
+  In case you do not have breeze image configured locally, you can also disable all checks that require breeze
+  the image by setting ``SKIP_BREEZE_PRE_COMMITS`` to "true". This will mark the tests as "green" automatically
   when run locally (note that those checks will anyway run in CI).
 
 .. note:: Mypy volume cache