diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml new file mode 100644 index 0000000000..7875107ddd --- /dev/null +++ b/.github/actions/setup-env/action.yml @@ -0,0 +1,34 @@ +name: "Setup Python and Rust Environment" +description: "Set up Python, virtual environment, and Rust toolchain" + +inputs: + + python-version: + description: "The Python version to set up" + required: true + default: "3.10" + + rust-toolchain: + description: "The Rust toolchain to set up" + required: true + default: "stable" + +runs: + using: "composite" + + steps: + + - name: Set up Python ${{ inputs.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + + - name: Install Rust toolchain + uses: actions-rs/toolchain@v1 + with: + profile: default + toolchain: ${{ inputs.rust-toolchain }} + override: true + components: rustfmt, clippy + + - uses: Swatinem/rust-cache@v2 \ No newline at end of file diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 079cd66fcc..a6f5d32afe 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -47,25 +47,13 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: rustfmt, clippy - - - uses: Swatinem/rust-cache@v2 - - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: '3.10' + - name: Setup Environment + uses: ./.github/actions/setup-env - name: Build and install deltalake run: | cd python - pip install virtualenv - virtualenv venv + python -m venv venv source venv/bin/activate make ${{ env.BUILD_ARGS }} diff --git a/.github/workflows/python_build.yml b/.github/workflows/python_build.yml index ce4a58c844..4bae1b5ab9 100644 --- a/.github/workflows/python_build.yml +++ b/.github/workflows/python_build.yml @@ -15,23 +15,17 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: "3.10" + + - name: Setup Environment + uses: ./.github/actions/setup-env - name: Check Python run: | + python -m venv venv + source venv/bin/activate pip install ruff==0.5.2 mypy==1.10.1 types-dataclasses typing-extensions make check-python - - name: Install minimal stable with clippy and rustfmt - uses: actions-rs/toolchain@v1 - with: - profile: default - toolchain: stable - override: true - - name: Check Rust run: make check-rust @@ -45,24 +39,14 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v2 + - name: Setup Environment + uses: ./.github/actions/setup-env with: python-version: 3.8 - - name: Install latest nightly - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: rustfmt, clippy - - - uses: Swatinem/rust-cache@v2 - - name: Build and install deltalake run: | - pip install virtualenv - virtualenv venv + python -m venv venv source venv/bin/activate make setup # Install minimum PyArrow version @@ -89,26 +73,15 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Install latest nightly - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: rustfmt, clippy - - - uses: Swatinem/rust-cache@v2 - - - uses: actions/setup-python@v3 - with: - python-version: "3.10" + - name: Setup Environment + uses: ./.github/actions/setup-env - name: Start emulated services run: docker-compose up -d - name: Build and install deltalake run: | - pip install virtualenv - virtualenv venv + python -m venv venv source venv/bin/activate make develop @@ -137,23 +110,12 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Install latest nightly - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: rustfmt, clippy - - - uses: Swatinem/rust-cache@v2 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" + - name: Setup Environment + uses: ./.github/actions/setup-env - name: Build deltalake in release mode run: | - pip install virtualenv - virtualenv venv + python -m venv venv source venv/bin/activate MATURIN_EXTRA_ARGS=--release make develop @@ -187,18 +149,8 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Install latest nightly - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: rustfmt, clippy - - - uses: Swatinem/rust-cache@v2 - - - uses: actions/setup-python@v3 - with: - python-version: "3.10" + - name: Setup Environment + uses: ./.github/actions/setup-env - uses: actions/setup-java@v2 with: @@ -207,8 +159,7 @@ jobs: - name: Build and install deltalake run: | - pip install virtualenv - virtualenv venv + python -m venv venv source venv/bin/activate make develop-pyspark @@ -231,15 +182,14 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + - name: Setup Environment + uses: ./.github/actions/setup-env with: python-version: ${{ matrix.python-version }} - name: Build and install deltalake run: | - pip install virtualenv - virtualenv venv + python -m venv venv source venv/bin/activate make setup maturin develop diff --git a/.gitignore b/.gitignore index 84fc17c5f2..c5c12bde5c 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ __blobstorage__ .githubchangeloggenerator.cache.log .githubchangeloggenerator.cache/ .githubchangeloggenerator* +data # Add all Cargo.lock files except for those in binary crates Cargo.lock @@ -30,4 +31,4 @@ Cargo.lock justfile site -__pycache__ +__pycache__ \ No newline at end of file