diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh
index 23a411d19..503947556 100755
--- a/.github/scripts/validate_binaries.sh
+++ b/.github/scripts/validate_binaries.sh
@@ -49,7 +49,8 @@ else
         export PATH=${OLD_PATH}
     fi
 
-    if [[ ${INCLUDE_TEST_OPS} == 'true' &&  ${MATRIX_GPU_ARCH_TYPE} == 'cuda' ]]; then
+    # We are only interested in CUDA tests and Python 3.8-3.11. Not all requirement libraries are available for 3.12 yet.
+    if [[ ${INCLUDE_TEST_OPS} == 'true' &&  ${MATRIX_GPU_ARCH_TYPE} == 'cuda' && ${MATRIX_PYTHON_VERSION} != "3.12" ]]; then
         source ./.github/scripts/validate_test_ops.sh
     fi
 
diff --git a/.github/scripts/validate_test_ops.sh b/.github/scripts/validate_test_ops.sh
index 60686ee09..e93f28918 100644
--- a/.github/scripts/validate_test_ops.sh
+++ b/.github/scripts/validate_test_ops.sh
@@ -18,7 +18,7 @@ retry git clone ${BRANCH} --depth 1 https://github.com/pytorch/pytorch.git
 retry git submodule update --init --recursive
 pushd pytorch
 
-pip install expecttest numpy pyyaml jinja2 packaging hypothesis unittest-xml-reporting
+pip install expecttest numpy pyyaml jinja2 packaging hypothesis unittest-xml-reporting scipy
 
 # Run test_ops validation
 export CUDA_LAUNCH_BLOCKING=1
diff --git a/.github/workflows/validate-linux-binaries.yml b/.github/workflows/validate-linux-binaries.yml
index f674afa04..de5bda999 100644
--- a/.github/workflows/validate-linux-binaries.yml
+++ b/.github/workflows/validate-linux-binaries.yml
@@ -90,7 +90,7 @@ jobs:
       ref: ${{ inputs.ref || github.ref }}
       job-name: ${{ matrix.build_name }}
       binary-matrix: ${{ toJSON(matrix) }}
-      timeout: 120
+      timeout: 180
       script: |
         set -ex
         export ENV_NAME="conda-env-${{ github.run_id }}"