diff --git a/.github/workflows/test_export_onnx.yml b/.github/workflows/test_export_onnx.yml
index d1fd4a9723f..187aa6a65c6 100644
--- a/.github/workflows/test_export_onnx.yml
+++ b/.github/workflows/test_export_onnx.yml
@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.9']
+        python-version: ["3.9"]
         os: [ubuntu-20.04]
 
     runs-on: ${{ matrix.os }}
@@ -27,13 +27,14 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies for pytorch export
         run: |
-          pip install .[tests,exporters]
+          pip install .[tests,exporters,diffusers]
       - name: Test with unittest
         working-directory: tests
         run: |
           pytest exporters/onnx/test_onnx_*.py -s -n auto -m "not tensorflow_test and not timm_test" --durations=0
       - name: Install dependencies for tensorflow export
         run: |
+          pip uninstall diffusers -y
           pip install .[tests,exporters-tf]
       - name: Test with unittest
         working-directory: tests
diff --git a/.github/workflows/test_export_onnx_cli.yml b/.github/workflows/test_export_onnx_cli.yml
index 618a140c147..41394fef92b 100644
--- a/.github/workflows/test_export_onnx_cli.yml
+++ b/.github/workflows/test_export_onnx_cli.yml
@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.9]
+        python-version: ["3.9"]
         os: [ubuntu-20.04]
 
     runs-on: ${{ matrix.os }}
@@ -27,7 +27,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies for pytorch export
         run: |
-          pip install .[tests,exporters]
+          pip install .[tests,exporters,diffusers]
       - name: Test with unittest
         working-directory: tests
         run: |
diff --git a/.github/workflows/test_exporters_slow.yml b/.github/workflows/test_exporters_slow.yml
index b5f142fc7dc..51424a18f3f 100644
--- a/.github/workflows/test_exporters_slow.yml
+++ b/.github/workflows/test_exporters_slow.yml
@@ -14,26 +14,27 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.9']
+        python-version: ["3.9"]
 
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v2
-    - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies for pytorch export
-      run: |
-        pip install .[tests,exporters]
-    - name: Test with unittest
-      working-directory: tests
-      run: |
-        RUN_SLOW=1 pytest exporters -s -m "not tensorflow_test and run_slow" --durations=0
-    - name: Install dependencies for tensorflow export
-      run: |
-        pip install .[tests,exporters-tf]
-    - name: Test with unittest
-      working-directory: tests
-      run: |
-        RUN_SLOW=1 pytest exporters -s -m "tensorflow_test and run_slow" --durations=0
+      - uses: actions/checkout@v2
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies for pytorch export
+        run: |
+          pip install .[tests,exporters,diffusers]
+      - name: Test with unittest
+        working-directory: tests
+        run: |
+          RUN_SLOW=1 pytest exporters -s -m "not tensorflow_test and run_slow" --durations=0
+      - name: Install dependencies for tensorflow export
+        run: |
+          pip uninstall diffusers -y
+          pip install .[tests,exporters-tf]
+      - name: Test with unittest
+        working-directory: tests
+        run: |
+          RUN_SLOW=1 pytest exporters -s -m "tensorflow_test and run_slow" --durations=0
diff --git a/.github/workflows/test_onnx.yml b/.github/workflows/test_onnx.yml
index 418a9e42c1a..79a55a6c4a1 100644
--- a/.github/workflows/test_onnx.yml
+++ b/.github/workflows/test_onnx.yml
@@ -27,7 +27,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        pip install .[tests,exporters]
+        pip install .[tests,exporters,diffusers]
     - name: Test with unittest
       working-directory: tests
       run: |
diff --git a/.github/workflows/test_onnxruntime.yml b/.github/workflows/test_onnxruntime.yml
index f461de7cbd2..e4463bb0a34 100644
--- a/.github/workflows/test_onnxruntime.yml
+++ b/.github/workflows/test_onnxruntime.yml
@@ -38,17 +38,23 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: "3.9"
 
       - name: Install dependencies
         run: |
           pip install --upgrade pip
           pip install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
-          pip install .[tests,onnxruntime]
+          pip install .[tests,onnxruntime,diffusers]
 
       - name: Install transformers ${{ matrix.transformers-version }}
         if: ${{ matrix.transformers-version != 'latest' }}
-        run: pip install transformers==${{ matrix.transformers-version }}
+        run: |
+          pip install "transformers==${{ matrix.transformers-version }}"
+
+      - name: Downgrade diffusers
+        if: matrix.transformers-version == '4.36.*'
+        run: |
+          pip install "diffusers<0.32.0"
 
       - name: Test with pytest (in series)
         run: |
diff --git a/.github/workflows/test_onnxruntime_slow.yml b/.github/workflows/test_onnxruntime_slow.yml
index 5e5745c9534..92e2d4a8fc5 100644
--- a/.github/workflows/test_onnxruntime_slow.yml
+++ b/.github/workflows/test_onnxruntime_slow.yml
@@ -42,7 +42,7 @@ jobs:
         run: |
           pip install --upgrade pip
           pip install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
-          pip install .[tests,onnxruntime]
+          pip install .[tests,onnxruntime,diffusers]
 
       - name: Test with pytest
         run: |
diff --git a/optimum/exporters/onnx/__main__.py b/optimum/exporters/onnx/__main__.py
index 280c6fc6554..20bea423cb3 100644
--- a/optimum/exporters/onnx/__main__.py
+++ b/optimum/exporters/onnx/__main__.py
@@ -256,7 +256,7 @@ def main_export(
 
     if task == "auto":
         try:
-            task = TasksManager.infer_task_from_model(model_name_or_path)
+            task = TasksManager.infer_task_from_model(model_name_or_path, library_name=library_name)
         except KeyError as e:
             raise KeyError(
                 f"The task could not be automatically inferred. Please provide the argument --task with the relevant task from {', '.join(TasksManager.get_all_tasks())}. Detailed error: {e}"
diff --git a/optimum/exporters/onnx/model_configs.py b/optimum/exporters/onnx/model_configs.py
index b587f163163..fd3c20d29d6 100644
--- a/optimum/exporters/onnx/model_configs.py
+++ b/optimum/exporters/onnx/model_configs.py
@@ -183,8 +183,9 @@ def inputs(self) -> Dict[str, Dict[int, str]]:
         return {"input_ids": dynamic_axis, "attention_mask": dynamic_axis}
 
 
-class ModernBertOnnxConfig(DistilBertOnnxConfig):
-    pass
+# TODO: uncomment when transformers>=4.48.0
+# class ModernBertOnnxConfig(DistilBertOnnxConfig):
+#     pass
 
 
 class MPNetOnnxConfig(DistilBertOnnxConfig):
diff --git a/optimum/exporters/tasks.py b/optimum/exporters/tasks.py
index 5651537162a..2f7324fcf71 100644
--- a/optimum/exporters/tasks.py
+++ b/optimum/exporters/tasks.py
@@ -893,15 +893,15 @@ class TasksManager:
             "image-classification",
             onnx="MobileNetV2OnnxConfig",
         ),
-        "modernbert": supported_tasks_mapping(
-            "feature-extraction",
-            "fill-mask",
-            "text-classification",
-            "multiple-choice",
-            "token-classification",
-            "question-answering",
-            onnx="ModernBertOnnxConfig",
-        ),
+        # "modernbert": supported_tasks_mapping(
+        #     "feature-extraction",
+        #     "fill-mask",
+        #     "text-classification",
+        #     "multiple-choice",
+        #     "token-classification",
+        #     "question-answering",
+        #     onnx="ModernBertOnnxConfig",
+        # ),
         "mpnet": supported_tasks_mapping(
             "feature-extraction",
             "fill-mask",
@@ -1782,6 +1782,7 @@ def _infer_task_from_model_name_or_path(
         revision: Optional[str] = None,
         cache_dir: str = HUGGINGFACE_HUB_CACHE,
         token: Optional[Union[bool, str]] = None,
+        library_name: Optional[str] = None,
     ) -> str:
         inferred_task_name = None
 
@@ -1803,13 +1804,14 @@ def _infer_task_from_model_name_or_path(
                 raise RuntimeError(
                     f"Hugging Face Hub is not reachable and we cannot infer the task from a cached model. Make sure you are not offline, or otherwise please specify the `task` (or `--task` in command-line) argument ({', '.join(TasksManager.get_all_tasks())})."
                 )
-            library_name = cls.infer_library_from_model(
-                model_name_or_path,
-                subfolder=subfolder,
-                revision=revision,
-                cache_dir=cache_dir,
-                token=token,
-            )
+            if library_name is None:
+                library_name = cls.infer_library_from_model(
+                    model_name_or_path,
+                    subfolder=subfolder,
+                    revision=revision,
+                    cache_dir=cache_dir,
+                    token=token,
+                )
 
             if library_name == "timm":
                 inferred_task_name = "image-classification"
@@ -1828,6 +1830,8 @@ def _infer_task_from_model_name_or_path(
                                     break
                             if inferred_task_name is not None:
                                 break
+            elif library_name == "sentence_transformers":
+                inferred_task_name = "feature-extraction"
             elif library_name == "transformers":
                 pipeline_tag = model_info.pipeline_tag
                 transformers_info = model_info.transformersInfo
@@ -1864,6 +1868,7 @@ def infer_task_from_model(
         revision: Optional[str] = None,
         cache_dir: str = HUGGINGFACE_HUB_CACHE,
         token: Optional[Union[bool, str]] = None,
+        library_name: Optional[str] = None,
     ) -> str:
         """
         Infers the task from the model repo, model instance, or model class.
@@ -1882,7 +1887,9 @@ def infer_task_from_model(
             token (`Optional[Union[bool,str]]`, defaults to `None`):
                 The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
                 when running `huggingface-cli login` (stored in `huggingface_hub.constants.HF_TOKEN_PATH`).
-
+            library_name (`Optional[str]`, defaults to `None`):
+                The library name of the model. Can be any of "transformers", "timm", "diffusers", "sentence_transformers". See `TasksManager.infer_library_from_model` for the priority should
+                none be provided.
         Returns:
             `str`: The task name automatically detected from the HF hub repo, model instance, or model class.
         """
@@ -1895,6 +1902,7 @@ def infer_task_from_model(
                 revision=revision,
                 cache_dir=cache_dir,
                 token=token,
+                library_name=library_name,
             )
         elif type(model) == type:
             inferred_task_name = cls._infer_task_from_model_or_model_class(model_class=model)
@@ -2170,6 +2178,9 @@ def get_model_from_task(
                 none be provided.
             model_kwargs (`Dict[str, Any]`, *optional*):
                 Keyword arguments to pass to the model `.from_pretrained()` method.
+            library_name (`Optional[str]`, defaults to `None`):
+                The library name of the model. Can be any of "transformers", "timm", "diffusers", "sentence_transformers". See `TasksManager.infer_library_from_model` for the priority should
+                none be provided.
 
         Returns:
             The instance of the model.
@@ -2189,7 +2200,12 @@ def get_model_from_task(
         original_task = task
         if task == "auto":
             task = TasksManager.infer_task_from_model(
-                model_name_or_path, subfolder=subfolder, revision=revision, cache_dir=cache_dir, token=token
+                model_name_or_path,
+                subfolder=subfolder,
+                revision=revision,
+                cache_dir=cache_dir,
+                token=token,
+                library_name=library_name,
             )
 
         model_type = None
diff --git a/optimum/exporters/tflite/__main__.py b/optimum/exporters/tflite/__main__.py
index 0c4c7b994fa..4d8d4ee7b7e 100644
--- a/optimum/exporters/tflite/__main__.py
+++ b/optimum/exporters/tflite/__main__.py
@@ -46,7 +46,7 @@ def main():
     task = args.task
     if task == "auto":
         try:
-            task = TasksManager.infer_task_from_model(args.model)
+            task = TasksManager.infer_task_from_model(args.model, library_name="transformers")
         except KeyError as e:
             raise KeyError(
                 "The task could not be automatically inferred. Please provide the argument --task with the task "
@@ -58,7 +58,12 @@ def main():
             )
 
     model = TasksManager.get_model_from_task(
-        task, args.model, framework="tf", cache_dir=args.cache_dir, trust_remote_code=args.trust_remote_code
+        task,
+        args.model,
+        framework="tf",
+        cache_dir=args.cache_dir,
+        trust_remote_code=args.trust_remote_code,
+        library_name="transformers",
     )
 
     tflite_config_constructor = TasksManager.get_exporter_config_constructor(
diff --git a/optimum/exporters/tflite/convert.py b/optimum/exporters/tflite/convert.py
index c1a2010355a..fb0706cacd5 100644
--- a/optimum/exporters/tflite/convert.py
+++ b/optimum/exporters/tflite/convert.py
@@ -194,7 +194,7 @@ def prepare_converter_for_quantization(
         if task is None:
             from ...exporters import TasksManager
 
-            task = TasksManager.infer_task_from_model(model)
+            task = TasksManager.infer_task_from_model(model, library_name="transformers")
 
         preprocessor_kwargs = {}
         if isinstance(preprocessor, PreTrainedTokenizerBase):
diff --git a/setup.py b/setup.py
index bc737f2d09d..0474be4ac7c 100644
--- a/setup.py
+++ b/setup.py
@@ -31,7 +31,6 @@
     "Pillow",
     "sacremoses",
     "torchvision",
-    "diffusers>=0.17.0,<0.32.0",
     "torchaudio",
     "einops",
     "timm",
@@ -93,7 +92,7 @@
         "torch-ort",
         "onnxruntime-training",
     ],
-    "diffusers": ["diffusers<0.32.0"],
+    "diffusers": ["diffusers"],
     "intel": "optimum-intel>=1.18.0",
     "openvino": "optimum-intel[openvino]>=1.18.0",
     "nncf": "optimum-intel[nncf]>=1.18.0",
diff --git a/tests/exporters/exporters_utils.py b/tests/exporters/exporters_utils.py
index ee31397fd8c..a36ef6afa08 100644
--- a/tests/exporters/exporters_utils.py
+++ b/tests/exporters/exporters_utils.py
@@ -125,7 +125,7 @@
     "mobilenet-v2": "hf-internal-testing/tiny-random-MobileNetV2Model",
     "mobilenet-v1": "google/mobilenet_v1_0.75_192",
     "mobilevit": "hf-internal-testing/tiny-random-mobilevit",
-    "modernbert": "hf-internal-testing/tiny-random-ModernBertForMaskedLM",
+    # "modernbert": "hf-internal-testing/tiny-random-ModernBertForMaskedLM",
     "mpnet": "hf-internal-testing/tiny-random-MPNetModel",
     "mpt": "hf-internal-testing/tiny-random-MptForCausalLM",
     "mt5": "lewtun/tiny-random-mt5",
@@ -269,7 +269,7 @@
     # "mobilenet_v1": "google/mobilenet_v1_0.75_192",
     # "mobilenet_v2": "google/mobilenet_v2_0.35_96",
     "mobilevit": "apple/mobilevit-small",
-    "modernbert": "answerdotai/ModernBERT-base",
+    # "modernbert": "answerdotai/ModernBERT-base",
     "mpt": "mosaicml/mpt-7b",
     "mt5": "lewtun/tiny-random-mt5",  # Not using google/mt5-small because it takes too much time for testing.
     "musicgen": "facebook/musicgen-small",
diff --git a/tests/onnxruntime/test_diffusion.py b/tests/onnxruntime/test_diffusion.py
index a2df69077e1..5ff25092648 100644
--- a/tests/onnxruntime/test_diffusion.py
+++ b/tests/onnxruntime/test_diffusion.py
@@ -54,6 +54,7 @@ def _generate_prompts(batch_size=1):
         "guidance_scale": 7.5,
         "output_type": "np",
     }
+
     return inputs
 
 
@@ -105,8 +106,7 @@ class ORTPipelineForText2ImageTest(ORTModelTestMixin):
     def generate_inputs(self, height=128, width=128, batch_size=1):
         inputs = _generate_prompts(batch_size=batch_size)
 
-        inputs["height"] = height
-        inputs["width"] = width
+        inputs["height"], inputs["width"] = height, width
 
         return inputs
 
@@ -224,17 +224,19 @@ def test_shape(self, model_arch: str):
             elif output_type == "pt":
                 self.assertEqual(outputs.shape, (batch_size, 3, height, width))
             else:
-                expected_height = height // pipeline.vae_scale_factor
-                expected_width = width // pipeline.vae_scale_factor
-
                 if model_arch == "flux":
+                    expected_height = height // (pipeline.vae_scale_factor * 2)
+                    expected_width = width // (pipeline.vae_scale_factor * 2)
                     channels = pipeline.transformer.config.in_channels
                     expected_shape = (batch_size, expected_height * expected_width, channels)
-                elif model_arch == "stable-diffusion-3":
-                    out_channels = pipeline.transformer.config.out_channels
-                    expected_shape = (batch_size, out_channels, expected_height, expected_width)
                 else:
-                    out_channels = pipeline.unet.config.out_channels
+                    expected_height = height // pipeline.vae_scale_factor
+                    expected_width = width // pipeline.vae_scale_factor
+                    out_channels = (
+                        pipeline.unet.config.out_channels
+                        if getattr(pipeline, "unet", None) is not None
+                        else pipeline.transformer.config.out_channels
+                    )
                     expected_shape = (batch_size, out_channels, expected_height, expected_width)
 
                 self.assertEqual(outputs.shape, expected_shape)
@@ -363,6 +365,7 @@ def generate_inputs(self, height=128, width=128, batch_size=1, channel=3, input_
             height=height, width=width, batch_size=batch_size, channel=channel, input_type=input_type
         )
 
+        inputs["height"], inputs["width"] = height, width
         inputs["strength"] = 0.75
 
         return inputs
@@ -602,9 +605,8 @@ def generate_inputs(self, height=128, width=128, batch_size=1, channel=3, input_
             height=height, width=width, batch_size=batch_size, channel=1, input_type=input_type
         )
 
+        inputs["height"], inputs["width"] = height, width
         inputs["strength"] = 0.75
-        inputs["height"] = height
-        inputs["width"] = width
 
         return inputs