diff --git a/autogen/oai/openai_utils.py b/autogen/oai/openai_utils.py
index 70662668d84..42e95d0517c 100644
--- a/autogen/oai/openai_utils.py
+++ b/autogen/oai/openai_utils.py
@@ -436,6 +436,9 @@ def config_list_from_json(
 
     Returns:
         List[Dict]: A list of configuration dictionaries that match the filtering criteria specified in `filter_dict`.
+
+    Raises:
+        FileNotFoundError: if env_or_file is neither found as an environment variable nor a file
     """
     env_str = os.environ.get(env_or_file)
 
@@ -453,12 +456,8 @@ def config_list_from_json(
         # The environment variable does not exist.
         # So, `env_or_file` is a filename. We should use the file location.
         config_list_path = os.path.join(file_location, env_or_file)
-        try:
-            with open(config_list_path) as json_file:
-                config_list = json.load(json_file)
-        except FileNotFoundError:
-            logging.warning(f"The specified config_list file '{config_list_path}' does not exist.")
-            return []
+        with open(config_list_path) as json_file:
+            config_list = json.load(json_file)
     return filter_config(config_list, filter_dict)
 
 
diff --git a/test/agentchat/contrib/test_compressible_agent.py b/test/agentchat/contrib/test_compressible_agent.py
index 47a11ef165b..298fd01e9da 100644
--- a/test/agentchat/contrib/test_compressible_agent.py
+++ b/test/agentchat/contrib/test_compressible_agent.py
@@ -6,25 +6,27 @@
 from autogen.agentchat.contrib.compressible_agent import CompressibleAgent
 
 here = os.path.abspath(os.path.dirname(__file__))
-KEY_LOC = "notebook"
-OAI_CONFIG_LIST = "OAI_CONFIG_LIST"
 
-
-config_list = autogen.config_list_from_json(
-    OAI_CONFIG_LIST,
-    file_location=KEY_LOC,
-    filter_dict={
-        "model": ["gpt-3.5-turbo", "gpt-35-turbo", "gpt-3.5-turbo-16k", "gpt-35-turbo-16k"],
-    },
-)
+sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
+from test_assistant_agent import OAI_CONFIG_LIST, KEY_LOC  # noqa: E402
 
 try:
     import openai
+
 except ImportError:
     skip = True
 else:
     skip = False or skip_openai
 
+if not skip:
+    config_list = autogen.config_list_from_json(
+        OAI_CONFIG_LIST,
+        file_location=KEY_LOC,
+        filter_dict={
+            "model": ["gpt-3.5-turbo", "gpt-35-turbo", "gpt-3.5-turbo-16k", "gpt-35-turbo-16k"],
+        },
+    )
+
 
 @pytest.mark.skipif(
     sys.platform in ["darwin", "win32"] or skip,
diff --git a/test/agentchat/contrib/test_gpt_assistant.py b/test/agentchat/contrib/test_gpt_assistant.py
index fe92501c46b..0a7e92e8ccc 100644
--- a/test/agentchat/contrib/test_gpt_assistant.py
+++ b/test/agentchat/contrib/test_gpt_assistant.py
@@ -12,14 +12,16 @@
     import openai
     from autogen.agentchat.contrib.gpt_assistant_agent import GPTAssistantAgent
     from autogen.oai.openai_utils import retrieve_assistants_by_name
+
 except ImportError:
     skip = True
 else:
     skip = False or skip_openai
 
-config_list = autogen.config_list_from_json(
-    OAI_CONFIG_LIST, file_location=KEY_LOC, filter_dict={"api_type": ["openai"]}
-)
+if not skip:
+    config_list = autogen.config_list_from_json(
+        OAI_CONFIG_LIST, file_location=KEY_LOC, filter_dict={"api_type": ["openai"]}
+    )
 
 
 def ask_ossinsight(question):
diff --git a/test/oai/test_utils.py b/test/oai/test_utils.py
index 7de8dc3227e..ab8d2544f71 100644
--- a/test/oai/test_utils.py
+++ b/test/oai/test_utils.py
@@ -73,7 +73,6 @@ def test_config_list_from_json():
         json_data = json.loads(JSON_SAMPLE)
         tmp_file.write(JSON_SAMPLE)
         tmp_file.flush()
-
         config_list = autogen.config_list_from_json(tmp_file.name)
 
         assert len(config_list) == len(json_data)
@@ -115,6 +114,10 @@ def test_config_list_from_json():
 
         del os.environ["config_list_test"]
 
+    # Test that an error is thrown when the config list is missing
+    with pytest.raises(FileNotFoundError):
+        autogen.config_list_from_json("OAI_CONFIG_LIST.missing")
+
 
 def test_config_list_openai_aoai():
     # Testing the functionality for loading configurations for different API types