Fixed flaky CI tests by replacing httpbin with a simple http server

CHANGELOG.md

@@ -10,6 +10,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added MacOS and Windows CI workflows ([#390](https://github.com/opensearch-project/opensearch-py/pull/390))
 ### Changed
 - Upgrading pytest-asyncio to latest version - 0.21.0 ([#339](https://github.com/opensearch-project/opensearch-py/pull/339))
+- Fixed flaky CI tests by replacing httpbin with a simple http_server ([#395](https://github.com/opensearch-project/opensearch-py/pull/395))
 ### Deprecated
 ### Removed
 ### Fixed

test_opensearchpy/TestHttpServer.py

@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+
+import json
+import threading
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+
+class TestHTTPRequestHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        self.send_response(200)
+        headers = self.headers
+        self.send_header("Content-type", "application/json")
+        self.end_headers()
+
+        Headers = {}
+        for header, value in headers.items():
+            capitalized_header = "-".join([word.title() for word in header.split("-")])
+            Headers.update({capitalized_header: value})
+        if "Connection" in Headers:
+            Headers.pop("Connection")
+
+        data = {"method": "GET", "headers": Headers}
+        self.wfile.write(json.dumps(data).encode("utf-8"))
+
+
+class TestHTTPServer(HTTPServer):
+    def __init__(self, host="localhost", port=8080):
+        super().__init__((host, port), TestHTTPRequestHandler)
+        self._server_thread = None
+
+    def start(self):
+        if self._server_thread is not None:
+            return
+
+        self._server_thread = threading.Thread(target=self.serve_forever)
+        self._server_thread.start()
+
+    def stop(self):
+        if self._server_thread is None:
+            return
+        self.socket.close()
+        self.shutdown()
+        self._server_thread.join()
+        self._server_thread = None

test_opensearchpy/test_async/test_connection.py

@@ -43,6 +43,7 @@
 from opensearchpy.compat import reraise_exceptions
 from opensearchpy.connection import Connection, async_connections
 from opensearchpy.exceptions import ConnectionError, TransportError
+from test_opensearchpy.TestHttpServer import TestHTTPServer
 
 pytestmark = pytest.mark.asyncio
 
@@ -319,72 +320,80 @@ async def test_json_errors_are_parsed(self):
             await con.close()
 
 
-class TestConnectionHttpbin:
+class TestConnectionHttpServer:
     """Tests the HTTP connection implementations against a live server E2E"""
 
-    async def httpbin_anything(self, conn, **kwargs):
-        status, headers, data = await conn.perform_request("GET", "/anything", **kwargs)
+    @classmethod
+    def setup_class(cls):
+        # Start server
+        cls.server = TestHTTPServer(port=8081)
+        cls.server.start()
+
+    @classmethod
+    def teardown_class(cls):
+        # Stop server
+        cls.server.stop()
+
+    async def httpserver(self, conn, **kwargs):
+        status, headers, data = await conn.perform_request("GET", "/", **kwargs)
         data = json.loads(data)
-        data["headers"].pop(
-            "X-Amzn-Trace-Id", None
-        )  # Remove this header as it's put there by AWS.
         return (status, data)
 
     async def test_aiohttp_connection(self):
         # Defaults
-        conn = AIOHttpConnection("httpbin.org", port=443, use_ssl=True)
+        conn = AIOHttpConnection("localhost", port=8081, use_ssl=False)
         user_agent = conn._get_default_user_agent()
-        status, data = await self.httpbin_anything(conn)
+        status, data = await self.httpserver(conn)
         assert status == 200
         assert data["method"] == "GET"
         assert data["headers"] == {
             "Content-Type": "application/json",
-            "Host": "httpbin.org",
+            "Host": "localhost:8081",
             "User-Agent": user_agent,
         }
 
         # http_compress=False
         conn = AIOHttpConnection(
-            "httpbin.org", port=443, use_ssl=True, http_compress=False
+            "localhost", port=8081, use_ssl=False, http_compress=False
         )
-        status, data = await self.httpbin_anything(conn)
+        status, data = await self.httpserver(conn)
         assert status == 200
         assert data["method"] == "GET"
         assert data["headers"] == {
             "Content-Type": "application/json",
-            "Host": "httpbin.org",
+            "Host": "localhost:8081",
             "User-Agent": user_agent,
         }
 
         # http_compress=True
         conn = AIOHttpConnection(
-            "httpbin.org", port=443, use_ssl=True, http_compress=True
+            "localhost", port=8081, use_ssl=False, http_compress=True
         )
-        status, data = await self.httpbin_anything(conn)
+        status, data = await self.httpserver(conn)
         assert status == 200
         assert data["headers"] == {
             "Accept-Encoding": "gzip,deflate",
             "Content-Type": "application/json",
-            "Host": "httpbin.org",
+            "Host": "localhost:8081",
             "User-Agent": user_agent,
         }
 
         # Headers
         conn = AIOHttpConnection(
-            "httpbin.org",
-            port=443,
-            use_ssl=True,
+            "localhost",
+            port=8081,
+            use_ssl=False,
             http_compress=True,
             headers={"header1": "value1"},
         )
-        status, data = await self.httpbin_anything(
+        status, data = await self.httpserver(
             conn, headers={"header2": "value2", "header1": "override!"}
         )
         assert status == 200
         assert data["headers"] == {
             "Accept-Encoding": "gzip,deflate",
             "Content-Type": "application/json",
-            "Host": "httpbin.org",
+            "Host": "localhost:8081",
             "Header1": "override!",
             "Header2": "value2",
             "User-Agent": user_agent,

test_opensearchpy/test_connection.py

@@ -72,6 +72,9 @@
 from opensearchpy import OpenSearch, serializer
 from opensearchpy.connection import connections
 
+if sys.version_info > (3, 0):
+    from test_opensearchpy.TestHttpServer import TestHTTPServer
+
 
 def gzip_decompress(data):
     buf = gzip.GzipFile(fileobj=io.BytesIO(data), mode="rb")
@@ -857,76 +860,87 @@ def send_raise(*_, **__):
         assert str(e.value) == "Wasn't modified!"
 
 
-class TestConnectionHttpbin:
+@pytest.mark.skipif(
+    sys.version_info < (3, 0),
+    reason="http_server is only available from python 3.x",
+)
+class TestConnectionHttpServer:
     """Tests the HTTP connection implementations against a live server E2E"""
 
-    def httpbin_anything(self, conn, **kwargs):
-        status, headers, data = conn.perform_request("GET", "/anything", **kwargs)
+    @classmethod
+    def setup_class(cls):
+        # Start server
+        cls.server = TestHTTPServer(port=8080)
+        cls.server.start()
+
+    @classmethod
+    def teardown_class(cls):
+        # Stop server
+        cls.server.stop()
+
+    def httpserver(self, conn, **kwargs):
+        status, headers, data = conn.perform_request("GET", "/", **kwargs)
         data = json.loads(data)
-        data["headers"].pop(
-            "X-Amzn-Trace-Id", None
-        )  # Remove this header as it's put there by AWS.
         return (status, data)
 
     def test_urllib3_connection(self):
         # Defaults
-        # httpbin.org can be slow sometimes. Hence the timeout
-        conn = Urllib3HttpConnection("httpbin.org", port=443, use_ssl=True, timeout=60)
+        conn = Urllib3HttpConnection("localhost", port=8080, use_ssl=False, timeout=60)
         user_agent = conn._get_default_user_agent()
-        status, data = self.httpbin_anything(conn)
+        status, data = self.httpserver(conn)
         assert status == 200
         assert data["method"] == "GET"
         assert data["headers"] == {
             "Accept-Encoding": "identity",
             "Content-Type": "application/json",
-            "Host": "httpbin.org",
+            "Host": "localhost:8080",
             "User-Agent": user_agent,
         }
 
         # http_compress=False
         conn = Urllib3HttpConnection(
-            "httpbin.org", port=443, use_ssl=True, http_compress=False, timeout=60
+            "localhost", port=8080, use_ssl=False, http_compress=False, timeout=60
         )
-        status, data = self.httpbin_anything(conn)
+        status, data = self.httpserver(conn)
         assert status == 200
         assert data["method"] == "GET"
         assert data["headers"] == {
             "Accept-Encoding": "identity",
             "Content-Type": "application/json",
-            "Host": "httpbin.org",
+            "Host": "localhost:8080",
             "User-Agent": user_agent,
         }
 
         # http_compress=True
         conn = Urllib3HttpConnection(
-            "httpbin.org", port=443, use_ssl=True, http_compress=True, timeout=60
+            "localhost", port=8080, use_ssl=False, http_compress=True, timeout=60
         )
-        status, data = self.httpbin_anything(conn)
+        status, data = self.httpserver(conn)
         assert status == 200
         assert data["headers"] == {
             "Accept-Encoding": "gzip,deflate",
             "Content-Type": "application/json",
-            "Host": "httpbin.org",
+            "Host": "localhost:8080",
             "User-Agent": user_agent,
         }
 
         # Headers
         conn = Urllib3HttpConnection(
-            "httpbin.org",
-            port=443,
-            use_ssl=True,
+            "localhost",
+            port=8080,
+            use_ssl=False,
             http_compress=True,
             headers={"header1": "value1"},
             timeout=60,
         )
-        status, data = self.httpbin_anything(
+        status, data = self.httpserver(
             conn, headers={"header2": "value2", "header1": "override!"}
         )
         assert status == 200
         assert data["headers"] == {
             "Accept-Encoding": "gzip,deflate",
             "Content-Type": "application/json",
-            "Host": "httpbin.org",
+            "Host": "localhost:8080",
             "Header1": "override!",
             "Header2": "value2",
             "User-Agent": user_agent,
@@ -939,62 +953,62 @@ def test_urllib3_connection_error(self):
 
     def test_requests_connection(self):
         # Defaults
-        conn = RequestsHttpConnection("httpbin.org", port=443, use_ssl=True, timeout=60)
+        conn = RequestsHttpConnection("localhost", port=8080, use_ssl=False, timeout=60)
         user_agent = conn._get_default_user_agent()
-        status, data = self.httpbin_anything(conn)
+        status, data = self.httpserver(conn)
         assert status == 200
         assert data["method"] == "GET"
         assert data["headers"] == {
             "Accept-Encoding": "identity",
             "Content-Type": "application/json",
-            "Host": "httpbin.org",
+            "Host": "localhost:8080",
             "User-Agent": user_agent,
         }
 
         # http_compress=False
         conn = RequestsHttpConnection(
-            "httpbin.org", port=443, use_ssl=True, http_compress=False, timeout=60
+            "localhost", port=8080, use_ssl=False, http_compress=False, timeout=60
         )
-        status, data = self.httpbin_anything(conn)
+        status, data = self.httpserver(conn)
         assert status == 200
         assert data["method"] == "GET"
         assert data["headers"] == {
             "Accept-Encoding": "identity",
             "Content-Type": "application/json",
-            "Host": "httpbin.org",
+            "Host": "localhost:8080",
             "User-Agent": user_agent,
         }
 
         # http_compress=True
         conn = RequestsHttpConnection(
-            "httpbin.org", port=443, use_ssl=True, http_compress=True, timeout=60
+            "localhost", port=8080, use_ssl=False, http_compress=True, timeout=60
         )
-        status, data = self.httpbin_anything(conn)
+        status, data = self.httpserver(conn)
         assert status == 200
         assert data["headers"] == {
             "Accept-Encoding": "gzip,deflate",
             "Content-Type": "application/json",
-            "Host": "httpbin.org",
+            "Host": "localhost:8080",
             "User-Agent": user_agent,
         }
 
         # Headers
         conn = RequestsHttpConnection(
-            "httpbin.org",
-            port=443,
-            use_ssl=True,
+            "localhost",
+            port=8080,
+            use_ssl=False,
             http_compress=True,
             headers={"header1": "value1"},
             timeout=60,
         )
-        status, data = self.httpbin_anything(
+        status, data = self.httpserver(
             conn, headers={"header2": "value2", "header1": "override!"}
         )
         assert status == 200
         assert data["headers"] == {
             "Accept-Encoding": "gzip,deflate",
             "Content-Type": "application/json",
-            "Host": "httpbin.org",
+            "Host": "localhost:8080",
             "Header1": "override!",
             "Header2": "value2",
             "User-Agent": user_agent,