diff --git a/test/test_core_sparqlstore.py b/test/test_core_sparqlstore.py deleted file mode 100644 index 622e4a2461..0000000000 --- a/test/test_core_sparqlstore.py +++ /dev/null @@ -1,26 +0,0 @@ -import unittest -from rdflib.graph import Graph - - -class TestSPARQLStoreGraphCore(unittest.TestCase): - - store_name = "SPARQLStore" - path = "http://dbpedia.org/sparql" - storetest = True - create = False - - def setUp(self): - self.graph = Graph(store="SPARQLStore") - self.graph.open(self.path, create=self.create) - ns = list(self.graph.namespaces()) - assert len(ns) > 0, ns - - def tearDown(self): - self.graph.close() - - def test(self): - print("Done") - - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_sparqlstore.py b/test/test_sparqlstore.py index 25a0ec1c01..7859a3755c 100644 --- a/test/test_sparqlstore.py +++ b/test/test_sparqlstore.py @@ -1,33 +1,40 @@ from rdflib import Graph, URIRef, Literal -from urllib.request import urlopen import unittest -from nose import SkipTest from http.server import BaseHTTPRequestHandler, HTTPServer import socket from threading import Thread from unittest.mock import patch from rdflib.namespace import RDF, XSD, XMLNS, FOAF, RDFS from rdflib.plugins.stores.sparqlstore import SPARQLConnector +from typing import ClassVar from . import helper -from .testutils import MockHTTPResponse, SimpleHTTPMock, ctx_http_server +from .testutils import ( + MockHTTPResponse, + ServedSimpleHTTPMock, +) -try: - assert len(urlopen("http://dbpedia.org/sparql").read()) > 0 -except: - raise SkipTest("No HTTP connection.") +class SPARQLStoreFakeDBPediaTestCase(unittest.TestCase): + store_name = "SPARQLStore" + path: ClassVar[str] + httpmock: ClassVar[ServedSimpleHTTPMock] + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.httpmock = ServedSimpleHTTPMock() + cls.path = f"{cls.httpmock.url}/sparql" -class SPARQLStoreDBPediaTestCase(unittest.TestCase): - store_name = "SPARQLStore" - path = "http://dbpedia.org/sparql" - storetest = True - create = False + @classmethod + def tearDownClass(cls) -> None: + super().tearDownClass() + cls.httpmock.stop() def setUp(self): + self.httpmock.reset() self.graph = Graph(store="SPARQLStore") - self.graph.open(self.path, create=self.create) + self.graph.open(self.path, create=True) ns = list(self.graph.namespaces()) assert len(ns) > 0, ns @@ -37,11 +44,29 @@ def tearDown(self): def test_Query(self): query = "select distinct ?Concept where {[] a ?Concept} LIMIT 1" _query = SPARQLConnector.query + self.httpmock.do_get_responses.append( + MockHTTPResponse( + 200, + "OK", + b"""\ + + + + + + + http://www.w3.org/2000/01/rdf-schema#Datatype + + +""", + {"Content-Type": ["application/sparql-results+xml; charset=UTF-8"]}, + ) + ) with patch("rdflib.plugins.stores.sparqlstore.SPARQLConnector.query") as mock: SPARQLConnector.query.side_effect = lambda *args, **kwargs: _query( self.graph.store, *args, **kwargs ) - res = helper.query_with_retry(self.graph, query, initNs={}) + res = self.graph.query(query, initNs={}) count = 0 for i in res: count += 1 @@ -56,24 +81,97 @@ def unpacker(query, default_graph=None, named_graph=None): (mquery, _, _) = unpacker(*args, *kwargs) for _, uri in self.graph.namespaces(): assert mquery.count(f"<{uri}>") == 1 + self.assertEqual(self.httpmock.do_get_mock.call_count, 1) + req = self.httpmock.do_get_requests.pop(0) + self.assertRegex(req.path, r"^/sparql") + self.assertIn(query, req.path_query["query"][0]) def test_initNs(self): query = """\ SELECT ?label WHERE { ?s a xyzzy:Concept ; xyzzy:prefLabel ?label . } LIMIT 10 """ - res = helper.query_with_retry(self.graph, + self.httpmock.do_get_responses.append( + MockHTTPResponse( + 200, + "OK", + """\ + + + + + + + 189 + + + 1899–1900 Scottish Football League + + + 1899–1900 United States collegiate men's ice hockey season + + + 1899–1900 Western Conference men's basketball season + + + 1899–1900 collegiate men's basketball independents season in the United States + + + 1899–1900 domestic association football cups + + + 1899–1900 domestic association football leagues + + + 1899–1900 in American ice hockey by league + + + 1899–1900 in American ice hockey by team + + + 1899–1900 in Belgian football + + +""".encode( + "utf8" + ), + {"Content-Type": ["application/sparql-results+xml; charset=UTF-8"]}, + ) + ) + res = self.graph.query( query, initNs={"xyzzy": "http://www.w3.org/2004/02/skos/core#"} ) for i in res: assert type(i[0]) == Literal, i[0].n3() + self.assertEqual(self.httpmock.do_get_mock.call_count, 1) + req = self.httpmock.do_get_requests.pop(0) + self.assertRegex(req.path, r"^/sparql") + self.assertIn(query, req.path_query["query"][0]) + def test_noinitNs(self): query = """\ SELECT ?label WHERE { ?s a xyzzy:Concept ; xyzzy:prefLabel ?label . } LIMIT 10 """ - self.assertRaises(ValueError, self.graph.query, query) + self.httpmock.do_get_responses.append( + MockHTTPResponse( + 400, + "Bad Request", + b"""\ +Virtuoso 37000 Error SP030: SPARQL compiler, line 1: Undefined namespace prefix in prefix:localpart notation at 'xyzzy:Concept' before ';' + +SPARQL query: +SELECT ?label WHERE { ?s a xyzzy:Concept ; xyzzy:prefLabel ?label . } LIMIT 10""", + {"Content-Type": ["text/plain"]}, + ) + ) + with self.assertRaises(ValueError): + self.graph.query(query) + self.assertEqual(self.httpmock.do_get_mock.call_count, 1) + req = self.httpmock.do_get_requests.pop(0) + self.assertRegex(req.path, r"^/sparql") + self.assertIn(query, req.path_query["query"][0]) def test_query_with_added_prolog(self): prologue = """\ @@ -83,9 +181,60 @@ def test_query_with_added_prolog(self): SELECT ?label WHERE { ?s a xyzzy:Concept ; xyzzy:prefLabel ?label . } LIMIT 10 """ + self.httpmock.do_get_responses.append( + MockHTTPResponse( + 200, + "OK", + """\ + + + + + + + 189 + + + 1899–1900 Scottish Football League + + + 1899–1900 United States collegiate men's ice hockey season + + + 1899–1900 Western Conference men's basketball season + + + 1899–1900 collegiate men's basketball independents season in the United States + + + 1899–1900 domestic association football cups + + + 1899–1900 domestic association football leagues + + + 1899–1900 in American ice hockey by league + + + 1899–1900 in American ice hockey by team + + + 1899–1900 in Belgian football + + +""".encode( + "utf8" + ), + {"Content-Type": ["application/sparql-results+xml; charset=UTF-8"]}, + ) + ) res = helper.query_with_retry(self.graph, prologue + query) for i in res: assert type(i[0]) == Literal, i[0].n3() + self.assertEqual(self.httpmock.do_get_mock.call_count, 1) + req = self.httpmock.do_get_requests.pop(0) + self.assertRegex(req.path, r"^/sparql") + self.assertIn(query, req.path_query["query"][0]) def test_query_with_added_rdf_prolog(self): prologue = """\ @@ -96,9 +245,60 @@ def test_query_with_added_rdf_prolog(self): SELECT ?label WHERE { ?s a xyzzy:Concept ; xyzzy:prefLabel ?label . } LIMIT 10 """ + self.httpmock.do_get_responses.append( + MockHTTPResponse( + 200, + "OK", + """\ + + + + + + + 189 + + + 1899–1900 Scottish Football League + + + 1899–1900 United States collegiate men's ice hockey season + + + 1899–1900 Western Conference men's basketball season + + + 1899–1900 collegiate men's basketball independents season in the United States + + + 1899–1900 domestic association football cups + + + 1899–1900 domestic association football leagues + + + 1899–1900 in American ice hockey by league + + + 1899–1900 in American ice hockey by team + + + 1899–1900 in Belgian football + + +""".encode( + "utf8" + ), + {"Content-Type": ["application/sparql-results+xml; charset=UTF-8"]}, + ) + ) res = helper.query_with_retry(self.graph, prologue + query) for i in res: assert type(i[0]) == Literal, i[0].n3() + self.assertEqual(self.httpmock.do_get_mock.call_count, 1) + req = self.httpmock.do_get_requests.pop(0) + self.assertRegex(req.path, r"^/sparql") + self.assertIn(query, req.path_query["query"][0]) def test_counting_graph_and_store_queries(self): query = """ @@ -111,21 +311,62 @@ def test_counting_graph_and_store_queries(self): g = Graph("SPARQLStore") g.open(self.path) count = 0 - result = helper.query_with_retry(g, query) + response = MockHTTPResponse( + 200, + "OK", + """\ + + + + + + + http://www.openlinksw.com/virtrdf-data-formats#default-iid + + + http://www.openlinksw.com/virtrdf-data-formats#default-iid-nullable + + + http://www.openlinksw.com/virtrdf-data-formats#default-iid-blank + + + http://www.openlinksw.com/virtrdf-data-formats#default-iid-blank-nullable + + + http://www.openlinksw.com/virtrdf-data-formats#default-iid-nonblank + + + """.encode( + "utf8" + ), + {"Content-Type": ["application/sparql-results+xml; charset=UTF-8"]}, + ) + + self.httpmock.do_get_responses.append(response) + + result = g.query(query) for _ in result: count += 1 - assert count == 5, "Graph(\"SPARQLStore\") didn't return 5 records" + assert count == 5, 'Graph("SPARQLStore") didn\'t return 5 records' from rdflib.plugins.stores.sparqlstore import SPARQLStore + st = SPARQLStore(query_endpoint=self.path) count = 0 - result = helper.query_with_retry(st, query) + self.httpmock.do_get_responses.append(response) + result = st.query(query) for _ in result: count += 1 assert count == 5, "SPARQLStore() didn't return 5 records" + self.assertEqual(self.httpmock.do_get_mock.call_count, 2) + for _ in range(2): + req = self.httpmock.do_get_requests.pop(0) + self.assertRegex(req.path, r"^/sparql") + self.assertIn(query, req.path_query["query"][0]) + class SPARQLStoreUpdateTestCase(unittest.TestCase): def setUp(self): @@ -218,7 +459,6 @@ def do_GET(self): class SPARQLMockTests(unittest.TestCase): def test_query(self): - httpmock = SimpleHTTPMock() triples = { (RDFS.Resource, RDF.type, RDFS.Class), (RDFS.Resource, RDFS.isDefinedBy, URIRef(RDFS)), @@ -230,7 +470,6 @@ def test_query(self): response = MockHTTPResponse( 200, "OK", response_body, {"Content-Type": ["text/csv; charset=utf-8"]} ) - httpmock.do_get_responses.append(response) graph = Graph(store="SPARQLStore", identifier="http://example.com") graph.bind("xsd", XSD) @@ -240,9 +479,9 @@ def test_query(self): assert len(list(graph.namespaces())) >= 4 - with ctx_http_server(httpmock.Handler) as server: - (host, port) = server.server_address - url = f"http://{host}:{port}/query" + with ServedSimpleHTTPMock() as httpmock: + httpmock.do_get_responses.append(response) + url = f"{httpmock.url}/query" graph.open(url) query_result = graph.query("SELECT ?s ?p ?o WHERE { ?s ?p ?o }") diff --git a/test/testutils.py b/test/testutils.py index d693c1a3e7..f95619d28f 100644 --- a/test/testutils.py +++ b/test/testutils.py @@ -1,11 +1,14 @@ import sys +from types import TracebackType import isodate import datetime import random -from contextlib import contextmanager +from contextlib import AbstractContextManager, contextmanager from typing import ( List, + Optional, + TYPE_CHECKING, Type, Iterator, Set, @@ -23,10 +26,16 @@ import email.message from nose import SkipTest from .earl import add_test, report +import unittest from rdflib import BNode, Graph, ConjunctiveGraph from rdflib.term import Node from unittest.mock import MagicMock, Mock +from urllib.error import HTTPError +from urllib.request import urlopen + +if TYPE_CHECKING: + import typing_extensions as te # TODO: make an introspective version (like this one) of @@ -177,6 +186,7 @@ def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: class MockHTTPRequests(NamedTuple): + method: str path: str parsed_path: ParseResult path_query: PathQueryT @@ -191,6 +201,40 @@ class MockHTTPResponse(NamedTuple): class SimpleHTTPMock: + """ + SimpleHTTPMock allows testing of code that relies on an HTTP server. + + NOTE: Currently only the GET method is supported. + + Objects of this class has a list of responses for each method (GET, POST, etc...) + and returns these responses for these methods in sequence. + + All request received are appended to a method specific list. + + Example usage: + >>> httpmock = SimpleHTTPMock() + >>> with ctx_http_server(httpmock.Handler) as server: + ... url = "http://{}:{}".format(*server.server_address) + ... # add a response the server should give: + ... httpmock.do_get_responses.append( + ... MockHTTPResponse(404, "Not Found", b"gone away", {}) + ... ) + ... + ... # send a request to get the first response + ... http_error: Optional[HTTPError] = None + ... try: + ... urlopen(f"{url}/bad/path") + ... except HTTPError as caught: + ... http_error = caught + ... + ... assert http_error is not None + ... assert http_error.code == 404 + ... + ... # get and validate request that the mock received + ... req = httpmock.do_get_requests.pop(0) + ... assert req.path == "/bad/path" + """ + # TODO: add additional methods (POST, PUT, ...) similar to get def __init__(self): self.do_get_requests: List[MockHTTPRequests] = [] @@ -205,7 +249,7 @@ def _do_GET(self): parsed_path = urlparse(self.path) path_query = parse_qs(parsed_path.query) request = MockHTTPRequests( - self.path, parsed_path, path_query, self.headers + "GET", self.path, parsed_path, path_query, self.headers ) self.http_mock.do_get_requests.append(request) @@ -222,6 +266,9 @@ def _do_GET(self): (do_GET, do_GET_mock) = make_spypair(_do_GET) + def log_message(self, format: str, *args: Any) -> None: + pass + self.Handler = Handler self.do_get_mock = Handler.do_GET_mock @@ -229,3 +276,132 @@ def reset(self): self.do_get_requests.clear() self.do_get_responses.clear() self.do_get_mock.reset_mock() + + +class SimpleHTTPMockTests(unittest.TestCase): + def test_example(self) -> None: + httpmock = SimpleHTTPMock() + with ctx_http_server(httpmock.Handler) as server: + url = "http://{}:{}".format(*server.server_address) + # add two responses the server should give: + httpmock.do_get_responses.append( + MockHTTPResponse(404, "Not Found", b"gone away", {}) + ) + httpmock.do_get_responses.append( + MockHTTPResponse(200, "OK", b"here it is", {}) + ) + + # send a request to get the first response + with self.assertRaises(HTTPError) as raised: + urlopen(f"{url}/bad/path") + assert raised.exception.code == 404 + + # get and validate request that the mock received + req = httpmock.do_get_requests.pop(0) + self.assertEqual(req.path, "/bad/path") + + # send a request to get the second response + resp = urlopen(f"{url}/") + self.assertEqual(resp.status, 200) + self.assertEqual(resp.read(), b"here it is") + + httpmock.do_get_responses.append( + MockHTTPResponse(404, "Not Found", b"gone away", {}) + ) + httpmock.do_get_responses.append( + MockHTTPResponse(200, "OK", b"here it is", {}) + ) + + +class ServedSimpleHTTPMock(SimpleHTTPMock, AbstractContextManager): + """ + ServedSimpleHTTPMock is a ServedSimpleHTTPMock with a HTTP server. + + Example usage: + >>> with ServedSimpleHTTPMock() as httpmock: + ... # add a response the server should give: + ... httpmock.do_get_responses.append( + ... MockHTTPResponse(404, "Not Found", b"gone away", {}) + ... ) + ... + ... # send a request to get the first response + ... http_error: Optional[HTTPError] = None + ... try: + ... urlopen(f"{httpmock.url}/bad/path") + ... except HTTPError as caught: + ... http_error = caught + ... + ... assert http_error is not None + ... assert http_error.code == 404 + ... + ... # get and validate request that the mock received + ... req = httpmock.do_get_requests.pop(0) + ... assert req.path == "/bad/path" + """ + + def __init__(self): + super().__init__() + host = get_random_ip() + self.server = HTTPServer((host, 0), self.Handler) + self.server_thread = Thread(target=self.server.serve_forever) + self.server_thread.daemon = True + self.server_thread.start() + + def stop(self) -> None: + self.server.shutdown() + self.server.socket.close() + self.server_thread.join() + + @property + def address_string(self) -> str: + (host, port) = self.server.server_address + return f"{host}:{port}" + + @property + def url(self) -> str: + return f"http://{self.address_string}" + + def __enter__(self) -> "ServedSimpleHTTPMock": + return self + + def __exit__( + self, + __exc_type: Optional[Type[BaseException]], + __exc_value: Optional[BaseException], + __traceback: Optional[TracebackType], + ) -> "te.Literal[False]": + self.stop() + return False + + +class ServedSimpleHTTPMockTests(unittest.TestCase): + def test_example(self) -> None: + with ServedSimpleHTTPMock() as httpmock: + # add two responses the server should give: + httpmock.do_get_responses.append( + MockHTTPResponse(404, "Not Found", b"gone away", {}) + ) + httpmock.do_get_responses.append( + MockHTTPResponse(200, "OK", b"here it is", {}) + ) + + # send a request to get the first response + with self.assertRaises(HTTPError) as raised: + urlopen(f"{httpmock.url}/bad/path") + assert raised.exception.code == 404 + + # get and validate request that the mock received + req = httpmock.do_get_requests.pop(0) + self.assertEqual(req.path, "/bad/path") + + # send a request to get the second response + resp = urlopen(f"{httpmock.url}/") + self.assertEqual(resp.status, 200) + self.assertEqual(resp.read(), b"here it is") + + httpmock.do_get_responses.append( + MockHTTPResponse(404, "Not Found", b"gone away", {}) + ) + httpmock.do_get_responses.append( + MockHTTPResponse(200, "OK", b"here it is", {}) + )