From 45bb1f267d5d2cfbbcaa9033cfe37f010eecc638 Mon Sep 17 00:00:00 2001 From: Evan Morris Date: Fri, 19 Apr 2024 14:56:56 -0400 Subject: [PATCH] implementing dependency injection & singleton patterns for metadata, fixing tests --- PLATER/services/app_trapi.py | 49 ++--- PLATER/services/util/metadata.py | 250 +++++++++++++----------- PLATER/tests/data/full_simple_spec.json | 17 ++ PLATER/tests/test_endpoint_factory.py | 52 ++++- 4 files changed, 219 insertions(+), 149 deletions(-) create mode 100644 PLATER/tests/data/full_simple_spec.json diff --git a/PLATER/services/app_trapi.py b/PLATER/services/app_trapi.py index cccc22c..33c0f8f 100644 --- a/PLATER/services/app_trapi.py +++ b/PLATER/services/app_trapi.py @@ -17,7 +17,7 @@ ) from PLATER.services.util.bl_helper import BLHelper, get_bl_helper from PLATER.services.util.graph_adapter import GraphInterface -from PLATER.services.util.metadata import GraphMetadata +from PLATER.services.util.metadata import get_graph_metadata, GraphMetadata from PLATER.services.util.overlay import Overlay from PLATER.services.util.question import Question from PLATER.services.config import config @@ -32,41 +32,23 @@ config.get('logging_format'), ) - -# read in and validate the meta kg json only once on startup, -# create an already-encoded object that is ready to be returned quickly -def get_meta_kg_response(graph_metadata_reader: GraphMetadata): - meta_kg_json = graph_metadata_reader.get_meta_kg() - try: - MetaKnowledgeGraph.parse_obj(meta_kg_json) - logger.info('Successfully validated meta kg') - return jsonable_encoder(meta_kg_json) - except ValidationError as e: - logger.error(f'Error validating meta kg: {e}') - return None - - -# process and store static objects ready to be returned by their respective endpoints -graph_metadata_reader = GraphMetadata() -GRAPH_METADATA = graph_metadata_reader.get_metadata() -META_KG_RESPONSE = get_meta_kg_response(graph_metadata_reader) -SRI_TEST_DATA = graph_metadata_reader.get_sri_testing_data() -FULL_SIMPLE_SPEC = graph_metadata_reader.get_full_simple_spec() - # get an example query for the /query endpoint, to be included in the open api spec -TRAPI_QUERY_EXAMPLE = graph_metadata_reader.get_example_qgraph() +# it would be nice to use Depends() for the graph metadata here, as it's used elsewhere, +# but because TRAPI_QUERY_EXAMPLE is included a function parameter, it's not possible +TRAPI_QUERY_EXAMPLE = get_graph_metadata().get_example_qgraph() -async def get_meta_knowledge_graph() -> ORJSONResponse: +async def get_meta_knowledge_graph(metadata_retriever: GraphMetadata = Depends(get_graph_metadata)) -> ORJSONResponse: """Handle /meta_knowledge_graph.""" - if META_KG_RESPONSE: + meta_kg_response = metadata_retriever.get_meta_kg_response() + if meta_kg_response: # we are intentionally returning a ORJSONResponse directly, - # we already validated with pydantic above and the content won't change + # we already validated with the pydantic model and the content won't change return ORJSONResponse(status_code=200, - content=META_KG_RESPONSE, + content=meta_kg_response, media_type="application/json") else: - # if META_KG_RESPONSE is None it means the meta kg did not validate + # if meta_kg_response is None it means the meta kg did not validate return ORJSONResponse(status_code=500, media_type="application/json", content={"description": "MetaKnowledgeGraph failed validation - " @@ -85,9 +67,9 @@ async def get_meta_knowledge_graph() -> ORJSONResponse: ) -async def get_sri_testing_data(): +async def get_sri_testing_data(metadata_retriever: GraphMetadata = Depends(get_graph_metadata)): """Handle /sri_testing_data.""" - return SRI_TEST_DATA + return metadata_retriever.get_sri_testing_data() APP.add_api_route( "/sri_testing_data", @@ -196,9 +178,9 @@ async def cypher( ) -async def metadata() -> Any: +async def metadata(metadata_retriever: GraphMetadata = Depends(get_graph_metadata)) -> Any: """Handle /metadata.""" - return GRAPH_METADATA + return metadata_retriever.get_metadata() APP.add_api_route( "/metadata", @@ -264,6 +246,7 @@ async def simple_spec( target: str = None, graph_interface: GraphInterface = Depends(get_graph_interface), bl_helper: BLHelper = Depends(get_bl_helper), + metadata_retriever: GraphMetadata = Depends(get_graph_metadata) ) -> SimpleSpecResponse: """Handle simple spec.""" source_id = source @@ -295,7 +278,7 @@ async def simple_spec( 'edge_type': x[1], }), minischema)) else: - return FULL_SIMPLE_SPEC + return metadata_retriever.get_full_simple_spec() APP.add_api_route( "/simple_spec", diff --git a/PLATER/services/util/metadata.py b/PLATER/services/util/metadata.py index c4ea7f9..1f13db6 100644 --- a/PLATER/services/util/metadata.py +++ b/PLATER/services/util/metadata.py @@ -1,125 +1,149 @@ import os import json + +from pydantic import ValidationError +from fastapi.encoders import jsonable_encoder + from PLATER.services.config import config +from PLATER.models.shared import MetaKnowledgeGraph +from PLATER.services.util.logutil import LoggingUtil + +logger = LoggingUtil.init_logging( + __name__, + config.get('logging_level'), + config.get('logging_format'), +) class GraphMetadata: + """ + Singleton class for retrieving metadata + """ - def __init__(self): - self.metadata = None - self.meta_kg = None - self.sri_testing_data = None - self.full_simple_spec = None - - def get_metadata(self): - if self.metadata is None: - self.retrieve_metadata() - return self.metadata - - def retrieve_metadata(self): - with open(os.path.join(os.path.dirname(__file__), '..', '..', 'metadata', 'metadata.json')) as f: - self.metadata = json.load(f) - - if not self.metadata: - with open(os.path.join(os.path.dirname(__file__), '..', '..', 'metadata', 'about.json')) as f: + class _GraphMetadata: + + def __init__(self): + self.metadata = None + self._retrieve_metadata() + self.meta_kg = None + self._retrieve_meta_kg() + self.sri_testing_data = None + self._retrieve_sri_test_data() + self.full_simple_spec = None + self._generate_full_simple_spec() + + def get_metadata(self): + return self.metadata + + def _retrieve_metadata(self): + with open(os.path.join(os.path.dirname(__file__), '..', '..', 'metadata', 'metadata.json')) as f: self.metadata = json.load(f) - def get_meta_kg(self): - if self.meta_kg is None: - self.retrieve_meta_kg() - return self.meta_kg - - def retrieve_meta_kg(self): - with open(os.path.join(os.path.dirname(__file__), '..', '..', 'metadata', 'meta_knowledge_graph.json')) as f: - self.meta_kg = json.load(f) - - try: - # We removed pydantic model conversion during a response for the meta kg for performance reasons, - # instead it is validated once on startup. This attempts to populate some optional but preferred - # fields that may not be coming from upstream tools. - for node_type, node_properties in self.meta_kg['nodes'].items(): - for attribute_info in node_properties['attributes']: - if 'attribute_source' not in attribute_info: - attribute_info['attribute_source'] = None - if 'constraint_use' not in attribute_info: - attribute_info['constraint_use'] = False - if 'constraint_name' not in attribute_info: - attribute_info['constraint_name'] = None - except KeyError as e: - # just move on if a key is missing here, it won't validate but don't crash the rest of the app - pass - - def get_sri_testing_data(self): - if self.sri_testing_data is None: - self.retrieve_sri_test_data() - return self.sri_testing_data - - def retrieve_sri_test_data(self): - with open(os.path.join(os.path.dirname(__file__), '..', '..', 'metadata', 'sri_testing_data.json')) as f: - self.sri_testing_data = json.load(f) - - # version is technically not part of the spec anymore - # but this ensures validation with the model until it's removed - if 'version' not in self.sri_testing_data: - self.sri_testing_data['version'] = config.get('BL_VERSION') - - def get_full_simple_spec(self): - if self.meta_kg is None: - self.retrieve_meta_kg() - if self.full_simple_spec is None: - self.generate_full_simple_spec() - return self.full_simple_spec - - def generate_full_simple_spec(self): - self.full_simple_spec = [] - for edge in self.meta_kg.get('edges', []): - self.full_simple_spec.append({ - "source_type": edge["subject"], - "target_type": edge["object"], - "edge_type": edge["predicate"] - }) - - def get_example_qgraph(self): - sri_test_data = self.get_sri_testing_data() - if not sri_test_data['edges']: - return {'error': 'Could not generate example without edges in sri_testing_data.'} - test_edge = sri_test_data['edges'][0] - example_trapi = { - "message": { - "query_graph": { - "nodes": { - "n0": { - "categories": [ - test_edge['subject_category'] - ], - "ids": [ - test_edge['subject_id'] - ] + if not self.metadata: + with open(os.path.join(os.path.dirname(__file__), '..', '..', 'metadata', 'about.json')) as f: + self.metadata = json.load(f) + + def get_meta_kg(self): + return self.meta_kg + + def get_meta_kg_response(self): + return self.meta_kg_response + + def _retrieve_meta_kg(self): + with open(os.path.join(os.path.dirname(__file__), '..', '..', 'metadata', 'meta_knowledge_graph.json')) as f: + self.meta_kg = json.load(f) + try: + # validate the meta kg with the pydantic model + MetaKnowledgeGraph.parse_obj(self.meta_kg) + logger.info('Successfully validated meta kg') + + # create an already-encoded object that is ready to be returned quickly + self.meta_kg_response = jsonable_encoder(self.meta_kg) + except ValidationError as e: + logger.error(f'Error validating meta kg: {e}') + self.meta_kg_response = None + + def get_sri_testing_data(self): + return self.sri_testing_data + + def _retrieve_sri_test_data(self): + with open(os.path.join(os.path.dirname(__file__), '..', '..', 'metadata', 'sri_testing_data.json')) as f: + self.sri_testing_data = json.load(f) + + # version is technically not part of the spec anymore + # but this ensures validation with the model until it's removed + if 'version' not in self.sri_testing_data: + self.sri_testing_data['version'] = config.get('BL_VERSION') + + def get_full_simple_spec(self): + return self.full_simple_spec + + def _generate_full_simple_spec(self): + self.full_simple_spec = [] + for edge in self.meta_kg.get('edges', []): + self.full_simple_spec.append({ + "source_type": edge["subject"], + "target_type": edge["object"], + "edge_type": edge["predicate"] + }) + + def get_example_qgraph(self): + sri_test_data = self.get_sri_testing_data() + if not sri_test_data['edges']: + return {'error': 'Could not generate example without edges in sri_testing_data.'} + test_edge = sri_test_data['edges'][0] + example_trapi = { + "message": { + "query_graph": { + "nodes": { + "n0": { + "categories": [ + test_edge['subject_category'] + ], + "ids": [ + test_edge['subject_id'] + ] + }, + "n1": { + "categories": [ + test_edge['object_category'] + ], + "ids": [ + test_edge['object_id'] + ] + } }, - "n1": { - "categories": [ - test_edge['object_category'] - ], - "ids": [ - test_edge['object_id'] - ] - } - }, - "edges": { - "e01": { - "subject": "n0", - "object": "n1", - "predicates": [ - test_edge['predicate'] - ] + "edges": { + "e01": { + "subject": "n0", + "object": "n1", + "predicates": [ + test_edge['predicate'] + ] + } } } - } - }, - "workflow": [ - { - "id": "lookup" - } - ] - } - return example_trapi + }, + "workflow": [ + { + "id": "lookup" + } + ] + } + return example_trapi + + # the following code implements a singleton pattern so that only one metadata object is ever created + instance = None + + def __init__(self): + # create a new instance if not already created. + if not GraphMetadata.instance: + GraphMetadata.instance = GraphMetadata._GraphMetadata() + + def __getattr__(self, item): + # proxy function calls to the inner object. + return getattr(self.instance, item) + + +def get_graph_metadata(): + return GraphMetadata() diff --git a/PLATER/tests/data/full_simple_spec.json b/PLATER/tests/data/full_simple_spec.json new file mode 100644 index 0000000..ffa4143 --- /dev/null +++ b/PLATER/tests/data/full_simple_spec.json @@ -0,0 +1,17 @@ +[ + { + "source_type": "biolink:ChemicalSubstance", + "target_type": "biolink:Gene", + "edge_type": "biolink:directly_interacts_with" + }, + { + "source_type": "biolink:ChemicalSubstance", + "target_type": "biolink:Disease", + "edge_type": "biolink:treats" + }, + { + "source_type": "biolink:Gene", + "target_type": "biolink:Disease", + "edge_type": "biolink:has_basis_in" + } +] \ No newline at end of file diff --git a/PLATER/tests/test_endpoint_factory.py b/PLATER/tests/test_endpoint_factory.py index 8a657c0..13c70f9 100644 --- a/PLATER/tests/test_endpoint_factory.py +++ b/PLATER/tests/test_endpoint_factory.py @@ -4,9 +4,10 @@ import json from functools import reduce from PLATER.services.util.graph_adapter import GraphInterface +from PLATER.services.util.metadata import GraphMetadata import os -from PLATER.services.app_trapi import APP, get_graph_interface +from PLATER.services.app_trapi import APP, get_graph_interface, get_graph_metadata class MockGraphInterface(GraphInterface): @@ -72,7 +73,30 @@ def graph_interface(): return _graph_interface() +class MockGraphMetadata(GraphMetadata): + def __init__(self, *args, **kwargs): + pass + + # TODO this isn't super useful for testing full_simple_spec processing, it skips the interesting part, + # it'd be better to generate it from a fake meta_kg + def get_full_simple_spec(self): + full_simple_spec_file = os.path.join(os.path.dirname(__file__), 'data', 'full_simple_spec.json') + with open(full_simple_spec_file) as s_file: + full_simple_spec = json.load(s_file) + return full_simple_spec + + +def _graph_metadata(): + return MockGraphMetadata() + + +@pytest.fixture() +def graph_metadata(): + return _graph_metadata() + + APP.dependency_overrides[get_graph_interface] = _graph_interface +APP.dependency_overrides[get_graph_metadata] = _graph_metadata @pytest.mark.asyncio @@ -112,7 +136,7 @@ async def test_cypher_response(graph_interface): # assert response.status_code == 200 # assert response.json() == graph_interface.get_schema() - +""" @pytest.mark.asyncio async def test_simple_one_hop_spec_response(graph_interface): # with out parameters it should return all the questions based on that @@ -133,9 +157,31 @@ async def test_simple_one_hop_spec_response(graph_interface): for item in specs: assert item['source_type'] in source_types assert item['target_type'] in target_types - + # test source param response = await ac.get("/simple_spec?source=SOME:CURIE") assert response.status_code == 200 response = await ac.get("/simple_spec?source=SOME:CURIE") assert response.status_code == 200 +""" + + +@pytest.mark.asyncio +async def test_simple_one_hop_spec_response(graph_interface, graph_metadata): + + async with AsyncClient(app=APP, base_url="http://test") as ac: + # test source param + response = await ac.get("/simple_spec?source=SOME:CURIE") + assert response.status_code == 200 + # test two params + response = await ac.get("/simple_spec?source=SOME:CURIE&target=SOME:CURIE") + assert response.status_code == 200 + # test empty params + response = await ac.get("/simple_spec") + assert response.status_code == 200 + specs = response.json() + assert len(specs) == 3 + for item in specs: + assert item['source_type'] + assert item['target_type'] +