AL-12102: rework entities as list (#467)

* AL-12102: allow setup subregion as region lvl 2 * AL-12102: update test * AL-12102: fix * AL-12102: delete es logs * AL-12102: rework entities as list
anodot · Oct 17, 2022 · 6aa118b · 6aa118b
1 parent c50fdf8
commit 6aa118b
Show file tree

Hide file tree

Showing 6 changed files with 120 additions and 63 deletions.
diff --git a/Makefile b/Makefile
@@ -327,4 +327,3 @@ show-all-logs:
 	echo "DC 2 logs"; docker logs dc2;
 	echo "Dummy logs"; docker logs dummy_destination;
 	echo "Agent logs"; docker exec -i anodot-agent cat /var/log/agent/agent.log;
-	echo "Elastic logs"; docker logs agent-es;
diff --git a/agent/src/agent/data_extractor/topology/topology.py b/agent/src/agent/data_extractor/topology/topology.py
@@ -29,11 +29,15 @@ def __init__(self, name: str, config: dict):
 
 
 def _create_topology_records(entities: list[Entity], data: dict) -> dict:
-    return {
-        entity_.name: [field.extract_fields(entity_.fields, row) for row in data]
-        for entity_ in entities
-    }
+    records = {}
+    for entity_ in entities:
+        new_records = [field.extract_fields(entity_.fields, row) for row in data]
+        if entity_.name in records:
+            records[entity_.name].extend(new_records)
+        else:
+            records[entity_.name] = new_records
+    return records
 
 
 def _create_entities(pipeline_: Pipeline) -> list[Entity]:
-    return [Entity(name.upper(), entity_config) for name, entity_config in pipeline_.config['entity'].items()]
+    return [Entity(entity['type'].upper(), entity['mapping']) for entity in pipeline_.config['entity']]
diff --git a/agent/src/agent/pipeline/config/validators/topology.py b/agent/src/agent/pipeline/config/validators/topology.py
@@ -10,16 +10,16 @@
 class TopologyValidator(Validator):
     def validate(self, pipeline_: Pipeline):
         errors = []
-        for entity_type, entity in pipeline_.config['entity'].items():
-            if entity_type.upper() not in topology.ENTITIES:
-                errors.append(f'Invalid entity type: {entity_type}')
+        for entity in pipeline_.config['entity']:
+            if entity['type'].upper() not in topology.ENTITIES:
+                errors.append(f'Invalid entity type: {entity["type"]}')
                 continue
-            with open(self._get_validation_file_path(entity_type)) as schema_file:
+            with open(self._get_validation_file_path(entity['type'])) as schema_file:
                 json_schema = json.load(schema_file)
                 try:
-                    jsonschema.validate(entity, json_schema)
+                    jsonschema.validate(entity['mapping'], json_schema)
                 except jsonschema.ValidationError as e:
-                    errors.append(f'Invalid entity config for {entity_type}: {e}')
+                    errors.append(f'Invalid entity config for {entity["type"]}: {e}')
         if errors:
             raise validators.ValidationException('\n'.join(errors))
 

diff --git a/agent/src/agent/pipeline/json_builder/json_schema_definitions/topology.json b/agent/src/agent/pipeline/json_builder/json_schema_definitions/topology.json
@@ -2,7 +2,7 @@
   "type": "object",
   "properties": {
     "interval": {"type": "integer"},
-    "entity": {"type": "object"},
+    "entity": {"type": "array"},
     "lookups": {"type": "object"}
   },
   "required": ["interval", "entity"]

diff --git a/agent/tests/input_files/topology/pipelines.json b/agent/tests/input_files/topology/pipelines.json
@@ -3,29 +3,58 @@
     "source": "topology_region_source",
     "pipeline_id": "topology_region",
     "interval": 300,
-    "entity": {
-      "region": {
-        "id": {
-          "type": "variable",
-          "value_path": "Full Name"
-        },
-        "name": {
-          "type": "variable",
-          "value_path": "Full Name"
-        },
-        "type": {
-          "type": "constant",
-          "value": "Province"
+    "entity": [
+      {
+        "type": "region",
+        "mapping": {
+          "id": {
+            "type": "variable",
+            "value_path": "Full Name"
+          },
+          "name": {
+            "type": "variable",
+            "value_path": "Full Name"
+          },
+          "type": {
+            "type": "constant",
+            "value": "Province"
+          }
+        }
+      },
+      {
+        "type" : "region",
+        "mapping" : {
+          "id": {
+            "type": "variable",
+            "value_path": [
+              "Full Name",
+              "Full Name"
+            ],
+            "concat_by": ""
+          },
+          "name": {
+            "type": "variable",
+            "value_path": "Full Name"
+          },
+          "type": {
+            "type": "constant",
+            "value": "Province"
+          },
+          "parentRegionId" : {
+            "type": "variable",
+            "value_path": "Full Name"
+          }
         }
       }
-    }
+    ]
   },
   {
     "source": "topology_site_source",
     "pipeline_id": "topology_site",
     "interval": 300,
-    "entity": {
-      "site": {
+    "entity": [{
+      "type": "site",
+      "mapping": {
         "id": {
           "type": "variable",
           "value_path": "attributes.name"
@@ -79,7 +108,7 @@
           "concat_by": ", "
         }
       }
-    },
+    }],
     "lookups": {
       "region": {
         "type": "file",

diff --git a/agent/tests/test_pipelines/expected_output/topology_region.json b/agent/tests/test_pipelines/expected_output/topology_region.json
@@ -1,35 +1,60 @@
 [
   {
-    "bulkSerNumber": 1,
-    "type": "REGION",
-    "rows": {
-      "Phnom Penh": "{\"type\": \"Province\", \"name\": \"Phnom Penh\", \"id\": \"Phnom Penh\"}",
-      "Steung Treng": "{\"type\": \"Province\", \"name\": \"Steung Treng\", \"id\": \"Steung Treng\"}",
-      "Preah Vihear": "{\"type\": \"Province\", \"name\": \"Preah Vihear\", \"id\": \"Preah Vihear\"}",
-      "Svay Rieng": "{\"type\": \"Province\", \"name\": \"Svay Rieng\", \"id\": \"Svay Rieng\"}",
-      "Siem Reap": "{\"type\": \"Province\", \"name\": \"Siem Reap\", \"id\": \"Siem Reap\"}",
-      "Kandal": "{\"type\": \"Province\", \"name\": \"Kandal\", \"id\": \"Kandal\"}",
-      "Pailin": "{\"type\": \"Province\", \"name\": \"Pailin\", \"id\": \"Pailin\"}",
-      "Sihanoukville": "{\"type\": \"Province\", \"name\": \"Sihanoukville\", \"id\": \"Sihanoukville\"}",
-      "Koh Kong": "{\"type\": \"Province\", \"name\": \"Koh Kong\", \"id\": \"Koh Kong\"}",
-      "Kampong Speu": "{\"type\": \"Province\", \"name\": \"Kampong Speu\", \"id\": \"Kampong Speu\"}",
-      "Mondulkiri": "{\"type\": \"Province\", \"name\": \"Mondulkiri\", \"id\": \"Mondulkiri\"}",
-      "Kampong Cham": "{\"type\": \"Province\", \"name\": \"Kampong Cham\", \"id\": \"Kampong Cham\"}",
-      "Kampong Thom": "{\"type\": \"Province\", \"name\": \"Kampong Thom\", \"id\": \"Kampong Thom\"}",
-      "Takeo": "{\"type\": \"Province\", \"name\": \"Takeo\", \"id\": \"Takeo\"}",
-      "Oddor Meanchey": "{\"type\": \"Province\", \"name\": \"Oddor Meanchey\", \"id\": \"Oddor Meanchey\"}",
-      "Battambang": "{\"type\": \"Province\", \"name\": \"Battambang\", \"id\": \"Battambang\"}",
-      "Pursat": "{\"type\": \"Province\", \"name\": \"Pursat\", \"id\": \"Pursat\"}",
-      "Tboung Khmum": "{\"type\": \"Province\", \"name\": \"Tboung Khmum\", \"id\": \"Tboung Khmum\"}",
-      "Kampong Chnnang": "{\"type\": \"Province\", \"name\": \"Kampong Chnnang\", \"id\": \"Kampong Chnnang\"}",
-      "Ratanakiri": "{\"type\": \"Province\", \"name\": \"Ratanakiri\", \"id\": \"Ratanakiri\"}",
-      "Prey Veng": "{\"type\": \"Province\", \"name\": \"Prey Veng\", \"id\": \"Prey Veng\"}",
-      "Kep": "{\"type\": \"Province\", \"name\": \"Kep\", \"id\": \"Kep\"}",
-      "Banteay Meanchey": "{\"type\": \"Province\", \"name\": \"Banteay Meanchey\", \"id\": \"Banteay Meanchey\"}",
-      "Kratie": "{\"type\": \"Province\", \"name\": \"Kratie\", \"id\": \"Kratie\"}",
-      "Kampot": "{\"type\": \"Province\", \"name\": \"Kampot\", \"id\": \"Kampot\"}"
-    },
-    "numberOfRows": 25,
-    "rollupId": "123456789"
-  }
-]
+        "bulkSerNumber": 1,
+        "type": "REGION",
+        "rows": {
+            "Kampong ThomKampong Thom": "{\"type\": \"Province\", \"parentRegionId\": \"Kampong Thom\", \"name\": \"Kampong Thom\", \"id\": \"Kampong ThomKampong Thom\"}",
+            "Steung Treng": "{\"type\": \"Province\", \"name\": \"Steung Treng\", \"id\": \"Steung Treng\"}",
+            "Preah Vihear": "{\"type\": \"Province\", \"name\": \"Preah Vihear\", \"id\": \"Preah Vihear\"}",
+            "PailinPailin": "{\"type\": \"Province\", \"parentRegionId\": \"Pailin\", \"name\": \"Pailin\", \"id\": \"PailinPailin\"}",
+            "Svay Rieng": "{\"type\": \"Province\", \"name\": \"Svay Rieng\", \"id\": \"Svay Rieng\"}",
+            "Phnom PenhPhnom Penh": "{\"type\": \"Province\", \"parentRegionId\": \"Phnom Penh\", \"name\": \"Phnom Penh\", \"id\": \"Phnom PenhPhnom Penh\"}",
+            "Pailin": "{\"type\": \"Province\", \"name\": \"Pailin\", \"id\": \"Pailin\"}",
+            "Sihanoukville": "{\"type\": \"Province\", \"name\": \"Sihanoukville\", \"id\": \"Sihanoukville\"}",
+            "Kampong Speu": "{\"type\": \"Province\", \"name\": \"Kampong Speu\", \"id\": \"Kampong Speu\"}",
+            "BattambangBattambang": "{\"type\": \"Province\", \"parentRegionId\": \"Battambang\", \"name\": \"Battambang\", \"id\": \"BattambangBattambang\"}",
+            "KandalKandal": "{\"type\": \"Province\", \"parentRegionId\": \"Kandal\", \"name\": \"Kandal\", \"id\": \"KandalKandal\"}",
+            "Preah VihearPreah Vihear": "{\"type\": \"Province\", \"parentRegionId\": \"Preah Vihear\", \"name\": \"Preah Vihear\", \"id\": \"Preah VihearPreah Vihear\"}",
+            "Steung TrengSteung Treng": "{\"type\": \"Province\", \"parentRegionId\": \"Steung Treng\", \"name\": \"Steung Treng\", \"id\": \"Steung TrengSteung Treng\"}",
+            "Mondulkiri": "{\"type\": \"Province\", \"name\": \"Mondulkiri\", \"id\": \"Mondulkiri\"}",
+            "KampotKampot": "{\"type\": \"Province\", \"parentRegionId\": \"Kampot\", \"name\": \"Kampot\", \"id\": \"KampotKampot\"}",
+            "Koh KongKoh Kong": "{\"type\": \"Province\", \"parentRegionId\": \"Koh Kong\", \"name\": \"Koh Kong\", \"id\": \"Koh KongKoh Kong\"}",
+            "MondulkiriMondulkiri": "{\"type\": \"Province\", \"parentRegionId\": \"Mondulkiri\", \"name\": \"Mondulkiri\", \"id\": \"MondulkiriMondulkiri\"}",
+            "Kampong ChamKampong Cham": "{\"type\": \"Province\", \"parentRegionId\": \"Kampong Cham\", \"name\": \"Kampong Cham\", \"id\": \"Kampong ChamKampong Cham\"}",
+            "Oddor MeancheyOddor Meanchey": "{\"type\": \"Province\", \"parentRegionId\": \"Oddor Meanchey\", \"name\": \"Oddor Meanchey\", \"id\": \"Oddor MeancheyOddor Meanchey\"}",
+            "Oddor Meanchey": "{\"type\": \"Province\", \"name\": \"Oddor Meanchey\", \"id\": \"Oddor Meanchey\"}",
+            "Battambang": "{\"type\": \"Province\", \"name\": \"Battambang\", \"id\": \"Battambang\"}",
+            "Tboung Khmum": "{\"type\": \"Province\", \"name\": \"Tboung Khmum\", \"id\": \"Tboung Khmum\"}",
+            "Kampong ChnnangKampong Chnnang": "{\"type\": \"Province\", \"parentRegionId\": \"Kampong Chnnang\", \"name\": \"Kampong Chnnang\", \"id\": \"Kampong ChnnangKampong Chnnang\"}",
+            "TakeoTakeo": "{\"type\": \"Province\", \"parentRegionId\": \"Takeo\", \"name\": \"Takeo\", \"id\": \"TakeoTakeo\"}",
+            "Ratanakiri": "{\"type\": \"Province\", \"name\": \"Ratanakiri\", \"id\": \"Ratanakiri\"}",
+            "Prey Veng": "{\"type\": \"Province\", \"name\": \"Prey Veng\", \"id\": \"Prey Veng\"}",
+            "Kep": "{\"type\": \"Province\", \"name\": \"Kep\", \"id\": \"Kep\"}",
+            "Banteay Meanchey": "{\"type\": \"Province\", \"name\": \"Banteay Meanchey\", \"id\": \"Banteay Meanchey\"}",
+            "PursatPursat": "{\"type\": \"Province\", \"parentRegionId\": \"Pursat\", \"name\": \"Pursat\", \"id\": \"PursatPursat\"}",
+            "Kampot": "{\"type\": \"Province\", \"name\": \"Kampot\", \"id\": \"Kampot\"}",
+            "Kampong SpeuKampong Speu": "{\"type\": \"Province\", \"parentRegionId\": \"Kampong Speu\", \"name\": \"Kampong Speu\", \"id\": \"Kampong SpeuKampong Speu\"}",
+            "Banteay MeancheyBanteay Meanchey": "{\"type\": \"Province\", \"parentRegionId\": \"Banteay Meanchey\", \"name\": \"Banteay Meanchey\", \"id\": \"Banteay MeancheyBanteay Meanchey\"}",
+            "Phnom Penh": "{\"type\": \"Province\", \"name\": \"Phnom Penh\", \"id\": \"Phnom Penh\"}",
+            "Prey VengPrey Veng": "{\"type\": \"Province\", \"parentRegionId\": \"Prey Veng\", \"name\": \"Prey Veng\", \"id\": \"Prey VengPrey Veng\"}",
+            "Siem ReapSiem Reap": "{\"type\": \"Province\", \"parentRegionId\": \"Siem Reap\", \"name\": \"Siem Reap\", \"id\": \"Siem ReapSiem Reap\"}",
+            "SihanoukvilleSihanoukville": "{\"type\": \"Province\", \"parentRegionId\": \"Sihanoukville\", \"name\": \"Sihanoukville\", \"id\": \"SihanoukvilleSihanoukville\"}",
+            "Siem Reap": "{\"type\": \"Province\", \"name\": \"Siem Reap\", \"id\": \"Siem Reap\"}",
+            "Kandal": "{\"type\": \"Province\", \"name\": \"Kandal\", \"id\": \"Kandal\"}",
+            "Koh Kong": "{\"type\": \"Province\", \"name\": \"Koh Kong\", \"id\": \"Koh Kong\"}",
+            "Kampong Cham": "{\"type\": \"Province\", \"name\": \"Kampong Cham\", \"id\": \"Kampong Cham\"}",
+            "Svay RiengSvay Rieng": "{\"type\": \"Province\", \"parentRegionId\": \"Svay Rieng\", \"name\": \"Svay Rieng\", \"id\": \"Svay RiengSvay Rieng\"}",
+            "Kampong Thom": "{\"type\": \"Province\", \"name\": \"Kampong Thom\", \"id\": \"Kampong Thom\"}",
+            "Takeo": "{\"type\": \"Province\", \"name\": \"Takeo\", \"id\": \"Takeo\"}",
+            "Pursat": "{\"type\": \"Province\", \"name\": \"Pursat\", \"id\": \"Pursat\"}",
+            "Kampong Chnnang": "{\"type\": \"Province\", \"name\": \"Kampong Chnnang\", \"id\": \"Kampong Chnnang\"}",
+            "KratieKratie": "{\"type\": \"Province\", \"parentRegionId\": \"Kratie\", \"name\": \"Kratie\", \"id\": \"KratieKratie\"}",
+            "RatanakiriRatanakiri": "{\"type\": \"Province\", \"parentRegionId\": \"Ratanakiri\", \"name\": \"Ratanakiri\", \"id\": \"RatanakiriRatanakiri\"}",
+            "Kratie": "{\"type\": \"Province\", \"name\": \"Kratie\", \"id\": \"Kratie\"}",
+            "Tboung KhmumTboung Khmum": "{\"type\": \"Province\", \"parentRegionId\": \"Tboung Khmum\", \"name\": \"Tboung Khmum\", \"id\": \"Tboung KhmumTboung Khmum\"}",
+            "KepKep": "{\"type\": \"Province\", \"parentRegionId\": \"Kep\", \"name\": \"Kep\", \"id\": \"KepKep\"}"
+        },
+        "numberOfRows": 50,
+        "rollupId": "123456789"
+    }
+]