Skip to content

Commit

Permalink
add backup kusto for test result data (#5729)
Browse files Browse the repository at this point in the history
What is the motivation for this PR?
To support kusto cluster HA for sonic test data

How did you do it?
To add a backup Kusto cluster for test data upload.

How did you verify/test it?
Verified by manually run the pipeline and check the results in Kusto.
  • Loading branch information
StormLiangMS authored May 31, 2022
1 parent 7063b36 commit a923c95
Showing 1 changed file with 21 additions and 0 deletions.
21 changes: 21 additions & 0 deletions test_reporting/report_data_storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,25 @@ def __init__(self, db_name: str):
tenant_id)
self._ingestion_client = KustoIngestClient(kcsb)

"""
Kusto performance depends on the work load of cluster, to improve the high availability of test result data service
by hosting a backup cluster, which is optional.
"""
ingest_cluster = os.getenv("TEST_REPORT_INGEST_KUSTO_CLUSTER_BACKUP")
tenant_id = os.getenv("TEST_REPORT_AAD_TENANT_ID_BACKUP")
service_id = os.getenv("TEST_REPORT_AAD_CLIENT_ID_BACKUP")
service_key = os.getenv("TEST_REPORT_AAD_CLIENT_KEY_BACKUP")

if not ingest_cluster or not tenant_id or not service_id or not service_key:
print("Could not load backup Kusto Credentials from environment")
self._ingestion_client_backup = None
else:
kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(ingest_cluster,
service_id,
service_key,
tenant_id)
self._ingestion_client_backup = KustoIngestClient(kcsb)

def upload_report(self, report_json: Dict, external_tracking_id: str = "", report_guid: str = "") -> None:
"""Upload a report to the back-end data store.
Expand Down Expand Up @@ -238,3 +257,5 @@ def _ingest_data(self, table, data):
temp.write(json.dumps(data))
temp.seek(0)
self._ingestion_client.ingest_from_file(temp.name, ingestion_properties=props)
if self._ingestion_client_backup:
self._ingestion_client_backup.ingest_from_file(temp.name, ingestion_properties=props)

0 comments on commit a923c95

Please sign in to comment.