Skip to content

Commit

Permalink
Remove opensearch error taxonomy upload
Browse files Browse the repository at this point in the history
The error taxonomy has been moved to the analytics DB, so we no longer
need to upload it to opensearch. Hopefully this also has the secondary
effect of lowering the stress on the opensearch cluster and causing
job log upload failures.
  • Loading branch information
mvandenburgh committed Oct 31, 2024
1 parent 5883529 commit 9122589
Show file tree
Hide file tree
Showing 7 changed files with 129 additions and 257 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/custom_docker_builds.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ jobs:
- docker-image: ./images/cache-indexer
image-tags: ghcr.io/spack/cache-indexer:0.0.3
- docker-image: ./analytics
image-tags: ghcr.io/spack/django:0.3.21
image-tags: ghcr.io/spack/django:0.3.22
- docker-image: ./images/ci-prune-buildcache
image-tags: ghcr.io/spack/ci-prune-buildcache:0.0.4
- docker-image: ./images/protected-publish
Expand Down
243 changes: 0 additions & 243 deletions analytics/analytics/core/job_failure_classifier/__init__.py

This file was deleted.

8 changes: 4 additions & 4 deletions analytics/analytics/core/job_log_uploader/__init__.py
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
import json
import re
from dataclasses import asdict
from datetime import datetime
import json
import re
from typing import Any

import gitlab
from celery import shared_task
from django.conf import settings
import gitlab
from opensearch_dsl import Date, Document, connections
from opensearchpy import ConnectionTimeout
from requests.exceptions import ReadTimeout
from urllib3.exceptions import ReadTimeoutError

from analytics import setup_gitlab_job_sentry_tags
from analytics.core.job_failure_classifier import _job_retry_data
from analytics.job_processor.dimensions import _job_retry_data


class JobLog(Document):
Expand Down
3 changes: 0 additions & 3 deletions analytics/analytics/core/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods

from analytics.core.job_failure_classifier import upload_job_failure_classification
from analytics.core.job_log_uploader import store_job_data
from analytics.job_processor import process_job

Expand All @@ -25,8 +24,6 @@ def webhook_handler(request: HttpRequest) -> HttpResponse:

# Store gitlab job log and failure data in opensearch
store_job_data.delay(request.body)
if job_input_data["build_status"] == "failed":
upload_job_failure_classification.delay(request.body)

# Store job data in postgres DB
process_job.delay(request.body)
Expand Down
Loading

0 comments on commit 9122589

Please sign in to comment.