-
Notifications
You must be signed in to change notification settings - Fork 134
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
dfd9165
commit 6c19460
Showing
11 changed files
with
422 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
{{ if .Values.dbPruningEnabled}} | ||
apiVersion: batch/v1beta1 | ||
kind: CronJob | ||
metadata: | ||
name: lookout-v2-db-pruner | ||
labels: | ||
{{- include "lookout_v2.labels.all" . | nindent 4 }} | ||
spec: | ||
schedule: {{ .Values.dbPruneSchedule | default "@hourly" | quote }} | ||
jobTemplate: | ||
metadata: | ||
name: lookout-v2-db-pruner | ||
labels: | ||
{{- include "lookout_v2.labels.all" . | nindent 8 }} | ||
spec: | ||
template: | ||
metadata: | ||
name: lookout-v2-db-pruner | ||
labels: | ||
{{- include "lookout_v2.labels.all" . | nindent 12 }} | ||
spec: | ||
restartPolicy: Never | ||
serviceAccountName: {{ .Values.customServiceAccount | default (include "lookout_v2.name" .) }} | ||
securityContext: | ||
runAsUser: 1000 | ||
runAsGroup: 2000 | ||
containers: | ||
- name: lookout-v2 | ||
imagePullPolicy: IfNotPresent | ||
image: {{ .Values.image.repository }}:{{ required "A value is required for .Values.image.tag" .Values.image.tag }} | ||
args: | ||
- --pruneDatabase | ||
- --config | ||
- /config/application_config.yaml | ||
{{- if .Values.env }} | ||
env: | ||
{{- toYaml .Values.env | nindent 16 -}} | ||
{{- end }} | ||
resources: | ||
{{- toYaml .Values.resources | nindent 16 }} | ||
volumeMounts: | ||
- name: user-config | ||
mountPath: /config/application_config.yaml | ||
subPath: {{ include "lookout_v2.config.filename" . }} | ||
readOnly: true | ||
{{- if .Values.additionalVolumeMounts }} | ||
{{- toYaml .Values.additionalVolumeMounts | nindent 16 -}} | ||
{{- end }} | ||
securityContext: | ||
allowPrivilegeEscalation: false | ||
volumes: | ||
- name: user-config | ||
secret: | ||
secretName: {{ include "lookout_v2.config.name" . }} | ||
{{- if .Values.additionalVolumes }} | ||
{{- toYaml .Values.additionalVolumes | nindent 12 }} | ||
{{- end }} | ||
{{- end }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,10 +1,22 @@ | ||
package configuration | ||
|
||
import "github.com/armadaproject/armada/internal/armada/configuration" | ||
import ( | ||
"time" | ||
|
||
"github.com/armadaproject/armada/internal/armada/configuration" | ||
) | ||
|
||
type LookoutV2Configuration struct { | ||
ApiPort int | ||
CorsAllowedOrigins []string | ||
|
||
Postgres configuration.PostgresConfig | ||
|
||
PrunerConfig PrunerConfig | ||
} | ||
|
||
type PrunerConfig struct { | ||
ExpireAfter time.Duration | ||
Timeout time.Duration | ||
BatchSize int | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,103 @@ | ||
package pruner | ||
|
||
import ( | ||
"context" | ||
"time" | ||
|
||
"github.com/jackc/pgx/v4" | ||
"github.com/pkg/errors" | ||
log "github.com/sirupsen/logrus" | ||
"k8s.io/apimachinery/pkg/util/clock" | ||
) | ||
|
||
func PruneDb(ctx context.Context, db *pgx.Conn, keepAfterCompletion time.Duration, batchLimit int, clock clock.Clock) error { | ||
now := clock.Now() | ||
cutOffTime := now.Add(-keepAfterCompletion) | ||
totalJobsToDelete, err := createJobIdsToDeleteTempTable(ctx, db, cutOffTime) | ||
if err != nil { | ||
return errors.WithStack(err) | ||
} | ||
if totalJobsToDelete == 0 { | ||
log.Infof("Found no jobs to be deleted. Exiting") | ||
return nil | ||
} | ||
|
||
_, err = db.Exec(ctx, "CREATE TEMP TABLE batch (job_id TEXT);") | ||
if err != nil { | ||
return errors.WithStack(err) | ||
} | ||
|
||
jobsDeleted := 0 | ||
keepGoing := true | ||
for keepGoing { | ||
batchStart := clock.Now() | ||
batchSize := 0 | ||
err = db.BeginTxFunc(ctx, pgx.TxOptions{ | ||
IsoLevel: pgx.ReadCommitted, | ||
AccessMode: pgx.ReadWrite, | ||
DeferrableMode: pgx.Deferrable, | ||
}, func(tx pgx.Tx) error { | ||
batchSize, err = deleteBatch(ctx, tx, batchLimit) | ||
if err != nil { | ||
return err | ||
} | ||
if batchSize == 0 { | ||
keepGoing = false | ||
return nil | ||
} | ||
return nil | ||
}) | ||
if err != nil { | ||
return errors.Wrap(err, "error deleting batch from postgres") | ||
} | ||
batchDuration := clock.Since(batchStart) | ||
jobsDeleted += batchSize | ||
log.Infof("Deleted %d jobs in %s. Deleted %d jobs out of %d", batchSize, batchDuration, jobsDeleted, totalJobsToDelete) | ||
} | ||
totalTime := clock.Since(now) | ||
log.Infof("Total jobs deleted: %d, time taken: %v", jobsDeleted, totalTime) | ||
return nil | ||
} | ||
|
||
// Returns total number of jobs to delete | ||
func createJobIdsToDeleteTempTable(ctx context.Context, db *pgx.Conn, cutOffTime time.Time) (int, error) { | ||
_, err := db.Exec(ctx, ` | ||
CREATE TEMP TABLE job_ids_to_delete AS ( | ||
SELECT job_id FROM job | ||
WHERE last_transition_time < $1 | ||
)`, cutOffTime) | ||
if err != nil { | ||
return -1, errors.WithStack(err) | ||
} | ||
totalJobsToDelete := 0 | ||
err = db.QueryRow(ctx, "SELECT COUNT(*) FROM job_ids_to_delete").Scan(&totalJobsToDelete) | ||
if err != nil { | ||
return -1, errors.WithStack(err) | ||
} | ||
return totalJobsToDelete, nil | ||
} | ||
|
||
func deleteBatch(ctx context.Context, tx pgx.Tx, batchLimit int) (int, error) { | ||
_, err := tx.Exec(ctx, "INSERT INTO batch (job_id) SELECT job_id FROM job_ids_to_delete LIMIT $1;", batchLimit) | ||
if err != nil { | ||
return -1, err | ||
} | ||
var batchSize int | ||
err = tx.QueryRow(ctx, "SELECT COUNT(*) FROM batch").Scan(&batchSize) | ||
if err != nil { | ||
return -1, err | ||
} | ||
if batchSize == 0 { | ||
return 0, nil | ||
} | ||
_, err = tx.Exec(ctx, ` | ||
DELETE FROM job WHERE job_id in (SELECT job_id from batch); | ||
DELETE FROM job_run WHERE job_id in (SELECT job_id from batch); | ||
DELETE FROM user_annotation_lookup WHERE job_id in (SELECT job_id from batch); | ||
DELETE FROM job_ids_to_delete WHERE job_id in (SELECT job_id from batch); | ||
TRUNCATE TABLE batch;`) | ||
if err != nil { | ||
return -1, err | ||
} | ||
return batchSize, nil | ||
} |
Oops, something went wrong.