Skip to content

Commit

Permalink
Revert only the API surface area while keeping everything else
Browse files Browse the repository at this point in the history
  • Loading branch information
@MasslessParticle
MasslessParticle committed Mar 11, 2022
1 parent 28b5c89 commit 0fcd262
Show file tree
Hide file tree
Showing 8 changed files with 82 additions and 74 deletions.
25 changes: 12 additions & 13 deletions docs/sources/operations/storage/logs-deletion.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,23 +28,23 @@ The Compactor exposes endpoints to allow for the deletion of log entries from sp
### Request log entry deletion

```
POST /loki/api/v1/delete
PUT /loki/api/v1/delete
POST /loki/api/admin/delete
PUT /loki/api/admin/delete
```

Query parameters:

* `query=<series_selector>`: A label matcher argument that identifies the streams from which to delete. This argument must be provided.
* `match[]=<series_selector>`: Repeated label matcher argument that identifies the streams from which to delete. At least one `match[]` argument must be provided.
* `start=<rfc3339 | unix_timestamp>`: A timestamp that identifies the start of the time window within which entries will be deleted. If not specified, defaults to 0, the Unix Epoch time.
* `end=<rfc3339 | unix_timestamp>`: A timestamp that identifies the end of the time window within which entries will be deleted. If not specified, defaults to the current time.

A 204 response indicates success.

URL encode the `query` parameter. This sample form of a cURL command URL encodes `query={foo="bar"}`:
URL encode the `match[]` parameter. This sample form of a cURL command URL encodes `match[]={foo="bar"}`:

```
curl -g -X POST \
'http://127.0.0.1:3100/loki/api/v1/delete?query={foo="bar"}&start=1591616227&end=1591619692' \
'http://127.0.0.1:3100/loki/api/admin/delete?match[]={foo="bar"}&start=1591616227&end=1591619692' \
-H 'x-scope-orgid: 1'
```

Expand All @@ -53,39 +53,38 @@ curl -g -X POST \
List the existing delete requests using the following API:

```
GET /loki/api/v1/delete
GET /loki/api/admin/delete
```

Sample form of a cURL command:

```
curl -X GET \
<compactor_addr>/loki/api/v1/delete \
<compactor_addr>/loki/api/admin/delete \
-H 'x-scope-orgid: <orgid>'
```

This endpoint returns both processed and unprocessed requests. It does not list canceled requests, as those requests will have been removed from storage.

### Request cancellation of a delete request

Loki allows cancellation of delete requests until the requests are picked up for processing. It is controlled by the `delete_request_cancel_period` YAML configuration or the equivalent command line option when invoking Loki.

Cancel a delete request using this Compactor endpoint:

```
DELETE /loki/api/v1/delete
POST /loki/api/admin/cancel_delete_request
PUT /loki/api/admin/cancel_delete_request
```

Query parameters:

* `request_id=<request_id>`: Identifies the delete request to cancel; IDs are found using the GET method on this endpoint.
* `request_id=<request_id>`: Identifies the delete request to cancel; IDs are found using the `delete` endpoint.

A 204 response indicates success.

Sample form of a cURL command:

```
curl -X DELETE \
'<compactor_addr>/loki/api/v1/delete?request_id=<request_id>' \
curl -X POST \
'<compactor_addr>/loki/api/admin/cancel_delete_request?request_id=<request_id>' \
-H 'x-scope-orgid: <tenant-id>'
```
6 changes: 3 additions & 3 deletions pkg/loki/modules.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -722,9 +722,9 @@ func (t *Loki) initCompactor() (services.Service, error) {
t.Server.HTTP.Path("/compactor/ring").Methods("GET", "POST").Handler(t.compactor)

if t.Cfg.CompactorConfig.RetentionEnabled && t.compactor.DeleteMode == deletion.V1 {
t.Server.HTTP.Path("/loki/api/v1/delete").Methods("PUT", "POST").Handler(t.HTTPAuthMiddleware.Wrap(http.HandlerFunc(t.compactor.DeleteRequestsHandler.AddDeleteRequestHandler)))
t.Server.HTTP.Path("/loki/api/v1/delete").Methods("GET").Handler(t.HTTPAuthMiddleware.Wrap(http.HandlerFunc(t.compactor.DeleteRequestsHandler.GetAllDeleteRequestsHandler)))
t.Server.HTTP.Path("/loki/api/v1/delete").Methods("DELETE").Handler(t.HTTPAuthMiddleware.Wrap(http.HandlerFunc(t.compactor.DeleteRequestsHandler.CancelDeleteRequestHandler)))
t.Server.HTTP.Path("/loki/api/admin/delete").Methods("PUT", "POST").Handler(t.HTTPAuthMiddleware.Wrap(http.HandlerFunc(t.compactor.DeleteRequestsHandler.AddDeleteRequestHandler)))
t.Server.HTTP.Path("/loki/api/admin/delete").Methods("GET").Handler(t.HTTPAuthMiddleware.Wrap(http.HandlerFunc(t.compactor.DeleteRequestsHandler.GetAllDeleteRequestsHandler)))
t.Server.HTTP.Path("/loki/api/admin/cancel_delete_request").Methods("PUT", "POST").Handler(t.HTTPAuthMiddleware.Wrap(http.HandlerFunc(t.compactor.DeleteRequestsHandler.CancelDeleteRequestHandler)))
}

return t.compactor, nil
Expand Down
18 changes: 11 additions & 7 deletions pkg/storage/stores/shipper/compactor/deletion/delete_request.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,21 +11,25 @@ type DeleteRequest struct {
RequestID string `json:"request_id"`
StartTime model.Time `json:"start_time"`
EndTime model.Time `json:"end_time"`
Query string `json:"logql_requests"`
Queries []string `json:"logql_requests"`
Status DeleteRequestStatus `json:"status"`
CreatedAt model.Time `json:"created_at"`

UserID string `json:"-"`
matchers []*labels.Matcher `json:"-"`
}

func (d *DeleteRequest) AddQuery(logQL string) error {
d.Query = logQL
matchers, err := parseDeletionQuery(logQL)
if err != nil {
return err
func (d *DeleteRequest) AddQueries(queries []string) error {
d.Queries = queries
for _, query := range queries {
matchers, err := parseDeletionQuery(query)
if err != nil {
return err
}

d.matchers = append(d.matchers, matchers...)
}
d.matchers = matchers

return nil
}

Expand Down
22 changes: 11 additions & 11 deletions pkg/storage/stores/shipper/compactor/deletion/delete_request_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,15 @@ func TestDeleteRequest_IsDeleted(t *testing.T) {
now := model.Now()
user1 := "user1"

lbls := `{foo="bar", fizz="buzz"}`
lbls := []string{`{foo="bar", fizz="buzz"}`}

chunkEntry := retention.ChunkEntry{
ChunkRef: retention.ChunkRef{
UserID: []byte(user1),
From: now.Add(-3 * time.Hour),
Through: now.Add(-time.Hour),
},
Labels: mustParseLabel(lbls),
Labels: mustParseLabel(lbls[0]),
}

type resp struct {
Expand All @@ -43,7 +43,7 @@ func TestDeleteRequest_IsDeleted(t *testing.T) {
UserID: user1,
StartTime: now.Add(-3 * time.Hour),
EndTime: now.Add(-time.Hour),
Query: lbls,
Queries: lbls,
},
expectedResp: resp{
isDeleted: true,
Expand All @@ -56,7 +56,7 @@ func TestDeleteRequest_IsDeleted(t *testing.T) {
UserID: user1,
StartTime: now.Add(-3 * time.Hour),
EndTime: now.Add(-2 * time.Hour),
Query: lbls,
Queries: lbls,
},
expectedResp: resp{
isDeleted: true,
Expand All @@ -74,7 +74,7 @@ func TestDeleteRequest_IsDeleted(t *testing.T) {
UserID: user1,
StartTime: now.Add(-2 * time.Hour),
EndTime: now,
Query: lbls,
Queries: lbls,
},
expectedResp: resp{
isDeleted: true,
Expand All @@ -92,7 +92,7 @@ func TestDeleteRequest_IsDeleted(t *testing.T) {
UserID: user1,
StartTime: now.Add(-2 * time.Hour),
EndTime: now,
Query: lbls,
Queries: lbls,
},
expectedResp: resp{
isDeleted: true,
Expand All @@ -110,7 +110,7 @@ func TestDeleteRequest_IsDeleted(t *testing.T) {
UserID: user1,
StartTime: now.Add(-(2*time.Hour + 30*time.Minute)),
EndTime: now.Add(-(time.Hour + 30*time.Minute)),
Query: lbls,
Queries: lbls,
},
expectedResp: resp{
isDeleted: true,
Expand All @@ -132,7 +132,7 @@ func TestDeleteRequest_IsDeleted(t *testing.T) {
UserID: user1,
StartTime: now.Add(-12 * time.Hour),
EndTime: now.Add(-10 * time.Hour),
Query: lbls,
Queries: lbls,
},
expectedResp: resp{
isDeleted: false,
Expand All @@ -144,7 +144,7 @@ func TestDeleteRequest_IsDeleted(t *testing.T) {
UserID: "user1",
StartTime: now.Add(-3 * time.Hour),
EndTime: now.Add(-time.Hour),
Query: `{foo1="bar"}`,
Queries: []string{`{foo1="bar"}`},
},
expectedResp: resp{
isDeleted: false,
Expand All @@ -156,15 +156,15 @@ func TestDeleteRequest_IsDeleted(t *testing.T) {
UserID: "user2",
StartTime: now.Add(-3 * time.Hour),
EndTime: now.Add(-time.Hour),
Query: lbls,
Queries: lbls,
},
expectedResp: resp{
isDeleted: false,
},
},
} {
t.Run(tc.name, func(t *testing.T) {
require.NoError(t, tc.deleteRequest.AddQuery(tc.deleteRequest.Query))
require.NoError(t, tc.deleteRequest.AddQueries(tc.deleteRequest.Queries))
isDeleted, nonDeletedIntervals := tc.deleteRequest.IsDeleted(chunkEntry)
require.Equal(t, tc.expectedResp.isDeleted, isDeleted)
require.Equal(t, tc.expectedResp.nonDeletedIntervals, nonDeletedIntervals)
Expand Down
30 changes: 15 additions & 15 deletions pkg/storage/stores/shipper/compactor/deletion/delete_requests_manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func (m mockDeleteRequestsStore) UpdateStatus(ctx context.Context, userID, reque
return nil
}

func (m mockDeleteRequestsStore) AddDeleteRequest(ctx context.Context, userID string, startTime, endTime model.Time, LogQLRequest string) error {
func (m mockDeleteRequestsStore) AddDeleteRequest(ctx context.Context, userID string, startTime, endTime model.Time, LogQLRequest []string) error {
panic("implement me")
}

Expand Down Expand Up @@ -90,7 +90,7 @@ func TestDeleteRequestsManager_Expired(t *testing.T) {
deleteRequestsFromStore: []DeleteRequest{
{
UserID: "different-user",
Query: lblFoo.String(),
Queries: []string{lblFoo.String()},
StartTime: now.Add(-24 * time.Hour),
EndTime: now,
},
Expand All @@ -105,7 +105,7 @@ func TestDeleteRequestsManager_Expired(t *testing.T) {
deleteRequestsFromStore: []DeleteRequest{
{
UserID: testUserID,
Query: lblFoo.String(),
Queries: []string{lblFoo.String()},
StartTime: now.Add(-24 * time.Hour),
EndTime: now,
},
Expand All @@ -120,7 +120,7 @@ func TestDeleteRequestsManager_Expired(t *testing.T) {
deleteRequestsFromStore: []DeleteRequest{
{
UserID: testUserID,
Query: lblFoo.String(),
Queries: []string{lblFoo.String()},
StartTime: now.Add(-48 * time.Hour),
EndTime: now.Add(-24 * time.Hour),
},
Expand All @@ -135,13 +135,13 @@ func TestDeleteRequestsManager_Expired(t *testing.T) {
deleteRequestsFromStore: []DeleteRequest{
{
UserID: testUserID,
Query: lblFoo.String(),
Queries: []string{lblFoo.String()},
StartTime: now.Add(-48 * time.Hour),
EndTime: now.Add(-24 * time.Hour),
},
{
UserID: testUserID,
Query: lblFoo.String(),
Queries: []string{lblFoo.String()},
StartTime: now.Add(-12 * time.Hour),
EndTime: now,
},
Expand All @@ -156,25 +156,25 @@ func TestDeleteRequestsManager_Expired(t *testing.T) {
deleteRequestsFromStore: []DeleteRequest{
{
UserID: testUserID,
Query: lblFoo.String(),
Queries: []string{lblFoo.String()},
StartTime: now.Add(-13 * time.Hour),
EndTime: now.Add(-11 * time.Hour),
},
{
UserID: testUserID,
Query: lblFoo.String(),
Queries: []string{lblFoo.String()},
StartTime: now.Add(-10 * time.Hour),
EndTime: now.Add(-8 * time.Hour),
},
{
UserID: testUserID,
Query: lblFoo.String(),
Queries: []string{lblFoo.String()},
StartTime: now.Add(-6 * time.Hour),
EndTime: now.Add(-5 * time.Hour),
},
{
UserID: testUserID,
Query: lblFoo.String(),
Queries: []string{lblFoo.String()},
StartTime: now.Add(-2 * time.Hour),
EndTime: now,
},
Expand Down Expand Up @@ -202,13 +202,13 @@ func TestDeleteRequestsManager_Expired(t *testing.T) {
deleteRequestsFromStore: []DeleteRequest{
{
UserID: testUserID,
Query: lblFoo.String(),
Queries: []string{lblFoo.String()},
StartTime: now.Add(-13 * time.Hour),
EndTime: now.Add(-6 * time.Hour),
},
{
UserID: testUserID,
Query: lblFoo.String(),
Queries: []string{lblFoo.String()},
StartTime: now.Add(-8 * time.Hour),
EndTime: now,
},
Expand All @@ -223,19 +223,19 @@ func TestDeleteRequestsManager_Expired(t *testing.T) {
deleteRequestsFromStore: []DeleteRequest{
{
UserID: testUserID,
Query: lblFoo.String(),
Queries: []string{lblFoo.String()},
StartTime: now.Add(-12 * time.Hour),
EndTime: now.Add(-6*time.Hour) - 1,
},
{
UserID: testUserID,
Query: lblFoo.String(),
Queries: []string{lblFoo.String()},
StartTime: now.Add(-6 * time.Hour),
EndTime: now.Add(-4*time.Hour) - 1,
},
{
UserID: testUserID,
Query: lblFoo.String(),
Queries: []string{lblFoo.String()},
StartTime: now.Add(-4 * time.Hour),
EndTime: now,
},
Expand Down
Loading

0 comments on commit 0fcd262

Please sign in to comment.