Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[dbnode] Re-add aggregator doc limit update #3137

Merged
merged 12 commits into from
Feb 9, 2021
Merged

[dbnode] Re-add aggregator doc limit update #3137

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
8501986
[dbnode] Add aggregate term limit regression test
arnikola Jan 29, 2021
892cdd6
Linter
arnikola Jan 29, 2021
77100c4
[dbnode] Revert #3133 with fix
arnikola Jan 29, 2021
01d2dbf
Added fixes to aggregate query limit calculation
arnikola Jan 29, 2021
969c7bc
Lint + build fix
arnikola Jan 29, 2021
a288e7e
Merge branch 'master' into arnikola/fix-agg-limit
arnikola Jan 29, 2021
2048c63
Lint + response
arnikola Feb 1, 2021
34c39eb
Merge branch 'master' into arnikola/fix-agg-limit
arnikola Feb 1, 2021
ed9886a
Merge branch 'master' into arnikola/fix-agg-limit
arnikola Feb 1, 2021
969a386
Merge branch 'master' into arnikola/fix-agg-limit
arnikola Feb 9, 2021
97f5761
codegen
arnikola Feb 9, 2021
1b8d366
Merge branch 'master' into arnikola/fix-agg-limit
arnikola Feb 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions src/dbnode/storage/index/aggregate_results.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,9 +225,14 @@ func (r *aggregatedResults) AddFields(batch []AggregateResultsEntry) (int, int)
}
}

docs := 0
numInserts := 0
for _, entry := range batch {
var (
docs int
numInserts int
entry AggregateResultsEntry
)

for idx := 0; idx < len(batch); idx++ {
entry = batch[idx]
r.aggregateOpts.AggregateUsageMetrics.IncTotalFields(1)

if docs >= remainingDocs || numInserts >= remainingInserts {
Expand Down
3 changes: 0 additions & 3 deletions src/dbnode/storage/index/block.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,6 @@ func NewBlock(
iopts,
)

aggAdded := opts.InstrumentOptions().MetricsScope().Counter("aggregate-added-counter")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why remove this?

Copy link
Collaborator Author

@arnikola arnikola Feb 1, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This metric is essentially a copy of the total aggregate results metric here, so not too useful

// NB(bodu): The length of coldMutableSegments is always at least 1.
coldSegs := []*mutableSegments{
newMutableSegments(
Expand All @@ -261,7 +260,6 @@ func NewBlock(
metrics: newBlockMetrics(scope),
logger: iopts.Logger(),
docsLimit: opts.QueryLimits().DocsLimit(),
aggregatedAddedCounter: aggAdded,
}
b.newFieldsAndTermsIteratorFn = newFieldsAndTermsIterator
b.newExecutorWithRLockFn = b.executorWithRLock
Expand Down Expand Up @@ -745,7 +743,6 @@ func (b *block) aggregateWithSpan(

batchedFields = 1
}

}

batch, fieldAppended, termAppended = b.appendFieldAndTermToBatch(batch, field, term, iterateTerms)
Expand Down
3 changes: 1 addition & 2 deletions src/dbnode/storage/index/block_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1951,8 +1951,7 @@ func TestBlockAggregate(t *testing.T) {
require.Equal(t, tracepoint.BlockAggregate, spans[0].OperationName)

snap := scope.Snapshot()
tallytest.AssertCounterValue(t, 4, snap, "query-limit.total-docs-matched", nil)
tallytest.AssertCounterValue(t, 8, snap, "aggregate-added-counter", nil)
tallytest.AssertCounterValue(t, 3, snap, "query-limit.total-docs-matched", nil)
}

func TestBlockAggregateNotExhaustive(t *testing.T) {
Expand Down