Skip to content

Commit

Permalink
Use dedicated config value for storage
Browse files Browse the repository at this point in the history
  • Loading branch information
timacdonald committed Dec 1, 2024
1 parent f2b9d85 commit cdf3040
Show file tree
Hide file tree
Showing 5 changed files with 47 additions and 11 deletions.
4 changes: 4 additions & 0 deletions config/pulse.php
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,10 @@
'storage' => [
'driver' => env('PULSE_STORAGE_DRIVER', 'database'),

'trim' => [
'keep' => env('PULSE_STORAGE_KEEP', '7 days'),
],

'database' => [
'connection' => env('PULSE_DB_CONNECTION'),
'chunk' => 1000,
Expand Down
12 changes: 8 additions & 4 deletions src/Storage/DatabaseStorage.php
Original file line number Diff line number Diff line change
Expand Up @@ -125,16 +125,20 @@ public function trim(): void
{
$now = CarbonImmutable::now();

$keep = $this->config->get('pulse.ingest.trim.keep');
$keep = $this->config->get('pulse.storage.trim.keep') ?? '7 days';

$before = $now->subMilliseconds(
(int) CarbonInterval::fromString($keep)->totalMilliseconds
);

$this->connection()
->table('pulse_values')
->where('timestamp', '<=', $now->sub($keep)->getTimestamp())
->where('timestamp', '<=', $before->getTimestamp())
->delete();

$this->connection()
->table('pulse_entries')
->where('timestamp', '<=', $now->sub($keep)->getTimestamp())
->where('timestamp', '<=', $before->getTimestamp())
->delete();

$this->connection()
Expand All @@ -144,7 +148,7 @@ public function trim(): void
->each(fn (int $period) => $this->connection()
->table('pulse_aggregates')
->where('period', $period)
->where('bucket', '<=', $now->subMinutes($period)->getTimestamp())
->where('bucket', '<=', max($now->subMinutes($period)->getTimestamp(), $before->getTimestamp()))
->delete());
}

Expand Down
29 changes: 25 additions & 4 deletions tests/Feature/Ingests/DatabaseTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,23 @@
expect(DB::table('pulse_aggregates')->where('period', 1440)->count())->toBe(1);
});

it('trims aggregates to the configured storage duration when configured trim is shorter than the bucket period duration', function () {
Config::set('pulse.storage.trim.keep', '23 minutes');
Date::setTestNow('2000-01-01 00:00:00'); // Bucket: 2000-01-01 00:00:00
Pulse::record('foo', 'xxxx', 1)->count();
Pulse::ingest();
Pulse::stopRecording();
expect(DB::table('pulse_aggregates')->where('period', 1440)->count())->toBe(1);

Date::setTestNow('2000-01-01 00:22:59');
App::make(DatabaseStorage::class)->trim();
expect(DB::table('pulse_aggregates')->where('period', 1440)->count())->toBe(1);

Date::setTestNow('2000-01-01 00:23:00');
Pulse::ignore(fn () => App::make(DatabaseStorage::class)->trim());
expect(DB::table('pulse_aggregates')->where('period', 1440)->count())->toBe(0);
});

it('trims aggregates once the 7 day bucket is no longer relevant', function () {
Date::setTestNow('2000-01-01 02:23:59'); // Bucket: 1999-12-31 23:36:00
Pulse::record('foo', 'xxxx', 1)->count();
Expand All @@ -124,19 +141,23 @@
});

it('can configure days of data to keep when trimming', function () {
Config::set('pulse.ingest.trim.keep', '14 days');
Config::set('pulse.storage.trim.keep', '14 days');

Date::setTestNow('2000-01-01 00:00:04');
Pulse::record('foo', 'xxxx', 1);
Pulse::record('foo', 'xxxx', 1)->count();
Pulse::set('type', 'foo', 'value');
Date::setTestNow('2000-01-01 00:00:05');
Pulse::record('bar', 'xxxx', 1);
Pulse::record('bar', 'xxxx', 1)->count();
Pulse::set('type', 'bar', 'value');
Date::setTestNow('2000-01-01 00:00:06');
Pulse::record('baz', 'xxxx', 1);
Pulse::record('baz', 'xxxx', 1)->count();
Pulse::set('type', 'baz', 'value');
Pulse::ingest();

Pulse::stopRecording();
Date::setTestNow('2000-01-15 00:00:05');
App::make(DatabaseStorage::class)->trim();

expect(DB::table('pulse_entries')->pluck('type')->all())->toBe(['baz']);
expect(DB::table('pulse_values')->pluck('key')->all())->toBe(['baz']);
});
2 changes: 1 addition & 1 deletion tests/Feature/PulseTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
});

it('can configure days of data to keep when trimming', function () {
Config::set('pulse.ingest.trim.keep', '30 days');
Config::set('pulse.storage.trim.keep', '30 days');
App::instance(Storage::class, $storage = new StorageFake);

Pulse::record('foo', 'delete', 0, now()->subMonth());
Expand Down
11 changes: 9 additions & 2 deletions tests/StorageFake.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

namespace Tests;

use Carbon\CarbonImmutable;
use Carbon\CarbonInterval;
use Illuminate\Support\Collection;
use Laravel\Pulse\Contracts\Storage;
Expand Down Expand Up @@ -31,9 +32,15 @@ public function store(Collection $items): void
*/
public function trim(): void
{
$keep = config('pulse.ingest.trim.keep');
$now = CarbonImmutable::now();

$this->stored = $this->stored->reject(fn ($record) => $record->timestamp <= now()->sub($keep)->timestamp);
$keep = config('pulse.storage.trim.keep') ?? '7 days';

$before = $now->subMilliseconds(
(int) CarbonInterval::fromString($keep)->totalMilliseconds
);

$this->stored = $this->stored->reject(fn ($record) => $record->timestamp <= $before->timestamp);
}

/**
Expand Down

0 comments on commit cdf3040

Please sign in to comment.