From 4b52b70a780dd85dfe1e45c3e0fab469676f4b2e Mon Sep 17 00:00:00 2001 From: Tharlei Aleixo Date: Thu, 28 Nov 2024 14:30:52 -0300 Subject: [PATCH 1/5] feat: add command to prune data with suggested before hours --- src/Commands/PruneCommand.php | 47 ++++++++++++++++++++ src/Contracts/Storage.php | 9 ++++ src/PulseServiceProvider.php | 1 + src/Storage/DatabaseStorage.php | 32 ++++++++++++++ tests/Feature/Commands/PruneCommandTest.php | 49 +++++++++++++++++++++ tests/StorageFake.php | 12 +++++ 6 files changed, 150 insertions(+) create mode 100644 src/Commands/PruneCommand.php create mode 100644 tests/Feature/Commands/PruneCommandTest.php diff --git a/src/Commands/PruneCommand.php b/src/Commands/PruneCommand.php new file mode 100644 index 00000000..a15e56c8 --- /dev/null +++ b/src/Commands/PruneCommand.php @@ -0,0 +1,47 @@ +confirmToProceed()) { + return Command::FAILURE; + } + + $pulse->prune( + now()->subHours( + (int) $this->option('hours') + ) + ); + + return Command::SUCCESS; + } +} diff --git a/src/Contracts/Storage.php b/src/Contracts/Storage.php index 40b81f49..95a86347 100644 --- a/src/Contracts/Storage.php +++ b/src/Contracts/Storage.php @@ -3,6 +3,7 @@ namespace Laravel\Pulse\Contracts; use Carbon\CarbonInterval; +use DateTimeInterface; use Illuminate\Support\Collection; interface Storage @@ -19,6 +20,14 @@ public function store(Collection $items): void; */ public function trim(): void; + /** + * Prune the storage. + * + * @param \DateTimeInterface $before + * @return void + */ + public function prune(DateTimeInterface $before): void; + /** * Purge the storage. * diff --git a/src/PulseServiceProvider.php b/src/PulseServiceProvider.php index 8724c6ab..149f9ba9 100644 --- a/src/PulseServiceProvider.php +++ b/src/PulseServiceProvider.php @@ -234,6 +234,7 @@ protected function registerCommands(): void Commands\CheckCommand::class, Commands\RestartCommand::class, Commands\ClearCommand::class, + Commands\PruneCommand::class, ]); AboutCommand::add('Pulse', fn () => [ diff --git a/src/Storage/DatabaseStorage.php b/src/Storage/DatabaseStorage.php index 3cabc64f..e6311d2b 100644 --- a/src/Storage/DatabaseStorage.php +++ b/src/Storage/DatabaseStorage.php @@ -5,6 +5,7 @@ use Carbon\CarbonImmutable; use Carbon\CarbonInterval; use Closure; +use DateTimeInterface; use Illuminate\Contracts\Config\Repository; use Illuminate\Database\Connection; use Illuminate\Database\DatabaseManager; @@ -146,6 +147,37 @@ public function trim(): void ->delete()); } + /** + * Prune the storage. + * + * @param \DateTimeInterface $before + * @return void + */ + public function prune(DateTimeInterface $before): void + { + $before = CarbonImmutable::parse($before); + + $this->connection() + ->table('pulse_values') + ->where('timestamp', '<', $before->getTimestamp()) + ->delete(); + + $this->connection() + ->table('pulse_entries') + ->where('timestamp', '<', $before->getTimestamp()) + ->delete(); + + $this->connection() + ->table('pulse_aggregates') + ->distinct() + ->pluck('period') + ->each(fn (int $period) => $this->connection() + ->table('pulse_aggregates') + ->where('period', $period) + ->where('bucket', '<', $before->subMinutes($period)->getTimestamp()) + ->delete()); + } + /** * Purge the storage. * diff --git a/tests/Feature/Commands/PruneCommandTest.php b/tests/Feature/Commands/PruneCommandTest.php new file mode 100644 index 00000000..2f0018e3 --- /dev/null +++ b/tests/Feature/Commands/PruneCommandTest.php @@ -0,0 +1,49 @@ +count())->toBe(0); + expect(DB::table('pulse_entries')->count())->toBe(0); + expect(DB::table('pulse_aggregates')->count())->toBe(0); +}); + +it('prune entries from the suggested before hours', function () { + // Entries will be pruned + Date::setTestNow('2000-01-01 00:00:04'); + Pulse::record('foo', 'xxxx', 1); + Date::setTestNow('2000-01-01 00:00:05'); + Pulse::record('bar', 'xxxx', 1); + + // Entries will be kept + Date::setTestNow("2000-01-07 00:00:00"); + Pulse::record('baz', 'xxxx', 1); + Pulse::ingest(); + + Pulse::stopRecording(); + Date::setTestNow('2000-01-09 00:00:00'); + + # Act + Artisan::call("pulse:prune --hours=168"); + + expect(DB::table('pulse_entries')->pluck('type')->all())->toBe(['baz']); +}); diff --git a/tests/StorageFake.php b/tests/StorageFake.php index 5f9c815f..2c69f774 100644 --- a/tests/StorageFake.php +++ b/tests/StorageFake.php @@ -3,6 +3,7 @@ namespace Tests; use Carbon\CarbonInterval; +use DateTimeInterface; use Illuminate\Support\Collection; use Laravel\Pulse\Contracts\Storage; @@ -34,6 +35,17 @@ public function trim(): void $this->stored = $this->stored->reject(fn ($record) => $record->timestamp <= now()->subWeek()->timestamp); } + /** + * Prune the storage. + * + * @param \DateTimeInterface $before + * @return void + */ + public function prune(DateTimeInterface $before): void + { + $this->stored = $this->stored->reject(fn ($record) => $record->timestamp <= $before->getTimestamp()); + } + /** * Purge the storage. * From d08c2830c39035c63ed019419b1eb7250a32d660 Mon Sep 17 00:00:00 2001 From: Tharlei Aleixo Date: Fri, 29 Nov 2024 08:44:13 -0300 Subject: [PATCH 2/5] chore: remove prune structure --- src/Commands/PruneCommand.php | 47 -------------------- src/Contracts/Storage.php | 8 ---- src/PulseServiceProvider.php | 1 - src/Storage/DatabaseStorage.php | 31 ------------- tests/Feature/Commands/PruneCommandTest.php | 49 --------------------- tests/StorageFake.php | 12 ----- 6 files changed, 148 deletions(-) delete mode 100644 src/Commands/PruneCommand.php delete mode 100644 tests/Feature/Commands/PruneCommandTest.php diff --git a/src/Commands/PruneCommand.php b/src/Commands/PruneCommand.php deleted file mode 100644 index a15e56c8..00000000 --- a/src/Commands/PruneCommand.php +++ /dev/null @@ -1,47 +0,0 @@ -confirmToProceed()) { - return Command::FAILURE; - } - - $pulse->prune( - now()->subHours( - (int) $this->option('hours') - ) - ); - - return Command::SUCCESS; - } -} diff --git a/src/Contracts/Storage.php b/src/Contracts/Storage.php index 95a86347..9290ad0f 100644 --- a/src/Contracts/Storage.php +++ b/src/Contracts/Storage.php @@ -20,14 +20,6 @@ public function store(Collection $items): void; */ public function trim(): void; - /** - * Prune the storage. - * - * @param \DateTimeInterface $before - * @return void - */ - public function prune(DateTimeInterface $before): void; - /** * Purge the storage. * diff --git a/src/PulseServiceProvider.php b/src/PulseServiceProvider.php index 149f9ba9..8724c6ab 100644 --- a/src/PulseServiceProvider.php +++ b/src/PulseServiceProvider.php @@ -234,7 +234,6 @@ protected function registerCommands(): void Commands\CheckCommand::class, Commands\RestartCommand::class, Commands\ClearCommand::class, - Commands\PruneCommand::class, ]); AboutCommand::add('Pulse', fn () => [ diff --git a/src/Storage/DatabaseStorage.php b/src/Storage/DatabaseStorage.php index e6311d2b..675f4445 100644 --- a/src/Storage/DatabaseStorage.php +++ b/src/Storage/DatabaseStorage.php @@ -147,37 +147,6 @@ public function trim(): void ->delete()); } - /** - * Prune the storage. - * - * @param \DateTimeInterface $before - * @return void - */ - public function prune(DateTimeInterface $before): void - { - $before = CarbonImmutable::parse($before); - - $this->connection() - ->table('pulse_values') - ->where('timestamp', '<', $before->getTimestamp()) - ->delete(); - - $this->connection() - ->table('pulse_entries') - ->where('timestamp', '<', $before->getTimestamp()) - ->delete(); - - $this->connection() - ->table('pulse_aggregates') - ->distinct() - ->pluck('period') - ->each(fn (int $period) => $this->connection() - ->table('pulse_aggregates') - ->where('period', $period) - ->where('bucket', '<', $before->subMinutes($period)->getTimestamp()) - ->delete()); - } - /** * Purge the storage. * diff --git a/tests/Feature/Commands/PruneCommandTest.php b/tests/Feature/Commands/PruneCommandTest.php deleted file mode 100644 index 2f0018e3..00000000 --- a/tests/Feature/Commands/PruneCommandTest.php +++ /dev/null @@ -1,49 +0,0 @@ -count())->toBe(0); - expect(DB::table('pulse_entries')->count())->toBe(0); - expect(DB::table('pulse_aggregates')->count())->toBe(0); -}); - -it('prune entries from the suggested before hours', function () { - // Entries will be pruned - Date::setTestNow('2000-01-01 00:00:04'); - Pulse::record('foo', 'xxxx', 1); - Date::setTestNow('2000-01-01 00:00:05'); - Pulse::record('bar', 'xxxx', 1); - - // Entries will be kept - Date::setTestNow("2000-01-07 00:00:00"); - Pulse::record('baz', 'xxxx', 1); - Pulse::ingest(); - - Pulse::stopRecording(); - Date::setTestNow('2000-01-09 00:00:00'); - - # Act - Artisan::call("pulse:prune --hours=168"); - - expect(DB::table('pulse_entries')->pluck('type')->all())->toBe(['baz']); -}); diff --git a/tests/StorageFake.php b/tests/StorageFake.php index 2c69f774..5f9c815f 100644 --- a/tests/StorageFake.php +++ b/tests/StorageFake.php @@ -3,7 +3,6 @@ namespace Tests; use Carbon\CarbonInterval; -use DateTimeInterface; use Illuminate\Support\Collection; use Laravel\Pulse\Contracts\Storage; @@ -35,17 +34,6 @@ public function trim(): void $this->stored = $this->stored->reject(fn ($record) => $record->timestamp <= now()->subWeek()->timestamp); } - /** - * Prune the storage. - * - * @param \DateTimeInterface $before - * @return void - */ - public function prune(DateTimeInterface $before): void - { - $this->stored = $this->stored->reject(fn ($record) => $record->timestamp <= $before->getTimestamp()); - } - /** * Purge the storage. * From d688aa6bcc3fa8a51d18c6eeeee3108f12fff9cb Mon Sep 17 00:00:00 2001 From: Tharlei Aleixo Date: Fri, 29 Nov 2024 09:53:13 -0300 Subject: [PATCH 3/5] feat: add environment to keep data in database storage --- config/pulse.php | 2 +- src/Storage/DatabaseStorage.php | 5 +++-- tests/Feature/Ingests/DatabaseTest.php | 19 +++++++++++++++++++ tests/Feature/PulseTest.php | 13 +++++++++++++ tests/StorageFake.php | 4 +++- 5 files changed, 39 insertions(+), 4 deletions(-) diff --git a/config/pulse.php b/config/pulse.php index 7c00f819..d78c73b4 100644 --- a/config/pulse.php +++ b/config/pulse.php @@ -83,7 +83,7 @@ 'trim' => [ 'lottery' => [1, 1_000], - 'keep' => '7 days', + 'keep' => env('PULSE_INGEST_KEEP', '7 days'), ], 'redis' => [ diff --git a/src/Storage/DatabaseStorage.php b/src/Storage/DatabaseStorage.php index 675f4445..a633806d 100644 --- a/src/Storage/DatabaseStorage.php +++ b/src/Storage/DatabaseStorage.php @@ -124,16 +124,17 @@ public function store(Collection $items): void */ public function trim(): void { + $keep = $this->config->get('pulse.ingest.trim.keep'); $now = CarbonImmutable::now(); $this->connection() ->table('pulse_values') - ->where('timestamp', '<=', $now->subWeek()->getTimestamp()) + ->where('timestamp', '<=', $now->sub($keep)->getTimestamp()) ->delete(); $this->connection() ->table('pulse_entries') - ->where('timestamp', '<=', $now->subWeek()->getTimestamp()) + ->where('timestamp', '<=', $now->sub($keep)->getTimestamp()) ->delete(); $this->connection() diff --git a/tests/Feature/Ingests/DatabaseTest.php b/tests/Feature/Ingests/DatabaseTest.php index 9fd0500c..3ad6ed1e 100644 --- a/tests/Feature/Ingests/DatabaseTest.php +++ b/tests/Feature/Ingests/DatabaseTest.php @@ -1,6 +1,7 @@ trim(); expect(DB::table('pulse_aggregates')->where('period', 10080)->count())->toBe(1); }); + +it('can configure days of data to keep when trimming', function () { + Config::set('pulse.ingest.trim.keep', '14 days'); + + Date::setTestNow('2000-01-01 00:00:04'); + Pulse::record('foo', 'xxxx', 1); + Date::setTestNow('2000-01-01 00:00:05'); + Pulse::record('bar', 'xxxx', 1); + Date::setTestNow('2000-01-01 00:00:06'); + Pulse::record('baz', 'xxxx', 1); + Pulse::ingest(); + + Pulse::stopRecording(); + Date::setTestNow('2000-01-15 00:00:05'); + App::make(DatabaseStorage::class)->trim(); + + expect(DB::table('pulse_entries')->pluck('type')->all())->toBe(['baz']); +}); diff --git a/tests/Feature/PulseTest.php b/tests/Feature/PulseTest.php index c9223a6a..c8245238 100644 --- a/tests/Feature/PulseTest.php +++ b/tests/Feature/PulseTest.php @@ -48,6 +48,19 @@ expect($storage->stored)->toHaveCount(1); }); +it('can configure days of data to keep when trimming', function () { + Config::set('pulse.ingest.trim.keep', '30 days'); + App::instance(Storage::class, $storage = new StorageFake); + + Pulse::record('foo', 'delete', 0, now()->subMonth()); + Pulse::record('foo', 'keep', 0, now()->subWeek()); + Pulse::record('foo', 'keep', 0); + + Pulse::ingest(); + + expect($storage->stored)->toHaveCount(2); +}); + it('can lazily capture entries', function () { App::instance(Storage::class, $storage = new StorageFake); diff --git a/tests/StorageFake.php b/tests/StorageFake.php index 5f9c815f..b52e5806 100644 --- a/tests/StorageFake.php +++ b/tests/StorageFake.php @@ -31,7 +31,9 @@ public function store(Collection $items): void */ public function trim(): void { - $this->stored = $this->stored->reject(fn ($record) => $record->timestamp <= now()->subWeek()->timestamp); + $keep = config('pulse.ingest.trim.keep'); + + $this->stored = $this->stored->reject(fn ($record) => $record->timestamp <= now()->sub($keep)->timestamp); } /** From 6e7c89762434eb24e880622312aacf01fdd20da5 Mon Sep 17 00:00:00 2001 From: Tharlei Aleixo Date: Fri, 29 Nov 2024 09:54:15 -0300 Subject: [PATCH 4/5] chore: remove unused DateTimeInterface imports from Storage contracts --- src/Contracts/Storage.php | 1 - src/Storage/DatabaseStorage.php | 1 - 2 files changed, 2 deletions(-) diff --git a/src/Contracts/Storage.php b/src/Contracts/Storage.php index 9290ad0f..40b81f49 100644 --- a/src/Contracts/Storage.php +++ b/src/Contracts/Storage.php @@ -3,7 +3,6 @@ namespace Laravel\Pulse\Contracts; use Carbon\CarbonInterval; -use DateTimeInterface; use Illuminate\Support\Collection; interface Storage diff --git a/src/Storage/DatabaseStorage.php b/src/Storage/DatabaseStorage.php index a633806d..5eeaadfd 100644 --- a/src/Storage/DatabaseStorage.php +++ b/src/Storage/DatabaseStorage.php @@ -5,7 +5,6 @@ use Carbon\CarbonImmutable; use Carbon\CarbonInterval; use Closure; -use DateTimeInterface; use Illuminate\Contracts\Config\Repository; use Illuminate\Database\Connection; use Illuminate\Database\DatabaseManager; From 73ca6f849d42c430aee39e459dc7d086f81aad56 Mon Sep 17 00:00:00 2001 From: Taylor Otwell Date: Fri, 29 Nov 2024 10:10:18 -0600 Subject: [PATCH 5/5] Update DatabaseStorage.php --- src/Storage/DatabaseStorage.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storage/DatabaseStorage.php b/src/Storage/DatabaseStorage.php index 5eeaadfd..a3213320 100644 --- a/src/Storage/DatabaseStorage.php +++ b/src/Storage/DatabaseStorage.php @@ -123,9 +123,10 @@ public function store(Collection $items): void */ public function trim(): void { - $keep = $this->config->get('pulse.ingest.trim.keep'); $now = CarbonImmutable::now(); + $keep = $this->config->get('pulse.ingest.trim.keep'); + $this->connection() ->table('pulse_values') ->where('timestamp', '<=', $now->sub($keep)->getTimestamp())