Add logging for retention policy

Also remove unused code from compression_api. The function
policy_compression_get_verbose_log was unused. Moved it to
policy_utils and renamed to policy_get_verbose_log so that it can
be used by all policies.

CHANGELOG.md

@@ -8,6 +8,7 @@ accidentally triggering the load of a previous DB version.**
 
 **Features**
 * #4169 Add support for chunk exclusion on DELETE on PG14
+* #4120 Add logging for retention policy
 
 **Bugfixes**
 * #4122 Fix segfault on INSERT into distributed hypertable

tsl/src/bgw_policy/compression_api.c

@@ -65,15 +65,6 @@ policy_compression_get_maxchunks_per_job(const Jsonb *config)
 	return (found && maxchunks > 0) ? maxchunks : 0;
 }
 
-bool
-policy_compression_get_verbose_log(const Jsonb *config)
-{
-	bool found;
-	bool verbose_log = ts_jsonb_get_bool_field(config, CONFIG_KEY_VERBOSE_LOG, &found);
-
-	return found ? verbose_log : false;
-}
-
 int32
 policy_compression_get_hypertable_id(const Jsonb *config)
 {

tsl/src/bgw_policy/compression_api.h

@@ -24,6 +24,5 @@ bool policy_compression_get_recompress(const Jsonb *config);
 int32 policy_compression_get_maxchunks_per_job(const Jsonb *config);
 int64 policy_recompression_get_recompress_after_int(const Jsonb *config);
 Interval *policy_recompression_get_recompress_after_interval(const Jsonb *config);
-bool policy_compression_get_verbose_log(const Jsonb *config);
 
 #endif /* TIMESCALEDB_TSL_BGW_POLICY_COMPRESSION_API_H */

tsl/src/bgw_policy/job.c

@@ -55,6 +55,27 @@
 
 #define REORDER_SKIP_RECENT_DIM_SLICES_N 3
 
+static void
+log_retention_boundary(int elevel, PolicyRetentionData *policy_data, const char *message)
+{
+	char *relname;
+	Datum boundary;
+	Oid outfuncid = InvalidOid;
+	bool isvarlena;
+
+	getTypeOutputInfo(policy_data->boundary_type, &outfuncid, &isvarlena);
+
+	relname = get_rel_name(policy_data->object_relid);
+	boundary = policy_data->boundary;
+
+	if (OidIsValid(outfuncid))
+		elog(elevel,
+			 "%s \"%s\": dropping data older than %s",
+			 message,
+			 relname,
+			 DatumGetCString(OidFunctionCall1(outfuncid, boundary)));
+}
+
 static void
 enable_fast_restart(int32 job_id, const char *job_name)
 {
@@ -250,9 +271,14 @@ bool
 policy_retention_execute(int32 job_id, Jsonb *config)
 {
 	PolicyRetentionData policy_data;
+	bool verbose_log;
 
 	policy_retention_read_and_validate_config(config, &policy_data);
 
+	verbose_log = policy_get_verbose_log(config);
+	if (verbose_log)
+		log_retention_boundary(LOG, &policy_data, "applying retention policy to hypertable");
+
 	chunk_invoke_drop_chunks(policy_data.object_relid,
 							 policy_data.boundary,
 							 policy_data.boundary_type);
@@ -552,6 +578,13 @@ job_execute(BgwJob *job)
 	StringInfo query;
 	Portal portal = ActivePortal;
 
+	if (job->fd.config)
+		elog(DEBUG1,
+			 "Executing %s with parameters %s",
+			 NameStr(job->fd.proc_name),
+			 DatumGetCString(DirectFunctionCall1(jsonb_out, JsonbPGetDatum(job->fd.config))));
+	else
+		elog(DEBUG1, "Executing %s with no parameters", NameStr(job->fd.proc_name));
 	/* Create a portal if there's no active */
 	if (!PortalIsValid(portal))
 	{

tsl/src/bgw_policy/job.h

@@ -16,7 +16,7 @@
 #include "bgw_policy/chunk_stats.h"
 
 /* Add config keys common across job types here */
-#define CONFIG_KEY_VERBOSE_LOG "verbose_log" /*used only by compression now*/
+#define CONFIG_KEY_VERBOSE_LOG "verbose_log" /*used only by retention now*/
 
 typedef struct PolicyReorderData
 {

tsl/src/bgw_policy/policy_utils.c

@@ -132,3 +132,12 @@ get_open_dimension_for_hypertable(const Hypertable *ht)
 	}
 	return open_dim;
 }
+
+bool
+policy_get_verbose_log(const Jsonb *config)
+{
+	bool found;
+	bool verbose_log = ts_jsonb_get_bool_field(config, CONFIG_KEY_VERBOSE_LOG, &found);
+
+	return found ? verbose_log : false;
+}

tsl/src/bgw_policy/policy_utils.h

@@ -8,10 +8,12 @@
 #define TIMESCALEDB_TSL_BGW_POLICY_UTILS_H
 
 #include <postgres.h>
+#include "job.h"
 bool policy_config_check_hypertable_lag_equality(Jsonb *config, const char *json_label,
 												 Oid dim_type, Oid lag_type, Datum lag_datum);
 int64 subtract_integer_from_now_internal(int64 interval, Oid time_dim_type, Oid now_func,
 										 bool *overflow);
 Datum subtract_interval_from_now(Interval *interval, Oid time_dim_type);
 const Dimension *get_open_dimension_for_hypertable(const Hypertable *ht);
+bool policy_get_verbose_log(const Jsonb *config);
 #endif /* TIMESCALEDB_TSL_BGW_POLICY_UTILS_H */

tsl/test/expected/bgw_reorder_drop_chunks.out

@@ -574,3 +574,88 @@ SELECT * FROM timescaledb_information.job_stats;
  public            | test_drop_chunks_table |   1001 | Fri Dec 31 16:00:01 1999 PST | Fri Dec 31 16:00:01 1999 PST | Success         | Scheduled  |                   | Fri Dec 31 16:00:02 1999 PST |          2 |               2 |              0
 (1 row)
 
+-- Test that add_retention_policy also works with timestamp (without time zone) and date types
+-- and that the policy execution is being logged
+-- Test for date
+CREATE TABLE test_drop_chunks_table_date(time date, drop_order int);
+SELECT create_hypertable('test_drop_chunks_table_date', 'time', chunk_time_interval => INTERVAL '1 week');
+NOTICE:  adding not-null constraint to column "time"
+            create_hypertable             
+------------------------------------------
+ (3,public,test_drop_chunks_table_date,t)
+(1 row)
+
+INSERT INTO test_drop_chunks_table_date VALUES (now() - INTERVAL '2 month',  4);
+INSERT INTO test_drop_chunks_table_date VALUES (now(),                       5);
+INSERT INTO test_drop_chunks_table_date VALUES (now() - INTERVAL '6 months', 2);
+INSERT INTO test_drop_chunks_table_date VALUES (now() - INTERVAL '4 months', 3);
+INSERT INTO test_drop_chunks_table_date VALUES (now() - INTERVAL '8 months', 1);
+-- Clear the job stats and reset timer, this will also clear the bgw_log
+\c :TEST_DBNAME :ROLE_SUPERUSER
+TRUNCATE bgw_log;
+TRUNCATE _timescaledb_internal.bgw_job_stat;
+DELETE FROM _timescaledb_config.bgw_job;
+SELECT ts_bgw_params_reset_time();
+ ts_bgw_params_reset_time 
+--------------------------
+
+(1 row)
+
+\c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER
+-- Test for timestamp 
+CREATE TABLE test_drop_chunks_table_tsntz(time date, drop_order int);
+SELECT create_hypertable('test_drop_chunks_table_tsntz', 'time', chunk_time_interval => INTERVAL '1 week');
+NOTICE:  adding not-null constraint to column "time"
+             create_hypertable             
+-------------------------------------------
+ (4,public,test_drop_chunks_table_tsntz,t)
+(1 row)
+
+INSERT INTO test_drop_chunks_table_tsntz VALUES (now() - INTERVAL '2 month',  4);
+INSERT INTO test_drop_chunks_table_tsntz VALUES (now(),                       5);
+INSERT INTO test_drop_chunks_table_tsntz VALUES (now() - INTERVAL '6 months', 2);
+INSERT INTO test_drop_chunks_table_tsntz VALUES (now() - INTERVAL '4 months', 3);
+INSERT INTO test_drop_chunks_table_tsntz VALUES (now() - INTERVAL '8 months', 1);
+-- Add retention policies for both tables
+SELECT add_retention_policy('test_drop_chunks_table_date', INTERVAL '4 months') as drop_chunks_date_job_id \gset
+SELECT add_retention_policy('test_drop_chunks_table_tsntz', INTERVAL '4 months') as drop_chunks_tsntz_job_id \gset
+-- Test that retention policy is being logged
+SELECT alter_job(id,config:=jsonb_set(config,'{verbose_log}', 'true'))
+ FROM _timescaledb_config.bgw_job WHERE id = :drop_chunks_date_job_id;
+                                                              alter_job                                                              
+-------------------------------------------------------------------------------------------------------------------------------------
+ (1002,"@ 1 day","@ 5 mins",-1,"@ 5 mins",t,"{""drop_after"": ""@ 4 mons"", ""verbose_log"": true, ""hypertable_id"": 3}",-infinity)
+(1 row)
+
+SELECT alter_job(id,config:=jsonb_set(config,'{verbose_log}', 'true'))
+ FROM _timescaledb_config.bgw_job WHERE id = :drop_chunks_tsntz_job_id;
+                                                              alter_job                                                              
+-------------------------------------------------------------------------------------------------------------------------------------
+ (1003,"@ 1 day","@ 5 mins",-1,"@ 5 mins",t,"{""drop_after"": ""@ 4 mons"", ""verbose_log"": true, ""hypertable_id"": 4}",-infinity)
+(1 row)
+
+CALL run_job(:drop_chunks_date_job_id);
+SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(1000);
+ ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish 
+------------------------------------------------------------
+
+(1 row)
+
+CALL run_job(:drop_chunks_tsntz_job_id);
+SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(1000);
+ ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish 
+------------------------------------------------------------
+
+(1 row)
+
+SELECT * FROM sorted_bgw_log;
+ msg_no | mock_time |    application_name     |                                         msg                                         
+--------+-----------+-------------------------+-------------------------------------------------------------------------------------
+      0 |         0 | DB Scheduler            | [TESTING] Registered new background worker
+      1 |         0 | DB Scheduler            | [TESTING] Registered new background worker
+      2 |         0 | DB Scheduler            | [TESTING] Wait until 1000000, started at 0
+      0 |         0 | Retention Policy [1002] | applying retention policy to hypertable "test_drop_chunks_table_date": dropping ...
+      0 |         0 | Retention Policy [1003] | applying retention policy to hypertable "test_drop_chunks_table_tsntz": dropping...
+      0 |   1000000 | DB Scheduler            | [TESTING] Wait until 2000000, started at 1000000
+(6 rows)
+

tsl/test/sql/bgw_reorder_drop_chunks.sql

@@ -285,3 +285,55 @@ SELECT show_chunks('test_drop_chunks_table');
 
 --test that views work
 SELECT * FROM timescaledb_information.job_stats;
+
+
+-- Test that add_retention_policy also works with timestamp (without time zone) and date types
+-- and that the policy execution is being logged
+
+-- Test for date
+CREATE TABLE test_drop_chunks_table_date(time date, drop_order int);
+SELECT create_hypertable('test_drop_chunks_table_date', 'time', chunk_time_interval => INTERVAL '1 week');
+
+INSERT INTO test_drop_chunks_table_date VALUES (now() - INTERVAL '2 month',  4);
+INSERT INTO test_drop_chunks_table_date VALUES (now(),                       5);
+INSERT INTO test_drop_chunks_table_date VALUES (now() - INTERVAL '6 months', 2);
+INSERT INTO test_drop_chunks_table_date VALUES (now() - INTERVAL '4 months', 3);
+INSERT INTO test_drop_chunks_table_date VALUES (now() - INTERVAL '8 months', 1);
+
+-- Clear the job stats and reset timer, this will also clear the bgw_log
+\c :TEST_DBNAME :ROLE_SUPERUSER
+TRUNCATE bgw_log;
+TRUNCATE _timescaledb_internal.bgw_job_stat;
+DELETE FROM _timescaledb_config.bgw_job;
+SELECT ts_bgw_params_reset_time();
+\c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER
+
+
+-- Test for timestamp 
+CREATE TABLE test_drop_chunks_table_tsntz(time date, drop_order int);
+SELECT create_hypertable('test_drop_chunks_table_tsntz', 'time', chunk_time_interval => INTERVAL '1 week');
+
+INSERT INTO test_drop_chunks_table_tsntz VALUES (now() - INTERVAL '2 month',  4);
+INSERT INTO test_drop_chunks_table_tsntz VALUES (now(),                       5);
+INSERT INTO test_drop_chunks_table_tsntz VALUES (now() - INTERVAL '6 months', 2);
+INSERT INTO test_drop_chunks_table_tsntz VALUES (now() - INTERVAL '4 months', 3);
+INSERT INTO test_drop_chunks_table_tsntz VALUES (now() - INTERVAL '8 months', 1);
+
+-- Add retention policies for both tables
+SELECT add_retention_policy('test_drop_chunks_table_date', INTERVAL '4 months') as drop_chunks_date_job_id \gset
+SELECT add_retention_policy('test_drop_chunks_table_tsntz', INTERVAL '4 months') as drop_chunks_tsntz_job_id \gset
+
+-- Test that retention policy is being logged
+SELECT alter_job(id,config:=jsonb_set(config,'{verbose_log}', 'true'))
+ FROM _timescaledb_config.bgw_job WHERE id = :drop_chunks_date_job_id;
+
+SELECT alter_job(id,config:=jsonb_set(config,'{verbose_log}', 'true'))
+ FROM _timescaledb_config.bgw_job WHERE id = :drop_chunks_tsntz_job_id;
+
+CALL run_job(:drop_chunks_date_job_id);
+SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(1000);
+
+CALL run_job(:drop_chunks_tsntz_job_id);
+SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(1000);
+
+SELECT * FROM sorted_bgw_log;