Skip to content

Commit

Permalink
Schedule compression policy more often
Browse files Browse the repository at this point in the history 
By default, the compression policy is scheduled for every
chunk_time_interval / 2 in the current implementation, equal to three
days and twelve hours with our default settings. This schedule interval
was sufficient for previous versions of TimescaleDB. However, with the
introduction of features like mutable compression and ON CONFLICT .. DO
UPDATE queries, regular DML operations decompress data. To ensure that
modified data is compressed earlier, this patch reduces the schedule
interval of the compression policy to run at least every 12 hours.
  • Loading branch information
@jnidzwetzki
jnidzwetzki committed Sep 22, 2023
1 parent 3589d37 commit 683e2bc
Show file tree
Hide file tree
Showing 12 changed files with 253 additions and 42 deletions.
2 changes: 1 addition & 1 deletion scripts/docker-build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ build_timescaledb()
cd /build/debug \
&& git config --global --add safe.directory /src \
&& cmake -DGENERATE_DOWNGRADE_SCRIPT=${GENERATE_DOWNGRADE_SCRIPT} -DENABLE_DEBUG_UTILS=off -DUSE_OPENSSL=${USE_OPENSSL} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} /src \
&& make && make install \
&& make -j $(nproc) && make install \
&& echo \"shared_preload_libraries = 'timescaledb'\" >> /usr/local/share/postgresql/postgresql.conf.sample \
&& echo \"timescaledb.telemetry_level=off\" >> /usr/local/share/postgresql/postgresql.conf.sample \
&& cd / && rm -rf /build"
Expand Down
22 changes: 17 additions & 5 deletions test/sql/updates/setup.policies.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,21 +20,33 @@ SELECT

DO LANGUAGE PLPGSQL $$
DECLARE
ts_version TEXT;
ts_major INT;
ts_minor INT;
BEGIN

SELECT extversion INTO ts_version FROM pg_extension WHERE extname = 'timescaledb';
WITH timescale_version AS (
SELECT string_to_array(extversion,'.') AS v
FROM pg_extension
WHERE extname = 'timescaledb'
)
SELECT v[1], v[2]
INTO ts_major, ts_minor
FROM timescale_version;

PERFORM add_reorder_policy('policy_test_timestamptz','policy_test_timestamptz_time_idx');

-- some policy API functions got renamed for 2.0 so we need to make
-- sure to use the right name for the version
IF ts_version < '2.0.0' THEN
-- sure to use the right name for the version. The schedule_interval
-- parameter of add_compression_policy was introduced in 2.8.0
IF ts_major < 2 THEN
PERFORM add_drop_chunks_policy('policy_test_timestamptz','60d'::interval);
PERFORM add_compress_chunks_policy('policy_test_timestamptz','10d'::interval);
ELSE
ELSIF ts_major <= 2 AND ts_minor < 8 THEN
PERFORM add_retention_policy('policy_test_timestamptz','60d'::interval);
PERFORM add_compression_policy('policy_test_timestamptz','10d'::interval);
ELSE
PERFORM add_retention_policy('policy_test_timestamptz','60d'::interval);
PERFORM add_compression_policy('policy_test_timestamptz','10d'::interval, schedule_interval => '3 days 12:00:00'::interval);
END IF;
END
$$;
Expand Down
20 changes: 18 additions & 2 deletions tsl/src/bgw_policy/compression_api.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@
#define DEFAULT_RETRY_PERIOD \
DatumGetIntervalP(DirectFunctionCall3(interval_in, CStringGetDatum("1 hour"), InvalidOid, -1))

/* Default max schedule period for the compression policy is 12 hours. The actual schedule period
* will be chunk_interval/2 if the chunk_interval is < 12 hours. */
#define DEFAULT_MAX_SCHEDULE_PERIOD (int64)(12 * 3600 * 1000 * (int64) 1000)

static Hypertable *validate_compress_chunks_hypertable(Cache *hcache, Oid user_htoid,
bool *is_cagg);

Expand Down Expand Up @@ -248,8 +252,20 @@ policy_compression_add_internal(Oid user_rel_oid, Datum compress_after_datum,
if (dim && IS_TIMESTAMP_TYPE(ts_dimension_get_partition_type(dim)) &&
!user_defined_schedule_interval)
{
default_schedule_interval = DatumGetIntervalP(
ts_internal_to_interval_value(dim->fd.interval_length / 2, INTERVALOID));
int64 hypertable_schedule_interval = dim->fd.interval_length / 2;

/* On hypertables with a small chunk_time_interval, schedule the compression job more often
* than DEFAULT_MAX_SCHEDULE_PERIOD */
if (DEFAULT_MAX_SCHEDULE_PERIOD > hypertable_schedule_interval)
{
default_schedule_interval = DatumGetIntervalP(
ts_internal_to_interval_value(hypertable_schedule_interval, INTERVALOID));
}
else
{
default_schedule_interval = DatumGetIntervalP(
ts_internal_to_interval_value(DEFAULT_MAX_SCHEDULE_PERIOD, INTERVALOID));
}
}

/* insert a new job into jobs table */
Expand Down
8 changes: 4 additions & 4 deletions tsl/test/expected/bgw_custom.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,7 @@ order by id;
select t.schedule_interval FROM alter_job(:job_id_4, next_start=> now() ) t;
schedule_interval
-------------------
@ 7 days 12 hours
@ 12 hours
(1 row)

SELECT wait_for_job_to_run(:job_id_4, 2);
Expand Down Expand Up @@ -917,9 +917,9 @@ ALTER TABLE sensor_data SET (timescaledb.compress, timescaledb.compress_orderby
SELECT add_compression_policy('sensor_data', INTERVAL '1' minute) AS compressjob_id \gset
-- set recompress to true
SELECT alter_job(id,config:=jsonb_set(config,'{recompress}', 'true')) FROM _timescaledb_config.bgw_job WHERE id = :compressjob_id;
alter_job
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
(1014,"@ 3 days 12 hours","@ 0",-1,"@ 1 hour",t,"{""recompress"": true, ""hypertable_id"": 4, ""compress_after"": ""@ 1 min""}",-infinity,_timescaledb_functions.policy_compression_check,f,,)
alter_job
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
(1014,"@ 12 hours","@ 0",-1,"@ 1 hour",t,"{""recompress"": true, ""hypertable_id"": 4, ""compress_after"": ""@ 1 min""}",-infinity,_timescaledb_functions.policy_compression_check,f,,)
(1 row)

-- create new chunks
Expand Down
8 changes: 4 additions & 4 deletions tsl/test/expected/bgw_db_scheduler_fixed.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1696,7 +1696,7 @@ select * from _timescaledb_config.bgw_job;
id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone
------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+------------+-----------+----------------+----------------------------------+---------------+--------------------------------------------------------------------------------+------------------------+-------------------------------------------+---------------
1026 | Retention Policy [1026] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"drop_after": "@ 2 years", "hypertable_id": 1} | _timescaledb_functions | policy_retention_check | Europe/Berlin
1027 | Compression Policy [1027] | @ 15 days | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"hypertable_id": 1, "compress_after": "@ 1 year"} | _timescaledb_functions | policy_compression_check | Europe/Berlin
1027 | Compression Policy [1027] | @ 12 hours | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"hypertable_id": 1, "compress_after": "@ 1 year"} | _timescaledb_functions | policy_compression_check | Europe/Berlin
1028 | Refresh Continuous Aggregate Policy [1028] | @ 21 days | @ 0 | -1 | @ 21 days | _timescaledb_functions | policy_refresh_continuous_aggregate | super_user | t | t | Fri Dec 31 16:00:00.005 1999 PST | 2 | {"end_offset": "@ 2 mons", "start_offset": "@ 1 year", "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | Europe/Athens
(3 rows)

Expand All @@ -1711,7 +1711,7 @@ SELECT * from _timescaledb_internal.bgw_job_stat;
job_id | last_start | last_finish | next_start | last_successful_finish | last_run_success | total_runs | total_duration | total_duration_failures | total_successes | total_failures | total_crashes | consecutive_failures | consecutive_crashes | flags
--------+----------------------------------+----------------------------------+----------------------------------+----------------------------------+------------------+------------+----------------+-------------------------+-----------------+----------------+---------------+----------------------+---------------------+-------
1026 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST | t | 1 | @ 0 | @ 0 | 1 | 0 | 0 | 0 | 0 | 0
1027 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 15 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST | t | 1 | @ 0 | @ 0 | 1 | 0 | 0 | 0 | 0 | 0
1027 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 04:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST | t | 1 | @ 0 | @ 0 | 1 | 0 | 0 | 0 | 0 | 0
1028 | Fri Dec 31 16:00:00.005 1999 PST | Fri Dec 31 16:00:00.005 1999 PST | Fri Jan 21 16:00:00.005 2000 PST | Fri Dec 31 16:00:00.005 1999 PST | t | 1 | @ 0 | @ 0 | 1 | 0 | 0 | 0 | 0 | 0
(3 rows)

Expand Down Expand Up @@ -1780,7 +1780,7 @@ select * from _timescaledb_config.bgw_job order by id;
id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone
------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+------------+-----------+----------------+----------------------------------+---------------+--------------------------------------------------------------------------------+------------------------+-------------------------------------------+---------------
1026 | Retention Policy [1026] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"drop_after": "@ 2 years", "hypertable_id": 1} | _timescaledb_functions | policy_retention_check | Europe/Berlin
1027 | Compression Policy [1027] | @ 15 days | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"hypertable_id": 1, "compress_after": "@ 1 year"} | _timescaledb_functions | policy_compression_check | Europe/Berlin
1027 | Compression Policy [1027] | @ 12 hours | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"hypertable_id": 1, "compress_after": "@ 1 year"} | _timescaledb_functions | policy_compression_check | Europe/Berlin
1028 | Refresh Continuous Aggregate Policy [1028] | @ 21 days | @ 0 | -1 | @ 21 days | _timescaledb_functions | policy_refresh_continuous_aggregate | super_user | t | t | Fri Dec 31 16:00:00.005 1999 PST | 2 | {"end_offset": "@ 2 mons", "start_offset": "@ 1 year", "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | Europe/Athens
1029 | User-Defined Action [1029] | @ 7 mons | @ 0 | -1 | @ 5 mins | public | job_test_fixed | super_user | t | t | Fri Dec 31 16:00:00.01 1999 PST | | | | |
1030 | User-Defined Action [1030] | @ 7 mons | @ 0 | -1 | @ 5 mins | public | job_test_fixed | super_user | t | t | Fri Dec 31 16:00:00.01 1999 PST | | | | | Europe/Athens
Expand All @@ -1798,7 +1798,7 @@ ORDER BY job_id;
job_id | last_start | last_finish | next_start | last_successful_finish
--------+------------------------------+------------------------------+------------------------------+------------------------------
1026 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST
1027 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 15 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST
1027 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 04:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST
1028 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Fri Jan 21 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST
1029 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Mon Jul 31 16:00:00 2000 PDT | Fri Dec 31 16:00:00 1999 PST
1030 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Mon Jul 31 16:00:00 2000 PDT | Fri Dec 31 16:00:00 1999 PST
Expand Down
10 changes: 5 additions & 5 deletions tsl/test/expected/bgw_policy.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -714,11 +714,11 @@ select create_hypertable('test_missing_schedint_integer', 'time', chunk_time_int
alter table test_missing_schedint_integer set (timescaledb.compress);
select add_compression_policy('test_missing_schedint_integer', BIGINT '600000') as compression_id_integer \gset
select * from _timescaledb_config.bgw_job where id in (:retenion_id_missing_schedint, :compression_id_missing_schedint, :compression_id_integer);
id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone
------+---------------------------+--------------------+-------------+-------------+--------------+------------------------+--------------------+---------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------+------------------------+--------------------------+----------
1008 | Retention Policy [1008] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | default_perm_user_2 | t | f | | 8 | {"drop_after": "@ 14 days", "hypertable_id": 8} | _timescaledb_functions | policy_retention_check |
1009 | Compression Policy [1009] | @ 15 days 12 hours | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | default_perm_user_2 | t | f | | 8 | {"hypertable_id": 8, "compress_after": "@ 60 days"} | _timescaledb_functions | policy_compression_check |
1010 | Compression Policy [1010] | @ 1 day | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | default_perm_user_2 | t | f | | 10 | {"hypertable_id": 10, "compress_after": 600000} | _timescaledb_functions | policy_compression_check |
id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone
------+---------------------------+-------------------+-------------+-------------+--------------+------------------------+--------------------+---------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------+------------------------+--------------------------+----------
1008 | Retention Policy [1008] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | default_perm_user_2 | t | f | | 8 | {"drop_after": "@ 14 days", "hypertable_id": 8} | _timescaledb_functions | policy_retention_check |
1009 | Compression Policy [1009] | @ 12 hours | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | default_perm_user_2 | t | f | | 8 | {"hypertable_id": 8, "compress_after": "@ 60 days"} | _timescaledb_functions | policy_compression_check |
1010 | Compression Policy [1010] | @ 1 day | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | default_perm_user_2 | t | f | | 10 | {"hypertable_id": 10, "compress_after": 600000} | _timescaledb_functions | policy_compression_check |
(3 rows)

-- test policy check functions with NULL args
Expand Down
Loading

0 comments on commit 683e2bc

Please sign in to comment.