diff --git a/.unreleased/feature_6195 b/.unreleased/feature_6195 new file mode 100644 index 00000000000..a930c0a7bcc --- /dev/null +++ b/.unreleased/feature_6195 @@ -0,0 +1 @@ +Implements: #6195 Restart scheduler on error diff --git a/src/bgw/scheduler.c b/src/bgw/scheduler.c index d44fc59d433..b11d11d43b7 100644 --- a/src/bgw/scheduler.c +++ b/src/bgw/scheduler.c @@ -879,6 +879,7 @@ ts_bgw_scheduler_process(int32 run_for_interval_ms, wait_for_all_jobs_to_shutdown(); check_for_stopped_and_timed_out_jobs(); scheduled_jobs = NIL; + proc_exit(ts_debug_bgw_scheduler_exit_status); } static void diff --git a/src/guc.c b/src/guc.c index 861760c17e2..1c17b83a7af 100644 --- a/src/guc.c +++ b/src/guc.c @@ -183,6 +183,20 @@ bool ts_guc_debug_require_batch_sorted_merge = false; bool ts_guc_debug_allow_cagg_with_deprecated_funcs = false; +/* + * Exit code for the scheduler. + * + * Normally it exits with a zero which means that it will not restart. If an + * error is raised, it exits with error code 1, which will trigger a + * restart. + * + * This variable exists to be able to trigger a restart for a normal exit, + * which is useful when debugging. + * + * See backend/postmaster/bgworker.c + */ +int ts_debug_bgw_scheduler_exit_status = 0; + #ifdef TS_DEBUG bool ts_shutdown_bgw = false; char *ts_current_timestamp_mock = NULL; @@ -1067,6 +1081,19 @@ _guc_init(void) /* assign_hook= */ NULL, /* show_hook= */ NULL); + DefineCustomIntVariable(/* name= */ MAKE_EXTOPTION("debug_bgw_scheduler_exit_status"), + /* short_desc= */ "exit status to use when shutting down the scheduler", + /* long_desc= */ "this is for debugging purposes", + /* valueAddr= */ &ts_debug_bgw_scheduler_exit_status, + /* bootValue= */ 0, + /* minValue= */ 0, + /* maxValue= */ 255, + /* context= */ PGC_SIGHUP, + /* flags= */ 0, + /* check_hook= */ NULL, + /* assign_hook= */ NULL, + /* show_hook= */ NULL); + DefineCustomStringVariable(/* name= */ MAKE_EXTOPTION("current_timestamp_mock"), /* short_desc= */ "set the current timestamp", /* long_desc= */ "this is for debugging purposes", diff --git a/src/guc.h b/src/guc.h index 34ebc0ef2d6..88d4ba18e39 100644 --- a/src/guc.h +++ b/src/guc.h @@ -92,6 +92,14 @@ extern TSDLLEXPORT bool ts_guc_auto_sparse_indexes; extern TSDLLEXPORT bool ts_guc_enable_columnarscan; extern TSDLLEXPORT int ts_guc_bgw_log_level; +/* + * Exit code to use when scheduler exits. + * + * Mostly used for debugging, but defined also for non-debug builds since that + * simplifies the code (and also simplifies debugging non-debug builds). + */ +extern TSDLLEXPORT int ts_debug_bgw_scheduler_exit_status; + #ifdef TS_DEBUG extern bool ts_shutdown_bgw; extern char *ts_current_timestamp_mock; diff --git a/src/loader/bgw_launcher.c b/src/loader/bgw_launcher.c index af2a2cc3e13..b599c8e3962 100644 --- a/src/loader/bgw_launcher.c +++ b/src/loader/bgw_launcher.c @@ -23,6 +23,7 @@ #include #include #include +#include #include /* and checking db list for whether we're in a template*/ @@ -40,11 +41,15 @@ /* for allocating the htab storage */ #include +#include +#include + /* for getting settings correct before loading the versioned scheduler */ #include "catalog/pg_db_role_setting.h" #include "../compat/compat.h" #include "../extension_constants.h" +#include "../utils.h" #include "bgw_counter.h" #include "bgw_launcher.h" #include "bgw_message_queue.h" @@ -84,6 +89,8 @@ typedef enum SchedulerState static volatile sig_atomic_t got_SIGHUP = false; +int ts_guc_bgw_scheduler_restart_time_sec = BGW_DEFAULT_RESTART_INTERVAL; + static void launcher_sighup(SIGNAL_ARGS) { @@ -124,6 +131,7 @@ typedef struct DbHashEntry } DbHashEntry; static void scheduler_state_trans_enabled_to_allocated(DbHashEntry *entry); +static void scheduler_modify_state(DbHashEntry *entry, SchedulerState new_state); static void bgw_on_postmaster_death(void) @@ -238,13 +246,27 @@ terminate_background_worker(BackgroundWorkerHandle *handle) } extern void -ts_bgw_cluster_launcher_register(void) +ts_bgw_cluster_launcher_init(void) { BackgroundWorker worker; + DefineCustomIntVariable(/* name= */ MAKE_EXTOPTION("bgw_scheduler_restart_time"), + /* short_desc= */ "Restart time for scheduler in seconds", + /* long_desc= */ + "The number of seconds until the scheduler restart on failure.", + /* valueAddr= */ &ts_guc_bgw_scheduler_restart_time_sec, + /* bootValue= */ BGW_DEFAULT_RESTART_INTERVAL, + /* minValue= */ 1, + /* maxValue= */ 3600, + /* context= */ PGC_SIGHUP, + /* flags= */ GUC_UNIT_S, + /* check_hook= */ NULL, + /* assign_hook= */ NULL, + /* show_hook= */ NULL); + memset(&worker, 0, sizeof(worker)); /* set up worker settings for our main worker */ - snprintf(worker.bgw_name, BGW_MAXLEN, "TimescaleDB Background Worker Launcher"); + snprintf(worker.bgw_name, BGW_MAXLEN, TS_BGW_TYPE_LAUNCHER); worker.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; worker.bgw_restart_time = BGW_LAUNCHER_RESTART_TIME_S; @@ -274,9 +296,10 @@ register_entrypoint_for_db(Oid db_id, VirtualTransactionId vxid, BackgroundWorke BackgroundWorker worker; memset(&worker, 0, sizeof(worker)); - snprintf(worker.bgw_name, BGW_MAXLEN, "TimescaleDB Background Worker Scheduler"); + snprintf(worker.bgw_type, BGW_MAXLEN, TS_BGW_TYPE_SCHEDULER); + snprintf(worker.bgw_name, BGW_MAXLEN, "%s for database %d", TS_BGW_TYPE_SCHEDULER, db_id); worker.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; - worker.bgw_restart_time = BGW_NEVER_RESTART; + worker.bgw_restart_time = ts_guc_bgw_scheduler_restart_time_sec, worker.bgw_start_time = BgWorkerStart_RecoveryFinished; snprintf(worker.bgw_library_name, BGW_MAXLEN, EXTENSION_NAME); snprintf(worker.bgw_function_name, BGW_MAXLEN, BGW_ENTRYPOINT_FUNCNAME); @@ -304,6 +327,20 @@ init_database_htab(void) HASH_BLOBS | HASH_CONTEXT | HASH_ELEM); } +static DbHashEntry * +db_hash_entry_update(HTAB *db_htab, Oid db_oid, SchedulerState state) +{ + bool found; + DbHashEntry *entry = (DbHashEntry *) hash_search(db_htab, &db_oid, HASH_FIND, &found); + if (!found) + { + elog(LOG, "could not find database entry for %d", db_oid); + return NULL; + } + scheduler_modify_state(entry, state); + return entry; +} + /* Insert a scheduler entry into the hash table. Correctly set entry values. */ static DbHashEntry * db_hash_entry_create_if_not_exists(HTAB *db_htab, Oid db_oid) @@ -332,15 +369,43 @@ db_hash_entry_create_if_not_exists(HTAB *db_htab, Oid db_oid) return db_he; } +/* + * Update the database hash table with information from the registered + * background worker list. + * + * This list contains background workers that are currently assigned a slot + * and are either running or will soon be running. + * + * In either case, we should not start new schedulers if there are old ones + * available from a previous launcher and they have not terminated, so we set + * the state to "STARTED" since it already has a slot. + * + * We do not update the state if it is terminated because the postmaster will + * reclaim this slot later, so we instead create a new scheduler. + */ +static void +update_database_htab(HTAB *db_htab) +{ + slist_iter siter; + slist_foreach(siter, &BackgroundWorkerList) + { + RegisteredBgWorker *rw = slist_container(RegisteredBgWorker, rw_lnode, siter.cur); + + /* We store the database id in the main arg for schedulers, so we can + * fetch the database OID from there. */ + if (strcmp(rw->rw_worker.bgw_type, TS_BGW_TYPE_SCHEDULER) == 0 && !rw->rw_terminate) + db_hash_entry_update(db_htab, rw->rw_worker.bgw_main_arg, STARTED); + } +} + /* * Model this on autovacuum.c -> get_database_list. * - * Note that we are not doing - * all the things around memory context that they do, because the hashtable - * we're using to store db entries is automatically created in its own memory - * context (a child of TopMemoryContext) This can get called at two different - * times 1) when the cluster launcher starts and is looking for dbs and 2) if - * it restarts due to a postmaster signal. + * Note that we are not doing all the things around memory context that they + * do, because the hashtable we're using to store db entries is automatically + * created in its own memory context (a child of TopMemoryContext) This can + * get called at two different times 1) when the cluster launcher starts and + * is looking for dbs and 2) if it restarts due to a postmaster signal. */ static void populate_database_htab(HTAB *db_htab) @@ -758,6 +823,7 @@ ts_bgw_cluster_launcher_main(PG_FUNCTION_ARGS) *htab_storage = db_htab; populate_database_htab(db_htab); + update_database_htab(db_htab); while (true) { diff --git a/src/loader/bgw_launcher.h b/src/loader/bgw_launcher.h index f90a65cb3a9..82c2ec1893b 100644 --- a/src/loader/bgw_launcher.h +++ b/src/loader/bgw_launcher.h @@ -8,7 +8,12 @@ #include #include -extern void ts_bgw_cluster_launcher_register(void); +#define TS_BGW_TYPE_LAUNCHER "TimescaleDB Background Worker Launcher" +#define TS_BGW_TYPE_SCHEDULER "TimescaleDB Background Worker Scheduler" + +extern int ts_guc_bgw_scheduler_restart_time_sec; + +extern void ts_bgw_cluster_launcher_init(void); /*called by postmaster at launcher bgw startup*/ TSDLLEXPORT extern Datum ts_bgw_cluster_launcher_main(PG_FUNCTION_ARGS); diff --git a/src/loader/loader.c b/src/loader/loader.c index 6537e49d8c5..fdff65236fa 100644 --- a/src/loader/loader.c +++ b/src/loader/loader.c @@ -591,7 +591,7 @@ _PG_init(void) timescaledb_shmem_request_hook(); #endif - ts_bgw_cluster_launcher_register(); + ts_bgw_cluster_launcher_init(); ts_bgw_counter_setup_gucs(); ts_bgw_interface_register_api_version(); diff --git a/tsl/test/expected/bgw_scheduler_restart.out b/tsl/test/expected/bgw_scheduler_restart.out new file mode 100644 index 00000000000..87214d3146e --- /dev/null +++ b/tsl/test/expected/bgw_scheduler_restart.out @@ -0,0 +1,85 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +\c :TEST_DBNAME :ROLE_SUPERUSER +CREATE VIEW tsdb_bgw AS + SELECT datname, pid, backend_type, application_name + FROM pg_stat_activity + WHERE application_name LIKE '%TimescaleDB%' + ORDER BY datname, backend_type, application_name; +-- Show the default scheduler restart time +SHOW timescaledb.bgw_scheduler_restart_time; + timescaledb.bgw_scheduler_restart_time +---------------------------------------- + 1min +(1 row) + +SELECT _timescaledb_functions.start_background_workers(); + start_background_workers +-------------------------- + t +(1 row) + +SELECT pg_sleep(10); -- Wait for scheduler to start. + pg_sleep +---------- + +(1 row) + +SELECT datname, application_name FROM tsdb_bgw; + datname | application_name +--------------------------+----------------------------------------- + db_bgw_scheduler_restart | TimescaleDB Background Worker Scheduler + | TimescaleDB Background Worker Launcher +(2 rows) + +ALTER SYSTEM SET timescaledb.shutdown_bgw_scheduler TO 'on'; +ALTER SYSTEM SET timescaledb.debug_bgw_scheduler_exit_status TO 1; +SELECT pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + +SELECT pg_sleep(20); -- Wait for scheduler to exit. + pg_sleep +---------- + +(1 row) + +SELECT datname, application_name FROM tsdb_bgw; + datname | application_name +---------+---------------------------------------- + | TimescaleDB Background Worker Launcher +(1 row) + +ALTER SYSTEM RESET timescaledb.shutdown_bgw_scheduler; +ALTER SYSTEM RESET timescaledb.debug_bgw_scheduler_exit_status; +SELECT pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + +SELECT pg_sleep(60); -- Wait for scheduler to restart. + pg_sleep +---------- + +(1 row) + +SELECT datname, application_name FROM tsdb_bgw; + datname | application_name +--------------------------+----------------------------------------- + db_bgw_scheduler_restart | TimescaleDB Background Worker Scheduler + | TimescaleDB Background Worker Launcher +(2 rows) + +SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = :'TEST_DBNAME' + AND application_name LIKE 'TimescaleDB%'; + pg_terminate_backend +---------------------- + t +(1 row) + diff --git a/tsl/test/sql/CMakeLists.txt b/tsl/test/sql/CMakeLists.txt index 041f9300c10..625b8c88b8c 100644 --- a/tsl/test/sql/CMakeLists.txt +++ b/tsl/test/sql/CMakeLists.txt @@ -69,51 +69,55 @@ if(CMAKE_BUILD_TYPE MATCHES Debug) TEST_FILES bgw_custom.sql bgw_db_scheduler.sql + bgw_db_scheduler_fixed.sql bgw_job_stat_history.sql bgw_job_stat_history_errors.sql bgw_job_stat_history_errors_permissions.sql - bgw_db_scheduler_fixed.sql bgw_reorder_drop_chunks.sql - scheduler_fixed.sql - compress_bgw_reorder_drop_chunks.sql + bgw_scheduler_control.sql + bgw_scheduler_restart.sql + cagg_bgw_drop_chunks.sql + cagg_drop_chunks.sql + cagg_dump.sql + cagg_joins.sql + cagg_migrate.sql + cagg_multi.sql + cagg_on_cagg.sql + cagg_on_cagg_joins.sql + cagg_policy_run.sql + cagg_tableam.sql chunk_api.sql chunk_merge.sql chunk_utils_compression.sql chunk_utils_internal.sql + compress_bgw_reorder_drop_chunks.sql compression_algos.sql compression_bgw.sql compression_ddl.sql compression_hypertable.sql - compression_merge.sql compression_indexscan.sql + compression_merge.sql compression_segment_meta.sql compression_sorted_merge_filter.sql - cagg_bgw_drop_chunks.sql - cagg_drop_chunks.sql - cagg_dump.sql - cagg_joins.sql - cagg_migrate.sql - cagg_multi.sql - cagg_on_cagg.sql - cagg_on_cagg_joins.sql - cagg_tableam.sql - cagg_policy_run.sql decompress_memory.sql decompress_vector_qual.sql exp_cagg_monthly.sql exp_cagg_next_gen.sql exp_cagg_origin.sql exp_cagg_timezone.sql + feature_flags.sql + fixed_schedules.sql hypertable_generalization.sql - insert_memory_usage.sql information_view_chunk_count.sql + insert_memory_usage.sql + job_errors_permissions.sql + license_tsl.sql read_only.sql + recompress_chunk_segmentwise.sql + scheduler_fixed.sql transparent_decompression_queries.sql + troubleshooting_job_errors.sql tsl_tables.sql - license_tsl.sql - fixed_schedules.sql - recompress_chunk_segmentwise.sql - feature_flags.sql vector_agg_default.sql vector_agg_memory.sql vector_agg_segmentby.sql) @@ -181,6 +185,7 @@ set(SOLO_TESTS # log level. bgw_custom bgw_scheduler_control + bgw_scheduler_restart bgw_db_scheduler bgw_job_stat_history_errors_permissions bgw_job_stat_history_errors diff --git a/tsl/test/sql/bgw_scheduler_restart.sql b/tsl/test/sql/bgw_scheduler_restart.sql new file mode 100644 index 00000000000..0f2f7cb27e9 --- /dev/null +++ b/tsl/test/sql/bgw_scheduler_restart.sql @@ -0,0 +1,41 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. + +\c :TEST_DBNAME :ROLE_SUPERUSER + +CREATE VIEW tsdb_bgw AS + SELECT datname, pid, backend_type, application_name + FROM pg_stat_activity + WHERE application_name LIKE '%TimescaleDB%' + ORDER BY datname, backend_type, application_name; + +-- Show the default scheduler restart time +SHOW timescaledb.bgw_scheduler_restart_time; + +SELECT _timescaledb_functions.start_background_workers(); + +SELECT pg_sleep(10); -- Wait for scheduler to start. + +SELECT datname, application_name FROM tsdb_bgw; + +ALTER SYSTEM SET timescaledb.shutdown_bgw_scheduler TO 'on'; +ALTER SYSTEM SET timescaledb.debug_bgw_scheduler_exit_status TO 1; +SELECT pg_reload_conf(); + +SELECT pg_sleep(20); -- Wait for scheduler to exit. + +SELECT datname, application_name FROM tsdb_bgw; + +ALTER SYSTEM RESET timescaledb.shutdown_bgw_scheduler; +ALTER SYSTEM RESET timescaledb.debug_bgw_scheduler_exit_status; +SELECT pg_reload_conf(); + +SELECT pg_sleep(60); -- Wait for scheduler to restart. + +SELECT datname, application_name FROM tsdb_bgw; + +SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = :'TEST_DBNAME' + AND application_name LIKE 'TimescaleDB%';