Skip to content

Commit

Permalink
Streaming improvements No 6 (netdata#19196)
Browse files Browse the repository at this point in the history
* pulse aral stats fixes

* double is not needed there

* control the number of pages evicted at once, to avoid gaps on data collection

* lower the aggresiveness of evictions to prevent evicting signifanctly more pages than really needed

* add aral judy statistics to pulse aral

* metadata sync shutsdown in the background

* renumber

* when just 1 host is still pending, log also its hostname

* call mallocz_release_as_much_memory_to_the_system() at most once per second

* improve ML logs and move metadata_sync_shutdown() at the proper place

* stack size of external plugins; moved all function evloop data off the stack

* fix for incorrect replication accounting

* metadata cmds in aral

* fix function payload parsing for external plugins

* split main service control and netdata shutdown to separate files - no code changes

* add more information to shutdown watcher

* fix exit log messages

* parallel dbengine exits for all tiers

* log with limit

* alerts not alarms

* minor fixes in health

* added logs to trace the root cause of delays when closing data files

* added STORAGE_PRIORITY_SYNCHRONOUS_FIRST

* Revert "added logs to trace the root cause of delays when closing data files"

This reverts commit 75515c1.

* log datafile lock wait

* print the number of writers

* print the number of writers again

* print the number of writers again again

* fix watcher messages

* single node agents use 1 replication and 1 ML thread; max threads for both is 256

* log the progress of flushers

* spawn intense flushing on quiesce

* cleanup logs added

* print the status of the cache on exit

* dbengine shutdown cleanup

* cleanup logs

* cleanup logs

* more cleanup on logs

* proper percentage calculation

* sentry include added
  • Loading branch information
ktsaou authored Dec 13, 2024
1 parent f8bc77e commit 4179fa8
Show file tree
Hide file tree
Showing 69 changed files with 1,016 additions and 904 deletions.
8 changes: 6 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1051,8 +1051,8 @@ set(DAEMON_FILES
src/daemon/signals.c
src/daemon/signals.h
src/daemon/service.c
src/daemon/watcher.c
src/daemon/watcher.h
src/daemon/daemon-shutdown-watcher.c
src/daemon/daemon-shutdown-watcher.h
src/daemon/static_threads.c
src/daemon/static_threads.h
src/daemon/commands.c
Expand Down Expand Up @@ -1114,6 +1114,10 @@ set(DAEMON_FILES
src/daemon/config/netdata-conf-global.c
src/daemon/config/netdata-conf-global.h
src/daemon/config/netdata-conf.c
src/daemon/daemon-shutdown.c
src/daemon/daemon-shutdown.h
src/daemon/daemon-service.c
src/daemon/daemon-service.h
)

set(H2O_FILES
Expand Down
1 change: 1 addition & 0 deletions src/collectors/apps.plugin/apps_plugin.c
Original file line number Diff line number Diff line change
Expand Up @@ -666,6 +666,7 @@ static bool apps_plugin_exit = false;

int main(int argc, char **argv) {
nd_log_initialize_for_external_plugins("apps.plugin");
netdata_threads_init_for_external_plugins(0);

pagesize = (size_t)sysconf(_SC_PAGESIZE);

Expand Down
1 change: 0 additions & 1 deletion src/collectors/cgroups.plugin/sys_fs_cgroup.c
Original file line number Diff line number Diff line change
Expand Up @@ -1327,7 +1327,6 @@ static void cgroup_main_cleanup(void *pptr) {

static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;

collector_info("cleaning up...");
worker_unregister();

usec_t max = 2 * USEC_PER_SEC, step = 50000;
Expand Down
1 change: 1 addition & 0 deletions src/collectors/cups.plugin/cups_plugin.c
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ void reset_metrics() {

int main(int argc, char **argv) {
nd_log_initialize_for_external_plugins("cups.plugin");
netdata_threads_init_for_external_plugins(0);

parse_command_line(argc, argv);

Expand Down
1 change: 1 addition & 0 deletions src/collectors/debugfs.plugin/debugfs_plugin.c
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ static void debugfs_parse_args(int argc, char **argv)
int main(int argc, char **argv)
{
nd_log_initialize_for_external_plugins("debugfs.plugin");
netdata_threads_init_for_external_plugins(0);

netdata_configured_host_prefix = getenv("NETDATA_HOST_PREFIX");
if (verify_netdata_host_prefix(true) == -1)
Expand Down
4 changes: 0 additions & 4 deletions src/collectors/diskspace.plugin/plugin_diskspace.c
Original file line number Diff line number Diff line change
Expand Up @@ -516,8 +516,6 @@ static void diskspace_slow_worker_cleanup(void *pptr) {
struct slow_worker_data *data = CLEANUP_FUNCTION_GET_PTR(pptr);
if(data) return;
collector_info("cleaning up...");
worker_unregister();
}
Expand Down Expand Up @@ -608,8 +606,6 @@ static void diskspace_main_cleanup(void *pptr) {
static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
collector_info("cleaning up...");
rrd_collector_finished();
worker_unregister();
Expand Down
1 change: 1 addition & 0 deletions src/collectors/ebpf.plugin/ebpf.c
Original file line number Diff line number Diff line change
Expand Up @@ -4006,6 +4006,7 @@ static void ebpf_manage_pid(pid_t pid)
int main(int argc, char **argv)
{
nd_log_initialize_for_external_plugins(NETDATA_EBPF_PLUGIN_NAME);
netdata_threads_init_for_external_plugins(0);

ebpf_set_global_variables();
if (ebpf_can_plugin_load_code(running_on_kernel, NETDATA_EBPF_PLUGIN_NAME))
Expand Down
1 change: 0 additions & 1 deletion src/collectors/freebsd.plugin/plugin_freebsd.c
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ static void freebsd_main_cleanup(void *pptr)

static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;

collector_info("cleaning up...");
worker_unregister();

static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
Expand Down
1 change: 0 additions & 1 deletion src/collectors/idlejitter.plugin/plugin_idlejitter.c
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ static void cpuidlejitter_main_cleanup(void *pptr) {

static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;

collector_info("cleaning up...");
worker_unregister();

static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
Expand Down
1 change: 0 additions & 1 deletion src/collectors/macos.plugin/plugin_macos.c
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ static void macos_main_cleanup(void *pptr)

static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;

collector_info("cleaning up...");
worker_unregister();

static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
Expand Down
1 change: 1 addition & 0 deletions src/collectors/network-viewer.plugin/network-viewer.c
Original file line number Diff line number Diff line change
Expand Up @@ -960,6 +960,7 @@ void network_viewer_function(const char *transaction, char *function __maybe_unu
int main(int argc __maybe_unused, char **argv __maybe_unused) {
nd_thread_tag_set("NETWORK-VIEWER");
nd_log_initialize_for_external_plugins("network-viewer.plugin");
netdata_threads_init_for_external_plugins(0);

netdata_configured_host_prefix = getenv("NETDATA_HOST_PREFIX");
if(verify_netdata_host_prefix(true) == -1) exit(1);
Expand Down
1 change: 1 addition & 0 deletions src/collectors/nfacct.plugin/plugin_nfacct.c
Original file line number Diff line number Diff line change
Expand Up @@ -748,6 +748,7 @@ void nfacct_signals()

int main(int argc, char **argv) {
nd_log_initialize_for_external_plugins("nfacct.plugin");
netdata_threads_init_for_external_plugins(0);

// ------------------------------------------------------------------------
// parse command line parameters
Expand Down
1 change: 1 addition & 0 deletions src/collectors/perf.plugin/perf_plugin.c
Original file line number Diff line number Diff line change
Expand Up @@ -1288,6 +1288,7 @@ void parse_command_line(int argc, char **argv) {

int main(int argc, char **argv) {
nd_log_initialize_for_external_plugins("perf.plugin");
netdata_threads_init_for_external_plugins(0);

parse_command_line(argc, argv);

Expand Down
2 changes: 0 additions & 2 deletions src/collectors/proc.plugin/plugin_proc.c
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,6 @@ static void proc_main_cleanup(void *pptr)

static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;

collector_info("cleaning up...");

nd_thread_join(netdev_thread);
worker_unregister();

Expand Down
2 changes: 0 additions & 2 deletions src/collectors/proc.plugin/proc_net_dev.c
Original file line number Diff line number Diff line change
Expand Up @@ -1685,8 +1685,6 @@ static void netdev_main_cleanup(void *pptr) {
if(CLEANUP_FUNCTION_GET_PTR(pptr) != (void *)0x01)
return;

collector_info("cleaning up...");

worker_unregister();
}

Expand Down
2 changes: 0 additions & 2 deletions src/collectors/profile.plugin/plugin_profile.cc
Original file line number Diff line number Diff line change
Expand Up @@ -186,8 +186,6 @@ static void profile_main_cleanup(void *pptr) {

static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;

netdata_log_info("cleaning up...");

static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
}

Expand Down
1 change: 1 addition & 0 deletions src/collectors/slabinfo.plugin/slabinfo.c
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,7 @@ void usage(void) {

int main(int argc, char **argv) {
nd_log_initialize_for_external_plugins("slabinfo.plugin");
netdata_threads_init_for_external_plugins(0);

program_name = argv[0];
int update_every = 1, i, n, freq = 0;
Expand Down
3 changes: 0 additions & 3 deletions src/collectors/statsd.plugin/statsd.c
Original file line number Diff line number Diff line change
Expand Up @@ -1086,8 +1086,6 @@ void statsd_collector_thread_cleanup(void *pptr) {
d->status->running = false;
spinlock_unlock(&d->status->spinlock);

collector_info("cleaning up...");

#ifdef HAVE_RECVMMSG
size_t i;
for (i = 0; i < d->size; i++)
Expand Down Expand Up @@ -2399,7 +2397,6 @@ static void statsd_main_cleanup(void *pptr) {
if(!static_thread) return;

static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
collector_info("cleaning up...");

if (statsd.collection_threads_status) {
int i;
Expand Down
1 change: 1 addition & 0 deletions src/collectors/systemd-journal.plugin/systemd-main.c
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ static bool journal_data_directories_exist() {
int main(int argc __maybe_unused, char **argv __maybe_unused) {
nd_thread_tag_set("sd-jrnl.plugin");
nd_log_initialize_for_external_plugins("systemd-journal.plugin");
netdata_threads_init_for_external_plugins(0);

netdata_configured_host_prefix = getenv("NETDATA_HOST_PREFIX");
if(verify_netdata_host_prefix(true) == -1) exit(1);
Expand Down
2 changes: 0 additions & 2 deletions src/collectors/tc.plugin/plugin_tc.c
Original file line number Diff line number Diff line change
Expand Up @@ -845,8 +845,6 @@ static void tc_main_cleanup(void *pptr) {

static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;

collector_info("cleaning up...");

if(tc_child_instance) {
collector_info("TC: stopping the running tc-qos-helper script");
int code = spawn_popen_wait(tc_child_instance); (void)code;
Expand Down
1 change: 0 additions & 1 deletion src/collectors/timex.plugin/plugin_timex.c
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ static void timex_main_cleanup(void *pptr)

static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;

netdata_log_info("cleaning up...");
worker_unregister();

static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
Expand Down
1 change: 1 addition & 0 deletions src/collectors/windows-events.plugin/windows-events.c
Original file line number Diff line number Diff line change
Expand Up @@ -1292,6 +1292,7 @@ void function_windows_events(const char *transaction, char *function, usec_t *st
int main(int argc __maybe_unused, char **argv __maybe_unused) {
nd_thread_tag_set("wevt.plugin");
nd_log_initialize_for_external_plugins("windows-events.plugin");
netdata_threads_init_for_external_plugins(0);

// ------------------------------------------------------------------------
// initialization
Expand Down
2 changes: 0 additions & 2 deletions src/collectors/windows.plugin/windows_plugin.c
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,6 @@ static void windows_main_cleanup(void *pptr) {

static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;

collector_info("cleaning up...");

static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;

worker_unregister();
Expand Down
1 change: 1 addition & 0 deletions src/collectors/xenstat.plugin/xenstat_plugin.c
Original file line number Diff line number Diff line change
Expand Up @@ -926,6 +926,7 @@ int main(int argc, char **argv) {
program_name = PLUGIN_XENSTAT_NAME;

nd_log_initialize_for_external_plugins(PLUGIN_XENSTAT_NAME);
netdata_threads_init_for_external_plugins(0);

// ------------------------------------------------------------------------
// parse command line parameters
Expand Down
1 change: 0 additions & 1 deletion src/daemon/analytics.c
Original file line number Diff line number Diff line change
Expand Up @@ -549,7 +549,6 @@ void analytics_main_cleanup(void *pptr)

static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;

netdata_log_debug(D_ANALYTICS, "Cleaning up...");
analytics_free_data();

static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
Expand Down
2 changes: 2 additions & 0 deletions src/daemon/config/netdata-conf.c
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

#include "netdata-conf.h"

struct config netdata_config = APPCONFIG_INITIALIZER;

bool netdata_conf_load(char *filename, char overwrite_used, const char **user) {
static bool run = false;
if(run) return false;
Expand Down
1 change: 1 addition & 0 deletions src/daemon/config/netdata-conf.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

#include "libnetdata/libnetdata.h"

extern struct config netdata_config;
bool netdata_conf_load(char *filename, char overwrite_used, const char **user);

#include "netdata-conf-backwards-compatibility.h"
Expand Down
Loading

0 comments on commit 4179fa8

Please sign in to comment.