diff --git a/collectors/apps.plugin/apps_groups.conf b/collectors/apps.plugin/apps_groups.conf index ff72349244c568..9e9d83436b1740 100644 --- a/collectors/apps.plugin/apps_groups.conf +++ b/collectors/apps.plugin/apps_groups.conf @@ -83,14 +83,14 @@ xenstat.plugin: xenstat.plugin perf.plugin: perf.plugin charts.d.plugin: *charts.d.plugin* python.d.plugin: *python.d.plugin* +systemd-journal.plugin:*systemd-journal.plugin* tc-qos-helper: *tc-qos-helper.sh* fping: fping ioping: ioping go.d.plugin: *go.d.plugin* -slabinfo.plugin: slabinfo.plugin +slabinfo.plugin: *slabinfo.plugin* ebpf.plugin: *ebpf.plugin* debugfs.plugin: *debugfs.plugin* -systemd-journal.plugin: systemd-journal # agent-service-discovery agent_sd: agent_sd @@ -137,7 +137,7 @@ modem: ModemManager netmanager: NetworkManager nm* systemd-networkd networkctl netplan connmand wicked* avahi-autoipd networkd-dispatcher firewall: firewalld ufw nft tor: tor -bluetooth: bluetooth bluez bluedevil obexd +bluetooth: bluetooth bluetoothd bluez bluedevil obexd # ----------------------------------------------------------------------------- # high availability and balancers @@ -160,7 +160,7 @@ chat: irssi *vines* *prosody* murmurd # ----------------------------------------------------------------------------- # monitoring -logs: ulogd* syslog* rsyslog* logrotate systemd-journald rotatelogs sysklogd metalog +logs: ulogd* syslog* rsyslog* logrotate *systemd-journal* rotatelogs sysklogd metalog nms: snmpd vnstatd smokeping zabbix* munin* mon openhpid tailon nrpe monit: monit splunk: splunkd @@ -210,7 +210,7 @@ proxmox-ve: pve* spiceproxy # ----------------------------------------------------------------------------- # containers & virtual machines -containers: lxc* docker* balena* +containers: lxc* docker* balena* containerd VMs: vbox* VBox* qemu* kvm* libvirt: virtlogd virtqemud virtstoraged virtnetworkd virtlockd virtinterfaced libvirt: virtnodedevd virtproxyd virtsecretd libvirtd @@ -239,7 +239,7 @@ dhcp: *dhcp* dhclient # ----------------------------------------------------------------------------- # name servers and clients -dns: named unbound nsd pdns_server knotd gdnsd yadifad dnsmasq systemd-resolve* pihole* avahi-daemon avahi-dnsconfd +dns: named unbound nsd pdns_server knotd gdnsd yadifad dnsmasq *systemd-resolve* pihole* avahi-daemon avahi-dnsconfd dnsdist: dnsdist # ----------------------------------------------------------------------------- @@ -272,7 +272,7 @@ backup: rsync lsyncd bacula* borg rclone # ----------------------------------------------------------------------------- # cron -cron: cron* atd anacron systemd-cron* incrond +cron: cron* atd anacron *systemd-cron* incrond # ----------------------------------------------------------------------------- # UPS @@ -320,7 +320,7 @@ airflow: *airflow* # ----------------------------------------------------------------------------- # GUI -X: X Xorg xinit xdm Xwayland xsettingsd +X: X Xorg xinit xdm Xwayland xsettingsd touchegg wayland: swaylock swayidle waypipe wayvnc kde: *kdeinit* kdm sddm plasmashell startplasma-* kwin* kwallet* krunner kactivitymanager* gnome: gnome-* gdm gconf* mutter @@ -354,11 +354,11 @@ kswapd: kswapd zswap: zswap kcompactd: kcompactd -system: systemd-* udisks* udevd* *udevd ipv6_addrconf dbus-* rtkit* +system: systemd* udisks* udevd* *udevd ipv6_addrconf dbus-* rtkit* system: mdadm acpid uuidd upowerd elogind* eudev mdev lvmpolld dmeventd system: accounts-daemon rngd haveged rasdaemon irqbalance start-stop-daemon system: supervise-daemon openrc* init runit runsvdir runsv auditd lsmd -system: abrt* nscd rtkit-daemon gpg-agent usbguard* +system: abrt* nscd rtkit-daemon gpg-agent usbguard* boltd geoclue kernel: kworker kthreadd kauditd lockd khelper kdevtmpfs khungtaskd rpciod kernel: fsnotify_mark kthrotld deferwq scsi_* kdmflush oom_reaper kdevtempfs diff --git a/collectors/plugins.d/pluginsd_parser.c b/collectors/plugins.d/pluginsd_parser.c index d568db5ca2d5b4..126db7f2a447f7 100644 --- a/collectors/plugins.d/pluginsd_parser.c +++ b/collectors/plugins.d/pluginsd_parser.c @@ -888,7 +888,7 @@ static void inflight_functions_delete_callback(const DICTIONARY_ITEM *item __may pf->result_cb(pf->result_body_wb, pf->code, pf->result_cb_data); string_freez(pf->function); - freez(pf->payload); + freez((void *)pf->payload); } void inflight_functions_init(PARSER *parser) { diff --git a/collectors/systemd-journal.plugin/README.md b/collectors/systemd-journal.plugin/README.md index e69de29bb2d1d6..03236383978781 100644 --- a/collectors/systemd-journal.plugin/README.md +++ b/collectors/systemd-journal.plugin/README.md @@ -0,0 +1,357 @@ + + +[KEY FEATURES](#key-features) | [JOURNAL SOURCES](#journal-sources) | [JOURNAL FIELDS](#journal-fields) | +[PLAY MODE](#play-mode) | [FULL TEXT SEARCH](#full-text-search) | [PERFORMANCE](#query-performance) | +[CONFIGURATION](#configuration-and-maintenance) | [FAQ](#faq) + +# SystemD Journal + +The SystemD Journal plugin by Netdata makes viewing, exploring and analyzing systemd journal logs simple and efficient. +It automatically discovers available journal sources, allows advanced filtering, offers interactive visual +representations and supports exploring the logs of both individual servers and the logs on infrastructure wide +journal centralization servers. + +![image](https://github.com/netdata/netdata/assets/2662304/691b7470-ec56-430c-8b81-0c9e49012679) + +## Key features: + +- Works on both **individual servers** and **journal centralization servers**. +- Supports `persistent` and `volatile` journals. +- Supports `system`, `user`, `namespaces` and `remote` journals. +- Allows filtering on **any journal field** or **field value**, for any time-frame. +- Allows **full text search** (`grep`) on all journal fields, for any time-frame. +- Provides a **histogram** for log entries over time, with a break down per field-value, for any field and any time-frame. +- Works directly on journal files, without any other third party components. +- Supports coloring log entries, the same way `journalctl` does. +- In PLAY mode provides the same experience as `journalctl -f`, showing new logs entries immediately after they are received. + +### Prerequisites + +`systemd-journal.plugin` is a Netdata Function Plugin. + +To protect your privacy, as with all Netdata Functions, a free Netdata Cloud user account is required to access it. + +### Limitations: + +- This plugin is not available when Netdata is installed in a container. The problem is that `libsystemd` is not available in Alpine Linux (there is a `libsystemd`, but it is a dummy that returns failure on all calls). We plan to change this, by shipping Netdata containers based on Debian. +- For the same reason (lack of `systemd` support for Alpine Linux), the plugin is not available on `static` builds of Netdata (which are based on `muslc`, not `glibc`). + +## Journal Sources + +The plugin automatically detects the available journal sources, based on the journal files available in +`/var/log/journal` (persistent logs) and `/run/log/journal` (volatile logs). + +![journal-sources](https://github.com/netdata/netdata/assets/2662304/28e63a3e-6809-4586-b3b0-80755f340e31) + +The plugin, by default, merges all journal sources together, to provide a unified view of all log messages available. + +> To improve query performance, we recommend selecting the relevant journal source, before doing more analysis on the logs. + +### `system` journals + +These are the default journals available on all systems. + +`system` journals contain: + +- kernel log messages (via `kmsg`), +- audit records, originating from the kernel audit subsystem, +- messages received via `syslog`, +- messages received via the standard output and error of service units, +- structured messages received via the native journal API. + +### `user` journals + +By default, each user, with a UID outside the range of system users (0 - 999), dynamic service users, +and the nobody user (65534), will get their own set of `user` journal files. For more information about +this policy check [Users, Groups, UIDs and GIDs on systemd Systems](https://systemd.io/UIDS-GIDS/). + +The plugin allows viewing, exploring and querying the journal files of all users. + +### `namespaces` journals + +Journal 'namespaces' are both a mechanism for logically isolating the log stream of projects consisting +of one or more services from the rest of the system and a mechanism for improving performance. Systemd service +units may be assigned to a specific journal namespace through the `LogNamespace=` unit file setting. + +The plugin auto-detects the namespaces available and provides a list of all namespaces at the "sources" list on the UI. + +### `remote` journals + +Remote journals are created by `systemd-journal-remote`. This feature allows creating logs centralization points within +your infrastructure. + +Usually `remote` journals are named by the IP of the server sending these logs. The Netdata plugin automatically +extracts these IPs and performs a reverse DNS lookup to find their hostnames. When this is successful, +`remote` journals are named by the hostnames of the origin servers. + +For information about configuring a journals' centralization server, check [this FAQ item](#how-do-i-configure-a-journals-centralization-server). + +## Journal Fields + +Fields found in the journal files are automatically added to the UI in multiple places to help you explore +and filter the data. + +The plugin automatically enriches certain fields to make them more user-friendly: + +- `_BOOT_ID`: the hex value is annotated with the timestamp of the first message encountered for this boot id. +- `PRIORITY`: the numeric value is replaced with the human-readable name of each priority. +- `SYSLOG_FACILITY`: the encoded value is replaced with the human-readable name of each value. +- `ERRNO`: the numeric value is annotated with the short name of each value. +- `_UID` `_AUDIT_LOGINUID` and `_SYSTEMD_OWNER_UID`: the local user database is consulted to annotate them with usernames. +- `_GID`: the local group database is consulted to annotate them with group names. +- `_CAP_EFFECTIVE`: the encoded value is annotated with a human-readable list of the linux capabilities. +- `_SOURCE_REALTIME_TIMESTAMP`: the numeric value is annotated with human-readable datetime in UTC. + +The values of all other fields are presented as found in the journals. + +> IMPORTANT:
+> `_UID` `_AUDIT_LOGINUID`, `_SYSTEMD_OWNER_UID` and `_GID` annotations are added during presentation and are taken +> from the server running the plugin. For `remote` sources, the names presented may not reflect the actual user and +> group names on the origin server. + +The annotations are not searchable with full text search. They are only added for the presentation of the fields. + +### Journal fields as columns in the table + +All journal fields available in the journal files are offered as columns on the UI. Use the gear button above the table: + +![image](https://github.com/netdata/netdata/assets/2662304/cd75fb55-6821-43d4-a2aa-033792c7f7ac) + +### Journal fields as additional info to each log entry + +When you click a log line, the sidebar, on the right of the screen, provides the full list of fields related to this +log line. You can close this info sidebar, by selecting the filter icon at its top. + +![image](https://github.com/netdata/netdata/assets/2662304/3207794c-a61b-444c-8ffe-6c07cbc90ae2) + +### Journal fields as filters + +The plugin presents a select list of fields as filters to the query, with counters for each of the possible values +for the field. This list can used to quickly check which fields and values are available for the entire time-frame +of the query. + +Internally the plugin has: + +1. A white-list of fields, to be presented as filters. +2. A black-list of fields, to prevent them from becoming filters. This list includes fields with a very high cardinality, like timestamps, unique message ids, etc. This is mainly for protecting the server's performance, to avoid building in memory indexes for the fields that almost each of their values is unique. + +Keep in mind that the values presented in the filters, and their sorting is affected by the "full data queries" +setting: + +![image](https://github.com/netdata/netdata/assets/2662304/ac710d46-07c2-487b-8ce3-e7f767b9ae0f) + +When "full data queries" is off, empty values are hidden and cannot be selected. This is due to a limitation of +`libsystemd` that does not allow negative or empty matches. Also, values with zero counters may appear in the list. + +When "full data queries" is on, Netdata is applying all filtering to the data (not `libsystemd`), but this means +that all the data of the entire time-frame, without any filtering applied, have to be read by the plugin to prepare +the response required. + +### Journal fields as histogram sources + +The plugin presents a histogram of the number of log entries across time. + +The data source of this histogram can be any of the fields that are available as filters. +For each of the values this field has, across the entire time-frame of the query, the histogram will get corresponding +dimensions, showing the number of log entries, per value, over time. + +The granularity of the histogram is adjusted automatically to have about 150 columns visible on screen. + +The histogram presented by the plugin is interactive: + +- **Zoom**, either with the global date-time picker, or the zoom tool in the histogram's toolbox. +- **Pan**, either with global date-time picker, or by dragging with the mouse the chart to the left or the right. +- **Click**, to quickly jump to the highlighted point in time in the log entries. + +![image](https://github.com/netdata/netdata/assets/2662304/d3dcb1d1-daf4-49cf-9663-91b5b3099c2d) + +## PLAY mode + +The plugin supports PLAY mode, to continuously update the screen with new log entries found in the journal files. + +On centralized log servers, this provides a unified view of all the logs encountered across the entire infrastructure. + +## Full-text search + +The plugin supports searching for any text on all fields of the log entries. + +Full text search is combined with the selected filters. + +## Query performance + +Journal files are designed to be accessed by multiple readers and one writer, concurrently. + +Readers (like this Netdata plugin), open the journal files and `libsystemd`, behind the scenes, maps regions +of the files into memory, to satisfy each query. + +On logs aggregation servers, the performance of the queries depend on the following factors: + +1. The number of files involved in each query. This is why we suggest to select a source when possible. +2. The speed of the disks hosting the journal files. Journal files perform a lot of reading while querying, so the fastest the disks, the faster the query will finish. +3. The memory available for caching parts of the files. Increased memory will help the kernel cache the most frequently used parts of the journal files, avoiding disk I/O and speeding up queries. +4. The number of filters applied. Queries are significantly faster when just a few filters are selected. + +In general, for a faster experience, keep a low number of rows within the visible timeframe. + +Even on long timeframes, selecting a couple of filters that will result in a few dozen thousand log entries +will provide fast / rapid responses, usually less than a second. To the contrary, viewing timeframes with millions +of entries may result in longer delays. + +The plugin aborts journal queries when your browser cancels inflight requests. This allows you to work on the UI +while there are background queries running. + +At the time of this writing, this Netdata plugin is about 25-30 times faster than `journalctl` on queries that access +multiple journal files, over long time-frames. + +During the development of this plugin, we submitted, to `systemd`, a number of patches to improve `journalctl` +performance by a factor of 14: + +- https://github.com/systemd/systemd/pull/29365 +- https://github.com/systemd/systemd/pull/29366 +- https://github.com/systemd/systemd/pull/29261 + +However, even after these patches are merged, `journalctl` will still be 2x slower than this Netdata plugin, +on multi-journal queries. + +The problem lies in the way `libsystemd` handles multi-journal file queries. To overcome this problem, +the Netdata plugin queries each file individually and it then it merges the results to be returned. +This is transparent, thanks to the `facets` library in `libnetdata` that handles on-the-fly indexing, filtering, +and searching of any dataset, independently of its source. + +## Configuration and maintenance + +This Netdata plugin does not require any configuration or maintenance. + +## FAQ + +### Can I use this plugin on journals' centralization servers? + +Yes. You can centralize your logs using systemd journal, and then install Netdata +on this logs centralization server to explore the logs of all your infrastructure. + +This plugin will automatically provide multi-node views of your logs and also give you the ability to combine the logs +of multiple servers, as you see fit. + +Check [configuring a logs centralization server](#configuring-a-journals-centralization-server). + +### Can I use this plugin from a parent Netdata? + +Yes. When your nodes are connected to a Netdata parent, all their functions are available +via the parent's UI. So, from the parent UI, you can access the functions of all your nodes. + +Keep in mind that to protect your privacy, in order to access Netdata functions, you need a +free Netdata Cloud account. + +### Is any of my data exposed to Netdata Cloud from this plugin? + +No. When you access the agent directly, none of your data passes through Netdata Cloud. +You need a free Netdata Cloud account only to verify your identity and enable the use of +Netdata Functions. Once this is done, all the data flow directly from your Netdata agent +to your web browser. + +When you access Netdata via `https://app.netdata.cloud`, your data travel via Netdata Cloud, +but they are not stored in Netdata Cloud. This is to allow you access your Netdata agents from +anywhere. All communication from/to Netdata Cloud is encrypted. + +### What are `volatile` and `persistent` journals? + +SystemD JournalD allows creating both `volatile` journals in a `tmpfs` ram drive, +and `persistent` journals stored on disk. + +`volatile` journals are particularly useful when the system monitored is sensitive to +disk I/O, or does not have any writable disks at all. + +For more information check `man systemd-journald`. + +### Is it worth to build a systemd logs centralization server? + +Yes. It is simple, fast and the software to do it is already in your systems. + +For application and system logs, systemd journal is ideal and the visibility you can get +by centralizing your system logs and the use of this Netdata plugin, is unparalleled. + +### How do I configure a journals' centralization server? + +A short summary to get journal server running can be found below. + +For more options and reference to documentation, check `man systemd-journal-remote` and `man systemd-journal-upload`. + +#### Configuring a journals' centralization server + +On the centralization server install `systemd-journal-remote`, and enable it with `systemctl`, like this: + +```sh +# change this according to your distro +sudo apt-get install systemd-journal-remote + +# enable receiving +sudo systemctl enable --now systemd-journal-remote.socket +sudo systemctl enable systemd-journal-remote.service +``` + +`systemd-journal-remote` is now listening for incoming journals from remote hosts, on port `19532`. +Please note that `systemd-journal-remote` supports using secure connections. +To learn more run `man systemd-journal-remote`. + +To change the protocol of the journal transfer (HTTP/HTTPS) and the save location, do: + +```sh +# copy the service file +sudo cp /lib/systemd/system/systemd-journal-remote.service /etc/systemd/system/ + +# edit it +# --listen-http=-3 specifies the incoming journal for http. +# If you want to use https, change it to --listen-https=-3. +nano /etc/systemd/system/systemd-journal-remote.service + +# reload systemd +sudo systemctl daemon-reload +``` + +To change the port, copy `/lib/systemd/system/systemd-journal-remote.socket` to `/etc/systemd/system/` and edit it. +Then do `sudo systemctrl daemon-reload` + + +#### Configuring journal clients to push their logs to the server + +On the clients you want to centralize their logs, install `systemd-journal-remote`, configure `systemd-journal-upload`, enable it and start it with `systemctl`. + +To install it run: + +```sh +# change this according to your distro +sudo apt-get install systemd-journal-remote +``` + +Then, edit `/etc/systemd/journal-upload.conf` and set the IP address and the port of the server, like this: + +``` +[Upload] +URL=http://centralization.server.ip:19532 +``` + +Remember to match the protocol (http/https) the server expects. + +Finally, enable and start `systemd-journal-upload`, like this: + +```sh +sudo systemctl enable systemd-journal-upload +sudo systemctl start systemd-journal-upload +``` + +Keep in mind that immediately after starting `systemd-journal-upload` on a server, a replication process starts pushing logs in the order they have been received. This means that depending on the size of the available logs, some time may be needed for Netdata to show the most recent logs of that server. + +#### Limitations when using a logs centralization server + +As of this writing `namespaces` support by systemd is limited: + +- Docker containers cannot log to namespaces. Check [this issue](https://github.com/moby/moby/issues/41879). +- `systemd-journal-upload` automatically uploads `system` and `user` journals, but not `namespaces` journals. For this you need to spawn a `systemd-journal-upload` per namespace. + diff --git a/collectors/systemd-journal.plugin/systemd-journal.c b/collectors/systemd-journal.plugin/systemd-journal.c index bfa51393fb9a34..e7134958968159 100644 --- a/collectors/systemd-journal.plugin/systemd-journal.c +++ b/collectors/systemd-journal.plugin/systemd-journal.c @@ -9,19 +9,97 @@ #include "libnetdata/libnetdata.h" #include "libnetdata/required_dummies.h" +#include #include #include +// ---------------------------------------------------------------------------- +// fstat64 overloading to speed up libsystemd +// https://github.com/systemd/systemd/pull/29261 + +#define ND_SD_JOURNAL_OPEN_FLAGS (0) + +#ifdef HAVE_SD_JOURNAL_OPEN_FILES_FD + +#include +#include + +#define FSTAT_CACHE_MAX 1024 +struct fdstat64_cache_entry { + bool enabled; + bool updated; + int err_no; + struct stat64 stat; + int ret; + size_t cached_count; +}; +struct fdstat64_cache_entry fstat64_cache[FSTAT_CACHE_MAX] = {0 }; + +static void fstat_cache_enable(int fd) { + if(fd >= 0 && fd < FSTAT_CACHE_MAX) { + fstat64_cache[fd].enabled = true; + fstat64_cache[fd].updated = false; + fstat64_cache[fd].cached_count = 0; + } +} + +static size_t fstat_cache_disable(int fd) { + size_t cached_count = 0; + + if(fd >= 0 && fd < FSTAT_CACHE_MAX) { + fstat64_cache[fd].enabled = false; + fstat64_cache[fd].updated = false; + cached_count = fstat64_cache[fd].cached_count; + fstat64_cache[fd].cached_count = 0; + } + + return cached_count; +} + +static size_t fstat_calls = 0; +static size_t fstat_cached_responses = 0; + +int fstat64(int fd, struct stat64 *buf) { + static int (*real_fstat)(int, struct stat64 *) = NULL; + if (!real_fstat) + real_fstat = dlsym(RTLD_NEXT, "fstat64"); + + fstat_calls++; + + if(fd >= 0 && fd < FSTAT_CACHE_MAX && fstat64_cache[fd].enabled && fstat64_cache[fd].updated) { + fstat_cached_responses++; + errno = fstat64_cache[fd].err_no; + *buf = fstat64_cache[fd].stat; + fstat64_cache[fd].cached_count++; + return fstat64_cache[fd].ret; + } + + int ret = real_fstat(fd, buf); + + if(fd >= 0 && fd < FSTAT_CACHE_MAX && fstat64_cache[fd].enabled) { + fstat64_cache[fd].ret = ret; + fstat64_cache[fd].updated = true; + fstat64_cache[fd].err_no = errno; + fstat64_cache[fd].stat = *buf; + } + + return ret; +} + +#endif // HAVE_SD_JOURNAL_OPEN_FILES_FD + +// ---------------------------------------------------------------------------- + #define FACET_MAX_VALUE_LENGTH 8192 +#define SYSTEMD_JOURNAL_MAX_SOURCE_LEN 64 #define SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION "View, search and analyze systemd journal entries." #define SYSTEMD_JOURNAL_FUNCTION_NAME "systemd-journal" -#define SYSTEMD_JOURNAL_DEFAULT_TIMEOUT 30 +#define SYSTEMD_JOURNAL_DEFAULT_TIMEOUT 60 #define SYSTEMD_JOURNAL_MAX_PARAMS 100 #define SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION (3 * 3600) #define SYSTEMD_JOURNAL_DEFAULT_ITEMS_PER_QUERY 200 -#define SYSTEMD_JOURNAL_EXCESS_ROWS_ALLOWED 50 -#define SYSTEMD_JOURNAL_WORKER_THREADS 2 +#define SYSTEMD_JOURNAL_WORKER_THREADS 5 #define JOURNAL_PARAMETER_HELP "help" #define JOURNAL_PARAMETER_AFTER "after" @@ -36,22 +114,60 @@ #define JOURNAL_PARAMETER_DATA_ONLY "data_only" #define JOURNAL_PARAMETER_SOURCE "source" #define JOURNAL_PARAMETER_INFO "info" +#define JOURNAL_PARAMETER_ID "id" +#define JOURNAL_PARAMETER_PROGRESS "progress" +#define JOURNAL_PARAMETER_SLICE "slice" +#define JOURNAL_PARAMETER_DELTA "delta" +#define JOURNAL_PARAMETER_TAIL "tail" + +#define JOURNAL_DEFAULT_SLICE_MODE true +#define JOURNAL_DEFAULT_DIRECTION FACETS_ANCHOR_DIRECTION_BACKWARD #define SYSTEMD_ALWAYS_VISIBLE_KEYS NULL -#define SYSTEMD_KEYS_EXCLUDED_FROM_FACETS NULL + +#define SYSTEMD_KEYS_EXCLUDED_FROM_FACETS \ + "*MESSAGE*" \ + "|CODE_LINE" \ + "|*DOCUMENTATION*" \ + "|TID" \ + "|*_RAW" \ + "|*_NSEC" \ + "|*TIMESTAMP*" \ + "|*_ID" \ + "|*_ID_*" \ + "|*_PID" \ + "|*_TID" \ + "|__*" \ + "" + #define SYSTEMD_KEYS_INCLUDED_IN_FACETS \ - "_TRANSPORT" \ + "_COMM" \ + "|CONTAINER_NAME" \ + "|CONTAINER_TAG" \ + "|_TRANSPORT" \ "|SYSLOG_IDENTIFIER" \ "|SYSLOG_FACILITY" \ "|PRIORITY" \ - "|_UID" \ - "|_GID" \ "|_SYSTEMD_UNIT" \ "|_SYSTEMD_SLICE" \ - "|_COMM" \ + "|_SYSTEMD_USER_UNIT" \ + "|_SYSTEMD_USER_SLICE" \ + "|_SYSTEMD_OWNER_UID" \ + "|_UID" \ + "|_GID" \ "|UNIT" \ - "|CONTAINER_NAME" \ + "|USER_UNIT" \ "|IMAGE_NAME" \ + "|ERRNO" \ + "|_NAMESPACE" \ + "|COREDUMP_COMM" \ + "|COREDUMP_UNIT" \ + "|COREDUMP_USER_UNIT" \ + "|COREDUMP_SIGNAL_NAME" \ + "|COREDUMP_CGROUP" \ + "|_HOSTNAME" \ + "|UNIT_RESULT" \ + "|_RUNTIME_SCOPE" \ "" static netdata_mutex_t stdout_mutex = NETDATA_MUTEX_INITIALIZER; @@ -59,34 +175,9 @@ static bool plugin_should_exit = false; // ---------------------------------------------------------------------------- -static inline sd_journal *netdata_open_systemd_journal(void) { - sd_journal *j = NULL; - int r; - - if(*netdata_configured_host_prefix) { -#ifdef HAVE_SD_JOURNAL_OS_ROOT - // Give our host prefix to systemd journal - r = sd_journal_open_directory(&j, netdata_configured_host_prefix, SD_JOURNAL_OS_ROOT); -#else - char buf[FILENAME_MAX + 1]; - snprintfz(buf, FILENAME_MAX, "%s/var/log/journal", netdata_configured_host_prefix); - r = sd_journal_open_directory(&j, buf, 0); -#endif - } - else { - // Open the system journal for reading - r = sd_journal_open(&j, 0); - } - - if (r < 0) { - netdata_log_error("SYSTEMD-JOURNAL: Failed to open SystemD Journal, with error %d", r); - return NULL; - } - - return j; -} - typedef enum { + ND_SD_JOURNAL_NO_FILE_MATCHED, + ND_SD_JOURNAL_FAILED_TO_OPEN, ND_SD_JOURNAL_FAILED_TO_SEEK, ND_SD_JOURNAL_TIMED_OUT, ND_SD_JOURNAL_OK, @@ -94,6 +185,69 @@ typedef enum { ND_SD_JOURNAL_CANCELLED, } ND_SD_JOURNAL_STATUS; +typedef enum { + SDJF_ALL = 0, + SDJF_LOCAL = (1 << 0), + SDJF_REMOTE = (1 << 1), + SDJF_SYSTEM = (1 << 2), + SDJF_USER = (1 << 3), + SDJF_NAMESPACE = (1 << 4), + SDJF_OTHER = (1 << 5), +} SD_JOURNAL_FILE_SOURCE_TYPE; + +typedef struct function_query_status { + bool *cancelled; // a pointer to the cancelling boolean + usec_t stop_monotonic_ut; + + usec_t started_monotonic_ut; + + // request + SD_JOURNAL_FILE_SOURCE_TYPE source_type; + STRING *source; + usec_t after_ut; + usec_t before_ut; + + struct { + usec_t start_ut; + usec_t stop_ut; + } anchor; + + FACETS_ANCHOR_DIRECTION direction; + size_t entries; + usec_t if_modified_since; + bool delta; + bool tail; + bool data_only; + bool slice; + size_t filters; + usec_t last_modified; + const char *query; + const char *histogram; + + // per file progress info + size_t cached_count; + + // progress statistics + usec_t matches_setup_ut; + size_t rows_useful; + size_t rows_read; + size_t bytes_read; + size_t files_matched; + size_t file_working; +} FUNCTION_QUERY_STATUS; + +struct journal_file { + STRING *source; + SD_JOURNAL_FILE_SOURCE_TYPE source_type; + usec_t file_last_modified_ut; + usec_t msg_first_ut; + usec_t msg_last_ut; + usec_t last_scan_ut; + size_t size; + bool logged_failure; + usec_t max_journal_vs_realtime_delta_ut; +}; + static inline bool netdata_systemd_journal_seek_to(sd_journal *j, usec_t timestamp) { if(sd_journal_seek_realtime_usec(j, timestamp) < 0) { netdata_log_error("SYSTEMD-JOURNAL: Failed to seek to %" PRIu64, timestamp); @@ -106,296 +260,1003 @@ static inline bool netdata_systemd_journal_seek_to(sd_journal *j, usec_t timesta return true; } -static inline void netdata_systemd_journal_process_row(sd_journal *j, FACETS *facets) { +#define JD_SOURCE_REALTIME_TIMESTAMP "_SOURCE_REALTIME_TIMESTAMP" + +#define JOURNAL_VS_REALTIME_DELTA_DEFAULT_UT (2 * USEC_PER_SEC) // assume always 2 seconds latency +#define JOURNAL_VS_REALTIME_DELTA_MAX_UT (2 * 60 * USEC_PER_SEC) // up to 2 minutes delta + +static inline bool parse_journal_field(const char *data, size_t data_length, const char **key, size_t *key_length, const char **value, size_t *value_length) { + const char *k = data; + const char *equal = strchr(k, '='); + if(unlikely(!equal)) + return false; + + size_t kl = equal - k; + + const char *v = ++equal; + size_t vl = data_length - kl - 1; + + *key = k; + *key_length = kl; + *value = v; + *value_length = vl; + + return true; +} + +static inline size_t netdata_systemd_journal_process_row(sd_journal *j, FACETS *facets, struct journal_file *jf, usec_t *msg_ut) { const void *data; - size_t length; + size_t length, bytes = 0; + SD_JOURNAL_FOREACH_DATA(j, data, length) { - const char *key = data; - const char *equal = strchr(key, '='); - if(unlikely(!equal)) - continue; + const char *key, *value; + size_t key_length, value_length; - const char *value = ++equal; - size_t key_length = value - key; // including '\0' + if(!parse_journal_field(data, length, &key, &key_length, &value, &value_length)) + continue; - char key_copy[key_length]; - memcpy(key_copy, key, key_length - 1); - key_copy[key_length - 1] = '\0'; + usec_t origin_journal_ut = *msg_ut; + + if(unlikely(key_length == sizeof(JD_SOURCE_REALTIME_TIMESTAMP) - 1 && + memcmp(key, JD_SOURCE_REALTIME_TIMESTAMP, sizeof(JD_SOURCE_REALTIME_TIMESTAMP) - 1) == 0)) { + usec_t ut = str2ull(value, NULL); + if(ut && ut < *msg_ut) { + usec_t delta = *msg_ut - ut; + *msg_ut = ut; + + if(delta > JOURNAL_VS_REALTIME_DELTA_MAX_UT) + delta = JOURNAL_VS_REALTIME_DELTA_MAX_UT; + + // update max_journal_vs_realtime_delta_ut if the delta increased + usec_t expected = jf->max_journal_vs_realtime_delta_ut; + do { + if(delta <= expected) + break; + } while(!__atomic_compare_exchange_n(&jf->max_journal_vs_realtime_delta_ut, &expected, delta, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); + + internal_error(delta > expected, + "increased max_journal_vs_realtime_delta_ut from %"PRIu64" to %"PRIu64", " + "journal %"PRIu64", actual %"PRIu64" (delta %"PRIu64")" + , expected, delta, origin_journal_ut, *msg_ut, origin_journal_ut - (*msg_ut)); + } + } - size_t value_length = length - key_length; // without '\0' - facets_add_key_value_length(facets, key_copy, key_length - 1, value, value_length <= FACET_MAX_VALUE_LENGTH ? value_length : FACET_MAX_VALUE_LENGTH); + bytes += length; + facets_add_key_value_length(facets, key, key_length, value, value_length <= FACET_MAX_VALUE_LENGTH ? value_length : FACET_MAX_VALUE_LENGTH); } + + return bytes; } -static inline ND_SD_JOURNAL_STATUS check_stop(size_t row_counter, const bool *cancelled, usec_t stop_monotonic_ut) { - if((row_counter % 1000) == 0) { - if(cancelled && __atomic_load_n(cancelled, __ATOMIC_RELAXED)) { - internal_error(true, "Function has been cancelled"); - return ND_SD_JOURNAL_CANCELLED; - } +#define FUNCTION_PROGRESS_UPDATE_ROWS(rows_read, rows) __atomic_fetch_add(&(rows_read), rows, __ATOMIC_RELAXED) +#define FUNCTION_PROGRESS_UPDATE_BYTES(bytes_read, bytes) __atomic_fetch_add(&(bytes_read), bytes, __ATOMIC_RELAXED) +#define FUNCTION_PROGRESS_EVERY_ROWS 10000 - if(now_monotonic_usec() > stop_monotonic_ut) { - internal_error(true, "Function timed out"); - return ND_SD_JOURNAL_TIMED_OUT; - } +static inline ND_SD_JOURNAL_STATUS check_stop(const bool *cancelled, const usec_t *stop_monotonic_ut) { + if(cancelled && __atomic_load_n(cancelled, __ATOMIC_RELAXED)) { + internal_error(true, "Function has been cancelled"); + return ND_SD_JOURNAL_CANCELLED; + } + + if(now_monotonic_usec() > __atomic_load_n(stop_monotonic_ut, __ATOMIC_RELAXED)) { + internal_error(true, "Function timed out"); + return ND_SD_JOURNAL_TIMED_OUT; } return ND_SD_JOURNAL_OK; } -ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_full( +ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_backward( sd_journal *j, BUFFER *wb __maybe_unused, FACETS *facets, - usec_t after_ut, usec_t before_ut, - usec_t if_modified_since, usec_t stop_monotonic_ut, usec_t *last_modified, - bool *cancelled) { - if(!netdata_systemd_journal_seek_to(j, before_ut)) + struct journal_file *jf, FUNCTION_QUERY_STATUS *fqs) { + + usec_t anchor_delta = __atomic_load_n(&jf->max_journal_vs_realtime_delta_ut, __ATOMIC_RELAXED); + + usec_t start_ut = ((fqs->data_only && fqs->anchor.start_ut) ? fqs->anchor.start_ut : fqs->before_ut) + anchor_delta; + usec_t stop_ut = (fqs->data_only && fqs->anchor.stop_ut) ? fqs->anchor.stop_ut : fqs->after_ut; + + if(!netdata_systemd_journal_seek_to(j, start_ut)) return ND_SD_JOURNAL_FAILED_TO_SEEK; size_t errors_no_timestamp = 0; - usec_t first_msg_ut = 0; - size_t row_counter = 0; - - // the entries are not guaranteed to be sorted, so we process up to 100 entries beyond - // the end of the query to find possibly useful logs for our time-frame - size_t excess_rows_allowed = SYSTEMD_JOURNAL_EXCESS_ROWS_ALLOWED; + usec_t earliest_msg_ut = 0; + size_t row_counter = 0, last_row_counter = 0; + size_t bytes = 0, last_bytes = 0; ND_SD_JOURNAL_STATUS status = ND_SD_JOURNAL_OK; facets_rows_begin(facets); while (status == ND_SD_JOURNAL_OK && sd_journal_previous(j) > 0) { - row_counter++; - - usec_t msg_ut; - if(sd_journal_get_realtime_usec(j, &msg_ut) < 0) { + usec_t msg_ut = 0; + if(sd_journal_get_realtime_usec(j, &msg_ut) < 0 || !msg_ut) { errors_no_timestamp++; continue; } - if(unlikely(!first_msg_ut)) { - if(msg_ut == if_modified_since) { - return ND_SD_JOURNAL_NOT_MODIFIED; - } - - first_msg_ut = msg_ut; - } + if(unlikely(msg_ut > earliest_msg_ut)) + earliest_msg_ut = msg_ut; - if (msg_ut > before_ut) + if (unlikely(msg_ut > start_ut)) continue; - if (msg_ut < after_ut) { - if(--excess_rows_allowed == 0) - break; + if (unlikely(msg_ut < stop_ut)) + break; - continue; + bytes += netdata_systemd_journal_process_row(j, facets, jf, &msg_ut); + if(facets_row_finished(facets, msg_ut)) + fqs->rows_useful++; + + row_counter++; + if(row_counter % 100 == 0 && fqs->data_only && facets_rows(facets) >= fqs->entries) { + // stop the data only query + usec_t oldest = facets_row_oldest_ut(facets); + if(oldest && msg_ut < (oldest - anchor_delta)) + break; } - netdata_systemd_journal_process_row(j, facets); - facets_row_finished(facets, msg_ut); + if(row_counter % FUNCTION_PROGRESS_EVERY_ROWS == 0) { + FUNCTION_PROGRESS_UPDATE_ROWS(fqs->rows_read, row_counter - last_row_counter); + last_row_counter = row_counter; + + FUNCTION_PROGRESS_UPDATE_BYTES(fqs->bytes_read, bytes - last_bytes); + last_bytes = bytes; - status = check_stop(row_counter, cancelled, stop_monotonic_ut); + status = check_stop(fqs->cancelled, &fqs->stop_monotonic_ut); + } } + FUNCTION_PROGRESS_UPDATE_ROWS(fqs->rows_read, row_counter - last_row_counter); + FUNCTION_PROGRESS_UPDATE_BYTES(fqs->bytes_read, bytes - last_bytes); + if(errors_no_timestamp) netdata_log_error("SYSTEMD-JOURNAL: %zu lines did not have timestamps", errors_no_timestamp); - *last_modified = first_msg_ut; + if(earliest_msg_ut > fqs->last_modified) + fqs->last_modified = earliest_msg_ut; return status; } -ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_data_forward( +ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_forward( sd_journal *j, BUFFER *wb __maybe_unused, FACETS *facets, - usec_t after_ut, usec_t before_ut, - usec_t anchor, size_t entries, usec_t stop_monotonic_ut, - bool *cancelled) { + struct journal_file *jf, FUNCTION_QUERY_STATUS *fqs) { + + usec_t anchor_delta = __atomic_load_n(&jf->max_journal_vs_realtime_delta_ut, __ATOMIC_RELAXED); - if(!netdata_systemd_journal_seek_to(j, anchor)) + usec_t start_ut = (fqs->data_only && fqs->anchor.start_ut) ? fqs->anchor.start_ut : fqs->after_ut; + usec_t stop_ut = ((fqs->data_only && fqs->anchor.stop_ut) ? fqs->anchor.stop_ut : fqs->before_ut) + anchor_delta; + + if(!netdata_systemd_journal_seek_to(j, start_ut)) return ND_SD_JOURNAL_FAILED_TO_SEEK; size_t errors_no_timestamp = 0; - size_t row_counter = 0; - size_t rows_added = 0; - - // the entries are not guaranteed to be sorted, so we process up to 100 entries beyond - // the end of the query to find possibly useful logs for our time-frame - size_t excess_rows_allowed = SYSTEMD_JOURNAL_EXCESS_ROWS_ALLOWED; + usec_t earliest_msg_ut = 0; + size_t row_counter = 0, last_row_counter = 0; + size_t bytes = 0, last_bytes = 0; ND_SD_JOURNAL_STATUS status = ND_SD_JOURNAL_OK; facets_rows_begin(facets); while (status == ND_SD_JOURNAL_OK && sd_journal_next(j) > 0) { - row_counter++; - - usec_t msg_ut; - if(sd_journal_get_realtime_usec(j, &msg_ut) < 0) { + usec_t msg_ut = 0; + if(sd_journal_get_realtime_usec(j, &msg_ut) < 0 || !msg_ut) { errors_no_timestamp++; continue; } - if (msg_ut > before_ut || msg_ut <= anchor) + if(likely(msg_ut > earliest_msg_ut)) + earliest_msg_ut = msg_ut; + + if (unlikely(msg_ut < start_ut)) continue; - if (msg_ut < after_ut) { - if(--excess_rows_allowed == 0) - break; + if (unlikely(msg_ut > stop_ut)) + break; - continue; + bytes += netdata_systemd_journal_process_row(j, facets, jf, &msg_ut); + if(facets_row_finished(facets, msg_ut)) + fqs->rows_useful++; + + row_counter++; + if(row_counter % 100 == 0 && fqs->data_only && facets_rows(facets) >= fqs->entries) { + usec_t newest = facets_row_newest_ut(facets); + if(newest && msg_ut > (newest + anchor_delta)) + break; } - if(rows_added > entries && --excess_rows_allowed == 0) - break; + if(row_counter % FUNCTION_PROGRESS_EVERY_ROWS == 0) { + FUNCTION_PROGRESS_UPDATE_ROWS(fqs->rows_read, row_counter - last_row_counter); + last_row_counter = row_counter; - netdata_systemd_journal_process_row(j, facets); - facets_row_finished(facets, msg_ut); - rows_added++; + FUNCTION_PROGRESS_UPDATE_BYTES(fqs->bytes_read, bytes - last_bytes); + last_bytes = bytes; - status = check_stop(row_counter, cancelled, stop_monotonic_ut); + status = check_stop(fqs->cancelled, &fqs->stop_monotonic_ut); + } } + FUNCTION_PROGRESS_UPDATE_ROWS(fqs->rows_read, row_counter - last_row_counter); + FUNCTION_PROGRESS_UPDATE_BYTES(fqs->bytes_read, bytes - last_bytes); + if(errors_no_timestamp) netdata_log_error("SYSTEMD-JOURNAL: %zu lines did not have timestamps", errors_no_timestamp); + if(earliest_msg_ut > fqs->last_modified) + fqs->last_modified = earliest_msg_ut; + return status; } -ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_data_backward( - sd_journal *j, BUFFER *wb __maybe_unused, FACETS *facets, - usec_t after_ut, usec_t before_ut, - usec_t anchor, size_t entries, usec_t stop_monotonic_ut, - bool *cancelled) { +bool netdata_systemd_journal_check_if_modified_since(sd_journal *j, usec_t seek_to, usec_t last_modified) { + // return true, if data have been modified since the timestamp - if(!netdata_systemd_journal_seek_to(j, anchor)) - return ND_SD_JOURNAL_FAILED_TO_SEEK; + if(!last_modified || !seek_to) + return false; + + if(!netdata_systemd_journal_seek_to(j, seek_to)) + return false; + + usec_t first_msg_ut = 0; + while (sd_journal_previous(j) > 0) { + usec_t msg_ut; + if(sd_journal_get_realtime_usec(j, &msg_ut) < 0) + continue; + + first_msg_ut = msg_ut; + break; + } + + return first_msg_ut != last_modified; +} + +#ifdef HAVE_SD_JOURNAL_RESTART_FIELDS +static bool netdata_systemd_filtering_by_journal(sd_journal *j, FACETS *facets, FUNCTION_QUERY_STATUS *fqs) { + const char *field = NULL; + const void *data = NULL; + size_t data_length; + size_t added_keys = 0; + size_t failures = 0; + size_t filters_added = 0; + + SD_JOURNAL_FOREACH_FIELD(j, field) { + bool interesting; + + if(fqs->data_only) + interesting = facets_key_name_is_filter(facets, field); + else + interesting = facets_key_name_is_facet(facets, field); + + if(interesting) { + if(sd_journal_query_unique(j, field) >= 0) { + bool added_this_key = false; + size_t added_values = 0; + + SD_JOURNAL_FOREACH_UNIQUE(j, data, data_length) { + const char *key, *value; + size_t key_length, value_length; + + if(!parse_journal_field(data, data_length, &key, &key_length, &value, &value_length)) + continue; + + facets_add_possible_value_name_to_key(facets, key, key_length, value, value_length); + + if(!facets_key_name_value_length_is_selected(facets, key, key_length, value, value_length)) + continue; + + if(added_keys && !added_this_key) { + if(sd_journal_add_conjunction(j) < 0) + failures++; + + added_this_key = true; + added_keys++; + } + else if(added_values) + if(sd_journal_add_disjunction(j) < 0) + failures++; + + if(sd_journal_add_match(j, data, data_length) < 0) + failures++; + + added_values++; + filters_added++; + } + } + } + } + + if(failures) { + netdata_log_error("failed to setup journal filter, will run the full query."); + sd_journal_flush_matches(j); + return true; + } + + return filters_added ? true : false; +} +#endif // HAVE_SD_JOURNAL_RESTART_FIELDS + +static ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_one_file( + const char *filename, BUFFER *wb, FACETS *facets, + struct journal_file *jf, FUNCTION_QUERY_STATUS *fqs) { + + sd_journal *j = NULL; + errno = 0; + +#ifdef HAVE_SD_JOURNAL_OPEN_FILES_FD + int fd = open(filename, O_RDONLY); + fstat_cache_enable(fd); + + if(sd_journal_open_files_fd(&j, &fd, 1, ND_SD_JOURNAL_OPEN_FLAGS) < 0 || !j) { + fqs->cached_count += fstat_cache_disable(fd); + close(fd); + return ND_SD_JOURNAL_FAILED_TO_OPEN; + } +#else // !HAVE_SD_JOURNAL_OPEN_FILES_FD + + const char *paths[2] = { + [0] = filename, + [1] = NULL, + }; + if(sd_journal_open_files(&j, paths, ND_SD_JOURNAL_OPEN_FLAGS) < 0 || !j) + return ND_SD_JOURNAL_FAILED_TO_OPEN; + +#endif // !HAVE_SD_JOURNAL_OPEN_FILES_FD + + ND_SD_JOURNAL_STATUS status; + bool matches_filters = true; + +#ifdef HAVE_SD_JOURNAL_RESTART_FIELDS + if(fqs->slice) { + usec_t started = now_monotonic_usec(); + + matches_filters = netdata_systemd_filtering_by_journal(j, facets, fqs) || !fqs->filters; + usec_t ended = now_monotonic_usec(); + + fqs->matches_setup_ut += (ended - started); + } +#endif // HAVE_SD_JOURNAL_RESTART_FIELDS + + if(matches_filters) { + if(fqs->direction == FACETS_ANCHOR_DIRECTION_FORWARD) + status = netdata_systemd_journal_query_forward(j, wb, facets, jf, fqs); + else + status = netdata_systemd_journal_query_backward(j, wb, facets, jf, fqs); + } + else + status = ND_SD_JOURNAL_NO_FILE_MATCHED; + + sd_journal_close(j); + +#ifdef HAVE_SD_JOURNAL_OPEN_FILES_FD + fqs->cached_count += fstat_cache_disable(fd); + close(fd); +#endif + + return status; +} + +// ---------------------------------------------------------------------------- +// journal files registry + +#define VAR_LOG_JOURNAL_MAX_DEPTH 10 +#define MAX_JOURNAL_DIRECTORIES 100 + +struct journal_directory { + char *path; + bool logged_failure; +}; + +static struct journal_directory journal_directories[MAX_JOURNAL_DIRECTORIES] = { 0 }; +static DICTIONARY *journal_files_registry = NULL; +static DICTIONARY *used_hashes_registry = NULL; + +static usec_t systemd_journal_session = 0; + +static void buffer_json_journal_versions(BUFFER *wb) { + buffer_json_member_add_object(wb, "versions"); + { + buffer_json_member_add_uint64(wb, "sources", + systemd_journal_session + dictionary_version(journal_files_registry)); + } + buffer_json_object_close(wb); +} + +static void journal_file_update_msg_ut(const char *filename, struct journal_file *jf) { + const char *files[2] = { + [0] = filename, + [1] = NULL, + }; + + sd_journal *j = NULL; + if(sd_journal_open_files(&j, files, ND_SD_JOURNAL_OPEN_FLAGS) < 0 || !j) { + if(!jf->logged_failure) { + netdata_log_error("cannot open journal file '%s', using file timestamps to understand time-frame.", filename); + jf->logged_failure = true; + } + + jf->msg_first_ut = 0; + jf->msg_last_ut = jf->file_last_modified_ut; + return; + } + + usec_t first_ut = 0, last_ut = 0; + + if(sd_journal_seek_head(j) < 0 || sd_journal_next(j) < 0 || sd_journal_get_realtime_usec(j, &first_ut) < 0 || !first_ut) { + internal_error(true, "cannot find the timestamp of the first message in '%s'", filename); + first_ut = 0; + } + + if(sd_journal_seek_tail(j) < 0 || sd_journal_previous(j) < 0 || sd_journal_get_realtime_usec(j, &last_ut) < 0 || !last_ut) { + internal_error(true, "cannot find the timestamp of the last message in '%s'", filename); + last_ut = jf->file_last_modified_ut; + } + + sd_journal_close(j); + + if(first_ut > last_ut) { + internal_error(true, "timestamps are flipped in file '%s'", filename); + usec_t t = first_ut; + first_ut = last_ut; + last_ut = t; + } + + jf->msg_first_ut = first_ut; + jf->msg_last_ut = last_ut; +} + +static STRING *string_strdupz_source(const char *s, const char *e, size_t max_len, const char *prefix) { + char buf[max_len]; + size_t len; + char *dst = buf; + + if(prefix) { + len = strlen(prefix); + memcpy(buf, prefix, len); + dst = &buf[len]; + max_len -= len; + } + + len = e - s; + if(len >= max_len) + len = max_len - 1; + memcpy(dst, s, len); + dst[len] = '\0'; + buf[max_len - 1] = '\0'; + + for(size_t i = 0; buf[i] ;i++) + if(!isalnum(buf[i]) && buf[i] != '-' && buf[i] != '.' && buf[i] != ':') + buf[i] = '_'; + + return string_strdupz(buf); +} + +static void files_registry_insert_cb(const DICTIONARY_ITEM *item, void *value, void *data __maybe_unused) { + struct journal_file *jf = value; + const char *filename = dictionary_acquired_item_name(item); + + // based on the filename + // decide the source to show to the user + const char *s = strrchr(filename, '/'); + if(s) { + if(strstr(filename, "/remote/")) + jf->source_type = SDJF_REMOTE; + else { + const char *t = s - 1; + while(t >= filename && *t != '.' && *t != '/') + t--; + + if(t >= filename && *t == '.') { + jf->source_type = SDJF_NAMESPACE; + jf->source = string_strdupz_source(t + 1, s, SYSTEMD_JOURNAL_MAX_SOURCE_LEN, "namespace-"); + } + else + jf->source_type = SDJF_LOCAL; + } + + if(strncmp(s, "/system", 7) == 0) + jf->source_type |= SDJF_SYSTEM; + + else if(strncmp(s, "/user", 5) == 0) + jf->source_type |= SDJF_USER; + + else if(strncmp(s, "/remote-", 8) == 0) { + jf->source_type |= SDJF_REMOTE; + + s = &s[8]; // skip "/remote-" + + char *e = strchr(s, '@'); + if(!e) + e = strstr(s, ".journal"); + + if(e) { + const char *d = s; + for(; d < e && (isdigit(*d) || *d == '.' || *d == ':') ; d++) ; + if(d == e) { + // a valid IP address + char ip[e - s + 1]; + memcpy(ip, s, e - s); + ip[e - s] = '\0'; + char buf[SYSTEMD_JOURNAL_MAX_SOURCE_LEN]; + if(ip_to_hostname(ip, buf, sizeof(buf))) + jf->source = string_strdupz_source(buf, &buf[strlen(buf)], SYSTEMD_JOURNAL_MAX_SOURCE_LEN, "remote-"); + else { + internal_error(true, "Cannot find the hostname for IP '%s'", ip); + jf->source = string_strdupz_source(s, e, SYSTEMD_JOURNAL_MAX_SOURCE_LEN, "remote-"); + } + } + else + jf->source = string_strdupz_source(s, e, SYSTEMD_JOURNAL_MAX_SOURCE_LEN, "remote-"); + } + else + jf->source_type |= SDJF_OTHER; + } + else + jf->source_type |= SDJF_OTHER; + } + else + jf->source_type = SDJF_LOCAL | SDJF_OTHER; + + journal_file_update_msg_ut(filename, jf); + + internal_error(true, + "found journal file '%s', type %d, source '%s', " + "file modified: %"PRIu64", " + "msg {first: %"PRIu64", last: %"PRIu64"}", + filename, jf->source_type, jf->source ? string2str(jf->source) : "", + jf->file_last_modified_ut, + jf->msg_first_ut, jf->msg_last_ut); +} + +static bool files_registry_conflict_cb(const DICTIONARY_ITEM *item, void *old_value, void *new_value, void *data __maybe_unused) { + struct journal_file *jf = old_value; + struct journal_file *njf = new_value; + + if(njf->last_scan_ut > jf->last_scan_ut) + jf->last_scan_ut = njf->last_scan_ut; + + if(njf->file_last_modified_ut > jf->file_last_modified_ut) { + jf->file_last_modified_ut = njf->file_last_modified_ut; + jf->size = njf->size; + + const char *filename = dictionary_acquired_item_name(item); + journal_file_update_msg_ut(filename, jf); + +// internal_error(true, +// "updated journal file '%s', type %d, " +// "file modified: %"PRIu64", " +// "msg {first: %"PRIu64", last: %"PRIu64"}", +// filename, jf->source_type, +// jf->file_last_modified_ut, +// jf->msg_first_ut, jf->msg_last_ut); + } + + return false; +} + +#define SDJF_SOURCE_ALL_NAME "all" +#define SDJF_SOURCE_LOCAL_NAME "all-local-logs" +#define SDJF_SOURCE_LOCAL_SYSTEM_NAME "all-local-system-logs" +#define SDJF_SOURCE_LOCAL_USERS_NAME "all-local-user-logs" +#define SDJF_SOURCE_LOCAL_OTHER_NAME "all-uncategorized" +#define SDJF_SOURCE_NAMESPACES_NAME "all-local-namespaces" +#define SDJF_SOURCE_REMOTES_NAME "all-remote-systems" + +struct journal_file_source { + usec_t first_ut; + usec_t last_ut; + size_t count; + uint64_t size; +}; + +static void human_readable_size_ib(uint64_t size, char *dst, size_t dst_len) { + if(size > 1024ULL * 1024 * 1024 * 1024) + snprintfz(dst, dst_len, "%0.2f TiB", (double)size / 1024.0 / 1024.0 / 1024.0 / 1024.0); + else if(size > 1024ULL * 1024 * 1024) + snprintfz(dst, dst_len, "%0.2f GiB", (double)size / 1024.0 / 1024.0 / 1024.0); + else if(size > 1024ULL * 1024) + snprintfz(dst, dst_len, "%0.2f MiB", (double)size / 1024.0 / 1024.0); + else if(size > 1024ULL) + snprintfz(dst, dst_len, "%0.2f KiB", (double)size / 1024.0); + else + snprintfz(dst, dst_len, "%"PRIu64" B", size); +} + +#define print_duration(dst, dst_len, pos, remaining, duration, one, many, printed) do { \ + if((remaining) > (duration)) { \ + uint64_t _count = (remaining) / (duration); \ + uint64_t _rem = (remaining) - (_count * (duration)); \ + (pos) += snprintfz(&(dst)[pos], (dst_len) - (pos), "%s%s%"PRIu64" %s", (printed) ? ", " : "", _rem ? "" : "and ", _count, _count > 1 ? (many) : (one)); \ + (remaining) = _rem; \ + (printed) = true; \ + } \ +} while(0) + +static void human_readable_duration_s(time_t duration_s, char *dst, size_t dst_len) { + if(duration_s < 0) + duration_s = -duration_s; + + size_t pos = 0; + dst[0] = 0 ; + + bool printed = false; + print_duration(dst, dst_len, pos, duration_s, 86400 * 365, "year", "years", printed); + print_duration(dst, dst_len, pos, duration_s, 86400 * 30, "month", "months", printed); + print_duration(dst, dst_len, pos, duration_s, 86400 * 1, "day", "days", printed); + print_duration(dst, dst_len, pos, duration_s, 3600 * 1, "hour", "hours", printed); + print_duration(dst, dst_len, pos, duration_s, 60 * 1, "min", "mins", printed); + print_duration(dst, dst_len, pos, duration_s, 1, "sec", "secs", printed); +} + +static int journal_file_to_json_array_cb(const DICTIONARY_ITEM *item, void *entry, void *data) { + struct journal_file_source *jfs = entry; + BUFFER *wb = data; + + const char *name = dictionary_acquired_item_name(item); + + buffer_json_add_array_item_object(wb); + { + char size_for_humans[100]; + human_readable_size_ib(jfs->size, size_for_humans, sizeof(size_for_humans)); + + char duration_for_humans[1024]; + human_readable_duration_s((time_t)((jfs->last_ut - jfs->first_ut) / USEC_PER_SEC), + duration_for_humans, sizeof(duration_for_humans)); + + char info[1024]; + snprintfz(info, sizeof(info), "%zu files, with a total size of %s, covering %s", + jfs->count, size_for_humans, duration_for_humans); + + buffer_json_member_add_string(wb, "id", name); + buffer_json_member_add_string(wb, "name", name); + buffer_json_member_add_string(wb, "pill", size_for_humans); + buffer_json_member_add_string(wb, "info", info); + } + buffer_json_object_close(wb); // options object + + return 1; +} + +static bool journal_file_merge_sizes(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value , void *data __maybe_unused) { + struct journal_file_source *jfs = old_value, *njfs = new_value; + jfs->count += njfs->count; + jfs->size += njfs->size; + + if(njfs->first_ut && njfs->first_ut < jfs->first_ut) + jfs->first_ut = njfs->first_ut; + + if(njfs->last_ut && njfs->last_ut > jfs->last_ut) + jfs->last_ut = njfs->last_ut; + + return false; +} + +static void available_journal_file_sources_to_json_array(BUFFER *wb) { + DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_NAME_LINK_DONT_CLONE|DICT_OPTION_DONT_OVERWRITE_VALUE); + dictionary_register_conflict_callback(dict, journal_file_merge_sizes, NULL); + + struct journal_file_source t = { 0 }; + + struct journal_file *jf; + dfe_start_read(journal_files_registry, jf) { + t.first_ut = jf->msg_first_ut; + t.last_ut = jf->msg_last_ut; + t.count = 1; + t.size = jf->size; + + dictionary_set(dict, SDJF_SOURCE_ALL_NAME, &t, sizeof(t)); + + if((jf->source_type & (SDJF_LOCAL)) == (SDJF_LOCAL)) + dictionary_set(dict, SDJF_SOURCE_LOCAL_NAME, &t, sizeof(t)); + if((jf->source_type & (SDJF_LOCAL | SDJF_SYSTEM)) == (SDJF_LOCAL | SDJF_SYSTEM)) + dictionary_set(dict, SDJF_SOURCE_LOCAL_SYSTEM_NAME, &t, sizeof(t)); + if((jf->source_type & (SDJF_LOCAL | SDJF_USER)) == (SDJF_LOCAL | SDJF_USER)) + dictionary_set(dict, SDJF_SOURCE_LOCAL_USERS_NAME, &t, sizeof(t)); + if((jf->source_type & (SDJF_LOCAL | SDJF_OTHER)) == (SDJF_LOCAL | SDJF_OTHER)) + dictionary_set(dict, SDJF_SOURCE_LOCAL_OTHER_NAME, &t, sizeof(t)); + if((jf->source_type & (SDJF_NAMESPACE)) == (SDJF_NAMESPACE)) + dictionary_set(dict, SDJF_SOURCE_NAMESPACES_NAME, &t, sizeof(t)); + if((jf->source_type & (SDJF_REMOTE)) == (SDJF_REMOTE)) + dictionary_set(dict, SDJF_SOURCE_REMOTES_NAME, &t, sizeof(t)); + if(jf->source) + dictionary_set(dict, string2str(jf->source), &t, sizeof(t)); + } + dfe_done(jf); + + dictionary_sorted_walkthrough_read(dict, journal_file_to_json_array_cb, wb); + + dictionary_destroy(dict); +} + +static void files_registry_delete_cb(const DICTIONARY_ITEM *item, void *value, void *data __maybe_unused) { + struct journal_file *jf = value; (void)jf; + const char *filename = dictionary_acquired_item_name(item); (void)filename; + + string_freez(jf->source); + internal_error(true, "removed journal file '%s'", filename); +} + +void journal_directory_scan(const char *dirname, int depth, usec_t last_scan_ut) { + static const char *ext = ".journal"; + static const size_t ext_len = sizeof(".journal") - 1; + + if (depth > VAR_LOG_JOURNAL_MAX_DEPTH) + return; + + DIR *dir; + struct dirent *entry; + struct stat info; + char absolute_path[FILENAME_MAX]; + + // Open the directory. + if ((dir = opendir(dirname)) == NULL) { + if(errno != ENOENT && errno != ENOTDIR) + netdata_log_error("Cannot opendir() '%s'", dirname); + return; + } + + // Read each entry in the directory. + while ((entry = readdir(dir)) != NULL) { + snprintfz(absolute_path, sizeof(absolute_path), "%s/%s", dirname, entry->d_name); + if (stat(absolute_path, &info) != 0) { + netdata_log_error("Failed to stat() '%s", absolute_path); + continue; + } + + if (S_ISDIR(info.st_mode)) { + // If entry is a directory, call traverse recursively. + if (strcmp(entry->d_name, ".") != 0 && strcmp(entry->d_name, "..") != 0) + journal_directory_scan(absolute_path, depth + 1, last_scan_ut); + + } + else if (S_ISREG(info.st_mode)) { + // If entry is a regular file, check if it ends with .journal. + char *filename = entry->d_name; + size_t len = strlen(filename); + + if (len > ext_len && strcmp(filename + len - ext_len, ext) == 0) { + struct journal_file t = { + .file_last_modified_ut = info.st_mtim.tv_sec * USEC_PER_SEC + info.st_mtim.tv_nsec / NSEC_PER_USEC, + .last_scan_ut = last_scan_ut, + .size = info.st_size, + .max_journal_vs_realtime_delta_ut = JOURNAL_VS_REALTIME_DELTA_DEFAULT_UT, + }; + dictionary_set(journal_files_registry, absolute_path, &t, sizeof(t)); + } + } + } + + closedir(dir); +} + +static void journal_files_registry_update() { + usec_t scan_ut = now_monotonic_usec(); + + for(unsigned i = 0; i < MAX_JOURNAL_DIRECTORIES ;i++) { + if(!journal_directories[i].path) + break; + + journal_directory_scan(journal_directories[i].path, 0, scan_ut); + } + + struct journal_file *jf; + dfe_start_write(journal_files_registry, jf) { + if(jf->last_scan_ut < scan_ut) + dictionary_del(journal_files_registry, jf_dfe.name); + } + dfe_done(jf); +} + +// ---------------------------------------------------------------------------- + +static bool jf_is_mine(struct journal_file *jf, FUNCTION_QUERY_STATUS *fqs) { + + if((fqs->source_type == SDJF_ALL || (jf->source_type & fqs->source_type) == fqs->source_type) && + (!fqs->source || fqs->source == jf->source)) { + + usec_t anchor_delta = JOURNAL_VS_REALTIME_DELTA_MAX_UT; + usec_t first_ut = jf->msg_first_ut; + usec_t last_ut = jf->msg_last_ut + anchor_delta; + + if(last_ut >= fqs->after_ut && first_ut <= fqs->before_ut) + return true; + } + + return false; +} + +static int journal_file_dict_items_backward_compar(const void *a, const void *b) { + const DICTIONARY_ITEM **ad = (const DICTIONARY_ITEM **)a, **bd = (const DICTIONARY_ITEM **)b; + struct journal_file *jfa = dictionary_acquired_item_value(*ad); + struct journal_file *jfb = dictionary_acquired_item_value(*bd); + + if(jfa->msg_last_ut < jfb->msg_last_ut) + return 1; + + if(jfa->msg_last_ut > jfb->msg_last_ut) + return -1; + + if(jfa->msg_first_ut < jfb->msg_first_ut) + return 1; + + if(jfa->msg_first_ut > jfb->msg_first_ut) + return -1; + + return 0; +} - size_t errors_no_timestamp = 0; - size_t row_counter = 0; - size_t rows_added = 0; +static int journal_file_dict_items_forward_compar(const void *a, const void *b) { + return -journal_file_dict_items_backward_compar(a, b); +} - // the entries are not guaranteed to be sorted, so we process up to 100 entries beyond - // the end of the query to find possibly useful logs for our time-frame - size_t excess_rows_allowed = SYSTEMD_JOURNAL_EXCESS_ROWS_ALLOWED; +static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QUERY_STATUS *fqs) { + ND_SD_JOURNAL_STATUS status = ND_SD_JOURNAL_NO_FILE_MATCHED; + struct journal_file *jf; - ND_SD_JOURNAL_STATUS status = ND_SD_JOURNAL_OK; + fqs->files_matched = 0; + fqs->file_working = 0; - facets_rows_begin(facets); - while (status == ND_SD_JOURNAL_OK && sd_journal_previous(j) > 0) { - row_counter++; + size_t files_used = 0; + size_t files_max = dictionary_entries(journal_files_registry); + const DICTIONARY_ITEM *file_items[files_max]; - usec_t msg_ut; - if(sd_journal_get_realtime_usec(j, &msg_ut) < 0) { - errors_no_timestamp++; + // count the files + bool files_are_newer = false; + dfe_start_read(journal_files_registry, jf) { + if(!jf_is_mine(jf, fqs)) continue; - } - if (msg_ut > before_ut || msg_ut >= anchor) - continue; + file_items[files_used++] = dictionary_acquired_item_dup(journal_files_registry, jf_dfe.item); - if (msg_ut < after_ut) { - if(--excess_rows_allowed == 0) - break; + if(jf->msg_last_ut > fqs->if_modified_since) + files_are_newer = true; + } + dfe_done(jf); - continue; - } + fqs->files_matched = files_used; - if(rows_added > entries && --excess_rows_allowed == 0) - break; + if(fqs->if_modified_since && !files_are_newer) { + buffer_flush(wb); + return HTTP_RESP_NOT_MODIFIED; + } - netdata_systemd_journal_process_row(j, facets); - facets_row_finished(facets, msg_ut); - rows_added++; + // We will not do an if_modified_since query + // we know something changed in the files + fqs->if_modified_since = 0; - status = check_stop(row_counter, cancelled, stop_monotonic_ut); + // sort the files, so that they are optimal for facets + if(files_used >= 2) { + if (fqs->direction == FACETS_ANCHOR_DIRECTION_BACKWARD) + qsort(file_items, files_used, sizeof(const DICTIONARY_ITEM *), + journal_file_dict_items_backward_compar); + else + qsort(file_items, files_used, sizeof(const DICTIONARY_ITEM *), + journal_file_dict_items_forward_compar); } - if(errors_no_timestamp) - netdata_log_error("SYSTEMD-JOURNAL: %zu lines did not have timestamps", errors_no_timestamp); + bool partial = false; + usec_t started_ut; + usec_t ended_ut = now_monotonic_usec(); - return status; -} + buffer_json_member_add_array(wb, "_journal_files"); + for(size_t f = 0; f < files_used ;f++) { + const char *filename = dictionary_acquired_item_name(file_items[f]); + jf = dictionary_acquired_item_value(file_items[f]); -bool netdata_systemd_journal_check_if_modified_since(sd_journal *j, usec_t seek_to, usec_t last_modified) { - // return true, if data have been modified since the timestamp + if(!jf_is_mine(jf, fqs)) + continue; - if(!last_modified || !seek_to) - return false; + fqs->file_working++; + fqs->cached_count = 0; - if(!netdata_systemd_journal_seek_to(j, seek_to)) - return false; + size_t rows_useful = fqs->rows_useful; + size_t rows_read = fqs->rows_read; + size_t bytes_read = fqs->bytes_read; + size_t matches_setup_ut = fqs->matches_setup_ut; - usec_t first_msg_ut = 0; - while (sd_journal_previous(j) > 0) { - usec_t msg_ut; - if(sd_journal_get_realtime_usec(j, &msg_ut) < 0) - continue; + ND_SD_JOURNAL_STATUS tmp_status = netdata_systemd_journal_query_one_file(filename, wb, facets, jf, fqs); - first_msg_ut = msg_ut; - break; - } + rows_useful = fqs->rows_useful - rows_useful; + rows_read = fqs->rows_read - rows_read; + bytes_read = fqs->bytes_read - bytes_read; + matches_setup_ut = fqs->matches_setup_ut - matches_setup_ut; - return first_msg_ut != last_modified; -} + started_ut = ended_ut; + ended_ut = now_monotonic_usec(); + usec_t duration_ut = ended_ut - started_ut; -static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, - usec_t after_ut, usec_t before_ut, - usec_t anchor, FACETS_ANCHOR_DIRECTION direction, size_t entries, - usec_t if_modified_since, bool data_only, - usec_t stop_monotonic_ut, - bool *cancelled) { - sd_journal *j = netdata_open_systemd_journal(); - if(!j) - return HTTP_RESP_INTERNAL_SERVER_ERROR; + buffer_json_add_array_item_object(wb); // journal file + { + // information about the file + buffer_json_member_add_string(wb, "_filename", filename); + buffer_json_member_add_uint64(wb, "_source_type", jf->source_type); + buffer_json_member_add_string(wb, "_source", string2str(jf->source)); + buffer_json_member_add_uint64(wb, "_last_modified_ut", jf->file_last_modified_ut); + buffer_json_member_add_uint64(wb, "_msg_first_ut", jf->msg_first_ut); + buffer_json_member_add_uint64(wb, "_msg_last_ut", jf->msg_last_ut); + buffer_json_member_add_uint64(wb, "_journal_vs_realtime_delta_ut", jf->max_journal_vs_realtime_delta_ut); + + // information about the current use of the file + buffer_json_member_add_uint64(wb, "duration_ut", ended_ut - started_ut); + buffer_json_member_add_uint64(wb, "rows_read", rows_read); + buffer_json_member_add_uint64(wb, "rows_useful", rows_useful); + buffer_json_member_add_double(wb, "rows_per_second", (double) rows_read / (double) duration_ut * (double) USEC_PER_SEC); + buffer_json_member_add_uint64(wb, "bytes_read", bytes_read); + buffer_json_member_add_double(wb, "bytes_per_second", (double) bytes_read / (double) duration_ut * (double) USEC_PER_SEC); + buffer_json_member_add_uint64(wb, "duration_matches_ut", matches_setup_ut); + } + buffer_json_object_close(wb); // journal file - usec_t last_modified = 0; + bool stop = false; + switch(tmp_status) { + case ND_SD_JOURNAL_OK: + case ND_SD_JOURNAL_NO_FILE_MATCHED: + status = (status == ND_SD_JOURNAL_OK) ? ND_SD_JOURNAL_OK : tmp_status; + break; - ND_SD_JOURNAL_STATUS status; + case ND_SD_JOURNAL_FAILED_TO_OPEN: + case ND_SD_JOURNAL_FAILED_TO_SEEK: + partial = true; + if(status == ND_SD_JOURNAL_NO_FILE_MATCHED) + status = tmp_status; + break; - if(data_only && anchor /* && !netdata_systemd_journal_check_if_modified_since(j, before_ut, if_modified_since) */) { - facets_data_only_mode(facets); + case ND_SD_JOURNAL_CANCELLED: + case ND_SD_JOURNAL_TIMED_OUT: + partial = true; + stop = true; + status = tmp_status; + break; - // we can do a data-only query - if(direction == FACETS_ANCHOR_DIRECTION_FORWARD) - status = netdata_systemd_journal_query_data_forward(j, wb, facets, after_ut, before_ut, anchor, entries, stop_monotonic_ut, cancelled); - else - status = netdata_systemd_journal_query_data_backward(j, wb, facets, after_ut, before_ut, anchor, entries, stop_monotonic_ut, cancelled); - } - else { - // we have to do a full query - status = netdata_systemd_journal_query_full(j, wb, facets, - after_ut, before_ut, if_modified_since, - stop_monotonic_ut, &last_modified, cancelled); + case ND_SD_JOURNAL_NOT_MODIFIED: + internal_fatal(true, "this should never be returned here"); + break; + } + + if(stop) + break; } + buffer_json_array_close(wb); // _journal_files - sd_journal_close(j); + // release the files + for(size_t f = 0; f < files_used ;f++) + dictionary_acquired_item_release(journal_files_registry, file_items[f]); - if(status != ND_SD_JOURNAL_OK && status != ND_SD_JOURNAL_TIMED_OUT) { - buffer_flush(wb); + switch (status) { + case ND_SD_JOURNAL_OK: + case ND_SD_JOURNAL_TIMED_OUT: + case ND_SD_JOURNAL_NO_FILE_MATCHED: + break; - switch (status) { - case ND_SD_JOURNAL_CANCELLED: - return HTTP_RESP_CLIENT_CLOSED_REQUEST; + case ND_SD_JOURNAL_CANCELLED: + buffer_flush(wb); + return HTTP_RESP_CLIENT_CLOSED_REQUEST; - case ND_SD_JOURNAL_NOT_MODIFIED: - return HTTP_RESP_NOT_MODIFIED; + case ND_SD_JOURNAL_NOT_MODIFIED: + buffer_flush(wb); + return HTTP_RESP_NOT_MODIFIED; - default: - case ND_SD_JOURNAL_FAILED_TO_SEEK: - return HTTP_RESP_INTERNAL_SERVER_ERROR; - } + default: + case ND_SD_JOURNAL_FAILED_TO_OPEN: + case ND_SD_JOURNAL_FAILED_TO_SEEK: + buffer_flush(wb); + return HTTP_RESP_INTERNAL_SERVER_ERROR; } buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK); - buffer_json_member_add_boolean(wb, "partial", status != ND_SD_JOURNAL_OK); + buffer_json_member_add_boolean(wb, "partial", partial); buffer_json_member_add_string(wb, "type", "table"); - if(!data_only) { + if(!fqs->data_only) { buffer_json_member_add_time_t(wb, "update_every", 1); buffer_json_member_add_string(wb, "help", SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION); - buffer_json_member_add_uint64(wb, "last_modified", last_modified); } - facets_report(facets, wb); + if(!fqs->data_only || fqs->tail) + buffer_json_member_add_uint64(wb, "last_modified", fqs->last_modified); + + facets_sort_and_reorder_keys(facets); + facets_report(facets, wb, used_hashes_registry); - buffer_json_member_add_time_t(wb, "expires", now_realtime_sec() + (data_only ? 3600 : 0)); + buffer_json_member_add_time_t(wb, "expires", now_realtime_sec() + (fqs->data_only ? 3600 : 0)); buffer_json_finalize(wb); return HTTP_RESP_OK; @@ -408,38 +1269,108 @@ static void netdata_systemd_journal_function_help(const char *transaction) { "\n" "%s\n" "\n" - "The following filters are supported:\n" + "The following parameters are supported:\n" "\n" - " help\n" + " "JOURNAL_PARAMETER_HELP"\n" " Shows this help message.\n" "\n" - " before:TIMESTAMP\n" + " "JOURNAL_PARAMETER_ID":STRING\n" + " Caller supplied unique ID of the request.\n" + " This can be used later to request a progress report of the query.\n" + " Optional, but if omitted no `"JOURNAL_PARAMETER_PROGRESS"` can be requested.\n" + "\n" + " "JOURNAL_PARAMETER_INFO"\n" + " Request initial configuration information about the plugin.\n" + " The key entity returned is the required_params array, which includes\n" + " all the available systemd journal sources.\n" + " When `"JOURNAL_PARAMETER_INFO"` is requested, all other parameters are ignored.\n" + "\n" + " "JOURNAL_PARAMETER_DELTA"\n" + " When doing data queries, include deltas for histogram and facets.\n" + "\n" + " "JOURNAL_PARAMETER_TAIL"\n" + " Do a tail query, to return the newest items between the anchor and before.\n" + "\n" + " "JOURNAL_PARAMETER_PROGRESS"\n" + " Request a progress report (the `id` of a running query is required).\n" + " When `"JOURNAL_PARAMETER_PROGRESS"` is requested, only parameter `"JOURNAL_PARAMETER_ID"` is used.\n" + "\n" + " "JOURNAL_PARAMETER_DATA_ONLY"\n" + " Quickly respond with data requested, without generating a\n" + " histogram and facets counters.\n" + "\n" + " "JOURNAL_PARAMETER_SLICE":true or "JOURNAL_PARAMETER_SLICE":false\n" + " When it is turned on, the plugin is executing filtering via libsystemd,\n" + " utilizing all the available indexes of the journal files.\n" + " When it is off, only the time constraint is handled by libsystemd and\n" + " all filtering is done by the plugin.\n" + " The default is: %s\n" + "\n" + " "JOURNAL_PARAMETER_SOURCE":SOURCE\n" + " Query only the specified journal sources.\n" + " Do an `"JOURNAL_PARAMETER_INFO"` query to find the sources.\n" + "\n" + " "JOURNAL_PARAMETER_BEFORE":TIMESTAMP_IN_SECONDS\n" " Absolute or relative (to now) timestamp in seconds, to start the query.\n" " The query is always executed from the most recent to the oldest log entry.\n" " If not given the default is: now.\n" "\n" - " after:TIMESTAMP\n" + " "JOURNAL_PARAMETER_AFTER":TIMESTAMP_IN_SECONDS\n" " Absolute or relative (to `before`) timestamp in seconds, to end the query.\n" " If not given, the default is %d.\n" "\n" - " last:ITEMS\n" + " "JOURNAL_PARAMETER_LAST":ITEMS\n" " The number of items to return.\n" " The default is %d.\n" "\n" - " anchor:NUMBER\n" - " The `timestamp` of the item last received, to return log entries after that.\n" - " If not given, the query will return the top `ITEMS` from the most recent.\n" + " "JOURNAL_PARAMETER_ANCHOR":TIMESTAMP_IN_MICROSECONDS\n" + " Return items relative to this timestamp.\n" + " The exact items to be returned depend on the query `"JOURNAL_PARAMETER_DIRECTION"`.\n" + "\n" + " "JOURNAL_PARAMETER_DIRECTION":forward or "JOURNAL_PARAMETER_DIRECTION":backward\n" + " When set to `backward` (default) the items returned are the newest before the\n" + " `"JOURNAL_PARAMETER_ANCHOR"`, (or `"JOURNAL_PARAMETER_BEFORE"` if `"JOURNAL_PARAMETER_ANCHOR"` is not set)\n" + " When set to `forward` the items returned are the oldest after the\n" + " `"JOURNAL_PARAMETER_ANCHOR"`, (or `"JOURNAL_PARAMETER_AFTER"` if `"JOURNAL_PARAMETER_ANCHOR"` is not set)\n" + " The default is: %s\n" + "\n" + " "JOURNAL_PARAMETER_QUERY":SIMPLE_PATTERN\n" + " Do a full text search to find the log entries matching the pattern given.\n" + " The plugin is searching for matches on all fields of the database.\n" + "\n" + " "JOURNAL_PARAMETER_IF_MODIFIED_SINCE":TIMESTAMP_IN_MICROSECONDS\n" + " Each successful response, includes a `last_modified` field.\n" + " By providing the timestamp to the `"JOURNAL_PARAMETER_IF_MODIFIED_SINCE"` parameter,\n" + " the plugin will return 200 with a successful response, or 304 if the source has not\n" + " been modified since that timestamp.\n" + "\n" + " "JOURNAL_PARAMETER_HISTOGRAM":facet_id\n" + " Use the given `facet_id` for the histogram.\n" + " This parameter is ignored in `"JOURNAL_PARAMETER_DATA_ONLY"` mode.\n" + "\n" + " "JOURNAL_PARAMETER_FACETS":facet_id1,facet_id2,facet_id3,...\n" + " Add the given facets to the list of fields for which analysis is required.\n" + " The plugin will offer both a histogram and facet value counters for its values.\n" + " This parameter is ignored in `"JOURNAL_PARAMETER_DATA_ONLY"` mode.\n" "\n" " facet_id:value_id1,value_id2,value_id3,...\n" " Apply filters to the query, based on the facet IDs returned.\n" " Each `facet_id` can be given once, but multiple `facet_ids` can be given.\n" "\n" - "Filters can be combined. Each filter can be given only one time.\n" + " There is special mode. By specifying:\n" + "\n" + " - `"JOURNAL_PARAMETER_DIRECTION":forward`,\n" + " - `"JOURNAL_PARAMETER_ANCHOR":TIMESTAMP_IN_USEC`,\n" + " - `"JOURNAL_PARAMETER_DATA_ONLY"`, and\n" + " - `"JOURNAL_PARAMETER_IF_MODIFIED_SINCE":TIMESTAMP_IN_USEC`\n" + "\n" , program_name , SYSTEMD_JOURNAL_FUNCTION_NAME , SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION + , JOURNAL_DEFAULT_SLICE_MODE ? "true" : "false" // slice , -SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION , SYSTEMD_JOURNAL_DEFAULT_ITEMS_PER_QUERY + , JOURNAL_DEFAULT_DIRECTION == FACETS_ANCHOR_DIRECTION_BACKWARD ? "backward" : "forward" ); netdata_mutex_lock(&stdout_mutex); @@ -449,6 +1380,140 @@ static void netdata_systemd_journal_function_help(const char *transaction) { buffer_free(wb); } +const char *errno_map[] = { + [1] = "1 (EPERM)", // "Operation not permitted", + [2] = "2 (ENOENT)", // "No such file or directory", + [3] = "3 (ESRCH)", // "No such process", + [4] = "4 (EINTR)", // "Interrupted system call", + [5] = "5 (EIO)", // "Input/output error", + [6] = "6 (ENXIO)", // "No such device or address", + [7] = "7 (E2BIG)", // "Argument list too long", + [8] = "8 (ENOEXEC)", // "Exec format error", + [9] = "9 (EBADF)", // "Bad file descriptor", + [10] = "10 (ECHILD)", // "No child processes", + [11] = "11 (EAGAIN)", // "Resource temporarily unavailable", + [12] = "12 (ENOMEM)", // "Cannot allocate memory", + [13] = "13 (EACCES)", // "Permission denied", + [14] = "14 (EFAULT)", // "Bad address", + [15] = "15 (ENOTBLK)", // "Block device required", + [16] = "16 (EBUSY)", // "Device or resource busy", + [17] = "17 (EEXIST)", // "File exists", + [18] = "18 (EXDEV)", // "Invalid cross-device link", + [19] = "19 (ENODEV)", // "No such device", + [20] = "20 (ENOTDIR)", // "Not a directory", + [21] = "21 (EISDIR)", // "Is a directory", + [22] = "22 (EINVAL)", // "Invalid argument", + [23] = "23 (ENFILE)", // "Too many open files in system", + [24] = "24 (EMFILE)", // "Too many open files", + [25] = "25 (ENOTTY)", // "Inappropriate ioctl for device", + [26] = "26 (ETXTBSY)", // "Text file busy", + [27] = "27 (EFBIG)", // "File too large", + [28] = "28 (ENOSPC)", // "No space left on device", + [29] = "29 (ESPIPE)", // "Illegal seek", + [30] = "30 (EROFS)", // "Read-only file system", + [31] = "31 (EMLINK)", // "Too many links", + [32] = "32 (EPIPE)", // "Broken pipe", + [33] = "33 (EDOM)", // "Numerical argument out of domain", + [34] = "34 (ERANGE)", // "Numerical result out of range", + [35] = "35 (EDEADLK)", // "Resource deadlock avoided", + [36] = "36 (ENAMETOOLONG)", // "File name too long", + [37] = "37 (ENOLCK)", // "No locks available", + [38] = "38 (ENOSYS)", // "Function not implemented", + [39] = "39 (ENOTEMPTY)", // "Directory not empty", + [40] = "40 (ELOOP)", // "Too many levels of symbolic links", + [42] = "42 (ENOMSG)", // "No message of desired type", + [43] = "43 (EIDRM)", // "Identifier removed", + [44] = "44 (ECHRNG)", // "Channel number out of range", + [45] = "45 (EL2NSYNC)", // "Level 2 not synchronized", + [46] = "46 (EL3HLT)", // "Level 3 halted", + [47] = "47 (EL3RST)", // "Level 3 reset", + [48] = "48 (ELNRNG)", // "Link number out of range", + [49] = "49 (EUNATCH)", // "Protocol driver not attached", + [50] = "50 (ENOCSI)", // "No CSI structure available", + [51] = "51 (EL2HLT)", // "Level 2 halted", + [52] = "52 (EBADE)", // "Invalid exchange", + [53] = "53 (EBADR)", // "Invalid request descriptor", + [54] = "54 (EXFULL)", // "Exchange full", + [55] = "55 (ENOANO)", // "No anode", + [56] = "56 (EBADRQC)", // "Invalid request code", + [57] = "57 (EBADSLT)", // "Invalid slot", + [59] = "59 (EBFONT)", // "Bad font file format", + [60] = "60 (ENOSTR)", // "Device not a stream", + [61] = "61 (ENODATA)", // "No data available", + [62] = "62 (ETIME)", // "Timer expired", + [63] = "63 (ENOSR)", // "Out of streams resources", + [64] = "64 (ENONET)", // "Machine is not on the network", + [65] = "65 (ENOPKG)", // "Package not installed", + [66] = "66 (EREMOTE)", // "Object is remote", + [67] = "67 (ENOLINK)", // "Link has been severed", + [68] = "68 (EADV)", // "Advertise error", + [69] = "69 (ESRMNT)", // "Srmount error", + [70] = "70 (ECOMM)", // "Communication error on send", + [71] = "71 (EPROTO)", // "Protocol error", + [72] = "72 (EMULTIHOP)", // "Multihop attempted", + [73] = "73 (EDOTDOT)", // "RFS specific error", + [74] = "74 (EBADMSG)", // "Bad message", + [75] = "75 (EOVERFLOW)", // "Value too large for defined data type", + [76] = "76 (ENOTUNIQ)", // "Name not unique on network", + [77] = "77 (EBADFD)", // "File descriptor in bad state", + [78] = "78 (EREMCHG)", // "Remote address changed", + [79] = "79 (ELIBACC)", // "Can not access a needed shared library", + [80] = "80 (ELIBBAD)", // "Accessing a corrupted shared library", + [81] = "81 (ELIBSCN)", // ".lib section in a.out corrupted", + [82] = "82 (ELIBMAX)", // "Attempting to link in too many shared libraries", + [83] = "83 (ELIBEXEC)", // "Cannot exec a shared library directly", + [84] = "84 (EILSEQ)", // "Invalid or incomplete multibyte or wide character", + [85] = "85 (ERESTART)", // "Interrupted system call should be restarted", + [86] = "86 (ESTRPIPE)", // "Streams pipe error", + [87] = "87 (EUSERS)", // "Too many users", + [88] = "88 (ENOTSOCK)", // "Socket operation on non-socket", + [89] = "89 (EDESTADDRREQ)", // "Destination address required", + [90] = "90 (EMSGSIZE)", // "Message too long", + [91] = "91 (EPROTOTYPE)", // "Protocol wrong type for socket", + [92] = "92 (ENOPROTOOPT)", // "Protocol not available", + [93] = "93 (EPROTONOSUPPORT)", // "Protocol not supported", + [94] = "94 (ESOCKTNOSUPPORT)", // "Socket type not supported", + [95] = "95 (ENOTSUP)", // "Operation not supported", + [96] = "96 (EPFNOSUPPORT)", // "Protocol family not supported", + [97] = "97 (EAFNOSUPPORT)", // "Address family not supported by protocol", + [98] = "98 (EADDRINUSE)", // "Address already in use", + [99] = "99 (EADDRNOTAVAIL)", // "Cannot assign requested address", + [100] = "100 (ENETDOWN)", // "Network is down", + [101] = "101 (ENETUNREACH)", // "Network is unreachable", + [102] = "102 (ENETRESET)", // "Network dropped connection on reset", + [103] = "103 (ECONNABORTED)", // "Software caused connection abort", + [104] = "104 (ECONNRESET)", // "Connection reset by peer", + [105] = "105 (ENOBUFS)", // "No buffer space available", + [106] = "106 (EISCONN)", // "Transport endpoint is already connected", + [107] = "107 (ENOTCONN)", // "Transport endpoint is not connected", + [108] = "108 (ESHUTDOWN)", // "Cannot send after transport endpoint shutdown", + [109] = "109 (ETOOMANYREFS)", // "Too many references: cannot splice", + [110] = "110 (ETIMEDOUT)", // "Connection timed out", + [111] = "111 (ECONNREFUSED)", // "Connection refused", + [112] = "112 (EHOSTDOWN)", // "Host is down", + [113] = "113 (EHOSTUNREACH)", // "No route to host", + [114] = "114 (EALREADY)", // "Operation already in progress", + [115] = "115 (EINPROGRESS)", // "Operation now in progress", + [116] = "116 (ESTALE)", // "Stale file handle", + [117] = "117 (EUCLEAN)", // "Structure needs cleaning", + [118] = "118 (ENOTNAM)", // "Not a XENIX named type file", + [119] = "119 (ENAVAIL)", // "No XENIX semaphores available", + [120] = "120 (EISNAM)", // "Is a named type file", + [121] = "121 (EREMOTEIO)", // "Remote I/O error", + [122] = "122 (EDQUOT)", // "Disk quota exceeded", + [123] = "123 (ENOMEDIUM)", // "No medium found", + [124] = "124 (EMEDIUMTYPE)", // "Wrong medium type", + [125] = "125 (ECANCELED)", // "Operation canceled", + [126] = "126 (ENOKEY)", // "Required key not available", + [127] = "127 (EKEYEXPIRED)", // "Key has expired", + [128] = "128 (EKEYREVOKED)", // "Key has been revoked", + [129] = "129 (EKEYREJECTED)", // "Key was rejected by service", + [130] = "130 (EOWNERDEAD)", // "Owner died", + [131] = "131 (ENOTRECOVERABLE)", // "State not recoverable", + [132] = "132 (ERFKILL)", // "Operation not possible due to RF-kill", + [133] = "133 (EHWPOISON)", // "Memory page has hardware error", +}; + static const char *syslog_facility_to_name(int facility) { switch (facility) { case LOG_FAC(LOG_KERN): return "kern"; @@ -489,11 +1554,17 @@ static const char *syslog_priority_to_name(int priority) { } } -static FACET_ROW_SEVERITY syslog_priority_to_facet_severity(int priority) { +static FACET_ROW_SEVERITY syslog_priority_to_facet_severity(FACETS *facets __maybe_unused, FACET_ROW *row, void *data __maybe_unused) { // same to // https://github.com/systemd/systemd/blob/aab9e4b2b86905a15944a1ac81e471b5b7075932/src/basic/terminal-util.c#L1501 // function get_log_colors() + FACET_ROW_KEY_VALUE *priority_rkv = dictionary_get(row->dict, "PRIORITY"); + if(!priority_rkv || priority_rkv->empty) + return FACET_ROW_SEVERITY_NORMAL; + + int priority = str2i(buffer_tostring(priority_rkv->wb)); + if(priority <= LOG_ERR) return FACET_ROW_SEVERITY_CRITICAL; @@ -533,7 +1604,7 @@ static char *gid_to_groupname(gid_t gid, char* buffer, size_t buffer_size) { return buffer; } -static void netdata_systemd_journal_transform_syslog_facility(FACETS *facets __maybe_unused, BUFFER *wb, void *data __maybe_unused) { +static void netdata_systemd_journal_transform_syslog_facility(FACETS *facets __maybe_unused, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope __maybe_unused, void *data __maybe_unused) { const char *v = buffer_tostring(wb); if(*v && isdigit(*v)) { int facility = str2i(buffer_tostring(wb)); @@ -545,7 +1616,10 @@ static void netdata_systemd_journal_transform_syslog_facility(FACETS *facets __m } } -static void netdata_systemd_journal_transform_priority(FACETS *facets __maybe_unused, BUFFER *wb, void *data __maybe_unused) { +static void netdata_systemd_journal_transform_priority(FACETS *facets __maybe_unused, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope __maybe_unused, void *data __maybe_unused) { + if(scope == FACETS_TRANSFORM_FACET_SORT) + return; + const char *v = buffer_tostring(wb); if(*v && isdigit(*v)) { int priority = str2i(buffer_tostring(wb)); @@ -554,8 +1628,23 @@ static void netdata_systemd_journal_transform_priority(FACETS *facets __maybe_un buffer_flush(wb); buffer_strcat(wb, name); } + } +} - facets_set_current_row_severity(facets, syslog_priority_to_facet_severity(priority)); +static void netdata_systemd_journal_transform_errno(FACETS *facets __maybe_unused, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope __maybe_unused, void *data __maybe_unused) { + if(scope == FACETS_TRANSFORM_FACET_SORT) + return; + + const char *v = buffer_tostring(wb); + if(*v && isdigit(*v)) { + unsigned err_no = str2u(buffer_tostring(wb)); + if(err_no > 0 && err_no < sizeof(errno_map) / sizeof(*errno_map)) { + const char *name = errno_map[err_no]; + if(name) { + buffer_flush(wb); + buffer_strcat(wb, name); + } + } } } @@ -637,7 +1726,78 @@ const char *gid_to_groupname_cached(gid_t gid, size_t *length) { return (*e)->str; } -static void netdata_systemd_journal_transform_uid(FACETS *facets __maybe_unused, BUFFER *wb, void *data __maybe_unused) { +DICTIONARY *boot_ids_to_first_ut = NULL; + +static void netdata_systemd_journal_transform_boot_id(FACETS *facets __maybe_unused, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope __maybe_unused, void *data __maybe_unused) { + const char *boot_id = buffer_tostring(wb); + if(*boot_id && isxdigit(*boot_id)) { + usec_t ut = UINT64_MAX; + usec_t *p_ut = dictionary_get(boot_ids_to_first_ut, boot_id); + if(!p_ut) { + struct journal_file *jf; + dfe_start_read(journal_files_registry, jf) { + const char *files[2] = { + [0] = jf_dfe.name, + [1] = NULL, + }; + + sd_journal *j = NULL; + if(sd_journal_open_files(&j, files, ND_SD_JOURNAL_OPEN_FLAGS) < 0 || !j) + continue; + + char m[100]; + size_t len = snprintfz(m, sizeof(m), "_BOOT_ID=%s", boot_id); + usec_t t_ut = 0; + if(sd_journal_add_match(j, m, len) < 0 || + sd_journal_seek_head(j) < 0 || + sd_journal_next(j) < 0 || + sd_journal_get_realtime_usec(j, &t_ut) < 0 || !t_ut) { + sd_journal_close(j); + continue; + } + + if(t_ut < ut) + ut = t_ut; + + sd_journal_close(j); + } + dfe_done(jf); + + dictionary_set(boot_ids_to_first_ut, boot_id, &ut, sizeof(ut)); + } + else + ut = *p_ut; + + if(ut != UINT64_MAX) { + time_t timestamp_sec = (time_t)(ut / USEC_PER_SEC); + struct tm tm; + char buffer[30]; + + gmtime_r(×tamp_sec, &tm); + strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", &tm); + + switch(scope) { + default: + case FACETS_TRANSFORM_DATA: + case FACETS_TRANSFORM_VALUE: + buffer_sprintf(wb, " (%s UTC) ", buffer); + break; + + case FACETS_TRANSFORM_FACET: + case FACETS_TRANSFORM_FACET_SORT: + case FACETS_TRANSFORM_HISTOGRAM: + buffer_flush(wb); + buffer_sprintf(wb, "%s UTC", buffer); + break; + } + } + } +} + +static void netdata_systemd_journal_transform_uid(FACETS *facets __maybe_unused, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope __maybe_unused, void *data __maybe_unused) { + if(scope == FACETS_TRANSFORM_FACET_SORT) + return; + const char *v = buffer_tostring(wb); if(*v && isdigit(*v)) { uid_t uid = str2i(buffer_tostring(wb)); @@ -647,7 +1807,10 @@ static void netdata_systemd_journal_transform_uid(FACETS *facets __maybe_unused, } } -static void netdata_systemd_journal_transform_gid(FACETS *facets __maybe_unused, BUFFER *wb, void *data __maybe_unused) { +static void netdata_systemd_journal_transform_gid(FACETS *facets __maybe_unused, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope __maybe_unused, void *data __maybe_unused) { + if(scope == FACETS_TRANSFORM_FACET_SORT) + return; + const char *v = buffer_tostring(wb); if(*v && isdigit(*v)) { gid_t gid = str2i(buffer_tostring(wb)); @@ -657,24 +1820,120 @@ static void netdata_systemd_journal_transform_gid(FACETS *facets __maybe_unused, } } +const char *linux_capabilities[] = { + [CAP_CHOWN] = "CHOWN", + [CAP_DAC_OVERRIDE] = "DAC_OVERRIDE", + [CAP_DAC_READ_SEARCH] = "DAC_READ_SEARCH", + [CAP_FOWNER] = "FOWNER", + [CAP_FSETID] = "FSETID", + [CAP_KILL] = "KILL", + [CAP_SETGID] = "SETGID", + [CAP_SETUID] = "SETUID", + [CAP_SETPCAP] = "SETPCAP", + [CAP_LINUX_IMMUTABLE] = "LINUX_IMMUTABLE", + [CAP_NET_BIND_SERVICE] = "NET_BIND_SERVICE", + [CAP_NET_BROADCAST] = "NET_BROADCAST", + [CAP_NET_ADMIN] = "NET_ADMIN", + [CAP_NET_RAW] = "NET_RAW", + [CAP_IPC_LOCK] = "IPC_LOCK", + [CAP_IPC_OWNER] = "IPC_OWNER", + [CAP_SYS_MODULE] = "SYS_MODULE", + [CAP_SYS_RAWIO] = "SYS_RAWIO", + [CAP_SYS_CHROOT] = "SYS_CHROOT", + [CAP_SYS_PTRACE] = "SYS_PTRACE", + [CAP_SYS_PACCT] = "SYS_PACCT", + [CAP_SYS_ADMIN] = "SYS_ADMIN", + [CAP_SYS_BOOT] = "SYS_BOOT", + [CAP_SYS_NICE] = "SYS_NICE", + [CAP_SYS_RESOURCE] = "SYS_RESOURCE", + [CAP_SYS_TIME] = "SYS_TIME", + [CAP_SYS_TTY_CONFIG] = "SYS_TTY_CONFIG", + [CAP_MKNOD] = "MKNOD", + [CAP_LEASE] = "LEASE", + [CAP_AUDIT_WRITE] = "AUDIT_WRITE", + [CAP_AUDIT_CONTROL] = "AUDIT_CONTROL", + [CAP_SETFCAP] = "SETFCAP", + [CAP_MAC_OVERRIDE] = "MAC_OVERRIDE", + [CAP_MAC_ADMIN] = "MAC_ADMIN", + [CAP_SYSLOG] = "SYSLOG", + [CAP_WAKE_ALARM] = "WAKE_ALARM", + [CAP_BLOCK_SUSPEND] = "BLOCK_SUSPEND", + [37 /*CAP_AUDIT_READ*/] = "AUDIT_READ", + [38 /*CAP_PERFMON*/] = "PERFMON", + [39 /*CAP_BPF*/] = "BPF", + [40 /* CAP_CHECKPOINT_RESTORE */] = "CHECKPOINT_RESTORE", +}; + +static void netdata_systemd_journal_transform_cap_effective(FACETS *facets __maybe_unused, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope __maybe_unused, void *data __maybe_unused) { + if(scope == FACETS_TRANSFORM_FACET_SORT) + return; + + const char *v = buffer_tostring(wb); + if(*v && isdigit(*v)) { + uint64_t cap = strtoul(buffer_tostring(wb), NULL, 16); + if(cap) { + buffer_fast_strcat(wb, " (", 2); + for (size_t i = 0, added = 0; i < sizeof(linux_capabilities) / sizeof(linux_capabilities[0]); i++) { + if (linux_capabilities[i] && (cap & (1ULL << i))) { + + if (added) + buffer_fast_strcat(wb, " | ", 3); + + buffer_strcat(wb, linux_capabilities[i]); + added++; + } + } + buffer_fast_strcat(wb, ")", 1); + } + } +} + +static void netdata_systemd_journal_transform_timestamp_usec(FACETS *facets __maybe_unused, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope __maybe_unused, void *data __maybe_unused) { + if(scope == FACETS_TRANSFORM_FACET_SORT) + return; + + const char *v = buffer_tostring(wb); + if(*v && isdigit(*v)) { + uint64_t ut = str2ull(buffer_tostring(wb), NULL); + if(ut) { + time_t timestamp_sec = ut / USEC_PER_SEC; + struct tm tm; + char buffer[30]; + + gmtime_r(×tamp_sec, &tm); + strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", &tm); + buffer_sprintf(wb, " (%s.%06llu UTC)", buffer, ut % USEC_PER_SEC); + } + } +} + // ---------------------------------------------------------------------------- static void netdata_systemd_journal_dynamic_row_id(FACETS *facets __maybe_unused, BUFFER *json_array, FACET_ROW_KEY_VALUE *rkv, FACET_ROW *row, void *data __maybe_unused) { FACET_ROW_KEY_VALUE *pid_rkv = dictionary_get(row->dict, "_PID"); const char *pid = pid_rkv ? buffer_tostring(pid_rkv->wb) : FACET_VALUE_UNSET; - FACET_ROW_KEY_VALUE *syslog_identifier_rkv = dictionary_get(row->dict, "SYSLOG_IDENTIFIER"); - const char *identifier = syslog_identifier_rkv ? buffer_tostring(syslog_identifier_rkv->wb) : FACET_VALUE_UNSET; + const char *identifier = NULL; + FACET_ROW_KEY_VALUE *container_name_rkv = dictionary_get(row->dict, "CONTAINER_NAME"); + if(container_name_rkv && !container_name_rkv->empty) + identifier = buffer_tostring(container_name_rkv->wb); + + if(!identifier) { + FACET_ROW_KEY_VALUE *syslog_identifier_rkv = dictionary_get(row->dict, "SYSLOG_IDENTIFIER"); + if(syslog_identifier_rkv && !syslog_identifier_rkv->empty) + identifier = buffer_tostring(syslog_identifier_rkv->wb); - if(strcmp(identifier, FACET_VALUE_UNSET) == 0) { - FACET_ROW_KEY_VALUE *comm_rkv = dictionary_get(row->dict, "_COMM"); - identifier = comm_rkv ? buffer_tostring(comm_rkv->wb) : FACET_VALUE_UNSET; + if(!identifier) { + FACET_ROW_KEY_VALUE *comm_rkv = dictionary_get(row->dict, "_COMM"); + if(comm_rkv && !comm_rkv->empty) + identifier = buffer_tostring(comm_rkv->wb); + } } buffer_flush(rkv->wb); - if(strcmp(pid, FACET_VALUE_UNSET) == 0) - buffer_strcat(rkv->wb, identifier); + if(!identifier) + buffer_strcat(rkv->wb, FACET_VALUE_UNSET); else buffer_sprintf(rkv->wb, "%s[%s]", identifier, pid); @@ -687,11 +1946,82 @@ static void netdata_systemd_journal_rich_message(FACETS *facets __maybe_unused, buffer_json_object_close(json_array); } +DICTIONARY *function_query_status_dict = NULL; + +static void function_systemd_journal_progress(BUFFER *wb, const char *transaction, const char *progress_id) { + if(!progress_id || !(*progress_id)) { + netdata_mutex_lock(&stdout_mutex); + pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_BAD_REQUEST, "missing progress id"); + netdata_mutex_unlock(&stdout_mutex); + return; + } + + const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(function_query_status_dict, progress_id); + + if(!item) { + netdata_mutex_lock(&stdout_mutex); + pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_NOT_FOUND, "progress id is not found here"); + netdata_mutex_unlock(&stdout_mutex); + return; + } + + FUNCTION_QUERY_STATUS *fqs = dictionary_acquired_item_value(item); + + usec_t now_monotonic_ut = now_monotonic_usec(); + if(now_monotonic_ut + 10 * USEC_PER_SEC > fqs->stop_monotonic_ut) + fqs->stop_monotonic_ut = now_monotonic_ut + 10 * USEC_PER_SEC; + + usec_t duration_ut = now_monotonic_ut - fqs->started_monotonic_ut; + + size_t files_matched = fqs->files_matched; + size_t file_working = fqs->file_working; + if(file_working > files_matched) + files_matched = file_working; + + size_t rows_read = __atomic_load_n(&fqs->rows_read, __ATOMIC_RELAXED); + size_t bytes_read = __atomic_load_n(&fqs->bytes_read, __ATOMIC_RELAXED); + + buffer_flush(wb); + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY); + buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK); + buffer_json_member_add_string(wb, "type", "table"); + buffer_json_member_add_uint64(wb, "running_duration_usec", duration_ut); + buffer_json_member_add_double(wb, "progress", (double)file_working * 100.0 / (double)files_matched); + char msg[1024 + 1]; + snprintfz(msg, 1024, + "Read %zu rows (%0.0f rows/s), " + "data %0.1f MB (%0.1f MB/s), " + "file %zu of %zu", + rows_read, (double)rows_read / (double)duration_ut * (double)USEC_PER_SEC, + (double)bytes_read / 1024.0 / 1024.0, ((double)bytes_read / (double)duration_ut * (double)USEC_PER_SEC) / 1024.0 / 1024.0, + file_working, files_matched + ); + buffer_json_member_add_string(wb, "message", msg); + buffer_json_finalize(wb); + + netdata_mutex_lock(&stdout_mutex); + pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "application/json", now_realtime_sec() + 1, wb); + netdata_mutex_unlock(&stdout_mutex); + + dictionary_acquired_item_release(function_query_status_dict, item); +} + static void function_systemd_journal(const char *transaction, char *function, int timeout, bool *cancelled) { + journal_files_registry_update(); + BUFFER *wb = buffer_create(0, NULL); buffer_flush(wb); buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY); + usec_t now_monotonic_ut = now_monotonic_usec(); + FUNCTION_QUERY_STATUS tmp_fqs = { + .cancelled = cancelled, + .started_monotonic_ut = now_monotonic_ut, + .stop_monotonic_ut = now_monotonic_ut + timeout * USEC_PER_SEC, + }; + FUNCTION_QUERY_STATUS *fqs = NULL; + const DICTIONARY_ITEM *fqs_item = NULL; + FACETS *facets = facets_create(50, FACETS_OPTION_ALL_KEYS_FTS, SYSTEMD_ALWAYS_VISIBLE_KEYS, SYSTEMD_KEYS_INCLUDED_IN_FACETS, @@ -709,9 +2039,22 @@ static void function_systemd_journal(const char *transaction, char *function, in facets_accepted_param(facets, JOURNAL_PARAMETER_HISTOGRAM); facets_accepted_param(facets, JOURNAL_PARAMETER_IF_MODIFIED_SINCE); facets_accepted_param(facets, JOURNAL_PARAMETER_DATA_ONLY); + facets_accepted_param(facets, JOURNAL_PARAMETER_ID); + facets_accepted_param(facets, JOURNAL_PARAMETER_PROGRESS); + facets_accepted_param(facets, JOURNAL_PARAMETER_DELTA); + facets_accepted_param(facets, JOURNAL_PARAMETER_TAIL); + +#ifdef HAVE_SD_JOURNAL_RESTART_FIELDS + facets_accepted_param(facets, JOURNAL_PARAMETER_SLICE); +#endif // HAVE_SD_JOURNAL_RESTART_FIELDS // register the fields in the order you want them on the dashboard + facets_register_row_severity(facets, syslog_priority_to_facet_severity, NULL); + + facets_register_key_name(facets, "_HOSTNAME", + FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS); + facets_register_dynamic_key_name(facets, "ND_JOURNAL_PROCESS", FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS, netdata_systemd_journal_dynamic_row_id, NULL); @@ -725,32 +2068,70 @@ static void function_systemd_journal(const char *transaction, char *function, in // FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS, // netdata_systemd_journal_rich_message, NULL); - facets_register_key_name_transformation(facets, "PRIORITY", FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS, + facets_register_key_name_transformation(facets, "PRIORITY", + FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW, netdata_systemd_journal_transform_priority, NULL); - facets_register_key_name_transformation(facets, "SYSLOG_FACILITY", FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS, + facets_register_key_name_transformation(facets, "SYSLOG_FACILITY", + FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW, netdata_systemd_journal_transform_syslog_facility, NULL); - facets_register_key_name(facets, "SYSLOG_IDENTIFIER", FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS); - facets_register_key_name(facets, "UNIT", FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS); - facets_register_key_name(facets, "USER_UNIT", FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS); + facets_register_key_name_transformation(facets, "ERRNO", + FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW, + netdata_systemd_journal_transform_errno, NULL); - facets_register_key_name_transformation(facets, "_UID", FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS, + facets_register_key_name(facets, "SYSLOG_IDENTIFIER", + FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS); + + facets_register_key_name(facets, "UNIT", + FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS); + + facets_register_key_name(facets, "USER_UNIT", + FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS); + + facets_register_key_name_transformation(facets, "_BOOT_ID", + FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW, + netdata_systemd_journal_transform_boot_id, NULL); + + facets_register_key_name_transformation(facets, "_SYSTEMD_OWNER_UID", + FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW, netdata_systemd_journal_transform_uid, NULL); - facets_register_key_name_transformation(facets, "_GID", FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS, + facets_register_key_name_transformation(facets, "_UID", + FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW, + netdata_systemd_journal_transform_uid, NULL); + + facets_register_key_name_transformation(facets, "_GID", + FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW, netdata_systemd_journal_transform_gid, NULL); - bool info = false; - bool data_only = false; + facets_register_key_name_transformation(facets, "_CAP_EFFECTIVE", + FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW, + netdata_systemd_journal_transform_cap_effective, NULL); + + facets_register_key_name_transformation(facets, "_AUDIT_LOGINUID", + FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW, + netdata_systemd_journal_transform_uid, NULL); + + facets_register_key_name_transformation(facets, "_SOURCE_REALTIME_TIMESTAMP", + FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW, + netdata_systemd_journal_transform_timestamp_usec, NULL); + + // ------------------------------------------------------------------------ + // parse the parameters + + bool info = false, data_only = false, progress = false, slice = JOURNAL_DEFAULT_SLICE_MODE, delta = false, tail = false; time_t after_s = 0, before_s = 0; usec_t anchor = 0; usec_t if_modified_since = 0; size_t last = 0; - FACETS_ANCHOR_DIRECTION direction = FACETS_ANCHOR_DIRECTION_BACKWARD; + FACETS_ANCHOR_DIRECTION direction = JOURNAL_DEFAULT_DIRECTION; const char *query = NULL; const char *chart = NULL; const char *source = NULL; + const char *progress_id = NULL; + SD_JOURNAL_FILE_SOURCE_TYPE source_type = SDJF_ALL; + size_t filters = 0; buffer_json_member_add_object(wb, "request"); @@ -767,11 +2148,82 @@ static void function_systemd_journal(const char *transaction, char *function, in else if(strcmp(keyword, JOURNAL_PARAMETER_INFO) == 0) { info = true; } - else if(strcmp(keyword, JOURNAL_PARAMETER_DATA_ONLY) == 0) { - data_only = true; + else if(strcmp(keyword, JOURNAL_PARAMETER_PROGRESS) == 0) { + progress = true; + } + else if(strncmp(keyword, JOURNAL_PARAMETER_DELTA ":", sizeof(JOURNAL_PARAMETER_DELTA ":") - 1) == 0) { + char *v = &keyword[sizeof(JOURNAL_PARAMETER_DELTA ":") - 1]; + + if(strcmp(v, "false") == 0 || strcmp(v, "no") == 0 || strcmp(v, "0") == 0) + delta = false; + else + delta = true; + } + else if(strncmp(keyword, JOURNAL_PARAMETER_TAIL ":", sizeof(JOURNAL_PARAMETER_TAIL ":") - 1) == 0) { + char *v = &keyword[sizeof(JOURNAL_PARAMETER_TAIL ":") - 1]; + + if(strcmp(v, "false") == 0 || strcmp(v, "no") == 0 || strcmp(v, "0") == 0) + tail = false; + else + tail = true; + } + else if(strncmp(keyword, JOURNAL_PARAMETER_DATA_ONLY ":", sizeof(JOURNAL_PARAMETER_DATA_ONLY ":") - 1) == 0) { + char *v = &keyword[sizeof(JOURNAL_PARAMETER_DATA_ONLY ":") - 1]; + + if(strcmp(v, "false") == 0 || strcmp(v, "no") == 0 || strcmp(v, "0") == 0) + data_only = false; + else + data_only = true; + } + else if(strncmp(keyword, JOURNAL_PARAMETER_SLICE ":", sizeof(JOURNAL_PARAMETER_SLICE ":") - 1) == 0) { + char *v = &keyword[sizeof(JOURNAL_PARAMETER_SLICE ":") - 1]; + + if(strcmp(v, "false") == 0 || strcmp(v, "no") == 0 || strcmp(v, "0") == 0) + slice = false; + else + slice = true; + } + else if(strncmp(keyword, JOURNAL_PARAMETER_ID ":", sizeof(JOURNAL_PARAMETER_ID ":") - 1) == 0) { + char *id = &keyword[sizeof(JOURNAL_PARAMETER_ID ":") - 1]; + + if(*id) + progress_id = id; } else if(strncmp(keyword, JOURNAL_PARAMETER_SOURCE ":", sizeof(JOURNAL_PARAMETER_SOURCE ":") - 1) == 0) { source = &keyword[sizeof(JOURNAL_PARAMETER_SOURCE ":") - 1]; + + if(strcmp(source, SDJF_SOURCE_ALL_NAME) == 0) { + source_type = SDJF_ALL; + source = NULL; + } + else if(strcmp(source, SDJF_SOURCE_LOCAL_NAME) == 0) { + source_type = SDJF_LOCAL; + source = NULL; + } + else if(strcmp(source, SDJF_SOURCE_REMOTES_NAME) == 0) { + source_type = SDJF_REMOTE; + source = NULL; + } + else if(strcmp(source, SDJF_SOURCE_NAMESPACES_NAME) == 0) { + source_type = SDJF_NAMESPACE; + source = NULL; + } + else if(strcmp(source, SDJF_SOURCE_LOCAL_SYSTEM_NAME) == 0) { + source_type = SDJF_LOCAL | SDJF_SYSTEM; + source = NULL; + } + else if(strcmp(source, SDJF_SOURCE_LOCAL_USERS_NAME) == 0) { + source_type = SDJF_LOCAL | SDJF_USER; + source = NULL; + } + else if(strcmp(source, SDJF_SOURCE_LOCAL_OTHER_NAME) == 0) { + source_type = SDJF_LOCAL | SDJF_OTHER; + source = NULL; + } + else { + source_type = SDJF_ALL; + // else, match the source, whatever it is + } } else if(strncmp(keyword, JOURNAL_PARAMETER_AFTER ":", sizeof(JOURNAL_PARAMETER_AFTER ":") - 1) == 0) { after_s = str2l(&keyword[sizeof(JOURNAL_PARAMETER_AFTER ":") - 1]); @@ -830,6 +2282,7 @@ static void function_systemd_journal(const char *transaction, char *function, in facets_register_facet_id_filter(facets, keyword, value, FACET_KEY_OPTION_FACET|FACET_KEY_OPTION_FTS|FACET_KEY_OPTION_REORDER); buffer_json_add_array_item_string(wb, value); + filters++; value = sep; } @@ -839,6 +2292,22 @@ static void function_systemd_journal(const char *transaction, char *function, in } } + // ------------------------------------------------------------------------ + // put this request into the progress db + + if(progress_id && *progress_id) { + fqs_item = dictionary_set_and_acquire_item(function_query_status_dict, progress_id, &tmp_fqs, sizeof(tmp_fqs)); + fqs = dictionary_acquired_item_value(fqs_item); + } + else { + // no progress id given, proceed without registering our progress in the dictionary + fqs = &tmp_fqs; + fqs_item = NULL; + } + + // ------------------------------------------------------------------------ + // validate parameters + time_t expires = now_realtime_sec() + 1; time_t now_s; @@ -862,18 +2331,103 @@ static void function_systemd_journal(const char *transaction, char *function, in if(!last) last = SYSTEMD_JOURNAL_DEFAULT_ITEMS_PER_QUERY; - buffer_json_member_add_string(wb, "source", source ? source : "default"); - buffer_json_member_add_time_t(wb, "after", after_s); - buffer_json_member_add_time_t(wb, "before", before_s); - buffer_json_member_add_uint64(wb, "if_modified_since", if_modified_since); - buffer_json_member_add_uint64(wb, "anchor", anchor); - buffer_json_member_add_string(wb, "direction", direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward"); - buffer_json_member_add_uint64(wb, "last", last); - buffer_json_member_add_string(wb, "query", query); - buffer_json_member_add_string(wb, "chart", chart); - buffer_json_member_add_time_t(wb, "timeout", timeout); + + // ------------------------------------------------------------------------ + // set query time-frame, anchors and direction + + fqs->after_ut = after_s * USEC_PER_SEC; + fqs->before_ut = before_s * USEC_PER_SEC; + fqs->if_modified_since = if_modified_since; + fqs->data_only = data_only; + fqs->delta = (fqs->data_only) ? delta : false; + fqs->tail = (fqs->data_only && fqs->if_modified_since) ? tail : false; + fqs->source = string_strdupz(source); + fqs->source_type = source_type; + fqs->entries = last; + fqs->last_modified = 0; + fqs->filters = filters; + fqs->query = (query && *query) ? query : NULL; + fqs->histogram = (chart && *chart) ? chart : NULL; + + if(anchor && anchor < fqs->after_ut) { + netdata_log_error("Received anchor %"PRIu64" is too small for query time-frame [%"PRIu64" - %"PRIu64"]", + anchor, fqs->after_ut, fqs->before_ut); + anchor = 0; + } + else if(anchor > fqs->before_ut) { + netdata_log_error("Received anchor %"PRIu64" is too big for query time-frame [%"PRIu64" - %"PRIu64"]", + anchor, fqs->after_ut, fqs->before_ut); + anchor = 0; + } + + fqs->direction = direction; + fqs->anchor.start_ut = anchor; + fqs->anchor.stop_ut = 0; + + if(fqs->anchor.start_ut && fqs->tail) { + // a tail request + // we need the top X entries from BEFORE + // but, we need to calculate the facets and the + // histogram up to the anchor + fqs->direction = direction = FACETS_ANCHOR_DIRECTION_BACKWARD; + fqs->anchor.start_ut = 0; + fqs->anchor.stop_ut = anchor; + } + + facets_set_anchor(facets, fqs->anchor.start_ut, fqs->anchor.stop_ut, fqs->direction); + + facets_set_additional_options(facets, + ((fqs->data_only) ? FACETS_OPTION_DATA_ONLY : 0) | + ((fqs->delta) ? FACETS_OPTION_SHOW_DELTAS : 0)); + + // ------------------------------------------------------------------------ + // set the rest of the query parameters + + + facets_set_items(facets, fqs->entries); + facets_set_query(facets, fqs->query); + +#ifdef HAVE_SD_JOURNAL_RESTART_FIELDS + fqs->slice = slice; + if(slice) + facets_enable_slice_mode(facets); +#else + fqs->slice = false; +#endif + + if(fqs->histogram) + facets_set_timeframe_and_histogram_by_id(facets, fqs->histogram, fqs->after_ut, fqs->before_ut); + else + facets_set_timeframe_and_histogram_by_name(facets, "PRIORITY", fqs->after_ut, fqs->before_ut); + + + // ------------------------------------------------------------------------ + // complete the request object + + buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_INFO, false); + buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_SLICE, fqs->slice); + buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_DATA_ONLY, fqs->data_only); + buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_PROGRESS, false); + buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_DELTA, fqs->delta); + buffer_json_member_add_boolean(wb, JOURNAL_PARAMETER_TAIL, fqs->tail); + buffer_json_member_add_string(wb, JOURNAL_PARAMETER_ID, progress_id); + buffer_json_member_add_string(wb, JOURNAL_PARAMETER_SOURCE, string2str(fqs->source)); + buffer_json_member_add_uint64(wb, "source_type", fqs->source_type); + buffer_json_member_add_uint64(wb, JOURNAL_PARAMETER_AFTER, fqs->after_ut / USEC_PER_SEC); + buffer_json_member_add_uint64(wb, JOURNAL_PARAMETER_BEFORE, fqs->before_ut / USEC_PER_SEC); + buffer_json_member_add_uint64(wb, "if_modified_since", fqs->if_modified_since); + buffer_json_member_add_uint64(wb, JOURNAL_PARAMETER_ANCHOR, anchor); + buffer_json_member_add_string(wb, JOURNAL_PARAMETER_DIRECTION, fqs->direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward"); + buffer_json_member_add_uint64(wb, JOURNAL_PARAMETER_LAST, fqs->entries); + buffer_json_member_add_string(wb, JOURNAL_PARAMETER_QUERY, fqs->query); + buffer_json_member_add_string(wb, JOURNAL_PARAMETER_HISTOGRAM, fqs->histogram); buffer_json_object_close(wb); // request + buffer_json_journal_versions(wb); + + // ------------------------------------------------------------------------ + // run the request + int response; if(info) { @@ -888,12 +2442,7 @@ static void function_systemd_journal(const char *transaction, char *function, in buffer_json_member_add_string(wb, "type", "select"); buffer_json_member_add_array(wb, "options"); { - buffer_json_add_array_item_object(wb); - { - buffer_json_member_add_string(wb, "id", "default"); - buffer_json_member_add_string(wb, "name", "default"); - } - buffer_json_object_close(wb); // options object + available_journal_file_sources_to_json_array(wb); } buffer_json_array_close(wb); // options array } @@ -901,6 +2450,8 @@ static void function_systemd_journal(const char *transaction, char *function, in } buffer_json_array_close(wb); // required_params array + facets_table_config(wb); + buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK); buffer_json_member_add_string(wb, "type", "table"); buffer_json_member_add_string(wb, "help", SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION); @@ -909,22 +2460,21 @@ static void function_systemd_journal(const char *transaction, char *function, in goto output; } - facets_set_items(facets, last); - facets_set_anchor(facets, anchor, direction); - facets_set_query(facets, query); + if(progress) { + function_systemd_journal_progress(wb, transaction, progress_id); + goto cleanup; + } - if(chart && *chart) - facets_set_histogram_by_id(facets, chart, - after_s * USEC_PER_SEC, before_s * USEC_PER_SEC); - else - facets_set_histogram_by_name(facets, "PRIORITY", - after_s * USEC_PER_SEC, before_s * USEC_PER_SEC); + response = netdata_systemd_journal_query(wb, facets, fqs); + + // ------------------------------------------------------------------------ + // cleanup query params + + string_freez(fqs->source); + fqs->source = NULL; - response = netdata_systemd_journal_query(wb, facets, after_s * USEC_PER_SEC, before_s * USEC_PER_SEC, - anchor, direction, last, - if_modified_since, data_only, - now_monotonic_usec() + (timeout - 1) * USEC_PER_SEC, - cancelled); + // ------------------------------------------------------------------------ + // handle error response if(response != HTTP_RESP_OK) { netdata_mutex_lock(&stdout_mutex); @@ -941,6 +2491,12 @@ static void function_systemd_journal(const char *transaction, char *function, in cleanup: facets_destroy(facets); buffer_free(wb); + + if(fqs_item) { + dictionary_del(function_query_status_dict, dictionary_acquired_item_name(fqs_item)); + dictionary_acquired_item_release(function_query_status_dict, fqs_item); + dictionary_garbage_collect(function_query_status_dict); + } } // ---------------------------------------------------------------------------- @@ -961,14 +2517,66 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) { netdata_configured_host_prefix = getenv("NETDATA_HOST_PREFIX"); if(verify_netdata_host_prefix() == -1) exit(1); + // ------------------------------------------------------------------------ + // setup the journal directories + + unsigned d = 0; + + journal_directories[d++].path = strdupz("/var/log/journal"); + journal_directories[d++].path = strdupz("/run/log/journal"); + + if(*netdata_configured_host_prefix) { + char path[PATH_MAX]; + snprintfz(path, sizeof(path), "%s/var/log/journal", netdata_configured_host_prefix); + journal_directories[d++].path = strdupz(path); + snprintfz(path, sizeof(path), "%s/run/log/journal", netdata_configured_host_prefix); + journal_directories[d++].path = strdupz(path); + } + + // terminate the list + journal_directories[d].path = NULL; + + // ------------------------------------------------------------------------ + + function_query_status_dict = dictionary_create_advanced( + DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, + NULL, sizeof(FUNCTION_QUERY_STATUS)); + + // ------------------------------------------------------------------------ + // initialize the used hashes files registry + + used_hashes_registry = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE); + + + // ------------------------------------------------------------------------ + // initialize the journal files registry + + systemd_journal_session = (now_realtime_usec() / USEC_PER_SEC) * USEC_PER_SEC; + + journal_files_registry = dictionary_create_advanced( + DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, + NULL, sizeof(struct journal_file)); + + dictionary_register_insert_callback(journal_files_registry, files_registry_insert_cb, NULL); + dictionary_register_delete_callback(journal_files_registry, files_registry_delete_cb, NULL); + dictionary_register_conflict_callback(journal_files_registry, files_registry_conflict_cb, NULL); + + boot_ids_to_first_ut = dictionary_create_advanced( + DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, + NULL, sizeof(usec_t)); + + journal_files_registry_update(); + + // ------------------------------------------------------------------------ // debug if(argc == 2 && strcmp(argv[1], "debug") == 0) { bool cancelled = false; char buf[] = "systemd-journal after:-2592000 before:0 last:500"; + // char buf[] = "systemd-journal after:1695332964 before:1695937764 direction:backward last:100 slice:true source:all DHKucpqUoe1:PtVoyIuX.MU"; // char buf[] = "systemd-journal after:1694511062 before:1694514662 anchor:1694514122024403"; - function_systemd_journal("123", buf, 30, &cancelled); + function_systemd_journal("123", buf, 600, &cancelled); exit(1); } diff --git a/configure.ac b/configure.ac index f75dfa2ac90b14..5db1b9f21cb7cd 100644 --- a/configure.ac +++ b/configure.ac @@ -1141,6 +1141,16 @@ fi AC_MSG_RESULT([${enable_plugin_systemd_journal}]) AM_CONDITIONAL([ENABLE_PLUGIN_SYSTEMD_JOURNAL], [test "${enable_plugin_systemd_journal}" = "yes"]) +AC_CHECK_LIB([systemd], [sd_journal_open_files_fd], [have_sd_journal_open_files_fd=yes], [have_sd_journal_open_files_fd=no]) +if test "${have_sd_journal_open_files_fd}" = "yes"; then + AC_DEFINE([HAVE_SD_JOURNAL_OPEN_FILES_FD], [1], [sd_journal_open_files_fd usability]) +fi + +AC_CHECK_LIB([systemd], [sd_journal_restart_fields], [have_sd_journal_restart_fields=yes], [have_sd_journal_restart_fields=no]) +if test "${have_sd_journal_restart_fields}" = "yes"; then + AC_DEFINE([HAVE_SD_JOURNAL_RESTART_FIELDS], [1], [sd_journal_restart_fields usability]) +fi + AC_MSG_NOTICE([OPTIONAL_SYSTEMD_LIBS is set to: ${OPTIONAL_SYSTEMD_LIBS}]) if test "${enable_plugin_systemd_journal}" = "yes"; then diff --git a/libnetdata/facets/facets.c b/libnetdata/facets/facets.c index 9982940a6d0aed..26029c41e7247f 100644 --- a/libnetdata/facets/facets.c +++ b/libnetdata/facets/facets.c @@ -1,9 +1,12 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "facets.h" -#define HISTOGRAM_COLUMNS 100 -#define FACETS_KEYS_HASHTABLE_ENTRIES 1000 -#define FACETS_VALUES_HASHTABLE_ENTRIES 20 +#define HISTOGRAM_COLUMNS 150 // the target number of points in a histogram +#define FACETS_KEYS_WITH_VALUES_MAX 200 // the max number of keys that can be facets +#define FACETS_KEYS_IN_ROW_MAX 500 // the max number of keys in a row + +#define FACETS_KEYS_HASHTABLE_ENTRIES 256 +#define FACETS_VALUES_HASHTABLE_ENTRIES 32 // ---------------------------------------------------------------------------- @@ -106,6 +109,7 @@ typedef struct facet_value { uint32_t rows_matching_facet_value; uint32_t final_facet_value_counter; + uint32_t order; uint32_t *histogram; uint32_t min, max, sum; @@ -131,9 +135,11 @@ struct facet_key { // members about the current row uint32_t key_found_in_row; uint32_t key_values_selected_in_row; + uint32_t order; struct { bool enabled; + uint32_t used; FACET_VALUE *hashtable[FACETS_VALUES_HASHTABLE_ENTRIES]; FACET_VALUE *ll; } values; @@ -150,14 +156,13 @@ struct facet_key { FACET_VALUE *v; } empty_value; - uint32_t order; - struct { facet_dynamic_row_t cb; void *data; } dynamic; struct { + bool view_only; facets_key_transformer_t cb; void *data; } transform; @@ -177,7 +182,8 @@ struct facets { FACETS_OPTIONS options; struct { - usec_t key; + usec_t start_ut; + usec_t stop_ut; FACETS_ANCHOR_DIRECTION direction; } anchor; @@ -187,10 +193,23 @@ struct facets { DICTIONARY *accepted_params; struct { + size_t count; FACET_KEY *hashtable[FACETS_KEYS_HASHTABLE_ENTRIES]; FACET_KEY *ll; } keys; + struct { + // this is like a stack, of the keys that are used as facets + size_t used; + FACET_KEY *array[FACETS_KEYS_WITH_VALUES_MAX]; + } keys_with_values; + + struct { + // this is like a stack, of the keys that need to clean up between each row + size_t used; + FACET_KEY *array[FACETS_KEYS_IN_ROW_MAX]; + } keys_in_row; + FACET_ROW *base; // double linked list of the selected facets rows uint32_t items_to_return; @@ -203,6 +222,12 @@ struct facets { } current_row; struct { + usec_t after_ut; + usec_t before_ut; + } timeframe; + + struct { + FACET_KEY *key; FACETS_HASH hash; char *chart; bool enabled; @@ -212,6 +237,11 @@ struct facets { usec_t before_ut; } histogram; + struct { + facet_row_severity_t cb; + void *data; + } severity; + struct { FACET_ROW *last_added; @@ -252,6 +282,24 @@ struct facets { } operations; }; +usec_t facets_row_oldest_ut(FACETS *facets) { + if(facets->base) + return facets->base->prev->usec; + + return 0; +} + +usec_t facets_row_newest_ut(FACETS *facets) { + if(facets->base) + return facets->base->usec; + + return 0; +} + +uint32_t facets_rows(FACETS *facets) { + return facets->items_to_return; +} + // ---------------------------------------------------------------------------- static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row); @@ -268,6 +316,7 @@ static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k); static inline void FACETS_VALUES_INDEX_CREATE(FACET_KEY *k) { k->values.ll = NULL; + k->values.used = 0; } static inline void FACETS_VALUES_INDEX_DESTROY(FACET_KEY *k) { @@ -280,6 +329,7 @@ static inline void FACETS_VALUES_INDEX_DESTROY(FACET_KEY *k) { v = next; } k->values.ll = NULL; + k->values.used = 0; memset(k->values.hashtable, 0, sizeof(k->values.hashtable)); k->values.enabled = false; } @@ -309,6 +359,11 @@ static inline FACET_VALUE **facets_values_hashtable_slot(FACET_KEY *k, FACETS_HA return v; } +static inline FACET_VALUE *FACET_VALUE_GET_FROM_INDEX(FACET_KEY *k, FACETS_HASH hash) { + FACET_VALUE **v_ptr = facets_values_hashtable_slot(k, hash); + return *v_ptr; +} + static inline FACET_VALUE *FACET_VALUE_ADD_TO_INDEX(FACET_KEY *k, const FACET_VALUE * const tv) { FACET_VALUE **v_ptr = facets_values_hashtable_slot(k, tv->hash); @@ -328,6 +383,7 @@ static inline FACET_VALUE *FACET_VALUE_ADD_TO_INDEX(FACET_KEY *k, const FACET_VA memcpy(v, tv, sizeof(*v)); DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(k->values.ll, v, prev, next); + k->values.used++; if(!v->selected) v->selected = k->default_selected_for_values; @@ -403,11 +459,15 @@ static inline void facet_key_late_init(FACETS *facets, FACET_KEY *k) { if(facets_key_is_facet(facets, k)) { FACETS_VALUES_INDEX_CREATE(k); k->values.enabled = true; + if(facets->keys_with_values.used < FACETS_KEYS_WITH_VALUES_MAX) + facets->keys_with_values.array[facets->keys_with_values.used++] = k; } } static inline void FACETS_KEYS_INDEX_CREATE(FACETS *facets) { facets->keys.ll = NULL; + facets->keys.count = 0; + facets->keys_with_values.used = 0; } static inline void FACETS_KEYS_INDEX_DESTROY(FACETS *facets) { @@ -424,6 +484,8 @@ static inline void FACETS_KEYS_INDEX_DESTROY(FACETS *facets) { } memset(facets->keys.hashtable, 0, sizeof(facets->keys.hashtable)); facets->keys.ll = NULL; + facets->keys.count = 0; + facets->keys_with_values.used = 0; } static inline FACET_KEY **facets_keys_hashtable_slot(FACETS *facets, FACETS_HASH hash) { @@ -441,45 +503,67 @@ static inline FACET_KEY *FACETS_KEY_GET_FROM_INDEX(FACETS *facets, FACETS_HASH h return *k; } -static inline FACET_KEY *FACETS_KEY_ADD_TO_INDEX(FACETS *facets, FACETS_HASH hash, const char *name, FACET_KEY_OPTIONS options) { - facets->operations.keys.registered++; +bool facets_key_name_value_length_is_selected(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length) { + FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length); + FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash); + if(!k || k->default_selected_for_values) + return false; - FACET_KEY **k_ptr = facets_keys_hashtable_slot(facets, hash); + hash = FACETS_HASH_FUNCTION(value, value_length); + FACET_VALUE *v = FACET_VALUE_GET_FROM_INDEX(k, hash); + return (v && v->selected) ? true : false; +} - if(likely(*k_ptr)) { - // already exists +void facets_add_possible_value_name_to_key(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length) { + FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_length); + FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash); + if(!k) return; - FACET_KEY *k = *k_ptr; + hash = FACETS_HASH_FUNCTION(value, value_length); + FACET_VALUE *v = FACET_VALUE_GET_FROM_INDEX(k, hash); + if(v && v->name) return; - if(!k->name && name) { - // an actual value, not a filter - k->name = strdupz(name); - facet_key_late_init(facets, k); - } + BUFFER *wb = buffer_create(0, NULL); + buffer_contents_replace(wb, value, value_length); - internal_fatal(k->name && name && strcmp(k->name, name) != 0, - "key hash conflict: '%s' and '%s' have the same hash '%s'", - k->name, name, - hash_to_static_string(hash)); + FACET_VALUE tv = { + .hash = hash, + .name = buffer_tostring(wb), + }; + FACET_VALUE_ADD_TO_INDEX(k, &tv); - if(k->options & FACET_KEY_OPTION_REORDER) { - k->order = facets->order++; - k->options &= ~FACET_KEY_OPTION_REORDER; - } + buffer_free(wb); +} - return k; - } +static void facet_key_set_name(FACET_KEY *k, const char *name, size_t name_length) { + internal_fatal(k->name && name && (strncmp(k->name, name, name_length) != 0 || k->name[name_length] != '\0'), + "key hash conflict: '%s' and '%s' have the same hash", + k->name, name); - // we have to add it + if(k->name || !name || !name_length) + return; + + // an actual value, not a filter + + char buf[name_length + 1]; + memcpy(buf, name, name_length); + buf[name_length] = '\0'; + + internal_fatal(strchr(buf, '='), "found = in key"); + + k->name = strdupz(buf); + facet_key_late_init(k->facets, k); +} + +static inline FACET_KEY *FACETS_KEY_CREATE(FACETS *facets, FACETS_HASH hash, const char *name, size_t name_length, FACET_KEY_OPTIONS options) { facets->operations.keys.unique++; FACET_KEY *k = callocz(1, sizeof(*k)); - *k_ptr = k; // add it to the index k->hash = hash; k->facets = facets; k->options = options; - k->current_value.b = buffer_create(0, NULL); + k->current_value.b = buffer_create(sizeof(FACET_VALUE_UNSET), NULL); k->default_selected_for_values = true; if(!(k->options & FACET_KEY_OPTION_REORDER)) @@ -488,17 +572,52 @@ static inline FACET_KEY *FACETS_KEY_ADD_TO_INDEX(FACETS *facets, FACETS_HASH has if((k->options & FACET_KEY_OPTION_FTS) || (facets->options & FACETS_OPTION_ALL_KEYS_FTS)) facets->keys_filtered_by_query++; - if(name) { - // an actual value, not a filter - k->name = strdupz(name); - facet_key_late_init(facets, k); - } + facet_key_set_name(k, name, name_length); DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->keys.ll, k, prev, next); + facets->keys.count++; + + return k; +} + +static inline FACET_KEY *FACETS_KEY_ADD_TO_INDEX(FACETS *facets, FACETS_HASH hash, const char *name, size_t name_length, FACET_KEY_OPTIONS options) { + facets->operations.keys.registered++; + + FACET_KEY **k_ptr = facets_keys_hashtable_slot(facets, hash); + + if(unlikely(!(*k_ptr))) { + // we have to add it + *k_ptr = FACETS_KEY_CREATE(facets, hash, name, name_length, options); + return (*k_ptr); + } + + // already in the index + + FACET_KEY *k = *k_ptr; + + facet_key_set_name(k, name, name_length); + + if(unlikely(k->options & FACET_KEY_OPTION_REORDER)) { + k->order = facets->order++; + k->options &= ~FACET_KEY_OPTION_REORDER; + } return k; } +bool facets_key_name_is_filter(FACETS *facets, const char *key) { + FACETS_HASH hash = FACETS_HASH_FUNCTION(key, strlen(key)); + FACET_KEY *k = FACETS_KEY_GET_FROM_INDEX(facets, hash); + return (!k || k->default_selected_for_values) ? false : true; +} + +bool facets_key_name_is_facet(FACETS *facets, const char *key) { + size_t key_len = strlen(key); + FACETS_HASH hash = FACETS_HASH_FUNCTION(key, key_len); + FACET_KEY *k = FACETS_KEY_ADD_TO_INDEX(facets, hash, key, key_len, 0); + return (k && (k->options & FACET_KEY_OPTION_FACET)); +} + // ---------------------------------------------------------------------------- static usec_t calculate_histogram_bar_width(usec_t after_ut, usec_t before_ut) { @@ -530,7 +649,7 @@ static inline usec_t facets_histogram_slot_baseline_ut(FACETS *facets, usec_t ut return ut - delta_ut; } -void facets_set_histogram_by_id(FACETS *facets, const char *key_id, usec_t after_ut, usec_t before_ut) { +void facets_set_timeframe_and_histogram_by_id(FACETS *facets, const char *key_id, usec_t after_ut, usec_t before_ut) { if(after_ut > before_ut) { usec_t t = after_ut; after_ut = before_ut; @@ -549,37 +668,50 @@ void facets_set_histogram_by_id(FACETS *facets, const char *key_id, usec_t after facets->histogram.hash = FACETS_HASH_ZERO; } + facets->timeframe.after_ut = after_ut; + facets->timeframe.before_ut = before_ut; + facets->histogram.slot_width_ut = calculate_histogram_bar_width(after_ut, before_ut); facets->histogram.after_ut = facets_histogram_slot_baseline_ut(facets, after_ut); facets->histogram.before_ut = facets_histogram_slot_baseline_ut(facets, before_ut) + facets->histogram.slot_width_ut; facets->histogram.slots = (facets->histogram.before_ut - facets->histogram.after_ut) / facets->histogram.slot_width_ut + 1; + internal_fatal(after_ut < facets->histogram.after_ut, "histogram after_ut is not less or equal to wanted after_ut"); + internal_fatal(before_ut > facets->histogram.before_ut, "histogram before_ut is not more or equal to wanted before_ut"); + if(facets->histogram.slots > 1000) { facets->histogram.slots = 1000 + 1; facets->histogram.slot_width_ut = (facets->histogram.before_ut - facets->histogram.after_ut) / 1000; } } -void facets_set_histogram_by_name(FACETS *facets, const char *key_name, usec_t after_ut, usec_t before_ut) { +void facets_set_timeframe_and_histogram_by_name(FACETS *facets, const char *key_name, usec_t after_ut, usec_t before_ut) { char hash_str[FACET_STRING_HASH_SIZE]; FACETS_HASH hash = FACETS_HASH_FUNCTION(key_name, strlen(key_name)); facets_hash_to_str(hash, hash_str); - facets_set_histogram_by_id(facets, hash_str, after_ut, before_ut); + facets_set_timeframe_and_histogram_by_id(facets, hash_str, after_ut, before_ut); } -static inline void facets_histogram_update_value(FACETS *facets, FACET_KEY *k __maybe_unused, FACET_VALUE *v, usec_t usec) { - if(!facets->histogram.enabled) +static inline void facets_histogram_update_value(FACETS *facets, usec_t usec) { + if(!facets->histogram.enabled || + !facets->histogram.key || + !facets->histogram.key->values.enabled || + !facets->histogram.key->current_value.v || + usec < facets->histogram.after_ut || + usec > facets->histogram.before_ut) return; + FACET_VALUE *v = facets->histogram.key->current_value.v; + if(unlikely(!v->histogram)) v->histogram = callocz(facets->histogram.slots, sizeof(*v->histogram)); usec_t base_ut = facets_histogram_slot_baseline_ut(facets, usec); - if(base_ut < facets->histogram.after_ut) + if(unlikely(base_ut < facets->histogram.after_ut)) base_ut = facets->histogram.after_ut; - if(base_ut > facets->histogram.before_ut) + if(unlikely(base_ut > facets->histogram.before_ut)) base_ut = facets->histogram.before_ut; uint32_t slot = (base_ut - facets->histogram.after_ut) / facets->histogram.slot_width_ut; @@ -591,6 +723,8 @@ static inline void facets_histogram_update_value(FACETS *facets, FACET_KEY *k __ } static inline void facets_histogram_value_names(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key, const char *first_key) { + BUFFER *tb = NULL; + buffer_json_member_add_array(wb, key); { if(first_key) @@ -602,12 +736,24 @@ static inline void facets_histogram_value_names(BUFFER *wb, FACETS *facets __may if (unlikely(!v->histogram)) continue; - buffer_json_add_array_item_string(wb, v->name); + if(!v->empty && k->transform.cb && k->transform.view_only) { + if(!tb) + tb = buffer_create(0, NULL); + + buffer_flush(tb); + buffer_strcat(tb, v->name); + k->transform.cb(facets, tb, FACETS_TRANSFORM_HISTOGRAM, k->transform.data); + buffer_json_add_array_item_string(wb, buffer_tostring(tb)); + } + else + buffer_json_add_array_item_string(wb, v->name); } foreach_value_in_key_done(v); } } buffer_json_array_close(wb); // key + + buffer_free(tb); } static inline void facets_histogram_value_units(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) { @@ -1122,7 +1268,7 @@ static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) { } } - if(included && !excluded && !(facets->options & FACETS_OPTION_DISABLE_ALL_FACETS)) { + if(included && !excluded) { k->options |= FACET_KEY_OPTION_FACET; k->options &= ~FACET_KEY_OPTION_NO_FACET; return true; @@ -1150,7 +1296,8 @@ FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const ch facets->visible_keys = simple_pattern_create(visible_keys, "|", SIMPLE_PATTERN_EXACT, true); facets->max_items_to_return = items_to_return; - facets->anchor.key = 0; + facets->anchor.start_ut = 0; + facets->anchor.stop_ut = 0; facets->anchor.direction = FACETS_ANCHOR_DIRECTION_BACKWARD; facets->order = 1; @@ -1183,7 +1330,7 @@ void facets_accepted_param(FACETS *facets, const char *param) { } static inline FACET_KEY *facets_register_key_name_length(FACETS *facets, const char *key, size_t key_length, FACET_KEY_OPTIONS options) { - return FACETS_KEY_ADD_TO_INDEX(facets, FACETS_HASH_FUNCTION(key, key_length), key, options); + return FACETS_KEY_ADD_TO_INDEX(facets, FACETS_HASH_FUNCTION(key, key_length), key, key_length, options); } inline FACET_KEY *facets_register_key_name(FACETS *facets, const char *key, FACET_KEY_OPTIONS options) { @@ -1194,6 +1341,7 @@ inline FACET_KEY *facets_register_key_name_transformation(FACETS *facets, const FACET_KEY *k = facets_register_key_name(facets, key, options); k->transform.cb = cb; k->transform.data = data; + k->transform.view_only = (options & FACET_KEY_OPTION_TRANSFORM_VIEW) ? true : false; return k; } @@ -1215,9 +1363,21 @@ void facets_set_items(FACETS *facets, uint32_t items) { facets->max_items_to_return = items; } -void facets_set_anchor(FACETS *facets, usec_t anchor, FACETS_ANCHOR_DIRECTION direction) { - facets->anchor.key = anchor; +void facets_set_anchor(FACETS *facets, usec_t start_ut, usec_t stop_ut, FACETS_ANCHOR_DIRECTION direction) { + facets->anchor.start_ut = start_ut; + facets->anchor.stop_ut = stop_ut; facets->anchor.direction = direction; + + if((facets->anchor.direction == FACETS_ANCHOR_DIRECTION_BACKWARD && facets->anchor.start_ut && facets->anchor.start_ut < facets->anchor.stop_ut) || + (facets->anchor.direction == FACETS_ANCHOR_DIRECTION_FORWARD && facets->anchor.stop_ut && facets->anchor.stop_ut < facets->anchor.start_ut)) { + internal_error(true, "start and stop anchors are flipped"); + facets->anchor.start_ut = stop_ut; + facets->anchor.stop_ut = start_ut; + } +} + +void facets_enable_slice_mode(FACETS *facets) { + facets->options |= FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS | FACETS_OPTION_SORT_FACETS_ALPHABETICALLY; } inline FACET_KEY *facets_register_facet_id(FACETS *facets, const char *key_id, FACET_KEY_OPTIONS options) { @@ -1229,7 +1389,7 @@ inline FACET_KEY *facets_register_facet_id(FACETS *facets, const char *key_id, F internal_error(strcmp(hash_to_static_string(hash), key_id) != 0, "Regenerating the user supplied key, does not produce the same hash string"); - FACET_KEY *k = FACETS_KEY_ADD_TO_INDEX(facets, hash, NULL, options); + FACET_KEY *k = FACETS_KEY_ADD_TO_INDEX(facets, hash, NULL, 0, options); k->options |= FACET_KEY_OPTION_FACET; k->options &= ~FACET_KEY_OPTION_NO_FACET; facet_key_late_init(facets, k); @@ -1251,22 +1411,33 @@ void facets_set_current_row_severity(FACETS *facets, FACET_ROW_SEVERITY severity facets->current_row.severity = severity; } -void facets_data_only_mode(FACETS *facets) { - facets->options |= FACETS_OPTION_DISABLE_ALL_FACETS | FACETS_OPTION_DISABLE_HISTOGRAM | FACETS_OPTION_DATA_ONLY; +void facets_register_row_severity(FACETS *facets, facet_row_severity_t cb, void *data) { + facets->severity.cb = cb; + facets->severity.data = data; +} + +void facets_set_additional_options(FACETS *facets, FACETS_OPTIONS options) { + facets->options |= options; } // ---------------------------------------------------------------------------- static inline void facets_key_set_empty_value(FACETS *facets, FACET_KEY *k) { + if(likely(!k->current_value.updated && facets->keys_in_row.used < FACETS_KEYS_IN_ROW_MAX)) + facets->keys_in_row.array[facets->keys_in_row.used++] = k; + k->current_value.updated = true; k->current_value.empty = true; facets->operations.values.registered++; facets->operations.values.empty++; - buffer_contents_replace(k->current_value.b, FACET_VALUE_UNSET, sizeof(FACET_VALUE_UNSET) - 1); + // no need to copy the UNSET value + // empty values are exported as empty + k->current_value.b->len = 0; + // buffer_contents_replace(k->current_value.b, FACET_VALUE_UNSET, sizeof(FACET_VALUE_UNSET) - 1); - if(k->values.enabled) + if(unlikely(k->values.enabled)) FACET_VALUE_ADD_EMPTY_VALUE_TO_INDEX(k); else { k->key_found_in_row++; @@ -1275,14 +1446,17 @@ static inline void facets_key_set_empty_value(FACETS *facets, FACET_KEY *k) { } static inline void facets_key_check_value(FACETS *facets, FACET_KEY *k) { + if(likely(!k->current_value.updated && facets->keys_in_row.used < FACETS_KEYS_IN_ROW_MAX)) + facets->keys_in_row.array[facets->keys_in_row.used++] = k; + k->current_value.updated = true; k->current_value.empty = false; facets->operations.values.registered++; - if(k->transform.cb) { + if(k->transform.cb && !k->transform.view_only) { facets->operations.values.transformed++; - k->transform.cb(facets, k->current_value.b, k->transform.data); + k->transform.cb(facets, k->current_value.b, FACETS_TRANSFORM_VALUE, k->transform.data); } // bool found = false; @@ -1326,7 +1500,8 @@ static void facet_row_key_value_insert_callback(const DICTIONARY_ITEM *item __ma FACET_ROW *row = data; (void)row; rkv->wb = buffer_create(0, NULL); - buffer_strcat(rkv->wb, rkv->tmp); + if(!rkv->empty) + buffer_strcat(rkv->wb, rkv->tmp); } static bool facet_row_key_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *data) { @@ -1334,8 +1509,11 @@ static bool facet_row_key_value_conflict_callback(const DICTIONARY_ITEM *item __ FACET_ROW_KEY_VALUE *n_rkv = new_value; FACET_ROW *row = data; (void)row; + rkv->empty = n_rkv->empty; + buffer_flush(rkv->wb); - buffer_strcat(rkv->wb, n_rkv->tmp); + if(!rkv->empty) + buffer_strcat(rkv->wb, n_rkv->tmp); return false; } @@ -1377,9 +1555,9 @@ static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into FACET_KEY *k; foreach_key_in_facets(facets, k) { FACET_ROW_KEY_VALUE t = { - .tmp = (k->current_value.updated) ? buffer_tostring(k->current_value.b) : FACET_VALUE_UNSET, + .tmp = (k->current_value.updated && !k->current_value.empty) ? buffer_tostring(k->current_value.b) : NULL, .wb = NULL, - .empty = k->current_value.empty, + .empty = !k->current_value.updated || k->current_value.empty, }; dictionary_set(row->dict, k->name, &t, sizeof(t)); } @@ -1418,10 +1596,8 @@ static void facets_row_keep_first_entry(FACETS *facets, usec_t usec) { facets->operations.first++; } -static void facets_row_keep(FACETS *facets, usec_t usec) { - facets->operations.rows.matched++; - - if(facets->anchor.key) { +static inline bool facets_is_entry_within_anchor(FACETS *facets, usec_t usec) { + if(facets->anchor.start_ut || facets->anchor.stop_ut) { // we have an anchor key // we don't want to keep rows on the other side of the direction @@ -1429,22 +1605,36 @@ static void facets_row_keep(FACETS *facets, usec_t usec) { default: case FACETS_ANCHOR_DIRECTION_BACKWARD: // we need to keep only the smaller timestamps - if (usec >= facets->anchor.key) { + if (facets->anchor.start_ut && usec >= facets->anchor.start_ut) { facets->operations.skips_before++; - return; + return false; + } + if (facets->anchor.stop_ut && usec <= facets->anchor.stop_ut) { + facets->operations.skips_after++; + return false; } break; case FACETS_ANCHOR_DIRECTION_FORWARD: // we need to keep only the bigger timestamps - if (usec <= facets->anchor.key) { + if (facets->anchor.start_ut && usec <= facets->anchor.start_ut) { facets->operations.skips_after++; - return; + return false; + } + if (facets->anchor.stop_ut && usec >= facets->anchor.stop_ut) { + facets->operations.skips_before++; + return false; } break; } } + return true; +} + +static void facets_row_keep(FACETS *facets, usec_t usec) { + facets->operations.rows.matched++; + if(unlikely(!facets->base)) { // the first row to keep facets_row_keep_first_entry(facets, usec); @@ -1510,81 +1700,112 @@ static void facets_row_keep(FACETS *facets, usec_t usec) { facets->items_to_return++; } +static inline void facets_reset_key(FACET_KEY *k) { + k->key_found_in_row = 0; + k->key_values_selected_in_row = 0; + k->current_value.updated = false; + k->current_value.empty = false; + k->current_value.hash = FACETS_HASH_ZERO; + k->current_value.v = NULL; +} + +static void facets_reset_keys_with_value_and_row(FACETS *facets) { + size_t entries = facets->keys_in_row.used; + + for(size_t p = 0; p < entries ;p++) { + FACET_KEY *k = facets->keys_in_row.array[p]; + facets_reset_key(k); + } + + facets->current_row.severity = FACET_ROW_SEVERITY_NORMAL; + facets->current_row.keys_matched_by_query = 0; + facets->keys_in_row.used = 0; +} + void facets_rows_begin(FACETS *facets) { FACET_KEY *k; foreach_key_in_facets(facets, k) { - k->key_found_in_row = 0; - k->key_values_selected_in_row = 0; - k->current_value.updated = false; - k->current_value.empty = false; - k->current_value.hash = FACETS_HASH_ZERO; - k->current_value.v = NULL; + facets_reset_key(k); } foreach_key_in_facets_done(k); - facets->current_row.severity = FACET_ROW_SEVERITY_NORMAL; - facets->current_row.keys_matched_by_query = 0; + facets->keys_in_row.used = 0; + facets_reset_keys_with_value_and_row(facets); } -void facets_row_finished(FACETS *facets, usec_t usec) { - if(facets->query && facets->keys_filtered_by_query && !facets->current_row.keys_matched_by_query) - goto cleanup; - +bool facets_row_finished(FACETS *facets, usec_t usec) { facets->operations.rows.evaluated++; - uint32_t total_keys = 0; - uint32_t selected_by = 0; + if((facets->query && facets->keys_filtered_by_query && !facets->current_row.keys_matched_by_query) || + (facets->timeframe.before_ut && usec > facets->timeframe.before_ut) || + (facets->timeframe.after_ut && usec < facets->timeframe.after_ut)) { + // this row is not useful + // 1. not matched by full text search, or + // 2. not in our timeframe + facets_reset_keys_with_value_and_row(facets); + return false; + } + + size_t entries = facets->keys_with_values.used; + size_t total_keys = 0; + size_t selected_keys = 0; + + for(size_t p = 0; p < entries ;p++) { + FACET_KEY *k = facets->keys_with_values.array[p]; - FACET_KEY *k; - foreach_key_in_facets(facets, k) { if(!k->key_found_in_row) { // put the FACET_VALUE_UNSET value into it facets_key_set_empty_value(facets, k); } - internal_fatal(!k->key_found_in_row, "all keys should be found in the row at this point"); internal_fatal(k->key_found_in_row != 1, "all keys should be matched exactly once at this point"); internal_fatal(k->key_values_selected_in_row > 1, "key values are selected in row more than once"); - k->key_found_in_row = 1; - total_keys++; - selected_by += (k->key_values_selected_in_row) ? 1 : 0; + + if(k->key_values_selected_in_row) + selected_keys++; + + if(unlikely(!facets->histogram.key && facets->histogram.hash == k->hash)) + facets->histogram.key = k; } - foreach_key_in_facets_done(k); - if(selected_by >= total_keys - 1) { - uint32_t found = 0; + bool within_anchor = facets_is_entry_within_anchor(facets, usec); - foreach_key_in_facets(facets, k) { - uint32_t counted_by = selected_by; + if(within_anchor && selected_keys >= total_keys - 1) { + size_t found = 0; (void)found; + + for(size_t p = 0; p < entries ;p++) { + FACET_KEY *k = facets->keys_with_values.array[p]; + + size_t counted_by = selected_keys; if (counted_by != total_keys && !k->key_values_selected_in_row) counted_by++; - if(counted_by == total_keys) { - if(k->values.enabled) { - FACET_VALUE *v = FACET_VALUE_GET_CURRENT_VALUE(k); - v->final_facet_value_counter++; - - if(selected_by == total_keys) - facets_histogram_update_value(facets, k, v, usec); - } + if (counted_by == total_keys) { + FACET_VALUE *v = FACET_VALUE_GET_CURRENT_VALUE(k); + v->final_facet_value_counter++; found++; } } - foreach_key_in_facets_done(k); internal_fatal(!found, "We should find at least one facet to count this row"); - (void)found; } - if(selected_by == total_keys) - facets_row_keep(facets, usec); + if(selected_keys == total_keys) { + // we need to keep this row + + facets_histogram_update_value(facets, usec); + + if(within_anchor) + facets_row_keep(facets, usec); + } + + facets_reset_keys_with_value_and_row(facets); -cleanup: - facets_rows_begin(facets); + return selected_keys == total_keys; } // ---------------------------------------------------------------------------- @@ -1635,49 +1856,307 @@ void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool w buffer_json_array_close(wb); // accepted_params } -void facets_report(FACETS *facets, BUFFER *wb) { - if(!(facets->options & FACETS_OPTION_DATA_ONLY)) { - buffer_json_member_add_boolean(wb, "show_ids", false); // do not show the column ids to the user - buffer_json_member_add_boolean(wb, "has_history", true); // enable date-time picker with after-before +static int facets_keys_reorder_compar(const void *a, const void *b) { + const FACET_KEY *ak = *((const FACET_KEY **)a); + const FACET_KEY *bk = *((const FACET_KEY **)b); - buffer_json_member_add_object(wb, "pagination"); - { - buffer_json_member_add_boolean(wb, "enabled", true); - buffer_json_member_add_string(wb, "key", "anchor"); - buffer_json_member_add_string(wb, "column", "timestamp"); - buffer_json_member_add_string(wb, "units", "timestamp_usec"); + const char *an = ak->name; + const char *bn = bk->name; + + if(!an) an = "0"; + if(!bn) bn = "0"; + + while(*an && ispunct(*an)) an++; + while(*bn && ispunct(*bn)) bn++; + + return strcasecmp(an, bn); +} + +void facets_sort_and_reorder_keys(FACETS *facets) { + size_t entries = facets->keys_with_values.used; + if(!entries) + return; + + FACET_KEY *keys[entries]; + memcpy(keys, facets->keys_with_values.array, sizeof(FACET_KEY *) * entries); + + qsort(keys, entries, sizeof(FACET_KEY *), facets_keys_reorder_compar); + + for(size_t i = 0; i < entries ;i++) + keys[i]->order = i + 1; +} + +static int facets_key_values_reorder_by_name_compar(const void *a, const void *b) { + const FACET_VALUE *av = *((const FACET_VALUE **)a); + const FACET_VALUE *bv = *((const FACET_VALUE **)b); + + const char *an = av->name; + const char *bn = bv->name; + + if(!an) an = "0"; + if(!bn) bn = "0"; + + while(*an && ispunct(*an)) an++; + while(*bn && ispunct(*bn)) bn++; + + int ret = strcasecmp(an, bn); + return ret; +} + +static int facets_key_values_reorder_by_count_compar(const void *a, const void *b) { + const FACET_VALUE *av = *((const FACET_VALUE **)a); + const FACET_VALUE *bv = *((const FACET_VALUE **)b); + + if(av->final_facet_value_counter < bv->final_facet_value_counter) + return 1; + + if(av->final_facet_value_counter > bv->final_facet_value_counter) + return -1; + + return facets_key_values_reorder_by_name_compar(a, b); +} + +static int facets_key_values_reorder_by_name_numeric_compar(const void *a, const void *b) { + const FACET_VALUE *av = *((const FACET_VALUE **)a); + const FACET_VALUE *bv = *((const FACET_VALUE **)b); + + const char *an = av->name; + const char *bn = bv->name; + + if(!an) an = "0"; + if(!bn) bn = "0"; + + if(strcmp(an, FACET_VALUE_UNSET) == 0) an = "0"; + if(strcmp(bn, FACET_VALUE_UNSET) == 0) bn = "0"; + + int64_t ad = str2ll(an, NULL); + int64_t bd = str2ll(bn, NULL); + + if(ad < bd) + return -1; + + if(ad > bd) + return 1; + + return facets_key_values_reorder_by_name_compar(a, b); +} + +static uint32_t facets_sort_and_reorder_values_internal(FACET_KEY *k) { + bool all_values_numeric = true; + size_t entries = k->values.used; + FACET_VALUE *values[entries], *v; + uint32_t used = 0; + foreach_value_in_key(k, v) { + if((k->facets->options & FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS) && v->empty) + continue; + + if(used >= entries) + break; + + values[used++] = v; + + if(all_values_numeric && !v->empty && v->name) { + const char *s = v->name; + while(isdigit(*s)) s++; + if(*s != '\0') + all_values_numeric = false; } - buffer_json_object_close(wb); // pagination + } + foreach_value_in_key_done(v); + + if(!used) + return 0; + + if(k->facets->options & FACETS_OPTION_SORT_FACETS_ALPHABETICALLY) { + if(all_values_numeric) + qsort(values, used, sizeof(FACET_VALUE *), facets_key_values_reorder_by_name_numeric_compar); + else + qsort(values, used, sizeof(FACET_VALUE *), facets_key_values_reorder_by_name_compar); + } + else + qsort(values, used, sizeof(FACET_VALUE *), facets_key_values_reorder_by_count_compar); + + for(size_t i = 0; i < used; i++) + values[i]->order = i + 1; + + return used; +} + +static uint32_t facets_sort_and_reorder_values(FACET_KEY *k) { + if(!k->values.enabled || !k->values.ll || !k->values.used) + return 0; + + if(!k->transform.cb || !(k->facets->options & FACETS_OPTION_SORT_FACETS_ALPHABETICALLY)) + return facets_sort_and_reorder_values_internal(k); + + // we have a transformation and has to be sorted alphabetically + + BUFFER *tb = buffer_create(0, NULL); + uint32_t ret = 0; + + size_t entries = k->values.used; + const char *values[entries]; + FACET_VALUE *v; + uint32_t used = 0; + + foreach_value_in_key(k, v) { + if(used >= entries) + break; + + values[used++] = v->name; + + buffer_flush(tb); + buffer_strcat(tb, v->name); + k->transform.cb(k->facets, tb, FACETS_TRANSFORM_FACET_SORT, k->transform.data); + v->name = strdupz(buffer_tostring(tb)); + } + foreach_value_in_key_done(v); + + ret = facets_sort_and_reorder_values_internal(k); + + used = 0; + foreach_value_in_key(k, v) { + if(used >= entries) + break; + + freez((void *)v->name); + v->name = values[used++]; + } + foreach_value_in_key_done(v); + + buffer_free(tb); + return ret; +} + +void facets_table_config(BUFFER *wb) { + buffer_json_member_add_boolean(wb, "show_ids", false); // do not show the column ids to the user + buffer_json_member_add_boolean(wb, "has_history", true); // enable date-time picker with after-before + + buffer_json_member_add_object(wb, "pagination"); + { + buffer_json_member_add_boolean(wb, "enabled", true); + buffer_json_member_add_string(wb, "key", "anchor"); + buffer_json_member_add_string(wb, "column", "timestamp"); + buffer_json_member_add_string(wb, "units", "timestamp_usec"); + } + buffer_json_object_close(wb); // pagination +} + +static const char *facets_json_key_name_string(FACET_KEY *k, DICTIONARY *used_hashes_registry) { + if(k->name) { + if(used_hashes_registry && !k->default_selected_for_values) { + char hash_str[FACET_STRING_HASH_SIZE]; + facets_hash_to_str(k->hash, hash_str); + dictionary_set(used_hashes_registry, hash_str, (void *)k->name, strlen(k->name) + 1); + } + + return k->name; + } + + // key has no name + const char *name = "[UNAVAILABLE_FIELD]"; + if(used_hashes_registry) { + char hash_str[FACET_STRING_HASH_SIZE]; + facets_hash_to_str(k->hash, hash_str); + const char *s = dictionary_get(used_hashes_registry, hash_str); + if(s) name = s; + } + + return name; +} + +static const char *facets_json_key_value_string(FACET_KEY *k, FACET_VALUE *v, DICTIONARY *used_hashes_registry) { + if(v->name) { + if(used_hashes_registry && !k->default_selected_for_values && v->selected) { + char hash_str[FACET_STRING_HASH_SIZE]; + facets_hash_to_str(v->hash, hash_str); + dictionary_set(used_hashes_registry, hash_str, (void *)v->name, strlen(v->name) + 1); + } + + return v->name; + } + + // key has no name + const char *name = "[unavailable field]"; + + if(used_hashes_registry) { + char hash_str[FACET_STRING_HASH_SIZE]; + facets_hash_to_str(v->hash, hash_str); + const char *s = dictionary_get(used_hashes_registry, hash_str); + if(s) name = s; + } + + return name; +} + +void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) { + if(!(facets->options & FACETS_OPTION_DATA_ONLY)) { + facets_table_config(wb); facets_accepted_parameters_to_json_array(facets, wb, true); } - if(!(facets->options & FACETS_OPTION_DISABLE_ALL_FACETS)) { - buffer_json_member_add_array(wb, "facets"); - { + // ------------------------------------------------------------------------ + // facets + + if(!(facets->options & FACETS_OPTION_DONT_SEND_FACETS)) { + bool show_facets = false; + + if(facets->options & FACETS_OPTION_DATA_ONLY) { + if(facets->options & FACETS_OPTION_SHOW_DELTAS) { + buffer_json_member_add_array(wb, "facets_delta"); + show_facets = true; + } + } + else { + buffer_json_member_add_array(wb, "facets"); + show_facets = true; + } + + if(show_facets) { + BUFFER *tb = NULL; FACET_KEY *k; foreach_key_in_facets(facets, k) { if(!k->values.enabled) continue; + if(!facets_sort_and_reorder_values(k)) + // no values for this key + continue; + buffer_json_add_array_item_object(wb); // key { buffer_json_member_add_string(wb, "id", hash_to_static_string(k->hash)); - buffer_json_member_add_string(wb, "name", k->name); - - if(!k->order) - k->order = facets->order++; + buffer_json_member_add_string(wb, "name", facets_json_key_name_string(k, used_hashes_registry)); + if(!k->order) k->order = facets->order++; buffer_json_member_add_uint64(wb, "order", k->order); + buffer_json_member_add_array(wb, "options"); { FACET_VALUE *v; foreach_value_in_key(k, v) { + if((facets->options & FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS) && v->empty) + continue; + buffer_json_add_array_item_object(wb); { buffer_json_member_add_string(wb, "id", hash_to_static_string(v->hash)); - buffer_json_member_add_string(wb, "name", v->name); + + if(!v->empty && k->transform.cb && k->transform.view_only) { + if(!tb) + tb = buffer_create(0, NULL); + + buffer_flush(tb); + buffer_strcat(tb, v->name); + k->transform.cb(facets, tb, FACETS_TRANSFORM_FACET, k->transform.data); + buffer_json_member_add_string(wb, "name", buffer_tostring(tb)); + } + else + buffer_json_member_add_string(wb, "name", facets_json_key_value_string(k, v, used_hashes_registry)); + buffer_json_member_add_uint64(wb, "count", v->final_facet_value_counter); + buffer_json_member_add_uint64(wb, "order", v->order); } buffer_json_object_close(wb); } @@ -1688,77 +2167,81 @@ void facets_report(FACETS *facets, BUFFER *wb) { buffer_json_object_close(wb); // key } foreach_key_in_facets_done(k); + buffer_free(tb); + buffer_json_array_close(wb); // facets } - buffer_json_array_close(wb); // facets } - if(!(facets->options & FACETS_OPTION_DATA_ONLY)) { - buffer_json_member_add_object(wb, "columns"); - { - size_t field_id = 0; - buffer_rrdf_table_add_field( - wb, field_id++, - "timestamp", "Timestamp", - RRDF_FIELD_TYPE_TIMESTAMP, - RRDF_FIELD_VISUAL_VALUE, - RRDF_FIELD_TRANSFORM_DATETIME_USEC, 0, NULL, NAN, - RRDF_FIELD_SORT_DESCENDING, - NULL, - RRDF_FIELD_SUMMARY_COUNT, - RRDF_FIELD_FILTER_RANGE, - RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_UNIQUE_KEY, - NULL); - - buffer_rrdf_table_add_field( - wb, field_id++, - "rowOptions", "rowOptions", - RRDF_FIELD_TYPE_NONE, - RRDR_FIELD_VISUAL_ROW_OPTIONS, - RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN, - RRDF_FIELD_SORT_FIXED, - NULL, - RRDF_FIELD_SUMMARY_COUNT, - RRDF_FIELD_FILTER_NONE, - RRDR_FIELD_OPTS_DUMMY, - NULL); + // ------------------------------------------------------------------------ + // columns - FACET_KEY *k; - foreach_key_in_facets(facets, k) { - RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_NONE; - bool visible = k->options & (FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_STICKY); - - if ((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE && k->values.enabled)) - visible = true; - - if (!visible) - visible = simple_pattern_matches(facets->visible_keys, k->name); - - if (visible) - options |= RRDF_FIELD_OPTS_VISIBLE; - - if (k->options & FACET_KEY_OPTION_MAIN_TEXT) - options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP; - - const char *hash_str = hash_to_static_string(k->hash); - - buffer_rrdf_table_add_field( - wb, field_id++, - hash_str, k->name ? k->name : hash_str, - RRDF_FIELD_TYPE_STRING, - (k->options & FACET_KEY_OPTION_RICH_TEXT) ? RRDF_FIELD_VISUAL_RICH : RRDF_FIELD_VISUAL_VALUE, - RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN, - RRDF_FIELD_SORT_ASCENDING, - NULL, - RRDF_FIELD_SUMMARY_COUNT, - (k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE - : RRDF_FIELD_FILTER_FACET, - options, - FACET_VALUE_UNSET); - } - foreach_key_in_facets_done(k); - } - buffer_json_object_close(wb); // columns + buffer_json_member_add_object(wb, "columns"); + { + size_t field_id = 0; + buffer_rrdf_table_add_field( + wb, field_id++, + "timestamp", "Timestamp", + RRDF_FIELD_TYPE_TIMESTAMP, + RRDF_FIELD_VISUAL_VALUE, + RRDF_FIELD_TRANSFORM_DATETIME_USEC, 0, NULL, NAN, + RRDF_FIELD_SORT_DESCENDING, + NULL, + RRDF_FIELD_SUMMARY_COUNT, + RRDF_FIELD_FILTER_RANGE, + RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_UNIQUE_KEY, + NULL); + + buffer_rrdf_table_add_field( + wb, field_id++, + "rowOptions", "rowOptions", + RRDF_FIELD_TYPE_NONE, + RRDR_FIELD_VISUAL_ROW_OPTIONS, + RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN, + RRDF_FIELD_SORT_FIXED, + NULL, + RRDF_FIELD_SUMMARY_COUNT, + RRDF_FIELD_FILTER_NONE, + RRDR_FIELD_OPTS_DUMMY, + NULL); + + FACET_KEY *k; + foreach_key_in_facets(facets, k) { + RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_NONE; + bool visible = k->options & (FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_STICKY); + + if ((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE && k->values.enabled)) + visible = true; + + if (!visible) + visible = simple_pattern_matches(facets->visible_keys, k->name); + + if (visible) + options |= RRDF_FIELD_OPTS_VISIBLE; + + if (k->options & FACET_KEY_OPTION_MAIN_TEXT) + options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP; + + const char *hash_str = hash_to_static_string(k->hash); + + buffer_rrdf_table_add_field( + wb, field_id++, + hash_str, k->name ? k->name : hash_str, + RRDF_FIELD_TYPE_STRING, + (k->options & FACET_KEY_OPTION_RICH_TEXT) ? RRDF_FIELD_VISUAL_RICH : RRDF_FIELD_VISUAL_VALUE, + RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN, + RRDF_FIELD_SORT_ASCENDING, + NULL, + RRDF_FIELD_SUMMARY_COUNT, + (k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE + : RRDF_FIELD_FILTER_FACET, + options, FACET_VALUE_UNSET); + } + foreach_key_in_facets_done(k); } + buffer_json_object_close(wb); // columns + + // ------------------------------------------------------------------------ + // rows data buffer_json_member_add_array(wb, "data"); { @@ -1767,10 +2250,10 @@ void facets_report(FACETS *facets, BUFFER *wb) { for(FACET_ROW *row = facets->base ; row ;row = row->next) { internal_fatal( - facets->anchor.key && ( - (facets->anchor.direction == FACETS_ANCHOR_DIRECTION_BACKWARD && row->usec >= facets->anchor.key) || - (facets->anchor.direction == FACETS_ANCHOR_DIRECTION_FORWARD && row->usec <= facets->anchor.key) - ), "Wrong data returned related to %s anchor!", facets->anchor.direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward"); + facets->anchor.start_ut && ( + (facets->anchor.direction == FACETS_ANCHOR_DIRECTION_BACKWARD && row->usec >= facets->anchor.start_ut) || + (facets->anchor.direction == FACETS_ANCHOR_DIRECTION_FORWARD && row->usec <= facets->anchor.start_ut) + ), "Wrong data returned related to %s start anchor!", facets->anchor.direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward"); internal_fatal(last_usec && row->usec > last_usec, "Wrong order of data returned!"); @@ -1780,6 +2263,9 @@ void facets_report(FACETS *facets, BUFFER *wb) { buffer_json_add_array_item_uint64(wb, row->usec); buffer_json_add_array_item_object(wb); { + if(facets->severity.cb) + row->severity = facets->severity.cb(facets, row, facets->severity.data); + buffer_json_member_add_string(wb, "severity", facets_severity_to_string(row->severity)); } buffer_json_object_close(wb); @@ -1796,8 +2282,13 @@ void facets_report(FACETS *facets, BUFFER *wb) { facets->operations.values.dynamic++; } else { - if(!rkv || rkv->empty) - buffer_json_add_array_item_string(wb, FACET_VALUE_UNSET); + if(!rkv || rkv->empty) { + buffer_json_add_array_item_string(wb, NULL); + } + else if(unlikely(k->transform.cb && k->transform.view_only)) { + k->transform.cb(facets, rkv->wb, FACETS_TRANSFORM_DATA, k->transform.data); + buffer_json_add_array_item_string(wb, buffer_tostring(rkv->wb)); + } else buffer_json_add_array_item_string(wb, buffer_tostring(rkv->wb)); } @@ -1814,7 +2305,10 @@ void facets_report(FACETS *facets, BUFFER *wb) { buffer_json_array_close(wb); } - if(facets->histogram.enabled && !(facets->options & FACETS_OPTION_DISABLE_HISTOGRAM)) { + // ------------------------------------------------------------------------ + // histogram + + if(facets->histogram.enabled && !(facets->options & FACETS_OPTION_DONT_SEND_HISTOGRAM)) { FACETS_HASH first_histogram_hash = 0; buffer_json_member_add_array(wb, "available_histograms"); { @@ -1840,31 +2334,60 @@ void facets_report(FACETS *facets, BUFFER *wb) { if(!k || !k->values.enabled) k = FACETS_KEY_GET_FROM_INDEX(facets, first_histogram_hash); - buffer_json_member_add_object(wb, "histogram"); - { + bool show_histogram = false; + + if(facets->options & FACETS_OPTION_DATA_ONLY) { + if(facets->options & FACETS_OPTION_SHOW_DELTAS) { + buffer_json_member_add_object(wb, "histogram_delta"); + show_histogram = true; + } + } + else { + buffer_json_member_add_object(wb, "histogram"); + show_histogram = true; + } + + if(show_histogram) { buffer_json_member_add_string(wb, "id", k ? hash_to_static_string(k->hash) : ""); buffer_json_member_add_string(wb, "name", k ? k->name : ""); buffer_json_member_add_object(wb, "chart"); - facets_histogram_generate(facets, k, wb); - buffer_json_object_close(wb); + { + facets_histogram_generate(facets, k, wb); + } + buffer_json_object_close(wb); // chart + buffer_json_object_close(wb); // histogram } - buffer_json_object_close(wb); // histogram } } - if(!(facets->options & FACETS_OPTION_DATA_ONLY)) { - buffer_json_member_add_object(wb, "items"); - { - buffer_json_member_add_uint64(wb, "evaluated", facets->operations.rows.evaluated); - buffer_json_member_add_uint64(wb, "matched", facets->operations.rows.matched); - buffer_json_member_add_uint64(wb, "returned", facets->items_to_return); - buffer_json_member_add_uint64(wb, "max_to_return", facets->max_items_to_return); - buffer_json_member_add_uint64(wb, "before", facets->operations.skips_before); - buffer_json_member_add_uint64(wb, "after", facets->operations.skips_after + facets->operations.shifts); + // ------------------------------------------------------------------------ + // items + + bool show_items = false; + if(facets->options & FACETS_OPTION_DATA_ONLY) { + if(facets->options & FACETS_OPTION_SHOW_DELTAS) { + buffer_json_member_add_object(wb, "items_delta"); + show_items = true; } + } + else { + buffer_json_member_add_object(wb, "items"); + show_items = true; + } + + if(show_items) { + buffer_json_member_add_uint64(wb, "evaluated", facets->operations.rows.evaluated); + buffer_json_member_add_uint64(wb, "matched", facets->operations.rows.matched); + buffer_json_member_add_uint64(wb, "returned", facets->items_to_return); + buffer_json_member_add_uint64(wb, "max_to_return", facets->max_items_to_return); + buffer_json_member_add_uint64(wb, "before", facets->operations.skips_before); + buffer_json_member_add_uint64(wb, "after", facets->operations.skips_after + facets->operations.shifts); buffer_json_object_close(wb); // items } + // ------------------------------------------------------------------------ + // stats + buffer_json_member_add_object(wb, "stats"); { buffer_json_member_add_uint64(wb, "first", facets->operations.first); diff --git a/libnetdata/facets/facets.h b/libnetdata/facets/facets.h index 2ae0e62d6f5181..3eaa39564e5d61 100644 --- a/libnetdata/facets/facets.h +++ b/libnetdata/facets/facets.h @@ -12,6 +12,14 @@ typedef enum __attribute__((packed)) { FACETS_ANCHOR_DIRECTION_BACKWARD, } FACETS_ANCHOR_DIRECTION; +typedef enum __attribute__((packed)) { + FACETS_TRANSFORM_VALUE, + FACETS_TRANSFORM_HISTOGRAM, + FACETS_TRANSFORM_FACET, + FACETS_TRANSFORM_DATA, + FACETS_TRANSFORM_FACET_SORT, +} FACETS_TRANSFORMATION_SCOPE; + typedef enum __attribute__((packed)) { FACET_KEY_OPTION_FACET = (1 << 0), // filterable values FACET_KEY_OPTION_NO_FACET = (1 << 1), // non-filterable value @@ -22,6 +30,7 @@ typedef enum __attribute__((packed)) { FACET_KEY_OPTION_MAIN_TEXT = (1 << 6), // full width and wrap FACET_KEY_OPTION_RICH_TEXT = (1 << 7), FACET_KEY_OPTION_REORDER = (1 << 8), // give the key a new order id on first encounter + FACET_KEY_OPTION_TRANSFORM_VIEW = (1 << 9), // when registering the transformation, do it only at the view, not on all data } FACET_KEY_OPTIONS; typedef enum __attribute__((packed)) { @@ -48,17 +57,22 @@ typedef struct facet_row { typedef struct facets FACETS; typedef struct facet_key FACET_KEY; -typedef void (*facets_key_transformer_t)(FACETS *facets __maybe_unused, BUFFER *wb, void *data); +typedef void (*facets_key_transformer_t)(FACETS *facets __maybe_unused, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope, void *data); typedef void (*facet_dynamic_row_t)(FACETS *facets, BUFFER *json_array, FACET_ROW_KEY_VALUE *rkv, FACET_ROW *row, void *data); +typedef FACET_ROW_SEVERITY (*facet_row_severity_t)(FACETS *facets, FACET_ROW *row, void *data); FACET_KEY *facets_register_dynamic_key_name(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facet_dynamic_row_t cb, void *data); FACET_KEY *facets_register_key_name_transformation(FACETS *facets, const char *key, FACET_KEY_OPTIONS options, facets_key_transformer_t cb, void *data); +void facets_register_row_severity(FACETS *facets, facet_row_severity_t cb, void *data); typedef enum __attribute__((packed)) { - FACETS_OPTION_ALL_FACETS_VISIBLE = (1 << 0), // all facets, should be visible by default in the table - FACETS_OPTION_ALL_KEYS_FTS = (1 << 1), // all keys are searchable by full text search - FACETS_OPTION_DISABLE_ALL_FACETS = (1 << 2), - FACETS_OPTION_DISABLE_HISTOGRAM = (1 << 3), - FACETS_OPTION_DATA_ONLY = (1 << 4), + FACETS_OPTION_ALL_FACETS_VISIBLE = (1 << 0), // all facets should be visible by default in the table + FACETS_OPTION_ALL_KEYS_FTS = (1 << 1), // all keys are searchable by full text search + FACETS_OPTION_DONT_SEND_FACETS = (1 << 2), // "facets" object will not be included in the report + FACETS_OPTION_DONT_SEND_HISTOGRAM = (1 << 3), // "histogram" object will not be included in the report + FACETS_OPTION_DATA_ONLY = (1 << 4), + FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS = (1 << 5), // empty facet values will not be included in the report + FACETS_OPTION_SORT_FACETS_ALPHABETICALLY = (1 << 6), + FACETS_OPTION_SHOW_DELTAS = (1 << 7), } FACETS_OPTIONS; FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys); @@ -67,23 +81,37 @@ void facets_destroy(FACETS *facets); void facets_accepted_param(FACETS *facets, const char *param); void facets_rows_begin(FACETS *facets); -void facets_row_finished(FACETS *facets, usec_t usec); +bool facets_row_finished(FACETS *facets, usec_t usec); FACET_KEY *facets_register_key_name(FACETS *facets, const char *key, FACET_KEY_OPTIONS options); void facets_set_query(FACETS *facets, const char *query); void facets_set_items(FACETS *facets, uint32_t items); -void facets_set_anchor(FACETS *facets, usec_t anchor, FACETS_ANCHOR_DIRECTION direction); +void facets_set_anchor(FACETS *facets, usec_t start_ut, usec_t stop_ut, FACETS_ANCHOR_DIRECTION direction); +void facets_enable_slice_mode(FACETS *facets); + FACET_KEY *facets_register_facet_id(FACETS *facets, const char *key_id, FACET_KEY_OPTIONS options); void facets_register_facet_id_filter(FACETS *facets, const char *key_id, char *value_id, FACET_KEY_OPTIONS options); -void facets_set_histogram_by_id(FACETS *facets, const char *key_id, usec_t after_ut, usec_t before_ut); -void facets_set_histogram_by_name(FACETS *facets, const char *key_name, usec_t after_ut, usec_t before_ut); +void facets_set_timeframe_and_histogram_by_id(FACETS *facets, const char *key_id, usec_t after_ut, usec_t before_ut); +void facets_set_timeframe_and_histogram_by_name(FACETS *facets, const char *key_name, usec_t after_ut, usec_t before_ut); void facets_add_key_value(FACETS *facets, const char *key, const char *value); void facets_add_key_value_length(FACETS *facets, const char *key, size_t key_len, const char *value, size_t value_len); -void facets_report(FACETS *facets, BUFFER *wb); +void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry); void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool with_keys); void facets_set_current_row_severity(FACETS *facets, FACET_ROW_SEVERITY severity); -void facets_data_only_mode(FACETS *facets); +void facets_set_additional_options(FACETS *facets, FACETS_OPTIONS options); + +bool facets_key_name_is_filter(FACETS *facets, const char *key); +bool facets_key_name_is_facet(FACETS *facets, const char *key); +bool facets_key_name_value_length_is_selected(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length); +void facets_add_possible_value_name_to_key(FACETS *facets, const char *key, size_t key_length, const char *value, size_t value_length); + +void facets_sort_and_reorder_keys(FACETS *facets); +usec_t facets_row_oldest_ut(FACETS *facets); +usec_t facets_row_newest_ut(FACETS *facets); +uint32_t facets_rows(FACETS *facets); + +void facets_table_config(BUFFER *wb); #endif diff --git a/libnetdata/socket/socket.c b/libnetdata/socket/socket.c index a48b6ec53f93b1..67dc4c71c06169 100644 --- a/libnetdata/socket/socket.c +++ b/libnetdata/socket/socket.c @@ -10,6 +10,40 @@ #include "../libnetdata.h" +bool ip_to_hostname(const char *ip, char *dst, size_t dst_len) { + if(!dst || !dst_len) + return false; + + struct sockaddr_in sa; + struct sockaddr_in6 sa6; + struct sockaddr *sa_ptr; + int sa_len; + + // Try to convert the IP address to sockaddr_in (IPv4) + if (inet_pton(AF_INET, ip, &(sa.sin_addr)) == 1) { + sa.sin_family = AF_INET; + sa_ptr = (struct sockaddr *)&sa; + sa_len = sizeof(sa); + } + // Try to convert the IP address to sockaddr_in6 (IPv6) + else if (inet_pton(AF_INET6, ip, &(sa6.sin6_addr)) == 1) { + sa6.sin6_family = AF_INET6; + sa_ptr = (struct sockaddr *)&sa6; + sa_len = sizeof(sa6); + } + + else { + dst[0] = '\0'; + return false; + } + + // Perform the reverse lookup + int res = getnameinfo(sa_ptr, sa_len, dst, dst_len, NULL, 0, NI_NAMEREQD); + if(res != 0) + return false; + + return true; +} SOCKET_PEERS socket_peers(int sock_fd) { SOCKET_PEERS peers; diff --git a/libnetdata/socket/socket.h b/libnetdata/socket/socket.h index c4bd473609220b..e4ca08d47fede3 100644 --- a/libnetdata/socket/socket.h +++ b/libnetdata/socket/socket.h @@ -243,5 +243,6 @@ typedef struct socket_peers { } SOCKET_PEERS; SOCKET_PEERS socket_peers(int sock_fd); +bool ip_to_hostname(const char *ip, char *dst, size_t dst_len); #endif //NETDATA_SOCKET_H