-
Notifications
You must be signed in to change notification settings - Fork 2
/
searchbnf.conf
6518 lines (5835 loc) · 333 KB
/
searchbnf.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Version 8.0.1
##################################################################
# Pseudo-BNF Definitions for Search Language
#########################
# LAST CHANGED: 11FEB19
#########################
# FORMATTING
#########################
# - Adjacent tokens implicitly allow no whitespace.
# - All literals are case-insensitive.
# - Aside from reserved characters ("<>()|?*+") and <tokens>, everything else is taken literally.
# Those characters need to be quoted. use \" to represent a quote.
# - Examples are now broken into a separate values -- example1,
# example2, example3. If there's a text comment that goes with
# exampleN then it should be in the commentN attribute. For example:
# "example2 = 5d" "comment2 = 5 days"
# - The only reserved characters are "<>()|?*+",
# - Whitespace (including newlines) matches \s+
# - Regex-like grouping
# (): grouping
# <term> : <term> is required
# (<term>)? : <term> is optional
# (<term>)* : <term> is optional and repeated 0 or more times
# (<term>)+ : <term> is required and repeated 1 or more times
# - <terms> can be named for readability with a colon and a default value
# "move <field:fromfield=localhost> to <field:tofield=localhost>"
#
#########################
# COMMAND STRUCTURE
#########################
#
# [command_name-command]
# simplesyntax = Use only if the command is complex. See SIMPLE SYNTAX section below.
# syntax = command_name (parameter_name=<datatype>) for example (maxlines=<int>)?
# See FORMATTING section above for more details
# alias = alias_name Optional. Only supply an alias if it it ABSOLUTELY necessary. Otherwise omit.
# shortdesc = Public commands only. Provide a one sentence description.
# description = Provide a full description. This should match the description in the Search Reference.
# See DESCRIPTION FORMATTING below.
# note = (optional) Add special notes of operation
# example1 = ... |command_name parameter parameter (NOTE: include common examples with 1 or more parameters)
# comment1 = The explanation for what the command is doing
# internal category = category for internal (legacy reference only)
# external category = category for external use (See CATEGORIES section below)
# maintainer = name of person who originally worked on the command or who is responsible for the command now.
# appears-in = The version that the command first appeared in
# usage = specify public, internal, deprecated, or fun (for easter eggs only)
# related = list of SPL commands related to this command
# tags = other words that users might search on that are similar to the command_name
#
#########################
# SIMPLE SYNTAX
#########################
# - To simplify the bnf for the end user, an optional "simplesyntax" can be specified,
# - To remove things like (m|min|mins|minute|minutes) so that the bnf becomes understandable
# - Potentially can be used to remove obscure/rarely used features
# of search commands, but that should be done sparingly and with some thought.
#########################
# DESCRIPTION FORMATTING
#########################
# - For a command's DESCRIPTION, when automatically converted to html:
# - multiple whitespace are removed
# - for convenience, \p\ will cause a paragraph break and \i\ a newline and indent (<br> )
# - <terms> are italicized, UPPERCASETERMS and quoted terms are put into <code/>
#########################
# CATEGORIES
#########################
# The external categories are:
# - correlation
# - data::managing, data::viewing
# - fields::adding, fields::extracting, fields::modifying
# - find_anomalies
# - geographic_location
# - indexes:manage_summary
# - predicting_trending
# - reporting
# - results::alerting, results::appending, results::filtering, results::formatting
# results::generating, results::grouping, results::reading, results::reordering
# results::writing
# - search
# - subsearch
# - time
#
#########################
# conventions in attributes
#
# fields that begin with "_" are considered Splunk internal fields
# and are not used in many of the commands that by default operate
# on all the fields
#
# any stanza that ends with "-command" is considered a command.
#
#########################
# common term definitions
#########################
#
# field is any field, non-wildcarded
# wc-field represents wildcarded fields
# wc-string is any wildcarded string
#
#########################
#
# <field-list> ::= <field> | <field-list> <ws> <field>
# <field> ::= <string>
# <wc-field-list> ::= <wc-field> | <wc-field-list> <ws> <wc-field>
# <wc-field> ::= <wc-string>
# <wc-string> ::= <string>
#
# <field-and-value> ::= ? a field and value separated by a :: eg host::localhost ?
# <field-and-value-list> ::= <field-and-value> | <field-and-value-list> <ws> <field-and-value>
#
# <tag> ::= <string>
# <tag-list> ::= <tag> | <tag-list>
#
# <bool> ::= <true> | <false>
# <true> ::= T | TRUE
# <false> ::= F | FALSE
#
# <string> ::= ? <unquoted-string> | <double-quoted-string> ?
# <unquoted-string> ::= ? any unbroken sequence of alphanumeric characters plus underscore ?
# <double-quoted-string> ::= any string enclosed by double quotes
#
# <int> ::= ? any integer ?
#
# we do not support 0 and 100th percentiles
# with the perc aggregator, to achieve, use the min() and max() aggregators.
# <percentile> ::= ? any integer between 1 and 99 ?#
# <num> ::= ? any real number ?
#
# <filename> ::= ? a valid path to a file on the server ?
#
# [filename]
# syntax = <string>
# description = A path on the filesystem of the server
#
# [term]
# syntax = [a-zA-Z0-9_-]+
#
# [command-pipeline]
# syntax = <generating-command> (| <command>)*
#
# [boolean-operator]
# syntax = <boolean-operator-not>|<boolean-operator-and>|<boolean-operator-or>
#
# [boolean-operator-not]
# syntax = NOT
# description = Case-sensitive boolean operator.
#
# [boolean-operator-and]
# syntax = AND
# description = Case-sensitive boolean operator.
#
# [boolean-operator-or]
# syntax = OR
# description = Case-sensitive boolean operator.
#
###########################################################################
# COMMAND LIST:
#
##################
# NOTE:
# pre* commands are not included in this list as all pre* commands are the
# map portion of the original command with exactly the same syntax and cannot be
# access externally (via CLI or UI)
# (e.g. prestats / stats, prediscretize / discretize, prededup / dedup)
##################
#
# !!PLEASE DEFINE THESE UNDEFINED STANZAS:
#
# Ignoring undefined stanza: search-pipeline
# Ignoring undefined stanza: search-directive
# Ignoring undefined stanza: quoted-str
##################
# abstract/excerpt
##################
[abstract-command]
syntax = abstract (maxterms=<int>)? (maxlines=<int>)?
alias = excerpt
shortdesc = Shortens the text of results to a brief summary representation.
description = Produce an abstract -- a summary or brief representation -- of the text of search results. The original text is replaced by the summary, which is produced by a scoring mechanism. If the event is larger than the selected maxlines, those with more terms and more terms on adjacent lines are preferred over those with fewer terms. If a line has a search term, its neighboring lines also partially match, and may be returned to provide context. When there are gaps between the selected lines, lines are prefixed with "...". \p\\
If the text of a result has fewer lines or an equal number of lines to maxlines, no change will occur.\i\\
* <maxlines> accepts values from 1 - 500. \i\\
* <maxterms> accepts values from 1 - 1000.
commentcheat = Show a summary of up to 5 lines for each search result.
examplecheat = ... |abstract maxlines=5
category = formatting
maintainer = david
appears-in = 3.0
usage = public
related = highlight
tags = condense summarize summary outline pare prune shorten skim snip sum trim
##################
# accum
##################
[accum-command]
syntax = accum <field> (AS <field>)?
shortdesc = Keeps a running total of a specified numeric field.
description = For each event where <field> is a number, keep a running total of the sum of this number and write it out to either the same field, or a new field if specified.
comment1 = Save the running total of "count" in a field called "total_count".
example1 = ... | accum count AS total_count
usage = public
appears-in = 4.0
maintainer = steveyz
category = fields::add
tags = total sum accumulate
related = autoregress, delta, streamstats, trendline
##################
# addcoltotals
##################
[addcoltotals-command]
syntax = addcoltotals (labelfield=<field>)? (label=<string>)? <field-list>?
shortdesc = Appends a new result to the end of the search result set.
description = Appends a new result to the end of the search result set.\
The result contains the sum of each numeric field or you can specify which fields\
to summarize. Results are displayed on the Statistics tab. If the labelfield argument\
is specified, a column is added to the statistical results table with the name\
specified.
comment1 = Compute the sums of all the fields, and put the sums in a summary event called "change_name".
example1 = ... | addcoltotals labelfield=change_name label=ALL
comment2 = Add a column total for two specific fields in a table.
example2 = sourcetype=access_* | table userId bytes avgTime duration | addcoltotals bytes duration
comment3 = Augment a chart with a total of the values present.
example3 = index=_internal source=*metrics.log group=pipeline |stats avg(cpu_seconds) by processor |addcoltotals labelfield=processor
category = reporting
appears-in = 3.0
maintainer = steveyz
usage = public
related = stats
tags = total add calculate sum
##################
# addinfo
##################
[addinfo-command]
syntax = addinfo
shortdesc = Add fields that contain common information about the current search.
description = Adds global information about the search to each event. The addinfo command is primarily an internal component of summary indexing. \i\\
Currently the following fields are added: \i\\
"info_min_time" - the earliest time bound for the search \i\\
"info_max_time" - the latest time bound for the search \i\\
"info_search_id" - query id of the search that generated the event \i\\
"info_search_time" - time when the search was executed.
maintainer = ledion
comment = Add information about the search to each event.
example = ... |addinfo
appears-in = 3.2
usage = public
tags = search info
category = fields::add
related = search
##################
# addtotals
##################
[addtotals-command]
syntax = addtotals (row=<bool>)? (col=<bool>)? (labelfield=<field>)? (label=<string>)? (fieldname=<field>)? <field-list>
shortdesc = Computes the sum of all numeric fields for each result.
description = If "row=t" (default if invoked as 'addtotals') for each result, computes the arithmetic sum of all\
numeric fields that match <field-list> (wildcarded field list).\
If list is empty all fields are considered.\
The sum is placed in the specified field or "Total" if none was specified.\
If "col=t" (default if invoked as 'addcoltotals'), adds a new result at the end that represents the sum of each field.\
LABELFIELD, if specified, is a field that will be added to this summary \
event with the value set by the 'label' option.
comment1 = Compute the sums of the numeric fields of each results.
example1 = ... | addtotals
comment2 = Compute the sums of the numeric fields that match the given list, and save the sums in the field "sum".
example2 = ... | addtotals fieldname=sum foobar* *baz*
comment3 = Compute the sums of all the fields, and put the sums in a summary event called "change_name".
example3 = ... | addtotals col=t labelfield=change_name label=ALL
commentcheat = Calculate the sums of the numeric fields of each result, and put the sums in the field "sum".
examplecheat = ... | addtotals fieldname=sum
category = reporting
appears-in = 3.0
maintainer = steveyz
usage = public
related = stats
tags = total add calculate sum
##################
# analyzefields
##################
[analyzefields-command]
syntax = analyzefields classfield=<field>
shortdesc = Finds degree of correlation between a target discrete field and other numerical fields.
description = Using <field> as a discrete random variable, analyze all *numerical* fields to determine the ability for each of those fields to "predict" the value of the classfield.\
In other words, analyzefields determines the stability of the relationship between values in the target classfield and numeric values in other fields. \i\\
As a reporting command, analyzefields consumes all input results, and generates one output result per identified numeric field. \i\\
For best results, classfield should have 2 distinct values, although multi-class analysis is possible.
comment1 = Analyze the numerical fields to predict the value of "is_activated".
example1 = ... | analyzefields classfield=is_activated
usage = public beta
appears-in = 4.0
maintainer = steveyz
alias = af
tags = analyze predict
category = reporting
related = anomalousvalue
##################
# anomalies
##################
[anomalies-command]
syntax = anomalies (threshold=<num>)? (labelonly=<bool>)? (normalize=<bool>)? (maxvalues=<int>)? (field=<field>)? (blacklist=<filename>)? (blacklistthreshold=<num>)? (<by-clause>)?
shortdesc = Computes an "unexpectedness" score for an event.
description = Determines the degree of "unexpectedness" of an event's field \
value, based on the previous MAXVALUE events. By default it \
removes events that are well-expected (unexpectedness > \
THRESHOLD). The default THRESHOLD is 0.01. If LABELONLY is true, \
no events are removed, and the "unexpectedness" attribute is set \
on all events. The FIELD analyzed by default is "_raw". \
By default, NORMALIZE is true, which normalizes numerics. For cases \
where FIELD contains numeric data that should not be normalized, but \
treated as categories, set NORMALIZE=false. The \
BLACKLIST is a name of a csv file of events in \
$SPLUNK_HOME/var/run/splunk/<BLACKLIST>.csv, such that any incoming \
events that are similar to the blacklisted events are treated as \
not anomalous (i.e., uninteresting) and given an unexpectedness \
score of 0.0. Events that match blacklisted events with a \
similarity score above BLACKLISTTHRESHOLD (defaulting to 0.05) are \
marked as unexpected. The inclusion of a 'by' clause, allows the \
specification of a list of fields to segregate results for anomaly \
detection. For each combination of values for the specified \
field(s), events with those values are treated entirely separately. \
Therefore, 'anomalies by source' will look for anomalies in each \
source separately -- a pattern in one source will not affect that \
it is anomalous in another source.
maintainer = david
appears-in = 4.0
comment1 = Return only anomalous events.
example1 = ... | anomalies
comment2 = Show most interesting events first, ignoring any in the blacklist 'boringevents'.
example2 = ... | anomalies blacklist=boringevents | sort -unexpectedness
comment3 = Use with transactions to find regions of time that look unusual.
example3 = ... | transaction maxpause=2s | anomalies
usage = public
related = anomalousvalue, cluster, kmeans, outlier
tags = anomaly unusual odd irregular dangerous unexpected outlier
category = results::filter
##################
# anomalousvalue
##################
[anomalousvalue-command]
syntax = anomalousvalue <av-option>* <anovalue-action-option>? <anovalue-pthresh-option>? <field-list>?
shortdesc = Finds and summarizes irregular, or uncommon, search results.
description = Identifies or summarizes the values in the data that are anomalous either by frequency of occurrence \
or number of standard deviations from the mean. If a field-list is given, only those fields are \
considered. Otherwise all non internal fields are considered. \p\\
For fields that are considered anomalous, a new field is added with the following scheme. \
If the field is numeric, e.g. \"size\", the new field will be \"Anomaly_Score_Num(size)\". \
If the field is non-numeric, e.g. \"name\", the new field will be \"Anomaly_Score_Cat(name)\".
maintainer = steveyz
appears-in = 3.0
comment1 = Return only uncommon values.
example1 = ... | anomalousvalue
commentcheat = Return events with uncommon values.
examplecheat = ... | anomalousvalue action=filter pthresh=0.02
category = reporting
usage = public
related = af, anomalies, cluster, kmeans, outlier
tags = anomaly unusual odd irregular dangerous unexpected
[av-option]
syntax=(minsupcount=<int>)|(maxanofreq=<num>)|(minsupfreq=<num>)|(minnormfreq=<num>)
description = Parameters to the anomalousvalue command. \
minsupcount is the minimum number of rows that must contain a field in order to consider the field at all. \
maxanofreq is the maximum frequency (as a decimal) for a value to be considered anomalous. \
minsupfreq is the minimum support frequency. A field must be in at least this fraction of overall events to be considered. \
minnormfreq is the minimum normal frequency. A field's values must be considered normal at least this fraction of times \
or else the field is not considered for determining if the event is anomalous
[anovalue-action-option]
syntax = action=(annotate|filter|summary)
description = If action is ANNOTATE, new fields will be added to the event containing anomalous values that \
indicate the anomaly scores of the values. \
If action is FILTER, events with anomalous value(s) are retained while non-anomalous values are dropped. \
If action is SUMMARY, a table summarizing the anomaly statistics for each field is generated.
default = "action=filter"
[anovalue-pthresh-option]
syntax = pthresh=<num>
description = Probability threshold (as a decimal) that has to be met for a value to be deemed anomalous
default = "pthresh=0.01"
##################
## anomalydetection
###################
[anomalydetection-command]
syntax = anomalydetection <anoma-method-option>? <anoma-action-option>? <anoma-pthresh-option>? <anoma-cutoff-option>? <field-list>?
shortdesc = Find anomalous events in a set of search results.
description = Identify anomalous events by computing a probability for each event and then detecting unusually small probabilities. \
The probability is defined as the product of the frequencies of each individual field value in the event. \
For categorical fields, the frequency of a value X is the number of times X occurs divided by the total number of events. \
For numerical fields, we first build a histogram for all the values, then compute the frequency of a value X \
as the size of the bin that contains X divided by the number of events. \
Missing values are treated by adding a special value and updating its count just like a normal value. \
Histograms are built using the standard Scott's rule to determine the bin width.
maintainer = nnguyen
appears-in = Ember
comment = The way probabilities are computed is called the Naive Bayes method, which means the individual fields are considered independent. \
This is a simplification to make the command reasonably fast.
example1 = ... | anomalydetection
comment1 = Return only anomalous events.
example2 = ... | anomalydetection action=summary
comment2 = Return a short summary of how many anomalous events are there and some other statistics such as the threshold value used to detect them.
category = streaming, reporting
usage = public
related = anomalies, anomalousvalue, outlier, cluster, kmeans
tags = anomaly unusual odd irregular dangerous unexpected Bayes
[anoma-method-option]
syntax = method=(histogram|zscore|iqr)
description = There are three methods instead of one because we've combined two older commands, anomalousvalue and outlier, with the new method. \
The new method is invoked by choosing the 'histogram' option. The anomalousvalue command is invoked by choosing 'zscore', \
and the outlier command is invoked by choosing 'iqr'. \
Below we will describe other options associated with the histogram method. For the other two methods, the associated options are \
exactly the same as before. That is, the queries '...| anomalousvalue ...' and '...| anomalydetection method=zscore ...' where the '...' are \
exactly the same in the two queries will produce exactly the same outputs. The same scheme applies to outlier.
example1 = ... | anomalydetection method=zscore action=filter pthresh=0.05
comment1 = This query returns the same output as that returned by the query '... | anomalousvalue action=filter pthresh=0.05'.
example2 = ... | anomalydetection method=iqr action=tf param=4 uselower=true mark=true
comment2 = This query returns the same output as that returned by the query '...| outlier action=tf param=4 uselower=true mark=true'.
default = "method=histogram"
[anoma-action-option]
syntax = action=(filter|annotate|summary|transform|tf|remove|rm)
description = If the method is zscore or iqr, then the actions have the same meaning as in the anomalousvalue and outlier commands. \
If the method is histogram, then the meanings are:\
If action is FILTER, anomalous events are retained while others are dropped. \
If action is ANNOTATE, new fields will be added to anomalous events that indicate the probability of the event as well as which field \
may be the cause of the anomaly. \
If the action is SUMMARY, a table summarizing the anomaly statistics for the search results is generated.
default = "action=filter"
comment = This is the default action when method=histogram. If method is zscore or iqr, then the default action is the default ones for those commands, \
i.e., 'filter' for zscore and 'transform' for iqr
[anoma-pthresh-option]
syntax = pthresh=<num>
description = First, this option only applies when the method is either histogram or zscore. An invalid argument error will be returned if the method is iqr. \
In the histogram case, it means the probability (as a decimal) that has to be met for an event to be deemed anomalous. \
In the zscore case, it means the same as in the annomalousvalue command.
default = If the method is zscore, then the default is 0.01 (as in the anomalousvalue command). If the method is histogram, the default is not any fixed value \
but is instead calculated for each data set during the analysis.
[anoma-cutoff-option]
syntax = cutoff=<bool>
description = This option applies to histogram method only. If the cutoff is false, the algorithm uses the formula threshold = 1st-quartile - 1.5*iqr without \
modification. If the cutoff is true, the algorithm modifies the above formula in order to come up with a smaller number of anomalies.
default = "cutoff=true"
##################
# append
##################
[append-command]
syntax = append (<subsearch-options>)? <subsearch>
shortdesc = Appends the results of a subsearch results to the current results.
description = Append the results of a subsearch as additional results at the end of the current results.
comment = Append the current results with the tabular results of errors.
example = ... | chart count by category1 | append [search error | chart count by category2]
maintainer = steveyz
appears-in = 4.0
usage = public
tags = append join combine unite combine
category = results::append
related = appendcols, join, set
[subsearch-options]
syntax = (extendtimerange=<bool>)? (maxtime=<int>)? (maxout=<int>)? (timeout=<int>)?
description = You can specify one or more of these options.\
The extendtimerange option specifies whether to include the subsearch time range \
in the time range for the entire search. The default is false. \
The maxtime option specifies the maximum number of seconds to run the subsearch before finalizing. \
The maxout option specifies the maximum number of results to return from the subsearch. \
The timeout option specifies the maximum amount of time, in seconds, to cache the subsearch results.
##################
# appendcols
##################
[appendcols-command]
syntax = appendcols (override=<bool> | <subsearch-options>)? <subsearch>
shortdesc = Appends the fields of the subsearch results to current results, first results to first result, second to second, etc.
description = Appends fields of the results of the subsearch into input search results by combining the external fields of the subsearch (fields that do not start with '_') into the current results. The first subsearch result is merged with the first main result, the second with the second, and so on. If option override is false (default), if a field is present in both a subsearch result and the main result, the main result is used. If it is true, the subsearch result's value for that field is used.
comment = Search for "404" events and append the fields in each event to the previous search results.
example = ... | appendcols [search 404]
maintainer = steveyz
appears-in = 4.0
usage = public
tags = append join combine unite
category = fields::add
related = append, join, set
##################
# appendpipe
##################
[appendpipe-command]
syntax = appendpipe (run_in_preview=<bool>)? [<subpipeline>]
description = Appends the result of the subpipeline applied to the current result set to results.
comment = Append subtotals for each action across all users
example = index=_audit | stats count by action user | appendpipe [stats sum(count) as count by action | eval user = "ALL USERS"] | sort action
maintainer = steveyz
appears-in = 4.2
usage = public
tags = append join combine unite combine
category = results::append
related = append appendcols join set
##################
# arules
##################
[arules-command]
syntax = arules (<arules-option> )* <fields>
shortdesc = Finds the association rules between field values.
description = Finding association rules between values. This is the algorithm behind most online \
shopping websites. When a customer buys an item, these sites are able to recommend \
related items that other customers also buy when they buy the first one. Arules finds such relationships and not only for \
shopping items but any kinds of fields. Note that stricly speaking, arules does not find relationships between fields, but rather \
between the values of the fields.
maintainer = nnguyen
appears-in = 6.0
usage = public
example1 = ... | arules field1 field2 field3
comment1 = Running arules with default support (=3) and confidence (=.5) \
The minimum number of fields is 2. There is no maximum restriction.
example2 = ... | arules sup=3 conf=.6 field1 field2 field3
comment2 = The sup option must be a positive integer \
The conf option must be a float between 0 and 1 \
In general, the higher the support, the less noisy the output will be. However, setting the support too high may exclude too much \
useful data in some circumstances. The conf option should be at least 0.5, otherwise the associations will not be significant. The higher \
the conf, the more significant the associations will be, but at the expense of retaining less associations.
category = streaming, reporting
related = associate, correlate
tags = associate contingency correlate correspond dependence independence
[arules-option]
syntax = (sup=<int>)|(conf=<num>)
default = ("sup=3" | "conf=.5")
description = The sup option specifies a required level of support, or computed level of association between fields. Support is expressed as the output Support and Implied Support fields. The conf option specifies a measure of how certain the algorithm is about that association. Confidence is expressed as the output Strength field. (For example a small number of datapoints that are entirely equivalent would have high support but low confidence.) For either option, associations which are below the limits will not be included in output results.
##################
# associate
##################
[associate-command]
syntax = associate (<associate-option> )* <field-list>?
shortdesc = Identifies correlations between fields.
description = Searches for relationships between pairs of fields. More specifically, this command tries to identify \
cases where the entropy of field1 decreases significantly based on the condition of field2=value2. \
field1 is known as the target key and field2 the reference key and value2 the reference value. \
If a list of fields is provided, analysis will be restrict to only those fields. By default all fields \
are used.
usage = public
maintainer = steveyz
appears-in = 3.0
comment1 = Analyze all fields to find a relationship.
example1 = ... | associate
comment2 = Analyze all events from host "reports" and return results associated with each other.
example2 = host="reports" | associate supcnt=50 supfreq=0.2 improv=0.5
commentcheat = Return results associated with each other (that have at least 3 references to each other).
examplecheat = ... | associate supcnt=3
category = reporting
related = correlate, contingency
tags = associate contingency correlate connect link correspond dependence independence
[associate-option]
syntax = <associate-supcnt-option>|<associate-supfreq-option>|<associate-improv-option>
description = Associate command options
[associate-supcnt-option]
syntax = supcnt=<int>
description = Minimum number of times the reference key=reference value combination must be appear. \
Must be a non-negative integer.
default = "supcnt=100"
[associate-supfreq-option]
syntax = supfreq=<num>
description = Minimum frequency of reference key=reference value combination, as a fraction of the number of total events.
default = "supfreq=0.1"
[associate-improv-option]
syntax = improv=<num>
description = Minimum entropy improvement for target key. That is, \
entropy(target key) - entropy(target key given reference key/value) \
must be greater than or equal to this.
default = "improv=0.5"
##################
# audit
##################
[audit-command]
syntax = audit
shortdesc = Returns audit trail information that is stored in the local audit index.
description = View audit trail information stored in the local "audit" index. Also validate signed audit events while checking for gaps and tampering.
commentcheat = View information in the "audit" index.
examplecheat = index=audit | audit
category = administrative
usage = public
appears-in = 3.2
maintainer = rdas
tags = audit trail security
related = metadata
##################
# autoregress
##################
[autoregress-command]
syntax = autoregress <field> (AS <field:newfield>)? (p=<int:p_start>("-"<int:p_end>)?)?
shortdesc = Prepares events or results for calculating the moving average.
description = Sets up data for auto-regression (e.g. moving average) by copying one or more of the previous values for <field> into each event. If <newfield> is provided, one prior value will be copied into <newfield> from a count of 'p' events prior. In this case, 'p' must be a single integer. If <newfield> is not provided, the single or multiple values will be copied into fields named '<field>_p<p-val>'. In this case 'p' may be a single integer, or a range <p_start>-<p_end>. For a range, the values will be copied from 'p_start' events prior to 'p_end' events prior. If 'p' option is unspecified, it defaults to 1 (i.e., copy only the previous one value of <field> into <field>_p1. The first few events will lack previous values, since they do not exist.
comment1 = Calculate a moving average of event size; the first N average numbers are omitted by eval since summing null fields results in null.
example1 = ... | eval rawlen=len(_raw) | autoregress rawlen p=1-4 | eval moving_average = (rawlen + rawlen_p1 + rawlen_p2 + rawlen_p3 + rawlen_p4) / 5
comment2 = For each event, copy the 2nd, 3rd, 4th, and 5th previous values of the 'count' field into the respective fields 'count_p2', 'count_p3', 'count_p4', and 'count_p5'.
example2 = ... | autoregress count p=2-5
comment3 = For each event, copy the 3rd previous value of the 'foo' field into the field 'oldfoo'.
example3 = ... | autoregress foo AS oldfoo p=3
usage = public
appears-in = 4.0
maintainer = steveyz
tags = average mean
alias = ar
category = reporting
related = accum, delta, streamstats, trendline
##################
# bin
##################
[bin-command]
syntax = bin (<bin-options> )* <field> (as <field>)?
alias = bucket, discretize
shortdesc = Puts continuous numerical values into discrete sets.
description = Puts continuous numerical field values into discrete sets, or bins. Adjusts the value of 'field', so that all items in the set have the same value for 'field'. Note: Bin is called by chart and timechart automatically and is only needed for statistical operations that timechart and chart cannot process.
usage = public
commentcheat1 = Separate search results into 10 bins, and return the count of raw events for each bin.
examplecheat1 = ... | bin size bins=10 | stats count(_raw) by size
commentcheat2 = Return the average "thruput" of each "host" for each 5 minute time span.
examplecheat2 = ... | bin _time span=5m | stats avg(thruput) by _time host
category = reporting
appears-in = 3.0
maintainer = steveyz
related = chart, timechart
tags = bucket band bracket bin round chunk lump span
[bin-options]
syntax = (<bin-bins> <bin-minspan>?)|<bin-span>|<bin-start-end>|<bin-aligntime>
description = Discretization options.
[bin-minspan]
syntax = minspan=(<span-length>)
description = Specifies the smallest span granularity to use automatically inferring span from the data time range.
[bin-bins]
syntax = bins=<int>
description = Sets the maximum number of bins to discretize into. \
Given this upper-bound guidance, the bins will snap to \
human sensible bounds.
note = The actual number of bins will almost certainly be smaller than the given number.
example1 = bins=10
[bin-span]
syntax = span=(<span-length>|<log-span>)
description = Sets the size of each bin.
comment1 = set span to 10 seconds
example1 = span=10
comment2 = set span to 2 days
example2 = span=2d
comment3 = set span to 5 minutes
example3 = span=5m
[bin-aligntime]
syntax = aligntime=(earliest|latest|<time-specifier>)
description = Align the bin times to something other than base UTC time (epoch 0). \
Only valid when doing a time based discretization. \
Ignored if span is in days/months/years.
example1 = aligntime=earliest
description = Align time bins to the earliest time of the search.
example2 = aligntime=@d+3h
description = Align time bins to 3am (local time). If span=12h, this means the bins \
will represent 3am - 3pm, then 3pm - 3am (next day), etc.
example3 = aligntime=1500567890
description = Align to the specific UTC time of 1500567890.
[log-span]
syntax = (<num>)?log(<num>)?
description = Sets to log based span, first number if coefficient, second number is base \
coefficient, if supplied, must be real number >= 1.0 and < base \
base, if supplied, must be real number > 1.0 (strictly greater than 1)
example1 = log
comment1 = set log span of base 10, coeff 1.0, e.g. ...,0.1,1,10,100,...
example2 = 2log5
comment2 = set log span of base 5, coeff 2.0, e.g. ...,0.4,2,10,50,250,1250,...
[bin-start-end]
syntax = (start|end)=<num>
description = Sets the minimum and maximum extents for numerical bins.
note = Data outside of the [start, end] range is discarded.
[span-length]
syntax = <int>(<timescale>)?
description = Span of each bin. \
If using a timescale, this is used as a time range.\
If not, this is an absolute bin "length."
comment1 = set span to 10 seconds
example1 = 10
comment2 = set span to 2 days
example2 = 2d
comment3 = set span to 5 minutes
example3 = 5m
[timescale]
syntax = <ts-sec>|<ts-min>|<ts-hr>|<ts-day>|<ts-month>|<ts-subseconds>
description = Time scale units.
[ts-subseconds]
syntax = us|ms|cs|ds
description = Time scale in microseconds("us"), milliseconds("ms"), \
centiseconds("cs"), or deciseconds("ds")
[ts-sec]
syntax = s|sec|secs|second|seconds
simplesyntax = seconds
description = Time scale in seconds.
[ts-min]
syntax = m|min|mins|minute|minutes
simplesyntax = minutes
description = Time scale in minutes.
[ts-hr]
syntax = h|hr|hrs|hour|hours
simplesyntax = hours
description = Time scale in hours.
[ts-day]
syntax = d|day|days
simplesyntax = days
description = Time scale in days.
[ts-month]
syntax = mon|month|months
simplesyntax = months
description = Time scale in months.
##################
# bucketdir
##################
[bucketdir-command]
syntax = bucketdir pathfield=<field> sizefield=<field> (maxcount=<int>)? (countfield=<field>)? (sep=<char>)?
shortdesc = Replaces PATHFIELD with higher-level grouping, such as replacing filenames with directories.
description = Returns at most MAXCOUNT events by taking the incoming events and rolling up multiple sources into directories, by preferring directories that have many files but few events. The field with the path is PATHFIELD (e.g., source), and strings are broken up by a SEP character. The default pathfield=source; sizefield=totalCount; maxcount=20; countfield=totalCount; sep="/" or "\\", depending on the os.
maintainer = david
usage = public
appears-in = 4.0
comment1 = get 10 best sources and directories
example1 = ... | top source|bucketdir pathfield=source sizefield=count maxcount=10
category = results::group
tags = cluster group collect gather
related = cluster dedup
##################
# chart
##################
[chart-command]
simplesyntax = chart (agg=<stats-agg-term>)? ( <stats-agg-term> | ( "(" <eval-expression> ")" ) )+ ( BY <field> (<bin-options> )* (<split-by-clause>)? )? | ( OVER <field> (<bin-options>)* (BY <split-by-clause>)? )? (<dedup_splitvals>)?
syntax = chart <chart-command-arguments>
shortdesc = Returns results in a tabular output for charting.
description = Creates a table of statistics suitable for charting. Whereas timechart generates a \
chart with _time as the x-axis, chart lets you select an arbitrary field as the \
x-axis with the "by" or "over" keyword. If necessary, the x-axis field is converted \
to discrete numerical quantities.\p\\
When chart includes a split-by-clause, the columns in the output table represents a \
distinct value of the split-by-field. (With stats, each row represents a single \
unique combination of values of the group-by-field. The table displays ten columns \
by default, but you can specify a where clause to adjust the number of columns.\p\\
When a where clause is not provided, you can use limit and agg options to specify \
series filtering. If limit=0, there is no series filtering. \p\\
When specifying multiple data series with a split-by-clause, you can use sep and \
format options to construct output field names.
commentcheat1 = Return the average (mean) "size" for each distinct "host".
examplecheat1 = ... | chart avg(size) by host
commentcheat2 = Return the the maximum "delay" by "size", where "size" is broken down into a maximum of 10 equal sized buckets.
examplecheat2 = ... | chart max(delay) by size bins=10
commentcheat3 = Return the ratio of the average (mean) "size" to the maximum "delay" for each distinct "host" and "user" pair.
examplecheat3 = ... | chart eval(avg(size)/max(delay)) by host user
commentcheat4 = Return max(delay) for each value of foo split by the value of bar.
examplecheat4 = ... | chart max(delay) over foo by bar
commentcheat5 = Return max(delay) for each value of foo.
examplecheat5 = ... | chart max(delay) over foo
category = reporting
appears-in = 3.0
usage = public
maintainer = steveyz
supports-multivalue = true
related = timechart, bucket, sichart
tags = chart graph report sparkline count dc mean avg stdev var min max mode median
[chart-command-arguments]
syntax = (sep=<string>)? (format=<string>)? (cont=<bool>)? (limit=<int>)? (agg=<stats-agg-term>)? ( <stats-agg-term> | <sparkline-agg-term> | ( "(" <eval-expression> ")" ) )+ ( BY <field> (<bin-options> )* (<split-by-clause>)? )? | ( OVER <field> (<bin-options>)* (BY <split-by-clause>)? )? (<dedup_splitvals>)?
description = See chart-command description. See the bin command for details about the <bin-options>.
##################
# cofilter
##################
[cofilter-command]
syntax = cofilter field1 field2
shortdesc = Find how many times field1 and field2 values occurred together.
description = For this command, we think of field1 values as "users" and field2 values as "items". \
The goal of the command is to compute, for each pair of item (i.e., field2 values), how many \
users (i.e., field1 values) used them both (i.e., occurred with each of them).
maintainer = nnguyen
appears-in = 6.0
usage = public
example1 = ... | cofilter field1 field2
comment1 = user field must be specified first and item field second
category = streaming, reporting
related = associate, correlate
tags = arules associate contingency correlate correspond dependence independence
##################
# collapse
##################
[collapse-command]
syntax = collapse (chunksize=<num>)? (force=<bool>)?
description = Purely internal operation that condenses multi-file results into as few files as chunksize option will allow. (default chunksize=50000). Operation automatically invoked by output* operators. If force=true and the results are entirely in memory, re-divide the results into appropriated chunked files (this option is new for 5.0).
maintainer = steveyz
example1 = ... | collapse
usage = internal
appears-in = 3.2
optout-in = lite, lite_free
##################
# concurrency
##################
[concurrency-command]
syntax = concurrency duration=<field> (start=<field>)? (output=<field>)?
shortdesc = Given a duration field, finds the number of "concurrent" events for each event.
description = If each event represents something that occurs over a span of time, where that \
span is specified in the duration field, calculate the number of concurrent events \
for each event start time. An event X is concurrent with event Y if \
the X start time, X.start, lies between Y.start and (Y.start + Y.duration). \
In other words, the concurrent set of events is calculated for each event start time, \
and that number is attached to the event. \
The units of start and duration are assumed to be the same. If you have different \
units, you will need to convert them to corresponding units prior to using the concurrency \
command. \
Unless specified, the start field is assumed to be _time and the output field will \
be 'concurrency' \
Limits: If concurrency exceeds limits.conf [concurrency] max_count \
(Defaults to 10 million), results will not be accurate.
maintainer = steveyz
appears-in = 4.1
usage = public
comment1 = Calculate the number of concurrent events for each event start time and emit as field 'foo'
example1 = ... | concurrency duration=total_time output=foo
commentcheat = Calculate the number of concurrent events using the 'et' field as the start time \
and 'length' as the duration.
examplecheat = ... | concurrency duration=length start=et
comment2 = Calculate the number of ongoing http requests at the start time of each http request in a splunk access log
example2 = ... | eval spent_in_seconds = spent / 1000 | concurrency duration=spent_in_seconds
category = reporting
related = timechart
tags = concurrency
##################
# contingency
##################
[contingency-command]
syntax = contingency (<contingency-option> )* <field> <field>
alias = counttable, ctable
shortdesc = Builds a contingency table for two fields.
description = In statistics, contingency tables are used to record \
and analyze the relationship between two or more (usually categorical) variables. Many metrics of \
association or independence can be calculated based on contingency tables, such as the phi \
coefficient or the V of Cramer.
maintainer = steveyz
appears-in = 3.0
usage = public
comment1 = Build a contingency table for fields "host" and "sourcetype".
example1 = ... | contingency host sourcetype
commentcheat = Build a contingency table of "datafields" from all events.
examplecheat = ... | contingency datafield1 datafield2 maxrows=5 maxcols=5 usetotal=F
category = reporting
related = associate, correlate
tags = associate contingency correlate connect link correspond dependence independence
[contingency-option]
syntax = <contingency-maxopts>|<contingency-mincover>|<contingency-usetotal>|<contingency-totalstr>
description = Options for the contingency table
[contingency-maxopts]
syntax = (maxrows|maxcols)=<int>
description = Maximum number of rows or columns. If the number of distinct values of the field exceeds this maximum, \
the least common values will be ignored. There is a ceiling on the values permitted for maxrows and maxcols\
from limits.conf, [ctable] stanza maxvalues. This limit defaults to 1000. Values for over this will be rejected \
and values of 0 for these settings mean to use this maxvalues setting.
default = ("maxrows=0" | "maxcols=0")
[contingency-mincover]
syntax = (mincolcover|minrowcover)=<num>
description = Cover only this percentage of values for the row or column field. If the number of entries needed to \
cover the required percentage of values exceeds maxrows or maxcols, maxrows or maxcols takes precedence.
default = ("mincolcover=1.0" | "minrowcover=1.0")
[contingency-usetotal]
syntax = usetotal=<bool>
description = Add row and column totals
default = "usetotal=true"
[contingency-totalstr]
syntax = totalstr=<field>
description = Field name for the totals row/column
default = "totalstr=TOTAL"
##################
# convert
##################
[convert-command]
simplesyntax = convert (timeformat=<string>)? ( (auto|dur2sec|mstime|memk|none|num|rmunit|rmcomma|ctime|mktime) "(" <field>? ")" (as <field>)?)+
syntax = convert (timeformat=<string>)? (<convert-function> (as <wc-field>)?)+
shortdesc = Converts field values into numerical values.
description = Converts the values of fields into numerical values. When renaming a field using "as", the original field is left intact. The timeformat option is used by ctime and mktime conversions. Default = "%m/%d/%Y %H:%M:%S".
maintainer = steveyz
appears-in = 3.0
commentcheat1 = Convert every field value to a number value except for values in the field "foo" (use the "none" argument to specify fields to ignore).
examplecheat1 = ... | convert auto(*) none(foo)
commentcheat2 = Change all memory values in the "virt" field to Kilobytes.
examplecheat2 = ... | convert memk(virt)
commentcheat3 = Change the sendmail syslog duration format (D+HH:MM:SS) to seconds. For example, if "delay="00:10:15"", the resulting value will be "delay="615"".
examplecheat3 = ... | convert dur2sec(delay)
commentcheat4 = Convert values of the "duration" field into number value by removing string values in the field value. For example, if "duration="212 sec"", the resulting value will be "duration="212"".
examplecheat4 = ... | convert rmunit(duration)
category = fields::convert
usage = public
tags = interchange transform translate convert ctime mktime dur2sec mstime memk
related = eval
[convert-function]
syntax = <convert-auto>|<convert-dur2sec>|<convert-mstime>|<convert-memk>|<convert-none>|<convert-num>|<convert-rmunit>|<convert-rmcomma>|<convert-ctime>|<convert-mktime>
[convert-auto]
syntax = auto("(" (<wc-field>)? ")")?
description = Automatically convert the field(s) to a number using the best conversion. \
Note that if not all values of a particular field can be converted using a known conversion type, \
the field is left untouched and no conversion at all in done for that field.
example1 = ... | convert auto(*)
example2 = ... | convert auto
example3 = ... | convert auto()
example4 = ... | convert auto(delay) auto(xdelay)
example5 = ... | convert auto(delay) as delay_secs
example6 = ... | convert auto(*delay) as *delay_secs
example7 = ... | convert auto(*) as *_num
[convert-ctime]
syntax = ctime"("<wc-field>?")"
description = Convert an epoch time to an ascii human readable time. Use timeformat option to specify exact format to convert to.
example1 = ... | convert timeformat="%H:%M:%S" ctime(_time) as timestr
[convert-mktime]
syntax = mktime"("<wc-field>?")"
description = Convert an human readable time string to an epoch time. Use timeformat option to specify exact format to convert from.
example1 = ... | convert mktime(timestr)
[convert-dur2sec]
syntax = dur2sec"("<wc-field>?")"
description = Convert a duration format "[D+]HH:MM:SS" to seconds.
example1 = ... | convert dur2sec(xdelay)
example2 = ... | convert dur2sec(*delay)
[convert-mstime]
syntax = mstime"(" <wc-field>? ")"