From 2b92da596631ff1a29b7deac05e00faaad9305f3 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 2 Oct 2023 10:28:44 +0200 Subject: [PATCH] Remove Zeek-specific documentation. This now lives Zeek-side. This keeps the section structure so that we make things easier to find, and put in pointers to the new Zeek-side documentation It leaves everything in the development section as is; hard to piece apart and seems fine to leave it here. --- doc/Makefile | 13 - doc/_static/record-spicy-batch.zeek | 1 - .../zeek => _static}/tftp-no-accept.spicy | 0 doc/autogen/zeek/init-bare.zeek | 38 - doc/autogen/zeek/init-framework.zeek | 81 -- doc/autogen/zeek/record-spicy-batch.zeek | 90 -- doc/autogen/zeek/tftp.evt | 16 - doc/autogen/zeek/tftp.spicy | 92 -- doc/autogen/zeek/tftp.zeek | 165 --- doc/autogen/zeek/zeek-functions.spicy | 219 ---- doc/conf.py | 4 +- doc/development/testing.rst | 2 +- doc/examples/my-http.zeek | 4 - doc/faq.rst | 82 +- doc/getting-started.rst | 122 +-- doc/index.rst | 5 +- doc/installation.rst | 4 +- doc/programming/language/conditional.rst | 7 +- doc/programming/parsing.rst | 2 +- doc/scripts/autogen-zeek-docs | 51 - doc/toolchain.rst | 11 +- .../examples/tftp-schedule-analyzer.zeek | 37 - .../tftp-single-request-more-args.zeek | 4 - .../examples/tftp-single-request.zeek | 4 - doc/tutorial/examples/tftp-two-requests.zeek | 9 - doc/tutorial/examples/tftp.zeek | 1 - doc/tutorial/index.rst | 415 +------- doc/zeek.rst | 975 +----------------- 28 files changed, 35 insertions(+), 2419 deletions(-) delete mode 120000 doc/_static/record-spicy-batch.zeek rename doc/{autogen/zeek => _static}/tftp-no-accept.spicy (100%) delete mode 100644 doc/autogen/zeek/init-bare.zeek delete mode 100644 doc/autogen/zeek/init-framework.zeek delete mode 100644 doc/autogen/zeek/record-spicy-batch.zeek delete mode 100644 doc/autogen/zeek/tftp.evt delete mode 100644 doc/autogen/zeek/tftp.spicy delete mode 100644 doc/autogen/zeek/tftp.zeek delete mode 100644 doc/autogen/zeek/zeek-functions.spicy delete mode 100644 doc/examples/my-http.zeek delete mode 100755 doc/scripts/autogen-zeek-docs delete mode 100644 doc/tutorial/examples/tftp-schedule-analyzer.zeek delete mode 100644 doc/tutorial/examples/tftp-single-request-more-args.zeek delete mode 100644 doc/tutorial/examples/tftp-single-request.zeek delete mode 100644 doc/tutorial/examples/tftp-two-requests.zeek delete mode 120000 doc/tutorial/examples/tftp.zeek diff --git a/doc/Makefile b/doc/Makefile index ae3f77dc5..4445b52e8 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -55,18 +55,5 @@ doxygen: check: @$(SPHINXBUILD) -q -b linkcheck $(SOURCEDIR) $(DESTDIR)/linkcheck - spicy-%: git clone https://github.com/zeek/$@ -zeek: - git clone https://github.com/zeek/zeek --depth 1 - -check-zeek-docs: spicy-tftp zeek - @echo Refreshing checkouts - @for REPO in $^; do (cd $$REPO && git pull && git reset HEAD --hard)>/dev/null; done - @ - @echo Checking whether docs for Zeek integration are up-to-date - @./scripts/autogen-zeek-docs zeek spicy-tftp - @ - @git diff --quiet autogen/zeek \ - || (echo "Zeek docs are not up-to-date, rerun './scripts/autogen-zeek-docs'." && exit 1) diff --git a/doc/_static/record-spicy-batch.zeek b/doc/_static/record-spicy-batch.zeek deleted file mode 120000 index 982b4ef70..000000000 --- a/doc/_static/record-spicy-batch.zeek +++ /dev/null @@ -1 +0,0 @@ -../autogen/zeek/record-spicy-batch.zeek \ No newline at end of file diff --git a/doc/autogen/zeek/tftp-no-accept.spicy b/doc/_static/tftp-no-accept.spicy similarity index 100% rename from doc/autogen/zeek/tftp-no-accept.spicy rename to doc/_static/tftp-no-accept.spicy diff --git a/doc/autogen/zeek/init-bare.zeek b/doc/autogen/zeek/init-bare.zeek deleted file mode 100644 index a9f9d2dbd..000000000 --- a/doc/autogen/zeek/init-bare.zeek +++ /dev/null @@ -1,38 +0,0 @@ - -module Spicy; - -export { -# doc-options-start - ## Constant for testing if Spicy is available. - const available = T; - - ## Show output of Spicy print statements. - const enable_print = F &redef; - - # Record and display profiling information, if compiled into analyzer. - const enable_profiling = F &redef; - - ## abort() instead of throwing HILTI exceptions. - const abort_on_exceptions = F &redef; - - ## Include backtraces when reporting unhandled exceptions. - const show_backtraces = F &redef; - - ## Maximum depth of recursive file analysis (Spicy analyzers only) - const max_file_depth: count = 5 &redef; -# doc-options-end - -# doc-types-start - ## Result type for `Spicy::resource_usage()`. The values reflect resource - ## usage as reported by the Spicy runtime system. - type ResourceUsage: record { - user_time : interval; ##< user CPU time of the Zeek process - system_time :interval; ##< system CPU time of the Zeek process - memory_heap : count; ##< memory allocated on the heap by the Zeek process - num_fibers : count; ##< number of fibers currently in use - max_fibers: count; ##< maximum number of fibers ever in use - max_fiber_stack_size: count; ##< maximum fiber stack size ever in use - cached_fibers: count; ##< number of fibers currently cached - }; -# doc-types-end -} diff --git a/doc/autogen/zeek/init-framework.zeek b/doc/autogen/zeek/init-framework.zeek deleted file mode 100644 index 886b0f417..000000000 --- a/doc/autogen/zeek/init-framework.zeek +++ /dev/null @@ -1,81 +0,0 @@ -@load base/misc/version - -# doc-common-start -module Spicy; - -export { -# doc-functions-start - ## Enable a specific Spicy protocol analyzer if not already active. If this - ## analyzer replaces an standard analyzer, that one will automatically be - ## disabled. - ## - ## tag: analyzer to toggle - ## - ## Returns: true if the operation succeeded - global enable_protocol_analyzer: function(tag: Analyzer::Tag) : bool; - - ## Disable a specific Spicy protocol analyzer if not already inactive. If - ## this analyzer replaces an standard analyzer, that one will automatically - ## be re-enabled. - ## - ## tag: analyzer to toggle - ## - ## Returns: true if the operation succeeded - global disable_protocol_analyzer: function(tag: Analyzer::Tag) : bool; - - - ## Enable a specific Spicy file analyzer if not already active. If this - ## analyzer replaces an standard analyzer, that one will automatically be - ## disabled. - ## - ## tag: analyzer to toggle - ## - ## Returns: true if the operation succeeded - global enable_file_analyzer: function(tag: Files::Tag) : bool; - - ## Disable a specific Spicy file analyzer if not already inactive. If - ## this analyzer replaces an standard analyzer, that one will automatically - ## be re-enabled. - ## - ## tag: analyzer to toggle - ## - ## Returns: true if the operation succeeded - global disable_file_analyzer: function(tag: Files::Tag) : bool; - - ## Returns current resource usage as reported by the Spicy runtime system. - global resource_usage: function() : ResourceUsage; -# doc-functions-end -} - -# Marked with &is_used to suppress complaints when there aren't any -# Spicy file analyzers loaded, and hence this event can't be generated. -# The attribute is only supported for Zeek 5.0 and higher. -event spicy_analyzer_for_mime_type(a: Files::Tag, mt: string) &is_used - { - Files::register_for_mime_type(a, mt); - } - -function enable_protocol_analyzer(tag: Analyzer::Tag) : bool - { - return Spicy::__toggle_analyzer(tag, T); - } - -function disable_protocol_analyzer(tag: Analyzer::Tag) : bool - { - return Spicy::__toggle_analyzer(tag, F); - } - -function enable_file_analyzer(tag: Files::Tag) : bool - { - return Spicy::__toggle_analyzer(tag, T); - } - -function disable_file_analyzer(tag: Files::Tag) : bool - { - return Spicy::__toggle_analyzer(tag, F); - } - -function resource_usage() : ResourceUsage - { - return Spicy::__resource_usage(); - } diff --git a/doc/autogen/zeek/record-spicy-batch.zeek b/doc/autogen/zeek/record-spicy-batch.zeek deleted file mode 100644 index ddea43205..000000000 --- a/doc/autogen/zeek/record-spicy-batch.zeek +++ /dev/null @@ -1,90 +0,0 @@ -##! Saves all input traffic in Spicy's batch format. - -module SpicyBatch; - -export { - const filename = "batch.dat" &redef; -} - -redef tcp_content_deliver_all_orig=T; -redef tcp_content_deliver_all_resp=T; -redef udp_content_deliver_all_orig=T; -redef udp_content_deliver_all_resp=T; - -global output: file; -global conns: set[conn_id]; -global num_conns = 0; - -function id(c: connection) : string - { - local cid = c$id; - local proto = "???"; - - if ( is_tcp_port(cid$orig_p) ) - proto = "tcp"; - else if ( is_udp_port(cid$orig_p) ) - proto = "udp"; - else if ( is_icmp_port(cid$orig_p) ) - proto = "icmp"; - - return fmt("%s-%d-%s-%d-%s", cid$orig_h, cid$orig_p, cid$resp_h, cid$resp_p, proto); - } - -function begin(c: connection, type_: string) - { - add conns[c$id]; - ++num_conns; - print fmt("tracking %s", c$id); - - local id_ = id(c); - print output, fmt("@begin-conn %s %s %s-orig %s%%orig %s-resp %s%%resp\n", id_, type_, id_, c$id$resp_p, id_, c$id$resp_p); - } - -event zeek_init() - { - output = open(filename); - enable_raw_output(output); - print output, "!spicy-batch v2\n"; - } - -event new_connection_contents(c: connection) - { - begin(c, "stream"); - } - -event tcp_contents(c: connection, is_orig: bool, seq: count, contents: string) - { - print output, fmt("@data %s-%s %d\n", id(c), (is_orig ? "orig" : "resp"), |contents|); - print output, contents; - print output, "\n"; - } - -event content_gap(c: connection, is_orig: bool, seq: count, length: count) - { - print output, fmt("@gap %s-%s %d\n", id(c), (is_orig ? "orig" : "resp"), length); - } - -event udp_contents(c: connection, is_orig: bool, contents: string) - { - if ( c$id !in conns ) - begin(c, "block"); - - print output, fmt("@data %s-%s %d\n", id(c), (is_orig ? "orig" : "resp"), |contents|); - print output, contents; - print output, "\n"; - } - -event connection_state_remove(c: connection) - { - if ( c$id !in conns ) - return; - - print output, fmt("@end-conn %s\n", id(c)); - } - -event zeek_done() - { - close(output); - print fmt("recorded %d session%s total", num_conns, (num_conns > 1 ? "s" : "")); - print fmt("output in %s", filename); - } diff --git a/doc/autogen/zeek/tftp.evt b/doc/autogen/zeek/tftp.evt deleted file mode 100644 index 85575e076..000000000 --- a/doc/autogen/zeek/tftp.evt +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2021 by the Zeek Project. See LICENSE for details. -# -# Note: When line numbers change in this file, update the documentation that pulls it in. - -protocol analyzer spicy::TFTP over UDP: - parse with TFTP::Packet, - port 69/udp; - -import TFTP; - -on TFTP::Request if ( is_read ) -> event tftp::read_request($conn, $is_orig, self.filename, self.mode); -on TFTP::Request if ( ! is_read ) -> event tftp::write_request($conn, $is_orig, self.filename, self.mode); - -on TFTP::Data -> event tftp::data($conn, $is_orig, self.num, self.data); -on TFTP::Acknowledgement -> event tftp::ack($conn, $is_orig, self.num); -on TFTP::Error -> event tftp::error($conn, $is_orig, self.code, self.msg); diff --git a/doc/autogen/zeek/tftp.spicy b/doc/autogen/zeek/tftp.spicy deleted file mode 100644 index 7927c6e90..000000000 --- a/doc/autogen/zeek/tftp.spicy +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright (c) 2021 by the Zeek Project. See LICENSE for details. -# -# Trivial File Transfer Protocol -# -# Specs from https://tools.ietf.org/html/rfc1350 - -module TFTP; - -import spicy; - -# Common header for all messages: -# -# 2 bytes -# --------------- -# | TFTP Opcode | -# --------------- - -public type Packet = unit { # public top-level entry point for parsing - op: uint16 &convert=Opcode($$); - switch ( self.op ) { - Opcode::RRQ -> rrq: Request(True); - Opcode::WRQ -> wrq: Request(False); - Opcode::DATA -> data: Data; - Opcode::ACK -> ack: Acknowledgement; - Opcode::ERROR -> error: Error; - }; -}; - -# TFTP supports five types of packets [...]: -# -# opcode operation -# 1 Read request (RRQ) -# 2 Write request (WRQ) -# 3 Data (DATA) -# 4 Acknowledgment (ACK) -# 5 Error (ERROR) -type Opcode = enum { - RRQ = 0x01, - WRQ = 0x02, - DATA = 0x03, - ACK = 0x04, - ERROR = 0x05 -}; - -# Figure 5-1: RRQ/WRQ packet -# -# 2 bytes string 1 byte string 1 byte -# ------------------------------------------------ -# | Opcode | Filename | 0 | Mode | 0 | -# ------------------------------------------------ - -type Request = unit(is_read: bool) { - filename: bytes &until=b"\x00"; - mode: bytes &until=b"\x00"; - - on %done { spicy::accept_input(); } -}; - -# Figure 5-2: DATA packet -# -# 2 bytes 2 bytes n bytes -# ---------------------------------- -# | Opcode | Block # | Data | -# ---------------------------------- - -type Data = unit { - num: uint16; - data: bytes &eod; -}; - -# Figure 5-3: ACK packet -# -# 2 bytes 2 bytes -# --------------------- -# | Opcode | Block # | -# --------------------- - -type Acknowledgement = unit { - num: uint16; -}; - -# Figure 5-4: ERROR packet -# -# 2 bytes 2 bytes string 1 byte -# ----------------------------------------- -# | Opcode | ErrorCode | ErrMsg | 0 | -# ----------------------------------------- - -type Error = unit { - code: uint16; - msg: bytes &until=b"\x00"; -}; diff --git a/doc/autogen/zeek/tftp.zeek b/doc/autogen/zeek/tftp.zeek deleted file mode 100644 index 6835fcbee..000000000 --- a/doc/autogen/zeek/tftp.zeek +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright (c) 2021 by the Zeek Project. See LICENSE for details. - -module TFTP; - -export { - redef enum Log::ID += { LOG }; - - type Info: record { - ## Timestamp for when the request happened. - ts: time &log; - ## Unique ID for the connection. - uid: string &log; - ## The connection's 4-tuple of endpoint addresses/ports. - id: conn_id &log; - ## True for write requests, False for read request. - wrq: bool &log; - ## File name of request. - fname: string &log; - ## Mode of request. - mode: string &log; - ## UID of data connection - uid_data: string &optional &log; - ## Number of bytes sent. - size: count &default=0 &log; - ## Highest block number sent. - block_sent: count &default=0 &log; - ## Highest block number ackknowledged. - block_acked: count &default=0 &log; - ## Any error code encountered. - error_code: count &optional &log; - ## Any error message encountered. - error_msg: string &optional &log; - - # Set to block number of final piece of data once received. - final_block: count &optional; - - # Set to true once logged. - done: bool &default=F; - }; - - ## Event that can be handled to access the TFTP logging record. - global log_tftp: event(rec: Info); -} - -# Maps a partial data connection ID to the request's Info record. -global expected_data_conns: table[addr, port, addr] of Info; - -redef record connection += { - tftp: Info &optional; -}; - -event zeek_init() &priority=5 - { - Log::create_stream(TFTP::LOG, [$columns = Info, $ev = log_tftp, $path="tftp"]); - } - -function log_pending(c: connection) - { - if ( ! c?$tftp || c$tftp$done ) - return; - - Log::write(TFTP::LOG, c$tftp); - c$tftp$done = T; - } - -function init_request(c: connection, is_orig: bool, fname: string, mode: string, is_read: bool) - { - log_pending(c); - - local info: Info; - info$ts = network_time(); - info$uid = c$uid; - info$id = c$id; - info$fname = fname; - info$mode = mode; - info$wrq = (! is_read); - c$tftp = info; - - # The data will come in from a different source port. - # - # TODO: Starting with Zeek 5.2, we will be able to use - # Analyzer::ANALYZER_SPICY_TFTP instead of get_tag(), - Analyzer::schedule_analyzer(c$id$resp_h, c$id$orig_h, c$id$orig_p, Analyzer::get_tag("Spicy_TFTP"), 1min); - expected_data_conns[c$id$resp_h, c$id$orig_p, c$id$orig_h] = info; - } - -event scheduled_analyzer_applied(c: connection, a: Analyzer::Tag) &priority=10 - { - local id = c$id; - if ( [c$id$orig_h, c$id$resp_p, c$id$resp_h] in expected_data_conns ) - { - c$tftp = expected_data_conns[c$id$orig_h, c$id$resp_p, c$id$resp_h]; - c$tftp$uid_data = c$uid; - add c$service["spicy_tftp_data"]; - } - } - -event tftp::read_request(c: connection, is_orig: bool, fname: string, mode: string) - { - init_request(c, is_orig, fname, mode, T); - } - -event tftp::write_request(c: connection, is_orig: bool, fname: string, mode: string) - { - init_request(c, is_orig, fname, mode, F); - } - -event tftp::data(c: connection, is_orig: bool, block_num: count, data: string) - { - if ( ! c?$tftp || c$tftp$done ) - return; - - local info = c$tftp; - - if ( block_num <= info$block_sent ) - # Duplicate (or previous gap; we don't track that) - return; - - info$size += |data|; - info$block_sent = block_num; - - if ( |data| < 512 ) - # Last block, per spec. - info$final_block = block_num; - } - -event tftp::ack(c: connection, is_orig: bool, block_num: count) - { - if ( ! c?$tftp || c$tftp$done ) - return; - - local info = c$tftp; - - info$block_acked = block_num; - - if ( block_num <= info$block_acked ) - # Duplicate (or previous gap, we don't track that) - return; - - info$block_acked = block_num; - - # If it's an ack for the last block, we're done. - if ( info?$final_block && info$final_block == block_num ) - log_pending(c); - } - -event tftp::error(c: connection, is_orig: bool, code: count, msg: string) - { - if ( ! c?$tftp || c$tftp$done ) - return; - - local info = c$tftp; - - info$error_code = code; - info$error_msg = msg; - log_pending(c); - } - -event connection_state_remove(c: connection) - { - if ( ! c?$tftp || c$tftp$done ) - return; - - log_pending(c); - } diff --git a/doc/autogen/zeek/zeek-functions.spicy b/doc/autogen/zeek/zeek-functions.spicy deleted file mode 100644 index 9ac138b4a..000000000 --- a/doc/autogen/zeek/zeek-functions.spicy +++ /dev/null @@ -1,219 +0,0 @@ -.. _spicy_confirm_protocol: - -.. rubric:: ``function zeek::confirm_protocol()`` - -[Deprecated] Triggers a DPD protocol confirmation for the current connection. - -This function has been deprecated and will be removed. Use ``spicy::accept_input`` -instead, which will have the same effect with Zeek. - -.. _spicy_reject_protocol: - -.. rubric:: ``function zeek::reject_protocol(reason: string)`` - -[Deprecated] Triggers a DPD protocol violation for the current connection. - -This function has been deprecated and will be removed. Use ``spicy::decline_input`` -instead, which will have the same effect with Zeek. - -.. _spicy_weird: - -.. rubric:: ``function zeek::weird(id: string, addl: string = "") : &cxxname="zeek::spicy::rt::weird";`` - -Reports a "weird" to Zeek. This should be used with similar semantics as in -Zeek: something quite unexpected happening at the protocol level, which however -does not prevent us from continuing to process the connection. - -id: the name of the weird, which (just like in Zeek) should be a *static* -string identifying the situation reported (e.g., ``unexpected_command``). - -addl: additional information to record along with the weird - -.. _spicy_is_orig: - -.. rubric:: ``function zeek::is_orig() : bool`` - -Returns true if we're currently parsing the originator side of a connection. - -.. _spicy_uid: - -.. rubric:: ``function zeek::uid() : string`` - -Returns the current connection's UID. - -.. _spicy_conn_id: - -.. rubric:: ``function zeek::conn_id() : tuple`` - -Returns the current connection's 4-tuple ID. - -.. _spicy_flip_roles: - -.. rubric:: ``function zeek::flip_roles()`` - -Instructs Zeek to flip the directionality of the current connection. - -.. _spicy_number_packets: - -.. rubric:: ``function zeek::number_packets() : uint64`` - -Returns the number of packets seen so far on the current side of the current connection. - -.. _spicy_protocol_begin: - -.. rubric:: ``function zeek::protocol_begin(analyzer: optional = Null)`` - -Adds a Zeek-side child protocol analyzer to the current connection. - -If the same analyzer was added previously with protocol_handle_get_or_create or -protocol_begin with same argument, and not closed with protocol_handle_close -or protocol_end, no new analyzer will be added. - -See `protocol_handle_get_or_create` for the error semantics of this function. - -analyzer: type of analyzer to instantiate, specified through its Zeek-side -name (similar to what Zeek's signature action `enable` takes); if not -specified, Zeek will perform its usual dynamic protocol detection to figure -out how to parse the data (the latter will work only for TCP protocols, though.) - -.. _spicy_protocol_handle_get_or_create: - -.. rubric:: ``function zeek::protocol_handle_get_or_create(analyzer: string) : ProtocolHandle`` - -Gets a handle to a Zeek-side child protocol analyzer for the current connection. - -If no such child exists it will be added; otherwise a handle to the -existing child protocol analyzer will be returned. - -This function will return an error - -- if not called from a protocol analyzer, or -- the requested child protocol analyzer is unknown, or -- creation of a child analyzer of the requested type was prevented by a - previous call of `disable_analyzer` with `prevent=T` - -analyzer: type of analyzer to instantiate, specified through its Zeek-side -name (similar to what Zeek's signature action `enable` takes). - -.. _spicy_protocol_data_in: - -.. rubric:: ``function zeek::protocol_data_in(is_orig: bool, data: bytes, h: optional = Null)`` - -Forwards protocol data to all previously instantiated Zeek-side child protocol analyzers. - -is_orig: true to feed the data to the child's originator side, false for the responder -data: chunk of data to forward to child analyzer -h: optional handle to the child analyzer to forward data into, else forward to all child analyzers - -.. _spicy_protocol_gap: - -.. rubric:: ``function zeek::protocol_gap(is_orig: bool, offset: uint64, len: uint64, h: optional = Null)`` - -Signals a gap in input data to all previously instantiated Zeek-side child protocol analyzers. - -is_orig: true to signal gap to the child's originator side, false for the responder -offset: start offset of gap in input stream -len: size of gap -h: optional handle to the child analyzer signal a gap to, else signal to all child analyzers - -.. _spicy_protocol_end: - -.. rubric:: ``function zeek::protocol_end()`` - -Signals end-of-data to all previously instantiated Zeek-side child protocol -analyzers and removes them. - -.. _spicy_protocol_handle_close: - -.. rubric:: ``function zeek::protocol_handle_close(handle: ProtocolHandle)`` - -Signals end-of-data to the given child analyzer and removes it. - -The given handle must be live, i.e., it must not have been used in a -previous protocol_handle_close call, and must not have been live when -protocol_end was called. If the handle is not live a runtime error will -be triggered. - -handle: handle to the child analyzer to remove - -.. _spicy_file_begin: - -.. rubric:: ``function zeek::file_begin(mime_type: optional = Null) : string`` - -Signals the beginning of a file to Zeek's file analysis, associating it with the current connection. -Optionally, a mime type can be provided. It will be passed on to Zeek's file analysis framework. -Returns the Zeek-side file ID of the new file. - -.. _spicy_fuid: - -.. rubric:: ``function zeek::fuid() : string`` - -Returns the current file's FUID. - -.. _spicy_terminate_session: - -.. rubric:: ``function zeek::terminate_session()`` - -Terminates the currently active Zeek-side session, flushing all state. Any -subsequent activity will start a new session from scratch. This can only be -called from inside a protocol analyzer. - -.. _spicy_file_set_size: - -.. rubric:: ``function zeek::file_set_size(size: uint64, fid: optional = Null)`` - -Signals the expected size of a file to Zeek's file analysis. - -size: expected size of file -fid: Zeek-side ID of the file to operate on; if not given, the file started by the most recent file_begin() will be used - -.. _spicy_file_data_in: - -.. rubric:: ``function zeek::file_data_in(data: bytes, fid: optional = Null)`` - -Passes file content on to Zeek's file analysis. - -data: chunk of raw data to pass into analysis -fid: Zeek-side ID of the file to operate on; if not given, the file started by the most recent file_begin() will be used - -.. _spicy_file_data_in_at_offset: - -.. rubric:: ``function zeek::file_data_in_at_offset(data: bytes, offset: uint64, fid: optional = Null)`` - -Passes file content at a specific offset on to Zeek's file analysis. - -data: chunk of raw data to pass into analysis -offset: position in file where data starts -fid: Zeek-side ID of the file to operate on; if not given, the file started by the most recent file_begin() will be used - -.. _spicy_file_gap: - -.. rubric:: ``function zeek::file_gap(offset: uint64, len: uint64, fid: optional = Null)`` - -Signals a gap in a file to Zeek's file analysis. - -offset: position in file where gap starts -len: size of gap -fid: Zeek-side ID of the file to operate on; if not given, the file started by the most recent file_begin() will be used - -.. _spicy_file_end: - -.. rubric:: ``function zeek::file_end(fid: optional = Null)`` - -Signals the end of a file to Zeek's file analysis. - -fid: Zeek-side ID of the file to operate on; if not given, the file started by the most recent file_begin() will be used - -.. _spicy_forward_packet: - -.. rubric:: ``function zeek::forward_packet(identifier: uint32)`` - -Inside a packet analyzer, forwards what data remains after parsing the top-level unit -on to another analyzer. The index specifies the target, per the current dispatcher table. - -.. _spicy_network_time: - -.. rubric:: ``function zeek::network_time() : time`` - -Gets the network time from Zeek. - diff --git a/doc/conf.py b/doc/conf.py index 4204bf630..97c8db5f9 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -23,7 +23,7 @@ # -- Project information ----------------------------------------------------- project = u'Spicy' -copyright = u'2020 by the Zeek Project' +copyright = u'2023 by the Zeek Project' author = u'Zeek Project' version = open('../VERSION').readline() @@ -59,6 +59,8 @@ "issue": ("https://github.com/zeek/spicy/issues/%s", "#%s"), "pr": ("https://github.com/zeek/spicy/pulls/%s", "#%s"), + "zeek": ("https://docs.zeek.org/en/master/%s", "%s"), + # Links to binary builds. "package-dev-tgz": ("https://api.cirrus-ci.com/v1/artifact/github/zeek/spicy/%s/packages/build/spicy-dev.tar.gz", "%s"), "package-dev-rpm": ("https://api.cirrus-ci.com/v1/artifact/github/zeek/spicy/%s/packages/spicy-dev.rpm", "%s"), diff --git a/doc/development/testing.rst b/doc/development/testing.rst index a8aebb3b9..d890217a1 100644 --- a/doc/development/testing.rst +++ b/doc/development/testing.rst @@ -67,7 +67,7 @@ libraries aren't in a standard runtime library path, you'll also need to set ``LD_LIBRARY_PATH`` (Linux) or ``DYLD_LIBRARY_PATH`` (macOS) to point there (e.g., ``LD_LIBRARY_PATH=/opt/clang9/lib/clang/9.0.1/lib/linux``). -When using Zeek's spicy support, and Zeek hasn't been compiled +When using Zeek's Spicy support, and Zeek hasn't been compiled with sanitizer support, you'll also need to set ``LD_PRELOAD`` (Linux) or ``DYLD_INSERT_LIBRARIES`` (macOS) to the shared ``asan`` library to use (e.g., diff --git a/doc/examples/my-http.zeek b/doc/examples/my-http.zeek deleted file mode 100644 index d140400ab..000000000 --- a/doc/examples/my-http.zeek +++ /dev/null @@ -1,4 +0,0 @@ -event MyHTTP::request_line(c: connection, method: string, uri: string, version: string) - { - print fmt("Zeek saw from %s: %s %s %s", c$id$orig_h, method, uri, version); - } diff --git a/doc/faq.rst b/doc/faq.rst index 3431f1ea3..84dc5ff92 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -57,84 +57,4 @@ refers to compiling generated C++ code). Zeek ---- -.. _faq_zeek_install_spicy_and_plugin_to_use_parsers: - -.. rubric:: Do I need to install Spicy and/or a Zeek plugin to use Spicy parsers in Zeek? - -If you're using Zeek >= 5.0 with a default build configuration, -there's nothing else you need to install. After installing Zeek, the -same folder containing the ``zeek`` binary will also have the relevant -Spicy tools, such as ``spicyc`` (provided by Spicy) and ``spicyz`` -(provided by Zeek). To double check that the Spicy support is indeed -available, look for ``Zeek::Spicy`` in the output of ``zeek -N``:: - - # zeek -N - <...> - Zeek::Spicy - Support for Spicy parsers (``*.spicy``, ``*.evt``, ``*.hlto``) (built-in) - -Note that it remains possible to build Zeek against an external Spicy -installation, or even without any Spicy support at all. Look at Zeek's -``configure`` for corresponding options. - -.. note:: - - For some historic background: Zeek 5.0 started bundling Spicy, as well - as the former Zeek plugin for Spicy, so that now nothing else needs to - be installed separately anymore to use Spicy parsers. Since Zeek 6.0, - the code for that former plugin has further moved into Zeek itself, - and is now maintained directly by the Zeek developers. - - -.. _faq_zeek_spicy_dpd_support: - -.. rubric:: Does Spicy support *Dynamic Protocol Detection (DPD)*? - -Yes, see the :ref:`corresponding section ` on how to add it -to your analyzers. - -.. _faq_zeek_layer2_analyzer: - -.. rubric:: Can I write a Layer 2 protocol analyzer with Spicy? - -Yes, you can. In Zeek terminology a layer 2 protocol analyzer is a packet -analyzer, see the :ref:`corresponding section ` on how -to declare such an analyzer. - -.. _faq_zeek_print_statements_no_effect: - -.. rubric:: I have ``print`` statements in my Spicy grammar, why do I not see any output when running Zeek? - -Zeek by default disables the output of Spicy-side ``print`` -statements. To enable them, add ``Spicy::enable_print=T`` to the Zeek -command line (or ``redef Spicy::enable_print=T;`` to a Zeek script -that you are loading). - -.. _faq_zeek_tcp_analyzer_not_all_messages_recognized: - -.. rubric:: My analyzer recognizes only one or two TCP packets even though there are more in the input. - -In Zeek, a Spicy analyzer parses the sending and receiving sides of a TCP -connection each according to the given Spicy grammar. This means that -if more than one message can be sent per side the grammar needs to -allow for that. For example, if the grammar parses messages of the -protocol as ``Message``, the top-level parsing unit given in the EVT -file needs to be able to parse a list of messages ``Message[]``. - -One way to express this is to introduce a parser which wraps -messages of the protocol in an :ref:`anonymous field `. - -.. warning:: Since in general the number of messages exchanged over a TCP - connection is unbounded, an anonymous field should be used. If a named field - was used instead the parser would need to store all messages over the - connection which would lead to unbounded memory growth. - -.. code-block:: spicy - - type Message = unit { - # Fields for messages of the protocol. - }; - - # Parser used e.g., in EVT file. - public type Messages = unit { - : Message[]; - }; +See Zeek's Spicy FAQ :zeek:`devel/spicy/faq.html`. diff --git a/doc/getting-started.rst b/doc/getting-started.rst index 16578ef47..9ce8aa876 100644 --- a/doc/getting-started.rst +++ b/doc/getting-started.rst @@ -197,126 +197,12 @@ If you want to see the actual parsing code that Spicy generates, use ``spicyc`` again: ``spicyc -c my-http.spicy`` will show the C++ code, and ``spicyc -p my-http.spicy`` will show the intermediary HILTI code. -Zeek Integration ----------------- - -Now let's use our ``RequestLine`` parser with Zeek. - -Since version 5.0, Zeek comes with Spicy support built-in by default, -so normally you won't need to install anything further to make use of -Spicy parsers. To double check that your Zeek indeed supports Spicy, -confirm that it shows up in the output of ``zeek -N``: - -.. code:: - - # zeek -N - <...> - Zeek::Spicy - Support for Spicy parsers (``*.spicy``, ``*.evt``, ``*.hlto``) (built-in) - -If you do not see the Spicy listed, it must have been disabled at Zeek -build time. Assuming Spicy is indeed available, you will also find the -Zeek-side compiler tool ``spicyz`` in the same location as the ``zeek`` -binary. - -Next, we prepare some input traffic for testing our ``RequestLine`` -parser with Zeek. - -.. rubric:: Preparations - -Because Zeek works from network packets, we first need a packet trace -with the payload we want to parse. We can't just use a normal HTTP -session as our simple parser wouldn't go further than just the first -line of the protocol exchange and then bail out with an error. So -instead, for our example we create a custom packet trace with a TCP -connection that carries just a single HTTP request line as its -payload:: - - # tcpdump -i lo0 -w request-line.pcap port 12345 & - # nc -l 12345 & - # echo "GET /index.html HTTP/1.0" | nc localhost 12345 - # killall tcpdump nc - -This gets us :download:`this trace file `. - -.. _example_zeek_my_http_adding_analyzer: - -.. rubric:: Adding a Protocol Analyzer - -Now we can go ahead and add a new protocol analyzer to Zeek. We -already got the Spicy grammar to parse our connection's payload, it's -in ``my-http.spicy``. In order to use this with Zeek, we have two -additional things to do: (1) We need to let Zeek know about our new -protocol analyzer, including when to use it; and (2) we need to define -at least one Zeek event that we want our parser to generate, so that -we can then write a Zeek script working with the information that it -extracts. - -We do both of these by creating an additional control file for Zeek: - -.. literalinclude:: examples/my-http.evt - :caption: my-http.evt - :linenos: - :language: spicy-evt - -The first block (lines 1-3) tells Zeek that we have a new protocol -analyzer to provide. The analyzer's Zeek-side name is -``spicy::MyHTTP``, and it's meant to run on top of TCP connections -(line 1). Lines 2-3 then provide Zeek with more specifics: The entry -point for originator-side payload is the ``MyHTTP::RequestLine`` unit -type that our Spicy grammar defines (line 2); and we want Zeek to -activate our analyzer for all connections with a responder port of -12345 (which, of course, matches the packet trace we created). - -The second block (line 5) tells Zeek that we want to -define one event. On the left-hand side of that line we give the unit -that is to trigger the event. The right-hand side defines its name and -arguments. What we are saying here is that every time a ``RequestLine`` -line has been fully parsed, we'd like a ``MyHTTP::request_line`` event -to go to Zeek. Each event instance will come with four parameters: -Three of them are the values of corresponding unit fields, accessed -just through normal Spicy expressions (inside an event argument -expression, ``self`` refers to the unit instance that has led to the -generation of the current event). The first parameter, ``$conn``, is a -"magic" keyword that passes the Zeek-side -connection ID (``conn_id``) to the event. - -Now we got everything in place that we need for our new protocol -analyzer---except for a Zeek script actually doing something with the -information we are parsing. Let's use this: - -.. literalinclude:: examples/my-http.zeek - :caption: my-http.zeek - :language: zeek - -You see an Zeek event handler for the event that we just defined, -having the expected signature of four parameters matching the types of -the parameter expressions that the ``*.evt`` file specifies. The -handler's body then just prints out what it gets. - -.. _example_zeek_my_http: - -Finally we can put together our pieces by compiling the Spicy grammar and the -EVT file into an HLTO file with ``spicyz``, and by pointing Zeek at the produced -file and the analyzer-specific Zeek scripts:: - - # spicyz my-http.spicy my-http.evt -o my-http.hlto - # zeek -Cr request-line.pcap my-http.hlto my-http.zeek - Zeek saw from 127.0.0.1: GET /index.html 1.0 - -When Zeek starts up here the Spicy integration registers a protocol analyzer to -the entry point of our Spicy grammar as specified in the EVT file. It then -begins processing the packet trace as usual, now activating our new analyzer -whenever it sees a TCP connection on port 12345. Accordingly, the -``MyHTTP::request_line`` event gets generated once the parser gets to process -the session's payload. The Zeek event handler then executes and prints the -output we would expect. - .. note:: - By default, Zeek suppresses any output from Spicy-side - ``print`` statements. You can add ``Spicy::enable_print=T`` to the - command line to see it. In the example above, you would then get - an additional line of output: ``GET, /index.html, 1.0``. + See :zeek:`Zeek's Spicy documentation + ` for how to use the + `RequestLine` parser with Zeek. + Custom Host Application ----------------------- diff --git a/doc/index.rst b/doc/index.rst index 4652fe78e..f7cab81b6 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -35,10 +35,7 @@ Overview Spicy comes with Zeek support that enables adding new protocol and file analyzers to `Zeek `_ without having to write any C++ code. You define the grammar, specify which Zeek - events to generate, and Spicy takes care of the rest. There's also - a `Zeek analyzers `_ - package that provides Zeek with several new, Spicy-based - analyzers. + events to generate, and Spicy takes care of the rest. See our :ref:`collection of example grammars ` to get a sense of what Spicy looks like. diff --git a/doc/installation.rst b/doc/installation.rst index cae8510f7..d05cfbf16 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -19,8 +19,8 @@ platforms to support and test. If your goal is to use Spicy with Zeek, you can skip these installation instructions. Zeek comes with Spicy support built-in - by default since version 5.0. See the :ref:`Zeek section - ` for more. + by default since version 5.0. See the :zeek:`Zeek documentation + ` for more. Pre-built binaries ------------------ diff --git a/doc/programming/language/conditional.rst b/doc/programming/language/conditional.rst index dada6d327..451421776 100644 --- a/doc/programming/language/conditional.rst +++ b/doc/programming/language/conditional.rst @@ -39,5 +39,8 @@ By default, Spicy currently provides just one pre-defined identifier: The current Spicy version in numerical format (e.g., 10000 for version 1.0; see the output of ``spicy-config --version-number``). -Zeek defines a couple of :ref:`additional -identifiers `. + +.. note:: + + When using Spicy with Zeek, Zeek defines a couple of :zeek:`additional + identifiers `. diff --git a/doc/programming/parsing.rst b/doc/programming/parsing.rst index 1b3704499..822fe666e 100644 --- a/doc/programming/parsing.rst +++ b/doc/programming/parsing.rst @@ -1980,7 +1980,7 @@ method, which will return a reference to it: By itself, this is not very useful. However, host applications can control how contexts are maintained, and they may assign the same context value to multiple units. For example, when parsing a protocol, -the :ref:`Zeek integration ` always creates a single context +the :zeek:`Zeek integration ` always creates a single context value shared by all top-level units belonging to the same connection, enabling parsers to maintain bi-directional, per-connection state. The batch mode of :ref:`spicy-driver ` does the same. diff --git a/doc/scripts/autogen-zeek-docs b/doc/scripts/autogen-zeek-docs deleted file mode 100755 index 76508b87c..000000000 --- a/doc/scripts/autogen-zeek-docs +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2020-2023 by the Zeek Project. See LICENSE for details. -# -# Tool to update autogenerated docs that require the Zeek integration. Because we are no longer -# shipping that with the Spicy distribution, this needs to be called manually. - -set -e - -if [ $# != 2 ]; then - echo "usage: $(basename "$0") " - exit 1 -fi - -zeek=$1 -tftp=$2 - -if [ ! -e "${zeek}"/scripts/spicy/zeek.spicy ]; then - echo "${zeek} does not seem to point to a (recent) Zeek repository." - exit 1 -fi - -if [ ! -d "${tftp}"/analyzer ]; then - echo "${tftp} does not seem to point to a spicy-tftp repository." - exit 1 -fi - -set -o errexit -set -o nounset - -ROOTDIR="$(cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/../.." -BUILDDIR="${ROOTDIR}/build" -AUTOGEN_FINAL="${ROOTDIR}/doc/autogen" - -# Autogen library docs. -if [[ ! -x ${BUILDDIR}/bin/spicy-doc ]]; then - >&2 echo "Warning: Could not find required executable ${BUILDDIR}/bin/spicy-doc, aborting" - exit 0 -fi - -"${ROOTDIR}/doc/scripts/autogen-spicy-lib" functions zeek < "${zeek}/scripts/spicy/zeek.spicy" > "${AUTOGEN_FINAL}/zeek/zeek-functions.spicy" || exit 1 - -# Copy some static files over. -cp "${zeek}"/scripts/base/frameworks/spicy/init-bare.zeek "${AUTOGEN_FINAL}"/zeek/ || exit 1 -cp "${zeek}"/scripts/base/frameworks/spicy/init-framework.zeek "${AUTOGEN_FINAL}"/zeek/ || exit 1 -cp "${zeek}"/scripts/policy/frameworks/spicy/record-spicy-batch.zeek "${AUTOGEN_FINAL}"/zeek/ || exit 1 - -cp "${tftp}"/scripts/main.zeek "${AUTOGEN_FINAL}"/zeek/tftp.zeek || exit 1 -cp "${tftp}"/analyzer/tftp.spicy "${AUTOGEN_FINAL}"/zeek/tftp.spicy || exit 1 -cp "${tftp}"/analyzer/tftp.evt "${AUTOGEN_FINAL}"/zeek/tftp.evt || exit 1 -cat "${tftp}"/analyzer/tftp.spicy | grep -v spicy::accept_input > "${AUTOGEN_FINAL}"/zeek/tftp-no-accept.spicy || exit 1 diff --git a/doc/toolchain.rst b/doc/toolchain.rst index 231cfa71c..c64d334d2 100644 --- a/doc/toolchain.rst +++ b/doc/toolchain.rst @@ -136,13 +136,14 @@ interleaved input flows in parallel, mimicking how host applications like Zeek would be employing Spicy parsers for processing many sessions concurrently. The batch input must be prepared in a specific format (see below) that provides embedded meta information about the -contained flows of input. The easiest way to generate such a batch -is :download:`a Zeek script coming with Spicy -`. If you run Zeek with this script -on a PCAP trace, it will record the contained TCP and UDP sessions +contained flows of input. If you have Zeek at hand, the easiest way to +generate such a batch is `a script coming with Zeek +`_. +If you run Zeek with this script on a PCAP trace, it will record the +contained TCP and UDP sessions into a Spicy batch file:: - # zeek -b -r http/methods.trace record-spicy-batch.zeek + # zeek -b -r http/methods.trace policy/frameworks/spicy/record-spicy-batch tracking [orig_h=128.2.6.136, orig_p=46562/tcp, resp_h=173.194.75.103, resp_p=80/tcp] tracking [orig_h=128.2.6.136, orig_p=46563/tcp, resp_h=173.194.75.103, resp_p=80/tcp] tracking [orig_h=128.2.6.136, orig_p=46564/tcp, resp_h=173.194.75.103, resp_p=80/tcp] diff --git a/doc/tutorial/examples/tftp-schedule-analyzer.zeek b/doc/tutorial/examples/tftp-schedule-analyzer.zeek deleted file mode 100644 index 122677aa6..000000000 --- a/doc/tutorial/examples/tftp-schedule-analyzer.zeek +++ /dev/null @@ -1,37 +0,0 @@ - -function schedule_tftp_analyzer(id: conn_id) - { - # Schedule the TFTP analyzer for the expected next packet coming in on different - # ports. We know that it will be exchanged between same IPs and reuse the - # originator's port. "Spicy_TFTP" is the Zeek-side name of the TFTP analyzer - # (generated from "Spicy::TFTP" in tftp.evt). - Analyzer::schedule_analyzer(id$resp_h, id$orig_h, id$orig_p, Analyzer::get_tag("Spicy_TFTP"), 1min); - } - -event tftp::read_request(c: connection, is_orig: bool, filename: string, mode: string) - { - print "TFTP read request", c$id, filename, mode; - schedule_tftp_analyzer(c$id); - } - -event tftp::write_request(c: connection, is_orig: bool, filename: string, mode: string) - { - print "TFTP write request", c$id, filename, mode; - schedule_tftp_analyzer(c$id); - } - -# Add handlers for other packet types so that we see their events being generated. -event tftp::data(c: connection, is_orig: bool, block_num: count, data: string) - { - print "TFTP data", block_num, data; - } - -event tftp::ack(c: connection, is_orig: bool, block_num: count) - { - print "TFTP ack", block_num; - } - -event tftp::error(c: connection, is_orig: bool, code: count, msg: string) - { - print "TFTP error", code, msg; - } diff --git a/doc/tutorial/examples/tftp-single-request-more-args.zeek b/doc/tutorial/examples/tftp-single-request-more-args.zeek deleted file mode 100644 index cf29b8666..000000000 --- a/doc/tutorial/examples/tftp-single-request-more-args.zeek +++ /dev/null @@ -1,4 +0,0 @@ -event tftp::request(c: connection, is_orig: bool, filename: string, mode: string) - { - print "TFTP request", c$id, is_orig, filename, mode; - } diff --git a/doc/tutorial/examples/tftp-single-request.zeek b/doc/tutorial/examples/tftp-single-request.zeek deleted file mode 100644 index 53c072aa3..000000000 --- a/doc/tutorial/examples/tftp-single-request.zeek +++ /dev/null @@ -1,4 +0,0 @@ -event tftp::request(c: connection) - { - print "TFTP request", c$id; - } diff --git a/doc/tutorial/examples/tftp-two-requests.zeek b/doc/tutorial/examples/tftp-two-requests.zeek deleted file mode 100644 index 45a24a464..000000000 --- a/doc/tutorial/examples/tftp-two-requests.zeek +++ /dev/null @@ -1,9 +0,0 @@ -event tftp::read_request(c: connection, is_orig: bool, filename: string, mode: string) - { - print "TFTP read request", c$id, is_orig, filename, mode; - } - -event tftp::write_request(c: connection, is_orig: bool, filename: string, mode: string) - { - print "TFTP write request", c$id, is_orig, filename, mode; - } diff --git a/doc/tutorial/examples/tftp.zeek b/doc/tutorial/examples/tftp.zeek deleted file mode 120000 index bdcfce316..000000000 --- a/doc/tutorial/examples/tftp.zeek +++ /dev/null @@ -1 +0,0 @@ -../../autogen/zeek/tftp.zeek \ No newline at end of file diff --git a/doc/tutorial/index.rst b/doc/tutorial/index.rst index 6dc961f1e..dc31edfc2 100644 --- a/doc/tutorial/index.rst +++ b/doc/tutorial/index.rst @@ -6,7 +6,7 @@ Tutorial: A Real Analyzer ========================= In this chapter we will develop a simple protocol analyzer from -scratch, including full Zeek integration. Our analyzer will parse the +scratch. Our analyzer will parse the *Trivial File Transfer Protocol (TFTP)* in its original incarnation, as described in `RFC 1350 `_. TFTP provides a small protocol for copying files from a server to a @@ -26,8 +26,8 @@ Creating a Spicy Grammar We start by developing Spicy grammar for TFTP. The protocol is packet-based, and our grammar will parse the content of one TFTP -packet at a time. While TFTP is running on top of UDP, we will leave -the lower layers to Zeek and have Spicy parse just the actual UDP +packet at a time. While TFTP is running on top of UDP, we will +Spicy parse just the actual UDP application-layer payload, as described in `Section 5 `_ of the protocol standard. @@ -503,7 +503,7 @@ really need two separate types here, and could instead define a single ``Request`` unit to cover both cases. Doing so is straight-forward, except for one issue: when parsing such a ``Request``, we would now lose the information whether we are seeing -read or a write operation. For our Zeek integration later it will be +read or a write operation. For a potential Zeek integration later it will be useful to retain that distinction, so let us leverage a Spicy capability that allows passing state into a sub-unit: :ref:`unit parameters `. Here's the corresponding excerpt after @@ -581,410 +581,9 @@ Combining everything discussed so far, this leaves us with the following complete grammar for TFTP, including the packet formats in comments as well: -.. literalinclude:: /autogen/zeek/tftp-no-accept.spicy +.. literalinclude:: /_static/tftp-no-accept.spicy :language: spicy -Zeek Integration -================ - -To turn the Spicy-side grammar into a Zeek analyzer, we need to -provide Zeek with a description of how to employ it. -There are two parts to that: Telling Zeek when to activate the -analyzer, and defining events to generate. In addition, we will need a -Zeek-side script to do something with our new TFTP events. We will -walk through this in the following, starting with the mechanics of -compiling the Spicy analyzer for Zeek. While we will build up the -files involved individually first, see the :ref:`final section -` for how the Zeek package manager, *zkg*, can be -used to bootstrap a new Zeek package with a skeleton of everything -needed for an analyzer. - -Before proceeding, make sure that your Zeek comes with Spicy support -built-in---which is the default since Zeek version 5.0:: - - # zeek -N Zeek::Spicy - Zeek::Spicy - Support for Spicy parsers (*.hlto) (built-in) - -You should also have ``spicyz`` in your ``PATH``:: - - # which spicyz - /usr/local/zeek/bin/spicyz - -Compiling the Analyzer ----------------------- - -Zeek comes with a tool :ref:`spicyz ` that compiles Spicy -analyzers into binary code that Zeek can load through a Spicy plugin. -The following command line produces a binary object file ``tftp.hlto`` -containing the executable analyzer code: - -.. code:: - - # spicyz -o tftp.hlto tftp.spicy - -Below, we will prepare an additional interface definition file -``tftp.evt`` that describes the analyzer's integration into Zeek. We -will need to give that to ``spicyz`` as well, and our full -compilation command hence becomes: - -.. code:: - - # spicyz -o tftp.hlto tftp.spicy tftp.evt - -When starting Zeek, we add ``tftp.hlto`` to its command line: - -.. code:: - - # zeek -r tftp_rrq.pcap tftp.hlto - - -Activating the Analyzer ------------------------ - -In *Getting Started*, :ref:`we already saw -` how to inform Zeek about a new -protocol analyzer. We follow the same scheme here and put the -following into ``tftp.evt``, the analyzer definition file: - -.. literalinclude:: examples/tftp.evt - :lines: 3-5 - :language: spicy-evt - -The first line provides our analyzer with a Zeek-side name -(``spicy::TFTP``) and also tells Zeek that we are adding an -application analyzer on top of UDP (``over UDP``). ``TFTP::Packet`` -provides the top-level entry point for parsing both sides of a TFTP -connection. Furthermore, we want Zeek to automatically activate our -analyzer for all sessions on UDP port 69 (i.e., TFTP's well known -port). See :ref:`zeek_evt_analyzer_setup` for more details on defining -such a ``protocol analyzer`` section. - -With this in place, we can already employ the analyzer inside Zeek. It -will not generate any events yet, but we can at least see the output of -the ``on %done { print self; }`` hook that still remains part of the -grammar from earlier: - -.. code:: - - # zeek -r tftp_rrq.pcap tftp.hlto Spicy::enable_print=T - [$opcode=Opcode::RRQ, $rrq=[$filename=b"rfc1350.txt", $mode=b"octet"], $wrq=(not set), $data=(not set), $ack=(not set), $error=(not set)] - -As by default, the Zeek plugin does not show the output of Spicy-side -``print`` statements, we added ``Spicy::enable_print=T`` to the -command line to turn that on. We see that Zeek took care of the -lower network layers, extracted the UDP payload from the Read Request, -and passed that into our Spicy parser. (If you want to view more about -the internals of what is happening here, there are a couple kinds of -:ref:`debug output available `.) - -You might be wondering why there is only one line of output, even -though there are multiple TFTP packets in our pcap trace. Shouldn't -the ``print`` execute multiple times? Yes, it should, but it does not -currently: Due to some intricacies of the TFTP protocol, our analyzer -gets to see only the first packet for now. We will fix this later. For -now, we focus on the Read Request packet that the output above shows. - -Defining Events ---------------- - -The core task of any Zeek analyzer is to generate events for Zeek -scripts to process. For binary protocols, events will often correspond -pretty directly to data units specified by their specifications---and -TFTP is no exception. We start with an event for Read/Write Requests -by adding this definition to ``tftp.evt``: - -.. literalinclude:: examples/tftp-single-request.evt - :lines: 5-7 - :language: spicy-evt - -The first line makes our Spicy TFTP grammar available to the rest of -the file. The line ``on ...`` defines one event: Every time a -``Request`` unit will be parsed, we want to receive an event -``tftp::request`` with one parameter: the connection it belongs to. -Here, ``$conn`` is a reserved identifier that will turn into the -standard `connection record -`_ -record on the Zeek side. - -Now we need a Zeek event handler for our new event. Let's put this -into ``tftp.zeek``: - -.. literalinclude:: examples/tftp-single-request.zeek - :language: zeek - -Running Zeek then gives us: - -.. code:: - - # spicyz -o tftp.hlto tftp.spicy tftp.evt - # zeek -r tftp_rrq.pcap tftp.hlto tftp.zeek - TFTP request, [orig_h=192.168.0.253, orig_p=50618/udp, resp_h=192.168.0.10, resp_p=69/udp] - -Let's extend the event signature a bit by passing further arguments: - -.. literalinclude:: examples/tftp-single-request-more-args.evt - :lines: 5-7 - :language: spicy-evt - -This shows how each parameter gets specified as a Spicy expression: -``self`` refers to the instance currently being parsed (``self``), and -``self.filename`` retrieves the value of its ``filename`` field. -``$is_orig`` is another reserved ID that turns into a boolean that -will be true if the event has been triggered by originator-side -traffic. On the Zeek side, our event now has the following signature: - -.. literalinclude:: examples/tftp-single-request-more-args.zeek - :language: zeek - -.. code:: - - # spicyz -o tftp.hlto tftp.spicy tftp.evt - # zeek -r tftp_rrq.pcap tftp.hlto tftp.zeek - TFTP request, [orig_h=192.168.0.253, orig_p=50618/udp, resp_h=192.168.0.10, resp_p=69/udp], T, rfc1350.txt, octet - -Going back to our earlier discussion of Read vs Write Requests, we do -not yet make that distinction with the ``request`` event that we are -sending to Zeek-land. However, since we had introduced the ``is_read`` -unit parameter, we can easily separate the two by gating event -generation through an additional ``if`` condition: - -.. literalinclude:: examples/tftp.evt - :lines: 9-10 - :language: spicy-evt - -This now defines two separate events, each being generated only for -the corresponding value of ``is_read``. Let's try it with a new -``tftp.zeek``: - -.. literalinclude:: examples/tftp-two-requests.zeek - :language: zeek - -.. code:: - - # spicyz -o tftp.hlto tftp.spicy tftp.evt - # zeek -r tftp_rrq.pcap tftp.hlto tftp.zeek - TFTP read request, [orig_h=192.168.0.253, orig_p=50618/udp, resp_h=192.168.0.10, resp_p=69/udp], T, rfc1350.txt, octet - -If we look at the ``conn.log`` that Zeek produces during this run, we -will see that the ``service`` field is not filled in yet. That's -because our analyzer does not yet confirm to Zeek that it has been -successful in parsing the content. To do that, we can call a library -function that Spicy makes available once we have successfully parsed a -request: :ref:`spicy::accept_input `. That -function signals the host application---i.e., Zeek in our case—--that -the parser is processing the expected protocol. With that, our request -looks like this now: - -.. spicy-code:: tftp-request-with-accept.spicy - - type Request = unit(is_read: bool) { - filename: bytes &until=b"\x00"; - mode: bytes &until=b"\x00"; - - on %done { spicy::accept_input(); } - }; - - -Let's try it again: - -.. code:: - - # spicyz -o tftp.hlto tftp.spicy tftp.evt - # zeek -r tftp_rrq.pcap tftp.hlto tftp.zeek - TFTP read request, [orig_h=192.168.0.253, orig_p=50618/udp, resp_h=192.168.0.10, resp_p=69/udp], T, rfc1350.txt, octet - # cat conn.log - [...] - 1367411051.972852 C1f7uj4uuv6zu2aKti 192.168.0.253 50618 192.168.0.10 69 udp spicy_tftp - - - S0 - -0 D 1 48 0 0 - - [...] - -Now the service field says TFTP! (There will be a 2nd connection in -the log that we are not showing here; see the next section on that). - -Turning to the other TFTP packet types, it is straight-forward to add -events for them as well. The following is our complete ``tftp.evt`` -file: - -.. literalinclude:: examples/tftp.evt - :lines: 3- - :language: spicy-evt - - -Detour: Zeek vs. TFTP ---------------------- - -We noticed above that Zeek seems to be seeing only a single TFTP -packet from our input trace, even though ``tcpdump`` shows that the -pcap file contains multiple different types of packets. The reason -becomes clear once we look more closely at the UDP ports that are in -use: - -.. code:: - - # tcpdump -ttnr tftp_rrq.pcap - 1367411051.972852 IP 192.168.0.253.50618 > 192.168.0.10.69: 20 RRQ "rfc1350.txtoctet" [tftp] - 1367411052.077243 IP 192.168.0.10.3445 > 192.168.0.253.50618: UDP, length 516 - 1367411052.081790 IP 192.168.0.253.50618 > 192.168.0.10.3445: UDP, length 4 - 1367411052.086300 IP 192.168.0.10.3445 > 192.168.0.253.50618: UDP, length 516 - 1367411052.088961 IP 192.168.0.253.50618 > 192.168.0.10.3445: UDP, length 4 - 1367411052.088995 IP 192.168.0.10.3445 > 192.168.0.253.50618: UDP, length 516 - [...] - -Turns out that only the first packet is using the well-known TFTP port -69/udp, whereas all the subsequent packets use ephemeral ports. Due to -the port difference, Zeek believes it is seeing two independent -network connections, and it does not associate TFTP with the second -one at all due to its lack of the well-known port (neither does -``tcpdump``!). Zeek's connection log confirms this by showing two -separate entries: - -.. code:: - - # cat conn.log - 1367411051.972852 CH3xFz3U1nYI1Dp1Dk 192.168.0.253 50618 192.168.0.10 69 udp spicy_tftp - - - S0 - - 0 D 1 48 0 0 - - 1367411052.077243 CfwsLw2TaTIeo3gE9g 192.168.0.10 3445 192.168.0.253 50618 udp - 0.181558 24795 196 SF - - 0 Dd 49 26167 49 1568 - - -Switching the ports for subsequent packets is a quirk in TFTP that -resembles similar behaviour in standard FTP, where data connections -get set up separately as well. Fortunately, Zeek provides a built-in -function to designate a specific analyzer for an anticipated future -connection. We can call that function when we see the initial request: - -.. literalinclude:: examples/tftp-schedule-analyzer.zeek - :language: zeek - -.. code:: - - # spicyz -o tftp.hlto tftp.spicy tftp.evt - # zeek -r tftp_rrq.pcap tftp.hlto tftp.zeek - TFTP read request, [orig_h=192.168.0.253, orig_p=50618/udp, resp_h=192.168.0.10, resp_p=69/udp], rfc1350.txt, octet - TFTP data, 1, \x0a\x0a\x0a\x0a\x0a\x0aNetwork Working Group [...] - TFTP ack, 1 - TFTP data, 2, B Official Protocol\x0a Standards" for the [...] - TFTP ack, 2 - TFTP data, 3, protocol was originally designed by Noel Chia [...] - TFTP ack, 3 - TFTP data, 4, r mechanism was suggested by\x0a PARC's EFT [...] - TFTP ack, 4 - [...] - -Now we are seeing all the packets as we would expect. - -Zeek Script ------------ - -Analyzers normally come along with a Zeek-side script that implements -a set of standard base functionality, such as recording activity into -a protocol specific log file. These scripts provide handlers for the -analyzers' events, and collect and correlate their activity as -desired. We have created such :download:`a script for TFTP -`, based on the events that our Spicy analyzer -generates. Once we add that to the Zeek command line, we will see a -new ``tftp.log``: - -.. code:: - - # spicyz -o tftp.hlto tftp.spicy tftp.evt - # zeek -r tftp_rrq.pcap tftp.hlto tftp.zeek - # cat tftp.log - #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p wrq fname mode uid_data size block_sent block_acked error_code error_msg - 1367411051.972852 CKWH8L3AIekSHYzBU 192.168.0.253 50618 192.168.0.10 69 F rfc1350.txt octet ClAr3P158Ei77Fql8h 24599 49 49 - - - -The TFTP script also labels the second session as TFTP data by -adding a corresponding entry to the ``service`` field inside the -Zeek-side connection record. With that, we are now seeing this in -``conn.log``: - -.. code:: - - 1367411051.972852 ChbSfq3QWKuNirt9Uh 192.168.0.253 50618 192.168.0.10 69 udp spicy_tftp - - - S0 - -0 D 1 48 0 0 - - 1367411052.077243 CowFQj20FHHduhHSYk 192.168.0.10 3445 192.168.0.253 50618 udp spicy_tftp_data 0.181558 24795 196 SF -- 0 Dd 49 26167 49 1568 - - -The TFTP script ends up being a bit more complex than one would expect -for such a simple protocol. That's because it tracks the two related -connections (initial request and follow-up traffic on a different -port), and combines them into a single TFTP transaction for logging. -Since there is nothing Spicy-specific in that Zeek script, we skip -discussing it here in more detail. - -.. _zkg_create_package: - -Creating a Zeek Package -======================= - -We have now assembled all the parts needed for providing a new -analyzer to Zeek. By adding a few further pieces, we can wrap that -analyzer into a full *Zeek package* for others to install easily -through *zkg*. To help create that wrapping, *zkg* provides a template -for instantiating a skeleton analyzer package as a starting point. The -skeleton comes in three different flavors, depending on which kind of -analyzer you want to create: protocol, file, or packet analyzer. -In each case, it creates all the necessary files along with the -appropriate directory layout, and even includes a couple of -standard test cases. - -To create the scaffolding for our TFTP analyzer, execute the following -command and provide the requested information:: - - # zkg create --features spicy-protocol-analyzer --packagedir spicy-tftp - "package-template" requires a "name" value (the name of the package, e.g. "FooBar" or "spicy-http"): - name: spicy-tftp - "package-template" requires a "analyzer" value (name of the Spicy analyzer, which typically corresponds to the protocol/format being parsed (e.g. "HTTP", "PNG")): - analyzer: TFTP - "package-template" requires a "protocol" value (transport protocol for the analyzer to use: TCP or UDP): - protocol: UDP - "package-template" requires a "unit_orig" value (name of the top-level Spicy parsing unit for the originator side of the connection (e.g. "Request")): - unit_orig: Packet - "package-template" requires a "unit_resp" value (name of the top-level Spicy parsing unit for the responder side of the connection (e.g. "Reply"); may be the same as originator side): - unit_resp: Packet - - -The above creates the following files (skipping anything related to -``.git``):: - - spicy-tftp/CMakeLists.txt - spicy-tftp/COPYING - spicy-tftp/README - spicy-tftp/analyzer/CMakeLists.txt - spicy-tftp/analyzer/tftp.evt - spicy-tftp/analyzer/tftp.spicy - spicy-tftp/cmake/FindSpicyPlugin.cmake - spicy-tftp/scripts/__load__.zeek - spicy-tftp/scripts/dpd.sig - spicy-tftp/scripts/main.zeek - spicy-tftp/testing/Baseline/tests.run-pcap/conn.log - spicy-tftp/testing/Baseline/tests.run-pcap/output - spicy-tftp/testing/Baseline/tests.standalone/ - spicy-tftp/testing/Baseline/tests.standalone/output - spicy-tftp/testing/Baseline/tests.trace/output - spicy-tftp/testing/Baseline/tests.trace/tftp.log - spicy-tftp/testing/Files/random.seed - spicy-tftp/testing/Makefile - spicy-tftp/testing/Scripts/README - spicy-tftp/testing/Scripts/diff-remove-timestamps - spicy-tftp/testing/Scripts/get-zeek-env - spicy-tftp/testing/Traces/tcp-port-12345.pcap - spicy-tftp/testing/Traces/udp-port-12345.pcap - spicy-tftp/testing/btest.cfg - spicy-tftp/testing/tests/availability.zeek - spicy-tftp/testing/tests/standalone.spicy - spicy-tftp/testing/tests/trace.zeek - spicy-tftp/zkg.meta - - -Note the ``*.evt``, ``*.spicy``, ``*.zeek`` files: they correspond to -the files we created for TFTP in the preceding sections; we can just -move our versions in there. Furthermore, the generated scaffolding -marks places with ``TODO`` that need manual editing: use ``git grep -TODO`` inside the ``spicy-tftp`` directory to find them. We won't go -through all the specific customizations for TFTP here, but for -reference you can find the full TFTP package as created from the *zkg* -template on `GitHub `_. - -If instead of a protocol analyzer, you'd like to create a file or -packet analyzer, run zkg with ``--features spicy-file-analyzer`` or -``--features spicy-packet-analyzer``, respectively. The generated -skeleton will be suitably adjusted then. - Next Steps ========== @@ -1003,4 +602,8 @@ show here. Some pointers for what to look at next: - :ref:`examples` summarizes grammars coming with the Spicy distribution. +- Zeek's :zeek:`Spicy tutorial ` continues + the TFTP example by turning the Spicy code developed here into + a full Zeek analyzer. + - :ref:`zeek_plugin` discusses Spicy's integration into Zeek. diff --git a/doc/zeek.rst b/doc/zeek.rst index b88562746..0d1e491b0 100644 --- a/doc/zeek.rst +++ b/doc/zeek.rst @@ -11,976 +11,5 @@ for its development from early on. While historically an external Zeek plugin was required to use Spicy parsers with Zeek, Zeek has now been shipping with built-in Spicy support since version 5.0. That means you can directly add new protocol, file, and packet analyzers to Zeek -through Spicy without needing to write any further code. In the -following, we dig deeper into how to use all of this. - -.. note:: - - There are a number of pieces involved in creating a full Zeek - analyzer, in particular if you want to distribute it as a Zeek - package. To help you get started with that, Zeek's package manager - can create a skeleton Spicy package by running:: - - # zkg create --features=spicy-protocol-analyzer --packagedir - - The generated files mark places that will need manual editing with - ``TODO``. See the :ref:`tutorial ` for more on - this. - -.. _zeek_terminology: - -Terminology -=========== - -In Zeek, the term "analyzer" generally refers to a component that -processes a particular protocol ("protocol analyzer"), file format -("file analyzer"), or low-level packet structure ("packet analyzer"). -"Processing" here means more than just parsing content: An analyzer -controls when it wants to be used (e.g., with connections on specific -ports, or with files of a specific MIME type); what events to generate -for Zeek's scripting layer; and how to handle any errors occurring -during parsing. While Spicy itself focuses just on the parsing part, -Spicy makes it possible to provide the remaining pieces to -Zeek, turning a Spicy parser into a full Zeek analyzer. That's what we -refer to as a "Spicy (protocol/file/packet) analyzer" for Zeek. - -.. _zeek_installation: - -Installation -============ - -Since Zeek version 5.0, support for Spicy is by default bundled with -Zeek. To confirm that Spicy is indeed available, you can inspect the -output of ``zeek -N``:: - - # zeek -N Zeek::Spicy - Zeek::Spicy - Support for Spicy parsers (*.hlto) (built-in) - -It remains possible to build Zeek against an external Spicy -installation through Zeek's ``configure`` option -``--with-spicy=PATH``, where ``PATH`` points to the Spicy installation -directory. In that case, you also need to ensure that the Spicy tools -(e.g., ``spicyc``, ``spicy-config``) are available in ``PATH``. - -Interface Definitions ("evt files") -=================================== - -Per above, a Spicy analyzer for Zeek does more than just parsing data. -Accordingly, we need to tell Zeek a couple of additional -pieces about analyzers we want it to provide to Zeek: - -Analyzer setup - Zeek needs to know what type of analyzers we are creating, - when we want Zeek to activate them, and what Spicy unit types to - use as their parsing entry point. - -Event definitions - We need to tell Zeek which events to provide and - when to trigger them. - -We define all of these through custom interface definition files that -Spicy's compiler for Zeek reads in. These files use an ``*.evt`` -extension, and the following subsections discuss their content in more -detail. - -Generally, empty lines and comments starting with ``#`` are ignored in -an ``*.evt``. - -.. note:: - - The syntax for ``*.evt`` files comes with some legacy pieces that - aren't particularly pretty. We may clean that up at some point. - -.. _zeek_evt_analyzer_setup: - -Analyzer Setup --------------- - -You can define protocol analyzers, packet analyzers and file analyzers in an -``*.evt`` file, per the following. - -.. rubric:: Protocol Analyzer - -To define a protocol analyzer, add a new section to an ``*.evt`` -file that looks like this:: - - protocol analyzer ANALYZER_NAME over TRANSPORT_PROTOCOL: - PROPERTY_1, - PROPERTY_2, - ... - PROPERTY_N; - -Here, ``ANALYZER_NAME`` is a name to identify your analyzer inside -Zeek. You can choose names arbitrarily as long as they are unique. As -a convention, however, we recommend name with a ``spicy::*`` prefix -(e.g., ``spicy::BitTorrent``). - -On the Zeek-side, through some normalization, these names -automatically turn into tags added to Zeek's ``Analyzer::Tag`` enum. -For example, ``spicy::BitTorrent`` turns into -``Analyzer::ANALYZER_SPICY_BITTORRENT``. - -The analyzer's name is also what goes into Zeek signatures to activate -an analyzer DPD-style. If the name is ``spicy::BitTorrent``, you'd -write ``enable "spicy::BitTorrent"`` into the signature. - -.. note:: - - Once you have made your analyzers available to Zeek (which we will - discuss below), running ``zeek -NN Zeek::Spicy`` will show you a - summary of what's now available, including their Zeek-side names - and tags. - -``TRANSPORT_PROTOCOL`` can be either ``tcp`` or ``udp``, depending on -the transport-layer protocol that your new analyzer wants to sit on -top of. - -Following that initial ``protocol analyzer ...`` line, a set of -properties defines further specifics of your analyzer. The following -properties are supported: - - ``parse [originator|responder] with SPICY_UNIT`` - Specifies the top-level Spicy unit(s) the analyzer uses for - parsing payload, with ``SPICY_UNIT`` being a fully-qualified - Spicy-side type name (e.g. ``HTTP::Request``). The unit type must - have been declared as ``public`` in Spicy. - - If ``originator`` is given, the unit is used only for parsing the - connection's originator-side payload; and if ``responder`` is - given, only for responder-side payload. If neither is given, it's - used for both sides. In other words, you can use different units - per side by specifying two properties ``parse originator with - ...`` and ``parse responder with ...``. - - ``port PORT`` or ``ports { PORT_1, ..., PORT_M }`` - Specifies one or more well-known ports for which you want Zeek to - automatically activate your analyzer with corresponding - connections. Each port must be specified in Spicy's :ref:`syntax - for port constants ` (e.g., ``80/tcp``), or as a port range - ``PORT_START-PORT_END`` where start and end port are port constants - forming a closed interval. The ports' transport protocol must match - that of the analyzer. - - .. note:: - - Zeek will also honor any ``%port`` :ref:`meta data - property ` that the responder-side - ``SPICY_UNIT`` may define (as long as the attribute's - direction is not ``originator``). - - ``replaces ANALYZER_NAME`` - Disables an existing analyzer that Zeek already provides - internally, allowing you to replace a built-in analyzer with a new - Spicy version. ``ANALYZER_NAME`` is the Zeek-side name of the - analyzer. To find that name, inspect the output of ``zeek -NN`` - for available analyzers:: - - # zeek -NN | grep '\[Analyzer\]' - ... - [Analyzer] SMTP (ANALYZER_SMTP, enabled) - ... - - Here, ``SMTP`` is the name you would write into ``replaces`` to - disable the built-in SMTP analyzer. - -As a full example, here's what a new HTTP analyzer could look like: - -.. code-block:: spicy-evt - - protocol analyzer spicy::HTTP over TCP: - parse originator with HTTP::Requests, - parse responder with HTTP::Replies, - port 80/tcp, - replaces HTTP; - - -.. _zeek_packet_analyzer: -.. rubric:: Packet Analyzer - -Defining packet analyzers works quite similar to protocol analyzers through -``*.evt`` sections like this:: - - packet analyzer ANALYZER_NAME: - PROPERTY_1, - PROPERTY_2, - ... - PROPERTY_N; - -Here, ``ANALYZER_NAME`` is again a name to identify your analyzer -inside Zeek. On the Zeek-side, the name will be added to Zeek's -``PacketAnalyzer::Tag`` enum. - -Packet analyzers support the following properties: - - ``parse with SPICY_UNIT`` - Specifies the top-level Spicy unit the analyzer uses for - parsing each packet, with ``SPICY_UNIT`` being a fully-qualified - Spicy-side type name. The unit type must have been declared as - ``public`` in Spicy. - - ``replaces ANALYZER_NAME`` - Disables an existing packet analyzer that Zeek already - provides internally, allowing you to replace a built-in - analyzer with a new Spicy version. ``ANALYZER_NAME`` is the - Zeek-side name of the existing analyzer. To find that name, - inspect the output of ``zeek -NN`` for available analyzers:: - - # zeek -NN | grep '\[Packet Analyzer\]' - ... - [Packet Analyzer] Ethernet (ANALYZER_ETHERNET) - ... - - Here, ``Ethernet`` is the name you would give to ``replaces`` - to disable the built-in Ethernet analyzer. - - When replacing an existing packet analyzer, you still need to - configure your new analyzer on the Zeek side through a - `zeek_init()` event handler. See below for more. - - .. note:: - - This feature requires Zeek >= 5.2. - -As a full example, here's what a new packet analyzer could look like:: - - packet analyzer spicy::RawLayer: - parse with RawLayer::Packet; - -In addition to the Spicy-side configuration, packet analyzers also -need to be registered with Zeek inside a ``zeek_init`` event handler; -see the `Zeek documentation -`_ -for more. The ``-NN`` output shows your new analyzer's ``ANALYZER_*`` -tag to use. - -.. note:: - - Depending on relative loading order between your analyzer and - Zeek's scripts, in some situations you may not have access to the - ``ANALYZER_*`` tag referring to your new analyzer. If you run into - this, use `PacketAnalyzer::try_register_packet_analyzer_by_name - `_ - to register your Spicy analyzer by name (as shown by ``-NN``). - Example: - - .. code-block:: zeek - - event zeek_init() - { - if ( ! PacketAnalyzer::try_register_packet_analyzer_by_name("Ethernet", 0x88b5, "spicy::RawLayer") ) - Reporter::error("unknown analyzer name used"); - } - - -.. rubric:: File Analyzer - -Defining file analyzers works quite similar to protocol analyzers, -through ``*.evt`` sections like this:: - - file analyzer ANALYZER_NAME: - PROPERTY_1, - PROPERTY_2, - ... - PROPERTY_N; - -Here, ``ANALYZER_NAME`` is again a name to identify your analyzer -inside Zeek. On the Zeek-side, the name will be added to Zeek's -``Files::Tag`` enum. - -File analyzers support the following properties: - - ``parse with SPICY_UNIT`` - Specifies the top-level Spicy unit the analyzer uses for - parsing file content, with ``SPICY_UNIT`` being a - fully-qualified Spicy-side type name. The unit type must have - been declared as ``public`` in Spicy. - - ``mime-type MIME-TYPE`` - Specifies a MIME type for which you want Zeek to automatically - activate your analyzer when it sees a corresponding file on - the network. The type is specified in standard - ``type/subtype`` notion, without quotes (e.g., ``image/gif``). - - .. note:: - - Zeek will also honor any ``%mime-type`` :ref:`meta - data property ` that the ``SPICY_UNIT`` - may define. - - .. note:: - - Keep in mind that Zeek identifies MIME types through - "content sniffing" (i.e., similar to libmagic), and - usually not by protocol-level headers (e.g., *not* through - HTTP's ``Content-Type`` header). If in doubt, examine - ``files.log`` for what it records as a file's type. - - ``replaces ANALYZER_NAME`` - Disables an existing file analyzer that Zeek already provides - internally, allowing you to replace a built-in analyzer with a new - Spicy version. ``ANALYZER_NAME`` is the Zeek-side name of the - analyzer. To find that name, inspect the output of ``zeek -NN`` - for available analyzers:: - - # zeek -NN | grep '\[File Analyzer\]' - ... - [File Analyzer] PE (ANALYZER_PE, enabled) - ... - - Here, ``PE`` is the name you would write into ``replaces`` to - disable the built-in PE analyzer. - - .. note:: - - This feature requires Zeek >= 4.1. - -As a full example, here's what a new GIF analyzer could look like: - -.. code-block:: spicy-evt - - file analyzer spicy::GIF: - parse with GIF::Image, - mime-type image/gif; - -.. _zeek_events: - -Event Definitions ------------------ - -To define a Zeek event that you want the Spicy analyzer to trigger, you -add lines of the form:: - - on HOOK_ID -> event EVENT_NAME(ARG_1, ..., ARG_N); - - on HOOK_ID if COND -> event EVENT_NAME(ARG_1, ..., ARG_N); - -Zeek automatically derives from this everything it needs to -register new events with Zeek, including a mapping of the arguments' -Spicy types to corresponding Zeek types. More specifically, these are -the pieces going into such an event definition: - -``on HOOK_ID`` - A Spicy-side ID that defines when you want to trigger the event. - This works just like an ``on ...`` :ref:`unit hook `, - and you can indeed use anything here that Spicy supports for those - as well (except :ref:`container hooks `). So, e.g., ``on - HTTP::Request::%done`` triggers an event whenever a - ``HTTP::Request`` unit has been fully parsed, and ``on - HTTP::Request::uri`` leads to an event each time the ``uri`` field - has been parsed. (In the former example you may skip the - ``%done``, actually: ``on HTTP::Request`` implicitly adds it.) - -``EVENT_NAME`` - The Zeek-side name of the event you want to generate, preferably - including a namespace (e.g., ``http::request``). - -``ARG_1, ..., ARG_N`` - Arguments to pass to the event, given as arbitrary Spicy - expressions. Each expression will be evaluated within the context of - the unit that the ``on ...`` triggers on, similar to code running - inside the body of a corresponding :ref:`unit hook `. - That means the expression has access to ``self`` for accessing - the unit instance that's currently being parsed. - - The Spicy type of the expression determines the Zeek-side type of - the corresponding event argument. Most Spicy types translate - over pretty naturally, the following summarizes the translation: - - .. _zeek-event-arg-types: - - .. csv-table:: Type Conversion from Spicy to Zeek - :header: "Spicy Type", "Zeek Type", "Notes" - - ``addr``, ``addr``, - ``bool``, ``bool``, - ``enum { ... }``, ``enum { ... }``, [1] - ``int(8|16|32|64)``, ``int``, - ``interval``, ``interval``, - ``list``, ``vector of T``, - "``map``", "``table[V] of K``", - ``optional``, ``T``, [2] - ``port``, ``port``, - ``real``, ``double``, - ``set``, ``set[T]``, - ``string``, ``string``, - ``time``, ``time``, - "``tuple``", "``record { T1, ..., T_N }``", [3] - ``uint(8|16|32|64)``, ``count``, - ``vector``, ``vector of T``, - - .. note:: - - [1] - A corresponding Zeek-side ``enum`` type is automatically - created. See :ref:`below ` for more. - - [2] - The optional value must have a value, otherwise a runtime - exception will be thrown. - - [3] - Must be mapped to a Zeek-side record type with matching - fields. - - If a tuple element is mapped to a record field with a - ``&default`` or ``&optional`` attribute, a couple special - cases are supported: - - - If the expression evaluates to ``Null``, the record - field is left unset. - - - If the element's expression uses the - :spicy:op:`.? ` operator and that - fails to produce a value, the record field is - likewise left unset. - - In addition to full Spicy expressions, there are three reserved - IDs with specific meanings when used as arguments: - - ``$conn`` - Refers to the connection that's currently being processed - by Zeek. On the Zeek-side this will turn into a parameter - of Zeek type ``connection``. This ID can be used only with - protocol analyzers. - - ``$file`` - Refers to the file that's currently being processed by - Zeek. On the Zeek-side this will turn into a parameter of - Zeek type ``fa_file``. This ID can be used with file or protocol - analyzers. For protocol analyzers it refers to the most recently - opened, but not yet closed file, see :ref:`zeek::file_begin - ` and :ref:`zeek::file_end `. - - ``$packet`` - Refers to the packet that's currently being processed by - Zeek. On the Zeek-side this will turn into a parameter of - Zeek type ``raw_pkt_hdr``, with any fields filled in that - have already been parsed by Zeek's built-in analyzers. - This ID can be used only with packet analyzers. (Note that - instantiation of ``raw_pkt_hdr`` values can be relatively - expensive on the Zeek side, so best to limit usage of this - ID to a small part of the overall traffic.) - - ``$is_orig`` - A boolean indicating if the data currently being processed - is coming from the originator (``True``) or responder - (``False``) of the underlying connection. This turns into - a corresponding boolean value on the Zeek side. This ID - can be used only with protocol analyzers. - - .. note:: - - Some tips: - - - If you want to force a specific type on the Zeek-side, you - have a couple of options: - - 1. Spicy may provide a ``cast`` operator from the actual - type into the desired type (e.g., ``cast(..)``). - - 2. Argument expressions have access to global functions - defined in the Spicy source files, so you can write a - conversion function taking an argument with its - original type and returning it with the desired type. - - - List comprehension can be convenient to fill Zeek vectors: - ``[some_func(i) for i in self.my_list]``. - -``if COND`` - If given, events are only generated if the expression ``COND`` - evaluates to true. Just like event arguments, the expression is - evaluated in the context of the current unit instance and has - access to ``self``. - - -.. _zeek_export_types: - -Exporting Types ---------------- - -As we :ref:`discuss above `, the type of each event -argument maps over to a corresponding Zeek type. On the Zeek side, -that corresponding type needs to be known by Zeek. That is always the -case for built-in atomic types (per :ref:`the conversion table -`), but it can turn out more challenging -to achieve for custom types, such as ``enum`` and ``record``, for -which you would normally need to manually create matching type declarations in -your Zeek scripts. While that's not necessarily hard, it can -become quite cumbersome. - -Fortunately, there's help: for most types, Zeek can -instantiate corresponding types automatically as it loads the -corresponding Spicy analyzer. While you will never actually see the -Zeek-side type declarations, they will be available inside your Zeek -scripts as if you had typed them out yourself--similar to other types -that are built into Zeek itself. - -To have the Zeek create a type for your analyzer automatically, -you need to ``export`` the Spicy type in your EVT file. The most basic syntax for -that is:: - - export SPICY_ID [as ZEEK_ID]; - -Here, ``SPICY_ID`` is the fully-scoped type ID on the Spicy side, and -``ZEEK_ID`` is the fully-scoped type ID you want in Zeek. If you leave -out the ``as ...`` part, the Zeek name will be the same as the Spicy -name, including its namespace. For example, say you have a Spicy unit -``TFTP::Request``. Adding ``export TFTP::Request;`` to your EVT file -will make a ``record`` type of the same name, and with the same -fields, available in Zeek. If you instead use ``export TFTP::Request -as TheOtherTFTP::Request``, it will be placed into a different -namespace instead. - -Exporting types generally works for most Spicy types as long as -there's an ID associated with them in your Spicy code. However, -exporting is most helpful with user-defined types, such as ``enum`` -and ``unit``, because it can save you quite a bit of typing there. We -discuss the more common type conversions in more detail below. When -exporting units, there are two optional extensions to the syntax, -which we describe :ref:`below `. - -To confirm the types made available to Zeek, you can see all exports -in the output of ``zeek -NN``. With our 2nd ``TFTP::Request`` -example, that looks like this:: - - [...] - # zeek -NN Zeek::Spicy - Zeek::Spicy - Support for Spicy parsers (*.hlto) - [Type] TheOtherTFTP::Request - [...] - -.. note:: - - Most, but not all, types can be exported automatically. For - example, self-recursive types are currently not supported. - Generally, if you run into trouble exporting a type, you can always - fall back to declaring a corresponding Zeek version yourself in - your Zeek script. Consider the ``export`` mechanism as a - convenience feature that helps avoid writing a lot of boiler plate - code in common cases. - -.. _zeek_enum: - -Enum Types -^^^^^^^^^^ - -When you export a Spicy ``enum`` type, Zeek creates a -corresponding Zeek ``enum`` type. For example, assume the following -Spicy declaration: - -.. spicy-code:: - - module Test; - - type MyEnum = enum { - A = 83, - B = 84, - C = 85 - }; - -Using ``export Test::MyEnum;``, Zeek will create the equivalent -of the following Zeek type for use in your scripts: - -.. code-block:: zeek - - module Test; - - export { - - type MyEnum: enum { - MyEnum_A = 83, - MyEnum_B = 84, - MyEnum_A = 85, - MyEnum_Undef = -1 - }; - - } - -(The odd naming is due to ID limitations on the Zeek side.) - -.. note:: - - For backward compatibility, the ``enum`` type comes with an - additional property: all *public* enum types are automatically - exported, even without adding any ``export`` to your EVT file. - This feature may go away at some point, and we suggest to not rely - on it on new code; always use ``export`` for `enum` types as well. - -.. _zeek_unit: - -Unit Types -^^^^^^^^^^ - -When you export a Spicy ``unit`` type, Zeek creates a -corresponding Zeek ``record`` type. For example, assume the following -Spicy declaration: - -.. spicy-code:: - - module Test; - - type MyRecord = unit { - a: bytes &size=4; - b: uint16; - - var c: bool; - }; - -Using ``export Test::MyRecord;``, Zeek will then create the -equivalent of the following Zeek type for use in your scripts: - -.. code-block:: zeek - - module Test; - - export { - - type MyRecord: record { - a: string &optional; - b: count &optional; - c: bool; - }; - - } - -The individual fields map over just like event arguments do, following -the :ref:`the table ` above. For aggregate -types, this works recursively: if, e.g., a field is itself of unit -type *and that type has been exported as well*, Zeek will map it -over accordingly to the corresponding ``record`` type. Note that such -dependent types must be exported *first* in the EVT file for this to -work. As a result, you cannot export self-recursive unit types. - -As you can see in the example, unit fields are always declared as -optional on the Zeek-side, as they may not have been set during -parsing. Unit variables are non-optional by default, unless declared -as ``&optional`` in Spicy. - -Sometimes you may not want to export all unit items to Zeek. For that -case, there is an extended form the ``export`` statement that takes a -list of fields to include or exclude. To include only selected fields, -add a ``with`` clause to the ``export`` statement:: - - export spicy_id [as ZEEK_ID] with { field_1, field_2, ..., field_n }; - -To export all but a some fields, use ``without`` instead:: - - export spicy_id [as zeek_id] without { field_1, field_2, ..., field_n }; - -In both cases, ``field_i`` is the name of a unit field to include or -exclude. - -If you intend to use the exported ``record`` type with `Zeek's logging -framework `_, -you can automatically have a ``&log`` attribute included with the type -by adding ``&log`` to the ``export`` statement:: - - export spicy_id [as ZEEK_ID] &log; - -Now Zeek will log all the exported ``record`` fields when writing out -an instance of the ``record`` type. - -If you want ``&log`` only for a subset of ``record`` fields, you can -use a ``with`` clause (see above) and add ``&log`` to just the -desired fields. The following is the full syntax for ``with``:: - - export spicy_id [as ZEEK_ID] with { field_1 [&log], field_2 [&log], ..., field_n [&log] }; - -.. _zeek_struct: - -Struct Types -^^^^^^^^^^^^ - -A Spicy ``struct`` type maps over to Zeek in the same way as ``unit`` -types do, treating each field like a unit variable. See :ref:`there -` for more information. - -Tuple Types -^^^^^^^^^^^ - -A Spicy ``tuple`` type maps over to a Zeek ``record`` similar to how -``unit`` types do, treating each tuple element like a unit variable. -See :ref:`there ` for more information. - -Exporting works only for ``tuple`` types that declare names for all -their elements. - -Importing Spicy Modules ------------------------ - -Code in an ``*.evt`` file may need access to additional Spicy modules, -such as when expressions for event parameters call Spicy -functions defined elsewhere. To make a Spicy module available, you can -insert ``import`` statements into the ``*.evt`` file that work -:ref:`just like in Spicy code `: - - ``import NAME`` - Imports Spicy module ``NAME``. - - ``import NAME from X.Y.Z;`` - Searches for the module ``NAME`` (i.e., for the filename - ``NAME.spicy``) inside a sub-directory ``X/Y/Z`` along the - search path, and then imports it. - -.. _zeek_conditional_compilation: - -Conditional Compilation ------------------------ - -``*.evt`` files offer the same basic form of :ref:`conditional -compilation ` through -``@if``/``@else``/``@endif`` blocks as Spicy scripts. Zeek -makes two additional identifiers available for testing to both -``*.evt`` and ``*.spicy`` code: - - ``HAVE_ZEEK`` - Always set to 1 by Zeek. This can be used for feature - testing from Spicy code to check if it's being compiled for - Zeek. - - ``ZEEK_VERSION`` - The numerical Zeek version that's being compiled for (see - ``zeek -e 'print Version::number'``). - -This is an example bracketing code by Zeek version in an EVT file: - -.. code-block:: spicy-evt - - @if ZEEK_VERSION < 30200 - - @else - - @endif - -.. _zeek_compiling: -.. _spicyz: - -Compiling Analyzers -==================== - -Once you have the ``*.spicy`` and ``*.evt`` source files for your new -analyzer, you need to precompile them into an ``*.hlto`` object file -containing their final executable code, which Zeek will then use. To -do that, pass the relevant ``*.spicy`` and ``*.evt`` files to -``spicyz``, then have Zeek load the output. To repeat the -:ref:`example ` from the *Getting Started* -guide:: - - # spicyz -o my-http-analyzer.hlto my-http.spicy my-http.evt - # zeek -Cr request-line.pcap my-http-analyzer.hlto my-http.zeek - Zeek saw from 127.0.0.1: GET /index.html 1.0 - -Instead of providing the precompiled analyzer on the Zeek command -line, you can also copy them into -``${prefix}/lib/spicy/Zeek_Spicy/modules``. Zeek will -automatically load any ``*.hlto`` object files it finds there. In -addition, Zeek also scans its plugin directory for ``*.hlto`` -files. Alternatively, you can override both of those locations by -setting the environment variable ``ZEEK_SPICY_MODULE_PATH`` to a set of -colon-separated directories to search instead. Zeek will then -*only* look there. In all cases, Zeek searches any directories -recursively, so it will find ``*.hlto`` also if they are nested in -subfolders. - -Run ``spicyz -h`` to see some additional options it provides, which -are similar to :ref:`spicy-driver`. - -.. _zeek_functions: - -Controlling Zeek from Spicy -=========================== - -Spicy grammars can import a provided library module ``zeek`` to gain -access to Zeek-specific functions that call back into Zeek's -processing: - -.. include:: /autogen/zeek/zeek-functions.spicy - -.. _zeek_dpd: - -Dynamic Protocol Detection (DPD) -================================ - -Spicy protocol analyzers support Zeek's *Dynamic Protocol Detection* -(DPD), i.e., analysis independent of any well-known ports. To use that -with your analyzer, add two pieces: - -1. A `Zeek signature - `_ to - activate your analyzer based on payload patterns. Just like with - any of Zeek's standard analyzers, a signature can activate a Spicy - analyzer through the ``enable ""`` keyword. The name of the - analyzer comes out of the EVT file: it is the ``ANALYZER_NAME`` - with the double colons replaced with an underscore (e.g., - ``spicy::HTTP`` turns into ``enable "spicy_HTTP"``. - -2. You should call :ref:`spicy::accept_input() ` - from a hook inside your grammar at a point when the parser can be - reasonably certain that it is processing the expected protocol. - Optionally, you may also call :ref:`spicy::decline_input() - ` when you're sure the parser is *not* parsing - the right protocol. However, Zeek will also trigger this - automatically whenever your parser aborts with an error. - -.. _zeek_configuration: - -Configuration -============= - -Options -------- - -Zeek provides a set of script-level options to tune Spicy -behavior. These all live in the ``Spicy::`` namespace: - -.. literalinclude:: /autogen/zeek/init-bare.zeek - :language: zeek - :start-after: doc-options-start - :end-before: doc-options-end - -Functions ---------- - -Zeek also adds the following new built-in functions for Spicy, which -likewise live in the ``Spicy::`` namespace: - -.. literalinclude:: /autogen/zeek/init-framework.zeek - :language: zeek - :start-after: doc-functions-start - :end-before: doc-functions-end - -.. _zeek_debugging: - -Debugging -========= - -If Zeek doesn't seem to be doing the right thing with your Spicy -analyzer, there are several ways to debug what's going on. To -facilitate that, compile your analyzer with ``spicyz -d`` and, if -possible, use a debug version of Zeek (i.e., build Zeek with -``./configure --enable-debug``). - -If your analyzer doesn't seem to be active at all, first make sure -Zeek actually knows about it: It should show up in the output of -``zeek -NN Zeek::Spicy``. If it doesn't, you might not have been using -the right ``*.spicy`` or ``*.evt`` files when precompiling, or Zeek is -not loading the ``*.hlto`` file. Also check your ``*.evt`` if it -defines your analyzer correctly. - -If Zeek knows about your analyzer and just doesn't seem to activate -it, double-check that ports or MIME types are correct in the ``*.evt`` -file. If you're using a signature instead, try a port/MIME type first, -just to make sure it's not a matter of signature mismatches. - -If there's nothing obviously wrong with your source files, you can -trace what the Zeek's Spicy support is compiling by running ``spicyz`` -with ``-D zeek``. For example, reusing the :ref:`HTTP example -` from the *Getting Started* guide:: - - # spicyz -D zeek my-http.spicy my-http.evt -o my-http.hlt - [debug/zeek] Loading Spicy file "/Users/robin/work/spicy/main/tests/spicy/doc/my-http.spicy" - [debug/zeek] Loading EVT file "/Users/robin/work/spicy/main/doc/examples/my-http.evt" - [debug/zeek] Loading events from /Users/robin/work/spicy/main/doc/examples/my-http.evt - [debug/zeek] Got protocol analyzer definition for spicy_MyHTTP - [debug/zeek] Got event definition for MyHTTP::request_line - [debug/zeek] Running Spicy driver - [debug/zeek] Got unit type 'MyHTTP::Version' - [debug/zeek] Got unit type 'MyHTTP::RequestLine' - [debug/zeek] Adding protocol analyzer 'spicy_MyHTTP' - [debug/zeek] Adding Spicy hook 'MyHTTP::RequestLine::0x25_done' for event MyHTTP::request_line - [debug/zeek] Done with Spicy driver - -You can see the main pieces in there: The files being loaded, unit -types provided by them, analyzers and events being created. - -If that all looks as expected, it's time to turn to the Zeek side and -see what it's doing at runtime. You'll need a debug version of Zeek -for that, as well as a small trace with traffic that you expect your -analyzer to process. Run Zeek with ``-B dpd`` (or ``-B file_analysis`` -if you're debugging a file analyzer) on your trace to record the -analyzer activity into ``debug.log``. For example, with the same HTTP -example, we get: - -.. code-block:: text - :linenos: - - # zeek -B dpd -Cr request-line.pcap my-http.hlto - # cat debug.log - [dpd] Registering analyzer SPICY_MYHTTP for port 12345/1 - [...[ - [dpd] Available analyzers after zeek_init(): - [...] - [dpd] spicy_MyHTTP (enabled) - [...] - [dpd] Analyzers by port: - [dpd] 12345/tcp: SPICY_MYHTTP - [...] - [dpd] TCP[5] added child SPICY_MYHTTP[7] - [dpd] 127.0.0.1:59619 > 127.0.0.1:12345 activated SPICY_MYHTTP analyzer due to port 12345 - [...] - [dpd] SPICY_MYHTTP[7] DeliverStream(25, T) [GET /index.html HTTP/1.0\x0a] - [dpd] SPICY_MYHTTP[7] EndOfData(T) - [dpd] SPICY_MYHTTP[7] EndOfData(F) - -The first few lines show that Zeek's analyzer system registers the -analyzer as expected. The subsequent lines show that the analyzer gets -activated for processing the connection in the trace, and that it then -receives the data that we know indeed constitutes its payload, before -it eventually gets shutdown. - -To see this from the Zeek side, set the ``zeek`` debug stream -through the ``HILTI_DEBUG`` environment variable:: - - # HILTI_DEBUG=zeek zeek -Cr request-line.pcap my-http.hlto - [zeek] Have Spicy protocol analyzer spicy_MyHTTP - [zeek] Registering Protocol::TCP protocol analyzer spicy_MyHTTP with Zeek - [zeek] Scheduling analyzer for port 12345/tcp - [zeek] Done with post-script initialization - [zeek] [SPICY_MYHTTP/7/orig] initial chunk: |GET /index.html HTTP/1.0\\x0a| (eod=false) - [zeek] [SPICY_MYHTTP/7/orig] -> event MyHTTP::request_line($conn, GET, /index.html, 1.0) - [zeek] [SPICY_MYHTTP/7/orig] done with parsing - [zeek] [SPICY_MYHTTP/7/orig] parsing finished, skipping further originator payload - [zeek] [SPICY_MYHTTP/7/resp] no unit specified for parsing - [zeek] [SPICY_MYHTTP/7/orig] skipping end-of-data delivery - [zeek] [SPICY_MYHTTP/7/resp] no unit specified for parsing - [zeek] [SPICY_MYHTTP/7/orig] skipping end-of-data delivery - [zeek] [SPICY_MYHTTP/7/resp] no unit specified for parsing - -After the initial initialization, you see the data arriving and the -event being generated for Zeek. Zeek also reports that we didn't -define a unit for the responder side---which we know in this case, but -if that appears unexpectedly you probably found a problem. - -So we know now that our analyzer is receiving the anticipated data to -parse. At this point, we can switch to debugging the Spicy side -:ref:`through the usual mechanisms `. In particular, -setting ``HILTI_DEBUG=spicy`` tends to be helpful:: - - # HILTI_DEBUG=spicy zeek -Cr request-line.pcap my-http.hlto - [spicy] MyHTTP::RequestLine - [spicy] method = GET - [spicy] anon_2 = - [spicy] uri = /index.html - [spicy] anon_3 = - [spicy] MyHTTP::Version - [spicy] anon = HTTP/ - [spicy] number = 1.0 - [spicy] version = [$number=b"1.0"] - [spicy] anon_4 = \n - -If everything looks right with the parsing, and the right events are -generated too, then the final part is to check out the events that -arrive on the Zeek side. To get Zeek to see an event that Zeek -raises, you need to have at least one handler implemented for it in -one of your Zeek scripts. You can then load Zeek's -``misc/dump-events`` to see them as they are being received, including -their full Zeek-side values:: - - # zeek -Cr request-line.pcap my-http.hlto misc/dump-events - [...] - 1580991211.780489 MyHTTP::request_line - [0] c: connection = [id=[orig_h=127.0.0.1, orig_p=59619/tcp, ...] ...] - [1] method: string = GET - [2] uri: string = /index.html - [3] version: string = 1.0 - [...] +through Spicy without needing to write any further code. See +:zeek:`Zeek's Spicy documentation ` for more.