From 5a5c180ee3a820bc951c05686807df48fe07dc4f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 1 Dec 2020 19:01:03 +0000 Subject: [PATCH 1/3] Support &eod for vector parsing. That tells a vector explicitly to parse until it hits the end of the input, without having to use the look-ahead machinery to figure out when to stop. --- doc/programming/parsing.rst | 3 +++ .../toolchain/src/compiler/codegen/grammar-builder.cc | 5 +++-- spicy/toolchain/src/compiler/visitors/validator.cc | 4 ++-- .../Baseline/spicy.types.regexp.invalid-field/output | 4 ++-- tests/Baseline/spicy.types.vector.parse-eod/output | 2 ++ tests/spicy/types/vector/parse-eod.spicy | 11 +++++++++++ 6 files changed, 23 insertions(+), 6 deletions(-) create mode 100644 tests/Baseline/spicy.types.vector.parse-eod/output create mode 100644 tests/spicy/types/vector/parse-eod.spicy diff --git a/doc/programming/parsing.rst b/doc/programming/parsing.rst index 2bdcdab4f..9df7087fe 100644 --- a/doc/programming/parsing.rst +++ b/doc/programming/parsing.rst @@ -929,6 +929,9 @@ It is possible to skip the ``SIZE`` (e.g., ``x: uint8[]``) and instead use another kind of end conditions to terminate a vector's parsing loop. To that end, vectors support the following attributes: +``&eod`` + Parses elements until the end of the input stream is reached. + ``&size=N`` Parses the vector from the subsequent ``N`` bytes of input data. This effectively limits the available input to the corresponding diff --git a/spicy/toolchain/src/compiler/codegen/grammar-builder.cc b/spicy/toolchain/src/compiler/codegen/grammar-builder.cc index e8e938638..3434e9e53 100644 --- a/spicy/toolchain/src/compiler/codegen/grammar-builder.cc +++ b/spicy/toolchain/src/compiler/codegen/grammar-builder.cc @@ -65,6 +65,7 @@ struct Visitor : public hilti::visitor::PreOrder { const auto& loc = p.node.location(); auto& field = currentField().first; auto id = cg->uniquer()->get(field.id()); + auto eod = AttributeSet::find(field.attributes(), "&eod"); auto count = AttributeSet::find(field.attributes(), "&count"); auto size = AttributeSet::find(field.attributes(), "&size"); auto parse_at = AttributeSet::find(field.attributes(), "&parse-at"); @@ -97,9 +98,9 @@ struct Visitor : public hilti::visitor::PreOrder { // Custom input, just iterate until EOD. return production::ForEach(id, sub, true, loc); - if ( while_ || until || until_including ) + if ( while_ || until || until_including || eod ) // The container parsing will evaluate the corresponding stop - // conditon. + // condition as necessary. return production::ForEach(id, sub, true, loc); // Nothing specified, use look-ahead to figure out when to stop diff --git a/spicy/toolchain/src/compiler/visitors/validator.cc b/spicy/toolchain/src/compiler/visitors/validator.cc index c0191d948..97545e6a7 100644 --- a/spicy/toolchain/src/compiler/visitors/validator.cc +++ b/spicy/toolchain/src/compiler/visitors/validator.cc @@ -275,8 +275,8 @@ struct PreTransformVisitor : public hilti::visitor::PreOrderparseType().isA() || f->ctor() ) - error("&eod is only valid for bytes fields", p); + if ( ! (f->parseType().isA() || f->parseType().isA()) || f->ctor() ) + error("&eod is only valid for bytes and vector fields", p); } } diff --git a/tests/Baseline/spicy.types.regexp.invalid-field/output b/tests/Baseline/spicy.types.regexp.invalid-field/output index c77669f20..44ed9f841 100644 --- a/tests/Baseline/spicy.types.regexp.invalid-field/output +++ b/tests/Baseline/spicy.types.regexp.invalid-field/output @@ -1,3 +1,3 @@ -[error] /Users/robin/work/spicy/topic2/tests/.tmp/spicy.types.regexp.invalid-field/invalid-field.spicy:7:5: need regexp constant for parsing a field -[error] /Users/robin/work/spicy/topic2/tests/.tmp/spicy.types.regexp.invalid-field/invalid-field.spicy:8:15: &eod is only valid for bytes fields +[error] /Users/robin/work/spicy/topic/tests/.tmp/spicy.types.regexp.invalid-field/invalid-field.spicy:7:5: need regexp constant for parsing a field +[error] /Users/robin/work/spicy/topic/tests/.tmp/spicy.types.regexp.invalid-field/invalid-field.spicy:8:15: &eod is only valid for bytes and vector fields [error] spicy-driver: aborting after errors diff --git a/tests/Baseline/spicy.types.vector.parse-eod/output b/tests/Baseline/spicy.types.vector.parse-eod/output new file mode 100644 index 000000000..e0a514d9c --- /dev/null +++ b/tests/Baseline/spicy.types.vector.parse-eod/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[$a=[49, 50, 51, 52, 53], $b=[]] diff --git a/tests/spicy/types/vector/parse-eod.spicy b/tests/spicy/types/vector/parse-eod.spicy new file mode 100644 index 000000000..f75f328c2 --- /dev/null +++ b/tests/spicy/types/vector/parse-eod.spicy @@ -0,0 +1,11 @@ +# @TEST-EXEC: printf "12345" | spicy-driver %INPUT >output +# @TEST-EXEC: btest-diff output + +module Test; + +public type Foo = unit { + a: int8[] &eod; + b: int8[] &eod; # Won't get anything here anymore + + on %done { print self; } + }; From ab761b86581a9c0f12a3a8988eae2494e251a3d2 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sun, 29 Nov 2020 15:05:38 +0000 Subject: [PATCH 2/3] Add support for manual backtracking to Spicy. If a field is marked with ``&try``, a later call to ``self.backtrack()`` anywhere down in the parse tree will return to that position and continue there. (This is ported over the old prototype). Example: export type test = unit { a: bytes &length=4; foo: Foo &try; bar: Bar; b: bytes &length=6; on %done { print self; } }; type Foo = unit { a: int8 { print "Foo.a", self; } b: int8 { print "Backtracking"; self.backtrack(); } c: int8 { print "Foo.c", self; } }; type Bar = unit { a: int8 { print "Bar.a", self; } b: int8 { print "Bar.b", self; } c: int8 { print "Bar.c", self; } }; # printf '1234\001\002\003567890' | pac-driver backtrack.pac2 Foo.a Backtracking Bar.a Bar.b Bar.c , b=567890> Closes #25. --- doc/autogen/types/unit.rst | 6 ++ doc/programming/examples/_backtrack.spicy | 22 +++++++ .../examples/_parse-backtrack.spicy | 22 +++++++ .../examples/_parse-backtrack.spicy.output | 6 ++ doc/programming/parsing.rst | 57 +++++++++++++++++++ spicy/lib/spicy_rt.hlt | 3 + spicy/runtime/include/parser.h | 17 ++++++ spicy/runtime/src/parser.cc | 1 + spicy/toolchain/include/ast/operators/unit.h | 14 +++++ .../include/ast/types/unit-items/switch.h | 5 +- .../compiler/detail/codegen/parser-builder.h | 6 ++ .../toolchain/src/compiler/codegen/codegen.cc | 5 ++ .../src/compiler/codegen/parser-builder.cc | 24 ++++++++ .../toolchain/src/compiler/parser/scanner.ll | 2 +- .../spicy.types.unit.backtrack-lah/output | 5 ++ .../output | 6 ++ .../spicy.types.unit.backtrack/output | 6 ++ tests/spicy/types/unit/backtrack-lah.spicy | 56 ++++++++++++++++++ .../spicy/types/unit/backtrack-on-error.spicy | 30 ++++++++++ tests/spicy/types/unit/backtrack.spicy | 25 ++++++++ tests/spicy/types/vector/parse-eod.spicy | 2 +- 21 files changed, 314 insertions(+), 6 deletions(-) create mode 100644 doc/programming/examples/_backtrack.spicy create mode 100644 doc/programming/examples/_parse-backtrack.spicy create mode 100644 doc/programming/examples/_parse-backtrack.spicy.output create mode 100644 tests/Baseline/spicy.types.unit.backtrack-lah/output create mode 100644 tests/Baseline/spicy.types.unit.backtrack-on-error/output create mode 100644 tests/Baseline/spicy.types.unit.backtrack/output create mode 100644 tests/spicy/types/unit/backtrack-lah.spicy create mode 100644 tests/spicy/types/unit/backtrack-on-error.spicy create mode 100644 tests/spicy/types/unit/backtrack.spicy diff --git a/doc/autogen/types/unit.rst b/doc/autogen/types/unit.rst index 567956c97..89b5ea13b 100644 --- a/doc/autogen/types/unit.rst +++ b/doc/autogen/types/unit.rst @@ -1,5 +1,11 @@ .. rubric:: Methods +.. spicy:method:: unit::backtrack unit backtrack False void () + + Aborts parsing at the current position and returns back to the most + recent ``&try`` attribute. Turns into a parse error if there's no + ``&try`` in scope. + .. spicy:method:: unit::connect_filter unit connect_filter False void (filter: strong_ref) Connects a separate filter unit to transform the unit's input diff --git a/doc/programming/examples/_backtrack.spicy b/doc/programming/examples/_backtrack.spicy new file mode 100644 index 000000000..479eb95ea --- /dev/null +++ b/doc/programming/examples/_backtrack.spicy @@ -0,0 +1,22 @@ +# Automatically generated; edit in Sphinx source code, not here. +module Test; + +public type test = unit { + foo: Foo &try; + bar: Bar; + + on %done { print self; } +}; + +type Foo = unit { + a: int8 { + if ( $$ != 1 ) + self.backtrack(); + } + b: int8; +}; + +type Bar = unit { + a: int8; + b: int8; +}; \ No newline at end of file diff --git a/doc/programming/examples/_parse-backtrack.spicy b/doc/programming/examples/_parse-backtrack.spicy new file mode 100644 index 000000000..479eb95ea --- /dev/null +++ b/doc/programming/examples/_parse-backtrack.spicy @@ -0,0 +1,22 @@ +# Automatically generated; edit in Sphinx source code, not here. +module Test; + +public type test = unit { + foo: Foo &try; + bar: Bar; + + on %done { print self; } +}; + +type Foo = unit { + a: int8 { + if ( $$ != 1 ) + self.backtrack(); + } + b: int8; +}; + +type Bar = unit { + a: int8; + b: int8; +}; \ No newline at end of file diff --git a/doc/programming/examples/_parse-backtrack.spicy.output b/doc/programming/examples/_parse-backtrack.spicy.output new file mode 100644 index 000000000..4d95477ca --- /dev/null +++ b/doc/programming/examples/_parse-backtrack.spicy.output @@ -0,0 +1,6 @@ +# Automatically generated; do not edit. -- printf '\001\002\003\004' | spicy-driver %INPUT; printf '\003\004' | spicy-driver %INPUT/printf '\001\002\003\004' | spicy-driver %INPUT; printf '\003\004' | spicy-driver %INPUT/False +# printf '\001\002\003\004' | spicy-driver backtrack.spicy; printf '\003\004' | spicy-driver backtrack.spicy +[$foo=[$a=1, $b=2], $bar=[$a=3, $b=4]] + +# printf '\001\002\003\004' | spicy-driver backtrack.spicy; printf '\003\004' | spicy-driver backtrack.spicy +[$foo=[$a=3, $b=(not set)], $bar=[$a=3, $b=4]] diff --git a/doc/programming/parsing.rst b/doc/programming/parsing.rst index 9df7087fe..291ab3dbb 100644 --- a/doc/programming/parsing.rst +++ b/doc/programming/parsing.rst @@ -1216,6 +1216,63 @@ once you have subunits that are recognizable by how they start: :exec: printf 'A ' | spicy-driver %INPUT; printf '\377\377' | spicy-driver %INPUT :show-with: foo.spicy +.. _backtracking: + +Backtracking +^^^^^^^^^^^^ + +Spicy supports a simple form of manual backtracking. If a field is +marked with ``&try``, a later call to the unit's ``backtrack()`` +method anywhere down in the parse tree originating at that field will +immediately transfer control over to the field following the ``&try``. +When doing so, the data position inside the input stream will be reset +to where it was when the ``&try`` field started its processing. Units +along the original path will be left in whatever state they were at +the time ``backtrack()`` executed (i.e., they will probably remain +just partially initialized). When ``backtrack()`` is called on a path +that involves multiple ``&try`` fields, control continues after the +most recent. + +Example: + +.. spicy-code:: parse-backtrack.spicy + + module Test; + + public type test = unit { + foo: Foo &try; + bar: Bar; + + on %done { print self; } + }; + + type Foo = unit { + a: int8 { + if ( $$ != 1 ) + self.backtrack(); + } + b: int8; + }; + + type Bar = unit { + a: int8; + b: int8; + }; + + +.. spicy-output:: parse-backtrack.spicy + :exec: printf '\001\002\003\004' | spicy-driver %INPUT; printf '\003\004' | spicy-driver %INPUT + :show-with: backtrack.spicy + +``backtrack()`` can be called from inside :ref:`%error hooks +`, so this provides a simple form of error recovery +as well. + +.. note:: + + This mechanism is preliminary and will probably see refinement + over time, both in terms of more automated backtracking and by + providing better control where to continue after backtracking. Changing Input ============== diff --git a/spicy/lib/spicy_rt.hlt b/spicy/lib/spicy_rt.hlt index f7a47e964..f4b7bcbee 100644 --- a/spicy/lib/spicy_rt.hlt +++ b/spicy/lib/spicy_rt.hlt @@ -1,6 +1,7 @@ module spicy_rt { public type ParseError = exception &cxxname="spicy::rt::ParseError"; +public type Backtrack = exception &cxxname="spicy::rt::Backtrack"; public type UnitAlreadyConnected = exception &cxxname="spicy::rt::UnitAlreadyConnected"; # State stored inside a unit to allow connecting it to a sink. @@ -67,6 +68,8 @@ declare public bool waitForEod(inout value_ref data, view cur, i declare public bool atEod(inout value_ref data, view cur) &cxxname="spicy::rt::detail::atEod" &have_prototype; declare public bool haveEod(inout value_ref data, view cur) &cxxname="spicy::rt::detail::haveEod" &have_prototype; +declare public void backtrack() &cxxname="spicy::rt::detail::backtrack" &have_prototype; + public type BitOrder = enum { LSB0, MSB0 } &cxxname="hilti::rt::integer::BitOrder"; # TODO: Should accept BitOrder instead of enum<*> here. declare public uint<*> extractBits(uint<*> v, uint<64> lower, uint<64> upper, enum<*> order) &cxxname="hilti::rt::integer::bits" &have_prototype; diff --git a/spicy/runtime/include/parser.h b/spicy/runtime/include/parser.h index 57f098378..cb87528a6 100644 --- a/spicy/runtime/include/parser.h +++ b/spicy/runtime/include/parser.h @@ -178,6 +178,17 @@ class ParseError : public hilti::rt::UserException { virtual ~ParseError(); /* required to create vtable, see hilti::rt::Exception */ }; +/** + * Exception triggering backtracking to the most recent ``&try``. Derived from + * ``ParseError`` so that if it's not caught, it turns into a regular parsing + * error. + */ +class Backtrack : public ParseError { +public: + Backtrack() : ParseError("backtracking outside of &try scope") {} + virtual ~Backtrack(); +}; + namespace detail { /** @@ -320,5 +331,11 @@ extern bool atEod(const hilti::rt::ValueReference& data, cons * @return true if end-of-data has been seen, but not necessarily reached. */ extern bool haveEod(const hilti::rt::ValueReference& data, const hilti::rt::stream::View& cur); + +/** + * Manually trigger a backtrack operation, reverting back to the most revent &try. + */ +inline void backtrack() { throw Backtrack(); } + } // namespace detail } // namespace spicy::rt diff --git a/spicy/runtime/src/parser.cc b/spicy/runtime/src/parser.cc index 73af1ce9e..0c282c960 100644 --- a/spicy/runtime/src/parser.cc +++ b/spicy/runtime/src/parser.cc @@ -11,6 +11,7 @@ using namespace spicy::rt; using namespace spicy::rt::detail; +HILTI_EXCEPTION_IMPL(Backtrack) HILTI_EXCEPTION_IMPL(ParseError) void detail::printParserState(const std::string& unit_id, const hilti::rt::ValueReference& data, diff --git a/spicy/toolchain/include/ast/operators/unit.h b/spicy/toolchain/include/ast/operators/unit.h index 65a454673..e42c6c131 100644 --- a/spicy/toolchain/include/ast/operators/unit.h +++ b/spicy/toolchain/include/ast/operators/unit.h @@ -116,4 +116,18 @@ this method will not do anything. } END_METHOD +BEGIN_METHOD(unit, Backtrack) + auto signature() const { + return hilti::operator_::Signature{.self = hilti::type::constant(spicy::type::Unit(type::Wildcard())), + .result = hilti::type::Void(), + .id = "backtrack", + .args = {}, + .doc = R"( +Aborts parsing at the current position and returns back to the most recent +``&try`` attribute. Turns into a parse error if there's no ``&try`` in scope. +)"}; + } +END_METHOD + + } // namespace spicy::operator_ diff --git a/spicy/toolchain/include/ast/types/unit-items/switch.h b/spicy/toolchain/include/ast/types/unit-items/switch.h index 27a744860..6ac614b9f 100644 --- a/spicy/toolchain/include/ast/types/unit-items/switch.h +++ b/spicy/toolchain/include/ast/types/unit-items/switch.h @@ -68,10 +68,7 @@ class Switch : public hilti::NodeBase, public spicy::trait::isUnitItem { _hooks_start(_cases_end), _hooks_end(-1) {} - auto expression() const { - return childs()[0].tryAs(); - ; - } + auto expression() const { return childs()[0].tryReferenceAs(); } Engine engine() const { return _engine; } auto condition() const { return childs()[1].tryReferenceAs(); } auto cases() const { return childs(_cases_start, _cases_end); } diff --git a/spicy/toolchain/include/compiler/detail/codegen/parser-builder.h b/spicy/toolchain/include/compiler/detail/codegen/parser-builder.h index c4e14fd47..e2f62288d 100644 --- a/spicy/toolchain/include/compiler/detail/codegen/parser-builder.h +++ b/spicy/toolchain/include/compiler/detail/codegen/parser-builder.h @@ -405,6 +405,12 @@ class ParserBuilder { */ void finalizeUnit(bool success, const Location& l); + /** Prepare for backtracking via ``&try``. */ + void initBacktracking(); + + /** Clean up after potential backtracking via ``&try``. */ + void finishBacktracking(); + CodeGen* cg() const { return _cg; } const std::shared_ptr& context() const; const hilti::Options& options() const; diff --git a/spicy/toolchain/src/compiler/codegen/codegen.cc b/spicy/toolchain/src/compiler/codegen/codegen.cc index b58439ef8..c5d3c2e8c 100644 --- a/spicy/toolchain/src/compiler/codegen/codegen.cc +++ b/spicy/toolchain/src/compiler/codegen/codegen.cc @@ -192,6 +192,11 @@ struct VisitorPassIterate : public hilti::visitor::PreOrderaddAssign(builder::index(__offsets, *field->index()), builder::tuple({cur_offset, builder::optional(hilti::type::UnsignedInteger(64))})); } + + if ( auto a = AttributeSet::find(field->attributes(), "&try") ) + pb->initBacktracking(); } } @@ -535,6 +538,9 @@ struct ProductionVisitor } } else { + if ( auto a = AttributeSet::find(field->attributes(), "&try") ) + pb->finishBacktracking(); + // We are the field's owner, record offsets and post-process the various attributes. if ( pb->options().getAuxOption("spicy.track_offsets", false) ) { assert(field->index()); @@ -1412,3 +1418,21 @@ void ParserBuilder::consumeLookAhead(std::optional dst) { builder()->addAssign(state().lahead, look_ahead::None); advanceInput(state().lahead_end); } + +void ParserBuilder::initBacktracking() { + auto try_cur = builder()->addTmp("try_cur", state().cur); + auto [body, try_] = builder()->addTry(); + auto catch_ = try_.addCatch(builder::parameter(ID("e"), builder::typeByID("spicy_rt::Backtrack"))); + pushBuilder(catch_, [&]() { builder()->addAssign(state().cur, try_cur); }); + + auto pstate = state(); + pstate.trim = builder::bool_(false); + pushState(std::move(pstate)); + pushBuilder(body); +} + +void ParserBuilder::finishBacktracking() { + popBuilder(); + popState(); + trimInput(); +} diff --git a/spicy/toolchain/src/compiler/parser/scanner.ll b/spicy/toolchain/src/compiler/parser/scanner.ll index 032bcab81..7197d6049 100644 --- a/spicy/toolchain/src/compiler/parser/scanner.ll +++ b/spicy/toolchain/src/compiler/parser/scanner.ll @@ -48,7 +48,7 @@ static std::string expandEscapes(Driver* driver, std::string s, spicy::detail::p address4 ({digits}"."){3}{digits} address6 ("["({hexs}:){7}{hexs}"]")|("["0x{hexs}({hexs}|:)*"::"({hexs}|:)*"]")|("["({hexs}|:)*"::"({hexs}|:)*"]")|("["({hexs}|:)*"::"({hexs}|:)*({digits}"."){3}{digits}"]") -attribute \&(bit-order|byte-order|chunked|convert|count|cxxname|default|eod|internal|ipv4|ipv6|length|no-emit|nosub|on-heap|optional|originator|parse-at|parse-from|priority|requires|responder|size|static|synchronize|transient|type|until|until-including|while|have_prototype) +attribute \&(bit-order|byte-order|chunked|convert|count|cxxname|default|eod|internal|ipv4|ipv6|length|no-emit|nosub|on-heap|optional|originator|parse-at|parse-from|priority|requires|responder|size|static|synchronize|transient|try|type|until|until-including|while|have_prototype) blank [ \t] comment [ \t]*#[^\n]*\n? digit [0-9] diff --git a/tests/Baseline/spicy.types.unit.backtrack-lah/output b/tests/Baseline/spicy.types.unit.backtrack-lah/output new file mode 100644 index 000000000..202c29da1 --- /dev/null +++ b/tests/Baseline/spicy.types.unit.backtrack-lah/output @@ -0,0 +1,5 @@ +[$a=[$payload=b"AAA"], $b=(not set), $c=(not set)] +[$a=(not set), $b=[$payload=b"BBB"], $c=(not set)] +[$a=(not set), $b=(not set), $c=[$payload=b"CCC"]] +[$a=(not set), $b=(not set), $c=(not set)] +could not parse: 'ddd DDD' diff --git a/tests/Baseline/spicy.types.unit.backtrack-on-error/output b/tests/Baseline/spicy.types.unit.backtrack-on-error/output new file mode 100644 index 000000000..43b1edc10 --- /dev/null +++ b/tests/Baseline/spicy.types.unit.backtrack-on-error/output @@ -0,0 +1,6 @@ +Foo.a, [$a=1, $b=(not set), $c=(not set)] +Error, backtracking +Bar.a, [$a=1, $b=(not set), $c=(not set)] +Bar.b, [$a=1, $b=2, $c=(not set)] +Bar.c, [$a=1, $b=2, $c=3] +[$a=b"1234", $foo=[$a=1, $b=(not set), $c=(not set)], $bar=[$a=1, $b=2, $c=3], $b=b"567890"] diff --git a/tests/Baseline/spicy.types.unit.backtrack/output b/tests/Baseline/spicy.types.unit.backtrack/output new file mode 100644 index 000000000..ec6b9a176 --- /dev/null +++ b/tests/Baseline/spicy.types.unit.backtrack/output @@ -0,0 +1,6 @@ +Foo.a, [$a=1, $b=(not set), $c=(not set)] +Backtracking +Bar.a, [$a=1, $b=(not set), $c=(not set)] +Bar.b, [$a=1, $b=2, $c=(not set)] +Bar.c, [$a=1, $b=2, $c=3] +[$a=b"1234", $foo=[$a=1, $b=2, $c=(not set)], $bar=[$a=1, $b=2, $c=3], $b=b"567890"] diff --git a/tests/spicy/types/unit/backtrack-lah.spicy b/tests/spicy/types/unit/backtrack-lah.spicy new file mode 100644 index 000000000..687fa45d3 --- /dev/null +++ b/tests/spicy/types/unit/backtrack-lah.spicy @@ -0,0 +1,56 @@ +# @TEST-EXEC: spicyc -j -o a.hlto %INPUT +# @TEST-EXEC: cat input.dat | spicy-driver a.hlto >>output +# @TEST-EXEC: btest-diff output + +module Mini; + +public type test = unit { + : LineWithFallback[] &eod; +}; + +type LineWithFallback = unit { + : Line &try { + print $$; + } + : bytes &until=b"\n" { + if ( |$$| > 0 ) + print "could not parse: '%s'" % $$; + } +}; + +type Line = unit { + switch { + -> a: A; + -> b: B; + -> c: C; + }; + + on %error { + self.backtrack(); + } +}; + +type A = unit { + : /aaa/; + : / +/; + payload: /[A-Z]+/; +}; + +type B = unit { + : /bbb/; + : / +/; + payload: /[A-Z]+/; +}; + +type C = unit { + : /ccc/; + : / +/; + payload: /[A-Z]+/; +}; + +@TEST-START-FILE input.dat +aaa AAA +bbb BBB +ccc CCC +ddd DDD + @TEST-END-FILE diff --git a/tests/spicy/types/unit/backtrack-on-error.spicy b/tests/spicy/types/unit/backtrack-on-error.spicy new file mode 100644 index 000000000..3789d281e --- /dev/null +++ b/tests/spicy/types/unit/backtrack-on-error.spicy @@ -0,0 +1,30 @@ +# @TEST-EXEC: printf '1234\001\002\003567890' | spicy-driver %INPUT >output +# @TEST-EXEC: btest-diff output + +module Mini; + +public type test = unit { + a: bytes &size=4; + foo: Foo &try; + bar: Bar; + b: bytes &size=6; + + on %done { print self; } +}; + +type Foo = unit { + a: int8 { print "Foo.a", self; } + b: b"XXX"; + c: int8 { print "Foo.c", self; } + + on %error { + print "Error, backtracking"; + self.backtrack(); + } +}; + +type Bar = unit { + a: int8 { print "Bar.a", self; } + b: int8 { print "Bar.b", self; } + c: int8 { print "Bar.c", self; } +}; diff --git a/tests/spicy/types/unit/backtrack.spicy b/tests/spicy/types/unit/backtrack.spicy new file mode 100644 index 000000000..3915e32e5 --- /dev/null +++ b/tests/spicy/types/unit/backtrack.spicy @@ -0,0 +1,25 @@ +# @TEST-EXEC: printf '1234\001\002\003567890' | spicy-driver %INPUT >output +# @TEST-EXEC: btest-diff output + +module Mini; + +public type test = unit { + a: bytes &size=4; + foo: Foo &try; + bar: Bar; + b: bytes &size=6; + + on %done { print self; } +}; + +type Foo = unit { + a: int8 { print "Foo.a", self; } + b: int8 { print "Backtracking"; self.backtrack(); } + c: int8 { print "Foo.c", self; } +}; + +type Bar = unit { + a: int8 { print "Bar.a", self; } + b: int8 { print "Bar.b", self; } + c: int8 { print "Bar.c", self; } +}; diff --git a/tests/spicy/types/vector/parse-eod.spicy b/tests/spicy/types/vector/parse-eod.spicy index f75f328c2..9d82e22d2 100644 --- a/tests/spicy/types/vector/parse-eod.spicy +++ b/tests/spicy/types/vector/parse-eod.spicy @@ -8,4 +8,4 @@ public type Foo = unit { b: int8[] &eod; # Won't get anything here anymore on %done { print self; } - }; +}; From 2038055e7a5d567fdf2328ad20dffb65d5ff8d19 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 2 Dec 2020 12:00:25 +0000 Subject: [PATCH 3/3] Fix Sphinx helper script that emits command lines. When an example contained multiple commands separated by semicolons, we'd execute them separately but still include all of them into each of the command lines emitted into the docs. (Couple of doc outputs show slightly changed content, including updates into this commit.) --- .../examples/_parse-backtrack.spicy.output | 4 ++-- doc/programming/examples/_parse-if.spicy.output | 4 ++-- .../examples/_parse-random-access.spicy.output | Bin 266 -> 278 bytes .../examples/_parse-switch-lhead-2.spicy.output | 4 ++-- .../examples/_parse-switch.spicy.output | 4 ++-- .../examples/_parse-unit-params.spicy.output | 2 +- .../examples/_unit-params.spicy.output | 2 +- doc/scripts/spicy.py | 10 ++++++++-- 8 files changed, 18 insertions(+), 12 deletions(-) diff --git a/doc/programming/examples/_parse-backtrack.spicy.output b/doc/programming/examples/_parse-backtrack.spicy.output index 4d95477ca..5246eee5a 100644 --- a/doc/programming/examples/_parse-backtrack.spicy.output +++ b/doc/programming/examples/_parse-backtrack.spicy.output @@ -1,6 +1,6 @@ # Automatically generated; do not edit. -- printf '\001\002\003\004' | spicy-driver %INPUT; printf '\003\004' | spicy-driver %INPUT/printf '\001\002\003\004' | spicy-driver %INPUT; printf '\003\004' | spicy-driver %INPUT/False -# printf '\001\002\003\004' | spicy-driver backtrack.spicy; printf '\003\004' | spicy-driver backtrack.spicy +# printf '\001\002\003\004' | spicy-driver backtrack.spicy [$foo=[$a=1, $b=2], $bar=[$a=3, $b=4]] -# printf '\001\002\003\004' | spicy-driver backtrack.spicy; printf '\003\004' | spicy-driver backtrack.spicy +# printf '\003\004' | spicy-driver backtrack.spicy [$foo=[$a=3, $b=(not set)], $bar=[$a=3, $b=4]] diff --git a/doc/programming/examples/_parse-if.spicy.output b/doc/programming/examples/_parse-if.spicy.output index dd492b13b..43797fac6 100644 --- a/doc/programming/examples/_parse-if.spicy.output +++ b/doc/programming/examples/_parse-if.spicy.output @@ -1,6 +1,6 @@ # Automatically generated; do not edit. -- printf '\01\02\03\04' | spicy-driver %INPUT; printf '\02\02\03\04' | spicy-driver %INPUT/printf '\01\02\03\04' | spicy-driver %INPUT; printf '\02\02\03\04' | spicy-driver %INPUT/False -# printf '\01\02\03\04' | spicy-driver foo.spicy; printf '\02\02\03\04' | spicy-driver foo.spicy +# printf '\01\02\03\04' | spicy-driver foo.spicy [$a=1, $b=2, $c=(not set), $d=3] -# printf '\01\02\03\04' | spicy-driver foo.spicy; printf '\02\02\03\04' | spicy-driver foo.spicy +# printf '\02\02\03\04' | spicy-driver foo.spicy [$a=2, $b=(not set), $c=2, $d=3] diff --git a/doc/programming/examples/_parse-random-access.spicy.output b/doc/programming/examples/_parse-random-access.spicy.output index 8af340a5682feac7a0809b448a8e872c16e7d069..47ee6266b9dbd743021861db5b256018176ad48b 100644 GIT binary patch delta 28 acmeBTn#Qz&iBTk`!oUCt8!FkwasdEw3kQS% delta 16 XcmbQn)Wx)ciIIbWfq_xUE|v=b8|MP3 diff --git a/doc/programming/examples/_parse-switch-lhead-2.spicy.output b/doc/programming/examples/_parse-switch-lhead-2.spicy.output index 43f529d99..255e8c330 100644 --- a/doc/programming/examples/_parse-switch-lhead-2.spicy.output +++ b/doc/programming/examples/_parse-switch-lhead-2.spicy.output @@ -1,6 +1,6 @@ # Automatically generated; do not edit. -- printf 'A ' | spicy-driver %INPUT; printf '\377\377' | spicy-driver %INPUT/printf 'A ' | spicy-driver %INPUT; printf '\377\377' | spicy-driver %INPUT/False -# printf 'A ' | spicy-driver foo.spicy; printf '\377\377' | spicy-driver foo.spicy +# printf 'A ' | spicy-driver foo.spicy [$a=[$a=b"A"], $b=(not set)] -# printf 'A ' | spicy-driver foo.spicy; printf '\377\377' | spicy-driver foo.spicy +# printf '\377\377' | spicy-driver foo.spicy [$a=(not set), $b=[$b=65535]] diff --git a/doc/programming/examples/_parse-switch.spicy.output b/doc/programming/examples/_parse-switch.spicy.output index 25b125c16..42a5926d9 100644 --- a/doc/programming/examples/_parse-switch.spicy.output +++ b/doc/programming/examples/_parse-switch.spicy.output @@ -1,6 +1,6 @@ # Automatically generated; do not edit. -- printf 'A\01' | spicy-driver %INPUT; printf 'B\01\02' | spicy-driver %INPUT/printf 'A\01' | spicy-driver %INPUT; printf 'B\01\02' | spicy-driver %INPUT/False -# printf 'A\01' | spicy-driver foo.spicy; printf 'B\01\02' | spicy-driver foo.spicy +# printf 'A\01' | spicy-driver foo.spicy [$x=b"A", $a8=1, $a16=(not set), $a32=(not set)] -# printf 'A\01' | spicy-driver foo.spicy; printf 'B\01\02' | spicy-driver foo.spicy +# printf 'B\01\02' | spicy-driver foo.spicy [$x=b"B", $a8=(not set), $a16=258, $a32=(not set)] diff --git a/doc/programming/examples/_parse-unit-params.spicy.output b/doc/programming/examples/_parse-unit-params.spicy.output index 43f0acfd0..785825e8d 100644 --- a/doc/programming/examples/_parse-unit-params.spicy.output +++ b/doc/programming/examples/_parse-unit-params.spicy.output @@ -1,4 +1,4 @@ # Automatically generated; do not edit. -- printf '\01\02' | spicy-driver %INPUT/printf '\01\02' | spicy-driver %INPUT/False # printf '\01\02' | spicy-driver foo.spicy -"Spicy": 1 +Spicy: 1 [$y=[$x=1]] diff --git a/doc/programming/examples/_unit-params.spicy.output b/doc/programming/examples/_unit-params.spicy.output index 5d37bcfbf..d44bee7ca 100644 --- a/doc/programming/examples/_unit-params.spicy.output +++ b/doc/programming/examples/_unit-params.spicy.output @@ -1,3 +1,3 @@ # Automatically generated; do not edit. -- printf '\05' | spicy-driver %INPUT/printf '\05' | spicy-driver %INPUT/False # printf '\05' | spicy-driver foo.spicy -"My multiplied integer": 25 +My multiplied integer: 25 diff --git a/doc/scripts/spicy.py b/doc/scripts/spicy.py index fe64c5480..44af17874 100644 --- a/doc/scripts/spicy.py +++ b/doc/scripts/spicy.py @@ -384,6 +384,10 @@ def update(self, source, destination, cmd): all_good = True first = True + show_as = [] + if self.show_as: + show_as = self.show_as.split(";") + for one_cmd in cmd.split(";"): one_cmd = one_cmd.strip() @@ -418,12 +422,14 @@ def update(self, source, destination, cmd): out = open(destination, "ab") out.write(b"\n") - if self.show_as: - one_cmd = "# %s\n" % self.show_as + if show_as: + one_cmd = "# %s\n" % show_as[0].strip() one_cmd = one_cmd.replace("%INPUT", self.show_with) output = output.replace( source.encode(), self.show_with.encode()) out.write(one_cmd.encode()) + show_as = show_as[1:] + out.write(output) out.close() first = False