Add support for semicolon delimiters to wit-parser (#1212)

* Add support for semicolon delimiters to `wit-parser`

This commit is an implementation of WebAssembly/component-model#142 to
add semicolon delimiters to the WIT text format. The intention of this
change is to be rolled out gradually to minimize ecosystem disruption
(although some is inevitable). The strategy implemented in this commit
is to add support to parse semicolons but optionally do so. Requiring
semicolons is configured via:

* A new `WIT_REQUIRE_SEMICOLONS` environment variable is now read. If
  set to `1` then semicolons are required during parsing.
* A new `SourceMap::set_require_semicolons` is provided to
  programmatically indicate what to do.

When semicolons are not required they are still parsed, but a failure is
not generated if they're not present. This should allow semicolon-using
WIT to coexist with non-semicolon-using WIT for a transitionary period.
After a release or two the default for requiring semicolons will
switch to `true` from the current `false`. That means that
`WIT_REQUIRE_SEMICOLONS=0` can be used to keep old WITs compiling. The
after a few releases of that support for optionally parsing semicolons
will be removed and they'll be unconditionally required.

* Update all in-repo tests to use semicolons

Additionally add environment-variable support to WIT printing to
optionally print semicolons, disabled for now.

crates/wit-component/src/printing.rs

@@ -4,6 +4,9 @@ use std::fmt::{self, Write};
 use std::mem;
 use wit_parser::*;
 
+// NB: keep in sync with `crates/wit-parser/src/ast/lex.rs`
+const PRINT_SEMICOLONS_DEFAULT: bool = false;
+
 /// A utility for printing WebAssembly interface definitions to a string.
 pub struct WitPrinter {
     output: Output,
@@ -14,6 +17,8 @@ pub struct WitPrinter {
 
     // Whether to print doc comments.
     emit_docs: bool,
+
+    print_semicolons: bool,
 }
 
 impl Default for WitPrinter {
@@ -22,6 +27,10 @@ impl Default for WitPrinter {
             output: Default::default(),
             any_items: false,
             emit_docs: true,
+            print_semicolons: match std::env::var("WIT_REQUIRE_SEMICOLONS") {
+                Ok(s) => s == "1",
+                Err(_) => PRINT_SEMICOLONS_DEFAULT,
+            },
         }
     }
 }
@@ -68,6 +77,12 @@ impl WitPrinter {
         Ok(std::mem::take(&mut self.output).into())
     }
 
+    fn print_semicolon(&mut self) {
+        if self.print_semicolons {
+            self.output.push_str(";");
+        }
+    }
+
     fn new_item(&mut self) {
         if self.any_items {
             self.output.push_str("\n");
@@ -106,6 +121,7 @@ impl WitPrinter {
             self.print_name(name);
             self.output.push_str(": ");
             self.print_function(resolve, func)?;
+            self.print_semicolon();
             self.output.push_str("\n");
         }
 
@@ -181,7 +197,9 @@ impl WitPrinter {
                     self.print_name(my_name);
                 }
             }
-            writeln!(&mut self.output, "}}")?;
+            write!(&mut self.output, "}}")?;
+            self.print_semicolon();
+            self.output.push_str("\n");
         }
 
         for id in types_to_declare {
@@ -205,6 +223,7 @@ impl WitPrinter {
         self.output.push_str("resource ");
         self.print_name(ty.name.as_ref().expect("resources must be named"));
         if funcs.is_empty() {
+            self.print_semicolon();
             self.output.push_str("\n");
             return Ok(());
         }
@@ -228,6 +247,7 @@ impl WitPrinter {
                 FunctionKind::Freestanding => unreachable!(),
             }
             self.print_function(resolve, func)?;
+            self.print_semicolon();
             self.output.push_str("\n");
         }
         self.output.push_str("}\n");
@@ -362,6 +382,7 @@ impl WitPrinter {
                     }
                     WorldItem::Function(f) => {
                         self.print_function(resolve, f)?;
+                        self.print_semicolon();
                         self.output.push_str("\n");
                     }
                     // Types are handled separately
@@ -374,6 +395,7 @@ impl WitPrinter {
                     _ => unreachable!(),
                 }
                 self.print_path_to_interface(resolve, *id, cur_pkg)?;
+                self.print_semicolon();
                 self.output.push_str("\n");
             }
         }
@@ -616,6 +638,7 @@ impl WitPrinter {
                             self.print_name(name);
                             self.output.push_str(" = ");
                             self.print_type_name(resolve, inner)?;
+                            self.print_semicolon();
                             self.output.push_str("\n");
                         }
                         None => bail!("unnamed type in document"),
@@ -646,6 +669,7 @@ impl WitPrinter {
                 // own<b>`. By forcing a handle to be printed here it's staying
                 // true to what's in the WIT document.
                 self.print_handle_type(resolve, handle, true)?;
+                self.print_semicolon();
                 self.output.push_str("\n");
 
                 Ok(())
@@ -690,6 +714,7 @@ impl WitPrinter {
             self.print_name(name);
             self.output.push_str(" = ");
             self.print_tuple_type(resolve, tuple)?;
+            self.print_semicolon();
             self.output.push_str("\n");
         }
         Ok(())
@@ -751,6 +776,7 @@ impl WitPrinter {
             self.print_name(name);
             self.output.push_str(" = ");
             self.print_option_type(resolve, payload)?;
+            self.print_semicolon();
             self.output.push_str("\n");
         }
         Ok(())
@@ -767,6 +793,7 @@ impl WitPrinter {
             self.print_name(name);
             self.output.push_str(" = ");
             self.print_result_type(resolve, result)?;
+            self.print_semicolon();
             self.output.push_str("\n");
         }
         Ok(())
@@ -795,7 +822,9 @@ impl WitPrinter {
             self.print_name(name);
             self.output.push_str(" = list<");
             self.print_type_name(resolve, ty)?;
-            self.output.push_str(">\n");
+            self.output.push_str(">");
+            self.print_semicolon();
+            self.output.push_str("\n");
             return Ok(());
         }
 

crates/wit-component/tests/components/adapt-empty-interface/adapt-old.wit

@@ -1,5 +1,5 @@
 world adapt-old {
   import new: interface {
-    thunk-that-is-not-called: func()
+    thunk-that-is-not-called: func();
   }
 }

crates/wit-component/tests/components/adapt-export-default/adapt-old.wit

@@ -1,3 +1,3 @@
 world adapt-old {
-  export entrypoint: func()
+  export entrypoint: func();
 }

crates/wit-component/tests/components/adapt-export-namespaced/adapt-old.wit

@@ -1,7 +1,7 @@
 interface new {
-  entrypoint: func()
+  entrypoint: func();
 }
 
 world adapt-old {
-  export new
+  export new;
 }

crates/wit-component/tests/components/adapt-export-reallocs/adapt-old.wit

@@ -1,6 +1,6 @@
 world adapt-old {
   import new: interface {
-    read: func(amt: u32) -> list<u8>
+    read: func(amt: u32) -> list<u8>;
   }
-  export entrypoint: func(args: list<string>)
+  export entrypoint: func(args: list<string>);
 }

crates/wit-component/tests/components/adapt-export-save-args/adapt-old.wit

@@ -1,3 +1,3 @@
 world adapt-old {
-  export entrypoint: func(nargs: u32)
+  export entrypoint: func(nargs: u32);
 }

crates/wit-component/tests/components/adapt-export-with-post-return/adapt-old.wit

@@ -1,7 +1,7 @@
 interface new {
-  foo: func() -> string
+  foo: func() -> string;
 }
 
 world adapt-old {
-  export new
+  export new;
 }

crates/wit-component/tests/components/adapt-import-only-used-in-adapter/adapt-unused.wit

@@ -1,9 +1,9 @@
 interface adapter-imports {
-  foo: func(x: string)
+  foo: func(x: string);
 }
 
 world adapt-unused {
-  import adapter-imports
+  import adapter-imports;
 
-  export adapter-bar: func(x: string)
+  export adapter-bar: func(x: string);
 }

crates/wit-component/tests/components/adapt-import-only-used-in-adapter/module.wit

@@ -1,6 +1,6 @@
 package foo:foo
 
 world module {
-  import foo: func(x: string)
-  export bar: func()
+  import foo: func(x: string);
+  export bar: func();
 }

crates/wit-component/tests/components/adapt-inject-stack-with-adapt-realloc/adapt-old.wit

@@ -1,5 +1,5 @@
 world adapt-old {
   import new: interface {
-    get-two: func() -> (a: u32, b: u32)
+    get-two: func() -> (a: u32, b: u32);
   }
 }

crates/wit-component/tests/components/adapt-inject-stack-with-realloc-no-state/adapt-old.wit

@@ -1,5 +1,5 @@
 world adapt-old {
   import new: interface {
-    get-two: func() -> (a: u32, b: u32)
+    get-two: func() -> (a: u32, b: u32);
   }
 }

crates/wit-component/tests/components/adapt-inject-stack-with-realloc/adapt-old.wit

@@ -1,5 +1,5 @@
 world adapt-old {
   import new: interface {
-    get-two: func() -> (a: u32, b: u32)
+    get-two: func() -> (a: u32, b: u32);
   }
 }

crates/wit-component/tests/components/adapt-inject-stack-with-reallocing-adapter/adapt-old.wit

@@ -1,5 +1,5 @@
 world adapt-old {
   import new: interface {
-    get-two: func() -> (a: u32, b: u32)
+    get-two: func() -> (a: u32, b: u32);
   }
 }

crates/wit-component/tests/components/adapt-inject-stack/adapt-old.wit

@@ -1,5 +1,5 @@
 world adapt-old {
   import new: interface {
-    get-two: func() -> (a: u32, b: u32)
+    get-two: func() -> (a: u32, b: u32);
   }
 }

crates/wit-component/tests/components/adapt-list-return/adapt-old.wit

@@ -1,5 +1,5 @@
 world adapt-old {
   import new: interface {
-    read: func() -> list<u8>
+    read: func() -> list<u8>;
   }
 }

crates/wit-component/tests/components/adapt-memory-simple/adapt-old.wit

@@ -1,5 +1,5 @@
 world adapt-old {
   import new: interface {
-    log: func(s: string)
+    log: func(s: string);
   }
 }

crates/wit-component/tests/components/adapt-multiple/adapt-foo.wit

@@ -1,8 +1,8 @@
 world adapt-foo {
   import other1: interface {
-    foo: func()
+    foo: func();
   }
   import other2: interface {
-    bar: func()
+    bar: func();
   }
 }

crates/wit-component/tests/components/adapt-preview1/adapt-wasi-snapshot-preview1.wit

@@ -2,11 +2,11 @@
 // to implement the `wasi_snapshot_preview1` interface.
 
 interface my-wasi {
-  random-get: func(size: u32) -> list<u8>
-  proc-exit: func(code: u32)
-  something-not-used: func()
+  random-get: func(size: u32) -> list<u8>;
+  proc-exit: func(code: u32);
+  something-not-used: func();
 }
 
 world adapt-wasi-snapshot-preview1 {
-  import my-wasi
+  import my-wasi;
 }

crates/wit-component/tests/components/adapt-preview1/module.wit

@@ -2,6 +2,6 @@ package foo:foo
 
 world module {
   import foo: interface {
-    foo: func()
+    foo: func();
   }
 }

crates/wit-component/tests/components/adapt-unused/adapt-old.wit

@@ -1,5 +1,5 @@
 world adapt-old {
   import new: interface {
-    log: func(s: string)
+    log: func(s: string);
   }
 }

crates/wit-component/tests/components/bare-funcs/module.wit

@@ -1,7 +1,7 @@
 package foo:foo
 world module {
-  import foo: func()
-  import bar: func() -> string
-  export baz: func()
-  export foo2: func(x: string) -> option<list<u8>>
+  import foo: func();
+  import bar: func() -> string;
+  export baz: func();
+  export foo2: func(x: string) -> option<list<u8>>;
 }

crates/wit-component/tests/components/ensure-default-type-exports/module.wit

@@ -1,17 +1,17 @@
 package foo:foo
 
 interface foo {
-  type foo = u8
+  type foo = u8;
 
   record bar {
       x: foo
   }
 
-  a: func(b: bar)
+  a: func(b: bar);
 }
 
 world module {
-  import foo
+  import foo;
 
-  export a: func(b: u8)
+  export a: func(b: u8);
 }

crates/wit-component/tests/components/error-adapt-missing-memory/adapt-old.wit

@@ -1,5 +1,5 @@
 world adapt-old {
   import new: interface {
-    log: func(s: string)
+    log: func(s: string);
   }
 }

crates/wit-component/tests/components/error-default-export-sig-mismatch/module.wit

@@ -1,5 +1,5 @@
 package foo:foo
 
 world module {
-  export a: func(x: string) -> string
+  export a: func(x: string) -> string;
 }

crates/wit-component/tests/components/error-export-sig-mismatch/module.wit

@@ -2,6 +2,6 @@ package foo:foo
 
 world module {
   export foo: interface {
-    a: func(x: string) -> string
+    a: func(x: string) -> string;
   }
 }

crates/wit-component/tests/components/error-import-resource-rep/module.wit

@@ -1,5 +1,5 @@
 package foo:bar
 
 world module {
-  resource a
+  resource a;
 }

crates/wit-component/tests/components/error-import-resource-wrong-signature/module.wit

@@ -1,5 +1,5 @@
 package foo:bar
 
 world module {
-  resource a
+  resource a;
 }

crates/wit-component/tests/components/error-import-sig-mismatch/module.wit

@@ -2,6 +2,6 @@ package foo:foo
 
 world module {
   import foo: interface {
-    bar: func(s: string)
+    bar: func(s: string);
   }
 }

crates/wit-component/tests/components/error-link-duplicate-symbols/lib-foo.wit

@@ -1,10 +1,10 @@
 package test:test
 
 interface test {
-   foo: func(v: s32) -> s32
+   foo: func(v: s32) -> s32;
 }
 
 world lib-foo {
-    import test
-    export test
+    import test;
+    export test;
 }