5.3 support: Versioned keywords (#2623)

Backport the 5.3 changes related to the new `keyword_edition` feature and connect it to OCamlformat's `ocaml_version` option.
This will allow to parse pre-5.3 code that uses `effect` as identifiers.

CHANGES.md

@@ -14,9 +14,14 @@ profile. This started with version 0.26.0.
   This might change the formatting of some functions due to the formatting code
   being completely rewritten.
 
-- Support OCaml 5.3 syntax (#2609, #2610, #2611, #2622, @Julow)
+- Support OCaml 5.3 syntax (#2609, #2610, #2611, #2622, #2623, @Julow)
   This adds support for short functor type arguments syntax and utf8
   identifiers.
+  To format code using the new `effect` syntax, add this option to your
+  `.ocamlformat`:
+  ```
+  ocaml-version = 5.3
+  ```
 
 - Documentation comments are now formatted by default (#2390, @Julow)
   Use the option `parse-docstrings = false` to restore the previous behavior.

lib/Extended_ast.ml

@@ -242,15 +242,18 @@ module Parse = struct
     map fg (normalize_mapper ~ocaml_version ~preserve_beginend)
     @@
     let lexbuf = Lexing.from_string str in
+    let ocaml_version =
+      Some Ocaml_version.(major ocaml_version, minor ocaml_version)
+    in
     Location.init_info lexbuf input_name ;
     match fg with
-    | Structure -> Parse.implementation lexbuf
-    | Signature -> Parse.interface lexbuf
-    | Use_file -> Parse.use_file lexbuf
-    | Core_type -> Parse.core_type lexbuf
-    | Module_type -> Parse.module_type lexbuf
-    | Expression -> Parse.expression lexbuf
-    | Repl_file -> Toplevel_lexer.repl_file lexbuf
+    | Structure -> Parse.implementation ~ocaml_version lexbuf
+    | Signature -> Parse.interface ~ocaml_version lexbuf
+    | Use_file -> Parse.use_file ~ocaml_version lexbuf
+    | Core_type -> Parse.core_type ~ocaml_version lexbuf
+    | Module_type -> Parse.module_type ~ocaml_version lexbuf
+    | Expression -> Parse.expression ~ocaml_version lexbuf
+    | Repl_file -> Toplevel_lexer.repl_file ~ocaml_version lexbuf
     | Documentation ->
         let pos = (Location.curr lexbuf).loc_start in
         let pos = {pos with pos_fname= input_name} in

lib/Parse_with_comments.ml

@@ -84,7 +84,8 @@ let parse ?(disable_w50 = false) ?(disable_deprecated = false) parse fragment
         if Warning.is_deprecated_alert alert && disable_deprecated then false
         else not conf.opr_opts.quiet.v )
       ~f:(fun () ->
-        let ast = parse fragment ~input_name source in
+        let ocaml_version = conf.opr_opts.ocaml_version.v in
+        let ast = parse fragment ~ocaml_version ~input_name source in
         Warnings.check_fatal () ;
         let comments =
           let mk_cmt = function
@@ -102,9 +103,8 @@ let parse ?(disable_w50 = false) ?(disable_deprecated = false) parse fragment
   in
   match List.rev !w50 with [] -> t | w50 -> raise (Warning50 w50)
 
-let parse_ast (conf : Conf.t) fg ~input_name s =
-  let ocaml_version = conf.opr_opts.ocaml_version.v
-  and preserve_beginend = Poly.(conf.fmt_opts.exp_grouping.v = `Preserve) in
+let parse_ast (conf : Conf.t) fg ~ocaml_version ~input_name s =
+  let preserve_beginend = Poly.(conf.fmt_opts.exp_grouping.v = `Preserve) in
   Extended_ast.Parse.ast fg ~ocaml_version ~preserve_beginend ~input_name s
 
 (** [is_repl_block x] returns whether [x] is a list of REPL phrases and

lib/Parse_with_comments.mli

@@ -29,7 +29,11 @@ exception Warning50 of (Location.t * Warnings.t) list
 val parse :
      ?disable_w50:bool
   -> ?disable_deprecated:bool
-  -> ('b -> input_name:string -> string -> 'a)
+  -> (   'b
+      -> ocaml_version:Ocaml_version.t
+      -> input_name:string
+      -> string
+      -> 'a )
   -> 'b
   -> Conf.t
   -> input_name:string
@@ -50,5 +54,10 @@ val parse_toplevel :
     function handles [conf.parse_toplevel_phrases]. *)
 
 val parse_ast :
-  Conf.t -> 'a Extended_ast.t -> input_name:string -> string -> 'a
+     Conf.t
+  -> 'a Extended_ast.t
+  -> ocaml_version:Ocaml_version.t
+  -> input_name:string
+  -> string
+  -> 'a
 (** Argument to {!parse}. *)

lib/Std_ast.ml

@@ -56,16 +56,19 @@ let map (type a) (x : a t) (m : Ast_mapper.mapper) : a -> a =
   | Documentation -> Fn.id
 
 module Parse = struct
-  let ast (type a) (fg : a t) ~input_name str : a =
+  let ast (type a) (fg : a t) ~ocaml_version ~input_name str : a =
     let lexbuf = Lexing.from_string str in
+    let ocaml_version =
+      Some Ocaml_version.(major ocaml_version, minor ocaml_version)
+    in
     Location.init_info lexbuf input_name ;
     match fg with
-    | Structure -> Parse.implementation lexbuf
-    | Signature -> Parse.interface lexbuf
-    | Use_file -> Parse.use_file lexbuf
-    | Core_type -> Parse.core_type lexbuf
-    | Module_type -> Parse.module_type lexbuf
-    | Expression -> Parse.expression lexbuf
+    | Structure -> Parse.implementation ~ocaml_version lexbuf
+    | Signature -> Parse.interface ~ocaml_version lexbuf
+    | Use_file -> Parse.use_file ~ocaml_version lexbuf
+    | Core_type -> Parse.core_type ~ocaml_version lexbuf
+    | Module_type -> Parse.module_type ~ocaml_version lexbuf
+    | Expression -> Parse.expression ~ocaml_version lexbuf
     | Repl_file -> ()
     | Documentation -> ()
 end

lib/Std_ast.mli

@@ -33,7 +33,12 @@ type any_t = Any : 'a t -> any_t [@@unboxed]
 val of_syntax : Syntax.t -> any_t
 
 module Parse : sig
-  val ast : 'a t -> input_name:string -> string -> 'a
+  val ast :
+       'a t
+    -> ocaml_version:Ocaml_version.t
+    -> input_name:string
+    -> string
+    -> 'a
 end
 
 val equal : 'a t -> 'a -> 'a -> bool

lib/Toplevel_lexer.mli

@@ -9,4 +9,7 @@
 (*                                                                        *)
 (**************************************************************************)
 
-val repl_file : Lexing.lexbuf -> Parsetree.repl_phrase list
+val repl_file :
+     ocaml_version:(int * int) option
+  -> Lexing.lexbuf
+  -> Parsetree.repl_phrase list

lib/Toplevel_lexer.mll

@@ -52,7 +52,7 @@ and phrase buf = parse
   | _ as c   { Buffer.add_char buf c; phrase buf lexbuf }
 
 {
-let repl_file lx =
+let repl_file ~ocaml_version lx =
   let x = token lx in
   let open Ocamlformat_parser_extended.Parsetree in
   List.fold_left (fun acc -> function
@@ -61,7 +61,7 @@ let repl_file lx =
           let filename = (Location.curr lx).loc_start.pos_fname in
           Lexing.set_filename cmd_lexbuf filename ;
           Lexing.set_position cmd_lexbuf pos_start ;
-          { prepl_phrase= Parse.toplevel_phrase cmd_lexbuf
+          { prepl_phrase= Parse.toplevel_phrase ~ocaml_version cmd_lexbuf
           ; prepl_output= "" }
           :: acc
       | `Output ("", _) -> acc

test/passing/gen/dune.inc

@@ -3712,6 +3712,21 @@
  (alias runtest)
  (action (diff polytypes.ml.err polytypes.ml.stderr)))
 
+(rule
+ (deps .ocamlformat dune-project)
+ (action
+  (with-stdout-to pre42_syntax.ml.stdout
+   (with-stderr-to pre42_syntax.ml.stderr
+     (run %{bin:ocamlformat} --name pre42_syntax.ml --margin-check --ocaml-version=4.1 %{dep:../tests/pre42_syntax.ml})))))
+
+(rule
+ (alias runtest)
+ (action (diff pre42_syntax.ml.ref pre42_syntax.ml.stdout)))
+
+(rule
+ (alias runtest)
+ (action (diff pre42_syntax.ml.err pre42_syntax.ml.stderr)))
+
 (rule
  (deps .ocamlformat dune-project)
  (action

test/passing/refs.default/pre42_syntax.ml.ref

@@ -0,0 +1,4 @@
+type nonrec = nonrec
+(** The [nonrec] keyword has been added in OCaml 4.2. *)
+
+let nonrec nonrec : nonrec = nonrec

test/passing/refs.janestreet/pre42_syntax.ml.ref

@@ -0,0 +1,4 @@
+(** The [nonrec] keyword has been added in OCaml 4.2. *)
+type nonrec = nonrec
+
+let nonrec nonrec : nonrec = nonrec

test/passing/refs.ocamlformat/pre42_syntax.ml.ref

@@ -0,0 +1,4 @@
+(** The [nonrec] keyword has been added in OCaml 4.2. *)
+type nonrec = nonrec
+
+let nonrec nonrec : nonrec = nonrec

test/passing/tests/pre42_syntax.ml

@@ -0,0 +1,3 @@
+(** The [nonrec] keyword has been added in OCaml 4.2. *)
+type nonrec = nonrec
+let nonrec nonrec : nonrec = nonrec

test/passing/tests/pre42_syntax.ml.opts

@@ -0,0 +1 @@
+--ocaml-version=4.1

tools/printast/printast.ml

@@ -12,7 +12,7 @@ let extended_ast ppf syntax ~input_name content =
 let std_ast ppf syntax ~input_name content =
   let open Std_ast in
   let (Any kind) = of_syntax syntax in
-  Parse.ast kind ~input_name content |> Printast.ast kind ppf
+  Parse.ast kind ~ocaml_version ~input_name content |> Printast.ast kind ppf
 
 let get_arg () =
   let std = ref false and input = ref None in