Patch to make dypgen compile with latest versions of ocaml

[mlemerre]

Hi,

Dypgen requires an old version of OCaml mainly because of the strings <-> bytes incompatibility. I made a quick patch that allows it to compile (attached).

But as it seems to be unmaintained, I am not sure where to send this patch to; I send it here because this is the Maintainer field for the opam package.

Is there a procedure in this kind of situation?

diff -u -r /tmp/dypgen/dypgen/dypgen.ml ./dypgen/dypgen.ml
--- /tmp/dypgen/dypgen/dypgen.ml    2011-11-25 21:37:13.000000000 +0100
+++ ./dypgen/dypgen.ml  2023-09-19 11:03:59.568726196 +0200
@@ -1620,12 +1620,12 @@

-let () = Insert_linenum.buffer := String.copy parser_code
+let () = Insert_linenum.buffer := Bytes.of_string parser_code
 let lexbuf = Lexing.from_string parser_code
 let parser_code = Insert_linenum.insert_linenum lexbuf

 let dest_file = open_out temp_output_file
-let () = output_string dest_file parser_code
+let () = output_bytes dest_file parser_code
 let () = close_out dest_file

@@ -1689,15 +1689,15 @@
     (List.fold_left aux "" non_terminal_start_list)^
     mli_code
   in
-  Insert_linenum.buffer := String.copy parser_code_mli;
+  Insert_linenum.buffer := Bytes.of_string parser_code_mli;
   let lexbuf = Lexing.from_string parser_code_mli in
   let parser_code_mli = Insert_linenum.insert_linenum lexbuf in
   let dest_file_mli = open_out output_file_mli in
-  output_string dest_file_mli parser_code_mli;
+  output_bytes dest_file_mli parser_code_mli;
   close_out dest_file_mli;
-  let lexbuf = Lexing.from_string parser_code in
+  let lexbuf = Lexing.from_string (Bytes.to_string parser_code) in
   (try Insert_linenum.replace_filename parser_code input_file_short lexbuf
   with Failure _ -> (error_regexp (input_file_short^".ml.temp"); exit 2));
   let dest_file = open_out output_file in
-  output_string dest_file parser_code;
+  output_bytes dest_file parser_code;
   close_out dest_file)
diff -u -r /tmp/dypgen/dypgen/dypgen_lexer.mll ./dypgen/dypgen_lexer.mll
--- /tmp/dypgen/dypgen/dypgen_lexer.mll 2010-06-24 22:57:53.000000000 +0200
+++ ./dypgen/dypgen_lexer.mll   2023-09-19 10:59:41.463143015 +0200
@@ -56,7 +56,7 @@
   | 'r' -> '\r'
   | c   -> c

-let string_of_char c = let x = " " in x.[0] <- c; x
+let string_of_char c = String.make 1 c

 }

diff -u -r /tmp/dypgen/dypgen/extract_type.mll ./dypgen/extract_type.mll
--- /tmp/dypgen/dypgen/extract_type.mll 2010-06-24 22:57:53.000000000 +0200
+++ ./dypgen/extract_type.mll   2023-09-19 10:58:21.106650076 +0200
@@ -121,10 +121,10 @@
     let len = String.length r in
     let s = String.sub r 1 (len-2) in
     if s = oldtp then
-      (let s = "'"^newtp^" " in
-      let len2 = String.length s in
+      (let s = Bytes.of_string ("'"^newtp^" ") in
+      let len2 = Bytes.length s in
       s.[len2-1] <- r.[len-1];
-      Buffer.add_string string_buf s)
+      Buffer.add_bytes string_buf s)
     else
       Buffer.add_string string_buf r;
     replace_tpar oldtp newtp lexbuf }
@@ -137,12 +137,14 @@
   | [^'_''['] * eof { () }
   | "_[" ['<''>']
     { let i = Lexing.lexeme_start lexbuf in
+      let fun_typ = Bytes.of_string fun_typ in
     fun_typ.[i] <- ' '; fun_typ.[i+2] <- ' ';
-    fix_variant fun_typ lexbuf}
+    fix_variant (Bytes.unsafe_to_string fun_typ) lexbuf}
   | "[" ['<''>']
     { let i = Lexing.lexeme_start lexbuf in
+      let fun_typ = Bytes.of_string fun_typ in
     fun_typ.[i+1] <- ' ';
-    fix_variant fun_typ lexbuf}
+    fix_variant (Bytes.unsafe_to_string fun_typ) lexbuf}
   | [^'_''['] + { fix_variant fun_typ lexbuf }
   | ['_''['] { fix_variant fun_typ lexbuf }

diff -u -r /tmp/dypgen/dypgen/insert_linenum.mll ./dypgen/insert_linenum.mll
--- /tmp/dypgen/dypgen/insert_linenum.mll   2010-06-24 22:57:53.000000000 +0200
+++ ./dypgen/insert_linenum.mll 2023-09-19 11:01:40.679874301 +0200
@@ -1,7 +1,7 @@
 {
 open Lexing

-let buffer = ref ""
+let buffer = ref (Bytes.empty)

 let update_loc lexbuf file line absolute chars =
   let pos = lexbuf.lex_curr_p in
@@ -24,12 +24,12 @@
       insert_linenum lexbuf }
   | "# insert-line-number"
       { let pos = Lexing.lexeme_start_p lexbuf in
-      let space = String.make 20 ' ' in
+      let space = Bytes.make 20 ' ' in
       let str = "# "^(string_of_int (pos.pos_lnum+1)) in
       String.blit str 0 space 0 (String.length str);
-      String.blit space 0 !buffer pos.pos_cnum 20;
+      String.blit (Bytes.unsafe_to_string space) 0 !buffer pos.pos_cnum 20;
       insert_linenum lexbuf }
-  | eof { let result = !buffer in buffer := ""; result }
+  | eof { let result = !buffer in buffer := Bytes.empty; result }
   | [^'#''\010''\013']+ { insert_linenum lexbuf }
   | _ { insert_linenum lexbuf }

diff -u -r /tmp/dypgen/dyplib/Makefile ./dyplib/Makefile
--- /tmp/dypgen/dyplib/Makefile 2011-11-25 15:26:27.000000000 +0100
+++ ./dyplib/Makefile   2023-09-19 10:54:33.853256015 +0200
@@ -4,7 +4,7 @@
 CAMLC=ocamlc
 OCAMLFIND=ocamlfind

-all: dyp opt prof
+all: dyp opt

 dyp: dyp.cmi priority_by_relation.cmo automaton.cmo dyplex.cmo dyp.cmo
    $(CAMLC) -a -o dyp.cma priority_by_relation.cmo automaton.cmo dyplex.cmo dyp.cmo
diff -u -r /tmp/dypgen/dyplib/dyp.ml ./dyplib/dyp.ml
--- /tmp/dypgen/dyplib/dyp.ml   2012-06-19 00:05:08.000000000 +0200
+++ ./dyplib/dyp.ml 2023-09-19 10:51:24.448094205 +0200
@@ -4572,7 +4572,7 @@

-let do_shifts tok_name tok_value prevTops lexbuf counters ppar layout lexeme next_lexeme_precursor =
+let do_shifts tok_name tok_value prevTops lexbuf counters ppar layout (lexeme:string) next_lexeme_precursor =
   let layout_flags = if layout then 3 else 2 in
   let pt =
     if !dypgen_verbose>2 then
@@ -5041,7 +5041,7 @@
   let old_start_pos = lexbuf.lb_lexbuf.lex_start_pos in
   (* ^ start might move at position 0 if end of buffer is reached *)
   (*Printf.fprintf !log_channel "first_curr_pos=%d\n" first_curr_pos;*)
-  let rec next_token res =
+  let rec next_token (res:string list) =
     let prev_curr_pos = lexbuf.lb_lexbuf.lex_curr_pos in
     let prev_start_pos = lexbuf.lb_lexbuf.lex_start_pos in
     (*Printf.fprintf !log_channel "prev_curr_pos=%d\n" prev_curr_pos;*)
@@ -5066,17 +5066,17 @@
         res
     | None, (Some ((action_id_l, pdev, _),_)) ->
       let lexeme =
-        try String.sub lexbuf.lb_lexbuf.lex_buffer prev_curr_pos len
+        try Bytes.sub lexbuf.lb_lexbuf.lex_buffer prev_curr_pos len
         with Invalid_argument("String.sub") ->
-          (Printf.printf "1; %s\n" lexbuf.lb_lexbuf.lex_buffer;
+          (Printf.printf "1; %s\n" (Bytes.unsafe_to_string lexbuf.lb_lexbuf.lex_buffer);
           raise (Invalid_argument("String.sub")))
       in
        (if !dypgen_verbose>2 then
          output_string !log_channel "next_lexeme called bis\n";
-       next_token (lexeme::res))
+       next_token ((Bytes.unsafe_to_string lexeme)::res))
     | (Some ((action_id_l, pdev, _),_)), _ ->
       let lexeme =
-        try String.sub lexbuf.lb_lexbuf.lex_buffer prev_curr_pos len
+        try Bytes.sub lexbuf.lb_lexbuf.lex_buffer prev_curr_pos len
         with Invalid_argument("String.sub") ->
           (Printf.printf "2; %i, %i\n" prev_curr_pos len;
           raise (Invalid_argument("String.sub")))
@@ -5084,7 +5084,7 @@
         (lexbuf.lb_lexbuf.lex_curr_pos <- first_curr_pos;
         (*Printf.printf "next_lexeme, lex_curr_pos=%d, lex_start_pos=%d, lex_abs_pos=%d\n"
         lexbuf.lb_lexbuf.lex_curr_pos lexbuf.lb_lexbuf.lex_start_pos lexbuf.lb_lexbuf.lex_abs_pos*)
-          lexeme::res)
+          (Bytes.unsafe_to_string lexeme)::res)
   in
   next_token []
   (*let l = next_token [] in
diff -u -r /tmp/dypgen/dyplib/dyp.mli ./dyplib/dyp.mli
--- /tmp/dypgen/dyplib/dyp.mli  2012-03-14 07:33:43.000000000 +0100
+++ ./dyplib/dyp.mli    2023-09-19 10:53:13.556763468 +0200
@@ -215,7 +215,7 @@

 val from_function :
   ('token,'obj,'global_data,'local_data,'lexbuf) parser_pilot ->
-  (string -> int -> int) ->
+  (bytes -> int -> int) ->
   'obj dyplexbuf

 val dyplex_lexbuf_position :
diff -u -r /tmp/dypgen/dyplib/dyplex.ml ./dyplib/dyplex.ml
--- /tmp/dypgen/dyplib/dyplex.ml    2012-04-19 11:36:11.000000000 +0200
+++ ./dyplib/dyplex.ml  2023-09-19 10:42:54.528966155 +0200
@@ -587,13 +587,13 @@
         lexbuf.lex_curr_pos <- p+1;
         if !dypgen_verbose>4 then
           (Printf.fprintf !log_channel
-          "lex_engine reads: `%c'\n" lexbuf.lex_buffer.[p]);
-        try Char.code lexbuf.lex_buffer.[p]
+          "lex_engine reads: `%c'\n" (Bytes.get lexbuf.lex_buffer p));
+        try Char.code (Bytes.get lexbuf.lex_buffer p)
         with Invalid_argument("index out of bounds")
         -> (Printf.printf "%d, %d, %s, %d, %d\n"
          lexbuf.lex_curr_pos lexbuf.lex_buffer_len
          (string_of_bool reset_start_pos)
-         p (String.length lexbuf.lex_buffer);
+         p (Bytes.length lexbuf.lex_buffer);
         raise (Invalid_argument("index out of bounds")))
     in
     let aux_lex (new_state_list,valid_lex) tbl state =

[kit-ty-kate]

cc-ing the original author @manu291 (the upstream repository is at https://github.com/manu291/dypgen)

@mlemerre do you want to take-over dypgen if the original author does not have time to maintain it anymore?

[mseri]

The fix has been sitting in a PR fo 3 years already: https://github.com/manu291/dypgen/pull/2

If the maintainer is unresponsive we could consider a -1 patch release in the meantime. Let’s wait a few days

[mlemerre]

@kit-ty-kate Thanks for the offer! I just wanted to fix @dypgen so that I can make a guix package with it. I probably won't be much more responsive than the original author

I can propose to make a proper patch, by merging how @Sylvain78 and I did it (there remains some TODO in his code, but he did not use unsafe_to_string as much as I did)

[manu291]

I've just merged @Sylvain78 PR.

[kit-ty-kate]

@manu291 Thank you so much! Would it be possible to have a release? If that can help you, we (opam-repository maintainers) can do the release in opam-repository for you but we just need a git tag to hold on to

[manu291]

@kit-ty-kate I've just created tag v-2023-09-20. Thanks for doing the release in opam-repository.

[vzsky]

Wondering if it is possible to install dypgen to ocaml > 4.6. I got package conflict error since the dependency of ocaml < 4.06 is stated.