End_of_file & Ssl.Write_error(5) errors

[Chris00]

When running the code (supposed to be in test.ml)

open Lwt

let mirage_repos = [
  "https://github.com/samoht/alcotest.git";
  "https://github.com/mirage/cowabloga.git";
  "https://github.com/mirage/dyntype.git";
  "https://github.com/mirage/ezjsonm.git";
  "https://github.com/avsm/ezxmlm.git";
  "https://github.com/mirage/io-page.git";
  "https://github.com/mirage/irmin.git";
  "https://github.com/magnuss/jitsu.git";
  "https://github.com/mirage/merge-queues.git";
  "https://github.com/mirage/merge-ropes.git";
  "https://github.com/mirage/mirage.git";
  "https://github.com/mirage/mirage-block-unix.git";
  "https://github.com/mirage/mirage-block-volume.git";
  "https://github.com/mirage/mirage-block-xen.git";
  "https://github.com/magnuss/mirage-bootvar.git";
  "https://github.com/mirage/mirage-bootvar-xen.git";
  "https://github.com/mirage/mirage-channel.git";
  "https://github.com/mirage/mirage-clock.git";
  "https://github.com/mirage/mirage-console.git";
  "https://github.com/mirage/mirage-decks.git";
  "https://github.com/mirage/mirage-entropy.git";
  "https://github.com/mirage/mirage-flow.git";
  "https://github.com/mirage/mirage-fs-unix.git";
  "https://github.com/mirage/mirage-http.git";
  "https://github.com/magnuss/mirage-mimic.git";
  "https://github.com/yomimono/mirage-nat.git";
  "https://github.com/mirage/mirage-net-macosx.git";
  "https://github.com/mirage/mirage-net-unix.git";
  "https://github.com/mirage/mirage-net-xen.git";
  "https://github.com/mirage/mirage-platform.git";
  "https://github.com/mirage/mirage-profile.git";
  "https://github.com/mirage/mirage-seal.git";
  "https://github.com/mirage/mirage-skeleton.git";
  "https://github.com/mirage/mirage-tc.git";
  "https://github.com/mirage/mirage-tcpip.git";
  "https://github.com/talex5/mirage-trace-viewer.git";
  "https://github.com/mirage/mirage-vagrant-vms.git";
  "https://github.com/magnuss/mirage-vnetif.git";
  "https://github.com/mirage/mirage-www.git";
  "https://github.com/mirage/mirage-xen-minios.git";
  "https://github.com/mirleft/ocaml-asn1-combinators.git";
  "https://github.com/mirage/ocaml-cohttp.git";
  "https://github.com/mirage/ocaml-conduit.git";
  "https://github.com/mirage/ocaml-cow.git";
  "https://github.com/mirage/ocaml-crunch.git";
  "https://github.com/mirage/ocaml-cstruct.git";
  "https://github.com/ocamllabs/ocaml-ctypes.git";
  "https://github.com/mirage/ocaml-dns.git";
  "https://github.com/mirage/ocaml-fat.git";
  "https://github.com/mirage/ocaml-git.git";
  "https://github.com/avsm/ocaml-github.git";
  "https://github.com/mirage/ocaml-hex.git";
  "https://github.com/mirage/ocaml-ipaddr.git";
  "https://github.com/mirage/ocaml-lazy-trie.git";
  "https://github.com/mirage/ocaml-magic-mime.git";
  "https://github.com/mirage/ocaml-mbr.git";
  "https://github.com/mirage/ocaml-mstruct.git";
  "https://github.com/mirleft/ocaml-nocrypto.git";
  "https://github.com/mirage/ocaml-pcap.git";
  "https://github.com/mirage/ocaml-tar.git";
  "https://github.com/mirleft/ocaml-tls.git";
  "https://github.com/mirage/ocaml-tuntap.git";
  "https://github.com/mirage/ocaml-uri.git";
  "https://github.com/mirage/ocaml-vchan.git";
  "https://github.com/mirage/ocaml-vmnet.git";
  "https://github.com/mirleft/ocaml-x509.git";
  "https://github.com/mirage/ocaml-xenstore.git";
  "https://github.com/djs55/ocaml-xenstore-clients.git";
  "https://github.com/mirage/shared-memory-ring.git";
  "https://github.com/mirage/synjitsu.git";
  "https://github.com/mirage/xen-disk.git";
]

let pull remote_uri =
  let dir = Filename.basename remote_uri in
  let dir = try Filename.chop_extension dir with _ -> dir in
  let root = Filename.concat "test_git" dir in
  let store = Irmin.basic (module Irmin_unix.Irmin_git.FS)
                          (module Irmin.Contents.String) in
  let config = Irmin_unix.Irmin_git.config ~root ~bare:true () in
  Irmin.create store config Irmin_unix.task >>= fun t ->
  let upstream = Irmin.remote_uri remote_uri in
  catch (fun () -> Irmin.pull_exn (t "Updating") upstream `Update)
        (fun e -> Lwt_io.printlf "Fail pull %s: %s"
                               remote_uri (Printexc.to_string e))

let () =
  Lwt_main.run (Lwt_list.iter_p pull mirage_repos)

(* Local Variables: *)
(* compile-command: "ocamlfind ocamlopt -o test.native -annot -package irmin,irmin.unix -linkpkg test.ml" *)
(* End: *)

for the first time, it finishes without errors. Re-running it generates many errors.

[samoht]

Thanks for the report. The Ssl.error looks suspicious, maybe using ocaml-tls could help:

Conduit_lwt_unix.tls_library := `Native

I'll have a proper look at it this week-end.

[Chris00]

Adding

let () =
  Conduit_lwt_unix.(tls_library := Native)

I get errors like

...
Fail pull https://github.com/mirage/mirage-xen-minios.git: End_of_file
Fail pull https://github.com/magnuss/mirage-bootvar.git: End_of_file
Fail pull https://github.com/mirage/mirage-www.git: Unix.Unix_error(Unix.EPIPE, "write", "")

[samoht]

This is an issue with the smart-HTTP protocol only. A workaround (before I do a proper fix in ocaml-git) is to use the normal git protocol: git://github.com/mirage/mirage-xen-minios.git, etc...

[Chris00]

Changing the schemes to git://..., I get errors like:

Fail pull git://github.com/mirage/ocaml-xenstore.git: Failure("resolution failed: name resolution failed")

Needless to say, the URIs work fine using Git. May it be because of too much parallelism?? If I use a smaller list (say of 30 elements — but not 40 for me), such errors do not occur.

[samoht]

May it be because of too much parallelism

Yes, that's possible that conduit doesn't like too much parallelism while resolving names...

[avsm]

I've got a testcase for this (parallel resolution) that I'll commit shortly.

[samoht]

This is fixed by ocaml-git 1.7.1.

[Chris00]

Cool! Thanks!