[4.14] Infinit memory leak while compiling functor heavy code #10779
Description
When compiling irmin-pack.2.9.0 with the 4.14 branch (3f170f3), the following problem arise:
#=== ERROR while compiling irmin-pack.2.9.0 ===================================#
# context 2.1.1 | linux/x86_64 | ocaml-variants.4.14.0+trunk | file:///home/opam/opam-repository
# path ~/.opam/4.14/.opam-switch/build/irmin-pack.2.9.0
# command ~/.opam/opam-init/hooks/sandbox.sh build dune build -p irmin-pack -j 31
# exit-code 1
# env-file ~/.opam/log/irmin-pack-23-7e89b3.env
# output-file ~/.opam/log/irmin-pack-23-7e89b3.out
### output ###
# ocamlc src/irmin-pack/.irmin_pack.objs/byte/irmin_pack__Ext.{cmo,cmt} (got signal KILL)
# (cd _build/default && /home/opam/.opam/4.14/bin/ocamlc.opt -w -40 -g -bin-annot -I src/irmin-pack/.irmin_pack.objs/byte -I /home/opam/.opam/4.14/lib/angstrom -I /home/opam/.opam/4.14/lib/astring -I /home/opam/.opam/4.14/lib/base64 -I /home/opam/.opam/4.14/lib/bheap -I /home/opam/.opam/4.14/lib/bigarray-compat -I /home/opam/.opam/4.14/lib/bigstringaf -I /home/opam/.opam/4.14/lib/bytes -I /home/opam/.opam/4.14/lib/cmdliner -I /home/opam/.opam/4.14/lib/digestif -I /home/opam/.opam/4.14/lib/either -I /home/opam/.opam/4.14/lib/eqaf -I /home/opam/.opam/4.14/lib/fmt -I /home/opam/.opam/4.14/lib/index -I /home/opam/.opam/4.14/lib/index/unix -I /home/opam/.opam/4.14/lib/irmin -I /home/opam/.opam/4.14/lib/irmin-layers -I /home/opam/.opam/4.14/lib/jsonm -I /home/opam/.opam/4.14/lib/logs -I /home/opam/.opam/4.14/lib/lru -I /home/opam/.opam/4.14/lib/lwt -I /home/opam/.opam/4.14/lib/lwt/unix -I /home/opam/.opam/4.14/lib/mmap -I /home/opam/.opam/4.14/lib/mtime -I /home/opam/.opam/4.14/lib/mtime/os -I /home/opam/.opam/4.14/lib/ocaml-compiler-libs/common -I /home/opam/.opam/4.14/lib/ocaml-compiler-libs/shadow -I /home/opam/.opam/4.14/lib/ocaml/compiler-libs -I /home/opam/.opam/4.14/lib/ocaml/threads -I /home/opam/.opam/4.14/lib/ocamlgraph -I /home/opam/.opam/4.14/lib/ocplib-endian -I /home/opam/.opam/4.14/lib/ocplib-endian/bigstring -I /home/opam/.opam/4.14/lib/optint -I /home/opam/.opam/4.14/lib/ppx_derivers -I /home/opam/.opam/4.14/lib/ppx_irmin -I /home/opam/.opam/4.14/lib/ppx_repr -I /home/opam/.opam/4.14/lib/ppx_repr/lib -I /home/opam/.opam/4.14/lib/ppxlib -I /home/opam/.opam/4.14/lib/ppxlib/ast -I /home/opam/.opam/4.14/lib/ppxlib/astlib -I /home/opam/.opam/4.14/lib/ppxlib/print_diff -I /home/opam/.opam/4.14/lib/ppxlib/stdppx -I /home/opam/.opam/4.14/lib/ppxlib/traverse_builtins -I /home/opam/.opam/4.14/lib/progress -I /home/opam/.opam/4.14/lib/progress/engine -I /home/opam/.opam/4.14/lib/psq -I /home/opam/.opam/4.14/lib/repr -I /home/opam/.opam/4.14/lib/result -I /home/opam/.opam/4.14/lib/semaphore-compat -I /home/opam/.opam/4.14/lib/seq -I /home/opam/.opam/4.14/lib/sexplib0 -I /home/opam/.opam/4.14/lib/stdlib-shims -I /home/opam/.opam/4.14/lib/stringext -I /home/opam/.opam/4.14/lib/terminal -I /home/opam/.opam/4.14/lib/uchar -I /home/opam/.opam/4.14/lib/uri -I /home/opam/.opam/4.14/lib/uucp -I /home/opam/.opam/4.14/lib/uutf -I /home/opam/.opam/4.14/lib/vector -intf-suffix .ml -no-alias-deps -open Irmin_pack__ -o src/irmin-pack/.irmin_pack.objs/byte/irmin_pack__Ext.cmo -c -impl src/irmin-pack/ext.pp.ml)
# ocamlc src/irmin-pack/layered/.irmin_pack_layered.objs/byte/irmin_pack_layered__Ext_layered.{cmo,cmt} (got signal KILL)
# (cd _build/default && /home/opam/.opam/4.14/bin/ocamlc.opt -w -40 -g -bin-annot -I src/irmin-pack/layered/.irmin_pack_layered.objs/byte -I /home/opam/.opam/4.14/lib/angstrom -I /home/opam/.opam/4.14/lib/astring -I /home/opam/.opam/4.14/lib/base64 -I /home/opam/.opam/4.14/lib/bheap -I /home/opam/.opam/4.14/lib/bigarray-compat -I /home/opam/.opam/4.14/lib/bigstringaf -I /home/opam/.opam/4.14/lib/bytes -I /home/opam/.opam/4.14/lib/cmdliner -I /home/opam/.opam/4.14/lib/digestif -I /home/opam/.opam/4.14/lib/either -I /home/opam/.opam/4.14/lib/eqaf -I /home/opam/.opam/4.14/lib/fmt -I /home/opam/.opam/4.14/lib/index -I /home/opam/.opam/4.14/lib/index/unix -I /home/opam/.opam/4.14/lib/irmin -I /home/opam/.opam/4.14/lib/irmin-layers -I /home/opam/.opam/4.14/lib/jsonm -I /home/opam/.opam/4.14/lib/logs -I /home/opam/.opam/4.14/lib/lru -I /home/opam/.opam/4.14/lib/lwt -I /home/opam/.opam/4.14/lib/lwt/unix -I /home/opam/.opam/4.14/lib/mmap -I /home/opam/.opam/4.14/lib/mtime -I /home/opam/.opam/4.14/lib/mtime/os -I /home/opam/.opam/4.14/lib/ocaml-compiler-libs/common -I /home/opam/.opam/4.14/lib/ocaml-compiler-libs/shadow -I /home/opam/.opam/4.14/lib/ocaml/compiler-libs -I /home/opam/.opam/4.14/lib/ocaml/threads -I /home/opam/.opam/4.14/lib/ocamlgraph -I /home/opam/.opam/4.14/lib/ocplib-endian -I /home/opam/.opam/4.14/lib/ocplib-endian/bigstring -I /home/opam/.opam/4.14/lib/optint -I /home/opam/.opam/4.14/lib/ppx_derivers -I /home/opam/.opam/4.14/lib/ppx_irmin -I /home/opam/.opam/4.14/lib/ppx_repr -I /home/opam/.opam/4.14/lib/ppx_repr/lib -I /home/opam/.opam/4.14/lib/ppxlib -I /home/opam/.opam/4.14/lib/ppxlib/ast -I /home/opam/.opam/4.14/lib/ppxlib/astlib -I /home/opam/.opam/4.14/lib/ppxlib/print_diff -I /home/opam/.opam/4.14/lib/ppxlib/stdppx -I /home/opam/.opam/4.14/lib/ppxlib/traverse_builtins -I /home/opam/.opam/4.14/lib/progress -I /home/opam/.opam/4.14/lib/progress/engine -I /home/opam/.opam/4.14/lib/psq -I /home/opam/.opam/4.14/lib/repr -I /home/opam/.opam/4.14/lib/result -I /home/opam/.opam/4.14/lib/semaphore-compat -I /home/opam/.opam/4.14/lib/seq -I /home/opam/.opam/4.14/lib/sexplib0 -I /home/opam/.opam/4.14/lib/stdlib-shims -I /home/opam/.opam/4.14/lib/stringext -I /home/opam/.opam/4.14/lib/terminal -I /home/opam/.opam/4.14/lib/uchar -I /home/opam/.opam/4.14/lib/uri -I /home/opam/.opam/4.14/lib/uucp -I /home/opam/.opam/4.14/lib/uutf -I /home/opam/.opam/4.14/lib/vector -I src/irmin-pack/.irmin_pack.objs/byte -intf-suffix .ml -no-alias-deps -open Irmin_pack_layered__ -o src/irmin-pack/layered/.irmin_pack_layered.objs/byte/irmin_pack_layered__Ext_layered.cmo -c -impl src/irmin-pack/layered/ext_layered.pp.ml)
# ocamlopt src/irmin-pack/.irmin_pack.objs/native/irmin_pack__Ext.{cmx,o} (got signal KILL)
# (cd _build/default && /home/opam/.opam/4.14/bin/ocamlopt.opt -w -40 -g -I src/irmin-pack/.irmin_pack.objs/byte -I src/irmin-pack/.irmin_pack.objs/native -I /home/opam/.opam/4.14/lib/angstrom -I /home/opam/.opam/4.14/lib/astring -I /home/opam/.opam/4.14/lib/base64 -I /home/opam/.opam/4.14/lib/bheap -I /home/opam/.opam/4.14/lib/bigarray-compat -I /home/opam/.opam/4.14/lib/bigstringaf -I /home/opam/.opam/4.14/lib/bytes -I /home/opam/.opam/4.14/lib/cmdliner -I /home/opam/.opam/4.14/lib/digestif -I /home/opam/.opam/4.14/lib/either -I /home/opam/.opam/4.14/lib/eqaf -I /home/opam/.opam/4.14/lib/fmt -I /home/opam/.opam/4.14/lib/index -I /home/opam/.opam/4.14/lib/index/unix -I /home/opam/.opam/4.14/lib/irmin -I /home/opam/.opam/4.14/lib/irmin-layers -I /home/opam/.opam/4.14/lib/jsonm -I /home/opam/.opam/4.14/lib/logs -I /home/opam/.opam/4.14/lib/lru -I /home/opam/.opam/4.14/lib/lwt -I /home/opam/.opam/4.14/lib/lwt/unix -I /home/opam/.opam/4.14/lib/mmap -I /home/opam/.opam/4.14/lib/mtime -I /home/opam/.opam/4.14/lib/mtime/os -I /home/opam/.opam/4.14/lib/ocaml-compiler-libs/common -I /home/opam/.opam/4.14/lib/ocaml-compiler-libs/shadow -I /home/opam/.opam/4.14/lib/ocaml/compiler-libs -I /home/opam/.opam/4.14/lib/ocaml/threads -I /home/opam/.opam/4.14/lib/ocamlgraph -I /home/opam/.opam/4.14/lib/ocplib-endian -I /home/opam/.opam/4.14/lib/ocplib-endian/bigstring -I /home/opam/.opam/4.14/lib/optint -I /home/opam/.opam/4.14/lib/ppx_derivers -I /home/opam/.opam/4.14/lib/ppx_irmin -I /home/opam/.opam/4.14/lib/ppx_repr -I /home/opam/.opam/4.14/lib/ppx_repr/lib -I /home/opam/.opam/4.14/lib/ppxlib -I /home/opam/.opam/4.14/lib/ppxlib/ast -I /home/opam/.opam/4.14/lib/ppxlib/astlib -I /home/opam/.opam/4.14/lib/ppxlib/print_diff -I /home/opam/.opam/4.14/lib/ppxlib/stdppx -I /home/opam/.opam/4.14/lib/ppxlib/traverse_builtins -I /home/opam/.opam/4.14/lib/progress -I /home/opam/.opam/4.14/lib/progress/engine -I /home/opam/.opam/4.14/lib/psq -I /home/opam/.opam/4.14/lib/repr -I /home/opam/.opam/4.14/lib/result -I /home/opam/.opam/4.14/lib/semaphore-compat -I /home/opam/.opam/4.14/lib/seq -I /home/opam/.opam/4.14/lib/sexplib0 -I /home/opam/.opam/4.14/lib/stdlib-shims -I /home/opam/.opam/4.14/lib/stringext -I /home/opam/.opam/4.14/lib/terminal -I /home/opam/.opam/4.14/lib/uchar -I /home/opam/.opam/4.14/lib/uri -I /home/opam/.opam/4.14/lib/uucp -I /home/opam/.opam/4.14/lib/uutf -I /home/opam/.opam/4.14/lib/vector -intf-suffix .ml -no-alias-deps -open Irmin_pack__ -o src/irmin-pack/.irmin_pack.objs/native/irmin_pack__Ext.cmx -c -impl src/irmin-pack/ext.pp.ml)
Testing this locally reveals that the KILLs are there because ocamlopt infinit loops and starts to take all my RAM and swap space (several GB) because being killed by the system.
Using lldb to investigate seems to indicate some kind of issue in the new Shape module introduced in #10718 (cc @voodoos):
* thread #1, queue = 'com.apple.main-thread', stop reason = signal SIGSTOP
* frame #0: 0x0000000100385a08 ocamlopt.opt`do_some_marking + 396
frame #1: 0x0000000100384870 ocamlopt.opt`mark_slice + 124
frame #2: 0x00000001003845c4 ocamlopt.opt`caml_major_collection_slice + 1140
frame #3: 0x0000000100386720 ocamlopt.opt`caml_alloc_small_dispatch + 268
frame #4: 0x00000001003a5574 ocamlopt.opt`caml_call_gc + 136
frame #5: 0x0000000100199714 ocamlopt.opt`camlShape__app_2353 + 212
frame #6: 0x00000001001997e0 ocamlopt.opt`camlShape__subst_2354 + 192
frame #7: 0x00000001001997e0 ocamlopt.opt`camlShape__subst_2354 + 192
frame #8: 0x00000001001998b0 ocamlopt.opt`camlShape__subst_2354 + 400
frame #9: 0x00000001001997e0 ocamlopt.opt`camlShape__subst_2354 + 192
frame #10: 0x00000001001998b0 ocamlopt.opt`camlShape__subst_2354 + 400
frame #11: 0x000000010034497c ocamlopt.opt`camlStdlib__Map__map_593 + 68
frame #12: 0x0000000100344964 ocamlopt.opt`camlStdlib__Map__map_593 + 44
frame #13: 0x0000000100344990 ocamlopt.opt`camlStdlib__Map__map_593 + 88
frame #14: 0x0000000100344964 ocamlopt.opt`camlStdlib__Map__map_593 + 44
frame #15: 0x0000000100344964 ocamlopt.opt`camlStdlib__Map__map_593 + 44
frame #16: 0x0000000100344990 ocamlopt.opt`camlStdlib__Map__map_593 + 88
frame #17: 0x0000000100199850 ocamlopt.opt`camlShape__subst_2354 + 304
frame #18: 0x00000001001997c4 ocamlopt.opt`camlShape__subst_2354 + 164
frame #19: 0x00000001001998b0 ocamlopt.opt`camlShape__subst_2354 + 400
frame #20: 0x000000010034497c ocamlopt.opt`camlStdlib__Map__map_593 + 68
frame #21: 0x0000000100344964 ocamlopt.opt`camlStdlib__Map__map_593 + 44
frame #22: 0x0000000100344990 ocamlopt.opt`camlStdlib__Map__map_593 + 88
frame #23: 0x0000000100344990 ocamlopt.opt`camlStdlib__Map__map_593 + 88
frame #24: 0x0000000100199850 ocamlopt.opt`camlShape__subst_2354 + 304
frame #25: 0x00000001001997c4 ocamlopt.opt`camlShape__subst_2354 + 164
frame #26: 0x00000001001997e0 ocamlopt.opt`camlShape__subst_2354 + 192
frame #27: 0x00000001001997e0 ocamlopt.opt`camlShape__subst_2354 + 192
frame #28: 0x00000001001997e0 ocamlopt.opt`camlShape__subst_2354 + 192
frame #29: 0x00000001001998b0 ocamlopt.opt`camlShape__subst_2354 + 400
frame #30: 0x000000010034497c ocamlopt.opt`camlStdlib__Map__map_593 + 68
frame #31: 0x0000000100344964 ocamlopt.opt`camlStdlib__Map__map_593 + 44
frame #32: 0x0000000100344990 ocamlopt.opt`camlStdlib__Map__map_593 + 88
frame #33: 0x0000000100344964 ocamlopt.opt`camlStdlib__Map__map_593 + 44
frame #34: 0x0000000100199850 ocamlopt.opt`camlShape__subst_2354 + 304
frame #35: 0x00000001001997c4 ocamlopt.opt`camlShape__subst_2354 + 164
frame #36: 0x00000001001997e0 ocamlopt.opt`camlShape__subst_2354 + 192
frame #37: 0x00000001001998b0 ocamlopt.opt`camlShape__subst_2354 + 400
frame #38: 0x000000010034497c ocamlopt.opt`camlStdlib__Map__map_593 + 68
frame #39: 0x0000000100344964 ocamlopt.opt`camlStdlib__Map__map_593 + 44
frame #40: 0x0000000100344964 ocamlopt.opt`camlStdlib__Map__map_593 + 44
frame #41: 0x0000000100344964 ocamlopt.opt`camlStdlib__Map__map_593 + 44
frame #42: 0x0000000100344990 ocamlopt.opt`camlStdlib__Map__map_593 + 88
frame #43: 0x0000000100199850 ocamlopt.opt`camlShape__subst_2354 + 304
frame #44: 0x00000001001997c4 ocamlopt.opt`camlShape__subst_2354 + 164
frame #45: 0x000000010034497c ocamlopt.opt`camlStdlib__Map__map_593 + 68
frame #46: 0x0000000100344990 ocamlopt.opt`camlStdlib__Map__map_593 + 88
frame #47: 0x0000000100199850 ocamlopt.opt`camlShape__subst_2354 + 304
frame #48: 0x00000001001997c4 ocamlopt.opt`camlShape__subst_2354 + 164
frame #49: 0x00000001001998b0 ocamlopt.opt`camlShape__subst_2354 + 400
frame #50: 0x000000010034497c ocamlopt.opt`camlStdlib__Map__map_593 + 68
frame #51: 0x0000000100344964 ocamlopt.opt`camlStdlib__Map__map_593 + 44
frame #52: 0x0000000100344990 ocamlopt.opt`camlStdlib__Map__map_593 + 88
frame #53: 0x0000000100344964 ocamlopt.opt`camlStdlib__Map__map_593 + 44
frame #54: 0x0000000100344990 ocamlopt.opt`camlStdlib__Map__map_593 + 88
frame #55: 0x0000000100344990 ocamlopt.opt`camlStdlib__Map__map_593 + 88
frame #56: 0x0000000100344990 ocamlopt.opt`camlStdlib__Map__map_593 + 88
frame #57: 0x0000000100199850 ocamlopt.opt`camlShape__subst_2354 + 304
frame #58: 0x000000010019979c ocamlopt.opt`camlShape__subst_2354 + 124
frame #59: 0x000000010019979c ocamlopt.opt`camlShape__subst_2354 + 124
frame #60: 0x000000010019979c ocamlopt.opt`camlShape__subst_2354 + 124
frame #61: 0x000000010019979c ocamlopt.opt`camlShape__subst_2354 + 124
frame #62: 0x000000010019979c ocamlopt.opt`camlShape__subst_2354 + 124
frame #63: 0x000000010019979c ocamlopt.opt`camlShape__subst_2354 + 124
frame #64: 0x000000010019979c ocamlopt.opt`camlShape__subst_2354 + 124
frame #65: 0x0000000100199678 ocamlopt.opt`camlShape__app_2353 + 56
frame #66: 0x0000000100227708 ocamlopt.opt`camlIncludemod__try_modtypes_2275 + 808
frame #67: 0x000000010022734c ocamlopt.opt`camlIncludemod__modtypes_2274 + 36
frame #68: 0x000000010022773c ocamlopt.opt`camlIncludemod__try_modtypes_2275 + 860
frame #69: 0x000000010022734c ocamlopt.opt`camlIncludemod__modtypes_2274 + 36
frame #70: 0x0000000100229174 ocamlopt.opt`camlIncludemod__signature_components_2280 + 796
frame #71: 0x00000001002289c8 ocamlopt.opt`camlIncludemod__pair_components_2458 + 104
frame #72: 0x0000000100229ffc ocamlopt.opt`camlIncludemod__compunit_2922 + 100
frame #73: 0x00000001002a3964 ocamlopt.opt`camlTypemod__fun_9442 + 804
frame #74: 0x00000001000eb650 ocamlopt.opt`camlMisc__try_finally_inner_3897 + 48
frame #75: 0x00000001000eb650 ocamlopt.opt`camlMisc__try_finally_inner_3897 + 48
frame #76: 0x000000010031faf4 ocamlopt.opt`camlCompile_common__typecheck_impl_1925 + 132
frame #77: 0x000000010031fe20 ocamlopt.opt`camlCompile_common__fun_2194 + 56
frame #78: 0x00000001000eb650 ocamlopt.opt`camlMisc__try_finally_inner_3897 + 48
frame #79: 0x00000001000eb650 ocamlopt.opt`camlMisc__try_finally_inner_3897 + 48
frame #80: 0x00000001000eb650 ocamlopt.opt`camlMisc__try_finally_inner_3897 + 48
frame #81: 0x0000000100305b40 ocamlopt.opt`camlCompenv__process_action_1711 + 1520
frame #82: 0x0000000100329f5c ocamlopt.opt`camlStdlib__List__iter_507 + 68
frame #83: 0x0000000100305fdc ocamlopt.opt`camlCompenv__process_deferred_actions_1781 + 356
frame #84: 0x00000001000e8eb4 ocamlopt.opt`camlOptmaindriver__code_begin + 532
frame #85: 0x000000010000ada8 ocamlopt.opt`camlOptmain__code_begin + 48
frame #86: 0x000000010000356c ocamlopt.opt`caml_program + 6636
frame #87: 0x00000001003a56bc ocamlopt.opt`caml_start_program + 104
frame #88: 0x000000010037fb2c ocamlopt.opt`caml_startup_common + 620
frame #89: 0x000000010037fb94 ocamlopt.opt`caml_main + 16
frame #90: 0x000000010037fbf4 ocamlopt.opt`main + 16
frame #91: 0x0000000100afd0f4 dyld`start + 520