Internal compiler error (unhandled IO exception)

[collinarnett]

Hello,

I'm trying to build a simple linear regression example to benchmark against some jax code (for fun) and I came across this error when trying to compile my code as a python library.

$ futhark pyopencl --library linear_regression.fut
Warning at linear_regression.fut:29:55-56:
  Unused variable "i".
Internal compiler error (unhandled IO exception).
Please report this at https://github.com/diku-dk/futhark/issues
dataDependencies':
Pattern size: 3
Op deps size: 2
Expression:
vjp(\ {eta_p_9576 : f32,
       eta_p_9577 : [m_8906][1i64]f32}
      : {f32} ->
      let {transpose_res_9578 : ((), [1i64][m_8906]f32)} =
        rearrange((1, 0), eta_p_9577)
      let {as_transformed_row_9579 : ((), [m_8906]f32)} =
        transpose_res_9578[0i64, 0i64 :+ m_8906 * 1i64]
      let {defunc_0_map_res_9580 : ((), [n_8905][1i64]f32)} =
        map(n_8905,
            {X_8907},
            \ {as_transformed_row_9581 : [m_8906]f32}
              : {[1i64]f32} ->
              let {defunc_0_map_res_9582 : ((), [m_8906]f32)} =
                map(m_8906,
                    {as_transformed_row_9581, as_transformed_row_9579},
                    \ {eta_p_9583 : f32,
                       eta_p_9584 : f32}
                      : {f32} ->
                      let {*_res_9585 : ((), f32)} =
                        fmul32(eta_p_9583, eta_p_9584)
                      in {*_res_9585})
              let {defunc_0_reduce_res_9586 : ((), f32)} =
                redomap(m_8906,
                        {defunc_0_map_res_9582},
                        {\ {eta_p_9587 : f32,
                            eta_p_9588 : f32}
                          : {f32} ->
                          let {+_res_9589 : ((), f32)} =
                            fadd32(eta_p_9587, eta_p_9588)
                          in {+_res_9589},
                        {0.0f32}},
                        \ {x_9590 : f32}
                          : {f32} ->
                          {x_9590})
              let {defunc_0_map_res_9591 : ((), [1i64]f32)} =
                replicate([1i64], defunc_0_reduce_res_9586)
              in {defunc_0_map_res_9591})
      let {defunc_0_map_res_transformed_9592 : ((), [n_8905]f32)} =
        defunc_0_map_res_9580[0i64 :+ n_8905 * 1i64, 0i64]
      let {defunc_0_map_res_9593 : ((), [n_8905][1i64]f32)} =
        map(n_8905,
            {defunc_0_map_res_transformed_9592},
            \ {defunc_0_map_res_transformed_row_9594 : f32}
              : {[1i64]f32} ->
              let {+_res_9595 : ((), f32)} =
                fadd32(eta_p_9576, defunc_0_map_res_transformed_row_9594)
              let {defunc_0_map_res_9596 : ((), [1i64]f32)} =
                replicate([1i64], +_res_9595)
              in {defunc_0_map_res_9596})
      let {zip_copy_transformed_9597 : ((), [n_8905]f32)} =
        defunc_0_map_res_9593[0i64 :+ n_8905 * 1i64, 0i64]
      let {defunc_0_map_res_9598 : ((), [n_8905][1i64]f32)} =
        map(n_8905,
            {zip_copy_transformed_9597, zip_copy_transformed_9572},
            \ {zip_copy_transformed_row_9599 : f32,
               zip_copy_transformed_row_9600 : f32}
              : {[1i64]f32} ->
              let {-_res_9601 : ((), f32)} =
                fsub32(zip_copy_transformed_row_9599, zip_copy_transformed_row_9600)
              let {defunc_0_map_res_9602 : ((), [1i64]f32)} =
                replicate([1i64], -_res_9601)
              in {defunc_0_map_res_9602})
      let {transpose_res_9603 : ((), [1i64][n_8905]f32)} =
        rearrange((1, 0), defunc_0_map_res_9598)
      let {map_arg1_9604 : ((), [n_8905]f32)} =
        transpose_res_9603[0i64, 0i64 :+ n_8905 * 1i64]
      let {defunc_0_map_res_9605 : ((), [n_8905]f32)} =
        map(n_8905,
            {map_arg1_9604},
            \ {eta_p_9606 : f32}
              : {f32} ->
              let {lifted_lambda_res_9607 : ((), f32)} =
                fpow32(eta_p_9606, 2.0f32)
              in {lifted_lambda_res_9607})
      let {defunc_res_9608 : ((), f32)} =
        redomap(n_8905,
                {defunc_0_map_res_9605},
                {\ {eta_p_9609 : f32,
                    eta_p_9610 : f32}
                  : {f32} ->
                  let {+_res_9611 : ((), f32)} =
                    fadd32(eta_p_9609, eta_p_9610)
                  in {+_res_9611},
                {0.0f32}},
                \ {x_9612 : f32}
                  : {f32} ->
                  {x_9612})
      let {mean_res_9613 : ((), f32)} =
        fdiv32(defunc_res_9608, i64_res_9571)
      in {mean_res_9613},
    {p'_9127, p'_9128},
    {1.0f32})

CallStack (from HasCallStack):
  error, called at src/Futhark/Analysis/DataDependencies.hs:55:15 in futhark-0.25.8-GROC5SrbCXD9LeCYaknxNz:Futhark.Analysis.DataDependencies

Here's my code as well. I assume my usage of vjp here is wrong but I figured I'd see if I could use it regardless.

--import "lib/github.com/diku-dk/cpprandom/random"
--import "lib/github.com/diku-dk/cpprandom/shuffle"
import "lib/github.com/diku-dk/linalg/linalg"
module linalg_f32 = mk_linalg f32
--module shuffle = mk_shuffle pcg32
--module dist = (normal_distribution f32 pcg32)
def mean [n] (vs: [n]f32) =
  f32.sum vs / f32.i64 n

def params [m] : {w: [m][1]f32, b: f32} =
  {w = linalg_f32.matzeros m 1, b=0f32}

def forward [n][m] (p: {w: [m][1]f32, b: f32}) (X:[n][m]f32) =
  linalg_f32.matadd  (replicate n [p.b]) (linalg_f32.matmul X p.w)

def loss_fn [n][m] (p: {w: [m][1]f32, b: f32}) (X:[n][m]f32) (y: [n][1]f32) =
  let err = linalg_f32.matsub (forward p X) y in
  mean (map (**2) (transpose err)[0])

def grad_fn [n][m] (p: {w: [m][1]f32, b: f32}) (X:[n][m]f32) (y: [n][1]f32) = vjp (\p -> loss_fn p X y ) p 1f32

def update [m] (p: {w: [m][1]f32, b: f32}) (grads: {w: [m][1]f32, b: f32}) =
  {w= map (\n -> [n]) (map2 (\x y -> x - 0.05 * y ) p.w[:, 0]  grads.w[:, 0]) , b= p.b - 0.05 * grads.b}

def train [n][m]
  (X:[n][m]f32)
  (y: [n][1]f32) =
  let grads = grad_fn (copy params) X y in
  loop (grads, p') = (grads, update params grads) for i < 50 do
    (grad_fn p' X y, update p' grads)

entry run_train [n][m]
  (X:[n][m]f32)
  (y: [n][1]f32) =
  let result = train X y in
  (result.1.w, result.1.b)

System Information

$ nix-shell -p nix-info --run "nix-info -m
∙ "
 - system: `"x86_64-linux"`
 - host os: `Linux 6.1.45, NixOS, 23.11 (Tapir), 23.11.20230816.caac0eb`
 - multi-user?: `yes`
 - sandbox: `yes`
 - version: `nix-env (Nix) 2.17.0`
 - nixpkgs: `/nix/store/hwp2x4k2p6ar2xpzbmjbl278sqxmxv0s-source`

Futhark Version

Futhark 0.25.8

[athas]

I cannot reproduce on the nightly version. I'm pretty sure this was fixed in 0.25.9.

[cpebble]

I am seeing an error which could be related, in the mss test.

cpj@utopia-1:/scratch-fast/cpj/futfut/bin$ futhark --version
Futhark 0.26.0 (prerelease - include info below when reporting bugs)
git: HEAD @ 5234eb8 (Tue Dec 5 14:54:42 2023 +0100)

-- Parallel maximum segment sum
--
-- ==
-- random input { [1000]i32 } auto output
-- random input { [1000000]i32 } auto output
-- random input { [1000000000]i32 } auto output

def main(xs: []i32): i32 =
  let max = i32.max
  let redOp (mssx, misx, mcsx, tsx) (mssy, misy, mcsy, tsy) =
    ( max mssx (max mssy (mcsx + misy))
    , max misx (tsx+misy)
    , max mcsy (mcsx+tsy)
    , tsx + tsy)
  let mapOp x =
    ( max x 0
    , max x 0
    , max x 0
    , x)
  in (reduce redOp (0,0,0,0) (map mapOp xs)).0

I get this error:

cpj@utopia-1:/scratch-fast/cpj/futfut/bin$ futhark c ../benches/mss.fut 
Internal compiler error (unhandled IO exception).
Please report this at https://github.com/diku-dk/futhark/issues
<<loop>>

[athas]

No, that is a very different bug - an infinite loop (and simple enough that the Haskell RTS can recognise it).

[athas]

But I also cannot reproduce that one.

[athas]

I am going to close this because I think it is already fixed.