`nerdctl` fails to run a minimal container

[tnytown]

Consider the following image:

{ writeShellScript, runtimeShell, nix-snapshotter }:
let
  hello-world = writeShellScript "hello-world"
  ''#!{runtimeShell}
   echo "Hello, world!"
  '';
  in (nix-snapshotter.buildImage {
    name = "repro";
    resolvedByNix = true;
    config.entrypoint = [ hello-world ];
})

Built and loaded by this procedure:

$ drv=$(nix eval --raw --apply 'p: let nix-snapshotter = (builtins.getFlake "github:pdtpartners/nix-snapshotter/6eb21bd"); pkgs = import (builtins.getFlake "nixpkgs/f292b49") { overlays = [ nix-snapshotter.overlays.default ]; }; container = (pkgs.callPackage p {}).copyToContainerd {}; in container.drvPath' --file ./repro.nix)
$ nix build "$drv^out"
$ sudo result/bin/copy-to-containerd

When this container is run with nerdctl run nix:0/nix/store/<path>:latest, an error occurs:

FATA[0002] failed to mount {Type:bind Source:/nix/store/c2lyvs0iz8b3l6ijk1f9fz8ma8khxcxm-hello-world Target:/nix/store/c2lyvs0iz8b3l6ijk1f9fz8ma8khxcxm-hello-world Options:[ro rbind]} on "/tmp/initialC595249715": no such file or directory

This issue seems to be related to how the Nix closure layer is generated. Directory mountpoints are generated correctly, but file mountpoints don't appear in the final image.

https://github.com/pdtpartners/nix-snapshotter/blob/6eb21bd3429535646da4aa396bb0c1f81a9b72c6/pkg/nix2container/generate.go#L255-L259

[usmcamp0811]

I have a similar problem:

╭─mcamp on chesty in /config on snapshotter-wip🔥
╰─🚧 sudo nerdctl image ls                      
REPOSITORY                         TAG       IMAGE ID        CREATED               PLATFORM       SIZE     BLOB SIZE
ghcr.io/pdtpartners/redis-shell    latest    6c6c48068af3    About a minute ago    linux/amd64    0.0 B    20.4 KiB
╭─mcamp on chesty in /config on snapshotter-wip🔥
╰─🚧 sudo nerdctl run 6c6c48068af3 
INFO[0000] apply failure, attempting cleanup             error="failed to extract layer sha256:cb559dbb3a6702d6977e6cd7cf5a175e59bf02792adda85e82b02f4296fdbaef: mount callback failed on /var/lib/containerd/tmpmounts/containerd-mount3791911954: archive/tar: invalid tar header: unknown" key="extract-359532708-B-98 sha256:cb559dbb3a6702d6977e6cd7cf5a175e59bf02792adda85e82b02f4296fdbaef"
FATA[0000] failed to extract layer sha256:cb559dbb3a6702d6977e6cd7cf5a175e59bf02792adda85e82b02f4296fdbaef: mount callback failed on /var/lib/containerd/tmpmounts/containerd-mount3791911954: archive/tar: invalid tar header: unknown 
╭─mcamp on chesty in /config on snapshotter-wip🔥
╰─🚧 

my container code:

{ lib
, writeText
, writeShellApplication
, substituteAll
, inputs
, pkgs
, hosts ? { }
, ...
}:
let
  inherit (lib) mapAttrsToList concatStringsSep;
  inherit (lib.campground) override-meta;
  inherit (pkgs.nix-snapshotter) buildImage;

      examples = rec {
        hello = buildImage {
          name = "ghcr.io/aicampground/my-hello2";
          tag = "latest";
          config = {
            entrypoint = ["${pkgs.hello}/bin/hello"];
          };
        };

        redis = buildImage {
          name = "ghcr.io/pdtpartners/my-redis";
          tag = "latest";
          config = {
            entrypoint = [ "${pkgs.redis}/bin/redis-server" ];
          };
        };

        mlflow = buildImage {
          name = "aicampground.com/testing/mlflow";
          tag = "latest";
          config = {
            entrypoint = [ "${pkgs.campground.mlflow}/bin/mlflow-server" ];
          };
        };

        redisWithShell = buildImage {
          name = "ghcr.io/pdtpartners/redis-shell";
          tag = "latest";
          fromImage = redis;
          config = {
            entrypoint = [ "/bin/sh" ];
          };
          copyToRoot = pkgs.buildEnv {
            name = "system-path";
            pathsToLink = [ "/bin" ];
            paths = [
              pkgs.bashInteractive
              pkgs.coreutils
              pkgs.redis
            ];
          };
        };
      };
in 
examples.redisWithShell

my snapshotter module:

{ inputs, lib, config, pkgs, ... }:
with lib;
with lib.campground;
let
  cfg = config.campground.services.nix-snapshotter;
in
{
  imports = [
    inputs.nix-snapshotter.nixosModules.default
  ];

  options.campground.services.nix-snapshotter = with types; {
    enable = mkBoolOpt false "Enable Nix Snapshotter;";
  };

  config = mkIf cfg.enable {

    services.nix-snapshotter = {
      enable = true;
      setContainerdSnapshotter = true;
      preloadContainerdImages = [
        pkgs.campground.containers
      ];
    };

    environment.systemPackages = [ 
      pkgs.nerdctl  
    ];

  };
}

my dotfiles are here

[elpdt852]

@tnytown Hey there, sorry it took a while to find the time, but there is now a fix in #128.

@usmcamp0811 I don't believe that is the same issue. That looks like the failure mode when CONTAINERD_SNAPSHOTTER=nix is not set. Alternatively you can pass it as a flag like nerdctl run --snapshotter=nix .... In the latest v0.2.0 NixOS / Home Manager modules, those env variables should be set for you.

[tnytown]

@elpdt852 No worries! Thanks for getting this fixed.

[elpdt852]

@tnytown The fix has finally landed. Kubernetes has some hidden assumptions that was difficult to reverse engineer, but that’s all resolved now. Thanks for your report!