NixOS install failing

[justinwaltrip]

I have been using this config on my Mac without issue for the last few months. Now I am trying to use the same configuration on NixOS, but I am running into issues during build-switch command. I have two Nvidia 3090 GPU's, which I think may be part of the problem.

Here is my hosts/nixos/default.nix:

{ config, inputs, pkgs, agenix, ... }:

let
  user = "justinwaltrip";
  keys = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOk8iAnIaa1deoc7jw8YACPNVka1ZFJxhnU4G74TmS+p" ];
in
{
  imports = [
    ../../modules/nixos/secrets.nix
    ../../modules/nixos/disk-config.nix
    ../../modules/shared
    ../../modules/shared/cachix
    agenix.nixosModules.default
  ];

  # Use the systemd-boot EFI boot loader.
  boot = {
    loader = {
      systemd-boot = {
        enable = true;
        configurationLimit = 42;
      };
      efi.canTouchEfiVariables = true;
    };
    initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" "v4l2loopback" ];
    kernelModules = [ "uinput" "v4l2loopback" ];
    extraModulePackages = [ pkgs.linuxPackages.v4l2loopback ];
  };

  # Set your time zone.
  time.timeZone = "America/New_York";

  # The global useDHCP flag is deprecated, therefore explicitly set to false here.
  # Per-interface useDHCP will be mandatory in the future, so this generated config
  # replicates the default behaviour.
  networking = {
    hostName = "justin"; # Define your hostname.
    useDHCP = false;
    interfaces.eno1.useDHCP = true;
  };

  # Turn on flag for proprietary software
  nix = {
    nixPath = [ "nixos-config=/home/${user}/.local/share/src/nixos-config:/etc/nixos" ];
    settings.allowed-users = [ "${user}" ];
    package = pkgs.nix;
    extraOptions = ''
      experimental-features = nix-command flakes
    '';
  };

  # Manages keys and such
  programs = {
    gnupg.agent.enable = true;

    # Needed for anything GTK related
    dconf.enable = true;

    # My shell
    zsh.enable = true;
  };

  services = {
    xserver = {
      enable = true;

      videoDrivers = [ "nvidia" ];

      # This helps fix tearing of windows for Nvidia cards
      screenSection = ''
        Option       "metamodes" "nvidia-auto-select +0+0 {ForceFullCompositionPipeline=On}"
        Option       "AllowIndirectGLXProtocol" "off"
        Option       "TripleBuffer" "on"
      '';

      # LightDM Display Manager
      displayManager.defaultSession = "none+bspwm";
      displayManager.lightdm = {
        enable = true;
        greeters.slick.enable = true;
        background = ../../modules/nixos/config/login-wallpaper.png;
      };

      # Tiling window manager
      windowManager.bspwm = {
        enable = true;
      };

      # Better support for general peripherals
      libinput.enable = true;

      # Turn Caps Lock into Ctrl
      xkb = {
        layout = "us";
        options = "ctrl:nocaps";
      };
    };

    # Enable CUPS to print documents
    printing = {
      enable = true;
      drivers = [ pkgs.brlaser ]; # Brother printer driver
    };

    # syncthing = {
    #   enable = true;
    #   openDefaultPorts = true;
    #   dataDir = "/home/${user}/.local/share/syncthing";
    #   configDir = "/home/${user}/.config/syncthing";
    #   user = "${user}";
    #   group = "users";
    #   guiAddress = "127.0.0.1:8384";
    #   overrideFolders = true;
    #   overrideDevices = true;

    #   settings = {
    #     devices = { };
    #     options.globalAnnounceEnabled = false; # Only sync on LAN
    #   };
    # };

    # Picom, my window compositor with fancy effects
    #
    # Notes on writing exclude rules:
    #
    #   class_g looks up index 1 in WM_CLASS value for an application
    #   class_i looks up index 0
    #
    #   To find the value for a specific application, use `xprop` at the
    #   terminal and then click on a window of the application in question
    #
    #   picom = {
    #     enable = true;
    #     settings = {
    #       animations = true;
    #       animation-stiffness = 300.0;
    #       animation-dampening = 35.0;
    #       animation-clamping = false;
    #       animation-mass = 1;
    #       animation-for-workspace-switch-in = "auto";
    #       animation-for-workspace-switch-out = "auto";
    #       animation-for-open-window = "slide-down";
    #       animation-for-menu-window = "none";
    #       animation-for-transient-window = "slide-down";
    #       corner-radius = 12;
    #       rounded-corners-exclude = [
    #         "class_i = 'polybar'"
    #         "class_g = 'i3lock'"
    #       ];
    #       round-borders = 3;
    #       round-borders-exclude = [ ];
    #       round-borders-rule = [ ];
    #       shadow = true;
    #       shadow-radius = 8;
    #       shadow-opacity = 0.4;
    #       shadow-offset-x = -8;
    #       shadow-offset-y = -8;
    #       fading = false;
    #       inactive-opacity = 0.8;
    #       frame-opacity = 0.7;
    #       inactive-opacity-override = false;
    #       active-opacity = 1.0;
    #       focus-exclude = [
    #       ];

    #       opacity-rule = [
    #         "100:class_g = 'i3lock'"
    #         "60:class_g = 'Dunst'"
    #         "100:class_g = 'Alacritty' && focused"
    #         "90:class_g = 'Alacritty' && !focused"
    #       ];

    #       blur-kern = "3x3box";
    #       blur = {
    #         method = "kernel";
    #         strength = 8;
    #         background = false;
    #         background-frame = false;
    #         background-fixed = false;
    #         kern = "3x3box";
    #       };

    #       shadow-exclude = [
    #         "class_g = 'Dunst'"
    #       ];

    #       blur-background-exclude = [
    #         "class_g = 'Dunst'"
    #       ];

    #       backend = "glx";
    #       vsync = false;
    #       mark-wmwin-focused = true;
    #       mark-ovredir-focused = true;
    #       detect-rounded-corners = true;
    #       detect-client-opacity = false;
    #       detect-transient = true;
    #       detect-client-leader = true;
    #       use-damage = true;
    #       log-level = "info";

    #       wintypes = {
    #         normal = { fade = true; shadow = false; };
    #         tooltip = { fade = true; shadow = false; opacity = 0.75; focus = true; full-shadow = false; };
    #         dock = { shadow = false; };
    #         dnd = { shadow = false; };
    #         popup_menu = { opacity = 1.0; };
    #         dropdown_menu = { opacity = 1.0; };
    #       };
    #     };
    #   };

    # Let's be able to SSH into this machine
    openssh.enable = true;

    # # My editor runs as a daemon
    # emacs = {
    #   enable = true;
    #   package = pkgs.emacs-unstable;
    # };

    gvfs.enable = true; # Mount, trash, and other functionalities
    tumbler.enable = true; # Thumbnail support for images
  };

  # systemd.user.services.emacs = {
  #   serviceConfig.TimeoutStartSec = "7min";
  # };

  # Enable sound
  sound.enable = true;
  hardware = {
    pulseaudio.enable = true;

    # Video support
    opengl = {
      enable = true;
      driSupport32Bit = true;
      driSupport = true;
    };

    nvidia.modesetting.enable = true;

    # Crypto wallet support
    ledger.enable = true;
  };

  # Add docker daemon
  virtualisation = {
    docker = {
      enable = true;
      logDriver = "json-file";
    };
  };

  # It's me, it's you, it's everyone
  users.users = {
    ${user} = {
      isNormalUser = true;
      extraGroups = [
        "wheel" # Enable ‘sudo’ for the user.
        "docker"
      ];
      shell = pkgs.zsh;
      openssh.authorizedKeys.keys = keys;
    };

    root = {
      openssh.authorizedKeys.keys = keys;
    };
  };

  # Don't require password for users in `wheel` group for these commands
  security.sudo = {
    enable = true;
    extraRules = [{
      commands = [
        {
          command = "${pkgs.systemd}/bin/reboot";
          options = [ "NOPASSWD" ];
        }
      ];
      groups = [ "wheel" ];
    }];
  };

  fonts.packages = with pkgs; [
    dejavu_fonts
    # emacs-all-the-icons-fonts
    feather-font # from overlay
    jetbrains-mono
    font-awesome
    noto-fonts
    noto-fonts-emoji
  ];

  environment.systemPackages = with pkgs; [
    agenix.packages."${pkgs.system}".default # "x86_64-linux"
    gitAndTools.gitFull
    linuxPackages.v4l2loopback
    v4l-utils
    inetutils
  ];

  system.stateVersion = "21.05"; # Don't change this

}

Here are the logs from journalctl -xb: https://gist.github.com/justinwaltrip/b0136e48eae1703e8ae4a6973244979e

Please let me know if you would need more information.

[dustinlyons]

The journalctl log is quite large. What are you experiencing when trying to install?

[justinwaltrip]

@dustinlyons it appears to be restarting from reboot and starting system services, then it appears to hang for a minute or two before giving me the option to drop to shell to view logs or reboot again due to failure

[justinwaltrip]

Here are some (potentially) relevant logs:

Jun 10 12:46:07 nixos dhcpcd[132364]: dhcpcd-10.0.6 starting
Jun 10 12:46:07 nixos dhcpcd[132379]: dev: loaded udev
Jun 10 12:46:07 nixos systemd-modules-load[132370]: Inserted module 'uinput'
Jun 10 12:46:07 nixos systemd-modules-load[132370]: Failed to find module 'v4l2loopback'
Jun 10 12:46:07 nixos systemd-modules-load[132370]: Inserted module 'veth'
Jun 10 12:46:07 nixos kernel: 8021q: 802.1Q VLAN Support v1.8
Jun 10 12:46:07 nixos systemd-modules-load[132370]: Inserted module 'br_netfilter'
Jun 10 12:46:07 nixos kernel: Bridge firewalling registered
Jun 10 12:46:07 nixos systemd[1]: Finished Store Sound Card State.
░░ Subject: A start job for unit alsa-store.service has finished successfully
░░ Defined-By: systemd
░░ Support: https://lists.freedesktop.org/mailman/listinfo/systemd-devel
░░ 
░░ A start job for unit alsa-store.service has finished successfully.
░░ 
░░ The job identifier is 4416.
Jun 10 12:46:07 nixos systemd-modules-load[132370]: Inserted module 'xt_nat'
Jun 10 12:46:07 nixos systemd-modules-load[132370]: Failed to find module 'nvidia'
Jun 10 12:46:07 nixos systemd-modules-load[132370]: Failed to find module 'nvidia_modeset'
Jun 10 12:46:07 nixos systemd-modules-load[132370]: Failed to find module 'nvidia_drm'
Jun 10 12:46:07 nixos systemd[1]: Finished Address configuration of eno1.

░░ The unit display-manager.service has entered the 'failed' state with result 'exit-code'.
Jun 10 12:44:42 nixos systemd[1]: display-manager.service: Triggering OnFailure= dependencies.
Jun 10 12:44:42 nixos systemd[1]: display-manager.service: Failed to enqueue OnFailure= job, ignoring: Unit plymouth-quit.service not found.
Jun 10 12:44:42 nixos systemd[1]: display-manager.service: Scheduled restart job, restart counter is at 2.

Jun 10 12:35:36 nixos rtkit-daemon[1577]: Successfully made thread 1576 of process 1558 owned by '132' RT at priority 20.
Jun 10 12:35:36 nixos .gnome-shell-wr[1558]: Made thread 'KMS thread' realtime scheduled
Jun 10 12:35:36 nixos .gnome-shell-wr[1558]: Device '/dev/dri/card0' prefers shadow buffer
Jun 10 12:35:36 nixos .gnome-shell-wr[1558]: Added device '/dev/dri/card0' (nouveau) using non-atomic mode setting.
Jun 10 12:35:37 nixos .gnome-shell-wr[1558]: g_close(fd:0) failed with EBADF. The tracking of file descriptors got messed up
Jun 10 12:35:37 nixos .gnome-shell-wr[1558]: Failed to open gpu '/dev/dri/card1': No suitable mode setting backend found
Jun 10 12:35:37 nixos .gnome-shell-wr[1558]: Created gbm renderer for '/dev/dri/card0'
Jun 10 12:35:37 nixos .gnome-shell-wr[1558]: Boot VGA GPU /dev/dri/card0 selected as primary
Jun 10 12:35:37 nixos /nix/store/yfqcil25q7n8nv5ci9bf2c80rrnjzy4j-gdm-46.2/libexec/gdm-wayland-session[1545]: dbus-daemon[1545]: [session uid=132 pid=1545] Activating service name='org.a11y.Bus' requested by ':1.4' (uid=132 pid=1558 comm="/nix/store/hxqdl71a8jakvzxxz26rrj18va3w9mzn-gnome-" label="kernel")
Jun 10 12:35:37 nixos /nix/store/yfqcil25q7n8nv5ci9bf2c80rrnjzy4j-gdm-46.2/libexec/gdm-wayland-session[1545]: dbus-daemon[1545]: [session uid=132 pid=1545] Successfully activated service 'org.a11y.Bus'
Jun 10 12:35:37 nixos .gnome-shell-wr[1558]: Using public X11 display :1024, (using :1025 for managed services)
Jun 10 12:35:37 nixos .gnome-shell-wr[1558]: Using Wayland display name 'wayland-0'
Jun 10 12:35:37 nixos /nix/store/yfqcil25q7n8nv5ci9bf2c80rrnjzy4j-gdm-46.2/libexec/gdm-wayland-session[1610]: dbus-daemon[1610]: Activating service name='org.a11y.atspi.Registry' requested by ':1.0' (uid=132 pid=1558 comm="/nix/store/hxqdl71a8jakvzxxz26rrj18va3w9mzn-gnome-" label="kernel")
Jun 10 12:35:37 nixos dbus-daemon[1258]: [system] Activating via systemd: service name='org.freedesktop.ColorManager' unit='colord.service' requested by ':1.33' (uid=132 pid=1558 comm="/nix/store/hxqdl71a8jakvzxxz26rrj18va3w9mzn-gnome-" label="kernel")
Jun 10 12:35:37 nixos /nix/store/yfqcil25q7n8nv5ci9bf2c80rrnjzy4j-gdm-46.2/libexec/gdm-wayland-session[1610]: dbus-daemon[1610]: Successfully activated service 'org.a11y.atspi.Registry'
Jun 10 12:35:37 nixos /nix/store/yfqcil25q7n8nv5ci9bf2c80rrnjzy4j-gdm-46.2/libexec/gdm-wayland-session[1614]: SpiRegistry daemon is running with well-known name - org.a11y.atspi.Registry
Jun 10 12:35:37 nixos systemd[1]: Starting Manage, Install and Generate Color Profiles...

Jun 10 12:38:35 nixos systemd[2066]: Failed to start Application launched by gnome-session-binary.
░░ Subject: A start job for unit UNIT has failed
░░ Defined-By: systemd
░░ Support: https://lists.freedesktop.org/mailman/listinfo/systemd-devel
░░ 
░░ A start job for unit UNIT has finished with a failure.
░░ 
░░ The job identifier is 121 and the job result is failed.
Jun 10 12:38:35 nixos systemd[2066]: app-gnome-gnome\x2dkeyring\x2dsecrets-2158.scope: PID 2158 vanished before we could move it to target cgroup '/user.slice/user-0.slice/user@0.service/app.slice/app-gnome-gnome\x2dkeyring\x2dsecrets-2158.scope', skipping: No such process
Jun 10 12:38:35 nixos systemd[2066]: app-gnome-gnome\x2dkeyring\x2dsecrets-2158.scope: No PIDs left to attach to the scope's control group, refusing.
Jun 10 12:38:35 nixos systemd[2066]: app-gnome-gnome\x2dkeyring\x2dsecrets-2158.scope: Failed with result 'resources'.

[justinwaltrip]

I was able to get nvidia working with a standard configuration.nix setup, so I think the problem is something else

[dustinlyons]

Also this line looks interesting:

39:33 nixos systemd-coredump[4375]: Process 4287 (gst-plugin-scan) of user 1000 dumped core.

and

Jun 10 12:39:33 nixos kernel: nouveau 0000:05:00.0: gst-plugin-scan[4287]: channel 1 killed!
Jun 10 12:39:33 nixos gst-plugin-scan[4287]: gst_va_filter_new: assertion 'GST_IS_VA_DISPLAY (display)' failed
Jun 10 12:39:33 nixos gst-plugin-scan[4287]: gst_va_filter_open: assertion 'GST_IS_VA_FILTER (self)' failed
Jun 10 12:39:33 nixos gst-plugin-scan[4287]: gst_va_filter_install_deinterlace_properties: assertion 'GST_IS_VA_FILTER (self)' failed
Jun 10 12:39:33 nixos gst-plugin-scan[4287]: gst_object_unref: assertion 'object != NULL' failed
Jun 10 12:39:33 nixos gst-plugin-scan[4287]: gst_object_unref: assertion 'object != NULL' failed
Jun 10 12:39:33 nixos gst-plugin-scan[4287]: gst_va_filter_new: assertion 'GST_IS_VA_DISPLAY (display)' failed
Jun 10 12:39:33 nixos gst-plugin-scan[4287]: gst_va_filter_open: assertion 'GST_IS_VA_FILTER (self)' failed
Jun 10 12:39:33 nixos gst-plugin-scan[4287]: gst_object_unref: assertion 'object != NULL' failed
Jun 10 12:39:33 nixos gst-plugin-scan[4287]: gst_object_unref: assertion 'object != NULL' failed

The kernel then stops all jobs after that core dump.

This is part of gstreamer. I would comment out packages in the modules/nixos/packages.nix file and modules/shared/packages.nix that have anything to do with image or video, like ffmpeg and vlc.

If that doesn't work, if you have the time, try to get something without many packages or services enabled working first, then turn things on one by one. Unfortunately, it's hard for me to debug Nix errors on different hardware than my own.

[dustinlyons]

And flake.nix vs configuration.nix shouldn't matter much, as it's Nix all the way down. So if you get it working there you could try to enable the same packages/services and see what breaks.

[justinwaltrip]

I'll try that and report back. Thanks for your help!

[justinwaltrip]

I was able to get the flake setup working after commenting out most of the imports, but I'm not quite sure which one was causing the boot issues. I don't have enough time to keep playing with this setup for my daily driver, so I'm going to dual boot and experiment with this as a secondary system. I'll try to update here if I figure out the issue

[dustinlyons]

It's also possible by the time you figure out the package, it's fixed upstream. But glad you're able to get this configuration working, albeit with less software 😃