frappe / bench

CLI to manage Multi-tenant deployments for Frappe apps
https://frappeframework.com/docs/user/en/bench
GNU General Public License v3.0
1.4k stars 1.2k forks source link

Introduction to Nix #1428

Open blaggacao opened 1 year ago

blaggacao commented 1 year ago

Sets up the Environment ...

(by leveraging PATH)

#! /nix/store/qqa28hmysc23yy081d178jfd9a1yk8aw-bash-5.2-p15/bin/bash -e

# .../bin/bench
# /nix/store/7772zxrjrbddvy6r3yrw2q7jc5r1z316-bench-5.16.1/bin/bench

PATH=${PATH:+':'$PATH':'}
PATH=${PATH/':''/nix/store/g5xjk98va4vqxdcpxppari5677rdw7dv-python3.10-watchdog-2.2.0/bin'':'/':'}
PATH='/nix/store/g5xjk98va4vqxdcpxppari5677rdw7dv-python3.10-watchdog-2.2.0/bin'$PATH
PATH=${PATH#':'}
PATH=${PATH%':'}
export PATH
PATH=${PATH:+':'$PATH':'}
PATH=${PATH/':''/nix/store/01vrw36wbpvlpbzvy8qg05d2lflkfpd4-python3.10-babel-2.11.0/bin'':'/':'}
PATH='/nix/store/01vrw36wbpvlpbzvy8qg05d2lflkfpd4-python3.10-babel-2.11.0/bin'$PATH
PATH=${PATH#':'}
PATH=${PATH%':'}
export PATH
PATH=${PATH:+':'$PATH':'}
PATH=${PATH/':''/nix/store/y7ashsjvli92qpn63aj2lgd1jvws5ky4-python3.10-staticjinja-4.1.3/bin'':'/':'}
PATH='/nix/store/y7ashsjvli92qpn63aj2lgd1jvws5ky4-python3.10-staticjinja-4.1.3/bin'$PATH
PATH=${PATH#':'}
PATH=${PATH%':'}
export PATH
PATH=${PATH:+':'$PATH':'}
PATH=${PATH/':''/nix/store/k8g3nrmvhx333kizrg6vjjgrpg2sy3mi-honcho-1.1.0/bin'':'/':'}
PATH='/nix/store/k8g3nrmvhx333kizrg6vjjgrpg2sy3mi-honcho-1.1.0/bin'$PATH
PATH=${PATH#':'}
PATH=${PATH%':'}
export PATH
PATH=${PATH:+':'$PATH':'}
PATH=${PATH/':''/nix/store/5ygj2xggw8zxqwkwv8pc420gzl7775li-python3.10-charset-normalizer-3.0.1/bin'':'/':'}
PATH='/nix/store/5ygj2xggw8zxqwkwv8pc420gzl7775li-python3.10-charset-normalizer-3.0.1/bin'$PATH
PATH=${PATH#':'}
PATH=${PATH%':'}
export PATH
PATH=${PATH:+':'$PATH':'}
PATH=${PATH/':''/nix/store/7cc9vik4l4x562i8fj8anbg8vlb1vfph-python3.10-supervisor-4.2.5/bin'':'/':'}
PATH='/nix/store/7cc9vik4l4x562i8fj8anbg8vlb1vfph-python3.10-supervisor-4.2.5/bin'$PATH
PATH=${PATH#':'}
PATH=${PATH%':'}
export PATH
PATH=${PATH:+':'$PATH':'}
PATH=${PATH/':''/nix/store/flz0j8byjnpd9fnn9vfkf9s923zfkhb3-python3.10-pip-22.3.1/bin'':'/':'}
PATH='/nix/store/flz0j8byjnpd9fnn9vfkf9s923zfkhb3-python3.10-pip-22.3.1/bin'$PATH
PATH=${PATH#':'}
PATH=${PATH%':'}
export PATH
PATH=${PATH:+':'$PATH':'}
PATH=${PATH/':''/nix/store/d6hwhgbh8hrzl7j57in9rz7qzy6hjyga-nginx-1.22.1/bin'':'/':'}
PATH='/nix/store/d6hwhgbh8hrzl7j57in9rz7qzy6hjyga-nginx-1.22.1/bin'$PATH
PATH=${PATH#':'}
PATH=${PATH%':'}
export PATH
PATH=${PATH:+':'$PATH':'}
PATH=${PATH/':''/nix/store/rrrqp9ras268q1994x40d4pqm4qk68m8-cron-4.1/bin'':'/':'}
PATH='/nix/store/rrrqp9ras268q1994x40d4pqm4qk68m8-cron-4.1/bin'$PATH
PATH=${PATH#':'}
PATH=${PATH%':'}
export PATH
PATH=${PATH:+':'$PATH':'}
PATH=${PATH/':''/nix/store/ggf8mmwmgprrvxj6ddckpbyhi8jmjzb3-yarn-1.22.19/bin'':'/':'}
PATH='/nix/store/ggf8mmwmgprrvxj6ddckpbyhi8jmjzb3-yarn-1.22.19/bin'$PATH
PATH=${PATH#':'}
PATH=${PATH%':'}
export PATH
PATH=${PATH:+':'$PATH':'}
PATH=${PATH/':''/nix/store/7nnhagzxb32ccjmhfqcq74s5zi63khk7-postgresql-14.6/bin'':'/':'}
PATH='/nix/store/7nnhagzxb32ccjmhfqcq74s5zi63khk7-postgresql-14.6/bin'$PATH
PATH=${PATH#':'}
PATH=${PATH%':'}
export PATH
PATH=${PATH:+':'$PATH':'}
PATH=${PATH/':''/nix/store/q9wlzalh2p0inc26bv9syqjxnlh85y8l-mariadb-server-10.6.11/bin'':'/':'}
PATH='/nix/store/q9wlzalh2p0inc26bv9syqjxnlh85y8l-mariadb-server-10.6.11/bin'$PATH
PATH=${PATH#':'}
PATH=${PATH%':'}
export PATH
PATH=${PATH:+':'$PATH':'}
PATH=${PATH/':''/nix/store/wb53r1aggfqm51cxw4zsa2prz4k24j7r-nodejs-18.14.0/bin'':'/':'}
PATH='/nix/store/wb53r1aggfqm51cxw4zsa2prz4k24j7r-nodejs-18.14.0/bin'$PATH
PATH=${PATH#':'}
PATH=${PATH%':'}
export PATH
PATH=${PATH:+':'$PATH':'}
PATH=${PATH/':''/nix/store/gl74nx1p7fklq5yr4j57krp9zqly5ijz-redis-7.0.8/bin'':'/':'}
PATH='/nix/store/gl74nx1p7fklq5yr4j57krp9zqly5ijz-redis-7.0.8/bin'$PATH
PATH=${PATH#':'}
PATH=${PATH%':'}
export PATH
PATH=${PATH:+':'$PATH':'}
PATH=${PATH/':''/nix/store/h4gphlrasiqmk894jm9bizacyr8ixxcb-git-minimal-2.39.1/bin'':'/':'}
PATH='/nix/store/h4gphlrasiqmk894jm9bizacyr8ixxcb-git-minimal-2.39.1/bin'$PATH
PATH=${PATH#':'}
PATH=${PATH%':'}
export PATH
PATH=${PATH:+':'$PATH':'}
PATH=${PATH/':''/nix/store/lljkbrqh1nkayprky3lrigj0fbxj35rv-coreutils-9.1/bin'':'/':'}
PATH='/nix/store/lljkbrqh1nkayprky3lrigj0fbxj35rv-coreutils-9.1/bin'$PATH
PATH=${PATH#':'}
PATH=${PATH%':'}
export PATH
PATH=${PATH:+':'$PATH':'}
PATH=${PATH/':''/nix/store/7772zxrjrbddvy6r3yrw2q7jc5r1z316-bench-5.16.1/bin'':'/':'}
PATH='/nix/store/7772zxrjrbddvy6r3yrw2q7jc5r1z316-bench-5.16.1/bin'$PATH
PATH=${PATH#':'}
PATH=${PATH%':'}
export PATH
PATH=${PATH:+':'$PATH':'}
PATH=${PATH/':''/nix/store/sp5x6s8n36gjlwck74xhj1i61p66vcpa-python3-3.10.9/bin'':'/':'}
PATH='/nix/store/sp5x6s8n36gjlwck74xhj1i61p66vcpa-python3-3.10.9/bin'$PATH
PATH=${PATH#':'}
PATH=${PATH%':'}
export PATH
export PYTHONNOUSERSITE='true'
exec -a "$0" "/nix/store/7772zxrjrbddvy6r3yrw2q7jc5r1z316-bench-5.16.1/bin/.bench-wrapped"  "$@" 

... and calls python to setup import paths

(by leveraging python's site.addsitedir())

... before it calls the bench cli

#!/nix/store/sp5x6s8n36gjlwck74xhj1i61p66vcpa-python3-3.10.9/bin/python3.10
# -*- coding: utf-8 -*-

# .../bin/.bench-wrapped
# /nix/store/7772zxrjrbddvy6r3yrw2q7jc5r1z316-bench-5.16.1/bin/.bench-wrapped

import sys;import site;import functools;sys.argv[0] = '/nix/store/7772zxrjrbddvy6r3yrw2q7jc5r1z316-bench-5.16.1/bin/bench';
functools.reduce(lambda k, p: site.addsitedir(p, k), [
'/nix/store/7772zxrjrbddvy6r3yrw2q7jc5r1z316-bench-5.16.1/lib/python3.10/site-packages',
'/nix/store/flz0j8byjnpd9fnn9vfkf9s923zfkhb3-python3.10-pip-22.3.1/lib/python3.10/site-packages',
'/nix/store/8d7cyaf477jns57d4f376qqlh41wxdkp-python3.10-psutil-5.9.4/lib/python3.10/site-packages',
'/nix/store/7cc9vik4l4x562i8fj8anbg8vlb1vfph-python3.10-supervisor-4.2.5/lib/python3.10/site-packages',
'/nix/store/yvv04lrgq2m4vkwlzs9jv720vj9jhdzy-python3.10-setuptools-65.6.3/lib/python3.10/site-packages',
'/nix/store/mb94x06narsvx5pmsah24asj238kqmq5-python3.10-click-8.1.3/lib/python3.10/site-packages',
'/nix/store/p5j8vgxjn7qq4vavmy72fc3jr8zrayrm-python3.10-gitpython-3.1.30/lib/python3.10/site-packages',
'/nix/store/hp64qlc40h3nlvl4xrbna9anb9ilnzng-python3.10-ddt-1.6.0/lib/python3.10/site-packages',
'/nix/store/rhr8p5q8gpbfxrh7pzjvrh7fkl3vzbxr-python3.10-gitdb-4.0.10/lib/python3.10/site-packages',
'/nix/store/l856mnq0ppya1v38mnrix24ax9ppbncd-python3.10-smmap-5.0.0/lib/python3.10/site-packages',
'/nix/store/3hxsj2ijs9hksi66h9pnr37wbx7g632w-python3.10-python-crontab-2.7.1/lib/python3.10/site-packages',
'/nix/store/ky1i1yagd5nmsdidgjlzkwic4baihjj6-python3.10-python-dateutil-2.8.2/lib/python3.10/site-packages',
'/nix/store/f7674syc19gcak5ja2m7kh0hnmm2cmp9-python3.10-six-1.16.0/lib/python3.10/site-packages',
'/nix/store/83xlqm06n4nw5vsm3rq71pmcyrxyimjj-python3.10-requests-2.28.2/lib/python3.10/site-packages',
'/nix/store/pq70cr5mvvi85k900c5cjks9j3dgpj1j-python3.10-brotlicffi-1.0.9.2/lib/python3.10/site-packages',
'/nix/store/320n0hhxf5fw3gp7n7n20cykqkfd2r4z-python3.10-cffi-1.15.1/lib/python3.10/site-packages',
'/nix/store/9igpaz82ss6j2hlyadaqbzjjp5kq6ksy-python3.10-pycparser-2.21/lib/python3.10/site-packages',
'/nix/store/ngpp0adl92575zd7mbjyd1056912p9rw-python3.10-certifi-2022.12.07/lib/python3.10/site-packages',
'/nix/store/5ygj2xggw8zxqwkwv8pc420gzl7775li-python3.10-charset-normalizer-3.0.1/lib/python3.10/site-packages',
'/nix/store/1dbaqx82pc37wlsjj61l17f2ixkb1b0a-python3.10-idna-3.4/lib/python3.10/site-packages',
'/nix/store/iny2gz590w08d5jp4qig1vd6qdcinx8y-python3.10-urllib3-1.26.13/lib/python3.10/site-packages',
'/nix/store/j9il0yxjng8p7ldshxhr8h58rgwybyqd-python3.10-brotli-1.0.9/lib/python3.10/site-packages',
'/nix/store/hrlynb3r3laflzysp58a5vnzj7iq1m0i-python3.10-pysocks-1.7.1/lib/python3.10/site-packages',
'/nix/store/8hm3jh41zi4pi8fh694swrxwjj0s9p6m-python3.10-semantic-version-2.10.0/lib/python3.10/site-packages',
'/nix/store/f8qybwiy65ly3r4sxm2k82y949an4vbz-python3.10-tomli-2.0.1/lib/python3.10/site-packages',
'/nix/store/k8g3nrmvhx333kizrg6vjjgrpg2sy3mi-honcho-1.1.0/lib/python3.10/site-packages',
'/nix/store/y7ashsjvli92qpn63aj2lgd1jvws5ky4-python3.10-staticjinja-4.1.3/lib/python3.10/site-packages',
'/nix/store/zfm8c04ks25n2dmhckgyp9xkgmhkinx1-python3.10-Jinja2-3.1.2/lib/python3.10/site-packages',
'/nix/store/01vrw36wbpvlpbzvy8qg05d2lflkfpd4-python3.10-babel-2.11.0/lib/python3.10/site-packages',
'/nix/store/x2ansvgbfrdffmixl6haz2aknafz46m5-python3.10-pytz-2022.7/lib/python3.10/site-packages',
'/nix/store/1xcy3dpllq6xw1pcbff9pixxqa4djl3b-python3.10-markupsafe-2.1.1/lib/python3.10/site-packages',
'/nix/store/9w3l7jf8scv2szns7x63mppqd36nyqk9-python3.10-docopt-ng-0.8.1/lib/python3.10/site-packages',
'/nix/store/vwbjy2hh6qns82gf72gb9nwk5cv5dfcf-python3.10-easywatch-0.0.5/lib/python3.10/site-packages',
'/nix/store/g5xjk98va4vqxdcpxppari5677rdw7dv-python3.10-watchdog-2.2.0/lib/python3.10/site-packages',
'/nix/store/c5z2slw9jd52ayb38id31vc3dib1iflc-python3.10-pathtools-0.1.2/lib/python3.10/site-packages',
'/nix/store/l1g7f4srglfsqbpk15cv2nphs527y7kw-python3.10-pyyaml-6.0/lib/python3.10/site-packages'
], site._init_pathinfo());
import re
import sys
from bench.cli import cli
if __name__ == '__main__':
    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
    sys.exit(cli())

Since all paths in /nix/* are reproducible and automatically fetched from cache.nixos.org, and provided that I didn't miss any intricacies while reverse-engineering the runtime deps from ansible roles or the pyproject.toml, we have a complete (non-root) bench environment (excluding services).

Since reverse engineering that stuff from all over the place is hard and likely also a significant chore for the different target OS systems, I get the impression that it might be worth it to dig a little deeper on the capabilities of this system since it looks like it can cut (really: abstract) a lot of maintenance complexity with quite impressive results.

Specifically, after analyzing the code, it could replace bench install and parts of bench setup with minimal declarative data input to the nix binary, that roughly looks like this:

{
  /* snip */
  # if it's a python lib it ends in the python wrapper
  # if it also has a `./bin` it ends up in the bash wrapper
  # ... that's the practical meaning of propagated*
  propagatedBuildInputs = [
    coreutils
    gitMinimal
    # non-python runtime deps
    redis
    nodejs
    mariadb
    postgresql
    yarn
    cron
    # wkhtmltopdf - has unmaintained dep; pdf printing not available out of the box
    # https://github.com/frappe/bench/issues/1427
    nginx
  ] ++ (with python3.pkgs; [
    # for bench's own environment management
    pip
    supervisor
    psutil
    # other
    click
    gitpython
    python-crontab
    requests
    semantic-version
    setuptools
    tomli
    # python; but not in pythonPackages
    honcho
    staticjinja
  ]);

  nativeBuildInputs = with python3.pkgs; [
    hatchling
  ];
  /* snip */
}
vasujain275 commented 1 week ago

@blaggacao Did you find a way to init a bench branch on nixos. The crobtab thing is driving me crazy. Here is the flake.nix I am using for dev shell -

{
  description = "Development environment for Bench installation with cron support";
  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
    flake-utils.url = "github:numtide/flake-utils";
  };
  outputs = { self, nixpkgs, flake-utils }:
    flake-utils.lib.eachDefaultSystem (system:
      let
        pkgs = import nixpkgs {
          inherit system;
          config.allowUnfree = true;
        };
        python3 = pkgs.python3;
      in
      {
        devShells.default = pkgs.mkShell {
          buildInputs = with pkgs; [
            coreutils
            gitMinimal
            redis
            nodejs
            mariadb
            postgresql
            yarn
            cronie
            wkhtmltopdf
            nginx
            xvfb-run
            fontconfig
            curl
            uv
            zsh
            sudo
          ] ++ (with python3.pkgs; [
            pip
            supervisor
            psutil
            click
            gitpython
            python-crontab
            requests
            semantic-version
            setuptools
            tomli
            honcho
            staticjinja
          ]);
          nativeBuildInputs = with python3.pkgs; [
            hatchling
          ];
          shellHook = ''
            export SHELL=${pkgs.zsh}/bin/zsh

            # Setup virtual environment using uv
            if [ ! -d ".venv" ]; then
              uv venv
            fi
            source .venv/bin/activate

            # Install frappe-bench
            uv pip install frappe-bench

            echo "Bench development environment activated!"
            echo "Node.js version: $(node --version)"
            echo "Run 'bench init <directory-name>' to create a new Bench instance."

            # Create a directory for the PID file if it doesn't exist
            mkdir -p /run/crond

            # Start crond in the background with a custom PID file location
            ${pkgs.cronie}/bin/crond -n -p /run/crond/crond.pid &

            # Notify about cron jobs
            echo "Cron daemon started. To add jobs for root, use 'sudo crontab -e'."

            # Start Zsh
            exec zsh
          '';
        };
      }
    );
}
blaggacao commented 1 week ago

@vasujain275 https://github.com/blaggacao/frappix

vasujain275 commented 1 week ago

@blaggacao Thanks alottt man, its a life saver ❤️❤️