docker to run multiple version parallel IN with caddy and nginx

[GurjinderSingh]

Target : Run multiple IN version with different database, public/storage directory on different subdomain/domain behind caddy (auto ssl).

requirement Mysql/MariaDB Caddy external ip of host server for example 2.2.2.2

multiple Nginx are used to keep public directory of each IN different.

these configuration is for docker swarm mode so might need little changes for normal docker mode.

invoice.domain.tld: for production use always online and only upgrade Friday after backup.

version: '3.3'
services:
  invoiceninja:
    image: invoiceninja/invoiceninja:5.1.65
    extra_hosts:
     - invoice.domain.tld:2.2.2.2
    environment:
      APP_DEBUG: 'false'
      APP_KEY: base64:key here
      APP_URL: https://invoice.domain.tld 
      DB_DATABASE1: inninja
      DB_HOST1: MariaDB_Host
      DB_PASSWORD1: password
      DB_PORT1: '3306'
      DB_TYPE: MariaDB
      DB_USERNAME1: invoiceninja
      IS_DOCKER: 'true'
      MAIL_MAILER: log
      MULTI_DB_ENABLED: 'false'
      PHANTOMJS_PDF_GENERATION: 'false'
      TRUSTED_PROXIES: "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
    volumes:
     - /gfdisk/gv1/invoiceninja/public:/var/www/app/public
     - /gfdisk/gv1/invoiceninja/storage:/var/www/app/storage
     - /gfdisk/gv1/invoiceninja/hosts:/etc/hosts
    networks:
     - invoiceninja
     - mydbnet
    logging:
      driver: json-file
    deploy:
      update_config:
        delay: 600s
  web:
    image: nginx:latest
    extra_hosts:
     - invoice.domain.tld:2.2.2.2
    volumes:
     - /gfdisk/gv1/invoiceninja/in-vhost.conf:/etc/nginx/conf.d/in-vhost.conf:ro
     - /gfdisk/gv1/invoiceninja/public:/var/www/app/public:ro
     - /gfdisk/gv1/invoiceninja/hosts:/etc/hosts:ro
    networks:
     - caddy_caddy
     - invoiceninja
    logging:
      driver: json-file
    deploy:
      labels:
        caddy: invoice.domain.tld
        caddy.2_reverse_proxy: '{{upstreams 80}}'
        caddy.3_encode: zstd gzip
networks:
  caddy_caddy:
    external: true
  invoiceninja:
    driver: overlay
  mydbnet:
    external: true

in-vhost.conf

most important configuration here is root directory and invoice ninja service name. in case IN container restart it gets new ip every time nginx need to get new ip but it crash. using set/variable will prevent nginx crash.

server {
    listen 80 default_server;
    server_name _;

    client_max_body_size 100M;

    root /var/www/app/public/;
    index index.php;

    location / {
        try_files $uri $uri/ /index.php?$query_string;
    }

    location = /favicon.ico { access_log off; log_not_found off; }
    location = /robots.txt  { access_log off; log_not_found off; }

    location ~ \.php$ {
        fastcgi_split_path_info ^(.+\.php)(/.+)$;
        resolver 127.0.0.11 ipv6=off;
        set $service invoiceninja_invoiceninja;   #<<<<<<
        fastcgi_pass $service:9000;
        fastcgi_index index.php;
        include fastcgi_params;
        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
        fastcgi_intercept_errors off;
        fastcgi_buffer_size 16k;
        fastcgi_buffers 4 16k;
    }
}

hosts

domain and external ip.

2.2.2.2 invoice.domain.tld

Caddy Config

need to add nginx service ip or name.

  invoice.domain.tld {
    reverse_proxy invoiceninja_nginx
    encode zstd gzip
    log {
        level debug
    }
  }

testninja.domain.tld: for testing new version and to allow employees play with so they don't clutter production db with fake client and invoices. most important recurring invoices playground they can fill db with 1000's invoices for no reason.

Most config are same so posting changes only from above.

version: '3.3'
services:
  testninja:
    image: invoiceninja/invoiceninja:latest
    extra_hosts:
     - testninja.domain.tld:2.2.2.2
    environment:
      APP_URL: https://testninja.domain.tld 
      DB_DATABASE1: testninja
      DB_USERNAME1: testninja
    volumes:
     - /gfdisk/gv1/testninja/public:/var/www/app/public
     - /gfdisk/gv1/testninja/storage:/var/www/app/storage
     - /gfdisk/gv1/testninja/hosts:/etc/hosts
    networks:
     - testninja
     - mydbnet
  web:
    image: nginx:latest
    extra_hosts:
     - testninja.domain.tld:2.2.2.2
    volumes:
     - /gfdisk/gv1/testninja/in-vhost.conf:/etc/nginx/conf.d/in-vhost.conf:ro
     - /gfdisk/gv1/testninja/public:/var/www/app/public:ro
     - /gfdisk/gv1/testninja/hosts:/etc/hosts:ro
    networks:
     - caddy_caddy
     - testninja
    logging:
      driver: json-file
    deploy:
      labels:
        caddy: testninja.domain.tld
        caddy.2_reverse_proxy: '{{upstreams 80}}'
        caddy.3_encode: zstd gzip
networks:
  caddy_caddy:
    external: true
  testninja:
    driver: overlay
  mydbnet:
    external: true

in-vhost.conf

    root /var/www/app/public/;
        set $service testninja_testninja;   #<<<<<<
        fastcgi_pass $service:9000;

hosts 2.2.2.2 testninja.domain.tld

Caddy Config

  testninja.domain.tld {
    reverse_proxy testninja_nginx
  }

that's it now you can have two separate parallel IN on different subdomain.

little more about my setup

I have docker swarm with 3 vps and for persistent storage using glusterfs. managing docker cluster with portainer and swarmpit (very useful to get logs from all replicated services across cluster) .

I have custom caddy working with consul.

https://github.com/lucaslorentz/caddy-docker-proxy https://github.com/pteich/caddy-tlsconsul

caddy-docker-proxy helps in auto creating caddy config and updating ip for nginx service. their is some learning time but it save way more than you ever imagine. caddy-tlsconsul save all tls certificate to consul so they can be accessed by every caddy server across cluster.

this is my first working caddy docker compose in order to use please create your caddy docker image.

version: '3.3'
services:
  consul:
    image: consul:latest
    command:
     - agent
     - -server
     - -ui
     - -bootstrap-expect=1
    environment:
      CONSUL_BIND_INTERFACE: eth0
      CONSUL_CLIENT_INTERFACE: eth0
      CONSUL_LOCAL_CONFIG: '{"datacenter":"east_tor_do","server":true}'
    volumes:
     - /gfdisk/gv1/CaddyConsul:/consul/data
    networks:
     - consul
    logging:
      driver: json-file
    deploy:
      placement:
        constraints:
         - node.role == manager
  consul_relay:
    image: alpine/socat:latest
    command:
     - TCP-LISTEN:8500,fork,reuseaddr
     - tcp-connect:consul:8500
    ports:
     - 8500:8500
    networks:
     - consul
    logging:
      driver: json-file
    deploy:
      placement:
        constraints:
         - node.role == manager
  controller:
    image: registry.gitlab.com/---------------------/caddy:latest
    environment:
      CADDY_CONTROLLER_NETWORK: 10.200.200.0/24
      CADDY_DOCKER_CADDYFILE_PATH: /Caddy/config/Caddyfile
      CADDY_DOCKER_MODE: controller
      CONSUL_HTTP_ADDR: consul:8500
    volumes:
     - /var/run/docker.sock:/var/run/docker.sock
     - /gfdisk/gv1/Caddy:/Caddy
    networks:
     - caddy
     - consul
     - controller
    logging:
      driver: json-file
    deploy:
      placement:
        constraints:
         - node.role == manager
  server:
    image: registry.gitlab.com/--------------------/caddy:v2.4.0-beta.1-v2.0
    environment:
      CADDY_CONTROLLER_NETWORK: 10.200.200.0/24
      CADDY_DOCKER_MODE: server
      CONSUL_HTTP_ADDR: consul:8500
    ports:
     - 80:80
     - 443:443
    volumes:
     - /gfdisk/gv1/Caddy/www:/www
     - /gfdisk/gv1/Caddy:/Caddy
    networks:
     - caddy
     - consul
     - controller
    logging:
      driver: json-file
    deploy:
      mode: global
      labels:
        caddy_controlled_server: ''
      update_config:
        delay: 10s
networks:
  caddy:
    driver: overlay
  consul:
    driver: overlay
  controller:
    driver: overlay

for reference

https://invoiceninja.slack.com/archives/C01N3QKA6JY/p1620675425040300

[turbo124]

Thanks I'll add these to the wiki