search

frappe / press

Full service cloud hosting for the Frappe stack - powers Frappe Cloud
https://frappe.cloud
GNU Affero General Public License v3.0
250 stars 176 forks source link

Allow custom queues/workers on benches #1903

Open adityahase opened 2 months ago

adityahase commented 2 months ago

Frappe and Bench do most of the heavy lifting anyway, but this needs

This will change memory allocation and memory calculation. For now, support this on dedicated servers.

References: https://github.com/frappe/frappe/blob/a4d396ae508aae83bc495ce0dc09cc548b84df9b/frappe/utils/background_jobs.py#L42-L61 https://github.com/frappe/bench/blob/07b1462e603984ea9f3bb58567bacec3d8ccc86c/bench/config/templates/supervisor.conf#L77-L92

LintangVienusa commented 1 month ago

Hello, this is the following supervisor.conf and common_site_config.json that are currently running on my server

this is my config for custom worker on common_site_config.json looks like

"workers": {
  "my_custom_worker": {
   "background_workers": 2,
   "timeout": 5000
  },
  "my_custom_worker2": {
   "background_workers": 1,
   "timeout": 5000
  }
}

and this is the supervisor.conf

; Notes:
; priority=1 --> Lower priorities indicate programs that start first and shut down last
; killasgroup=true --> send kill signal to child processes too

[program:frappe-bench-frappe-web]
command=/home/ubuntu/app_directory/frappe-bench/env/bin/gunicorn -b 127.0.0.1:8001 -w 5 --max-requests 5000 --max-requests-jitter 500 -t 1200 frappe.app:application --preload
priority=4
autostart=true
autorestart=true
stdout_logfile=/home/ubuntu/app_directory/frappe-bench/logs/web.log
stderr_logfile=/home/ubuntu/app_directory/frappe-bench/logs/web.error.log
user=ubuntu
directory=/home/ubuntu/app_directory/frappe-bench/sites

[program:frappe-bench-frappe-schedule]
command=/usr/local/bin/bench schedule
priority=3
autostart=true
autorestart=true
stdout_logfile=/home/ubuntu/app_directory/frappe-bench/logs/schedule.log
stderr_logfile=/home/ubuntu/app_directory/frappe-bench/logs/schedule.error.log
user=ubuntu
directory=/home/ubuntu/app_directory/frappe-bench

[program:frappe-bench-frappe-default-worker]
command=/usr/local/bin/bench worker --queue default
priority=4
autostart=true
autorestart=true
stdout_logfile=/home/ubuntu/app_directory/frappe-bench/logs/worker.log
stderr_logfile=/home/ubuntu/app_directory/frappe-bench/logs/worker.error.log
user=ubuntu
stopwaitsecs=1560
directory=/home/ubuntu/app_directory/frappe-bench
killasgroup=true
numprocs=3
process_name=%(program_name)s-%(process_num)d

[program:frappe-bench-frappe-short-worker]
command=/usr/local/bin/bench worker --queue short
priority=4
autostart=true
autorestart=true
stdout_logfile=/home/ubuntu/app_directory/frappe-bench/logs/worker.log
stderr_logfile=/home/ubuntu/app_directory/frappe-bench/logs/worker.error.log
user=ubuntu
stopwaitsecs=360
directory=/home/ubuntu/app_directory/frappe-bench
killasgroup=true
numprocs=3
process_name=%(program_name)s-%(process_num)d

[program:frappe-bench-frappe-long-worker]
command=/usr/local/bin/bench worker --queue long
priority=4
autostart=true
autorestart=true
stdout_logfile=/home/ubuntu/app_directory/frappe-bench/logs/worker.log
stderr_logfile=/home/ubuntu/app_directory/frappe-bench/logs/worker.error.log
user=ubuntu
stopwaitsecs=1560
directory=/home/ubuntu/app_directory/frappe-bench
killasgroup=true
numprocs=3
process_name=%(program_name)s-%(process_num)d

[program:frappe-bench-frappe-my_custom_worker-worker]
command=/usr/local/bin/bench worker --queue my_custom_worker
priority=4
autostart=true
autorestart=true
stdout_logfile=/home/ubuntu/app_directory/frappe-bench/logs/my_custom_worker.log
stderr_logfile=/home/ubuntu/app_directory/frappe-bench/logs/my_custom_worker.error.log
user=ubuntu
stopwaitsecs=5000
directory=/home/ubuntu/app_directory/frappe-bench
killasgroup=true
numprocs=2
process_name=%(program_name)s-%(process_num)d

[program:frappe-bench-frappe-my_custom_worker2-worker]
command=/usr/local/bin/bench worker --queue my_custom_worker2
priority=4
autostart=true
autorestart=true
stdout_logfile=/home/ubuntu/app_directory/frappe-bench/logs/my_custom_worker2.log
stderr_logfile=/home/ubuntu/app_directory/frappe-bench/logs/my_custom_worker2.error.log
user=ubuntu
stopwaitsecs=5000
directory=/home/ubuntu/app_directory/frappe-bench
killasgroup=true
numprocs=1
process_name=%(program_name)s-%(process_num)d

[program:frappe-bench-redis-cache]
command=/usr/bin/redis-server /home/ubuntu/app_directory/frappe-bench/config/redis_cache.conf
priority=1
autostart=true
autorestart=true
stdout_logfile=/home/ubuntu/app_directory/frappe-bench/logs/redis-cache.log
stderr_logfile=/home/ubuntu/app_directory/frappe-bench/logs/redis-cache.error.log
user=ubuntu
directory=/home/ubuntu/app_directory/frappe-bench/sites

[program:frappe-bench-redis-queue]
command=/usr/bin/redis-server /home/ubuntu/app_directory/frappe-bench/config/redis_queue.conf
priority=1
autostart=true
autorestart=true
stdout_logfile=/home/ubuntu/app_directory/frappe-bench/logs/redis-queue.log
stderr_logfile=/home/ubuntu/app_directory/frappe-bench/logs/redis-queue.error.log
user=ubuntu
directory=/home/ubuntu/app_directory/frappe-bench/sites

[program:frappe-bench-redis-socketio]
command=/usr/bin/redis-server /home/ubuntu/app_directory/frappe-bench/config/redis_socketio.conf
priority=1
autostart=true
autorestart=true
stdout_logfile=/home/ubuntu/app_directory/frappe-bench/logs/redis-socketio.log
stderr_logfile=/home/ubuntu/app_directory/frappe-bench/logs/redis-socketio.error.log
user=ubuntu
directory=/home/ubuntu/app_directory/frappe-bench/sites

[program:frappe-bench-node-socketio]
command=/usr/bin/node /home/ubuntu/app_directory/frappe-bench/apps/frappe/socketio.js
priority=4
autostart=true
autorestart=true
stdout_logfile=/home/ubuntu/app_directory/frappe-bench/logs/node-socketio.log
stderr_logfile=/home/ubuntu/app_directory/frappe-bench/logs/node-socketio.error.log
user=ubuntu
directory=/home/ubuntu/app_directory/frappe-bench

[group:frappe-bench-web]
programs=frappe-bench-frappe-web,frappe-bench-node-socketio

[group:frappe-bench-workers]
programs=frappe-bench-frappe-schedule,frappe-bench-frappe-default-worker,frappe-bench-frappe-short-worker,frappe-bench-frappe-long-worker, frappe-bench-frappe-my_custom_worker-worker, frappe-bench-frappe-my_custom_worker2-worker

[group:frappe-bench-redis]
programs=frappe-bench-redis-cache,frappe-bench-redis-queue,frappe-bench-redis-socketio