Open abdelati-mulhim opened 1 month ago
Following is my metrics endpoint output, some metrics like celery_task_failed_total are not working
celery_task_failed_total
# HELP celery_task_sent_total Sent when a task message is published. # TYPE celery_task_sent_total counter celery_task_sent_total{hostname="celery-beat-5d6d58c7f-ghqxb",name="app.celery.cleanup_task_results",queue_name="medium"} 8.0 celery_task_sent_total{hostname="celery-beat-5d6d58c7f-ghqxb",name="apps.celery.cron_run",queue_name="high"} 8.0 celery_task_sent_total{hostname="api-c7cc56d6-4cqlm",name="apps.badges.tasks.badge_trigger",queue_name="medium"} 35.0 celery_task_sent_total{hostname="api-c7cc56d6-4cqlm",name="apps.webhook.tasks.trainee_created_webhook_sender",queue_name="medium"} 1.0 celery_task_sent_total{hostname="api-c7cc56d6-pghvr",name="apps.badges.tasks.badge_trigger",queue_name="medium"} 34.0 celery_task_sent_total{hostname="api-c7cc56d6-pghvr",name="apps.webhook.tasks.trainee_created_webhook_sender",queue_name="medium"} 3.0 celery_task_sent_total{hostname="api-c7cc56d6-pghvr",name="apps.webhook.tasks.lx_webhook_sender",queue_name="medium"} 1.0 celery_task_sent_total{hostname="api-c7cc56d6-pghvr",name="apps.courses.tasks.fire_team_start",queue_name="medium"} 4.0 celery_task_sent_total{hostname="api-c7cc56d6-pghvr",name="apps.courses.tasks.calculate_user_progress",queue_name="critical"} 4.0 celery_task_sent_total{hostname="api-c7cc56d6-pghvr",name="apps.reports.tasks.feed_reporting",queue_name="high"} 4.0 celery_task_sent_total{hostname="api-c7cc56d6-fpbn2",name="apps.badges.tasks.badge_trigger",queue_name="medium"} 30.0 celery_task_sent_total{hostname="api-c7cc56d6-4cqlm",name="apps.courses.tasks.fire_team_start",queue_name="medium"} 8.0 celery_task_sent_total{hostname="api-c7cc56d6-4cqlm",name="apps.courses.tasks.calculate_user_progress",queue_name="critical"} 8.0 celery_task_sent_total{hostname="api-c7cc56d6-4cqlm",name="apps.reports.tasks.feed_reporting",queue_name="high"} 8.0 celery_task_sent_total{hostname="api-c7cc56d6-4cqlm",name="apps.webhook.tasks.lx_webhook_sender",queue_name="medium"} 2.0 celery_task_sent_total{hostname="api-c7cc56d6-fpbn2",name="apps.courses.tasks.fire_team_start",queue_name="medium"} 2.0 celery_task_sent_total{hostname="api-c7cc56d6-fpbn2",name="apps.courses.tasks.calculate_user_progress",queue_name="critical"} 2.0 celery_task_sent_total{hostname="api-c7cc56d6-fpbn2",name="apps.reports.tasks.feed_reporting",queue_name="high"} 2.0 celery_task_sent_total{hostname="celery-beat-5d6d58c7f-ghqxb",name="apps.celery.refresh_user_stat_materialized_view",queue_name="medium"} 3.0 celery_task_sent_total{hostname="celery-beat-5d6d58c7f-ghqxb",name="apps.notifications.tasks.daily_due_date_reminder",queue_name="medium"} 1.0 celery_task_sent_total{hostname="api-c7cc56d6-fpbn2",name="apps.courses.tasks.update_done_tries",queue_name="medium"} 1.0 celery_task_sent_total{hostname="api-c7cc56d6-4cqlm",name="apps.courses.tasks.update_done_tries",queue_name="medium"} 4.0 celery_task_sent_total{hostname="api-c7cc56d6-pghvr",name="apps.courses.tasks.update_done_tries",queue_name="medium"} 2.0 celery_task_sent_total{hostname="api-c7cc56d6-fpbn2",name="apps.celery.enqueue_send_email",queue_name="medium"} 18.0 celery_task_sent_total{hostname="api-c7cc56d6-pghvr",name="apps.celery.enqueue_send_email",queue_name="medium"} 20.0 celery_task_sent_total{hostname="api-c7cc56d6-4cqlm",name="apps.celery.enqueue_send_email",queue_name="medium"} 17.0 celery_task_sent_total{hostname="api-c7cc56d6-fpbn2",name="apps.webhook.tasks.trainee_created_webhook_sender",queue_name="medium"} 2.0 celery_task_sent_total{hostname="celery-beat-757f657d5d-d59t4",name="app.celery.cleanup_task_results",queue_name="medium"} 19.0 celery_task_sent_total{hostname="api-6b667bcd5b-l5dh7",name="apps.badges.tasks.badge_trigger",queue_name="medium"} 76.0 celery_task_sent_total{hostname="api-6b667bcd5b-l5dh7",name="apps.webhook.tasks.lx_webhook_sender",queue_name="medium"} 2.0 celery_task_sent_total{hostname="api-6b667bcd5b-l5dh7",name="apps.courses.tasks.fire_team_start",queue_name="medium"} 2.0 celery_task_sent_total{hostname="api-6b667bcd5b-l5dh7",name="apps.courses.tasks.calculate_user_progress",queue_name="critical"} 2.0 celery_task_sent_total{hostname="api-6b667bcd5b-l5dh7",name="apps.reports.tasks.feed_reporting",queue_name="high"} 2.0 celery_task_sent_total{hostname="celery-critical-64b77fd969-6dsp8",name="apps.badges.tasks.badge_trigger",queue_name="medium"} 2.0 celery_task_sent_total{hostname="celery-critical-64b77fd969-6dsp8",name="apps.webhook.tasks.lx_webhook_sender",queue_name="medium"} 2.0 celery_task_sent_total{hostname="celery-high-694788bdb9-qth45",name="apps.courses.tasks.calculate_user_progress",queue_name="critical"} 6.0 celery_task_sent_total{hostname="celery-high-694788bdb9-qth45",name="apps.badges.tasks.badge_trigger",queue_name="medium"} 12.0 celery_task_sent_total{hostname="celery-critical-64b77fd969-6dsp8",name="apps.notifications.tasks.push_bulk_notification",queue_name="medium"} 2.0 celery_task_sent_total{hostname="celery-critical-64b77fd969-6dsp8",name="apps.celery.enqueue_send_email",queue_name="medium"} 2.0 celery_task_sent_total{hostname="celery-critical-64b77fd969-6dsp8",name="apps.courses.tasks.container_user_update_points",queue_name="high"} 12.0 celery_task_sent_total{hostname="celery-high-694788bdb9-qth45",name="apps.courses.tasks.calculate_user_certificate",queue_name="critical"} 12.0 celery_task_sent_total{hostname="api-6b667bcd5b-cmhtf",name="apps.badges.tasks.badge_trigger",queue_name="medium"} 46.0 celery_task_sent_total{hostname="api-6b667bcd5b-cmhtf",name="apps.webhook.tasks.lx_webhook_sender",queue_name="medium"} 2.0 celery_task_sent_total{hostname="api-6b667bcd5b-cmhtf",name="apps.courses.tasks.fire_team_start",queue_name="medium"} 2.0 celery_task_sent_total{hostname="api-6b667bcd5b-cmhtf",name="apps.courses.tasks.calculate_user_progress",queue_name="critical"} 2.0 celery_task_sent_total{hostname="api-6b667bcd5b-cmhtf",name="apps.reports.tasks.feed_reporting",queue_name="high"} 2.0 celery_task_sent_total{hostname="celery-beat-757f657d5d-d59t4",name="apps.celery.cron_run",queue_name="high"} 18.0 celery_task_sent_total{hostname="api-6b667bcd5b-l5dh7",name="apps.webhook.tasks.trainee_created_webhook_sender",queue_name="medium"} 8.0 celery_task_sent_total{hostname="api-6b667bcd5b-zlgrd",name="apps.courses.tasks.fire_team_start",queue_name="medium"} 2.0 celery_task_sent_total{hostname="api-6b667bcd5b-zlgrd",name="apps.courses.tasks.calculate_user_progress",queue_name="critical"} 2.0 celery_task_sent_total{hostname="api-6b667bcd5b-zlgrd",name="apps.badges.tasks.badge_trigger",queue_name="medium"} 47.0 celery_task_sent_total{hostname="api-6b667bcd5b-zlgrd",name="apps.reports.tasks.feed_reporting",queue_name="high"} 2.0 celery_task_sent_total{hostname="celery-high-694788bdb9-qth45",name="apps.reports.tasks.feed_reporting",queue_name="high"} 1.0 celery_task_sent_total{hostname="api-6b667bcd5b-zlgrd",name="apps.webhook.tasks.trainee_created_webhook_sender",queue_name="medium"} 5.0 celery_task_sent_total{hostname="celery-beat-757f657d5d-d59t4",name="apps.celery.refresh_user_stat_materialized_view",queue_name="medium"} 6.0 celery_task_sent_total{hostname="api-6b667bcd5b-cmhtf",name="apps.celery.enqueue_send_email",queue_name="medium"} 33.0 celery_task_sent_total{hostname="api-6b667bcd5b-l5dh7",name="apps.celery.enqueue_send_email",queue_name="medium"} 53.0 celery_task_sent_total{hostname="api-6b667bcd5b-zlgrd",name="apps.celery.enqueue_send_email",queue_name="medium"} 30.0 celery_task_sent_total{hostname="api-6b667bcd5b-zlgrd",name="apps.customers.tasks.change_customer_has_assignment",queue_name="medium"} 1.0 celery_task_sent_total{hostname="api-6b667bcd5b-cmhtf",name="apps.webhook.tasks.trainee_created_webhook_sender",queue_name="medium"} 1.0 celery_task_sent_total{hostname="api-6b667bcd5b-l5dh7",name="apps.customers.tasks.call_netlify_build_task",queue_name="medium"} 3.0 celery_task_sent_total{hostname="api-6b667bcd5b-cmhtf",name="apps.customers.tasks.call_netlify_build_task",queue_name="medium"} 1.0 celery_task_sent_total{hostname="api-6b667bcd5b-zlgrd",name="apps.customers.tasks.call_netlify_build_task",queue_name="medium"} 2.0 celery_task_sent_total{hostname="celery-beat-757f657d5d-d59t4",name="apps.users.tasks.get_scfhs_users_data",queue_name="medium"} 1.0 celery_task_sent_total{hostname="api-6b667bcd5b-l5dh7",name="apps.customers.tasks.change_customer_has_assignment",queue_name="medium"} 2.0 celery_task_sent_total{hostname="celery-beat-757f657d5d-d59t4",name="apps.base.tasks.daily_cache_check",queue_name="medium"} 1.0 celery_task_sent_total{hostname="celery-beat-757f657d5d-d59t4",name="apps.reports.tasks.feed_reporting_with_median_time",queue_name="medium"} 1.0 celery_task_sent_total{hostname="celery-medium-7496c9c6fb-tjr59",name="apps.notifications.tasks.send_slack_message",queue_name="medium"} 1.0 celery_task_sent_total{hostname="celery-beat-757f657d5d-d59t4",name="apps.users.tasks.update_user_stats_csv",queue_name="low"} 1.0 celery_task_sent_total{hostname="celery-low-5f48c6d9-7qmmv",name="apps.users.tasks.upload_user_stats_csv_to_s3",queue_name="low"} 159.0 celery_task_sent_total{hostname="celery-low-5f48c6d9-7qmmv",name="apps.customers.tasks.call_netlify_build_task",queue_name="medium"} 154.0 celery_task_sent_total{hostname="celery-beat-757f657d5d-d59t4",name="apps.users.tasks.add_scfhs_interns_to_team",queue_name="medium"} 1.0 celery_task_sent_total{hostname="celery-beat-757f657d5d-d59t4",name="celery.backend_cleanup",queue_name="medium"} 1.0 celery_task_sent_total{hostname="celery-beat-757f657d5d-d59t4",name="apps.webhook.tasks.time_log_in_past_24h_schedule",queue_name="medium"} 1.0 celery_task_sent_total{hostname="celery-beat-757f657d5d-d59t4",name="apps.notifications.tasks.send_customer_observatory_notification",queue_name="medium"} 1.0 celery_task_sent_total{hostname="celery-medium-7496c9c6fb-tjr59",name="apps.celery.enqueue_send_email",queue_name="medium"} 9.0 celery_task_sent_total{hostname="celery-beat-757f657d5d-d59t4",name="apps.notifications.tasks.daily_due_date_reminder",queue_name="medium"} 1.0 # HELP celery_task_sent_created Sent when a task message is published. # TYPE celery_task_sent_created gauge celery_task_sent_created{hostname="celery-beat-5d6d58c7f-ghqxb",name="app.celery.cleanup_task_results",queue_name="medium"} 1.716194888990151e+09 celery_task_sent_created{hostname="celery-beat-5d6d58c7f-ghqxb",name="apps.celery.cron_run",queue_name="high"} 1.7161956000044553e+09 celery_task_sent_created{hostname="api-c7cc56d6-4cqlm",name="apps.badges.tasks.badge_trigger",queue_name="medium"} 1.716195731272917e+09 celery_task_sent_created{hostname="api-c7cc56d6-4cqlm",name="apps.webhook.tasks.trainee_created_webhook_sender",queue_name="medium"} 1.716195731281773e+09 celery_task_sent_created{hostname="api-c7cc56d6-pghvr",name="apps.badges.tasks.badge_trigger",queue_name="medium"} 1.7161967025909996e+09 celery_task_sent_created{hostname="api-c7cc56d6-pghvr",name="apps.webhook.tasks.trainee_created_webhook_sender",queue_name="medium"} 1.716196702599095e+09 celery_task_sent_created{hostname="api-c7cc56d6-pghvr",name="apps.webhook.tasks.lx_webhook_sender",queue_name="medium"} 1.7161967080115736e+09 celery_task_sent_created{hostname="api-c7cc56d6-pghvr",name="apps.courses.tasks.fire_team_start",queue_name="medium"} 1.716196708029697e+09 celery_task_sent_created{hostname="api-c7cc56d6-pghvr",name="apps.courses.tasks.calculate_user_progress",queue_name="critical"} 1.716196708032497e+09 celery_task_sent_created{hostname="api-c7cc56d6-pghvr",name="apps.reports.tasks.feed_reporting",queue_name="high"} 1.7161967080369968e+09 celery_task_sent_created{hostname="api-c7cc56d6-fpbn2",name="apps.badges.tasks.badge_trigger",queue_name="medium"} 1.716197888070591e+09 celery_task_sent_created{hostname="api-c7cc56d6-4cqlm",name="apps.courses.tasks.fire_team_start",queue_name="medium"} 1.7161985129468699e+09 celery_task_sent_created{hostname="api-c7cc56d6-4cqlm",name="apps.courses.tasks.calculate_user_progress",queue_name="critical"} 1.7161985129472618e+09 celery_task_sent_created{hostname="api-c7cc56d6-4cqlm",name="apps.reports.tasks.feed_reporting",queue_name="high"} 1.7161985129478815e+09 celery_task_sent_created{hostname="api-c7cc56d6-4cqlm",name="apps.webhook.tasks.lx_webhook_sender",queue_name="medium"} 1.7161986227159774e+09 celery_task_sent_created{hostname="api-c7cc56d6-fpbn2",name="apps.courses.tasks.fire_team_start",queue_name="medium"} 1.7161986246666076e+09 celery_task_sent_created{hostname="api-c7cc56d6-fpbn2",name="apps.courses.tasks.calculate_user_progress",queue_name="critical"} 1.7161986246669247e+09 celery_task_sent_created{hostname="api-c7cc56d6-fpbn2",name="apps.reports.tasks.feed_reporting",queue_name="high"} 1.7161986246674087e+09 celery_task_sent_created{hostname="celery-beat-5d6d58c7f-ghqxb",name="apps.celery.refresh_user_stat_materialized_view",queue_name="medium"} 1.7161991400039644e+09 celery_task_sent_created{hostname="celery-beat-5d6d58c7f-ghqxb",name="apps.notifications.tasks.daily_due_date_reminder",queue_name="medium"} 1.7161992000111895e+09 celery_task_sent_created{hostname="api-c7cc56d6-fpbn2",name="apps.courses.tasks.update_done_tries",queue_name="medium"} 1.716201386411996e+09 celery_task_sent_created{hostname="api-c7cc56d6-4cqlm",name="apps.courses.tasks.update_done_tries",queue_name="medium"} 1.716201423262146e+09 celery_task_sent_created{hostname="api-c7cc56d6-pghvr",name="apps.courses.tasks.update_done_tries",queue_name="medium"} 1.7162014727684107e+09 celery_task_sent_created{hostname="api-c7cc56d6-fpbn2",name="apps.celery.enqueue_send_email",queue_name="medium"} 1.7162047909888752e+09 celery_task_sent_created{hostname="api-c7cc56d6-pghvr",name="apps.celery.enqueue_send_email",queue_name="medium"} 1.7162047913114533e+09 celery_task_sent_created{hostname="api-c7cc56d6-4cqlm",name="apps.celery.enqueue_send_email",queue_name="medium"} 1.7162049190572798e+09 celery_task_sent_created{hostname="api-c7cc56d6-fpbn2",name="apps.webhook.tasks.trainee_created_webhook_sender",queue_name="medium"} 1.7162058604372587e+09 celery_task_sent_created{hostname="celery-beat-757f657d5d-d59t4",name="app.celery.cleanup_task_results",queue_name="medium"} 1.7162236890491798e+09 celery_task_sent_created{hostname="api-6b667bcd5b-l5dh7",name="apps.badges.tasks.badge_trigger",queue_name="medium"} 1.7162239681318974e+09 celery_task_sent_created{hostname="api-6b667bcd5b-l5dh7",name="apps.webhook.tasks.lx_webhook_sender",queue_name="medium"} 1.7162239681452491e+09 celery_task_sent_created{hostname="api-6b667bcd5b-l5dh7",name="apps.courses.tasks.fire_team_start",queue_name="medium"} 1.7162239681566901e+09 celery_task_sent_created{hostname="api-6b667bcd5b-l5dh7",name="apps.courses.tasks.calculate_user_progress",queue_name="critical"} 1.7162239681569684e+09 celery_task_sent_created{hostname="api-6b667bcd5b-l5dh7",name="apps.reports.tasks.feed_reporting",queue_name="high"} 1.7162239681811526e+09 celery_task_sent_created{hostname="celery-critical-64b77fd969-6dsp8",name="apps.badges.tasks.badge_trigger",queue_name="medium"} 1.7162239682458663e+09 celery_task_sent_created{hostname="celery-critical-64b77fd969-6dsp8",name="apps.webhook.tasks.lx_webhook_sender",queue_name="medium"} 1.7162239682462006e+09 celery_task_sent_created{hostname="celery-high-694788bdb9-qth45",name="apps.courses.tasks.calculate_user_progress",queue_name="critical"} 1.7162239682577775e+09 celery_task_sent_created{hostname="celery-high-694788bdb9-qth45",name="apps.badges.tasks.badge_trigger",queue_name="medium"} 1.7162239682580862e+09 celery_task_sent_created{hostname="celery-critical-64b77fd969-6dsp8",name="apps.notifications.tasks.push_bulk_notification",queue_name="medium"} 1.7162239685157936e+09 celery_task_sent_created{hostname="celery-critical-64b77fd969-6dsp8",name="apps.celery.enqueue_send_email",queue_name="medium"} 1.7162239685388286e+09 celery_task_sent_created{hostname="celery-critical-64b77fd969-6dsp8",name="apps.courses.tasks.container_user_update_points",queue_name="high"} 1.7162239685496924e+09 celery_task_sent_created{hostname="celery-high-694788bdb9-qth45",name="apps.courses.tasks.calculate_user_certificate",queue_name="critical"} 1.7162239685620062e+09 celery_task_sent_created{hostname="api-6b667bcd5b-cmhtf",name="apps.badges.tasks.badge_trigger",queue_name="medium"} 1.716224365593098e+09 celery_task_sent_created{hostname="api-6b667bcd5b-cmhtf",name="apps.webhook.tasks.lx_webhook_sender",queue_name="medium"} 1.7162243656070251e+09 celery_task_sent_created{hostname="api-6b667bcd5b-cmhtf",name="apps.courses.tasks.fire_team_start",queue_name="medium"} 1.7162243656182756e+09 celery_task_sent_created{hostname="api-6b667bcd5b-cmhtf",name="apps.courses.tasks.calculate_user_progress",queue_name="critical"} 1.7162243656225781e+09 celery_task_sent_created{hostname="api-6b667bcd5b-cmhtf",name="apps.reports.tasks.feed_reporting",queue_name="high"} 1.716224365637065e+09 celery_task_sent_created{hostname="celery-beat-757f657d5d-d59t4",name="apps.celery.cron_run",queue_name="high"} 1.7162244000050237e+09 celery_task_sent_created{hostname="api-6b667bcd5b-l5dh7",name="apps.webhook.tasks.trainee_created_webhook_sender",queue_name="medium"} 1.7162250060081868e+09 celery_task_sent_created{hostname="api-6b667bcd5b-zlgrd",name="apps.courses.tasks.fire_team_start",queue_name="medium"} 1.716229397277507e+09 celery_task_sent_created{hostname="api-6b667bcd5b-zlgrd",name="apps.courses.tasks.calculate_user_progress",queue_name="critical"} 1.7162293972819135e+09 celery_task_sent_created{hostname="api-6b667bcd5b-zlgrd",name="apps.badges.tasks.badge_trigger",queue_name="medium"} 1.716229397286609e+09 celery_task_sent_created{hostname="api-6b667bcd5b-zlgrd",name="apps.reports.tasks.feed_reporting",queue_name="high"} 1.7162293972899244e+09 celery_task_sent_created{hostname="celery-high-694788bdb9-qth45",name="apps.reports.tasks.feed_reporting",queue_name="high"} 1.7162293972960076e+09 celery_task_sent_created{hostname="api-6b667bcd5b-zlgrd",name="apps.webhook.tasks.trainee_created_webhook_sender",queue_name="medium"} 1.7162310788618793e+09 celery_task_sent_created{hostname="celery-beat-757f657d5d-d59t4",name="apps.celery.refresh_user_stat_materialized_view",queue_name="medium"} 1.7162315400030348e+09 celery_task_sent_created{hostname="api-6b667bcd5b-cmhtf",name="apps.celery.enqueue_send_email",queue_name="medium"} 1.7162385631828709e+09 celery_task_sent_created{hostname="api-6b667bcd5b-l5dh7",name="apps.celery.enqueue_send_email",queue_name="medium"} 1.7162385776223674e+09 celery_task_sent_created{hostname="api-6b667bcd5b-zlgrd",name="apps.celery.enqueue_send_email",queue_name="medium"} 1.7162390544229074e+09 celery_task_sent_created{hostname="api-6b667bcd5b-zlgrd",name="apps.customers.tasks.change_customer_has_assignment",queue_name="medium"} 1.7162392506929896e+09 celery_task_sent_created{hostname="api-6b667bcd5b-cmhtf",name="apps.webhook.tasks.trainee_created_webhook_sender",queue_name="medium"} 1.7162397095317886e+09 celery_task_sent_created{hostname="api-6b667bcd5b-l5dh7",name="apps.customers.tasks.call_netlify_build_task",queue_name="medium"} 1.71624094556108e+09 celery_task_sent_created{hostname="api-6b667bcd5b-cmhtf",name="apps.customers.tasks.call_netlify_build_task",queue_name="medium"} 1.7162410074289992e+09 celery_task_sent_created{hostname="api-6b667bcd5b-zlgrd",name="apps.customers.tasks.call_netlify_build_task",queue_name="medium"} 1.7162414756173782e+09 celery_task_sent_created{hostname="celery-beat-757f657d5d-d59t4",name="apps.users.tasks.get_scfhs_users_data",queue_name="medium"} 1.7162424000047145e+09 celery_task_sent_created{hostname="api-6b667bcd5b-l5dh7",name="apps.customers.tasks.change_customer_has_assignment",queue_name="medium"} 1.716242500494445e+09 celery_task_sent_created{hostname="celery-beat-757f657d5d-d59t4",name="apps.base.tasks.daily_cache_check",queue_name="medium"} 1.7162496000034285e+09 celery_task_sent_created{hostname="celery-beat-757f657d5d-d59t4",name="apps.reports.tasks.feed_reporting_with_median_time",queue_name="medium"} 1.7162496000090144e+09 celery_task_sent_created{hostname="celery-medium-7496c9c6fb-tjr59",name="apps.notifications.tasks.send_slack_message",queue_name="medium"} 1.7162496001390681e+09 celery_task_sent_created{hostname="celery-beat-757f657d5d-d59t4",name="apps.users.tasks.update_user_stats_csv",queue_name="low"} 1.7162532000036764e+09 celery_task_sent_created{hostname="celery-low-5f48c6d9-7qmmv",name="apps.users.tasks.upload_user_stats_csv_to_s3",queue_name="low"} 1.7162532059585462e+09 celery_task_sent_created{hostname="celery-low-5f48c6d9-7qmmv",name="apps.customers.tasks.call_netlify_build_task",queue_name="medium"} 1.7162532070926368e+09 celery_task_sent_created{hostname="celery-beat-757f657d5d-d59t4",name="apps.users.tasks.add_scfhs_interns_to_team",queue_name="medium"} 1.7162568000040486e+09 celery_task_sent_created{hostname="celery-beat-757f657d5d-d59t4",name="celery.backend_cleanup",queue_name="medium"} 1.7162640000037065e+09 celery_task_sent_created{hostname="celery-beat-757f657d5d-d59t4",name="apps.webhook.tasks.time_log_in_past_24h_schedule",queue_name="medium"} 1.7162694000044763e+09 celery_task_sent_created{hostname="celery-beat-757f657d5d-d59t4",name="apps.notifications.tasks.send_customer_observatory_notification",queue_name="medium"} 1.7162784000029979e+09 celery_task_sent_created{hostname="celery-medium-7496c9c6fb-tjr59",name="apps.celery.enqueue_send_email",queue_name="medium"} 1.7162784002010388e+09 celery_task_sent_created{hostname="celery-beat-757f657d5d-d59t4",name="apps.notifications.tasks.daily_due_date_reminder",queue_name="medium"} 1.7162856000045645e+09 # HELP celery_task_received_total Sent when the worker receives a task. # TYPE celery_task_received_total counter # HELP celery_task_started_total Sent just before the worker executes the task. # TYPE celery_task_started_total counter # HELP celery_task_succeeded_total Sent if the task executed successfully. # TYPE celery_task_succeeded_total counter # HELP celery_task_failed_total Sent if the execution of the task failed. # TYPE celery_task_failed_total counter # HELP celery_task_rejected_total The task was rejected by the worker, possibly to be re-queued or moved to a dead letter queue. # TYPE celery_task_rejected_total counter # HELP celery_task_revoked_total Sent if the task has been revoked. # TYPE celery_task_revoked_total counter # HELP celery_task_retried_total Sent if the task failed, but will be retried in the future. # TYPE celery_task_retried_total counter # HELP celery_worker_up Indicates if a worker has recently sent a heartbeat. # TYPE celery_worker_up gauge celery_worker_up{hostname="saleor-celery-worker-84b97c86df-cjh46"} 1.0 celery_worker_up{hostname="celery-critical-64b77fd969-6dsp8"} 1.0 celery_worker_up{hostname="celery-high-694788bdb9-qth45"} 1.0 celery_worker_up{hostname="celery-low-5f48c6d9-7qmmv"} 1.0 celery_worker_up{hostname="celery-medium-7496c9c6fb-tjr59"} 1.0 # HELP celery_worker_tasks_active The number of tasks the worker is currently processing # TYPE celery_worker_tasks_active gauge celery_worker_tasks_active{hostname="saleor-celery-worker-84b97c86df-cjh46"} 0.0 celery_worker_tasks_active{hostname="celery-critical-64b77fd969-6dsp8"} 0.0 celery_worker_tasks_active{hostname="celery-high-694788bdb9-qth45"} 0.0 celery_worker_tasks_active{hostname="celery-low-5f48c6d9-7qmmv"} 0.0 celery_worker_tasks_active{hostname="celery-medium-7496c9c6fb-tjr59"} 0.0 # HELP celery_task_runtime Histogram of task runtime measurements. # TYPE celery_task_runtime histogram # HELP celery_queue_length The number of message in broker queue. # TYPE celery_queue_length gauge celery_queue_length{queue_name="xapi_service"} 0.0 celery_queue_length{queue_name="medium"} 0.0 celery_queue_length{queue_name="low"} 0.0 celery_queue_length{queue_name="celery"} 0.0 celery_queue_length{queue_name="high"} 0.0 celery_queue_length{queue_name="critical"} 0.0 # HELP celery_active_consumer_count The number of active consumer in broker queue. # TYPE celery_active_consumer_count gauge # HELP celery_active_worker_count The number of active workers in broker queue. # TYPE celery_active_worker_count gauge celery_active_worker_count{queue_name="xapi_service"} 1.0 celery_active_worker_count{queue_name="medium"} 1.0 celery_active_worker_count{queue_name="low"} 1.0 celery_active_worker_count{queue_name="celery"} 1.0 celery_active_worker_count{queue_name="high"} 1.0 celery_active_worker_count{queue_name="critical"} 1.0 # HELP celery_active_process_count The number of active processes in broker queue. # TYPE celery_active_process_count gauge celery_active_process_count{queue_name="xapi_service"} 1.0 celery_active_process_count{queue_name="medium"} 1.0 celery_active_process_count{queue_name="low"} 4.0 celery_active_process_count{queue_name="celery"} 4.0 celery_active_process_count{queue_name="high"} 1.0 celery_active_process_count{queue_name="critical"} 1.0
This is my configuration
# CELERY CELERY_BROKER_URL = "redis://localhost:6379/0" CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" CELERY_WORKER_SEND_TASK_EVENTS = True CELERY_TASK_SEND_SENT_EVENT = True CELERY_ACCEPT_CONTENT = ["application/json"] CELERY_TASK_SERIALIZER = "json" CELERY_RESULT_BACKEND = "django-db" CELERY_RESULT_SERIALIZER = "json" CELERY_TASK_RESULT_EXPIRES = env("CELERY_RESULT_EXPIRES", default=3600)
The database tables does contain the results
db=# select task_name, count(*) from django_celery_results_taskresult where date_created > NOW() - INTERVAL '60 minutes' and date_created < NOW() - INTERVAL '10 minutes' group by task_name; task_name | count ---------------------------------------------------+-------- apps.badges.tasks.badge_trigger | 311572 apps.celery.cron_run | 1 apps.celery.enqueue_send_email | 20449 apps.courses.tasks.calculate_user_certificate | 191390 apps.courses.tasks.calculate_user_progress | 191393 apps.courses.tasks.container_user_update_points | 191393 apps.courses.tasks.fire_team_start | 95697 apps.notifications.tasks.daily_due_date_reminder | 1 apps.reports.tasks.feed_reporting | 95697 apps.webhook.tasks.certificate_webhook_sender | 5349 apps.webhook.tasks.lx_webhook_sender | 14730 apps.webhook.tasks.new_certificate_webhook_sender | 5349 apps.webhook.tasks.trainee_created_webhook_sender | 9751
How can I adjust my setting so that I start seeing those metrics in Grafana? What is the requirement regarding celery_results_backend should it be the same as the broker for the exporter to work properly?
Following is my metrics endpoint output, some metrics like
celery_task_failed_totalare not workingThis is my configuration
The database tables does contain the results
How can I adjust my setting so that I start seeing those metrics in Grafana? What is the requirement regarding celery_results_backend should it be the same as the broker for the exporter to work properly?