[infra] pagerduty

[AmitKumarDas]

// https://www.pagerduty.com/docs/guides/prometheus-integration-guide/

// Objective
//
// Send event data from Prometheus to PagerDuty
// Create high and low urgency incidents based 
// on the severity of the event from the 
// Prometheus event payload.

// How It Works
//
// Prometheus sends events to PagerDuty via a 
// Prometheus Alertmanager configuration file
// Events from Prometheus will trigger a new 
// incident on the corresponding PagerDuty 
// service, or group as alerts into an existing
// incident. 
//
// You can resolve PagerDuty incidents from the 
// Prometheus server as long as the **send_resolved** 
// configuration option is not set to **false**. The 
// default value is true, so there’s no need to 
// specify send_resolved: true to have PagerDuty 
// incidents be resolved automatically.

// Requirements
//
// Prometheus Alertmanager v0.11 and later: 
// Alertmanager now supports Events API v2. 
// However, if you set the routing_key property and
// use v2, the integration type of the integration 
// corresponding to the routing_key value must also 
// be Events API v2. 
//
// If you select Prometheus as the integration type 
// in PagerDuty, you will need to use the Events API
// v1 type and set a value for the service_key 
// property instead.
//
// A Manager base role or higher is required to configure
// this integration. If you’re not sure what role you have,
// or if you need your permissions adjusted, visit our 
// sections on Checking Your User Role or Changing User Roles.

// Integration
//
// There are two ways to integrate with PagerDuty: 
// via global event routing or directly through an 
// integration on a PagerDuty service. 
//
// Integrating with **global event routing** may be beneficial
// if you want to build different routing rules based on 
// the events coming from the integrated tool. 
//
// Integrating with a **PagerDuty service** directly can be 
// beneficial if you don’t need to route alerts from the
// integrated tool to different responders based on the 
// event payload. 

// Integration w.r.t Prometheus
//
// Install Prometheus alertmanager
// create/update alertmanager config file

//  receivers: // receive the event
//  - name: YOUR-PAGER-DUTY-NAME // name of the service to integrate
//    pagerduty_configs: // receive the event at pager duty
//    - service_key: YOUR-INTEGRATION-KEY // generated in pager duty

//  route: // default route to send all alerts
//    group_by: [cluster]
//    receiver: YOUR-PAGER-DUTY-NAME

//  routes: // send alert to receiver(s)
//  - match:
//      severity: 'warning' // send warning alerts to PG
//    receiver: YOUR-PAGER-DUTY-NAME

//  route: // send
//    group_by: [cluster]
//    receiver: **DEFAULT-RECEIVER**
//    group_interval: 5m
//    routes:
//    - match:
//      service: database
//      receiver: **DATABASE-RECEIVER**
//
//  receivers: // receive
//  - name: **DEFAULT-RECEIVER**
//    pagerduty_configs:
//    - service_key: **PRIMARY-INTEGRATION-KEY**
//  - name: **DATABASE-RECEIVER**
//    pagerduty_configs:
//    - service_key: **DATABASE-INTEGRATION-KEY**

// alert manager need to be restarted
// verify k8s docs if required
// should this be handled via gitops?

// test via 
// curl -d '[{"labels": {"Alertname": "PagerDuty Test"}}]' http://localhost:9093/api/v1/alerts