👋 Recently been seeing a lot of 429 error code: 1015 responses when the worker attempts to post to Discord. With the old logic, this would be treated the same as an excluded status type, and so the incident would be written to KV without being posted, and would be ignored.
This isn't great as it means random errors from Discord can lead to an incident that should be posted not being posted. So, this PR updates all the logic around posting to Discord to throw an actual error when it fails. This means that if an incident fails to post, it won't be written to KV and can be retried until it succeeds (while marking the worker as erroring to make the developer aware).
Also, this includes some improvements to the logging, such as ensuring all log messages have an incident ID associated with them to make it easier to trace a specific incident's processing, and to log the headers in Discord responses to inspect ratelimit issues etc.
👋 Recently been seeing a lot of 429 error code: 1015 responses when the worker attempts to post to Discord. With the old logic, this would be treated the same as an excluded status type, and so the incident would be written to KV without being posted, and would be ignored.
This isn't great as it means random errors from Discord can lead to an incident that should be posted not being posted. So, this PR updates all the logic around posting to Discord to throw an actual error when it fails. This means that if an incident fails to post, it won't be written to KV and can be retried until it succeeds (while marking the worker as erroring to make the developer aware).
Also, this includes some improvements to the logging, such as ensuring all log messages have an incident ID associated with them to make it easier to trace a specific incident's processing, and to log the headers in Discord responses to inspect ratelimit issues etc.
Example logs from me running this just now:
``` { "outcome": "ok", "scriptVersion": { "id": "c38dcfc1-dd23-484c-a207-91d31b999ae0" }, "scriptName": "status-worker", "diagnosticsChannelEvents": [], "exceptions": [], "logs": [ { "message": [ "[259gtn3rlsxf] In KV: true" ], "level": "log", "timestamp": 1720047356775 }, { "message": [ "[wtfb50fmwfgx] In KV: true" ], "level": "log", "timestamp": 1720047356778 }, { "message": [ "[nrgqw98yj9bz] In KV: true" ], "level": "log", "timestamp": 1720047356783 }, { "message": [ "[449n8d04vst3] In KV: true" ], "level": "log", "timestamp": 1720047356787 }, { "message": [ "[wsqv56rmx3g0] In KV: false" ], "level": "log", "timestamp": 1720047356792 }, { "message": [ "[wsqv56rmx3g0] New incident: [snip]" ], "level": "log", "timestamp": 1720047356792 }, { "message": [ "[wsqv56rmx3g0] Sending POST to Discord with body: [snip]" ], "level": "log", "timestamp": 1720047356792 }, { "message": [ "[h9wdsyt9jjwd] In KV: true" ], "level": "log", "timestamp": 1720047356793 }, { "message": [ "[wtpkr8hhwwdm] In KV: true" ], "level": "log", "timestamp": 1720047356836 }, { "message": [ "[tqkjrpq9bpkv] In KV: true" ], "level": "log", "timestamp": 1720047356843 }, { "message": [ "[lcdn900frz8z] In KV: true" ], "level": "log", "timestamp": 1720047356846 }, { "message": [ "[yqjv7c5yn7lw] In KV: true" ], "level": "log", "timestamp": 1720047356860 }, { "message": [ "[5cxwn2d0bzxc] In KV: true" ], "level": "log", "timestamp": 1720047356877 }, { "message": [ "[shyfhwzrk7yq] In KV: true" ], "level": "log", "timestamp": 1720047356884 }, { "message": [ "[m0y2kdhdkdg7] In KV: true" ], "level": "log", "timestamp": 1720047356912 }, { "message": [ "[lzlv9c5bjp1v] In KV: true" ], "level": "log", "timestamp": 1720047356912 }, { "message": [ "[r6mcnt4vzvxw] In KV: true" ], "level": "log", "timestamp": 1720047356923 }, { "message": [ "[kpcl7hzg8phn] In KV: true" ], "level": "log", "timestamp": 1720047356932 }, { "message": [ "[yzl464rhsqrg] In KV: true" ], "level": "log", "timestamp": 1720047356949 }, { "message": [ "[kd1vbrxg968f] In KV: true" ], "level": "log", "timestamp": 1720047356956 }, { "message": [ "[s3m5n57g368s] In KV: true" ], "level": "log", "timestamp": 1720047356989 }, { "message": [ "[3h2bvy498kwl] In KV: true" ], "level": "log", "timestamp": 1720047356990 }, { "message": [ "[91tj2rm59jgc] In KV: true" ], "level": "log", "timestamp": 1720047356994 }, { "message": [ "[5s1c6nl5rf0j] In KV: true" ], "level": "log", "timestamp": 1720047357006 }, { "message": [ "[k8trfbny4ym1] In KV: true" ], "level": "log", "timestamp": 1720047357020 }, { "message": [ "[mysrzvnmxr3d] In KV: true" ], "level": "log", "timestamp": 1720047357050 }, { "message": [ "[vrm689bbbhsr] In KV: true" ], "level": "log", "timestamp": 1720047357067 }, { "message": [ "[21gg18q1ddmv] In KV: true" ], "level": "log", "timestamp": 1720047357069 }, { "message": [ "[dwzxptsph123] In KV: true" ], "level": "log", "timestamp": 1720047357081 }, { "message": [ "[z1sf9hg9tjbm] In KV: true" ], "level": "log", "timestamp": 1720047357085 }, { "message": [ "[rtgb9cp8hrzy] In KV: true" ], "level": "log", "timestamp": 1720047357113 }, { "message": [ "[3tv36753kk25] In KV: true" ], "level": "log", "timestamp": 1720047357128 }, { "message": [ "[82hy4zzd316g] In KV: true" ], "level": "log", "timestamp": 1720047357133 }, { "message": [ "[2nng21f5nvfn] In KV: true" ], "level": "log", "timestamp": 1720047357142 }, { "message": [ "[65lj1w0mxvmb] In KV: true" ], "level": "log", "timestamp": 1720047357148 }, { "message": [ "[gkl208t787zz] In KV: true" ], "level": "log", "timestamp": 1720047357169 }, { "message": [ "[1755wn45yqyw] In KV: true" ], "level": "log", "timestamp": 1720047357173 }, { "message": [ "[l9d26ggtbw8h] In KV: true" ], "level": "log", "timestamp": 1720047357197 }, { "message": [ "[s5v9qwsvlxjv] In KV: true" ], "level": "log", "timestamp": 1720047357199 }, { "message": [ "[dj2p47c4zss1] In KV: true" ], "level": "log", "timestamp": 1720047357208 }, { "message": [ "[cd70bby9qynm] In KV: true" ], "level": "log", "timestamp": 1720047357226 }, { "message": [ "[g4sdvwy4z4zm] In KV: true" ], "level": "log", "timestamp": 1720047357235 }, { "message": [ "[n9211k9rmml7] In KV: true" ], "level": "log", "timestamp": 1720047357243 }, { "message": [ "[rccs80k3p2pb] In KV: true" ], "level": "log", "timestamp": 1720047357266 }, { "message": [ "[yp30nqsv310k] In KV: true" ], "level": "log", "timestamp": 1720047357274 }, { "message": [ "[q2sl05rc7mgp] In KV: true" ], "level": "log", "timestamp": 1720047357283 }, { "message": [ "[w1sspyd728kg] In KV: true" ], "level": "log", "timestamp": 1720047357288 }, { "message": [ "[wsqv56rmx3g0] Discord send response: {\"status\":429,\"body\":\"error code: 1015\",\"headers\":{\"cache-control\":\"private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0\",\"cf-ray\":\"89da774f275537c7-MAD\",\"connection\":\"keep-alive\",\"content-length\":\"16\",\"content-security-policy\":\"frame-ancestors 'none'; default-src 'none'\",\"content-type\":\"text/plain; charset=UTF-8\",\"date\":\"Wed, 03 Jul 2024 22:55:57 GMT\",\"expires\":\"Thu, 01 Jan 1970 00:00:01 GMT\",\"referrer-policy\":\"same-origin\",\"retry-after\":\"1730\",\"server\":\"cloudflare\",\"set-cookie\":\"_cfuvid=IaITirzLi9bg5n1Qr1qf6.5uwxfobHAZjQyp3.XJs6c-1720047357310-0.0.1.1-604800000; path=/; domain=.discord.com; HttpOnly; Secure; SameSite=None\",\"strict-transport-security\":\"max-age=31536000; includeSubDomains; preload\",\"x-content-type-options\":\"nosniff\",\"x-frame-options\":\"SAMEORIGIN\"}}" ], "level": "log", "timestamp": 1720047357300 }, { "message": [ "[y2cm9wm1bvv7] In KV: true" ], "level": "log", "timestamp": 1720047357312 }, { "message": [ "[1q1jhbr85zvv] In KV: true" ], "level": "log", "timestamp": 1720047357320 }, { "message": [ "[swc8grzsb33n] In KV: true" ], "level": "log", "timestamp": 1720047357323 }, { "message": [ "[39r97dl3v2l0] In KV: true" ], "level": "log", "timestamp": 1720047357346 }, { "message": [ "[r9w0yrbyy9ls] In KV: true" ], "level": "log", "timestamp": 1720047357359 }, { "message": [ "Processed 50 incidents (49 successful)" ], "level": "log", "timestamp": 1720047357359 }, { "message": [ "Error: [wsqv56rmx3g0] Failed to send message to Discord: {\"status\":429,\"body\":\"error code: 1015\",\"headers\":{\"cache-control\":\"private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0\",\"cf-ray\":\"89da774f275537c7-MAD\",\"connection\":\"keep-alive\",\"content-length\":\"16\",\"content-security-policy\":\"frame-ancestors 'none'; default-src 'none'\",\"content-type\":\"text/plain; charset=UTF-8\",\"date\":\"Wed, 03 Jul 2024 22:55:57 GMT\",\"expires\":\"Thu, 01 Jan 1970 00:00:01 GMT\",\"referrer-policy\":\"same-origin\",\"retry-after\":\"1730\",\"server\":\"cloudflare\",\"set-cookie\":\"_cfuvid=IaITirzLi9bg5n1Qr1qf6.5uwxfobHAZjQyp3.XJs6c-1720047357310-0.0.1.1-604800000; path=/; domain=.discord.com; HttpOnly; Secure; SameSite=None\",\"strict-transport-security\":\"max-age=31536000; includeSubDomains; preload\",\"x-content-type-options\":\"nosniff\",\"x-frame-options\":\"SAMEORIGIN\"}}" ], "level": "error", "timestamp": 1720047357359 } ], "eventTimestamp": 1720047355319, "event": { "cron": "* * * * *", "scheduledTime": 1720047355000 }, "id": 0 } ```