dsotirho-ucsc
commented
10 months ago Additional occurrences on prod:
## /aws/lambda/azul-service-prod-servicecachehealth
[
{
"@timestamp": "2023-11-22 18:27:29.875",
"@message": "START RequestId: 2538841d-417b-4651-845a-df7e69a86829 Version: $LATEST\n"
},
{
"@timestamp": "2023-11-22 18:27:29.875",
"@message": "[INFO]\t2023-11-22T18:27:29.875Z\t2538841d-417b-4651-845a-df7e69a86829\tazul.health\tGetting health property 'elasticsearch'"
},
{
"@timestamp": "2023-11-22 18:27:29.875",
"@message": "[INFO]\t2023-11-22T18:27:29.875Z\t2538841d-417b-4651-845a-df7e69a86829\telasticsearch\tMaking HEAD request to [https://vpc-azul-index-prod-mjluewlvxikvhillefxotw2tbm.us-east-1.es.amazonaws.com:443/](https://vpc-azul-index-prod-mjluewlvxikvhillefxotw2tbm.us-east-1.es.amazonaws.com/)"
},
{
"@timestamp": "2023-11-22 18:27:29.875",
"@message": "[INFO]\t2023-11-22T18:27:29.875Z\t2538841d-417b-4651-845a-df7e69a86829\telasticsearch\t… without request body"
},
{
"@timestamp": "2023-11-22 18:27:29.909",
"@message": "[INFO]\t2023-11-22T18:27:29.909Z\t2538841d-417b-4651-845a-df7e69a86829\telasticsearch\tGot 200 response after 0.033s from HEAD to [https://vpc-azul-index-prod-mjluewlvxikvhillefxotw2tbm.us-east-1.es.amazonaws.com:443/](https://vpc-azul-index-prod-mjluewlvxikvhillefxotw2tbm.us-east-1.es.amazonaws.com/)"
},
{
"@timestamp": "2023-11-22 18:27:29.909",
"@message": "[INFO]\t2023-11-22T18:27:29.909Z\t2538841d-417b-4651-845a-df7e69a86829\telasticsearch\t… with response body ''"
},
{
"@timestamp": "2023-11-22 18:27:29.909",
"@message": "[INFO]\t2023-11-22T18:27:29.909Z\t2538841d-417b-4651-845a-df7e69a86829\tazul.health\tGetting health property 'api_endpoints'"
},
{
"@timestamp": "2023-11-22 18:27:29.909",
"@message": "[INFO]\t2023-11-22T18:27:29.909Z\t2538841d-417b-4651-845a-df7e69a86829\tazul.health\tMaking HEAD request to https://service.azul.data.humancellatlas.org/index/samples?size=1"
},
{
"@timestamp": "2023-11-22 18:27:39.912",
"@message": "2023-11-22T18:27:39.912Z 2538841d-417b-4651-845a-df7e69a86829 Task timed out after 10.04 seconds\n\n"
},
{
"@timestamp": "2023-11-22 18:27:39.912",
"@message": "END RequestId: 2538841d-417b-4651-845a-df7e69a86829\n"
},
{
"@timestamp": "2023-11-22 18:27:39.912",
"@message": "REPORT RequestId: 2538841d-417b-4651-845a-df7e69a86829\tDuration: 10037.37 ms\tBilled Duration: 10000 ms\tMemory Size: 128 MB\tMax Memory Used: 121 MB\t\n"
}
]
## /aws/lambda/azul-service-prod-servicecachehealth
[
{
"@timestamp": "2023-11-22 18:29:29.919",
"@message": "START RequestId: 1ed57fbc-0c3a-436a-b56b-f5885d5e7758 Version: $LATEST\n"
},
{
"@timestamp": "2023-11-22 18:29:29.920",
"@message": "[INFO]\t2023-11-22T18:29:29.919Z\t1ed57fbc-0c3a-436a-b56b-f5885d5e7758\tazul.health\tGetting health property 'elasticsearch'"
},
{
"@timestamp": "2023-11-22 18:29:29.920",
"@message": "[INFO]\t2023-11-22T18:29:29.920Z\t1ed57fbc-0c3a-436a-b56b-f5885d5e7758\telasticsearch\tMaking HEAD request to [https://vpc-azul-index-prod-mjluewlvxikvhillefxotw2tbm.us-east-1.es.amazonaws.com:443/](https://vpc-azul-index-prod-mjluewlvxikvhillefxotw2tbm.us-east-1.es.amazonaws.com/)"
},
{
"@timestamp": "2023-11-22 18:29:29.920",
"@message": "[INFO]\t2023-11-22T18:29:29.920Z\t1ed57fbc-0c3a-436a-b56b-f5885d5e7758\telasticsearch\t… without request body"
},
{
"@timestamp": "2023-11-22 18:29:29.956",
"@message": "[INFO]\t2023-11-22T18:29:29.956Z\t1ed57fbc-0c3a-436a-b56b-f5885d5e7758\telasticsearch\tGot 200 response after 0.036s from HEAD to [https://vpc-azul-index-prod-mjluewlvxikvhillefxotw2tbm.us-east-1.es.amazonaws.com:443/](https://vpc-azul-index-prod-mjluewlvxikvhillefxotw2tbm.us-east-1.es.amazonaws.com/)"
},
{
"@timestamp": "2023-11-22 18:29:29.957",
"@message": "[INFO]\t2023-11-22T18:29:29.956Z\t1ed57fbc-0c3a-436a-b56b-f5885d5e7758\telasticsearch\t… with response body ''"
},
{
"@timestamp": "2023-11-22 18:29:29.957",
"@message": "[INFO]\t2023-11-22T18:29:29.957Z\t1ed57fbc-0c3a-436a-b56b-f5885d5e7758\tazul.health\tGetting health property 'api_endpoints'"
},
{
"@timestamp": "2023-11-22 18:29:29.957",
"@message": "[INFO]\t2023-11-22T18:29:29.957Z\t1ed57fbc-0c3a-436a-b56b-f5885d5e7758\tazul.health\tMaking HEAD request to https://service.azul.data.humancellatlas.org/index/files?size=1"
},
{
"@timestamp": "2023-11-22 18:29:39.952",
"@message": "2023-11-22T18:29:39.952Z 1ed57fbc-0c3a-436a-b56b-f5885d5e7758 Task timed out after 10.03 seconds\n\n"
},
{
"@timestamp": "2023-11-22 18:29:39.952",
"@message": "END RequestId: 1ed57fbc-0c3a-436a-b56b-f5885d5e7758\n"
},
{
"@timestamp": "2023-11-22 18:29:39.952",
"@message": "REPORT RequestId: 1ed57fbc-0c3a-436a-b56b-f5885d5e7758\tDuration: 10032.73 ms\tBilled Duration: 10000 ms\tMemory Size: 128 MB\tMax Memory Used: 110 MB\t\n"
}
]
## /aws/lambda/azul-service-prod
[
{
"@timestamp": "2023-11-22 18:27:30.069",
"@message": "START RequestId: 2ad129f4-96c1-4170-bce0-60553f22e71f Version: $LATEST\n"
},
{
"@timestamp": "2023-11-22 18:27:30.069",
"@message": "[INFO]\t2023-11-22T18:27:30.069Z\t2ad129f4-96c1-4170-bce0-60553f22e71f\tazul.chalice\tReceived HEAD request for '/index/samples', with {\"query\": {\"size\": \"1\"}, \"headers\": {\"accept\": \"*/*\", \"accept-encoding\": \"gzip, deflate\", \"cloudfront-forwarded-proto\": \"https\", \"cloudfront-is-desktop-viewer\": \"true\", \"cloudfront-is-mobile-viewer\": \"false\", \"cloudfront-is-smarttv-viewer\": \"false\", \"cloudfront-is-tablet-viewer\": \"false\", \"cloudfront-viewer-asn\": \"14618\", \"cloudfront-viewer-country\": \"US\", \"host\": \"[service.azul.data.humancellatlas.org](http://service.azul.data.humancellatlas.org/)\", \"user-agent\": \"python-requests/2.31.0\", \"via\": \"1.1 [c2ab43aa4f5ff7b5cc58894f6a8494bc.cloudfront.net](http://c2ab43aa4f5ff7b5cc58894f6a8494bc.cloudfront.net/) (CloudFront)\", \"x-amz-cf-id\": \"NChQqwFjEPCCIxOFGbvIfq9ueYAgd0bvKcKdym5ODd3lom3errfhUw==\", \"x-amzn-trace-id\": \"Root=1-655e4812-48aa39ae5b3b16b2045b02af\", \"x-forwarded-for\": \"52.3.190.111, 18.68.30.171\", \"x-forwarded-port\": \"443\", \"x-forwarded-proto\": \"https\"}}."
},
{
"@timestamp": "2023-11-22 18:27:30.070",
"@message": "[INFO]\t2023-11-22T18:27:30.070Z\t2ad129f4-96c1-4170-bce0-60553f22e71f\tazul.chalice\tDid not authenticate request."
},
{
"@timestamp": "2023-11-22 18:27:30.077",
"@message": "[DEBUG]\t2023-11-22T18:27:30.077Z\t2ad129f4-96c1-4170-bce0-60553f22e71f\tazul.terra\t_request('GET', https://data.terra.bio/api/repository/v1/snapshots/roleMap, headers=None, timeout=5.0, body=None)"
},
{
"@timestamp": "2023-11-22 18:27:35.083",
"@message": "[WARNING]\t2023-11-22T18:27:35.083Z\t2ad129f4-96c1-4170-bce0-60553f22e71f\turllib3.connectionpool\tRetrying (Retry(total=None, connect=0, read=0, redirect=0, status=0)) after connection broken by 'ReadTimeoutError(\"HTTPSConnectionPool(host='data.terra.bio', port=443): Read timed out. (read timeout=5.0)\")': /api/repository/v1/snapshots/roleMap"
},
{
"@timestamp": "2023-11-22 18:27:39.790",
"@message": "[DEBUG]\t2023-11-22T18:27:39.790Z\t2ad129f4-96c1-4170-bce0-60553f22e71f\tazul.terra\t_request(…) -> 9.713s 200 b'{\"roleMap\":{\"f3283538-c86e-4d2b-b6fb-c51d3479b9cc\":[\"reader\"],\"82336861-0e39-497f-bdbe-03923f5f36ab\":[\"reader\"],\"5f006aa6-bae4-4241-a334-4f0e0b731c35\":[\"reader\"],\"c25f206f-c5de-48f2-9dfa-7ab41221d6eb\":[\"reader\"],\"bc7f344d-ff18-4a10-8f59-518324ad4126\":[\"...'"
},
{
"@timestamp": "2023-11-22 18:27:39.837",
"@message": "[INFO]\t2023-11-22T18:27:39.837Z\t2ad129f4-96c1-4170-bce0-60553f22e71f\telasticsearch\tMaking POST request to [https://vpc-azul-index-prod-mjluewlvxikvhillefxotw2tbm.us-east-1.es.amazonaws.com:443/azul_v2_prod_dcp32_samples_aggregate/_search](https://vpc-azul-index-prod-mjluewlvxikvhillefxotw2tbm.us-east-1.es.amazonaws.com/azul_v2_prod_dcp32_samples_aggregate/_search)"
},
{
"@timestamp": "2023-11-22 18:27:39.837",
"@message": "[INFO]\t2023-11-22T18:27:39.837Z\t2ad129f4-96c1-4170-bce0-60553f22e71f\telasticsearch\t… with request body b'{\"post_filter\":{\"bool\":{\"must\":[{\"constant_score\":{\"filter\":{\"terms\":{\"sources.id.keyword\":[\"4e6b4bd3-8534-4ebf-868b-8a3ed739...'"
},
{
"@timestamp": "2023-11-22 18:27:40.243",
"@message": "[INFO]\t2023-11-22T18:27:40.243Z\t2ad129f4-96c1-4170-bce0-60553f22e71f\telasticsearch\tGot 200 response after 0.406s from POST to [https://vpc-azul-index-prod-mjluewlvxikvhillefxotw2tbm.us-east-1.es.amazonaws.com:443/azul_v2_prod_dcp32_samples_aggregate/_search](https://vpc-azul-index-prod-mjluewlvxikvhillefxotw2tbm.us-east-1.es.amazonaws.com/azul_v2_prod_dcp32_samples_aggregate/_search)"
},
{
"@timestamp": "2023-11-22 18:27:40.243",
"@message": "[INFO]\t2023-11-22T18:27:40.243Z\t2ad129f4-96c1-4170-bce0-60553f22e71f\telasticsearch\t… with response body '{\"took\":355,\"timed_out\":false,\"_shards\":{\"total\":1,\"successful\":1,\"skipped\":0,\"failed\":0},\"hits\":{\"total\":{\"value\":19742,\"relat…'"
},
{
"@timestamp": "2023-11-22 18:27:40.270",
"@message": "[DEBUG]\t2023-11-22T18:27:40.270Z\t2ad129f4-96c1-4170-bce0-60553f22e71f\tazul.chalice\tReturning 200 response with headers {\"Access-Control-Allow-Origin\": \"*\", \"Access-Control-Allow-Headers\": \"Authorization,Content-Type,X-Amz-Date,X-Amz-Security-Token,X-Api-Key\", \"Strict-Transport-Security\": \"max-age=31536000; includeSubDomains\"}. See next line for the first 1024 characters of the body.\n{\"pagination\": {\"count\": 1, \"total\": 19742, \"size\": 1, \"next\": \"[https://service.azul.data.humancellatlas.org/index/samples?catalog=dcp32&filters=%7B%7D&search_after=%5B%22001C_specimen%22%2C+%221e26ce75-1be0-4153-a21e-3d4a08a0d614%22%5D&sort=sampleId&order=asc&size=1\](https://service.azul.data.humancellatlas.org/index/samples?catalog=dcp32&filters=%7B%7D&search_after=%5B%22001C_specimen%22%2C+%221e26ce75-1be0-4153-a21e-3d4a08a0d614%22%5D&sort=sampleId&order=asc&size=1%5C)", \"previous\": null, \"pages\": 19742, \"sort\": \"sampleId\", \"order\": \"asc\"}, \"termFacets\": {\"organ\": {\"terms\": [{\"term\": \"blood\", \"count\": 4969}, {\"term\": \"lung\", \"count\": 1907}, {\"term\": \"pancreas\", \"count\": 1733}, {\"term\": \"adipose tissue\", \"count\": 1441}, {\"term\": \"brain\", \"count\": 1397}, {\"term\": null, \"count\": 1072}, {\"term\": \"skin of body\", \"count\": 803}, {\"term\": \"colon\", \"count\": 635}, {\"term\": \"liver\", \"count\": 600}, {\"term\": \"kidney\", \"count\": 575}, {\"term\": \"heart\", \"count\": 340}, {\"term\": \"embryo\", \"count\": 334}, {\"term\": \"eye\", \"count\": 319}, {\"term\": \"wall of lateral ventricle\", \"count\": 250}, {\"term\": \"small intestine\", \"count\": 195}, {\"term\": \"large intestine\", \"count\": 193}, {\"term\": \"spinal cord\", \"count\": 161}, {\"term\": \"stomach\", "
},
{
"@timestamp": "2023-11-22 18:27:40.282",
"@message": "END RequestId: 2ad129f4-96c1-4170-bce0-60553f22e71f\n"
},
{
"@timestamp": "2023-11-22 18:27:40.282",
"@message": "REPORT RequestId: 2ad129f4-96c1-4170-bce0-60553f22e71f\tDuration: 10212.67 ms\tBilled Duration: 10213 ms\tMemory Size: 2048 MB\tMax Memory Used: 211 MB\t\n"
},
{
"@timestamp": "2023-11-22 18:29:30.125",
"@message": "START RequestId: be4c63b2-8594-44fa-9657-7d56ee386986 Version: $LATEST\n"
},
{
"@timestamp": "2023-11-22 18:29:30.126",
"@message": "[INFO]\t2023-11-22T18:29:30.126Z\tbe4c63b2-8594-44fa-9657-7d56ee386986\tazul.chalice\tReceived HEAD request for '/index/files', with {\"query\": {\"size\": \"1\"}, \"headers\": {\"accept\": \"*/*\", \"accept-encoding\": \"gzip, deflate\", \"cloudfront-forwarded-proto\": \"https\", \"cloudfront-is-desktop-viewer\": \"true\", \"cloudfront-is-mobile-viewer\": \"false\", \"cloudfront-is-smarttv-viewer\": \"false\", \"cloudfront-is-tablet-viewer\": \"false\", \"cloudfront-viewer-asn\": \"14618\", \"cloudfront-viewer-country\": \"US\", \"host\": \"[service.azul.data.humancellatlas.org](http://service.azul.data.humancellatlas.org/)\", \"user-agent\": \"python-requests/2.31.0\", \"via\": \"1.1 [8917feb7a0623473126b94dc50f359fa.cloudfront.net](http://8917feb7a0623473126b94dc50f359fa.cloudfront.net/) (CloudFront)\", \"x-amz-cf-id\": \"IHoTJ0B2s7nJrj5fVccfhJQZFS4tDlDGC3Ku9_A3Yrk3yffNs_Fq2A==\", \"x-amzn-trace-id\": \"Root=1-655e488a-4e53902b30b0d6675818e01a\", \"x-forwarded-for\": \"52.3.190.111, 18.68.30.134\", \"x-forwarded-port\": \"443\", \"x-forwarded-proto\": \"https\"}}."
},
{
"@timestamp": "2023-11-22 18:29:30.126",
"@message": "[INFO]\t2023-11-22T18:29:30.126Z\tbe4c63b2-8594-44fa-9657-7d56ee386986\tazul.chalice\tDid not authenticate request."
},
{
"@timestamp": "2023-11-22 18:29:30.135",
"@message": "[DEBUG]\t2023-11-22T18:29:30.135Z\tbe4c63b2-8594-44fa-9657-7d56ee386986\tazul.terra\t_request('GET', https://data.terra.bio/api/repository/v1/snapshots/roleMap, headers=None, timeout=5.0, body=None)"
},
{
"@timestamp": "2023-11-22 18:29:35.141",
"@message": "[WARNING]\t2023-11-22T18:29:35.141Z\tbe4c63b2-8594-44fa-9657-7d56ee386986\turllib3.connectionpool\tRetrying (Retry(total=None, connect=0, read=0, redirect=0, status=0)) after connection broken by 'ReadTimeoutError(\"HTTPSConnectionPool(host='data.terra.bio', port=443): Read timed out. (read timeout=5.0)\")': /api/repository/v1/snapshots/roleMap"
},
{
"@timestamp": "2023-11-22 18:29:35.462",
"@message": "[DEBUG]\t2023-11-22T18:29:35.462Z\tbe4c63b2-8594-44fa-9657-7d56ee386986\tazul.terra\t_request(…) -> 5.327s 200 b'{\"roleMap\":{\"f3283538-c86e-4d2b-b6fb-c51d3479b9cc\":[\"reader\"],\"82336861-0e39-497f-bdbe-03923f5f36ab\":[\"reader\"],\"5f006aa6-bae4-4241-a334-4f0e0b731c35\":[\"reader\"],\"c25f206f-c5de-48f2-9dfa-7ab41221d6eb\":[\"reader\"],\"bc7f344d-ff18-4a10-8f59-518324ad4126\":[\"...'"
},
{
"@timestamp": "2023-11-22 18:29:35.508",
"@message": "[INFO]\t2023-11-22T18:29:35.508Z\tbe4c63b2-8594-44fa-9657-7d56ee386986\telasticsearch\tMaking POST request to [https://vpc-azul-index-prod-mjluewlvxikvhillefxotw2tbm.us-east-1.es.amazonaws.com:443/azul_v2_prod_dcp32_files_aggregate/_search](https://vpc-azul-index-prod-mjluewlvxikvhillefxotw2tbm.us-east-1.es.amazonaws.com/azul_v2_prod_dcp32_files_aggregate/_search)"
},
{
"@timestamp": "2023-11-22 18:29:35.508",
"@message": "[INFO]\t2023-11-22T18:29:35.508Z\tbe4c63b2-8594-44fa-9657-7d56ee386986\telasticsearch\t… with request body b'{\"post_filter\":{\"bool\":{\"must\":[{\"constant_score\":{\"filter\":{\"terms\":{\"sources.id.keyword\":[\"4e6b4bd3-8534-4ebf-868b-8a3ed739...'"
},
{
"@timestamp": "2023-11-22 18:29:39.952",
"@message": "[INFO]\t2023-11-22T18:29:39.952Z\tbe4c63b2-8594-44fa-9657-7d56ee386986\telasticsearch\tGot 200 response after 4.444s from POST to [https://vpc-azul-index-prod-mjluewlvxikvhillefxotw2tbm.us-east-1.es.amazonaws.com:443/azul_v2_prod_dcp32_files_aggregate/_search](https://vpc-azul-index-prod-mjluewlvxikvhillefxotw2tbm.us-east-1.es.amazonaws.com/azul_v2_prod_dcp32_files_aggregate/_search)"
},
{
"@timestamp": "2023-11-22 18:29:39.953",
"@message": "[INFO]\t2023-11-22T18:29:39.953Z\tbe4c63b2-8594-44fa-9657-7d56ee386986\telasticsearch\t… with response body '{\"took\":4372,\"timed_out\":false,\"_shards\":{\"total\":1,\"successful\":1,\"skipped\":0,\"failed\":0},\"hits\":{\"total\":{\"value\":508690,\"rel…'"
},
{
"@timestamp": "2023-11-22 18:29:39.978",
"@message": "[DEBUG]\t2023-11-22T18:29:39.978Z\tbe4c63b2-8594-44fa-9657-7d56ee386986\tazul.chalice\tReturning 200 response with headers {\"Access-Control-Allow-Origin\": \"*\", \"Access-Control-Allow-Headers\": \"Authorization,Content-Type,X-Amz-Date,X-Amz-Security-Token,X-Api-Key\", \"Strict-Transport-Security\": \"max-age=31536000; includeSubDomains\"}. See next line for the first 1024 characters of the body.\n{\"pagination\": {\"count\": 1, \"total\": 508690, \"size\": 1, \"next\": \"[https://service.azul.data.humancellatlas.org/index/files?catalog=dcp32&filters=%7B%7D&search_after=%5B%2200027d78-bcde-444c-b51c-b7446cdf9ea9_qc.bam%22%2C+%221cfe9091-ab53-58d8-9815-2c1216f240b4%22%5D&sort=fileName&order=asc&size=1\](https://service.azul.data.humancellatlas.org/index/files?catalog=dcp32&filters=%7B%7D&search_after=%5B%2200027d78-bcde-444c-b51c-b7446cdf9ea9_qc.bam%22%2C+%221cfe9091-ab53-58d8-9815-2c1216f240b4%22%5D&sort=fileName&order=asc&size=1%5C)", \"previous\": null, \"pages\": 508690, \"sort\": \"fileName\", \"order\": \"asc\"}, \"termFacets\": {\"organ\": {\"terms\": [{\"term\": \"brain\", \"count\": 85138}, {\"term\": \"blood\", \"count\": 46661}, {\"term\": \"pancreas\", \"count\": 36960}, {\"term\": \"adipose tissue\", \"count\": 33236}, {\"term\": \"heart\", \"count\": 30090}, {\"term\": \"decidua\", \"count\": 25162}, {\"term\": \"lung\", \"count\": 23468}, {\"term\": null, \"count\": 21101}, {\"term\": \"bone tissue\", \"count\": 20303}, {\"term\": \"large intestine\", \"count\": 18584}, {\"term\": \"skin of body\", \"count\": 17414}, {\"term\": \"embryo\", \"count\": 15518}, {\"term\": \"liver\", \"count\": 14815}, {\"term\": \"lymph node\", \"count\": 13720}, {\"term\": \"tumor\", \"count\": 12739}, {\"term\": \"hematopoietic system\", \"count\": 12024}, {\"te"
},
{
"@timestamp": "2023-11-22 18:29:39.989",
"@message": "END RequestId: be4c63b2-8594-44fa-9657-7d56ee386986\n"
},
{
"@timestamp": "2023-11-22 18:29:39.989",
"@message": "REPORT RequestId: be4c63b2-8594-44fa-9657-7d56ee386986\tDuration: 9863.97 ms\tBilled Duration: 9864 ms\tMemory Size: 2048 MB\tMax Memory Used: 216 MB\t\n"
}
]
hannes-ucsc
Note the two retries on read, both after waiting five seconds.