prashant-shahi
commented
1 year ago @abhinavramana Kind suggestion to either use collapsed sections or GitHub Gist URL.
Closed abhinavramana closed 1 year ago
prashant-shahi
commented
1 year ago @abhinavramana Kind suggestion to either use collapsed sections or GitHub Gist URL.
makeavish
commented
1 year ago @abhinavramana Can you please verify if this issue exists in latest version of SigNoz as well?
abhinavramana
commented
1 year ago Yes this is fixed
Bug description
Even though there is data being sent (I verified using
opentelemetry-instrument --traces_exporter otlp_proto_grpc --metrics_exporter otlp_proto_grpc,console --logs_exporter otlp_proto_grpc celery -A wombo.celery_paint.celeryapp worker --max-tasks-per-child 1 --gid=nogroup --loglevel=info --pool=threadsaddition in metrics-exporter Docker command):Even then These show as empty. There are 2 weird things:
Reproduce:
https://drive.google.com/file/d/1FVTwX4sI2YASdHYCy7nr_x6d74jJuJot/view?usp=share_link
The network tab gave response:
``` { "status": "success", "data": { "id": 5, "uuid": "921311ad-a04f-45cb-ae45-caaf9c39736b", "created_at": "2022-12-01T21:04:16.982395691Z", "updated_at": "2023-03-23T22:39:32.003194991Z", "data": { "description": "", "layout": [ { "h": 2, "i": "952e7f93-76fb-47d1-91d6-5494e0eed9f3", "moved": false, "static": false, "w": 6, "x": 6, "y": 2 }, { "h": 2, "i": "45e3bc8c-3af0-44bf-a813-eff5c27e5909", "moved": false, "static": false, "w": 6, "x": 0, "y": 2 }, { "h": 2, "i": "7b9b4650-5154-4541-899a-657c397a8613", "moved": false, "static": false, "w": 6, "x": 6, "y": 0 }, { "h": 2, "i": "c3bbe07f-107f-4a5b-b72f-4b495d591a46", "moved": false, "static": false, "w": 6, "x": 0, "y": 0 } ], "tags": [], "title": "PROD Diffusion", "widgets": [ { "description": "", "id": "c3bbe07f-107f-4a5b-b72f-4b495d591a46", "isStacked": false, "nullZeroValues": "zero", "opacity": "1", "panelTypes": "TIME_SERIES", "query": { "clickHouse": [ { "disabled": false, "legend": "", "name": "A", "rawQuery": "" } ], "metricsBuilder": { "formulas": [], "queryBuilder": [ { "aggregateOperator": 5, "disabled": false, "groupBy": [ "service_name" ], "legend": "Argos Translations", "metricName": "argos_translations", "name": "A", "reduceTo": 1, "tagFilters": { "items": [ { "id": "8ee3a134", "key": "service_name", "op": "IN", "value": [ "translate-service-prod" ] } ], "op": "AND" } }, { "aggregateOperator": 5, "disabled": false, "groupBy": [ "service_name" ], "legend": "GCP Translations", "metricName": "gcp_translations", "name": "B", "reduceTo": 1, "tagFilters": { "items": [ { "id": "46669690", "key": "service_name", "op": "IN", "value": [ "translate-service-prod" ] } ], "op": "AND" } } ] }, "promQL": [ { "disabled": false, "legend": "{{service_name}}", "name": "A", "query": "histogram_quantile(0.5, rate(time_total_for_user_task_bucket{service_name=\"prod-standard-ml-diffusion\"}[5m]))" } ], "queryType": 3 }, "queryData": { "data": { "queryData": [] }, "error": false, "errorMessage": "", "loading": false }, "timePreferance": "GLOBAL_TIME", "title": "Standard Diffusion time total for user task", "yAxisUnit": "none" }, { "description": "", "id": "7b9b4650-5154-4541-899a-657c397a8613", "isStacked": false, "nullZeroValues": "zero", "opacity": "1", "panelTypes": "TIME_SERIES", "query": { "clickHouse": [ { "disabled": false, "legend": "", "name": "A", "rawQuery": "" } ], "metricsBuilder": { "formulas": [], "queryBuilder": [ { "aggregateOperator": 1, "disabled": false, "groupBy": [], "legend": "", "metricName": null, "name": "A", "reduceTo": 1, "tagFilters": { "items": [], "op": "AND" } } ] }, "promQL": [ { "disabled": false, "legend": "{{service_name}}", "name": "A", "query": "histogram_quantile(0.5, rate(time_total_for_user_task_bucket{service_name=\"prod-premium-ml-diffusion\"}[5m]))" } ], "queryType": 3 }, "queryData": { "data": { "queryData": [ { "legend": "{{service_name}}", "metric": { "service_name": "prod-premium-ml-diffusion", "telemetry_auto_version": "0.34b0", "telemetry_sdk_language": "python", "telemetry_sdk_name": "opentelemetry", "telemetry_sdk_version": "1.13.0" }, "queryName": "A", "values": [ [ 1669907408, "5.074991112857356" ], [ 1669907468, "5.106109925672191" ], [ 1669907528, "5.133090187385623" ], [ 1669907588, "5.156896914692053" ], [ 1669907648, "5.181326716451258" ], [ 1669907708, "5.21742347877566" ], [ 1669907768, "5.182697554442185" ], [ 1669907828, "5.1347909658817885" ], [ 1669907888, "5.103743367941693" ], [ 1669907948, "5.074727182277169" ], [ 1669908008, "5.082588858561545" ], [ 1669908068, "5.108653632418745" ], [ 1669908128, "5.123003061120054" ], [ 1669908188, "5.113655975511168" ], [ 1669908248, "5.134633490248823" ], [ 1669908308, "5.11563045002607" ], [ 1669908368, "5.100940438871473" ], [ 1669908428, "5.089377029766925" ], [ 1669908488, "5.059526649378838" ], [ 1669908548, "5.036855977349317" ], [ 1669908608, "5.047015673435236" ], [ 1669908668, "5.045984239624655" ], [ 1669908728, "5.0468169228538935" ], [ 1669908788, "5.047521421703634" ], [ 1669908848, "5.030369942970708" ], [ 1669908908, "5.034613923148778" ], [ 1669908968, "5.040914437382705" ], [ 1669909028, "5.027111255495208" ], [ 1669909088, "5.062919694570164" ], [ 1669909148, "5.104189681413432" ], [ 1669909208, "5.111156058521872" ], [ 1669909268, "5.1011594501706" ], [ 1669909328, "5.129954891759339" ], [ 1669909388, "5.13409211411098" ], [ 1669909448, "5.12311319649145" ], [ 1669909508, "5.093469539170077" ], [ 1669909568, "5.097197928017688" ], [ 1669909628, "5.091163344560963" ], [ 1669909688, "5.067401166397855" ], [ 1669909748, "5.045345484946063" ], [ 1669909808, "5.048884689793689" ], [ 1669909868, "5.0283018575547915" ], [ 1669909928, "5.018152000496972" ], [ 1669909988, "5.009999914276685" ], [ 1669910048, "5.0130483991500006" ], [ 1669910108, "5.0046027083800135" ], [ 1669910168, "5.032613597789252" ], [ 1669910228, "5.03075924708399" ], [ 1669910288, "5.096191452862156" ], [ 1669910348, "5.122469393555273" ], [ 1669910408, "5.137750443334876" ], [ 1669910468, "5.089158191205739" ], [ 1669910528, "5.090999207215871" ], [ 1669910588, "5.056655259943623" ], [ 1669910648, "5.0575208407855365" ], [ 1669910708, "5.07603635722128" ], [ 1669910768, "5.114864753428937" ], [ 1669910828, "5.108462662192404" ], [ 1669910888, "5.088579924191856" ], [ 1669910948, "5.078612609376896" ], [ 1669911008, "5.04242194420992" ], [ 1669911068, "5.020352715526826" ], [ 1669911128, "4.998808372381498" ], [ 1669911188, "4.968376405586977" ], [ 1669911248, "4.934490259177162" ], [ 1669911308, "4.941983632399069" ], [ 1669911368, "4.942019258306055" ], [ 1669911428, "4.978897628641098" ], [ 1669911488, "5.0031743096646935" ], [ 1669911548, "5.033496485866606" ], [ 1669911608, "5.028454962557658" ], [ 1669911668, "5.00936791694085" ], [ 1669911728, "4.995258301012779" ], [ 1669911788, "4.996093276801938" ], [ 1669911848, "5.000451766406649" ], [ 1669911908, "5.022539916305968" ], [ 1669911968, "5.044594070574965" ], [ 1669912028, "5.030475416497359" ], [ 1669912088, "5.026924639621854" ], [ 1669912148, "5.027834209122584" ], [ 1669912208, "5.033495156516279" ], [ 1669912268, "5.057768017625992" ], [ 1669912328, "5.094077952447003" ], [ 1669912388, "5.13604056588512" ], [ 1669912448, "5.159829364215157" ], [ 1669912508, "5.178090787479643" ], [ 1669912568, "5.193519658170826" ], [ 1669912628, "5.205941956700495" ], [ 1669912688, "5.197073361026472" ], [ 1669912748, "5.217601796720575" ], [ 1669912808, "5.218164696783039" ], [ 1669912868, "5.233287894912751" ], [ 1669912928, "5.232338554453068" ], [ 1669912988, "5.24505338161219" ], [ 1669913048, "5.231640172926733" ], [ 1669913108, "5.227282298644403" ], [ 1669913168, "5.237649103971298" ], [ 1669913228, "5.238853068063857" ], [ 1669913288, "5.243384754210321" ], [ 1669913348, "5.251871206458351" ], [ 1669913408, "5.249248716372682" ], [ 1669913468, "5.218405565902648" ], [ 1669913528, "5.202626538385328" ], [ 1669913588, "5.176672051125882" ], [ 1669913648, "5.1687618372765405" ], [ 1669913708, "5.174917935981381" ], [ 1669913768, "5.194417297036652" ], [ 1669913828, "5.197882388136245" ], [ 1669913888, "5.2128931590953425" ], [ 1669913948, "5.220978847439947" ], [ 1669914008, "5.223021327113274" ], [ 1669914068, "5.226224250391028" ], [ 1669914128, "5.253151658377523" ], [ 1669914188, "5.272673202346282" ], [ 1669914248, "5.283772794730529" ], [ 1669914308, "5.300767251145776" ], [ 1669914368, "5.297169659828483" ], [ 1669914428, "5.281367328923928" ], [ 1669914488, "5.255339510752709" ], [ 1669914548, "5.217187950783135" ], [ 1669914608, "5.2051321647978686" ], [ 1669914668, "5.195039416186329" ], [ 1669914728, "5.20416878715791" ], [ 1669914788, "5.262646479833834" ], [ 1669914848, "5.34017670930679" ], [ 1669914908, "5.393084887043092" ], [ 1669914968, "5.431510622413255" ], [ 1669915028, "5.441607776068726" ], [ 1669915088, "5.43705845259344" ], [ 1669915148, "5.417878296406908" ], [ 1669915208, "5.398357999777769" ], [ 1669915268, "5.387905778616191" ], [ 1669915328, "5.397703357067314" ], [ 1669915388, "5.394306524631947" ], [ 1669915448, "5.372713220970611" ], [ 1669915508, "5.350943609847225" ], [ 1669915568, "5.317827513091985" ], [ 1669915628, "5.285927685690118" ], [ 1669915688, "5.265278265417988" ], [ 1669915748, "5.257293636295655" ], [ 1669915808, "5.226388741557937" ], [ 1669915868, "5.251010921407341" ], [ 1669915928, "5.2559872299626695" ], [ 1669915988, "5.239496226799918" ], [ 1669916048, "5.216381690416561" ], [ 1669916108, "5.200260287930913" ], [ 1669916168, "5.165598725250881" ], [ 1669916228, "5.12087081931186" ], [ 1669916288, "5.082277308242658" ], [ 1669916348, "5.06181575525524" ], [ 1669916408, "5.086253373842043" ], [ 1669916468, "5.089123745953476" ], [ 1669916528, "5.11792468252619" ], [ 1669916588, "5.12417121874635" ], [ 1669916648, "5.148122143600215" ], [ 1669916708, "5.132193889762149" ], [ 1669916768, "5.120805281476307" ], [ 1669916828, "5.082900781081574" ], [ 1669916888, "5.09997748851252" ], [ 1669916948, "5.071606880393013" ], [ 1669917008, "5.051480404984735" ], [ 1669917068, "5.051078047455291" ], [ 1669917128, "5.081464281862656" ], [ 1669917188, "5.083675263774912" ], [ 1669917248, "5.120339261762277" ], [ 1669917308, "5.165600355188727" ], [ 1669917368, "5.164545763002096" ], [ 1669917428, "5.2029319495811945" ], [ 1669917488, "5.247522317586011" ], [ 1669917548, "5.255664857427729" ], [ 1669917608, "5.26680841845365" ], [ 1669917668, "5.30029046006539" ], [ 1669917728, "5.270333020843373" ], [ 1669917788, "5.249623708112392" ], [ 1669917848, "5.245182757250697" ], [ 1669917908, "5.235036282094341" ], [ 1669917968, "5.227905926056074" ], [ 1669918028, "5.235153371034702" ], [ 1669918088, "5.224225304966717" ], [ 1669918148, "5.204608392029379" ], [ 1669918208, "5.173772273104598" ], [ 1669918268, "5.155027320444961" ], [ 1669918328, "5.135350340599265" ], [ 1669918388, "5.092507274830346" ], [ 1669918448, "5.067832195496881" ], [ 1669918508, "5.05076077445225" ], [ 1669918568, "5.023279103522366" ], [ 1669918628, "5.019110646729909" ], [ 1669918688, "5.044319386375075" ], [ 1669918748, "5.057214365189694" ], [ 1669918808, "5.083846952102711" ], [ 1669918868, "5.0952536158077" ], [ 1669918928, "5.111840312471614" ], [ 1669918988, "5.147348294227422" ], [ 1669919048, "5.2001577940748716" ], [ 1669919108, "5.200098409053632" ], [ 1669919168, "5.222986511985819" ], [ 1669919228, "5.25413024198657" ], [ 1669919288, "5.275410056808365" ], [ 1669919348, "5.264379707869979" ], [ 1669919408, "5.304245808099381" ], [ 1669919468, "5.310730681897056" ], [ 1669919528, "5.308986135395376" ], [ 1669919588, "5.298903196780117" ], [ 1669919648, "5.27815546356209" ], [ 1669919708, "5.235928870159307" ], [ 1669919768, "5.2322014065794775" ], [ 1669919828, "5.199361178524704" ], [ 1669919888, "5.160267710674638" ], [ 1669919948, "5.15362968278259" ], [ 1669920008, "5.162469762025371" ], [ 1669920068, "5.160283425187138" ], [ 1669920128, "5.174026643726889" ], [ 1669920188, "5.190018024123148" ], [ 1669920248, "5.205967515194468" ], [ 1669920308, "5.189729403762683" ], [ 1669920368, "5.163637837952179" ], [ 1669920428, "5.134684375058851" ], [ 1669920488, "5.107985112342303" ], [ 1669920548, "5.0806921063715995" ], [ 1669920608, "5.078249603634729" ], [ 1669920668, "5.083273286654942" ], [ 1669920728, "5.085381740930126" ], [ 1669920788, "5.102975686147578" ], [ 1669920848, "5.113716803648629" ], [ 1669920908, "5.125425324394008" ], [ 1669920968, "5.12054060565067" ], [ 1669921028, "5.1230594255167246" ], [ 1669921088, "5.1034320879880335" ], [ 1669921148, "5.077137784590128" ], [ 1669921208, "5.048980559829699" ], [ 1669921268, "5.040548047912413" ], [ 1669921328, "5.049782468832531" ], [ 1669921388, "5.064837425067726" ], [ 1669921448, "5.0747155479839" ], [ 1669921508, "5.084375976446461" ], [ 1669921568, "5.093850823289466" ], [ 1669921628, "5.0799908279535755" ], [ 1669921688, "5.057019952814141" ], [ 1669921748, "5.074861798725662" ], [ 1669921808, "5.096766594483878" ], [ 1669921868, "5.100384909927961" ], [ 1669921928, "5.111073595182217" ], [ 1669921988, "5.130646415313352" ], [ 1669922048, "5.119227047441804" ], [ 1669922108, "5.124165824937625" ], [ 1669922168, "5.103928101971294" ], [ 1669922228, "5.066375705646455" ], [ 1669922288, "5.058455520052558" ], [ 1669922348, "5.0713333765720225" ], [ 1669922408, "5.057562109788277" ], [ 1669922468, "5.060129504848017" ], [ 1669922528, "5.086128673610083" ], [ 1669922588, "5.078992537703958" ], [ 1669922648, "5.06976531842968" ], [ 1669922708, "5.068684667499273" ], [ 1669922768, "5.075460245308251" ], [ 1669922828, "5.060281376983823" ], [ 1669922888, "5.029455813605195" ], [ 1669922948, "4.974820352902438" ], [ 1669923008, "4.975512908180329" ], [ 1669923068, "4.977179459502932" ], [ 1669923128, "4.973251520256346" ], [ 1669923188, "5.000377821063944" ], [ 1669923248, "5.00291287798689" ], [ 1669923308, "4.939062770040902" ], [ 1669923368, "4.90501319541539" ], [ 1669923428, "4.90131038492162" ], [ 1669923488, "4.887648542343442" ], [ 1669923548, "4.8884638242518195" ], [ 1669923608, "4.907531671077836" ], [ 1669923668, "4.877498212066438" ], [ 1669923728, "4.879605283509141" ], [ 1669923788, "4.899379812910228" ], [ 1669923848, "4.954144050017483" ], [ 1669923908, "4.965989179520448" ], [ 1669923968, "5.049031197051803" ], [ 1669924028, "5.096719475745358" ], [ 1669924088, "5.1304102351600545" ], [ 1669924148, "5.110855174504784" ], [ 1669924208, "5.14492451804632" ], [ 1669924268, "5.1547621309319505" ], [ 1669924328, "5.164458149142476" ], [ 1669924388, "5.162629041165725" ], [ 1669924448, "5.145588069952464" ], [ 1669924508, "5.121496562971553" ], [ 1669924568, "5.089570291811899" ], [ 1669924628, "5.061214121366977" ], [ 1669924688, "5.025477083234387" ], [ 1669924748, "5.0383243717838" ], [ 1669924808, "5.038251109724107" ], [ 1669924868, "5.01841393555529" ], [ 1669924928, "4.986453775292613" ], [ 1669924988, "4.949773867786584" ], [ 1669925048, "4.940719559853851" ], [ 1669925108, "4.934000996801742" ], [ 1669925168, "4.94952322855419" ], [ 1669925228, "4.939708814443154" ], [ 1669925288, "4.976525796102945" ], [ 1669925348, "4.9631217637574965" ], [ 1669925408, "4.94819519066895" ], [ 1669925468, "4.9335300748154385" ], [ 1669925528, "4.941764433783075" ], [ 1669925588, "4.929427189665326" ], [ 1669925648, "4.958350364782679" ], [ 1669925708, "4.987865917610775" ], [ 1669925768, "4.992184967955755" ], [ 1669925828, "4.996814048770541" ], [ 1669925888, "4.968388593967163" ], [ 1669925948, "4.90738085319724" ], [ 1669926008, "4.861408878898216" ], [ 1669926068, "4.843038925235567" ], [ 1669926128, "4.835312868973561" ], [ 1669926188, "4.836574896136888" ], [ 1669926248, "4.858497299877541" ], [ 1669926308, "4.9183950396245795" ], [ 1669926368, "4.962862747665622" ], [ 1669926428, "4.997578737344602" ], [ 1669926488, "5.0466551253136815" ], [ 1669926548, "5.082061219127692" ], [ 1669926608, "5.060485226068298" ], [ 1669926668, "5.061975931919572" ], [ 1669926728, "5.088149111821707" ], [ 1669926788, "5.114657246578116" ], [ 1669926848, "5.1658381447223" ], [ 1669926908, "5.220315786148194" ], [ 1669926968, "5.244263180835801" ], [ 1669927028, "5.223540398450471" ], [ 1669927088, "5.201617649135858" ], [ 1669927148, "5.164724311834779" ], [ 1669927208, "5.1045401869576335" ], [ 1669927268, "5.076313592558683" ], [ 1669927328, "5.0584454691842256" ], [ 1669927388, "5.040878341713996" ], [ 1669927448, "5.045227794912132" ], [ 1669927508, "5.076569341940863" ], [ 1669927568, "5.079563356775029" ], [ 1669927628, "5.097297949845792" ], [ 1669927688, "5.100167065804944" ], [ 1669927748, "5.078052794256993" ], [ 1669927808, "5.075131705714814" ], [ 1669927868, "5.077315263962688" ], [ 1669927928, "5.096439165628836" ], [ 1669927988, "5.14142038938692" ], [ 1669928048, "5.124659571197543" ], [ 1669928108, "5.118245382593388" ], [ 1669928168, "5.150954911515643" ], [ 1669928228, "5.191823252147804" ], [ 1669928288, "5.195773703290048" ], [ 1669928348, "5.28555503912958" ], [ 1669928408, "5.321726671207474" ], [ 1669928468, "5.340734048431254" ], [ 1669928528, "5.323346943448784" ], [ 1669928588, "5.336681409686235" ], [ 1669928648, "5.322386276894269" ], [ 1669928708, "5.311551711869347" ], [ 1669928768, "5.280373472846655" ], [ 1669928828, "5.254524318737176" ], [ 1669928888, "5.256551784390276" ], [ 1669928948, "5.255159005079708" ], [ 1669929008, "5.21319321378839" ] ] } ] }, "error": false, "errorMessage": "", "loading": false }, "timePreferance": "GLOBAL_TIME", "title": "Premium Diffusion time total for user task", "yAxisUnit": "s" }, { "description": "", "id": "08b59740-daab-443c-a51a-a48c16d6983a", "isStacked": false, "nullZeroValues": "zero", "opacity": "0", "panelTypes": "TIME_SERIES", "query": { "clickHouse": [ { "disabled": false, "legend": "", "name": "A", "rawQuery": "" } ], "metricsBuilder": { "formulas": [], "queryBuilder": [ { "aggregateOperator": 1, "disabled": false, "groupBy": [], "legend": "", "metricName": null, "name": "A", "reduceTo": 1, "tagFilters": { "items": [], "op": "AND" } } ] }, "promQL": [ { "disabled": false, "legend": "{{service_name}}", "name": "A", "query": "histogram_quantile(0.5, rate(time_total_for_user_task_bucket{service_name=\"prod-standard-ml-diffusion\"}[5m]))" }, { "disabled": false, "legend": "{{service_name}}", "name": "B", "query": "histogram_quantile(0.5, rate(time_total_for_user_task_bucket{service_name=\"prod-premium-ml-diffusion\"}[5m]))" } ], "queryType": 3 }, "queryData": { "data": { "queryData": [ { "legend": "{{service_name}}", "metric": { "service_name": "prod-premium-ml-diffusion", "telemetry_auto_version": "0.34b0", "telemetry_sdk_language": "python", "telemetry_sdk_name": "opentelemetry", "telemetry_sdk_version": "1.13.0" }, "queryName": "B", "values": [ [ 1669907337, "5.057065618541483" ], [ 1669907397, "5.068580624336297" ], [ 1669907457, "5.099744336339748" ], [ 1669907517, "5.126156130455854" ], [ 1669907577, "5.160085664680333" ], [ 1669907637, "5.188724950075931" ], [ 1669907697, "5.226323846242039" ], [ 1669907757, "5.195027554729762" ], [ 1669907817, "5.148472570976593" ], [ 1669907877, "5.104143126742081" ], [ 1669907937, "5.076826541693517" ], [ 1669907997, "5.082661317433579" ], [ 1669908057, "5.108496554665012" ], [ 1669908117, "5.123704898769945" ], [ 1669908177, "5.1132110693244215" ], [ 1669908237, "5.1337507753993865" ], [ 1669908297, "5.1148324712073965" ], [ 1669908357, "5.102284350494374" ], [ 1669908417, "5.087470310925579" ], [ 1669908477, "5.0606015632729795" ], [ 1669908537, "5.039318538849064" ], [ 1669908597, "5.048193750621054" ], [ 1669908657, "5.046589326726586" ], [ 1669908717, "5.048254067864987" ], [ 1669908777, "5.0508358588327305" ], [ 1669908837, "5.030637743573384" ], [ 1669908897, "5.03632868643582" ], [ 1669908957, "5.041266165676336" ], [ 1669909017, "5.028948375759052" ], [ 1669909077, "5.061244910634187" ], [ 1669909137, "5.102397390825533" ], [ 1669909197, "5.11142802892712" ], [ 1669909257, "5.099354563921105" ], [ 1669909317, "5.13090913484386" ], [ 1669909377, "5.132059945245367" ], [ 1669909437, "5.121137319436826" ], [ 1669909497, "5.08868900759336" ], [ 1669909557, "5.0951303410504964" ], [ 1669909617, "5.086652983921454" ], [ 1669909677, "5.065605531971867" ], [ 1669909737, "5.046955632351346" ], [ 1669909797, "5.050794968879274" ], [ 1669909857, "5.026593545380053" ], [ 1669909917, "5.0277481994173066" ], [ 1669909977, "5.012919753360245" ], [ 1669910037, "5.012912392975074" ], [ 1669910097, "5.012505484861782" ], [ 1669910157, "5.03179517063044" ], [ 1669910217, "5.0245311095722895" ], [ 1669910277, "5.094153489046859" ], [ 1669910337, "5.12890174254707" ], [ 1669910397, "5.131974114330832" ], [ 1669910457, "5.09036890319386" ], [ 1669910517, "5.09236041150387" ], [ 1669910577, "5.048697458491654" ], [ 1669910637, "5.049993842642707" ], [ 1669910697, "5.067440066410319" ], [ 1669910757, "5.1104975537697275" ], [ 1669910817, "5.109364141540145" ], [ 1669910877, "5.088277944737193" ], [ 1669910937, "5.069873900656296" ], [ 1669910997, "5.042953659021272" ], [ 1669911057, "5.021178998988288" ], [ 1669911117, "4.9908237971053415" ], [ 1669911177, "4.969064348168305" ], [ 1669911237, "4.9301496070379285" ], [ 1669911297, "4.9325727961217165" ], [ 1669911357, "4.93168170765319" ], [ 1669911417, "4.968086716601755" ], [ 1669911477, "4.992139279148718" ], [ 1669911537, "5.021911121584998" ], [ 1669911597, "5.023584250883439" ], [ 1669911657, "5.005102267946866" ], [ 1669911717, "5.001327563265256" ], [ 1669911777, "5.0035395917772085" ], [ 1669911837, "5.012489617065331" ], [ 1669911897, "5.027618357979402" ], [ 1669911957, "5.049842756225211" ], [ 1669912017, "5.019330724756177" ], [ 1669912077, "5.009200962918383" ], [ 1669912137, "5.015045478377824" ], [ 1669912197, "5.038331666535571" ], [ 1669912257, "5.056586363686897" ], [ 1669912317, "5.103532044983077" ], [ 1669912377, "5.150908456563163" ], [ 1669912437, "5.166621554922222" ], [ 1669912497, "5.168407016587901" ], [ 1669912557, "5.1831967805926356" ], [ 1669912617, "5.194713473504083" ], [ 1669912677, "5.186060189200235" ], [ 1669912737, "5.2101866046929475" ], [ 1669912797, "5.210960397238706" ], [ 1669912857, "5.225590813136947" ], [ 1669912917, "5.235025443536294" ], [ 1669912977, "5.2490305036651685" ], [ 1669913037, "5.2320923489475" ], [ 1669913097, "5.228039068864996" ], [ 1669913157, "5.238524334418381" ], [ 1669913217, "5.234195461851273" ], [ 1669913277, "5.2373981497391" ], [ 1669913337, "5.243907598578439" ], [ 1669913397, "5.241395119198758" ], [ 1669913457, "5.216269946105886" ], [ 1669913517, "5.208280998655826" ], [ 1669913577, "5.176378433637365" ], [ 1669913637, "5.170482553431216" ], [ 1669913697, "5.182673871361339" ], [ 1669913757, "5.202744671765208" ], [ 1669913817, "5.193957490528478" ], [ 1669913877, "5.21420207711105" ], [ 1669913937, "5.222390088501534" ], [ 1669913997, "5.2127276706977765" ], [ 1669914057, "5.208317645188552" ], [ 1669914117, "5.23547593450215" ], [ 1669914177, "5.261165316225458" ], [ 1669914237, "5.2721806722517455" ], [ 1669914297, "5.294709934911434" ], [ 1669914357, "5.2982767705940805" ], [ 1669914417, "5.288202492324074" ], [ 1669914477, "5.261409100155964" ], [ 1669914537, "5.230258522464197" ], [ 1669914597, "5.218405258759601" ], [ 1669914657, "5.19581539693663" ], [ 1669914717, "5.199185232538287" ], [ 1669914777, "5.243613042613514" ], [ 1669914837, "5.314420377816554" ], [ 1669914897, "5.368010371766242" ], [ 1669914957, "5.421546430964844" ], [ 1669915017, "5.444678713057833" ], [ 1669915077, "5.44564180676755" ], [ 1669915137, "5.42562279100063" ], [ 1669915197, "5.411966152739855" ], [ 1669915257, "5.404267535913493" ], [ 1669915317, "5.407166468146289" ], [ 1669915377, "5.401581992342578" ], [ 1669915437, "5.3859062182817405" ], [ 1669915497, "5.3589165226912385" ], [ 1669915557, "5.325600529431755" ], [ 1669915617, "5.300296593677473" ], [ 1669915677, "5.266393956660252" ], [ 1669915737, "5.253427836892095" ], [ 1669915797, "5.222168843002456" ], [ 1669915857, "5.245968732802869" ], [ 1669915917, "5.257937252147346" ], [ 1669915977, "5.254911593025351" ], [ 1669916037, "5.237179573755436" ], [ 1669916097, "5.2146791308741784" ], [ 1669916157, "5.187041382397221" ], [ 1669916217, "5.128455741025234" ], [ 1669916277, "5.082173360675546" ], [ 1669916337, "5.047107021912008" ], [ 1669916397, "5.085466527097113" ], [ 1669916457, "5.08171159981654" ], [ 1669916517, "5.116960781435045" ], [ 1669916577, "5.129790572366658" ], [ 1669916637, "5.153480498853819" ], [ 1669916697, "5.131068494744976" ], [ 1669916757, "5.120889338379225" ], [ 1669916817, "5.082450711784447" ], [ 1669916877, "5.099505983420355" ], [ 1669916937, "5.078565492021277" ], [ 1669916997, "5.058539784803401" ], [ 1669917057, "5.055878519766404" ], [ 1669917117, "5.0884309814948026" ], [ 1669917177, "5.082920400976051" ], [ 1669917237, "5.112493993117484" ], [ 1669917297, "5.157595411920699" ], [ 1669917357, "5.15757380340482" ], [ 1669917417, "5.190171575144283" ], [ 1669917477, "5.249498140348284" ], [ 1669917537, "5.2641952369838485" ], [ 1669917597, "5.268489290745992" ], [ 1669917657, "5.307496773518434" ], [ 1669917717, "5.270462519097572" ], [ 1669917777, "5.24269009213275" ], [ 1669917837, "5.238432959917224" ], [ 1669917897, "5.235089376092107" ], [ 1669917957, "5.222341790952757" ], [ 1669918017, "5.235923499281019" ], [ 1669918077, "5.2259257728550175" ], [ 1669918137, "5.206228273862685" ], [ 1669918197, "5.175406157981449" ], [ 1669918257, "5.157182013155316" ], [ 1669918317, "5.13805535526097" ], [ 1669918377, "5.09466359058294" ], [ 1669918437, "5.077826776961645" ], [ 1669918497, "5.0677478117668295" ], [ 1669918557, "5.033493190429054" ], [ 1669918617, "5.02068176442594" ], [ 1669918677, "5.0464404113052765" ], [ 1669918737, "5.051168368796698" ], [ 1669918797, "5.0707067370292975" ], [ 1669918857, "5.089824427895853" ], [ 1669918917, "5.106284460146673" ], [ 1669918977, "5.1419109013105535" ], [ 1669919037, "5.195219482089129" ], [ 1669919097, "5.209138266746862" ], [ 1669919157, "5.23237329698892" ], [ 1669919217, "5.256083596848435" ], [ 1669919277, "5.2774131171388525" ], [ 1669919337, "5.272882586488763" ], [ 1669919397, "5.292836951719152" ], [ 1669919457, "5.298783297425871" ], [ 1669919517, "5.3101234482741635" ], [ 1669919577, "5.300023112387023" ], [ 1669919637, "5.280060466979929" ], [ 1669919697, "5.2443379663138945" ], [ 1669919757, "5.233767229855587" ], [ 1669919817, "5.201051711155911" ], [ 1669919877, "5.169325773558368" ], [ 1669919937, "5.147922953152739" ], [ 1669919997, "5.167145045164558" ], [ 1669920057, "5.157874237503446" ], [ 1669920117, "5.168405100782355" ], [ 1669920177, "5.177441812797298" ], [ 1669920237, "5.207575448234902" ], [ 1669920297, "5.184159011845627" ], [ 1669920357, "5.172070687488612" ], [ 1669920417, "5.1436834083678065" ], [ 1669920477, "5.117074330255754" ], [ 1669920537, "5.082095581528139" ], [ 1669920597, "5.08005610668786" ], [ 1669920657, "5.092838384220073" ], [ 1669920717, "5.094500105726076" ], [ 1669920777, "5.112062375057574" ], [ 1669920837, "5.1227722209189075" ], [ 1669920897, "5.126792402416901" ], [ 1669920957, "5.114682912829319" ], [ 1669921017, "5.117253516288741" ], [ 1669921077, "5.105150667881065" ], [ 1669921137, "5.071574862441087" ], [ 1669921197, "5.050933699632042" ], [ 1669921257, "5.042508703762315" ], [ 1669921317, "5.051705390480727" ], [ 1669921377, "5.059297198562817" ], [ 1669921437, "5.07693089812762" ], [ 1669921497, "5.086682079752802" ], [ 1669921557, "5.096082632822373" ], [ 1669921617, "5.073698593908405" ], [ 1669921677, "5.0590282999672676" ], [ 1669921737, "5.068371923735039" ], [ 1669921797, "5.0902608099592035" ], [ 1669921857, "5.101864441422947" ], [ 1669921917, "5.121035988622397" ], [ 1669921977, "5.132523446498109" ], [ 1669922037, "5.129103049488051" ], [ 1669922097, "5.149813778064621" ], [ 1669922157, "5.1291932920027685" ], [ 1669922217, "5.0919390807261165" ], [ 1669922277, "5.084232582836375" ], [ 1669922337, "5.080520864617747" ], [ 1669922397, "5.051140409667825" ], [ 1669922457, "5.04609413418374" ], [ 1669922517, "5.072233646289902" ], [ 1669922577, "5.064874552725299" ], [ 1669922637, "5.071978189971463" ], [ 1669922697, "5.070852190248484" ], [ 1669922757, "5.076523340948443" ], [ 1669922817, "5.070313681792905" ], [ 1669922877, "5.039399520539457" ], [ 1669922937, "4.985754093403033" ], [ 1669922997, "4.986140518088756" ], [ 1669923057, "4.987612416655703" ], [ 1669923117, "4.974981026268078" ], [ 1669923177, "5.002085468580686" ], [ 1669923237, "5.003608036426399" ], [ 1669923297, "4.939851626524967" ], [ 1669923357, "4.897464245088892" ], [ 1669923417, "4.893935676027855" ], [ 1669923477, "4.880883611803537" ], [ 1669923537, "4.8817062022848985" ], [ 1669923597, "4.901161979384707" ], [ 1669923657, "4.8774384940357844" ], [ 1669923717, "4.878789009153513" ], [ 1669923777, "4.9083272840402365" ], [ 1669923837, "4.954020900918956" ], [ 1669923897, "4.9657754922832975" ], [ 1669923957, "5.049553067056555" ], [ 1669924017, "5.0971035275059515" ], [ 1669924077, "5.122185163839434" ], [ 1669924137, "5.112432203407694" ], [ 1669924197, "5.148813814284371" ], [ 1669924257, "5.158065125455176" ], [ 1669924317, "5.168327775090587" ], [ 1669924377, "5.173758265009257" ], [ 1669924437, "5.165252809752892" ], [ 1669924497, "5.138139153666574" ], [ 1669924557, "5.106371421267894" ], [ 1669924617, "5.06686390024942" ], [ 1669924677, "5.023045178105995" ], [ 1669924737, "5.036244551026428" ], [ 1669924797, "5.037366439157012" ], [ 1669924857, "5.018566551998466" ], [ 1669924917, "4.9877374062715685" ], [ 1669924977, "4.951091430987295" ], [ 1669925037, "4.941523827799025" ], [ 1669925097, "4.934484825710401" ], [ 1669925157, "4.949590341622848" ], [ 1669925217, "4.939502512871339" ], [ 1669925277, "4.965896961115346" ], [ 1669925337, "4.962961415149333" ], [ 1669925397, "4.937824907034283" ], [ 1669925457, "4.922653578043302" ], [ 1669925517, "4.9319007573864635" ], [ 1669925577, "4.930115985136204" ], [ 1669925637, "4.95780849921533" ], [ 1669925697, "4.996703911427758" ], [ 1669925757, "5.001030379368862" ], [ 1669925817, "5.005393421940058" ], [ 1669925877, "4.9877079939616715" ], [ 1669925937, "4.917326177423243" ], [ 1669925997, "4.8725138802502075" ], [ 1669926057, "4.85308493730508" ], [ 1669926117, "4.833592732387567" ], [ 1669926177, "4.8250013808692245" ], [ 1669926237, "4.84818639149235" ], [ 1669926297, "4.879657083924197" ], [ 1669926357, "4.936993857339852" ], [ 1669926417, "4.99856413780081" ], [ 1669926477, "5.0477671701828335" ], [ 1669926537, "5.081731391885141" ], [ 1669926597, "5.073570774929942" ], [ 1669926657, "5.061846800997351" ], [ 1669926717, "5.074514402579115" ], [ 1669926777, "5.10084302256743" ], [ 1669926837, "5.15173952055063" ], [ 1669926897, "5.220322115688737" ], [ 1669926957, "5.255705496671746" ], [ 1669927017, "5.234464812351819" ], [ 1669927077, "5.2137150559855705" ], [ 1669927137, "5.178588896603226" ], [ 1669927197, "5.134205858983915" ], [ 1669927257, "5.091810040624353" ], [ 1669927317, "5.07480414295987" ], [ 1669927377, "5.054914994773059" ], [ 1669927437, "5.059662136858683" ], [ 1669927497, "5.076241325722023" ], [ 1669927557, "5.0688509268394" ], [ 1669927617, "5.084704693424449" ], [ 1669927677, "5.101390625484381" ], [ 1669927737, "5.0783903670575095" ], [ 1669927797, "5.0593704880918375" ], [ 1669927857, "5.076810311588633" ], [ 1669927917, "5.082269418466941" ], [ 1669927977, "5.114764033980314" ], [ 1669928037, "5.137780142002266" ], [ 1669928097, "5.120714942700176" ], [ 1669928157, "5.159515266896097" ], [ 1669928217, "5.217001897623907" ], [ 1669928277, "5.223147223059417" ], [ 1669928337, "5.2741541113455765" ], [ 1669928397, "5.33330381214402" ], [ 1669928457, "5.3532577631657166" ], [ 1669928517, "5.332883514307034" ], [ 1669928577, "5.334597959792511" ], [ 1669928637, "5.312860743309432" ], [ 1669928697, "5.300255463113198" ], [ 1669928757, "5.272606072764937" ], [ 1669928817, "5.245057962811563" ], [ 1669928877, "5.243375391752889" ], [ 1669928937, "5.238452049324498" ] ] }, { "legend": "{{service_name}}", "metric": { "service_name": "prod-standard-ml-diffusion", "telemetry_auto_version": "0.34b0", "telemetry_sdk_language": "python", "telemetry_sdk_name": "opentelemetry", "telemetry_sdk_version": "1.13.0" }, "queryName": "A", "values": [ [ 1669907337, "3.5184568847500146" ], [ 1669907397, "3.525399130354094" ], [ 1669907457, "3.509866331296" ], [ 1669907517, "3.5005874282213223" ], [ 1669907577, "3.491239060203635" ], [ 1669907637, "3.486128333609269" ], [ 1669907697, "3.4692957736617878" ], [ 1669907757, "3.4464196362283213" ], [ 1669907817, "3.40968635552634" ], [ 1669907877, "3.3829990801082137" ], [ 1669907937, "3.3709430554140214" ], [ 1669907997, "3.3728238098896157" ], [ 1669908057, "3.3774594008407997" ], [ 1669908117, "3.407658030433579" ], [ 1669908177, "3.4413640415504823" ], [ 1669908237, "3.4778608833958797" ], [ 1669908297, "3.50328881750553" ], [ 1669908357, "3.534751311210342" ], [ 1669908417, "3.5822516658631693" ], [ 1669908477, "3.602048679183379" ], [ 1669908537, "3.6035173198191033" ], [ 1669908597, "3.6144231496031116" ], [ 1669908657, "3.6398511257354094" ], [ 1669908717, "3.620643459766597" ], [ 1669908777, "3.6201299657275583" ], [ 1669908837, "3.6183468132099126" ], [ 1669908897, "3.614655741681219" ], [ 1669908957, "3.6064874080193725" ], [ 1669909017, "3.5949516997956525" ], [ 1669909077, "3.590127121034519" ], [ 1669909137, "3.5818549131733284" ], [ 1669909197, "3.5717805652186603" ], [ 1669909257, "3.5620596723188753" ], [ 1669909317, "3.5565010427511226" ], [ 1669909377, "3.5473206111042788" ], [ 1669909437, "3.546020518768598" ], [ 1669909497, "3.5472909309175176" ], [ 1669909557, "3.5549387354776796" ], [ 1669909617, "3.5651915219253287" ], [ 1669909677, "3.576596902763634" ], [ 1669909737, "3.57297647486135" ], [ 1669909797, "3.570314909068231" ], [ 1669909857, "3.556426122499408" ], [ 1669909917, "3.53395786953345" ], [ 1669909977, "3.5214957978010837" ], [ 1669910037, "3.5038165844826525" ], [ 1669910097, "3.492028412485195" ], [ 1669910157, "3.482402986370345" ], [ 1669910217, "3.4953411272029316" ], [ 1669910277, "3.496965360187197" ], [ 1669910337, "3.5131727335379987" ], [ 1669910397, "3.519279760151118" ], [ 1669910457, "3.543678473084049" ], [ 1669910517, "3.5531233360803904" ], [ 1669910577, "3.55292445694369" ], [ 1669910637, "3.5500031218063093" ], [ 1669910697, "3.548862411550747" ], [ 1669910757, "3.5515995203059676" ], [ 1669910817, "3.558850223047493" ], [ 1669910877, "3.574443126904589" ], [ 1669910937, "3.5827157923295565" ], [ 1669910997, "3.5882558247758505" ], [ 1669911057, "3.5659676947636516" ], [ 1669911117, "3.549925527039414" ], [ 1669911177, "3.5132399792546485" ], [ 1669911237, "3.4856043412476723" ], [ 1669911297, "3.473362778385221" ], [ 1669911357, "3.4855874030825524" ], [ 1669911417, "3.4908753940777943" ], [ 1669911477, "3.5143678696119496" ], [ 1669911537, "3.5259832984817945" ], [ 1669911597, "3.5266791517879152" ], [ 1669911657, "3.5045224279955707" ], [ 1669911717, "3.488176694056384" ], [ 1669911777, "3.478242878871432" ], [ 1669911837, "3.4814893254573147" ], [ 1669911897, "3.4886991577154207" ], [ 1669911957, "3.501087185740355" ], [ 1669912017, "3.516827493511323" ], [ 1669912077, "3.5303370256961" ], [ 1669912137, "3.5279739749720602" ], [ 1669912197, "3.5270078308614607" ], [ 1669912257, "3.544400773059036" ], [ 1669912317, "3.545749681079004" ], [ 1669912377, "3.5421358148218958" ], [ 1669912437, "3.555514662146085" ], [ 1669912497, "3.55067674607418" ], [ 1669912557, "3.5346509833221096" ], [ 1669912617, "3.527368011269863" ], [ 1669912677, "3.5206786006561424" ], [ 1669912737, "3.5106217594888993" ], [ 1669912797, "3.503453223967266" ], [ 1669912857, "3.476302165269871" ], [ 1669912917, "3.446722884600804" ], [ 1669912977, "3.418953294802013" ], [ 1669913037, "3.387760404637628" ], [ 1669913097, "3.379947480896406" ], [ 1669913157, "3.387459593904452" ], [ 1669913217, "3.39903599991749" ], [ 1669913277, "3.4306611086528016" ], [ 1669913337, "3.4600958957757126" ], [ 1669913397, "3.470445138322438" ], [ 1669913457, "3.4793745115983024" ], [ 1669913517, "3.4970861416978916" ], [ 1669913577, "3.497053207576224" ], [ 1669913637, "3.51650583297417" ], [ 1669913697, "3.53396087808834" ], [ 1669913757, "3.5396928689531726" ], [ 1669913817, "3.5312603368566715" ], [ 1669913877, "3.5188688780758577" ], [ 1669913937, "3.501353021604171" ], [ 1669913997, "3.495376373171146" ], [ 1669914057, "3.4965849221831062" ], [ 1669914117, "3.5186208742926355" ], [ 1669914177, "3.528896624522648" ], [ 1669914237, "3.5305010035395257" ], [ 1669914297, "3.541988398610542" ], [ 1669914357, "3.5609125245758704" ], [ 1669914417, "3.5474407717908045" ], [ 1669914477, "3.559456816812853" ], [ 1669914537, "3.5579543670895104" ], [ 1669914597, "3.5376596349072353" ], [ 1669914657, "3.500116744562896" ], [ 1669914717, "3.4860809494357485" ], [ 1669914777, "3.463464761556704" ], [ 1669914837, "3.454639669935065" ], [ 1669914897, "3.458815729227988" ], [ 1669914957, "3.4656334621762546" ], [ 1669915017, "3.457443276482377" ], [ 1669915077, "3.4587076441328355" ], [ 1669915137, "3.4576667274976183" ], [ 1669915197, "3.441212514052483" ], [ 1669915257, "3.436731546490236" ], [ 1669915317, "3.4230914702107196" ], [ 1669915377, "3.3965759601368495" ], [ 1669915437, "3.3642496565944806" ], [ 1669915497, "3.359125435849821" ], [ 1669915557, "3.3644496628154563" ], [ 1669915617, "3.378179049678546" ], [ 1669915677, "3.388255655733597" ], [ 1669915737, "3.4115499622875567" ], [ 1669915797, "3.430355099760151" ], [ 1669915857, "3.4444873881590565" ], [ 1669915917, "3.477221193454518" ], [ 1669915977, "3.4984598376687797" ], [ 1669916037, "3.5191438776881596" ], [ 1669916097, "3.528372291498743" ], [ 1669916157, "3.538429877183355" ], [ 1669916217, "3.529502474909279" ], [ 1669916277, "3.533228649291391" ], [ 1669916337, "3.520465558607937" ], [ 1669916397, "3.518126916460997" ], [ 1669916457, "3.5338494212159395" ], [ 1669916517, "3.5396103116952644" ], [ 1669916577, "3.5440113942742113" ], [ 1669916637, "3.555352681145494" ], [ 1669916697, "3.565287848824435" ], [ 1669916757, "3.5473156950246527" ], [ 1669916817, "3.5405064031022766" ], [ 1669916877, "3.5396648095633227" ], [ 1669916937, "3.543197826451157" ], [ 1669916997, "3.534150667059161" ], [ 1669917057, "3.538108686055852" ], [ 1669917117, "3.5496743140939406" ], [ 1669917177, "3.5743349742185133" ], [ 1669917237, "3.6023410115172756" ], [ 1669917297, "3.6266660615541784" ], [ 1669917357, "3.6363246616634393" ], [ 1669917417, "3.6443678358742364" ], [ 1669917477, "3.6147628121726716" ], [ 1669917537, "3.569776586206956" ], [ 1669917597, "3.5364614898720452" ], [ 1669917657, "3.5175863003948544" ], [ 1669917717, "3.4933880333968874" ], [ 1669917777, "3.496271554813311" ], [ 1669917837, "3.5133274753444184" ], [ 1669917897, "3.5165369969141884" ], [ 1669917957, "3.5342454417791447" ], [ 1669918017, "3.535009728345196" ], [ 1669918077, "3.530920206660901" ], [ 1669918137, "3.5221492769839613" ], [ 1669918197, "3.5385643769696147" ], [ 1669918257, "3.552935268891212" ], [ 1669918317, "3.574244971964093" ], [ 1669918377, "3.5829766551316578" ], [ 1669918437, "3.607315227733896" ], [ 1669918497, "3.6180514144277343" ], [ 1669918557, "3.618149529823877" ], [ 1669918617, "3.627996164908917" ], [ 1669918677, "3.667257364523488" ], [ 1669918737, "3.6774512396586285" ], [ 1669918797, "3.6807852132955032" ], [ 1669918857, "3.687663955982376" ], [ 1669918917, "3.6704591178958257" ], [ 1669918977, "3.6210034224533043" ], [ 1669919037, "3.59479783236295" ], [ 1669919097, "3.5675383087242327" ], [ 1669919157, "3.538774998975936" ], [ 1669919217, "3.5151713519791024" ], [ 1669919277, "3.5184785572391233" ], [ 1669919337, "3.520036053834576" ], [ 1669919397, "3.531227884349501" ], [ 1669919457, "3.5482494031966945" ], [ 1669919517, "3.579756277778579" ], [ 1669919577, "3.5768916882899466" ], [ 1669919637, "3.583136509740548" ], [ 1669919697, "3.5889034672701796" ], [ 1669919757, "3.578538684476218" ], [ 1669919817, "3.561095891756019" ], [ 1669919877, "3.5735401768117576" ], [ 1669919937, "3.572004481889575" ], [ 1669919997, "3.561982282173779" ], [ 1669920057, "3.5685821602542065" ], [ 1669920117, "3.5663040338405656" ], [ 1669920177, "3.558877857686553" ], [ 1669920237, "3.551263789426624" ], [ 1669920297, "3.5632804150680437" ], [ 1669920357, "3.5769560137017002" ], [ 1669920417, "3.5823413568484246" ], [ 1669920477, "3.6005664825861987" ], [ 1669920537, "3.620606041643853" ], [ 1669920597, "3.6059621484261495" ], [ 1669920657, "3.603055265934147" ], [ 1669920717, "3.631585515043967" ], [ 1669920777, "3.637873296940227" ], [ 1669920837, "3.6589637256230523" ], [ 1669920897, "3.7066774563251146" ], [ 1669920957, "3.7219579043753717" ], [ 1669921017, "3.715657104232029" ], [ 1669921077, "3.710640102455799" ], [ 1669921137, "3.681785125532633" ], [ 1669921197, "3.646289452315725" ], [ 1669921257, "3.6322826829750285" ], [ 1669921317, "3.6008353311695536" ], [ 1669921377, "3.5814822819278964" ], [ 1669921437, "3.583724633996799" ], [ 1669921497, "3.5589828430092836" ], [ 1669921557, "3.5547838484815615" ], [ 1669921617, "3.5640501961902253" ], [ 1669921677, "3.5491819066691366" ], [ 1669921737, "3.539184989790721" ], [ 1669921797, "3.537134678789529" ], [ 1669921857, "3.526807924565189" ], [ 1669921917, "3.5251071253717075" ], [ 1669921977, "3.523788410792565" ], [ 1669922037, "3.5131992186616605" ], [ 1669922097, "3.5083340677048582" ], [ 1669922157, "3.4928004618336574" ], [ 1669922217, "3.484261346511495" ], [ 1669922277, "3.4952749709255766" ], [ 1669922337, "3.508773580828339" ], [ 1669922397, "3.505947971953196" ], [ 1669922457, "3.504927598725849" ], [ 1669922517, "3.507264749717161" ], [ 1669922577, "3.519489686107395" ], [ 1669922637, "3.5258922693419548" ], [ 1669922697, "3.5387364996474373" ], [ 1669922757, "3.5533278436646754" ], [ 1669922817, "3.571445659026433" ], [ 1669922877, "3.562128654772307" ], [ 1669922937, "3.5500356099992874" ], [ 1669922997, "3.543907365672099" ], [ 1669923057, "3.5328061536453568" ], [ 1669923117, "3.5495498923885513" ], [ 1669923177, "3.5747083634848282" ], [ 1669923237, "3.574936545469564" ], [ 1669923297, "3.5901987840412413" ], [ 1669923357, "3.6186148897731307" ], [ 1669923417, "3.6101772841734965" ], [ 1669923477, "3.6019714699216907" ], [ 1669923537, "3.6202248576424694" ], [ 1669923597, "3.6088134257778175" ], [ 1669923657, "3.570083558903142" ], [ 1669923717, "3.5606193863304054" ], [ 1669923777, "3.51214739820817" ], [ 1669923837, "3.478120358563001" ], [ 1669923897, "3.455798796791444" ], [ 1669923957, "3.4611417956556756" ], [ 1669924017, "3.448869708344745" ], [ 1669924077, "3.489009584836429" ], [ 1669924137, "3.5196572330044247" ], [ 1669924197, "3.5503641388771205" ], [ 1669924257, "3.556808489842262" ], [ 1669924317, "3.557776031883399" ], [ 1669924377, "3.5250510243171007" ], [ 1669924437, "3.519226158167379" ], [ 1669924497, "3.5145249648447017" ], [ 1669924557, "3.540623284739201" ], [ 1669924617, "3.56670933315054" ], [ 1669924677, "3.6107876850529967" ], [ 1669924737, "3.6498101564533347" ], [ 1669924797, "3.6542426478892476" ], [ 1669924857, "3.649995948210285" ], [ 1669924917, "3.654919018355142" ], [ 1669924977, "3.6554606871485644" ], [ 1669925037, "3.62091304820286" ], [ 1669925097, "3.6140625850271264" ], [ 1669925157, "3.610292309171289" ], [ 1669925217, "3.6228701865721513" ], [ 1669925277, "3.6085803568784973" ], [ 1669925337, "3.593739456051905" ], [ 1669925397, "3.591023904528316" ], [ 1669925457, "3.5974877936513554" ], [ 1669925517, "3.569508531994678" ], [ 1669925577, "3.563473069719217" ], [ 1669925637, "3.6038463667520264" ], [ 1669925697, "3.616254927710887" ], [ 1669925757, "3.6189379157667263" ], [ 1669925817, "3.631096335147656" ], [ 1669925877, "3.62716645111133" ], [ 1669925937, "3.594010162099451" ], [ 1669925997, "3.604110813226095" ], [ 1669926057, "3.6141547505466702" ], [ 1669926117, "3.6362266282778437" ], [ 1669926177, "3.6822728936241815" ], [ 1669926237, "3.7057703126443724" ], [ 1669926297, "3.703381674403091" ], [ 1669926357, "3.700557238941303" ], [ 1669926417, "3.677220955684821" ], [ 1669926477, "3.6544457795644942" ], [ 1669926537, "3.672419312592765" ], [ 1669926597, "3.6882024454100426" ], [ 1669926657, "3.6868737555151387" ], [ 1669926717, "3.7225702159499092" ], [ 1669926777, "3.7412188942679374" ], [ 1669926837, "3.734514128348148" ], [ 1669926897, "3.725449991421308" ], [ 1669926957, "3.7496088084011725" ], [ 1669927017, "3.7233496864821003" ], [ 1669927077, "3.7121064929784424" ], [ 1669927137, "3.696583807891952" ], [ 1669927197, "3.6882947293636166" ], [ 1669927257, "3.645601353505743" ], [ 1669927317, "3.61459050752392" ], [ 1669927377, "3.5959930805003406" ], [ 1669927437, "3.5829797319364647" ], [ 1669927497, "3.551508519319678" ], [ 1669927557, "3.534600909488255" ], [ 1669927617, "3.530224656096632" ], [ 1669927677, "3.5166669666480677" ], [ 1669927737, "3.508674029324129" ], [ 1669927797, "3.5219296607211175" ], [ 1669927857, "3.5150245186257316" ], [ 1669927917, "3.5077485131042616" ], [ 1669927977, "3.4997947156654052" ], [ 1669928037, "3.4913497810616105" ], [ 1669928097, "3.465157521768514" ], [ 1669928157, "3.463101307508416" ], [ 1669928217, "3.4291013357526756" ], [ 1669928277, "3.379902877689271" ], [ 1669928337, "3.349726658601246" ], [ 1669928397, "3.3199340366867087" ], [ 1669928457, "3.299799921018315" ], [ 1669928517, "3.2997270702639003" ], [ 1669928577, "3.3061621102019254" ], [ 1669928637, "3.2765492741287177" ], [ 1669928697, "3.2760113582450288" ], [ 1669928757, "3.271484167510641" ], [ 1669928817, "3.283701111807578" ], [ 1669928877, "3.3009399630749963" ], [ 1669928937, "3.3529470518779587" ] ] } ] }, "error": false, "errorMessage": "", "loading": false }, "queryType": 0, "timePreferance": "GLOBAL_TIME", "title": "", "yAxisUnit": "none" }, { "description": "", "id": "45e3bc8c-3af0-44bf-a813-eff5c27e5909", "isStacked": false, "nullZeroValues": "zero", "opacity": "1", "panelTypes": "TIME_SERIES", "query": { "clickHouse": [ { "disabled": false, "legend": "", "name": "A", "rawQuery": "" } ], "metricsBuilder": { "formulas": [], "queryBuilder": [ { "aggregateOperator": 1, "disabled": false, "groupBy": [], "legend": "", "metricName": null, "name": "A", "reduceTo": 1, "tagFilters": { "items": [], "op": "AND" } } ] }, "promQL": [ { "disabled": false, "legend": "{{service_name}}", "name": "A", "query": "histogram_quantile(0.5, rate(time_total_for_user_task_bucket{service_name=\"prod-premium-ml-diffusion\"}[5m]))" }, { "disabled": false, "legend": "{{service_name}}", "name": "B", "query": "histogram_quantile(0.5, rate(time_total_for_user_task_bucket{service_name=\"prod-standard-ml-diffusion\"}[5m]))" } ], "queryType": 3 }, "queryData": { "data": { "queryData": [ { "legend": "{{service_name}}", "metric": { "service_name": "prod-premium-ml-diffusion", "telemetry_auto_version": "0.34b0", "telemetry_sdk_language": "python", "telemetry_sdk_name": "opentelemetry", "telemetry_sdk_version": "1.13.0" }, "queryName": "A", "values": [ [ 1669982163, "5.1744321761013525" ], [ 1669982223, "5.177215571906014" ], [ 1669982283, "5.1808170874917305" ], [ 1669982343, "5.160802546837008" ], [ 1669982403, "5.12816595812815" ], [ 1669982463, "5.110756875876856" ], [ 1669982523, "5.089172314246569" ], [ 1669982583, "5.091642569109142" ], [ 1669982643, "5.099411235587538" ], [ 1669982703, "5.141884267827632" ], [ 1669982763, "5.150553700954201" ], [ 1669982823, "5.163128795601948" ], [ 1669982883, "5.182778254233142" ], [ 1669982943, "5.2017437647481435" ], [ 1669983003, "5.215040073532836" ], [ 1669983063, "5.234744539980413" ], [ 1669983123, "5.226331319853203" ], [ 1669983183, "5.206662578107643" ], [ 1669983243, "5.2017580190203665" ], [ 1669983303, "5.151505608636229" ], [ 1669983363, "5.1072227307470905" ], [ 1669983423, "5.098838898766095" ], [ 1669983483, "5.06207573441168" ], [ 1669983543, "5.027935196500636" ], [ 1669983603, "5.007546515091495" ], [ 1669983663, "4.963913981981008" ], [ 1669983723, "4.905648640303147" ], [ 1669983783, "4.853310729657441" ], [ 1669983843, "4.818412830100841" ], [ 1669983903, "4.803530479648385" ], [ 1669983963, "4.822862066273994" ], [ 1669984023, "4.823133506060336" ], [ 1669984083, "4.855664438960242" ], [ 1669984143, "4.868769782397436" ], [ 1669984203, "4.893721426959658" ], [ 1669984263, "4.887502264478608" ], [ 1669984323, "4.906997433758091" ], [ 1669984383, "4.936523083232125" ], [ 1669984443, "4.960183151219175" ], [ 1669984503, "4.966806549257018" ], [ 1669984563, "5.021476180675396" ], [ 1669984623, "5.055265941018465" ], [ 1669984683, "5.052372140709296" ], [ 1669984743, "5.023423929943368" ], [ 1669984803, "4.976617766483356" ], [ 1669984863, "4.917229906496033" ], [ 1669984923, "4.863537067590859" ], [ 1669984983, "4.829952807739898" ], [ 1669985043, "4.796463186532903" ], [ 1669985103, "4.788269941725672" ], [ 1669985163, "4.768178010967651" ], [ 1669985223, "4.747695197175444" ], [ 1669985283, "4.765077966923897" ], [ 1669985343, "4.775572345119437" ], [ 1669985403, "4.79614984521676" ], [ 1669985463, "4.787843663007755" ], [ 1669985523, "4.769991189644444" ], [ 1669985583, "4.735531494555645" ], [ 1669985643, "4.726276699088818" ], [ 1669985703, "4.668245503525627" ], [ 1669985763, "4.6518015294543" ], [ 1669985823, "4.6541288107604375" ], [ 1669985883, "4.659283969576416" ], [ 1669985943, "4.636880866727023" ], [ 1669986003, "4.678795028351047" ], [ 1669986063, "4.699796690236646" ], [ 1669986123, "4.716847474696311" ], [ 1669986183, "4.716799504550596" ], [ 1669986243, "4.734716354858176" ], [ 1669986303, "4.7064791449299985" ], [ 1669986363, "4.691824957849667" ], [ 1669986423, "4.695451670787023" ], [ 1669986483, "4.6818773665384565" ], [ 1669986543, "4.658956425004142" ], [ 1669986603, "4.629666506577329" ], [ 1669986663, "4.607970877044516" ], [ 1669986723, "4.599239483760175" ], [ 1669986783, "4.604563459365289" ], [ 1669986843, "4.618024435114864" ], [ 1669986903, "4.6659579789894945" ], [ 1669986963, "4.715344864014649" ], [ 1669987023, "4.750961363729623" ], [ 1669987083, "4.790107053597036" ], [ 1669987143, "4.815712282530101" ], [ 1669987203, "4.8341855037322725" ], [ 1669987263, "4.821783926049557" ], [ 1669987323, "4.766970062506854" ], [ 1669987383, "4.709096463433695" ], [ 1669987443, "4.680691675115466" ], [ 1669987503, "4.644784975017845" ], [ 1669987563, "4.613670422378864" ], [ 1669987623, "4.58863649572011" ], [ 1669987683, "4.5778607867180785" ], [ 1669987743, "4.5865033676257605" ], [ 1669987803, "4.586920573440504" ], [ 1669987863, "4.5892793041557365" ], [ 1669987923, "4.690747880148901" ], [ 1669987983, "4.731536663947234" ], [ 1669988043, "4.78944382787948" ], [ 1669988103, "4.781815977595187" ], [ 1669988163, "4.810483115274613" ], [ 1669988223, "4.779914403301049" ], [ 1669988283, "4.780167911946655" ], [ 1669988343, "4.789078090729614" ], [ 1669988403, "4.848598625637479" ], [ 1669988463, "4.858062759512627" ], [ 1669988523, "4.883839662679518" ], [ 1669988583, "4.920066854253648" ], [ 1669988643, "4.911948556670716" ], [ 1669988703, "4.935704713028379" ], [ 1669988763, "4.9911305285860035" ], [ 1669988823, "4.983370913396277" ], [ 1669988883, "4.982702527822827" ], [ 1669988943, "4.9480703194303395" ], [ 1669989003, "4.915115768298789" ], [ 1669989063, "4.878791450789156" ], [ 1669989123, "4.859519680325338" ], [ 1669989183, "4.8336898873429766" ], [ 1669989243, "4.820260001301346" ], [ 1669989303, "4.762184280859306" ], [ 1669989363, "4.7079731358058625" ], [ 1669989423, "4.704987749212193" ], [ 1669989483, "4.670991252678794" ], [ 1669989543, "4.650357885336519" ], [ 1669989603, "4.641330606243936" ], [ 1669989663, "4.623482541461785" ], [ 1669989723, "4.608823415850322" ], [ 1669989783, "4.692616609658114" ], [ 1669989843, "4.816780244154724" ], [ 1669989903, "4.845189635249683" ], [ 1669989963, "4.900876892757975" ], [ 1669990023, "4.955492046002342" ], [ 1669990083, "4.955172067235617" ], [ 1669990143, "4.961048008523795" ], [ 1669990203, "5.072349398089462" ], [ 1669990263, "5.138089805342794" ], [ 1669990323, "5.142639807957844" ], [ 1669990383, "5.0996719497364085" ], [ 1669990443, "5.041983011030914" ], [ 1669990503, "4.966474403146542" ], [ 1669990563, "4.902254867216813" ], [ 1669990623, "4.880932136056849" ], [ 1669990683, "4.904651111320139" ], [ 1669990743, "4.903706510944706" ], [ 1669990803, "4.886498655063726" ], [ 1669990863, "4.84253444622898" ], [ 1669990923, "4.7630042074824965" ], [ 1669990983, "4.6589564267469195" ], [ 1669991043, "4.590917661680972" ], [ 1669991103, "4.610169197420127" ], [ 1669991163, "4.663618580722931" ], [ 1669991223, "4.750866828456487" ], [ 1669991283, "4.809322965022058" ], [ 1669991343, "4.845539966533659" ], [ 1669991403, "4.797617395516154" ], [ 1669991463, "4.757661143542798" ], [ 1669991523, "4.724432318959025" ], [ 1669991583, "4.796357967735775" ], [ 1669991643, "4.793502609874226" ], [ 1669991703, "4.848224321127078" ], [ 1669991763, "4.844438857094206" ], [ 1669991823, "4.809015408640512" ], [ 1669991883, "4.764847947867842" ], [ 1669991943, "4.797088552335538" ], [ 1669992003, "4.770372344241629" ], [ 1669992063, "4.751380830525223" ], [ 1669992123, "4.79192165688853" ], [ 1669992183, "4.7805190503316215" ], [ 1669992243, "4.755195218046468" ], [ 1669992303, "4.727729060248406" ], [ 1669992363, "4.746981019972774" ], [ 1669992423, "4.720435881158905" ], [ 1669992483, "4.7395964563953195" ], [ 1669992543, "4.759592867974492" ], [ 1669992603, "4.818992849606114" ], [ 1669992663, "4.871776059887607" ], [ 1669992723, "4.905195392726141" ], [ 1669992783, "4.908203937910624" ], [ 1669992843, "4.871905428063513" ], [ 1669992903, "4.82308448010612" ], [ 1669992963, "4.735090200407023" ], [ 1669993023, "4.673650552573877" ], [ 1669993083, "4.631651321586437" ], [ 1669993143, "4.645089685772828" ], [ 1669993203, "4.654395076494723" ], [ 1669993263, "4.662229463846532" ], [ 1669993323, "4.706807244269201" ], [ 1669993383, "4.663464563362133" ], [ 1669993443, "4.622817755946463" ], [ 1669993503, "4.5747392106290325" ], [ 1669993563, "4.590493638885194" ], [ 1669993623, "4.554137410421601" ], [ 1669993683, "4.544679422270046" ], [ 1669993743, "4.565344881419437" ], [ 1669993803, "4.566213340067097" ], [ 1669993863, "4.5345455844527" ], [ 1669993923, "4.505272057805071" ], [ 1669993983, "4.4820665703217175" ], [ 1669994043, "4.431211309903248" ], [ 1669994103, "4.406591073172804" ], [ 1669994163, "4.404810340510766" ], [ 1669994223, "4.407448178575813" ], [ 1669994283, "4.452553908857252" ], [ 1669994343, "4.5060999236605905" ], [ 1669994403, "4.533988988672254" ], [ 1669994463, "4.560510027163746" ], [ 1669994523, "4.543453971062826" ], [ 1669994583, "4.521248573184652" ], [ 1669994643, "4.487711267605633" ], [ 1669994703, "4.4781393958872515" ], [ 1669994763, "4.480995715837889" ], [ 1669994823, "4.496446078088301" ], [ 1669994883, "4.507348528200762" ], [ 1669994943, "4.52799868578457" ], [ 1669995003, "4.579575241692268" ], [ 1669995063, "4.60372453762891" ], [ 1669995123, "4.622126645012604" ], [ 1669995183, "4.671689098592983" ], [ 1669995243, "4.717488127687076" ], [ 1669995303, "4.694525951158189" ], [ 1669995363, "4.670130041711492" ], [ 1669995423, "4.653177646257231" ], [ 1669995483, "4.649461733465628" ], [ 1669995543, "4.6312541042663105" ], [ 1669995603, "4.65394518664312" ], [ 1669995663, "4.6502816097087" ], [ 1669995723, "4.687212435914289" ], [ 1669995783, "4.686761637150553" ], [ 1669995843, "4.70363747744168" ], [ 1669995903, "4.783331864038833" ], [ 1669995963, "4.873939903746708" ], [ 1669996023, "4.891669856197715" ], [ 1669996083, "4.952970627412417" ], [ 1669996143, "5.012699316316979" ], [ 1669996203, "4.9941530552307265" ], [ 1669996263, "4.981517162538944" ], [ 1669996323, "4.994632709743327" ], [ 1669996383, "4.969335564430834" ], [ 1669996443, "4.953482020937641" ], [ 1669996503, "4.971513565831289" ], [ 1669996563, "4.898818922909045" ], [ 1669996623, "4.8271785067418" ], [ 1669996683, "4.762164482848968" ], [ 1669996743, "4.685304094193342" ], [ 1669996803, "4.640006164887016" ], [ 1669996863, "4.686425447551095" ], [ 1669996923, "4.784976523253874" ], [ 1669996983, "4.828446455003576" ], [ 1669997043, "4.882297176498722" ], [ 1669997103, "4.856517797596191" ], [ 1669997163, "4.827110414294255" ], [ 1669997223, "4.761803526795289" ], [ 1669997283, "4.75706209306207" ], [ 1669997343, "4.726743863956418" ], [ 1669997403, "4.758231511550385" ], [ 1669997463, "4.74021671644926" ], [ 1669997523, "4.714551843787625" ], [ 1669997583, "4.741814543115381" ], [ 1669997643, "4.768240329185517" ], [ 1669997703, "4.752277054324248" ], [ 1669997763, "4.795683228980922" ], [ 1669997823, "4.806727292100935" ], [ 1669997883, "4.784642951951154" ], [ 1669997943, "4.785941736448361" ], [ 1669998003, "4.803758742719274" ], [ 1669998063, "4.778045802137967" ], [ 1669998123, "4.792006356298329" ], [ 1669998183, "4.800643472362865" ], [ 1669998243, "4.816278692984289" ], [ 1669998303, "4.832015188081152" ], [ 1669998363, "4.86426278638697" ], [ 1669998423, "4.928450849972767" ], [ 1669998483, "4.989210501565415" ], [ 1669998543, "5.031295370088489" ], [ 1669998603, "5.021654985896113" ], [ 1669998663, "5.053655376581408" ], [ 1669998723, "5.067062657529312" ], [ 1669998783, "5.0513635621201045" ], [ 1669998843, "5.0646000200724774" ], [ 1669998903, "5.1004556558543195" ], [ 1669998963, "5.0978614559514135" ], [ 1669999023, "5.080835873070558" ], [ 1669999083, "5.078888998328726" ], [ 1669999143, "5.060761336980938" ], [ 1669999203, "5.060491613321178" ], [ 1669999263, "5.061797678625662" ], [ 1669999323, "5.063506977352201" ], [ 1669999383, "5.045537057121606" ], [ 1669999443, "5.0289652681834305" ], [ 1669999503, "5.025024345326029" ], [ 1669999563, "5.017235354987838" ], [ 1669999623, "5.011578720117119" ], [ 1669999683, "5.0481626315486805" ], [ 1669999743, "5.077618813249641" ], [ 1669999803, "5.056583616700761" ], [ 1669999863, "5.046778619548556" ], [ 1669999923, "5.042276619369078" ], [ 1669999983, "5.012165450121655" ], [ 1670000043, "4.993704056905692" ], [ 1670000103, "4.988671830135564" ], [ 1670000163, "4.975777452367992" ], [ 1670000223, "4.964809774299665" ], [ 1670000283, "4.941958256441585" ], [ 1670000343, "4.910392041004558" ], [ 1670000403, "4.899826519600517" ], [ 1670000463, "4.9292274662260125" ], [ 1670000523, "4.948439373213684" ], [ 1670000583, "4.981811083561834" ], [ 1670000643, "5.020677493766816" ], [ 1670000703, "5.050219538968167" ], [ 1670000763, "5.046323056581143" ], [ 1670000823, "5.048203443630392" ], [ 1670000883, "5.059303366529351" ], [ 1670000943, "5.067316502856828" ], [ 1670001003, "5.075527860629492" ], [ 1670001063, "5.079452128909434" ], [ 1670001123, "5.054494252283128" ], [ 1670001183, "5.041027161504118" ], [ 1670001243, "5.0640414649662615" ], [ 1670001303, "5.0852712922644825" ], [ 1670001363, "5.0809141234982205" ], [ 1670001423, "5.100903817402976" ], [ 1670001483, "5.100013353914098" ], [ 1670001543, "5.112152364780966" ], [ 1670001603, "5.11299450248454" ], [ 1670001663, "5.1338370990792805" ], [ 1670001723, "5.1536299491262785" ], [ 1670001783, "5.168436617532426" ], [ 1670001843, "5.14832710196633" ], [ 1670001903, "5.148455683965292" ], [ 1670001963, "5.156264029722418" ], [ 1670002023, "5.117500350318382" ], [ 1670002083, "5.085254626865718" ], [ 1670002143, "5.045889443274353" ], [ 1670002203, "4.989480073540687" ], [ 1670002263, "4.930414020492345" ], [ 1670002323, "4.920701529729356" ], [ 1670002383, "4.9110087385044405" ], [ 1670002443, "4.9162528087234065" ], [ 1670002503, "4.91719821920145" ], [ 1670002563, "4.936963950683461" ], [ 1670002623, "4.951459237819585" ], [ 1670002683, "4.962911100813303" ], [ 1670002743, "4.961600381811521" ], [ 1670002803, "4.976462902975978" ], [ 1670002863, "4.975496298788695" ], [ 1670002923, "4.973745731591074" ], [ 1670002983, "5.007514030698493" ], [ 1670003043, "5.033441052376985" ], [ 1670003103, "5.032850301908711" ], [ 1670003163, "5.028246139835807" ], [ 1670003223, "5.028626445039626" ], [ 1670003283, "4.988905133708693" ], [ 1670003343, "4.9736464113197" ], [ 1670003403, "5.005064557857997" ], [ 1670003463, "5.016341636116934" ], [ 1670003523, "5.053199092198978" ], [ 1670003583, "5.118965595632391" ], [ 1670003643, "5.187712509835912" ], [ 1670003703, "5.224685609692957" ], [ 1670003763, "5.269504523034074" ] ] }, { "legend": "{{service_name}}", "metric": { "service_name": "prod-standard-ml-diffusion", "telemetry_auto_version": "0.34b0", "telemetry_sdk_language": "python", "telemetry_sdk_name": "opentelemetry", "telemetry_sdk_version": "1.13.0" }, "queryName": "B", "values": [ [ 1669982163, "3.3691666301685625" ], [ 1669982223, "3.386107044444666" ], [ 1669982283, "3.410665790968622" ], [ 1669982343, "3.430191619360781" ], [ 1669982403, "3.4204918903727144" ], [ 1669982463, "3.4447540092335975" ], [ 1669982523, "3.436963145385831" ], [ 1669982583, "3.443729342230812" ], [ 1669982643, "3.423251215008051" ], [ 1669982703, "3.42652767553757" ], [ 1669982763, "3.4164709505351403" ], [ 1669982823, "3.424294312171515" ], [ 1669982883, "3.4061868761071636" ], [ 1669982943, "3.410721252450006" ], [ 1669983003, "3.3982555434050052" ], [ 1669983063, "3.3826349019794186" ], [ 1669983123, "3.367416110804074" ], [ 1669983183, "3.3630017939248606" ], [ 1669983243, "3.356744049084177" ], [ 1669983303, "3.369570215511983" ], [ 1669983363, "3.3867886426778595" ], [ 1669983423, "3.407189967453117" ], [ 1669983483, "3.4110263272843744" ], [ 1669983543, "3.427690383877791" ], [ 1669983603, "3.438664944424705" ], [ 1669983663, "3.4420622186552245" ], [ 1669983723, "3.4357603354518584" ], [ 1669983783, "3.422496524609984" ], [ 1669983843, "3.406716168439461" ], [ 1669983903, "3.3852108056138794" ], [ 1669983963, "3.3434768465316065" ], [ 1669984023, "3.3206093301714965" ], [ 1669984083, "3.327327425014456" ], [ 1669984143, "3.332233142248942" ], [ 1669984203, "3.3320582266173178" ], [ 1669984263, "3.3646361727641545" ], [ 1669984323, "3.3547899153511618" ], [ 1669984383, "3.323622615587562" ], [ 1669984443, "3.3159549356029983" ], [ 1669984503, "3.3144791852388216" ], [ 1669984563, "3.307805113504471" ], [ 1669984623, "3.34115247650584" ], [ 1669984683, "3.367200863328365" ], [ 1669984743, "3.359581375744428" ], [ 1669984803, "3.3534080232072103" ], [ 1669984863, "3.3468308364058394" ], [ 1669984923, "3.3383419798388796" ], [ 1669984983, "3.3395586395697237" ], [ 1669985043, "3.362870928488051" ], [ 1669985103, "3.3834394351627486" ], [ 1669985163, "3.413071796345228" ], [ 1669985223, "3.4427849115910805" ], [ 1669985283, "3.4617920865551315" ], [ 1669985343, "3.489911890184161" ], [ 1669985403, "3.5429896610201657" ], [ 1669985463, "3.5701994826557923" ], [ 1669985523, "3.599880778796005" ], [ 1669985583, "3.6486486486486487" ], [ 1669985643, "3.643609290132537" ], [ 1669985703, "3.618053706744522" ], [ 1669985763, "3.6118365990999264" ], [ 1669985823, "3.6006712228230775" ], [ 1669985883, "3.590819921108697" ], [ 1669985943, "3.583025272811289" ], [ 1669986003, "3.5764429257730566" ], [ 1669986063, "3.5301364214975943" ], [ 1669986123, "3.502999451845556" ], [ 1669986183, "3.4651771378965956" ], [ 1669986243, "3.4867294984292063" ], [ 1669986303, "3.46923824086377" ], [ 1669986363, "3.511820420632401" ], [ 1669986423, "3.4885231024393293" ], [ 1669986483, "3.4935367472132253" ], [ 1669986543, "3.47880732423127" ], [ 1669986603, "3.4855850404072024" ], [ 1669986663, "3.4744181753272585" ], [ 1669986723, "3.5015164198931337" ], [ 1669986783, "3.499649933655334" ], [ 1669986843, "3.5014199625485833" ], [ 1669986903, "3.4911098525236586" ], [ 1669986963, "3.4690380211136116" ], [ 1669987023, "3.441203547978458" ], [ 1669987083, "3.432320484785019" ], [ 1669987143, "3.3849243981101114" ], [ 1669987203, "3.3409421346464105" ], [ 1669987263, "3.2907083315148604" ], [ 1669987323, "3.2410004337140386" ], [ 1669987383, "3.192718686598272" ], [ 1669987443, "3.1793669769047943" ], [ 1669987503, "3.2143735847848873" ], [ 1669987563, "3.228566216347517" ], [ 1669987623, "3.261541354871917" ], [ 1669987683, "3.2689523136923793" ], [ 1669987743, "3.2895134630032437" ], [ 1669987803, "3.27866552471906" ], [ 1669987863, "3.3000447240236057" ], [ 1669987923, "3.314099285713714" ], [ 1669987983, "3.331155407098728" ], [ 1669988043, "3.3340278293538232" ], [ 1669988103, "3.3466310379022435" ], [ 1669988163, "3.36464544493924" ], [ 1669988223, "3.3718503210846866" ], [ 1669988283, "3.3912831981164615" ], [ 1669988343, "3.400341205443651" ], [ 1669988403, "3.424065212652815" ], [ 1669988463, "3.425163002825689" ], [ 1669988523, "3.4120514834313487" ], [ 1669988583, "3.386021630771236" ], [ 1669988643, "3.401525025905809" ], [ 1669988703, "3.3994772484612357" ], [ 1669988763, "3.4062291947759467" ], [ 1669988823, "3.452307077892133" ], [ 1669988883, "3.5145216049152" ], [ 1669988943, "3.5541778965060122" ], [ 1669989003, "3.5825880483333576" ], [ 1669989063, "3.5924844316800737" ], [ 1669989123, "3.5846727913052523" ], [ 1669989183, "3.576799622493473" ], [ 1669989243, "3.5510200144322415" ], [ 1669989303, "3.5382676603916234" ], [ 1669989363, "3.578598544612568" ], [ 1669989423, "3.606276281283309" ], [ 1669989483, "3.6302707440552675" ], [ 1669989543, "3.635889612043571" ], [ 1669989603, "3.6138059312056" ], [ 1669989663, "3.576123444791284" ], [ 1669989723, "3.5611320335875494" ], [ 1669989783, "3.5472292818102487" ], [ 1669989843, "3.5510419029094864" ], [ 1669989903, "3.579170046403219" ], [ 1669989963, "3.6005639771232905" ], [ 1669990023, "3.6004541605727036" ], [ 1669990083, "3.5896079496755235" ], [ 1669990143, "3.6230796143081223" ], [ 1669990203, "3.6524566822956213" ], [ 1669990263, "3.638569097932987" ], [ 1669990323, "3.626025314801433" ], [ 1669990383, "3.663263500016532" ], [ 1669990443, "3.662621505005512" ], [ 1669990503, "3.6599490252966307" ], [ 1669990563, "3.6747229705468385" ], [ 1669990623, "3.6815839333080707" ], [ 1669990683, "3.6421180117306413" ], [ 1669990743, "3.620851857478325" ], [ 1669990803, "3.611562939432135" ], [ 1669990863, "3.6223869136856237" ], [ 1669990923, "3.582499704171635" ], [ 1669990983, "3.5409203569400614" ], [ 1669991043, "3.5083431845048647" ], [ 1669991103, "3.525824268528951" ], [ 1669991163, "3.5240877258379206" ], [ 1669991223, "3.548715816381919" ], [ 1669991283, "3.6121314812824417" ], [ 1669991343, "3.7332215485555285" ], [ 1669991403, "3.7669408254664343" ], [ 1669991463, "3.8140688427715963" ], [ 1669991523, "3.900408748957613" ], [ 1669991583, "3.930914901850869" ], [ 1669991643, "3.860434859940613" ], [ 1669991703, "3.7947860005036227" ], [ 1669991763, "3.7319318492943774" ], [ 1669991823, "3.7168462551534724" ], [ 1669991883, "3.702365900170964" ], [ 1669991943, "3.7009004468745887" ], [ 1669992003, "3.7303025458307704" ], [ 1669992063, "3.7829778948788544" ], [ 1669992123, "3.7507539969479744" ], [ 1669992183, "3.736120634142654" ], [ 1669992243, "3.7184771140888397" ], [ 1669992303, "3.7074456763099044" ], [ 1669992363, "3.732595926982108" ], [ 1669992423, "3.7945758588919434" ], [ 1669992483, "3.822571638036878" ], [ 1669992543, "3.8418812559918356" ], [ 1669992603, "3.8583288259074022" ], [ 1669992663, "3.801990414562608" ], [ 1669992723, "3.750826899558708" ], [ 1669992783, "3.7487315399070864" ], [ 1669992843, "3.7466977007260227" ], [ 1669992903, "3.7237634084246443" ], [ 1669992963, "3.726958206280184" ], [ 1669993023, "3.7102163786013067" ], [ 1669993083, "3.687874765705782" ], [ 1669993143, "3.6675085585190077" ], [ 1669993203, "3.6775753480474394" ], [ 1669993263, "3.6871993270130177" ], [ 1669993323, "3.6765437221717483" ], [ 1669993383, "3.705227670851838" ], [ 1669993443, "3.6958802483962057" ], [ 1669993503, "3.7245393116654126" ], [ 1669993563, "3.7165979867059153" ], [ 1669993623, "3.725553063084785" ], [ 1669993683, "3.6844269889951824" ], [ 1669993743, "3.6883153600356677" ], [ 1669993803, "3.6448218622218445" ], [ 1669993863, "3.610300002967717" ], [ 1669993923, "3.6140434802554577" ], [ 1669993983, "3.6262777770381254" ], [ 1669994043, "3.6115679943220123" ], [ 1669994103, "3.6018962740735345" ], [ 1669994163, "3.611575314945405" ], [ 1669994223, "3.59735621809955" ], [ 1669994283, "3.598136962776178" ], [ 1669994343, "3.6049142183298577" ], [ 1669994403, "3.6186834998017425" ], [ 1669994463, "3.6386999621161364" ], [ 1669994523, "3.6592931236548543" ], [ 1669994583, "3.6708800405823316" ], [ 1669994643, "3.6823922798194753" ], [ 1669994703, "3.6812154381938416" ], [ 1669994763, "3.6746092252313387" ], [ 1669994823, "3.673086852943783" ], [ 1669994883, "3.6732347788352326" ], [ 1669994943, "3.6768373156144745" ], [ 1669995003, "3.6670502831675638" ], [ 1669995063, "3.672574066975912" ], [ 1669995123, "3.6472993583986373" ], [ 1669995183, "3.65996889846346" ], [ 1669995243, "3.685990117884155" ], [ 1669995303, "3.7330638691798246" ], [ 1669995363, "3.7571266599792614" ], [ 1669995423, "3.8318411635855503" ], [ 1669995483, "3.8578540450263237" ], [ 1669995543, "3.8826784626015156" ], [ 1669995603, "3.9338157087083347" ], [ 1669995663, "3.991250988944696" ], [ 1669995723, "4.016176448478205" ], [ 1669995783, "4.042748213158268" ], [ 1669995843, "4.040297749254535" ], [ 1669995903, "4.011908356307677" ], [ 1669995963, "3.992380007934695" ], [ 1669996023, "3.990092739030477" ], [ 1669996083, "3.968412490676705" ], [ 1669996143, "3.967624027522966" ], [ 1669996203, "3.9702742345831545" ], [ 1669996263, "3.9586436738026127" ], [ 1669996323, "3.944536154372578" ], [ 1669996383, "3.9421405441481028" ], [ 1669996443, "3.9591877132080593" ], [ 1669996503, "3.962052033337042" ], [ 1669996563, "3.974460741927345" ], [ 1669996623, "4.015281355754128" ], [ 1669996683, "4.068453553309951" ], [ 1669996743, "4.07631668302021" ], [ 1669996803, "4.114045494472534" ], [ 1669996863, "4.186590652571977" ], [ 1669996923, "4.19799655216859" ], [ 1669996983, "4.2267489954302455" ], [ 1669997043, "4.280843854374968" ], [ 1669997103, "4.3245720195694615" ], [ 1669997163, "4.3437997875730225" ], [ 1669997223, "4.393449675979456" ], [ 1669997283, "4.398818195887568" ], [ 1669997343, "4.395085228473602" ], [ 1669997403, "4.379291701165698" ], [ 1669997463, "4.344650261367761" ], [ 1669997523, "4.322520936696831" ], [ 1669997583, "4.320713018881769" ], [ 1669997643, "4.324230467255853" ], [ 1669997703, "4.32359291095616" ], [ 1669997763, "4.369455118110237" ], [ 1669997823, "4.404996679357772" ], [ 1669997883, "4.434147355399972" ], [ 1669997943, "4.43949011472799" ], [ 1669998003, "4.4523174920980155" ], [ 1669998063, "4.452047393244428" ], [ 1669998123, "4.4333432315245656" ], [ 1669998183, "4.44690805634308" ], [ 1669998243, "4.481639367678096" ], [ 1669998303, "4.478663606370628" ], [ 1669998363, "4.465122134285959" ], [ 1669998423, "4.493332803448614" ], [ 1669998483, "4.497171833354121" ], [ 1669998543, "4.50708000591541" ], [ 1669998603, "4.5356068802269425" ], [ 1669998663, "4.57408870144952" ], [ 1669998723, "4.5897974676359725" ], [ 1669998783, "4.613551498993026" ], [ 1669998843, "4.634651408006512" ], [ 1669998903, "4.668053840804916" ], [ 1669998963, "4.702510365942421" ], [ 1669999023, "4.728662375410396" ], [ 1669999083, "4.743652788186703" ], [ 1669999143, "4.784252969828394" ], [ 1669999203, "4.830862821364197" ], [ 1669999263, "4.85114742377143" ], [ 1669999323, "4.888547495337063" ], [ 1669999383, "4.911602624581868" ], [ 1669999443, "4.901850253339806" ], [ 1669999503, "4.901570905649079" ], [ 1669999563, "4.912505855715177" ], [ 1669999623, "4.916608946082899" ], [ 1669999683, "4.946790507635967" ], [ 1669999743, "4.9769170474268565" ], [ 1669999803, "4.990040252711095" ], [ 1669999863, "4.995191610689605" ], [ 1669999923, "4.998477521915192" ], [ 1669999983, "4.985430067925035" ], [ 1670000043, "4.971516958466699" ], [ 1670000103, "4.952617059527613" ], [ 1670000163, "4.935348254511287" ], [ 1670000223, "4.915906138767523" ], [ 1670000283, "4.895321564047372" ], [ 1670000343, "4.896380576437465" ], [ 1670000403, "4.9687851857809235" ], [ 1670000463, "5.023806144436844" ], [ 1670000523, "5.068008041960791" ], [ 1670000583, "5.13794091805246" ], [ 1670000643, "5.190524341357206" ], [ 1670000703, "5.167314423515913" ], [ 1670000763, "5.170282015804771" ], [ 1670000823, "5.194974093324858" ], [ 1670000883, "5.1971035777449766" ], [ 1670000943, "5.194124193575221" ], [ 1670001003, "5.174964378794577" ], [ 1670001063, "5.143596337704056" ], [ 1670001123, "5.114830339181452" ], [ 1670001183, "5.102876342749729" ], [ 1670001243, "5.101200073192037" ], [ 1670001303, "5.109417980771849" ], [ 1670001363, "5.105539533480864" ], [ 1670001423, "5.077760694735419" ], [ 1670001483, "5.05969089390142" ], [ 1670001543, "5.049583376367228" ], [ 1670001603, "5.045971202757111" ], [ 1670001663, "5.060576118066236" ], [ 1670001723, "5.069975875896666" ], [ 1670001783, "5.061834851333231" ], [ 1670001843, "5.004393663242525" ], [ 1670001903, "4.9627898443271325" ], [ 1670001963, "4.932439188712533" ], [ 1670002023, "4.887085506779474" ], [ 1670002083, "4.848959225972073" ], [ 1670002143, "4.852579477612883" ], [ 1670002203, "4.8302954346551195" ], [ 1670002263, "4.7872924081146415" ], [ 1670002323, "4.773894382879501" ], [ 1670002383, "4.754294366985219" ], [ 1670002443, "4.7381673117753715" ], [ 1670002503, "4.737867927775038" ], [ 1670002563, "4.744347665504514" ], [ 1670002623, "4.749632819037917" ], [ 1670002683, "4.760905790543408" ], [ 1670002743, "4.753123087182954" ], [ 1670002803, "4.743671866379029" ], [ 1670002863, "4.747164358667236" ], [ 1670002923, "4.747650876661539" ], [ 1670002983, "4.73965287217314" ], [ 1670003043, "4.729034823519703" ], [ 1670003103, "4.741611879529407" ], [ 1670003163, "4.730998568399837" ], [ 1670003223, "4.719296890422363" ], [ 1670003283, "4.726590794513562" ], [ 1670003343, "4.750308360281446" ], [ 1670003403, "4.7521328224035555" ], [ 1670003463, "4.769241794467536" ], [ 1670003523, "4.776010211070295" ], [ 1670003583, "4.799252861608428" ], [ 1670003643, "4.827567852913899" ], [ 1670003703, "4.871425364758698" ], [ 1670003763, "4.92079103477376" ] ] } ] }, "error": false, "errorMessage": "", "loading": false }, "timePreferance": "GLOBAL_TIME", "title": "PROD Diffusion time total for user task", "yAxisUnit": "s" }, { "description": "", "id": "952e7f93-76fb-47d1-91d6-5494e0eed9f3", "isStacked": false, "nullZeroValues": "zero", "opacity": "1", "panelTypes": "TIME_SERIES", "query": { "clickHouse": [ { "disabled": false, "legend": "", "name": "A", "rawQuery": "" } ], "metricsBuilder": { "formulas": [], "queryBuilder": [ { "aggregateOperator": 1, "disabled": false, "groupBy": [], "legend": "", "metricName": null, "name": "A", "reduceTo": 1, "tagFilters": { "items": [], "op": "AND" } } ] }, "promQL": [ { "disabled": false, "legend": "{{service_name}}", "name": "A", "query": "histogram_quantile(0.5, rate(time_gpu_process_bucket{service_name=\"prod-premium-ml-diffusion\"}[5m]))" }, { "disabled": false, "legend": "{{service_name}}", "name": "B", "query": "histogram_quantile(0.5, rate(time_gpu_process_bucket{service_name=\"prod-standard-ml-diffusion\"}[5m]))" }, { "disabled": false, "legend": "{{service_name}}", "name": "C", "query": "histogram_quantile(0.5, rate(time_gpu_process_bucket{service_name=\"coreweave-prod-premium-ml-diffusion\"}[5m]))" }, { "disabled": false, "legend": "{{service_name}}", "name": "D", "query": "histogram_quantile(0.5, rate(time_gpu_process_bucket{service_name=\"coreweave-prod-standard-ml-diffusion\"}[5m]))" } ], "queryType": 3 }, "queryData": { "data": { "queryData": [ { "legend": "{{service_name}}", "metric": { "service_name": "coreweave-prod-standard-ml-diffusion", "telemetry_auto_version": "0.35b0", "telemetry_sdk_language": "python", "telemetry_sdk_name": "opentelemetry", "telemetry_sdk_version": "1.14.0" }, "queryName": "D", "values": [ [ 1671133998, "2.6081621047275556" ], [ 1671134058, "2.630040737003401" ], [ 1671134118, "2.6160152568008948" ], [ 1671134178, "2.6161004174397027" ], [ 1671134238, "2.628944689514761" ], [ 1671134298, "2.5974631434502653" ], [ 1671134358, "2.587689915556327" ], [ 1671134418, "2.5756014802513554" ], [ 1671134478, "2.553777680163303" ], [ 1671134538, "2.515958400573785" ], [ 1671134598, "2.517491413306195" ], [ 1671134658, "2.4154448779260003" ], [ 1671134718, "2.4164150297294777" ], [ 1671134778, "2.5181745371653887" ], [ 1671134838, "2.729142838988436" ], [ 1671134898, "2.9562756357670223" ], [ 1671134958, "3.0566640711180333" ], [ 1671135018, "2.9459273042250804" ], [ 1671135078, "2.8364698041953607" ], [ 1671135138, "2.6193561998914943" ], [ 1671135198, "2.518001090105855" ], [ 1671135258, "2.6190239468227143" ], [ 1671135318, "2.719069989872848" ], [ 1671135378, "2.8179146470555465" ], [ 1671135438, "2.815190397350993" ], [ 1671135498, "2.7151779369157603" ], [ 1671135558, "2.616165718927701" ], [ 1671135618, "2.5182393648409422" ], [ 1671135678, "2.51840384819815" ], [ 1671135738, "2.6231790623028175" ], [ 1671135798, "2.6264889393079978" ] ] }, { "legend": "{{service_name}}", "metric": { "service_name": "coreweave-prod-premium-ml-diffusion", "telemetry_auto_version": "0.35b0", "telemetry_sdk_language": "python", "telemetry_sdk_name": "opentelemetry", "telemetry_sdk_version": "1.14.0" }, "queryName": "C", "values": [ [ 1671133998, "2.5845968569849163" ], [ 1671134058, "2.6298779751117554" ], [ 1671134118, "2.4982849763932045" ], [ 1671134178, "2.5577540976939033" ], [ 1671134238, "2.6536668079694787" ], [ 1671134298, "2.614609571788413" ], [ 1671134358, "2.573896218937946" ], [ 1671134418, "2.706085455330168" ], [ 1671134478, "2.6591856461064456" ], [ 1671134538, "2.5959083695577174" ], [ 1671134598, "2.510312196700097" ], [ 1671134658, "2.510406417921744" ], [ 1671134718, "2.5104773405313257" ], [ 1671134778, "2.51067371010927" ], [ 1671134838, "2.5108856088560887" ], [ 1671134898, "2.6298921649950993" ], [ 1671134958, "2.627022058823529" ], [ 1671135018, "2.7537077467439586" ], [ 1671135078, "2.727379592933432" ], [ 1671135138, "2.705761316872428" ], [ 1671135198, "2.6008851503656056" ], [ 1671135258, "2.5990179149579107" ], [ 1671135318, "2.5101142008501047" ], [ 1671135378, "2.510028653295129" ], [ 1671135438, "2.5100843904251366" ], [ 1671135498, "2.510151597184624" ], [ 1671135558, "2.5105103506998026" ], [ 1671135618, "2.510990161189031" ], [ 1671135678, "2.511321531494443" ], [ 1671135738, "2.5115114102110003" ], [ 1671135798, "2.5113189710930897" ] ] }, { "legend": "{{service_name}}", "metric": { "service_name": "prod-premium-ml-diffusion", "telemetry_auto_version": "0.35b0", "telemetry_sdk_language": "python", "telemetry_sdk_name": "opentelemetry", "telemetry_sdk_version": "1.14.0" }, "queryName": "A", "values": [ [ 1671133998, "2.72203947368421" ], [ 1671134058, "2.719495310234179" ], [ 1671134118, "2.718009514661159" ], [ 1671134178, "2.71513674210928" ], [ 1671134238, "2.711209727356313" ], [ 1671134298, "2.711956866950384" ], [ 1671134358, "2.7127232217581" ], [ 1671134418, "2.7131510080719474" ], [ 1671134478, "2.710786925773278" ], [ 1671134538, "2.708849586042914" ], [ 1671134598, "2.704309063893016" ], [ 1671134658, "2.699996156104106" ], [ 1671134718, "2.6995087193602902" ], [ 1671134778, "2.704675273391089" ], [ 1671134838, "2.7102785988563216" ], [ 1671134898, "2.714643645868037" ], [ 1671134958, "2.7268470503145514" ], [ 1671135018, "2.728211212178981" ], [ 1671135078, "2.7198520068000565" ], [ 1671135138, "2.714909369415369" ], [ 1671135198, "2.7142111831740894" ], [ 1671135258, "2.704638448754219" ], [ 1671135318, "2.703375371423144" ], [ 1671135378, "2.705817948554329" ], [ 1671135438, "2.704292925086209" ], [ 1671135498, "2.703124322918924" ], [ 1671135558, "2.692891103684252" ], [ 1671135618, "2.680540060600859" ], [ 1671135678, "2.6779848107289923" ], [ 1671135738, "2.6815020563906544" ], [ 1671135798, "2.69009142995666" ] ] }, { "legend": "{{service_name}}", "metric": { "service_name": "prod-standard-ml-diffusion", "telemetry_auto_version": "0.35b0", "telemetry_sdk_language": "python", "telemetry_sdk_name": "opentelemetry", "telemetry_sdk_version": "1.14.0" }, "queryName": "B", "values": [ [ 1671133998, "2.5681830388313016" ], [ 1671134058, "2.576962017306567" ], [ 1671134118, "2.5821283185382375" ], [ 1671134178, "2.5902334379738616" ], [ 1671134238, "2.5863321704596913" ], [ 1671134298, "2.5680556984096663" ], [ 1671134358, "2.556514130340459" ], [ 1671134418, "2.5631149062558007" ], [ 1671134478, "2.5695895035249356" ], [ 1671134538, "2.582841320892055" ], [ 1671134598, "2.595806897671112" ], [ 1671134658, "2.609830365530417" ], [ 1671134718, "2.609654737573874" ], [ 1671134778, "2.606475089166056" ], [ 1671134838, "2.602092440854617" ], [ 1671134898, "2.5980444058739542" ], [ 1671134958, "2.593698892130814" ], [ 1671135018, "2.5897363243659384" ], [ 1671135078, "2.5892590100194406" ], [ 1671135138, "2.5897453643318413" ], [ 1671135198, "2.590493432354574" ], [ 1671135258, "2.6015760069869627" ], [ 1671135318, "2.6129911351861956" ], [ 1671135378, "2.6209159651772818" ], [ 1671135438, "2.6221345206775255" ], [ 1671135498, "2.6334151185323016" ], [ 1671135558, "2.6404694975725245" ], [ 1671135618, "2.6394160549989305" ], [ 1671135678, "2.638166379489612" ], [ 1671135738, "2.6524073183024814" ], [ 1671135798, "2.648559553962148" ] ] } ] }, "error": false, "errorMessage": "", "loading": false }, "timePreferance": "GLOBAL_TIME", "title": "Average GPU Processing Time ", "yAxisUnit": "s" } ] } } } ```