search

observeinc / entity-explorer

Observe Entity Explorer provides advanced relationship analysis of Observe entities
Apache License 2.0
1 stars 0 forks source link

Descendant datasets should be linked to from the output stage of dataset #1

Closed danielodievich closed 7 months ago

danielodievich commented 11 months ago

Per @georgios-observeinc In the descendants graph, it would be better if the links to the descendants started from the published (== last) stage of the dataset, rather from that small box that has the same name as the big box

danielodievich commented 11 months ago

image

danielodievich commented 11 months ago

We want something like that image

digraph observe_entity_explorer {rankdir=LR node [shape="rect"] label="ObsDataset: UJET/Environment/41254206/Dataset, ResourceDataset"

// Nodes
  subgraph cluster_dataset_DS_41254206 {
    label="🎈🎫 Dataset UJET/Environment" style="filled" fillcolor="lavender"
//     "DS_41254206" [label="🗃UJET/
// Environment" shape="rectangle" color="blue"]
    "ST_41254206_stage-g1qqnvyd" [label="📑🏁All Together [table]" shape="parallelogram" color="black" tooltip="union @"Container Logs_-08tc"
union @"K8S Namespace_-vm3d"

make_resource options(expiry:duration_hr(4)), 
   primary_key(environment)

set_label environment"]
    "ST_41254206_stage-08tclebf" [label="📑Container Logs [table]" shape="parallelogram" color="black" tooltip="// Select all UJET containers. 
// crm-adaptor
// crm-adaptor
// crm-server
// event-sourcer
// log-proxy
// ujet-clearview-exporter
// ujet-client
// ujet-data-exporter
// ujet-java-in-queue-key-capturer
// ujet-node-chatbot-server
// ujet-node-email-service
// ujet-node-media-service
// ujet-node-media-service-download
// ujet-node-rmq-event-generator
// ujet-node-rmq-event-sender
// ujet-node-theme-service
// ujet-node-virtual-agent-streaming-service
// ujet-prefect-looker-dump
// ujet-prefect-replication-monitor
// ujet-prefect-snowflake-backfill
// ujet-python-ae-carrier-data-importer
// ujet-python-ae-google-oem-carrier-importer
// ujet-python-ae-snowflake-backfiller
// ujet-python-alvaria-exporter
// ujet-python-data-exporter
// ujet-python-nice-exporter
// ujet-python-oem-billing-service
// ujet-server
// ujet-sidekiq
// ujet-verint-exporter
filter (match_regex(containerName, /^ujet.*$/, "si") or match_regex(containerName, /^crm.*$/, "si")) or containerName = "event-sourcer" or containerName =...1250"]
    "ST_41254206_stage-vm3d7rdq" [label="📑K8S Namespace [table]" shape="parallelogram" color="black" tooltip="extract_regex name, /^(?P<environment>[a-z]{2,3}\d{2,3})/, "i"
filter is_null(environment) = false

timechart 1h, group_by(environment)"]
    "ST_41254206_stage-zfsdfkoz" [label="📑File Logs [table]" shape="parallelogram" color="black" tooltip="// Select all UJET log files from /srv/ujet/was/shared/log path
filter match_regex(Logfile, /^\/srv\/ujet\/was\/shared\/.*$/, "si")

// Get environment based on the host name
// web001.qca01.ujet.xyz
//        ^^^^^
make_col environment:string(get_item(split(host, "."), 1))

filter is_null(environment) = false

timechart 1h, group_by(environment)"]
  }
  subgraph cluster_ds_app_GCP {
    label="📊 App Datasets [GCP] (2)" style="filled" fillcolor="lightyellow"
    "DS_41217852" [label="📅GCP/
PubSub Events" shape="rectangle" color="purple"]
    "DS_41217854" [label="📅GCP/
Logs" shape="rectangle" color="purple"]
  }
  subgraph cluster_ds_app_kubernetes {
    label="📊 App Datasets [kubernetes] (3)" style="filled" fillcolor="lightyellow"
    "DS_41218411" [label="📅kubernetes/
Container Logs" shape="rectangle" color="purple"]
    "DS_41218416" [label="📅kubernetes/
API Update" shape="rectangle" color="purple"]
    "DS_41218438" [label="🗃kubernetes/
Namespace" shape="rectangle" color="blue"]
  }
  subgraph cluster_ds_app_Server {
    label="📊 App Datasets [Server] (2)" style="filled" fillcolor="lightyellow"
    "DS_41218603" [label="📅Server/
Fluentbit Events" shape="rectangle" color="purple"]
    "DS_41218643" [label="📅Server/
Fluentbit Logs" shape="rectangle" color="purple"]
  }
  subgraph cluster_ds_user {
    label="👋 User Datasets (5)" style="filled" fillcolor="palegreen"
    "DS_41282619" [label="📅UJET/
Application Events" shape="rectangle" color="purple"]
    "DS_41272968" [label="📅UJET/
Application Log Level Metrics" shape="rectangle" color="purple"]
    "DS_41282620" [label="📅UJET/
Application Events - High Volume" shape="rectangle" color="purple"]
    "DS_41251154" [label="📅UJET/
Application Logs" shape="rectangle" color="purple"]
  }
  subgraph cluster_ds_metric_support {
    label="📈 Metric Support Datasets (1)" style="filled" fillcolor="seashell1"
    "DS_41272969" [label="⏲metric-sma-for-41272968" shape="diamond" color="maroon"]
  }
  subgraph cluster_ds_monitor_support {
    label="📟 Monitor Support Datasets (4)" style="filled" fillcolor="seashell"
    "DS_41251167" [label="🗃monitor/
individual 41251165" shape="diamond" color="blue"]
    "DS_41251170" [label="🗃monitor/
individual 41251168" shape="diamond" color="blue"]
    "DS_41251194" [label="🗃monitor/
individual 41251192" shape="diamond" color="blue"]
    "DS_41251205" [label="🗃monitor/
individual 41251203" shape="diamond" color="blue"]
  }
  subgraph cluster_ds_datastream {
    label="🎏 DataStreams (3)"
    "DS_41217848" [label="📅GCP" shape="hexagon" color="green"]
    "DS_41217896" [label="📅Host" shape="hexagon" color="green"]
    "DS_41218200" [label="📅Kubernetes" shape="hexagon" color="green"]
  }
  subgraph cluster_da_user {
    label="👋 User Dashboards (13)" style="filled" fillcolor="ivory"
    "DA_41257742" [label="📈Instacart Agent Network Monitoring" shape="tab" color="black"]
    "DA_41257663" [label="📈DAP API Metrics" shape="tab" color="black"]
    "DA_41256997" [label="📈PTOSC Monitoring" shape="tab" color="black"]
    "DA_41257667" [label="📈Public API Usage" shape="tab" color="black"]
    "DA_41257731" [label="📈Sessionless Outbound SMS" shape="tab" color="black"]
    "DA_41298618" [label="📈UJET/
PTOSC Monitoring Copy" shape="tab" color="black"]
    "DA_41258033" [label="📈eu_abandoned count" shape="tab" color="black"]
    "DA_41265213" [label="📈OTEL Span Delivery" shape="tab" color="black"]
    "DA_41265098" [label="📈OTELCOL Metrics" shape="tab" color="black"]
    "DA_41245433" [label="📈Presence expired monitoring" shape="tab" color="black"]
    "DA_41258262" [label="📈prod http status >= 500" shape="tab" color="black"]
    "DA_41258402" [label="📈Satsuki-Production Monitoring" shape="tab" color="black"]
    "DA_41258462" [label="📈voice VA monitoring" shape="tab" color="black"]
  }

// The main focus entity
// "DS_41254206" [label="🗃UJET/
// Environment" shape="rectangle" color="blue" tooltip="List of Environments that the application stack is deployed to" style="filled" fillcolor="pink"]

// Edges
"ST_41254206_stage-08tclebf"->"ST_41254206_stage-g1qqnvyd" [color="black" tooltip="Container Logs_-08tc"]
"ST_41254206_stage-zfsdfkoz"->"ST_41254206_stage-g1qqnvyd" [color="black" tooltip="File Logs_-zfsd"]
"DS_41217848"->"DS_41217852" [color="black" tooltip="ObsDataset: GCP/41217848/Dataset, DatastreamDataset, EventDataset to ObsDataset: GCP/PubSub Events/41217852/Dataset, EventDataset as ProvidesData"]
"DS_41217854"->"DS_41218411" [color="black" tooltip="ObsDataset: GCP/Logs/41217854/Dataset, EventDataset, InterfaceLogDataset to ObsDataset: kubernetes/Container Logs/41218411/Dataset, EventDataset, InterfaceLogDataset as ProvidesData"]
"DS_41217852"->"DS_41217854" [color="black" tooltip="ObsDataset: GCP/PubSub Events/41217852/Dataset, EventDataset to ObsDataset: GCP/Logs/41217854/Dataset, EventDataset, InterfaceLogDataset as ProvidesData"]
"DS_41217896"->"DS_41218603" [color="black" tooltip="ObsDataset: Host/41217896/Dataset, DatastreamDataset, EventDataset to ObsDataset: Server/Fluentbit Events/41218603/Dataset, EventDataset as ProvidesData"]
"ST_41254206_stage-vm3d7rdq"->"ST_41254206_stage-g1qqnvyd" [color="black" tooltip="K8S Namespace_-vm3d"]
"DS_41218200"->"DS_41218416" [color="black" tooltip="ObsDataset: Kubernetes/41218200/Dataset, DatastreamDataset, EventDataset to ObsDataset: kubernetes/API Update/41218416/Dataset, EventDataset as ProvidesData"]
"DS_41218200"->"DS_41218411" [color="black" tooltip="ObsDataset: Kubernetes/41218200/Dataset, DatastreamDataset, EventDataset to ObsDataset: kubernetes/Container Logs/41218411/Dataset, EventDataset, InterfaceLogDataset as ProvidesData"]
"DS_41218416"->"DS_41218438" [color="black" tooltip="ObsDataset: kubernetes/API Update/41218416/Dataset, EventDataset to ObsDataset: kubernetes/Namespace/41218438/Dataset, ResourceDataset as ProvidesData"]
"DS_41218411"->"ST_41254206_stage-08tclebf" [color="black" tooltip="kubernetes/Container Logs"]
// "DS_41218411"->"DS_41254206" [color="black" tooltip="ObsDataset: kubernetes/Container Logs/41218411/Dataset, EventDataset, InterfaceLogDataset to ObsDataset: UJET/Environment/41254206/Dataset, ResourceDataset as ProvidesData"]
"DS_41218438"->"ST_41254206_stage-vm3d7rdq" [color="black" tooltip="kubernetes/Namespace"]
// "DS_41218438"->"DS_41254206" [color="black" tooltip="ObsDataset: kubernetes/Namespace/41218438/Dataset, ResourceDataset to ObsDataset: UJET/Environment/41254206/Dataset, ResourceDataset as ProvidesData"]
"DS_41218603"->"DS_41218643" [color="black" tooltip="ObsDataset: Server/Fluentbit Events/41218603/Dataset, EventDataset to ObsDataset: Server/Fluentbit Logs/41218643/Dataset, EventDataset as ProvidesData"]
"DS_41218643"->"ST_41254206_stage-zfsdfkoz" [color="black" tooltip="Server/Fluentbit Logs"]
// "DS_41218643"->"DS_41254206" [color="black" tooltip="ObsDataset: Server/Fluentbit Logs/41218643/Dataset, EventDataset to ObsDataset: UJET/Environment/41254206/Dataset, ResourceDataset as ProvidesData"]
"DS_41282620"->"DA_41257742" [color="black" tooltip="OpenTelemetry/Span Event"]
"DS_41282619"->"DA_41257663" [color="black" tooltip="OpenTelemetry/Span Event"]
"DS_41282619"->"DA_41256997" [color="black" tooltip="OpenTelemetry/Span Event"]
"DS_41282619"->"DA_41257667" [color="black" tooltip="OpenTelemetry/Span Event"]
"DS_41282619"->"DA_41257731" [color="black" tooltip="OpenTelemetry/Span Event"]
"DS_41282619"->"DA_41298618" [color="black" tooltip="OpenTelemetry/Span Event"]
"DS_41272968"->"DS_41272969" [color="black" tooltip="ObsDataset: UJET/Application Log Level Metrics/41272968/Dataset, EventDataset, InterfaceMetricDataset to ObsDataset: metric-sma-for-41272968/41272969/Dataset, IntervalDataset, MetricSMADataset as ProvidesData"]
"DS_41251154"->"DA_41258033" [color="black" tooltip="UJET/Application Logs"]
"DS_41251154"->"DS_41251167" [color="black" tooltip="ObsDataset: UJET/Application Logs/41251154/Dataset, EventDataset, InterfaceLogDataset to ObsDataset: monitor/individual 41251165/41251167/Dataset, ResourceDataset, MonitorSupportDataset as ProvidesData"]
"DS_41251154"->"DS_41251170" [color="black" tooltip="ObsDataset: UJET/Application Logs/41251154/Dataset, EventDataset, InterfaceLogDataset to ObsDataset: monitor/individual 41251168/41251170/Dataset, ResourceDataset, MonitorSupportDataset as ProvidesData"]
"DS_41251154"->"DS_41251194" [color="black" tooltip="ObsDataset: UJET/Application Logs/41251154/Dataset, EventDataset, InterfaceLogDataset to ObsDataset: monitor/individual 41251192/41251194/Dataset, ResourceDataset, MonitorSupportDataset as ProvidesData"]
"DS_41251154"->"DS_41251205" [color="black" tooltip="ObsDataset: UJET/Application Logs/41251154/Dataset, EventDataset, InterfaceLogDataset to ObsDataset: monitor/individual 41251203/41251205/Dataset, ResourceDataset, MonitorSupportDataset as ProvidesData"]
"DS_41251154"->"DA_41265213" [color="black" tooltip="UJET/Application Logs"]
"DS_41251154"->"DA_41265098" [color="black" tooltip="UJET/Application Logs"]
"DS_41251154"->"DA_41245433" [color="black" tooltip="UJET/Application Logs"]
"DS_41251154"->"DA_41258262" [color="black" tooltip="UJET/Application Logs"]
"DS_41251154"->"DA_41258402" [color="black" tooltip="UJET/Application Logs"]
"DS_41251154"->"DS_41282619" [color="black" tooltip="ObsDataset: UJET/Application Logs/41251154/Dataset, EventDataset, InterfaceLogDataset to ObsDataset: UJET/Application Events/41282619/Dataset, EventDataset as ProvidesData"]
"DS_41251154"->"DS_41272968" [color="black" tooltip="ObsDataset: UJET/Application Logs/41251154/Dataset, EventDataset, InterfaceLogDataset to ObsDataset: UJET/Application Log Level Metrics/41272968/Dataset, EventDataset, InterfaceMetricDataset as ProvidesData"]
"DS_41251154"->"DA_41258462" [color="black" tooltip="UJET/Application Logs"]
"ST_41254206_stage-g1qqnvyd"->"DS_41282620" [color="blue" tooltip="ObsDataset: UJET/Environment/41254206/Dataset, ResourceDataset to ObsDataset: UJET/Application Events - High Volume/41282620/Dataset, EventDataset as Linked"]
"ST_41254206_stage-g1qqnvyd"->"DS_41282619" [color="blue" tooltip="ObsDataset: UJET/Environment/41254206/Dataset, ResourceDataset to ObsDataset: UJET/Application Events/41282619/Dataset, EventDataset as Linked"]
"ST_41254206_stage-g1qqnvyd"->"DS_41251154" [color="blue" tooltip="ObsDataset: UJET/Environment/41254206/Dataset, ResourceDataset to ObsDataset: UJET/Application Logs/41251154/Dataset, EventDataset, InterfaceLogDataset as Linked"]
}