search

internet-equity / nm-exp-active-netrics

Netrics - Active Measurements of Internet Performance
Apache License 2.0
12 stars 7 forks source link

Malformed JSON #81

Closed marcwitasee closed 2 years ago

marcwitasee commented 2 years ago

Some JSON files with results from nm-exp-active-netrics are malformed. I've confirmed that this is occurring on the Raspberry Pi, as the file is malformed before it is uploaded to S3.

Here is an example of a malformed JSON:

{
   "Measurements":{
      "connected_devices_arp":{
         "devices_1day":25,
         "devices_1week":33,
         "devices_active":21,
         "devices_total":74
      },
      "dns_latency":{
         "dns_query_avg_ms":28.6,
         "dns_query_max_ms":67,
         "error":false
      },
      "hops_to_target":{
         "error":false,
         "hops_to_google":10
      },
      "ipquery":{
         "ipv4":"71.239.164.240"
      },
      "last_mile_rtt":{
         "Cloudflare_DNS_last_mile_ping_packet_loss_pct":0,
         "Cloudflare_DNS_last_mile_ping_rtt_avg_ms":12.994,
         "Cloudflare_DNS_last_mile_ping_rtt_max_ms":22.788,
         "Cloudflare_DNS_last_mile_ping_rtt_mdev_ms":4.772,
         "Cloudflare_DNS_last_mile_ping_rtt_min_ms":8.908,
         "Cloudflare_DNS_last_mile_tr_rtt_max_ms":16.462,
         "Cloudflare_DNS_last_mile_tr_rtt_median_ms":16.416,
         "Cloudflare_DNS_last_mile_tr_rtt_min_ms":10.588
      },
      "ping_latency":{
         "Atlanta_packet_loss_pct":0,
         "Atlanta_rtt_avg_ms":42.375,
         "Atlanta_rtt_max_ms":45.565,
         "Atlanta_rtt_mdev_ms":2.468,
         "Atlanta_rtt_min_ms":38.484,
         "Chicago_packet_loss_pct":0,
         "Chicago_rtt_avg_ms":16.593,
         "Chicago_rtt_max_ms":20.237,
         "Chicago_rtt_mdev_ms":2.487,
         "Chicago_rtt_min_ms":13.459,
         "Denver_packet_loss_pct":0,
         "Denver_rtt_avg_ms":39.974,
         "Denver_rtt_max_ms":46.504,
         "Denver_rtt_mdev_ms":2.871,
         "Denver_rtt_min_ms":36.514,
         "Hong_Kong_packet_loss_pct":0,
         "Hong_Kong_rtt_avg_ms":214.675,
         "Hong_Kong_rtt_max_ms":224.821,
         "Hong_Kong_rtt_mdev_ms":5.093,
         "Hong_Kong_rtt_min_ms":208.148,
         "Johannesburg_packet_loss_pct":0,
         "Johannesburg_rtt_avg_ms":229.051,
         "Johannesburg_rtt_max_ms":234.597,
         "Johannesburg_rtt_mdev_ms":4.009,
         "Johannesburg_rtt_min_ms":222.773,
         "Paris_packet_loss_pct":0,
         "Paris_rtt_avg_ms":109.445,
         "Paris_rtt_max_ms":115.615,
         "Paris_rtt_mdev_ms":3.744,
         "Paris_rtt_min_ms":105.569,
         "Sao_Paulo_packet_loss_pct":0,
         "Sao_Paulo_rtt_avg_ms":173.797,
         "Sao_Paulo_rtt_max_ms":175.649,
         "Sao_Paulo_rtt_mdev_ms":1.527,
         "Sao_Paulo_rtt_min_ms":171.764,
         "Seattle_packet_loss_pct":0,
         "Seattle_rtt_avg_ms":67.49,
         "Seattle_rtt_max_ms":93.586,
         "Seattle_rtt_mdev_ms":9.003,
         "Seattle_rtt_min_ms":61.565,
         "Stockholm_packet_loss_pct":0,
         "Stockholm_rtt_avg_ms":133.031,
         "Stockholm_rtt_max_ms":174.746,
         "Stockholm_rtt_mdev_ms":14.04,
         "Stockholm_rtt_min_ms":126.031,
         "Sydney_packet_loss_pct":0,
         "Sydney_rtt_avg_ms":224.508,
         "Sydney_rtt_max_ms":235.657,
         "Sydney_rtt_mdev_ms":4.08,
         "Sydney_rtt_min_ms":220.811,
         "Tunis_packet_loss_pct":0,
         "Tunis_rtt_avg_ms":133.207,
         "Tunis_rtt_max_ms":136.605,
         "Tunis_rtt_mdev_ms":2.201,
         "Tunis_rtt_min_ms":130.643,
         "Washington_DC_packet_loss_pct":0,
         "Washington_DC_rtt_avg_ms":30.724,
         "Washington_DC_rtt_max_ms":34.578,
         "Washington_DC_rtt_mdev_ms":1.822,
         "Washington_DC_rtt_min_ms":28.316,
         "amazon_packet_loss_pct":0,
         "amazon_rtt_avg_ms":14.103,
         "amazon_rtt_max_ms":21.547,
         "amazon_rtt_mdev_ms":3.108,
         "amazon_rtt_min_ms":11.162,
         "error":false,
         "facebook_packet_loss_pct":0,
         "facebook_rtt_avg_ms":17.124,
         "facebook_rtt_max_ms":33.068,
         "facebook_rtt_mdev_ms":7.997,
         "facebook_rtt_min_ms":9.001,
         "google_packet_loss_pct":0,
         "google_rtt_avg_ms":11.352,
         "google_rtt_max_ms":18.616,
         "google_rtt_mdev_ms":3.601,
         "google_rtt_min_ms":8.755,
         "suntimes_packet_loss_pct":0,
         "suntimes_rtt_avg_ms":13.081,
         "suntimes_rtt_max_ms":15.753,
         "suntimes_rtt_mdev_ms":1.612,
         "suntimes_rtt_min_ms":11.266,
         "tribune_packet_loss_pct":0,
         "tribune_rtt_avg_ms":11.761,
         "tribune_rtt_max_ms":17.612,
         "tribune_rtt_mdev_ms":2.486,
         "tribune_rtt_min_ms":8.717,
         "uchicago_packet_loss_pct":0,
         "uchicago_rtt_avg_ms":16.794,
         "uchicago_rtt_max_ms":25.562,
         "uchicago_rtt_mdev_ms":3.818,
         "uchicago_rtt_min_ms":12.444,
         "wikipedia_packet_loss_pct":0,
         "wikipedia_rtt_avg_ms":28.849,
         "wikipedia_rtt_max_ms":31.417,
         "wikipedia_rtt_mdev_ms":1.624,
         "wikipedia_rtt_min_ms":27.189,
         "youtube_packet_loss_pct":0,
         "youtube_rtt_avg_ms":11.715,
         "youtube_rtt_max_ms":15.982,
         "youtube_rtt_mdev_ms":2.807,
         "youtube_rtt_min_ms":8.638
      },
      "test_bytes_consumed":0,
      "total_bytes_consumed":212457802339
   },
   "Meta":{
      "Extended":{
         "Annotation":null,
         "dataver":"1",
         "debhash":"d74a91d0f11fdb31dff65b243afff1ce5c0d3002",
         "gitlog":"fix: max_monthly checks"
      },
      "Id":"nm-mngd-20210927-8da43959",
      "Time":1643574363.891662
   }
}
},
"dns_latency":{
   "dns_query_avg_ms":28.6,
   "dns_query_max_ms":67,
   "error":false
},
"hops_to_target":{
   "hops_to_google":10,
   "error":false
},
"connected_devices_arp":{
   "devices_active":21,
   "devices_total":74,
   "devices_1day":25,
   "devices_1week":33
},
"test_bytes_consumed":0
}
}
jesteria commented 2 years ago

@marcwitasee Does the example JSON indicate a malformed structure or the expected structure?

marcwitasee commented 2 years ago

@jesteria Just updated the description. It's an example of a malformed JSON.

jesteria commented 2 years ago

Oh wow that is screwy. FTR: it's quite hard to tell what's wrong with such a large structure; luckily, computers: "line 148 column 1":

{
  "Measurements":{
    …
  },
  "Meta":{
    …
  }
}
},
"dns_latency":{
   "dns_query_avg_ms":28.6,
   "dns_query_max_ms":67,
   "error":false
},
"hops_to_target":{
   "hops_to_google":10,
   "error":false
},
"connected_devices_arp":{
   "devices_active":21,
   "devices_total":74,
   "devices_1day":25,
   "devices_1week":33
},
"test_bytes_consumed":0
}
}

So it looks like a regular measurements file with a few subsections recopied at the bottom. That's a mess.

If the code that writes this is just taking a proper Python dict and writing it in one swoop, e.g.:

with open(…) as fd:
    json.dump(data, fd)

…Then I don't really see how the code could be responsible, and rather the sync system might be doing something very bad.

(Of course the writer could be doing all sorts of things as well.)

ggmartins commented 2 years ago

seems to be fixed