search

dimfalk / netatmo.weather

R wrapper for Netatmo Weather API
GNU General Public License v3.0
3 stars 0 forks source link

`get_measure()`: tail cut-off or data delay? last tuples seem to be missing. #71

Closed dimfalk closed 5 months ago

dimfalk commented 5 months ago 
library(netatmo.weather)
#> 0.5.17

p <- get_period("2024-04-01/2024-04-02")
p
#> [1] 1711929600 1712016000

as.POSIXct(p, tz = "UTC")
'> [1] "2024-04-01 UTC" "2024-04-02 UTC"
obs <- get_measure(devices = stations_tiled[1, ], 
                   period = p, 
                   par = "temperature", 
                   res = 5)

zoo::index(obs[[1]]) |> range()
#> "2024-04-01 00:00:00 UTC" "2024-04-01 23:50:00 UTC"

# enforce `NA` values for missing timestamps
timeseriesIO::xts_make_equidistant(xts) |> length()
#> 287

23:55 missing

dimfalk commented 5 months ago 
library(netatmo.weather)
#> 0.5.17

Sys.time()
#> "2024-04-06 23:05:52 CEST"
#> == "2024-04-06 21:05:52 UTC"

p <- get_period(res = 60)
p
#>[1] 1708750800 1712437200

as.POSIXct(p, tz = "UTC")
'> [1] "2024-02-24 05:00:00 UTC" "2024-04-06 21:00:00 UTC"
obs <- get_measure(devices = stations_tiled[1, ], 
                   period = p, 
                   par = "temperature", 
                   res = 60)

zoo::index(obs[[1]]) |> range()
#> "2024-02-24 05:00:00 UTC" "2024-04-06 19:00:00 UTC"

length(obs[[1]])
#> 1023

20:00 missing

dimfalk commented 5 months ago

Investigate behaviour without lubridate::floor_date(unit = "hour")?

https://github.com/dimfalk/netatmo.weather/blob/aba5dd48e72b5df9e5a34e369d98e45958183829/R/get_period.R#L51-L52

dimfalk commented 5 months ago

It seems like the last value of the interval queried is dismissed:

p <- get_period("2024-04-01 00:00/2024-04-02 00:00")
as.POSIXct(p, tz = "UTC")
#> "2024-04-01 UTC" "2024-04-02 UTC"

obs <- get_measure(devices = stations_tiled[1, ], 
                   period = p, 
                   par = "temperature", 
                   res = 60)

zoo::index(obs[[1]]) |> range()
#> "2024-04-01 00:00:00 UTC" "2024-04-01 22:00:00 UTC"

length(obs[[1]])
#> 23
p <- get_period("2024-04-01 00:00/2024-04-02 00:00")
as.POSIXct(p, tz = "UTC")
#> "2024-04-01 00:00:00 UTC" "2024-04-02 01:00:00 UTC"

obs <- get_measure(devices = stations_tiled[1, ], 
                   period = p, 
                   par = "temperature", 
                   res = 60)

zoo::index(obs[[1]]) |> range()
#> "2024-04-01 00:00:00 UTC" "2024-04-01 23:00:00 UTC"

length(obs[[1]])
#> 24
dimfalk commented 5 months ago

However, the raw API response provides 24 values using the upper request, so we're loosing the last one during processing:

{
  "body": {
    "1711929600": [
      9.5
    ],
    "1711933200": [
      9.8
    ],
    "1711936800": [
      9.9
    ],
    "1711940400": [
      9.7
    ],
    "1711944000": [
      9.3
    ],
    "1711947600": [
      9.4
    ],
    "1711951200": [
      9.6
    ],
    "1711954800": [
      10.1
    ],
    "1711958400": [
      10.5
    ],
    "1711962000": [
      10.6
    ],
    "1711965600": [
      11
    ],
    "1711969200": [
      12.3
    ],
    "1711972800": [
      13.3
    ],
    "1711976400": [
      13.4
    ],
    "1711980000": [
      13.6
    ],
    "1711983600": [
      14.1
    ],
    "1711987200": [
      13.9
    ],
    "1711990800": [
      12.6
    ],
    "1711994400": [
      11.8
    ],
    "1711998000": [
      10.9
    ],
    "1712001600": [
      9.4
    ],
    "1712005200": [
      8.4
    ],
    "1712008800": [
      8.3
    ],
    "1712012400": [
      9.2
    ]
  },
  "status": "ok",
  "time_exec": 0.08574914932250977,
  "time_server": 1712438269
}
dimfalk commented 5 months ago 
library(netatmo.weather)
#> 0.5.18

p <- get_period("2024-04-01/2024-04-02")
p
#> [1] 1711929600 1712016000

as.POSIXct(p, tz = "UTC")
#> [1] "2024-04-01 UTC" "2024-04-02 UTC"
obs <- get_measure(devices = stations_tiled[1, ], 
                   period = p, 
                   par = "temperature", 
                   res = 5)

zoo::index(obs[[1]]) |> range()
#> [1] "2024-04-01 00:00:00 UTC" "2024-04-01 23:55:00 UTC"

# enforce `NA` values for missing timestamps
timeseriesIO::xts_make_equidistant(obs[[1]]) |> length()
#> [1] 288
obs <- get_measure(devices = stations_tiled[1, ], 
                   period = p, 
                   par = "temperature", 
                   res = 60)

zoo::index(obs[[1]]) |> range()
#> [1] "2024-04-01 00:00:00 UTC" "2024-04-01 23:00:00 UTC"

length(obs[[1]])
#> [1] 24