Docker container uptime_ns may be negative

gotson commented 5 years ago

Relevant telegraf.conf:

# Read metrics about docker containers
[[inputs.docker]]
  ## Docker Endpoint
  ##   To use TCP, set endpoint = "tcp://[ip]:[port]"
  ##   To use environment variables (ie, docker-machine), set endpoint = "ENV"
  endpoint = "unix:///var/run/docker.sock"

  ## Set to true to collect Swarm metrics(desired_replicas, running_replicas)
  gather_services = false

  ## Only collect metrics for these containers, collect all if empty
  container_names = []

  ## Containers to include and exclude. Globs accepted.
  ## Note that an empty array for both will include all containers
  container_name_include = []
  container_name_exclude = []

  ## Container states to include and exclude. Globs accepted.
  ## When empty only containers in the "running" state will be captured.
  ## example: container_state_include = ["created", "restarting", "running", "removing", "paused", "exited", "dead"]
  ## example: container_state_exclude = ["created", "restarting", "running", "removing", "paused", "exited", "dead"]
  # container_state_include = []
  # container_state_exclude = []

  ## Timeout for docker list, info, and stats commands
  timeout = "5s"

  ## Whether to report for each container per-device blkio (8:0, 8:1...) and
  ## network (eth0, eth1, ...) stats or not
  perdevice = true
  ## Whether to report for each container total blkio and network stats or not
  total = false
  ## Which environment variables should we use as a tag
  ##tag_env = ["JAVA_HOME", "HEAP_SIZE"]

  ## docker labels to include and exclude as tags.  Globs accepted.
  ## Note that an empty array for both will include all labels as tags
  docker_label_include = []
  docker_label_exclude = []

System info:

Telegraf v1.12.4 running inside Docker on Ubuntu 19.04

Steps to reproduce:

Run some containers

Expected behavior:

docker_container_status.uptime_ns should be positive.

Actual behavior:

For some containers, docker_container_status.uptime_ns is negative.

Additional info:

I have 23 containers running, and 7 of them have a negative uptime.

Here is a list of the uptimes extracted from Chronograf:

1225819351060780
1791733604813900
-1002174896
22861001160934
-852099304
16538947268714
889544877443432
889547034686139
-295827770
94724723108097
207720147185631
15383184878473
15383266074614
-35407018583
-28587512725
-484100573
-41658487591
6101729087168
429114049532334
414113293415570
28229441453406
475919373816345
381711068671320

gotson commented 5 years ago

I just noticed that while the positive values increase over time, the negative values remain the same over time.

danielnelson commented 5 years ago

Can you run telegraf --input-filter=docker --test and add the output? Make sure to remove anything from the output if it is sensitive.

gotson commented 5 years ago

Here it is, i ran it inside the container that is running telegraf. I have omitted lines that are not about docker_container_status.

2019-11-06T10:52:00Z I! Starting Telegraf 1.12.4
2019-11-06T10:52:00Z I! Using config file: /etc/telegraf/telegraf.conf
> docker,engine_host=leon,host=leon,server_version=19.03.1 n_containers=24i,n_containers_paused=0i,n_containers_running=23i,n_containers_stopped=1i,n_cpus=8i,n_goroutines=406i,n_images=28i,n_listener_events=0i,n_used_file_descriptors=425i 1573037521000000000
> docker,engine_host=leon,host=leon,server_version=19.03.1 memory_total=16678854656i 1573037521000000000
> docker_container_status,com.docker.compose.config-hash=27c64618914b2226ae6d6dd6f5a7f3e6c9050c6a20f52df2249f983db87d1aab,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=komga,com.docker.compose.service=komga-beta,com.docker.compose.version=1.23.2,com.ouroboros.enable=True,container_image=gotson/komga,container_name=komga-beta,container_status=running,container_version=beta,engine_host=leon,host=leon,server_version=19.03.1 container_id="6e31b7fbc58479e395adfdb80e002db7dbddb5c677359b3d939bd0144dcf1d68",exitcode=0i,oomkilled=false,pid=6661i,started_at=1572925879467672988i,uptime_ns=111642859440277i 1573037522000000000
> docker_container_status,MAINTAINER=sparkyballs\,TheLamer,build_version=Linuxserver.io\ version:-\ 32234c97-ls2\ Build-date:-\ 2019-07-12T08:18:37+00:00,com.docker.compose.config-hash=082c4e417a3bf19a202cd648ea50ce6c2796bb2be2823207b76c5e4435a78cd5,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=lsio,com.docker.compose.service=flexget,com.docker.compose.version=1.23.2,com.ouroboros.enable=True,container_image=cpoppema/docker-flexget,container_name=flexget,container_status=running,container_version=latest,engine_host=leon,host=leon,server_version=19.03.1 container_id="30a8c55f2427162f16d11728383a80a1b9c88cee64c86d06a75e780447d19c75",exitcode=0i,finished_at=1572969605133092224i,oomkilled=false,pid=13460i,started_at=1572969606019045664i,uptime_ns=-885953440i 1573037524000000000
> docker_container_status,com.docker.compose.config-hash=36f4e97912c3f9354db4b4ca7653e37c7b9bd9e563e53fbb863254e81ae84e40,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=influx,com.docker.compose.service=telegraf,com.docker.compose.version=1.23.2,com.ouroboros.enable=True,container_image=telegraf,container_name=telegraf,container_status=running,container_version=unknown,engine_host=leon,host=leon,server_version=19.03.1 container_id="ac5eeb434e53c1d2812be4b82c98ddbb6d303feac940e3a4b9370387e3c7a6e5",exitcode=0i,oomkilled=false,pid=31854i,started_at=1572913033261929631i,uptime_ns=124490265776264i 1573037524000000000
> docker_container_status,container_image=homeassistant/amd64-hassio-supervisor,container_name=hassio_supervisor,container_status=running,container_version=unknown,engine_host=leon,host=leon,io.hass.arch=amd64,io.hass.base.image=homeassistant/{arch}-base-python,io.hass.base.name=python,io.hass.base.version=10.2,io.hass.type=supervisor,io.hass.version=192,server_version=19.03.1 container_id="54f52d6f9c0c9bd289f10128efd30a572bf35fbf606b68653d30b9984b908b69",exitcode=0i,oomkilled=false,pid=19584i,started_at=1572051715653410525i,uptime_ns=985807888965726i 1573037524000000000
> docker_container_status,MAINTAINER=sparkyballs\,TheLamer,build_version=Linuxserver.io\ version:-\ v0.2.0.1358-ls34\ Build-date:-\ 2019-09-17T06:17:06+01:00,com.docker.compose.config-hash=c0da8c83cccf2a90515875f22623484ae333f5fff96385bad366a6f58aa0a3c6,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=lsio,com.docker.compose.service=radarr,com.docker.compose.version=1.23.2,com.ouroboros.enable=True,container_image=linuxserver/radarr,container_name=radarr,container_status=running,container_version=latest,engine_host=leon,host=leon,maintainer=sparklyballs,server_version=19.03.1 container_id="a139ee43301dcd328e8462518ce09da19cd1bcb2179635db499f5569efa03bf3",exitcode=0i,oomkilled=false,pid=2983i,started_at=1572935160943768976i,uptime_ns=102362613497188i 1573037524000000000
> docker_container_status,container_image=homeassistant/amd64-addon-configurator,container_name=addon_core_configurator,container_status=running,container_version=3.6,engine_host=leon,host=leon,io.hass.arch=amd64,io.hass.base.image=homeassistant/{arch}-base-python,io.hass.base.name=python,io.hass.base.version=9.0,io.hass.description=Browser-based\ configuration\ file\ editor\ for\ Home\ Assistant,io.hass.name=Configurator,io.hass.type=addon,io.hass.url=https://home-assistant.io/addons/configurator,io.hass.version=3.6,server_version=19.03.1 container_id="dd29bcf93ecd85aefd62fb40010a2d38357ffd04dc8d1c66bc161ff1ec4aa6c2",exitcode=0i,oomkilled=false,pid=15007i,started_at=1571149529143772534i,uptime_ns=1887994428328780i 1573037524000000000
> docker_container_status,MAINTAINER=sparkyballs\,TheLamer,build_version=Linuxserver.io\ version:-\ 2.1.2-ls21\ Build-date:-\ 2019-09-19T16:54:53-04:00,com.docker.compose.config-hash=bbf75532407948a92d95e1e150ea238e151a7797a061be8bfb8de5f1494715ff,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=lsio,com.docker.compose.service=ubooquity,com.docker.compose.version=1.23.2,com.ouroboros.enable=True,container_image=linuxserver/ubooquity,container_name=ubooquity,container_status=running,container_version=latest,engine_host=leon,host=leon,maintainer=sparklyballs,server_version=19.03.1 container_id="88c913823c97e0e61e1c6d99df79da3b5209af1d603d6a0593382dc115377c40",exitcode=0i,oomkilled=false,pid=20871i,started_at=1572559551726477651i,uptime_ns=477971860515230i 1573037524000000000
> docker_container_status,com.docker.compose.config-hash=e7b96c723063cc3bb527e7b983b3533a6f9154cc545f1e559ea7b2a162dde1aa,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=komga,com.docker.compose.service=komga,com.docker.compose.version=1.23.2,com.ouroboros.enable=True,container_image=gotson/komga,container_name=komga,container_status=running,container_version=latest,engine_host=leon,host=leon,server_version=19.03.1 container_id="48558f9199566fb154e7b92ebbb24ab6805351a189a5e4b7589a3232933bd120",exitcode=0i,oomkilled=false,pid=6900i,started_at=1572925879579018774i,uptime_ns=111644023178854i 1573037524000000000
> docker_container_status,com.docker.compose.config-hash=d6667c16592640501acd5449367066afa7b0cd8ee41442827ba5524186625727,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=influx,com.docker.compose.service=chronograf,com.docker.compose.version=1.23.2,com.ouroboros.enable=True,container_image=chronograf,container_name=chronograf,container_status=running,container_version=unknown,engine_host=leon,host=leon,server_version=19.03.1 container_id="156027238e20f1a5af87666f066f71c4b19adf670ab350d2171ece557524b209",exitcode=0i,oomkilled=false,pid=18291i,started_at=1572918401809233682i,uptime_ns=119121808433089i 1573037524000000000
> docker_container_status,container_image=homeassistant/intel-nuc-homeassistant,container_name=homeassistant,container_status=running,container_version=0.101.2,engine_host=leon,host=leon,io.hass.arch=amd64,io.hass.base.image=homeassistant/{arch}-homeassistant-base,io.hass.base.name=python,io.hass.base.version=4.3,io.hass.machine=intel-nuc,io.hass.type=homeassistant,io.hass.version=0.101.2,server_version=19.03.1 container_id="a6c808e84a55adb3df71241fbbaff6a26936c4d852b1aab10569c098bc31e767",exitcode=0i,finished_at=1572923728081562972i,oomkilled=false,pid=9187i,started_at=1572923728377390742i,uptime_ns=-295827770i 1573037524000000000
> docker_container_status,com.docker.compose.config-hash=e016247fdda99f40e5a2dd6855ddaa386fd97c245068bc03a894b572b6a2f8c8,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=lsio,com.docker.compose.service=portainer,com.docker.compose.version=1.23.2,com.ouroboros.enable=True,container_image=portainer/portainer,container_name=portainer,container_status=running,container_version=latest,engine_host=leon,host=leon,server_version=19.03.1 container_id="5781bfe7cfd2a3dbbf7a6670bbaf075cdf4a2eef84c419a730c39234ae080adc",exitcode=0i,oomkilled=false,pid=20408i,started_at=1573021553628750022i,uptime_ns=15970020798628i 1573037524000000000
> docker_container_status,caddy_version=1.0.3,com.docker.compose.config-hash=9471ce71d1b37bd0592931f2e26cacb9c831f6e9a731df499cc9602d8552e1a6,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=lsio,com.docker.compose.service=caddy,com.docker.compose.version=1.23.2,com.ouroboros.enable=True,container_image=abiosoft/caddy,container_name=caddy,container_status=running,container_version=php-no-stats,engine_host=leon,host=leon,maintainer=Abiola\ Ibrahim\ <abiola89@gmail.com>,server_version=19.03.1 container_id="6d9b2a45a80065d12d5d1bcc2b3e1bf7cb67cd647b4f0d7df70a91b0f8a5c0aa",exitcode=0i,finished_at=1572918646347234198i,oomkilled=false,pid=30916i,started_at=1572918647349409094i,uptime_ns=-1002174896i 1573037524000000000
> docker_container_status,com.docker.compose.config-hash=7cf5d70785af3f9204de2cc510d267668e58faabbab29925aad0fdf40a297264,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=influx,com.docker.compose.service=grafana,com.docker.compose.version=1.23.2,com.ouroboros.enable=True,container_image=grafana/grafana,container_name=grafana,container_status=running,container_version=unknown,engine_host=leon,host=leon,server_version=19.03.1 container_id="33661d7c0682e7ae29e91b591c613b3bf58d9a43defc67bdfac78b6e6cef474b",exitcode=0i,oomkilled=false,pid=5770i,started_at=1572924723590557839i,uptime_ns=112800089642786i 1573037524000000000
> docker_container_status,com.docker.compose.config-hash=dafa1d2ee5a7083fff798b61d800fe605106b43553ec905c8cfcbedff22fc20a,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=lsio,com.docker.compose.service=plex,com.docker.compose.version=1.23.2,com.ouroboros.enable=True,container_image=plexinc/pms-docker,container_name=plex,container_status=running,container_version=plexpass,engine_host=leon,host=leon,server_version=19.03.1 container_id="1543aca673015b79ce6cb16e27c5aad6ca33699bbc4949738fa93ef5c2a86d10",exitcode=0i,finished_at=1572969605004286495i,oomkilled=false,pid=13018i,started_at=1572969605474302976i,uptime_ns=-470016481i 1573037524000000000
> docker_container_status,com.docker.compose.config-hash=abb3aaa6eb9c7fab1ed274882fbb8a468d1f1dc967f40bc59be9579c903f4bb5,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=lsio,com.docker.compose.service=ouroboros,com.docker.compose.version=1.23.2,container_image=pyouroboros/ouroboros,container_name=ouroboros,container_status=running,container_version=unknown,engine_host=leon,host=leon,maintainers=dirtycajunrice\,circa10a,server_version=19.03.1 container_id="a1f1696ae18584ea8effe647d4df4444a2c0fb5a9bbf98ac456bc5dea863ba14",exitcode=0i,finished_at=1571149459435681518i,oomkilled=false,pid=4184i,started_at=1571149488023194243i,uptime_ns=-28587512725i 1573037524000000000
> docker_container_status,MAINTAINER=sparkyballs\,TheLamer,build_version=Linuxserver.io\ version:-\ 2.6.3-ls27\ Build-date:-\ 2019-09-19T07:54:28-04:00,com.docker.compose.config-hash=deb3d57863e6ced688635df89e740e776b4fda6992efce6059ee00ec2bb1e880,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=lsio,com.docker.compose.service=sync,com.docker.compose.version=1.23.2,com.ouroboros.enable=True,container_image=linuxserver/resilio-sync,container_name=sync,container_status=running,container_version=latest,engine_host=leon,host=leon,maintainer=sparklyballs,server_version=19.03.1 container_id="03ce29857e00eb8f493220181046c50e1c2d8a36a9c4c6fc9bfa97692bb1c6c7",exitcode=0i,oomkilled=false,pid=5163i,started_at=1572527149198871152i,uptime_ns=510374526750035i 1573037524000000000
> docker_container_status,container_image=homeassistant/amd64-hassio-dns,container_name=hassio_dns,container_status=running,container_version=1,engine_host=leon,host=leon,io.hass.arch=amd64,io.hass.base.image=homeassistant/{arch}-base,io.hass.base.name=alpine,io.hass.base.version=7.1,io.hass.type=dns,io.hass.version=1,server_version=19.03.1 container_id="c91258b63fa0c9609ae00fef3e168dcb3053ba48ba4e6385841f7e1d7012b3cd",exitcode=0i,oomkilled=false,pid=20111i,started_at=1572051717720252887i,uptime_ns=985806020742994i 1573037524000000000
> docker_container_status,com.docker.compose.config-hash=bbb7245d5d80fcf1d6cdd7790646d34a98a0dc87e6a65a23e5e928aead3a02c4,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=nestor,com.docker.compose.service=nestor,com.docker.compose.version=1.23.2,com.ouroboros.enable=True,container_image=gotson/nestor,container_name=nestor,container_status=running,container_version=latest,engine_host=leon,host=leon,server_version=19.03.1 container_id="96209d22ecb11b23911e82156a3294f5506c475408af13344f750b68bab3447f",exitcode=0i,finished_at=1571149451395473290i,oomkilled=false,pid=4111i,started_at=1571149486802491873i,uptime_ns=-35407018583i 1573037524000000000
> docker_container_status,MAINTAINER=sparkyballs\,TheLamer,build_version=Linuxserver.io\ version:-\ 2.94-r1-ls27\ Build-date:-\ 2019-09-18T14:24:24-04:00,com.docker.compose.config-hash=8604b3e03dad911816e1d18361413183aaa66191215af8b975bad1668b9f204b,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=lsio,com.docker.compose.service=transmission,com.docker.compose.version=1.23.2,com.ouroboros.enable=True,container_image=linuxserver/transmission,container_name=transmission,container_status=running,container_version=latest,engine_host=leon,host=leon,maintainer=sparklyballs,server_version=19.03.1 container_id="7a9eddde2464ebb9691263229c20c284e2d4a9dd29b843d0719178f32895e56c",exitcode=0i,oomkilled=false,pid=31763i,started_at=1572465343194160023i,uptime_ns=572180577109506i 1573037524000000000
> docker_container_status,com.docker.compose.config-hash=c8f6b23a238e5ef7b4f194eff2c8f8b5a40c74ef93de3c9c1ede8521f9f5f491,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=influx,com.docker.compose.service=influxdb,com.docker.compose.version=1.23.2,com.ouroboros.enable=True,container_image=influxdb,container_name=influxdb,container_status=running,container_version=unknown,engine_host=leon,host=leon,server_version=19.03.1 container_id="31a70449d84109e16b8be5c8e02b08fcecce0d8f7dbdb0c89d797cb897513b50",exitcode=0i,oomkilled=false,pid=2706i,started_at=1572846537919612431i,uptime_ns=190985867122774i 1573037524000000000
> docker_container_status,MAINTAINER=sparkyballs\,TheLamer,build_version=Linuxserver.io\ version:-\ v0.11.709-ls40\ Build-date:-\ 2019-09-20T15:07:16-04:00,com.docker.compose.config-hash=8f3b17c3a0381f59f0dc533a9d75d60d8d4f0134189361be87127c95d491f2fc,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=lsio,com.docker.compose.service=jackett,com.docker.compose.version=1.23.2,com.ouroboros.enable=True,container_image=linuxserver/jackett,container_name=jackett,container_status=running,container_version=latest,engine_host=leon,host=leon,maintainer=sparklyballs,server_version=19.03.1 container_id="2901df80ce972f011458df6246ab79822d2b15258a20732056c4879340ae5609",exitcode=0i,oomkilled=false,pid=17900i,started_at=1572733542510858508i,uptime_ns=303981291047412i 1573037524000000000
> docker_container_status,MAINTAINER=sparkyballs\,TheLamer,build_version=Linuxserver.io\ version:-\ 2.0.0.5338-ls34\ Build-date:-\ 2019-09-19T03:34:37-04:00,com.docker.compose.config-hash=1231cc32cb28ee318f4da8d7b1188deaed3214abd010c8f4ea39cb52679bbbbb,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=lsio,com.docker.compose.service=sonarr,com.docker.compose.version=1.23.2,com.ouroboros.enable=True,container_image=linuxserver/sonarr,container_name=sonarr,container_status=running,container_version=latest,engine_host=leon,host=leon,maintainer=sparklyballs,server_version=19.03.1 container_id="1bab8e91a72a30afc65434ef29ea4070b3348bf452d52f701f56484f17047a38",exitcode=0i,oomkilled=false,pid=4950i,started_at=1572512148533353762i,uptime_ns=525375283763649i 1573037524000000000
> docker_container_status,container_image=hassioaddons/ssh-amd64,container_name=addon_a0d7b954_ssh,container_status=running,container_version=6.4.0,engine_host=leon,host=leon,io.hass.arch=amd64,io.hass.base.image=hassioaddons/base,io.hass.base.name=alpine,io.hass.base.version=5.0.1,io.hass.description=SSH\ &\ Web\ Terminal\ access\ to\ your\ Home\ Assistant\ instance,io.hass.name=SSH\ &\ Web\ Terminal,io.hass.type=addon,io.hass.version=6.4.0,maintainer=Franck\ Nijhof\ <frenck@addons.community>,org.label-schema.build-date=2019-10-20T21:16:53Z,org.label-schema.description=SSH\ &\ Web\ Terminal\ access\ to\ your\ Home\ Assistant\ instance,org.label-schema.name=SSH\ &\ Web\ Terminal,org.label-schema.schema-version=1.0,org.label-schema.url=https://community.home-assistant.io/t/community-hass-io-add-on-ssh-web-terminal/33820?u\=frenck,org.label-schema.usage=https://github.com/hassio-addons/addon-ssh/tree/master/README.md,org.label-schema.vcs-ref=2047667be0c89a7f22dd416782e2b741bc98075b,org.label-schema.vcs-url=https://github.com/hassio-addons/addon-ssh,org.label-schema.vendor=Community\ Hass.io\ Add-ons,server_version=19.03.1 container_id="99cd435068c8bd4f2bee9a97a0249b0b2c206034a5f8d4be2dae2cc4bf4fec86",exitcode=0i,oomkilled=false,pid=15996i,started_at=1571715443276858027i,uptime_ns=1322080555388222i 1573037524000000000

gotson commented 5 years ago

You can see that most of the negative figures i gave in the first post are still the same in the telegraf output, showing they haven't changed.

danielnelson commented 5 years ago

Let's check what docker has to say about the container, can you run this curl command to grab the raw source data? Make sure to substitute a container_id that is reporting the negative uptime:

curl --unix-socket /var/run/docker.sock 'http://localhost/containers/30a8c55f2427162f16d11728383a80a1b9c88cee64c86d06a75e780447d19c75/json'

gotson commented 5 years ago

Here is the telegraf output (just to make sure it's up to date ):

> docker_container_status,com.docker.compose.config-hash=dafa1d2ee5a7083fff798b61d800fe605106b43553ec905c8cfcbedff22fc20a,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=lsio,com.docker.compose.service=plex,com.docker.compose.version=1.23.2,com.ouroboros.enable=True,container_image=plexinc/pms-docker,container_name=plex,container_status=running,container_version=plexpass,engine_host=leon,host=leon,server_version=19.03.1 container_id="1543aca673015b79ce6cb16e27c5aad6ca33699bbc4949738fa93ef5c2a86d10",exitcode=0i,finished_at=1573056004624365805i,oomkilled=false,pid=6071i,started_at=1573056005099844819i,uptime_ns=-475479014i 1573088182000000000

And the curl output:

{
  "Id": "1543aca673015b79ce6cb16e27c5aad6ca33699bbc4949738fa93ef5c2a86d10",
  "Created": "2019-09-04T14:46:56.32347611Z",
  "Path": "/init",
  "Args": [],
  "State": {
    "Status": "running",
    "Running": true,
    "Paused": false,
    "Restarting": false,
    "OOMKilled": false,
    "Dead": false,
    "Pid": 6071,
    "ExitCode": 0,
    "Error": "",
    "StartedAt": "2019-11-06T16:00:05.099844819Z",
    "FinishedAt": "2019-11-06T16:00:04.624365805Z",
    "Health": {
      "Status": "healthy",
      "FailingStreak": 0,
      "Log": [
        {
          "Start": "2019-11-07T08:34:10.45579616+08:00",
          "End": "2019-11-07T08:34:10.689659949+08:00",
          "ExitCode": 0,
          "Output": ""
        },
        {
          "Start": "2019-11-07T08:37:30.69951039+08:00",
          "End": "2019-11-07T08:37:30.969858721+08:00",
          "ExitCode": 0,
          "Output": ""
        },
        {
          "Start": "2019-11-07T08:40:50.988528916+08:00",
          "End": "2019-11-07T08:40:51.242986708+08:00",
          "ExitCode": 0,
          "Output": ""
        },
        {
          "Start": "2019-11-07T08:44:11.261928719+08:00",
          "End": "2019-11-07T08:44:11.530576021+08:00",
          "ExitCode": 0,
          "Output": ""
        },
        {
          "Start": "2019-11-07T08:47:31.537824342+08:00",
          "End": "2019-11-07T08:47:31.787807519+08:00",
          "ExitCode": 0,
          "Output": ""
        }
      ]
    }
  },
  "Image": "sha256:bffed2f1e11ca465e82ccfab6731c64477e4c0ecc5788180be5e8387548f31ec",
  "ResolvConfPath": "/var/lib/docker/containers/1543aca673015b79ce6cb16e27c5aad6ca33699bbc4949738fa93ef5c2a86d10/resolv.conf",
  "HostnamePath": "/var/lib/docker/containers/1543aca673015b79ce6cb16e27c5aad6ca33699bbc4949738fa93ef5c2a86d10/hostname",
  "HostsPath": "/var/lib/docker/containers/1543aca673015b79ce6cb16e27c5aad6ca33699bbc4949738fa93ef5c2a86d10/hosts",
  "LogPath": "/var/lib/docker/containers/1543aca673015b79ce6cb16e27c5aad6ca33699bbc4949738fa93ef5c2a86d10/1543aca673015b79ce6cb16e27c5aad6ca33699bbc4949738fa93ef5c2a86d10-json.log",
  "Name": "/plex",
  "RestartCount": 0,
  "Driver": "overlay2",
  "Platform": "linux",
  "MountLabel": "",
  "ProcessLabel": "",
  "AppArmorProfile": "docker-default",
  "ExecIDs": null,
  "HostConfig": {
    "Binds": [
      "/etc/timezone:/etc/timezone:ro",
      "/home/groebroeck/opt/appdata/plex:/config:rw",
      "/mnt/nas:/data:rw",
      "71351a260157765e5442e62d30e33d452cf0ac6d8703fbe593666994eaf404d6:/transcode:rw"
    ],
    "ContainerIDFile": "",
    "LogConfig": { "Type": "json-file", "Config": {} },
    "NetworkMode": "host",
    "PortBindings": {},
    "RestartPolicy": { "Name": "unless-stopped", "MaximumRetryCount": 0 },
    "AutoRemove": false,
    "VolumeDriver": "",
    "VolumesFrom": [],
    "CapAdd": null,
    "CapDrop": null,
    "Capabilities": null,
    "Dns": [],
    "DnsOptions": [],
    "DnsSearch": [],
    "ExtraHosts": null,
    "GroupAdd": null,
    "IpcMode": "shareable",
    "Cgroup": "",
    "Links": null,
    "OomScoreAdj": 0,
    "PidMode": "",
    "Privileged": false,
    "PublishAllPorts": false,
    "ReadonlyRootfs": false,
    "SecurityOpt": null,
    "UTSMode": "",
    "UsernsMode": "",
    "ShmSize": 67108864,
    "Runtime": "runc",
    "ConsoleSize": [0, 0],
    "Isolation": "",
    "CpuShares": 0,
    "Memory": 0,
    "NanoCpus": 0,
    "CgroupParent": "",
    "BlkioWeight": 0,
    "BlkioWeightDevice": null,
    "BlkioDeviceReadBps": null,
    "BlkioDeviceWriteBps": null,
    "BlkioDeviceReadIOps": null,
    "BlkioDeviceWriteIOps": null,
    "CpuPeriod": 0,
    "CpuQuota": 0,
    "CpuRealtimePeriod": 0,
    "CpuRealtimeRuntime": 0,
    "CpusetCpus": "",
    "CpusetMems": "",
    "Devices": null,
    "DeviceCgroupRules": null,
    "DeviceRequests": null,
    "KernelMemory": 0,
    "KernelMemoryTCP": 0,
    "MemoryReservation": 0,
    "MemorySwap": 0,
    "MemorySwappiness": null,
    "OomKillDisable": false,
    "PidsLimit": null,
    "Ulimits": null,
    "CpuCount": 0,
    "CpuPercent": 0,
    "IOMaximumIOps": 0,
    "IOMaximumBandwidth": 0,
    "MaskedPaths": [
      "/proc/asound",
      "/proc/acpi",
      "/proc/kcore",
      "/proc/keys",
      "/proc/latency_stats",
      "/proc/timer_list",
      "/proc/timer_stats",
      "/proc/sched_debug",
      "/proc/scsi",
      "/sys/firmware"
    ],
    "ReadonlyPaths": [
      "/proc/bus",
      "/proc/fs",
      "/proc/irq",
      "/proc/sys",
      "/proc/sysrq-trigger"
    ]
  },
  "GraphDriver": {
    "Data": {
      "LowerDir": "/var/lib/docker/overlay2/b4fac527b2f082d88f169254223e5c85685586ab2e23e5ea293b985a16732d49-init/diff:/var/lib/docker/overlay2/3e7aabe3fe8e5d46729d3cfb59351798ab5703913446364557ab294a41c4ac77/diff:/var/lib/docker/overlay2/bccddc92fa08e9d3b17ad328026c0948deb7782f7964422454cf4ea34d0d9cd5/diff:/var/lib/docker/overlay2/cfb47db39b48f7b342fcd11dd50692080a9398fb15c561514f5ae894439dafed/diff:/var/lib/docker/overlay2/f8c8f4e267ab1145284bc2ba4424725a4c8bd70f402922c8cd71ad839331d4cb/diff:/var/lib/docker/overlay2/bcc8552066124f332ea4bc4c87eaa3d2ead9509621935ec83ad98a7714d73a5e/diff:/var/lib/docker/overlay2/6aa48e5f4699d4136bc15b2958b576813acc97a245c27a2c147f80f47dadc5ff/diff:/var/lib/docker/overlay2/d78c3cb14ec6a848245ec2269028b210d69feeac87c6c7a8fbb253fce2f47cb4/diff",
      "MergedDir": "/var/lib/docker/overlay2/b4fac527b2f082d88f169254223e5c85685586ab2e23e5ea293b985a16732d49/merged",
      "UpperDir": "/var/lib/docker/overlay2/b4fac527b2f082d88f169254223e5c85685586ab2e23e5ea293b985a16732d49/diff",
      "WorkDir": "/var/lib/docker/overlay2/b4fac527b2f082d88f169254223e5c85685586ab2e23e5ea293b985a16732d49/work"
    },
    "Name": "overlay2"
  },
  "Mounts": [
    {
      "Type": "bind",
      "Source": "/etc/timezone",
      "Destination": "/etc/timezone",
      "Mode": "ro",
      "RW": false,
      "Propagation": "rprivate"
    },
    {
      "Type": "volume",
      "Name": "71351a260157765e5442e62d30e33d452cf0ac6d8703fbe593666994eaf404d6",
      "Source": "/var/lib/docker/volumes/71351a260157765e5442e62d30e33d452cf0ac6d8703fbe593666994eaf404d6/_data",
      "Destination": "/transcode",
      "Driver": "local",
      "Mode": "rw",
      "RW": true,
      "Propagation": ""
    },
    {
      "Type": "bind",
      "Source": "/home/groebroeck/opt/appdata/plex",
      "Destination": "/config",
      "Mode": "rw",
      "RW": true,
      "Propagation": "rprivate"
    },
    {
      "Type": "bind",
      "Source": "/mnt/nas",
      "Destination": "/data",
      "Mode": "rw",
      "RW": true,
      "Propagation": "rprivate"
    }
  ],
  "Config": {
    "Hostname": "leon",
    "Domainname": "",
    "User": "",
    "AttachStdin": false,
    "AttachStdout": false,
    "AttachStderr": false,
    "ExposedPorts": {
      "1900/udp": {},
      "3005/tcp": {},
      "32400/tcp": {},
      "32410/udp": {},
      "32412/udp": {},
      "32413/udp": {},
      "32414/udp": {},
      "32469/tcp": {},
      "8324/tcp": {}
    },
    "Tty": false,
    "OpenStdin": false,
    "StdinOnce": false,
    "Env": [
      "PLEX_UID=1000",
      "PLEX_GID=1000",
      "affinity:container==e1e46acb3a2c771cf22f9ef5aea9efebed89590555ef9d56abbc84ce4a3bd1a8",
      "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
      "TERM=xterm",
      "LANG=C.UTF-8",
      "LC_ALL=C.UTF-8",
      "CHANGE_CONFIG_DIR_OWNERSHIP=true",
      "HOME=/config"
    ],
    "Cmd": null,
    "Healthcheck": {
      "Test": ["CMD-SHELL", "/healthcheck.sh || exit 1"],
      "Interval": 200000000000,
      "Timeout": 100000000000
    },
    "Image": "plexinc/pms-docker:plexpass",
    "Volumes": {
      "/config": {},
      "/data": {},
      "/etc/timezone": {},
      "/transcode": {}
    },
    "WorkingDir": "",
    "Entrypoint": ["/init"],
    "OnBuild": null,
    "Labels": {
      "com.docker.compose.config-hash": "dafa1d2ee5a7083fff798b61d800fe605106b43553ec905c8cfcbedff22fc20a",
      "com.docker.compose.container-number": "1",
      "com.docker.compose.oneoff": "False",
      "com.docker.compose.project": "lsio",
      "com.docker.compose.service": "plex",
      "com.docker.compose.version": "1.23.2",
      "com.ouroboros.enable": "True"
    }
  },
  "NetworkSettings": {
    "Bridge": "",
    "SandboxID": "8231c2a79edddd7a73db75cf46634e4301538e50cdcfb02abcc911e0e6221158",
    "HairpinMode": false,
    "LinkLocalIPv6Address": "",
    "LinkLocalIPv6PrefixLen": 0,
    "Ports": {},
    "SandboxKey": "/var/run/docker/netns/default",
    "SecondaryIPAddresses": null,
    "SecondaryIPv6Addresses": null,
    "EndpointID": "",
    "Gateway": "",
    "GlobalIPv6Address": "",
    "GlobalIPv6PrefixLen": 0,
    "IPAddress": "",
    "IPPrefixLen": 0,
    "IPv6Gateway": "",
    "MacAddress": "",
    "Networks": {
      "host": {
        "IPAMConfig": null,
        "Links": null,
        "Aliases": null,
        "NetworkID": "39807ac7201d7f0ea1d100f05602cb28b7083d1bef12a9b48bfd23a25975d56c",
        "EndpointID": "b3d8b744be22feca2800f562e00ea04cb8b28334b54d3f3c076d071ea8f000a7",
        "Gateway": "",
        "IPAddress": "",
        "IPPrefixLen": 0,
        "IPv6Gateway": "",
        "GlobalIPv6Address": "",
        "GlobalIPv6PrefixLen": 0,
        "MacAddress": "",
        "DriverOpts": null
      }
    }
  }
}

danielnelson commented 5 years ago

I think I know the issue. The values we are using for uptime are:

"StartedAt": "2019-11-06T16:00:05.099844819Z",
"FinishedAt": "2019-11-06T16:00:04.624365805Z",

Normally, I see this for a running container. In the code we check if the FinishedAt is this particular value and if so we ignore it and instead use the current time when calculating the time difference.

"StartedAt": "2019-11-07T01:22:44.564196479Z",
"FinishedAt": "0001-01-01T00:00:00Z"

It looks like if we docker restart the plugin though then we reproduce your values:

"StartedAt": "2019-11-07T01:25:10.660975449Z",
"FinishedAt": "2019-11-07T01:25:10.372043515Z"

For the fix I think we just need to ignore the FinishedAt time when the container is not exited.

influxdata / telegraf