search

varnishcache / varnish-cache

Varnish Cache source code repository
https://www.varnish-cache.org
Other
3.68k stars 377 forks source link

Assert error in h2_del_req(), http2/cache_http2_proto.c line 175 #2591

Closed simonvik closed 6 years ago

simonvik commented 6 years ago 
Panic at: Wed, 28 Feb 2018 19:43:36 GMT
Assert error in h2_del_req(), http2/cache_http2_proto.c line 175:
  Condition(h2->rxthr == pthread_self()) not true.
version = varnish-trunk revision 60b699b91581e2d2f46bddf83ff4b42f28bac930, vrt api = 6.2
ident = Linux,4.4.30-20161130-1423-0a2a05d,x86_64,-junix,-smalloc,-sdefault,-hcritbit,epoll
now = 34663112.017212 (mono), 1519847001.652218 (real)
Backtrace:
  0x438dbb: /usr/sbin/varnishd() [0x438dbb]
  0x497da0: /usr/sbin/varnishd(VAS_Fail+0x40) [0x497da0]
  0x4644c5: /usr/sbin/varnishd(h2_del_req+0x1d5) [0x4644c5]
  0x467ca8: /usr/sbin/varnishd() [0x467ca8]
  0x45410f: /usr/sbin/varnishd() [0x45410f]
  0x454570: /usr/sbin/varnishd() [0x454570]
  0x7fa676d986ba: /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7fa676d986ba]
  0x7fa676ace41d: /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d) [0x7fa676ace41d]
thread = (cache-worker)
thr.req = 0x7f95d0311020 {
  vxid = 720601103, transport = H2
  step = 0x0,
  req_body = R_BODY_INIT,
  err_code = 1, err_reason = (null),
  restarts = 0, esi_level = 0,
  sp = 0x7fa5fd968820 {
    fd = 1793, vxid = 720601103,
    t_open = 1519847000.027619,
    t_idle = 1519847001.652020,
    ws = 0x7fa5fd968860 {
      id = \"ses\",
      {s, f, r, e} = {0x7fa5fd968898, +192, (nil), +352},
    },
    transport = H2 {
      streams {
        0x00000000 idle  
        0x00000043 c-rem 
        0x00000045 c-rem 
      }
    }
    client = ***.***.***.*** 59579 127.0.0.1:81,
  },
  ws = 0x7f95d03111c0 {
    id = \"req\",
    {s, f, r, e} = {0x7f95d0313030, +624, (nil), +122824},
  },
  http_conn = 0x7f95d0311130 {
    fd = 1793 (@0x7fa5fd968844),
    doclose = NULL,
    ws = 0x7f95d03111c0 {
      [Already dumped, see above]
    },
    {rxbuf_b, rxbuf_e} = {0x7f95d03132a0, 0x7f95d03132a0},
    {pipeline_b, pipeline_e} = {(nil), (nil)},
    content_length = 0,
    body_status = none,
    first_byte_timeout = 0.000000,
    between_bytes_timeout = 0.000000,
  },
  http[req] = 0x7f95d0311260 {
    ws = (nil) {
    },
    hdrs {
    },
  },
  vmods = {
    std = {Varnish trunk 60b699b91581e2d2f46bddf83ff4b42f28bac930, 0.0},
    one = {Varnish trunk 60b699b91581e2d2f46bddf83ff4b42f28bac930, 0.0},
    maxminddb = {Varnish trunk 60b699b91581e2d2f46bddf83ff4b42f28bac930, 0.0},
    directors = {Varnish trunk 60b699b91581e2d2f46bddf83ff4b42f28bac930, 0.0},
    xkey = {Varnish trunk 60b699b91581e2d2f46bddf83ff4b42f28bac930, 0.0},
    header = {Varnish trunk 60b699b91581e2d2f46bddf83ff4b42f28bac930, 6.2},
    cookie = {Varnish trunk 60b699b91581e2d2f46bddf83ff4b42f28bac930, 6.2},
    vsthrottle = {Varnish trunk 60b699b91581e2d2f46bddf83ff4b42f28bac930, 6.2},
    tcp = {Varnish trunk 60b699b91581e2d2f46bddf83ff4b42f28bac930, 0.0},
    bodyaccess = {Varnish trunk 60b699b91581e2d2f46bddf83ff4b42f28bac930, 0.0},
    blob = {Varnish trunk 60b699b91581e2d2f46bddf83ff4b42f28bac930, 0.0},
  },
  flags = {
  },
  privs = 0x7f95d0311248 {
  },
},
thr.busyobj = (nil) {
},

Don't know if this is a duplicate of #2563, close it in that case. Varnish built from commit 60b699b91581e2d2f46bddf83ff4b42f28bac930 on ubuntu 16.04

nm -an /usr/sbin/varnishd : nm_60b699b91581e2d2f46bddf83ff4b42f28bac930.txt

simonvik commented 6 years ago

Got two new asserts close to this on on bd844c3511d76b4f04a945a463c5a83047f17c26. nm -an /usr/sbin/varnishd : nm_bd844c3511d76b4f04a945a463c5a83047f17c26.txt

 Panic at: Wed, 14 Mar 2018 01:18:00 GMT
Assert error in h2_del_req(), http2/cache_http2_proto.c line 173:
  Condition(h2->rxthr == pthread_self()) not true.
version = varnish-trunk revision bd844c3511d76b4f04a945a463c5a83047f17c26, vrt api = 7.0
ident = Linux,4.4.30-20161130-1423-0a2a05d,x86_64,-junix,-smalloc,-sdefault,-hcritbit,epoll
now = 30899662.037118 (mono), 1520990268.295802 (real)
Backtrace:
  0x438f38: /usr/sbin/varnishd() [0x438f38]
  0x495640: /usr/sbin/varnishd(VAS_Fail+0x40) [0x495640]
  0x465703: /usr/sbin/varnishd(h2_del_req+0x143) [0x465703]
  0x466d93: /usr/sbin/varnishd(h2_rxframe+0x593) [0x466d93]
  0x468f8d: /usr/sbin/varnishd() [0x468f8d]
  0x45533f: /usr/sbin/varnishd() [0x45533f]
  0x4557a0: /usr/sbin/varnishd() [0x4557a0]
  0x7f6d92dad6ba: /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7f6d92dad6ba]
  0x7f6d92ae341d: /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d) [0x7f6d92ae341d]
thread = (cache-worker)
thr.req = 0x7f6b46082020 {
  vxid = 813564361, transport = H2
  step = 0x0,
  req_body = R_BODY_INIT,
  err_code = 1, err_reason = (null),
  restarts = 0, esi_level = 0,
  sp = 0x7f59e6e75820 {
    fd = 8797, vxid = 813564361,
    t_open = 1520990230.819031,
    t_idle = 1520990263.291833,
    ws = 0x7f59e6e75860 {
      id = \"ses\",
      {s, f, r, e} = {0x7f59e6e75898, +192, (nil), +352},
    },
    transport = H2 {
      streams {
        0x00000000 idle  
        0x0000005f closed
        0x00000003 c-rem 
      }
    }
    client = *********** 19557 127.0.0.1:81,
  },
  ws = 0x7f6b46082168 {
    id = \"req\",
    {s, f, r, e} = {0x7f6b46084098, +576, (nil), +122720},
  },
  http_conn = 0x7f6b46084038 {
    fd = 8797 (@0x7f59e6e75844),
    doclose = NULL,
    ws = 0x7f6b46082168 {
      [Already dumped, see above]
    },
    {rxbuf_b, rxbuf_e} = {0x7f6b460842d8, 0x7f6b460842d8},
    {pipeline_b, pipeline_e} = {(nil), (nil)},
    content_length = 0,
    body_status = none,
    first_byte_timeout = 0.000000,
    between_bytes_timeout = 0.000000,
  },
  http[req] = 0x7f6b46082208 {
    ws = (nil) {
    },
    hdrs {
    },
  },
  vmods = {
    std = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 0.0},
    one = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 0.0},
    maxminddb = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 0.0},
    directors = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 0.0},
    xkey = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 0.0},
    header = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 7.0},
    cookie = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 7.0},
    vsthrottle = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 7.0},
    tcp = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 0.0},
    bodyaccess = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 0.0},
    blob = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 0.0},
  },
  flags = {
  },
  privs = 0x7f6b460821f0 {
  },
},
thr.busyobj = (nil) {
},
Panic at: Wed, 14 Mar 2018 02:17:01 GMT
Assert error in h2_del_req(), http2/cache_http2_proto.c line 169:
  Condition((r2)->magic == 0x03411584) not true.
version = varnish-trunk revision bd844c3511d76b4f04a945a463c5a83047f17c26, vrt api = 7.0
ident = Linux,4.4.30-20161130-1423-0a2a05d,x86_64,-junix,-smalloc,-sdefault,-hcritbit,epoll
now = 30901881.481166 (mono), 1520993820.716943 (real)
Backtrace:
  0x438f38: /usr/sbin/varnishd() [0x438f38]
  0x495640: /usr/sbin/varnishd(VAS_Fail+0x40) [0x495640]
  0x46577f: /usr/sbin/varnishd(h2_del_req+0x1bf) [0x46577f]
  0x469218: /usr/sbin/varnishd() [0x469218]
  0x45533f: /usr/sbin/varnishd() [0x45533f]
  0x4557a0: /usr/sbin/varnishd() [0x4557a0]
  0x7f13ffcbb6ba: /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7f13ffcbb6ba]
  0x7f13ff9f141d: /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d) [0x7f13ff9f141d]
thread = (cache-worker)
thr.req = 0x7f1324d06020 {
  vxid = 194019365, transport = H2
  step = 0x0,
  req_body = R_BODY_INIT,
  err_code = 1, err_reason = (null),
  restarts = 0, esi_level = 0,
  sp = 0x7f13c1501420 {
    fd = 1839, vxid = 194019365,
    t_open = 1520993815.405180,
    t_idle = 1520993815.714666,
    ws = 0x7f13c1501460 {
      id = \"ses\",
      {s, f, r, e} = {0x7f13c1501498, +192, (nil), +352},
    },
    transport = H2 {
      streams {
        0x00000000 idle  
        MAGIC at 0x7f1203d15098 is 0x00000000 (Should be: H2_REQ_MAGIC/0x03411584)
        0x00000000 idle  
      }
    }
    client = ********** 52254 127.0.0.1:81,
  },
  ws = 0x7f1324d06168 {
    id = \"req\",
    {s, f, r, e} = {0x7f1324d08098, +496, (nil), +122720},
  },
  http_conn = 0x7f1324d08038 {
    fd = 1839 (@0x7f13c1501444),
    doclose = NULL,
    ws = 0x7f1324d06168 {
      [Already dumped, see above]
    },
    {rxbuf_b, rxbuf_e} = {0x7f1324d08288, 0x7f1324d08288},
    {pipeline_b, pipeline_e} = {(nil), (nil)},
    content_length = 0,
    body_status = none,
    first_byte_timeout = 0.000000,
    between_bytes_timeout = 0.000000,
  },
  http[req] = 0x7f1324d06208 {
    ws = (nil) {
    },
    hdrs {
    },
  },
  vmods = {
    std = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 0.0},
    one = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 0.0},
    maxminddb = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 0.0},
    directors = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 0.0},
    xkey = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 0.0},
    header = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 7.0},
    cookie = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 7.0},
    vsthrottle = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 7.0},
    tcp = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 0.0},
    bodyaccess = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 0.0},
    blob = {Varnish trunk bd844c3511d76b4f04a945a463c5a83047f17c26, 0.0},
  },
  flags = {
  },
  privs = 0x7f1324d061f0 {
  },
},
thr.busyobj = (nil) {
},
simonvik commented 6 years ago

I havent seen this since 70d49d5997503fcc91a693723993fbe06806b56b

fgsch commented 6 years ago

@daghf to update.

daghf commented 6 years ago

Fixed via https://github.com/varnishcache/varnish-cache/commit/70d49d5997503fcc91a693723993fbe06806b56b