Huge latency on publish/subscribe

[ZinkevychSergii]

After nchan running for a few days at our dev env we are facing to publish latency, about 20 sec, and it's growing. Also even not able to subscribe to channel. When open at browser, i can see at network tab that request at pending state. Nchan is behind nginx-ingress. Checked logs of nchan and nginx-ingress, there is no line that i tried to access data by subscribe url. Only restart helps

Here is stats total published messages: 828 stored messages: 0 shared memory used: 56K shared memory limit: 131072K channels: 1 subscribers: 4 redis pending commands: 0 redis connected servers: 4 total interprocess alerts received: 67497 interprocess alerts in transit: 239 interprocess queued alerts: 239 total interprocess send delay: 0 total interprocess receive delay: 0 nchan version: 1.2.6

Active connections: 15 server accepts handled requests 1812 1812 11314 Reading: 0 Writing: 14 Waiting: 1

[slact]

Anything in the error logs? I also need to see your config (edit out whatever you need to keep private).

there is no line that i tried to access data by subscribe url

that's normal, the request is logged when it is completed with a response. In our case, this doesn't happen until the subscriber disconnects. Think of this quirk as the price of having Nchan be a part of Nginx.

[ZinkevychSergii]

nginx.conf

user  nginx;
worker_processes  1;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;

load_module "modules/ngx_nchan_module.so";

events {
    accept_mutex on;
    worker_connections 10000;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    keepalive_timeout  120;

    #gzip  on;

    include /etc/nginx/conf.d/*.conf
}

default.conf

upstream redis_cluster {
  nchan_redis_server redis://:@******:6379;
  # you don't need to specify all the nodes, they will be autodiscovered
  # however, it's recommended that you do specify at least a few master nodes.
  nchan_redis_storage_mode distributed;
  nchan_redis_nostore_fastpublish on;
}

server {
  listen 80;
  server_name nchan;

  location = /auth {
    proxy_pass https://*******;
    proxy_pass_request_body off;
    proxy_set_header Content-Length "";
    proxy_set_header X-Subscriber-Type $nchan_subscriber_type;
    proxy_set_header X-Publisher-Type $nchan_publisher_type;
    proxy_set_header X-Prev-Message-Id $nchan_prev_message_id;
    proxy_set_header X-Channel-Id $nchan_channel_id;
    proxy_set_header X-Original-URI $request_uri;
    proxy_set_header X-Forwarded-For $remote_addr;
  }

  location ~ /pub/(\w+)$ {
    #publisher endpoint
    nchan_channel_id $1;
    nchan_publisher;
    nchan_authorize_request /auth;
    nchan_channel_group nchan;
    nchan_message_buffer_length $arg_max_msg;
    nchan_message_timeout $arg_ttl;
    nchan_redis_pass redis_cluster;
    #nchan_publisher_upstream_request /upstream/pub;
  }

  location ~ /nchan/sub/user/(\w+)$ {
    nchan_channel_id $1;
    nchan_subscriber;
    nchan_subscribe_request /upstream/sub;
    #nchan_unsubscribe_request /upstream/unsub;
    nchan_message_buffer_length 1000;
    nchan_subscribe_existing_channels_only on;
    nchan_subscriber_first_message 1;
    nchan_channel_group nchan;
    nchan_redis_pass redis_cluster;
  }

  location ~ /nchan/sub/shared/(\w+)$ {
    nchan_channel_id $1;
    nchan_subscriber;
    nchan_subscribe_request /upstream/sub;
    #nchan_unsubscribe_request /upstream/unsub;
    nchan_message_buffer_length 1000;
    nchan_subscribe_existing_channels_only on;
    nchan_subscriber_first_message 1;
    nchan_channel_group nchan;
    nchan_redis_pass redis_cluster;
  }

  location = /nchan/nchan_stub_status {
    nchan_stub_status;
  }

  location /nchan/nginx_stub_status {
  stub_status   on;
  access_log    off;
}

  location = /upstream/pub {
    proxy_pass https://********;
    proxy_set_header X-Publisher-Type $nchan_publisher_type;
    proxy_set_header X-Prev-Message-Id $nchan_prev_message_id;
    proxy_set_header X-Channel-Id $nchan_channel_id;
    proxy_set_header X-Original-URI $request_uri;
  }

  location = /upstream/unsub {
    internal;
    proxy_ignore_client_abort on;  #!!!important!!!!
    proxy_set_header X-Subscriber-Type $nchan_subscriber_type;
    proxy_set_header X-Message-Id $nchan_message_id;
    proxy_set_header X-Channel-Id $nchan_channel_id;
    proxy_set_header X-Original-URI $request_uri;
    proxy_pass https://********;
  }

  location = /upstream/sub {
    internal;
    proxy_set_header X-Subscriber-Type $nchan_subscriber_type;
    proxy_set_header X-Message-Id $nchan_message_id;
    proxy_set_header X-Channel-Id $nchan_channel_id;
    proxy_set_header X-Original-URI $request_uri;
    proxy_pass https://*****;
  }
}

[slact]

What about the error log?

[ZinkevychSergii]

as i have restarted pods, all logs missed. will post as soon as get it