Hybrid gateway not reloading on new events

[ilijabojanovic]

Branch/Environment/Version

• Branch/Version: Release-2.9
• Environment: Hybrid

Describe the bug When we update api definition, policy or token, hybrid gateway will not reload on mdcb events.

Reproduction steps Steps to reproduce the behavior:

1. Start mdcb env with one hybrid gateway
2. On Dashboard update api or policy

Actual behavior MDCB registers and sends reload events, but hybrid gateway not doing reload

Expected behavior Hybrid gateway should reload and pick up propagated changes

Screenshots/Video https://screencast.com/t/1C79emLQNH

Logs (debug mode or log file): MDCB:

NFO[1879] Policies found: 7
WARN[1884] Restart signal (redis) received, Checking
INFO[1884] ---- Reload added to group (aaa-reload.5d84baee9633d70782364f08.ungrouped) backlog: 2019-09-26 10:51:37.068609 +0200 CEST m=+1884.444877988
INFO[1889] Ping!
INFO[1889] Org/Group (aaa-reload.5d84baee9633d70782364f08.ungrouped) has 1 reload events
INFO[1889] Policies found: 7
INFO[1899] Ping!
INFO[1899] Forwarding analytics to Tyk Pump
INFO[1909] Ping!
INFO[1919] Ping!
INFO[1919] Forwarding analytics to Tyk Pump
INFO[1929] Ping!
INFO[1939] Ping!
WARN[1943] Restart signal (redis) received, Checking
INFO[1943] ---- Reload added to group (aaa-reload.5d84baee9633d70782364f08.ungrouped) backlog: 2019-09-26 10:52:36.193529 +0200 CEST m=+1943.570148607
INFO[1949] Ping!

Configuration (tyk config file):

{
    "listen_port": 8080,
    "secret": "352d20ee67be67f6340b4c0605b044b7",
    "node_secret": "",
    "template_path": "./templates",
    "middleware_path": "./middleware",
    "use_db_app_configs": false,
    "app_path": "./test_apps/",
        "storage": {
        "type": "redis",
        "host": "localhost",
        "port": 6380,
        "hosts": {},
        "username": "",
        "password": "",
        "database": 0,
        "optimisation_max_idle": 3000,
        "optimisation_max_active": 5000,
        "enable_cluster": false
    },
    "enable_analytics": true,
    "analytics_config": {
        "type": "rpc",
        "csv_dir": "/tmp",
        "mongo_url": "localhost",
        "mongo_db_name": "tyk_analytics",
        "mongo_collection": "tyk_analytics",
        "purge_delay": 10,
        "enable_detailed_recording": true,
        "ignored_ips": [],
        "enable_geo_ip": true,
        "geo_ip_db_path": "./GeoLite2-City.mmdb",
        "normalise_urls": {
            "enabled": true,
            "normalise_uuids": true,
            "normalise_numbers": true,
            "custom_patterns": []
        }
    },
    "health_check": {
        "enable_health_checks": false,
        "health_check_value_timeouts": 60
    },
    "optimisations_use_async_session_write": true,
    "allow_master_keys": false,
    "policies": {
      "policy_source": "rpc",
      "policy_record_name": "tyk_policies"
    },
    "hash_keys": true,
    "hash_key_function": "murmur64",
    "suppress_redis_signal_reload": false,
    "use_sentry": false,
    "sentry_code": "",
    "enforce_org_data_age": true,
    "http_server_options": {
        "enable_websockets": true
    },
    "monitor": {
        "enable_trigger_monitors": true,
        "configuration": {
        "method": "POST",
            "target_path": "http://cloud.tyk.io/1337/tyk/webhook",
            "template_path": "templates/monitor_template.json",
            "header_map": {"x-tyk-monitor-secret": "sjdkfhjKHKJHkjsdhsufdudfhjHKIHJ1"},
            "event_timeout": 10
        },
        "global_trigger_limit": 80.0,
        "monitor_user_keys": false,
        "monitor_org_keys": true
    },
    "slave_options": {
        "use_rpc": true,
        "rpc_key": "5d84baee9633d70782364f08",
        "api_key": "8ee599501ee54e2563845a7eee1dc7fd",
        "connection_string": "localhost:9090",
        "enable_rpc_cache": false,
        "use_ssl": false,
        "ssl_insecure_skip_verify": true,
        "bind_to_slugs": true,
        "call_timeout": 2
    },
    "local_session_cache": {
        "disable_cached_session_state": false,
        "cached_session_timeout": 5,
        "cached_session_eviction": 10
    },
    "enforce_org_quotas": false,
    "experimental_process_org_off_thread": true,
    "enable_non_transactional_rate_limiter": true,
    "enable_sentinel_rate_limiter": false,
    "local_session_cache": {
        "disable_cached_session_state": false
        },
    "auth_override": {
        "force_auth_provider": true,
        "auth_provider": {
            "name": "",
            "storage_engine": "rpc",
            "meta": {}
        }
    },
    "http_server_options": {
        "skip_url_cleaning": true,
        "override_defaults": true,
        "read_timeout": 0,
        "ssl_insecure_skip_verify": true,
        "write_timeout": 0,
        "use_ssl": false,
        "use_ssl_le": false,
        "enable_websockets": true,
        "certificates": [
        {
            "domain_name": "*.tyk-test.com",
            "cert_file": "./server.crt",
            "key_file": "./server.key"
                }
        ],
        "server_name": "",
        "min_version": 0,
        "flush_interval": 0
    },
    "enable_context_vars": true,
    "hostname": "",
    "enable_api_segregation": false,
    "control_api_hostname": "",
    "enable_custom_domains": true,
    "enable_jsvm": false,
    "coprocess_options": {
        "enable_coprocess": true
    },
    "enable_bundle_downloader": true,
    "bundle_base_url": "http://192.168.1.50:8000/",
    "hide_generator_header": false,
    "event_handlers": {
        "events": {}
    },
    "pid_file_location": "./tyk-gateway.pid",
    "allow_insecure_configs": true,
    "public_key_path": "",
    "log_level": "info",
    "allow_remote_config": false,
    "enable_bundle_downloader": true,
    "service_discovery": {
        "default_cache_timeout": 20
    },
    "close_idle_connections": true,
    "enable_key_logging": false,
    "close_connections": false,
    "max_idle_connections_per_host": 400,
    "disable_dashboard_zeroconf": true,
    "uptime_tests": {
        "disable": false,
        "config": {
            "failure_trigger_sample_size": 1,
            "time_wait": 7,
            "checker_pool_size": 50,
            "enable_uptime_analytics": false
            }
    },
    "proxy_ssl_insecure_skip_verify": true
}

Additional context Add any other context about the problem here.

[buger]

Can't replicate all work as expected.

[ilijabojanovic]

@buger issue is still visible

[buger]

Fixed by recent sink update