TykTechnologies / tyk

Tyk Open Source API Gateway written in Go, supporting REST, GraphQL, TCP and gRPC protocols
Other
9.65k stars 1.08k forks source link

K/V: Gateway crashes if k/v store is down or unavailable #3089

Closed ilijabojanovic closed 4 years ago

ilijabojanovic commented 4 years ago

Branch/Environment/Version

Describe the bug When api definition contains some value from k/v store during proxy gateway will stop.

Reproduction steps

  1. Enable Vault k/v store
  2. In api definition add dynamic value in global header e.g $secret_vault.secret/test.api
  3. Send traffic to that api

Actual behavior Gateway will stop during proxy request

Expected behavior If there is no connection to k/v store we should proxy request without that data

Screenshots/Video If applicable, add screenshots or video to help explain your problem.

Logs (debug mode or log file): Log from console or from log file.

Configuration (tyk config file): Attach tyk configuration file

Additional context Add any other context about the problem here.

ilijabojanovic commented 4 years ago

@adelowo

On release-2.10 branch

I was able to reproduce issue:

May 26 12:03:14]  INFO auth-mgr: Reset quota for key. inbound-key=eyJvcmciOiI1ZTQxM2Y4MGNjZTlmMzA4MDkyMWEwZGQiLCJpZCI6ImMxMGY1MTY4N2QyMzQ0NmE5NGViNWNmNWEzZTU5MjFmIiwiaCI6Im11cm11cjY0In0= key=quota-118543918d85b7b5
[May 26 12:03:14]  INFO api: Generated new key: (eyJvcmciOiI1ZTQxM2Y4MGNjZTlmMzA4MDkyMWEwZGQiLCJpZCI6ImMxMGY1MTY4N2QyMzQ0NmE5NGViNWNmNWEzZTU5MjFmIiwiaCI6Im11cm11cjY0In0=) api_id=-- key=eyJvcmciOiI1ZTQxM2Y4MGNjZTlmMzA4MDkyMWEwZGQiLCJpZCI6ImMxMGY1MTY4N2QyMzQ0NmE5NGViNWNmNWEzZTU5MjFmIiwiaCI6Im11cm11cjY0In0= org_id=5e413f80cce9f3080921a0dd path=-- server_name=system status=ok user_id=system user_ip=127.0.0.1
[May 26 12:03:14]  INFO api: Retrieved key detail. key=eyJvcmciOiI1ZTQxM2Y4MGNjZTlmMzA4MDkyMWEwZGQiLCJpZCI6ImMxMGY1MTY4N2QyMzQ0NmE5NGViNWNmNWEzZTU5MjFmIiwiaCI6Im11cm11cjY0In0= status=ok
[May 26 12:03:19] FATAL an error occurred while setting up vault... you must provide a root token in other to use vault

Here is my gateway conf:

{
    "listen_address": "",
    "listen_port": 8181,
    "secret": "352d20ee67be67f6340b4c0605b044b7",
    "node_secret": "352d20ee67be67f6340b4c0605b044b7",
    "template_path": "templates",
    "tyk_js_path": "./js",
    "middleware_path": "middleware",
    "policies": {
        "policy_source": "service",
        "policy_connection_string": "http://www.tyk-test.com:3000",
        "policy_record_name": "tyk_policies",
        "allow_explicit_policy_id": false
    },
    "use_db_app_configs": true,
    "db_app_conf_options": {
        "connection_string": "http://www.tyk-test.com:3000",
        "node_is_segmented": false,
        "tags": null
    },
    "disable_dashboard_zeroconf": false,
    "app_path": "apps/",
    "storage": {
        "type": "redis",
        "host": "tyk-redis",
        "port": 6379,
        "hosts": {},
        "username": "",
        "password": "",
        "database": 0,
        "timeout": 5,
        "optimisation_max_idle": 2000,
        "optimisation_max_active": 4000,
        "enable_cluster": false,
        "use_ssl": false,
        "ssl_insecure_skip_verify": false
    },
    "enable_separate_cache_store": false,
    "cache_storage": {
        "type": "redis",
        "host": "tyk-redis",
        "port": 6379,
        "hosts": null,
        "username": "",
        "password": "",
        "database": 0,
        "timeout": 5,
        "optimisation_max_idle": 0,
        "optimisation_max_active": 0,
        "enable_cluster": true,
        "use_ssl": false,
        "ssl_insecure_skip_verify": false
    },
    "enable_analytics": true,
    "analytics_config": {
        "type": "mongo",
        "ignored_ips": [],
        "enable_detailed_recording": true,
        "enable_geo_ip": true,
        "geo_ip_db_path": "./GeoLite2-City.mmdb",
        "normalise_urls": {
            "enabled": true,
            "normalise_uuids": true,
            "normalise_numbers": true,
            "custom_patterns": null
        },
        "pool_size": 0,
        "records_buffer_size": 0,
        "storage_expiration_time": 0
    },
    "health_check": {
        "enable_health_checks": false,
        "health_check_value_timeouts": 0
    },
    "optimisations_use_async_session_write": false,
    "session_update_pool_size": 0,
    "session_update_buffer_size": 0,
    "allow_master_keys": false,
    "hash_keys": true,
    "hash_key_function": "murmur64",
    "suppress_redis_signal_reload": false,
    "suppress_default_org_store": false,
    "use_redis_log": true,
    "sentry_code": "",
    "use_sentry": false,
    "use_syslog": false,
    "use_graylog": false,
    "use_logstash": false,
    "graylog_network_addr": "",
    "logstash_network_addr": "",
    "syslog_transport": "",
    "logstash_transport": "",
    "syslog_network_addr": "",
    "statsd_connection_string": "localhost:8126",
    "statsd_prefix": "tyk-gateway",
    "enforce_org_data_age": false,
    "enforce_org_data_detail_logging": false,
    "enforce_org_quotas": true,
    "experimental_process_org_off_thread": true,
    "enable_non_transactional_rate_limiter": true,
    "enable_sentinel_rate_limiter": false,
    "enable_redis_rolling_limiter": false,
    "management_node": false,
    "monitor": {
        "enable_trigger_monitors": true,
    "configuration": {
          "method": "POST",
          "target_path": "https://webhook.site/f7701316-3403-44c8-b4bf-d7020dc10a57",
          "template_path": "templates/monitor_template.json",
          "header_map": {"x-tyk-monitor-secret": "12345"},
          "event_timeout": 10
      },
        "global_trigger_limit": 80,
        "monitor_user_keys": true,
        "monitor_org_keys": true
    },
    "oauth_refresh_token_expire": 0,
    "oauth_token_expire": 0,
    "oauth_token_expired_retain_period": 0,
    "oauth_redirect_uri_separator": "",
    "slave_options": {
        "use_rpc": false,
        "use_ssl": false,
        "ssl_insecure_skip_verify": false,
        "connection_string": "",
        "rpc_key": "",
        "api_key": "",
        "enable_rpc_cache": false,
        "bind_to_slugs": false,
        "disable_keyspace_sync": false,
        "group_id": "",
        "call_timeout": 0,
        "ping_timeout": 0,
        "rpc_pool_size": 0
    },
    "disable_virtual_path_blobs": false,
    "local_session_cache": {
        "disable_cached_session_state": false,
        "cached_session_timeout": 10,
        "cached_session_eviction": 0
    },
    "http_server_options": {
        "override_defaults": false,
        "read_timeout": 0,
        "write_timeout": 0,
        "use_ssl": true,
        "use_ssl_le": false,
        "ssl_insecure_skip_verify": false,
        "enable_websockets": true,
        "certificates": [
            {
                "domain_name": "*.tyk-test.com",
                "cert_file": "./server.crt",
                "key_file": "./server.key"
            }
        ],
        "ssl_certificates": null,
        "server_name": "www.tyk-test.com",
        "min_version": 0,
        "flush_interval": 0,
        "skip_url_cleaning": true,
        "skip_target_path_escaping": true,
        "ssl_ciphers": null
    },
    "service_discovery": {
        "default_cache_timeout": 0
    },
    "proxy_close_connections": false,
    "close_connections": true,
    "auth_override": {
        "force_auth_provider": false,
        "auth_provider": {
            "name": "",
            "storage_engine": "",
            "meta": null
        },
        "force_session_provider": false,
        "session_provider": {
            "name": "",
            "storage_engine": "",
            "meta": null
        }
    },
    "uptime_tests": {
        "disable": false,
        "config": {
            "failure_trigger_sample_size": 2,
            "time_wait": 5,
            "checker_pool_size": 50,
            "enable_uptime_analytics": true
        }
    },
    "hostname": "",
    "enable_api_segregation": false,
    "control_api_hostname": "",
    "control_api_port": 0,
    "enable_custom_domains": true,
    "enable_jsvm": true,
    "jsvm_timeout": 10,
    "coprocess_options": {
        "enable_coprocess": true,
        "coprocess_grpc_server": "",
        "python_path_prefix": "/Users/ilijabojanovic/work/src/github.com/TykTechnologies/tyk"
    },
    "hide_generator_header": false,
    "event_handlers": {},
    "event_trigers_defunct": null,
    "pid_file_location": "",
    "allow_insecure_configs": true,
    "close_idle_connections": false,
    "drl_notification_frequency": 0,
    "global_session_lifetime": 0,
    "force_global_session_lifetime": false,
    "bundle_base_url": "http://192.168.1.7:8000/",
    "enable_bundle_downloader": true,
    "allow_remote_config": true,
    "legacy_enable_allowance_countdown": false,
    "max_idle_connections": 400,
    "max_idle_connections_per_host": 0,
    "max_conn_time": 0,
    "reload_wait_time": 0,
    "proxy_ssl_insecure_skip_verify": false,
    "proxy_ssl_min_version": 0,
    "proxy_ssl_ciphers": null,
    "proxy_default_timeout": 10,
    "proxy_ssl_disable_renegotiation": false,
    "log_level": "info",
    "enable_websockets": true,
    "public_key_path": "/Users/ilijabojanovic/work/src/github.com/TykTechnologies/tyk/.ssh/pubkey.pem",
    "security": {
        "private_certificate_encoding_secret": "",
        "control_api_use_mutual_tls": false,
        "pinned_public_keys": null,
        "certificates": {
            "apis": null,
            "upstream": null,
            "control_api": null,
            "dashboard_api": null,
            "mdcb_api": null
        }
    },
    "enable_key_logging": true,
    "newrelic": {
        "app_name": "",
        "license_key": ""
    },
    "health_check_endpoint_name": "ilija",
    "track_404_logs": true,
    "version_header": "",
    "enable_hashed_keys_listing": true,
    "min_token_length": 0,
    "disable_regexp_cache": false,
    "regexp_cache_expire": 0,
    "ports_whitelist": {
        "http": {
          "ranges": [
            {
              "from": 8000,
              "to": 9000
            }
          ]
        },
        "tcp": {
          "ranges": [
            {
              "from": 7001,
              "to": 7900
            }
          ]
        },
        "tls": {
          "ports": [
            6000,
            6015
          ]
        }
      }
    },
    "dns_cache": {
        "enabled": true,
        "ttl": 3600,
        "multiple_ips_handle_strategy": "no_cache"
    }
}
adelowo commented 4 years ago

I will take a look on 2.10 branch again.

ilijabojanovic commented 4 years ago

Verified