Kong random 404 after worker restart

We’re running 3 Kong instances in Mesos in front of Stolon. After upgrading to Kong 2.8.1 (from 1.5), we’re seeing a lot of 404’s from Kong, however not always: roughly 50% are good. I have no clue however as to why.

From the Kong logs it seems to start with a SIGKILL (due to memory limits) after which a new worker is launched and for whatever reason, this worker starts to return 404’s.

Aug 1, 2022 @ 12:57:21.118	2022/08/01 12:57:12 [info] 42#0: *312090 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:57:32.612	2022/08/01 12:57:29 [info] 33#0: *311135 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:57:35.076	2022/08/01 12:57:28 [info] 34#0: *311185 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:57:36.119	2022/08/01 12:57:32 [info] 42#0: *312392 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:57:50.076	2022/08/01 12:57:48 [info] 34#0: *311488 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:57:57.612	2022/08/01 12:57:49 [info] 33#0: *311444 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:58:01.119	2022/08/01 12:57:52 [info] 42#0: *312715 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:58:12.613	2022/08/01 12:58:09 [info] 33#0: *311742 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:58:15.077	2022/08/01 12:58:11 [notice] 29#0: signal 29 (SIGIO) received
	Aug 1, 2022 @ 12:58:15.077	2022/08/01 12:58:11 [alert] 29#0: worker process 34 exited on signal 9
	Aug 1, 2022 @ 12:58:15.077	2022/08/01 12:58:11 [info] 24161#0: *311846 [kong] handler.lua:57 [acme] acme renew timer started on worker 0, context: init_worker_by_lua*
	Aug 1, 2022 @ 12:58:15.077	2022/08/01 12:58:11 [notice] 29#0: start worker process 24161
	Aug 1, 2022 @ 12:58:15.077	2022/08/01 12:58:11 [notice] 24161#0: *311847 [lua] warmup.lua:70: finished warming up DNS entries' into the cache (in 19ms), context: ngx.timer
	Aug 1, 2022 @ 12:58:15.077	2022/08/01 12:58:11 [notice] 29#0: signal 17 (SIGCHLD) received from 34
	Aug 1, 2022 @ 12:58:15.077	2022/08/01 12:58:11 [notice] 24161#0: *311847 [lua] warmup.lua:36: warming up DNS entries ..., context: ngx.timer
	Aug 1, 2022 @ 12:58:15.077	2022/08/01 12:58:11 [notice] 24161#0: *311846 [lua] warmup.lua:104: single_dao(): Preloading 'services' into the core_cache..., context: init_worker_by_lua*
	Aug 1, 2022 @ 12:58:15.077	2022/08/01 12:58:08 [info] 34#0: *311796 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:58:15.077	2022/08/01 12:58:11 [notice] 24161#0: *311846 [lua] warmup.lua:145: single_dao(): finished preloading 'services' into the core_cache (in 0ms), context: init_worker_by_lua*
	Aug 1, 2022 @ 12:58:16.120	2022/08/01 12:58:12 [info] 42#0: *313025 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:58:30.078	2022/08/01 12:58:28 [info] 36#0: *312148 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:58:37.614	2022/08/01 12:58:29 [info] 33#0: *312052 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:58:41.121	2022/08/01 12:58:32 [info] 42#0: *313326 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:58:52.614	2022/08/01 12:58:49 [info] 33#0: *312366 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:58:55.078	2022/08/01 12:58:48 [info] 36#0: *312454 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:58:56.122	2022/08/01 12:58:52 [info] 42#0: *313636 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:58:59.516	pname=kong timestamp=2022-08-01T12:58:58+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
	Aug 1, 2022 @ 12:58:59.517	pname=kong timestamp=2022-08-01T12:58:59+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
	Aug 1, 2022 @ 12:59:00.518	pname=kong timestamp=2022-08-01T12:58:59+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:00.519	pname=kong timestamp=2022-08-01T12:58:59+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
	Aug 1, 2022 @ 12:59:00.519	pname=kong timestamp=2022-08-01T12:59:00+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
	Aug 1, 2022 @ 12:59:00.520	pname=kong timestamp=2022-08-01T12:59:00+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
	Aug 1, 2022 @ 12:59:03.520	pname=kong timestamp=2022-08-01T12:59:03+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:03.521	pname=kong timestamp=2022-08-01T12:59:03+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
	Aug 1, 2022 @ 12:59:04.522	pname=kong timestamp=2022-08-01T12:59:03+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:04.522	pname=kong timestamp=2022-08-01T12:59:04+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
	Aug 1, 2022 @ 12:59:05.523	pname=kong timestamp=2022-08-01T12:59:04+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
	Aug 1, 2022 @ 12:59:05.524	pname=kong timestamp=2022-08-01T12:59:05+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:06.525	pname=kong timestamp=2022-08-01T12:59:05+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:09.525	pname=kong timestamp=2022-08-01T12:59:08+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:09.527	pname=kong timestamp=2022-08-01T12:59:09+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:09.528	pname=kong timestamp=2022-08-01T12:59:09+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
	Aug 1, 2022 @ 12:59:10.079	2022/08/01 12:59:08 [info] 36#0: *312762 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:59:10.529	pname=kong timestamp=2022-08-01T12:59:09+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:11.529	pname=kong timestamp=2022-08-01T12:59:10+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:11.530	pname=kong timestamp=2022-08-01T12:59:11+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=Go-http-client/2.0 method=POST
	Aug 1, 2022 @ 12:59:14.531	pname=kong timestamp=2022-08-01T12:59:13+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:14.531	pname=kong timestamp=2022-08-01T12:59:14+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:15.532	pname=kong timestamp=2022-08-01T12:59:14+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:16.533	pname=kong timestamp=2022-08-01T12:59:15+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:17.615	2022/08/01 12:59:09 [info] 33#0: *312666 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:59:19.534	pname=kong timestamp=2022-08-01T12:59:18+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
	Aug 1, 2022 @ 12:59:19.534	pname=kong timestamp=2022-08-01T12:59:18+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:20.535	pname=kong timestamp=2022-08-01T12:59:19+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
	Aug 1, 2022 @ 12:59:21.122	2022/08/01 12:59:12 [info] 42#0: *313956 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:59:23.536	pname=kong timestamp=2022-08-01T12:59:22+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:23.536	pname=kong timestamp=2022-08-01T12:59:21+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:23.537	pname=kong timestamp=2022-08-01T12:59:22+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
	Aug 1, 2022 @ 12:59:24.538	pname=kong timestamp=2022-08-01T12:59:23+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:24.538	pname=kong timestamp=2022-08-01T12:59:24+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:24.539	pname=kong timestamp=2022-08-01T12:59:24+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
	Aug 1, 2022 @ 12:59:27.539	pname=kong timestamp=2022-08-01T12:59:25+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:27.540	pname=kong timestamp=2022-08-01T12:59:25+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
	Aug 1, 2022 @ 12:59:27.541	pname=kong timestamp=2022-08-01T12:59:26+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
	Aug 1, 2022 @ 12:59:27.541	pname=kong timestamp=2022-08-01T12:59:26+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:27.542	pname=kong timestamp=2022-08-01T12:59:26+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
	Aug 1, 2022 @ 12:59:27.542	pname=kong timestamp=2022-08-01T12:59:27+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
	Aug 1, 2022 @ 12:59:30.543	pname=kong timestamp=2022-08-01T12:59:29+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:30.544	pname=kong timestamp=2022-08-01T12:59:30+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:32.616	2022/08/01 12:59:29 [info] 33#0: *312979 client 127.0.0.1 closed keepalive connection
	Aug 1, 2022 @ 12:59:33.545	pname=kong timestamp=2022-08-01T12:59:31+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:33.545	pname=kong timestamp=2022-08-01T12:59:32+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:33.546	pname=kong timestamp=2022-08-01T12:59:32+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:33.546	pname=kong timestamp=2022-08-01T12:59:33+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:34.547	pname=kong timestamp=2022-08-01T12:59:33+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
	Aug 1, 2022 @ 12:59:34.547	pname=kong timestamp=2022-08-01T12:59:33+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST

        ....

I see high latency metrics (see screenshot), and timeouts on the Kong routes are set on 60s, but if timeouts would be the case I would expect 5xx status codes. Also, the 404’s start well before hitting 60s latency.

We are not adding/deleting any routes or services. In fact, these remain static so it’s not due to strict consistency (In which case I would expect fix(router) detect changes on router/plugins iterator while rebuilding (backported) by bungle · Pull Request #5431 · Kong/kong · GitHub)

To be clear: no Kong instances crash, also requests return normal (2xx) in about 50% of the time, so this looks like a sync/cache problem but I’m very unsure where to look for now.

Any help is very much appreciated!

Kong configuration:

{
  "tagline": "Welcome to kong",
  "timers": {
    "running": 0,
    "pending": 8
  },
  "node_id": "96f5eca2-3fa1-4b31-9efb-bcd2cc8fb50f",
  "plugins": {
    "available_on_server": {
      "jwt": true,
      "acl": true,
      "correlation-id": true,
      "cors": true,
      "oauth2": true,
      "tcp-log": true,
      "udp-log": true,
      "file-log": true,
      "http-log": true,
      "key-auth": true,
      "hmac-auth": true,
      "basic-auth": true,
      "ip-restriction": true,
      "request-transformer": true,
      "response-transformer": true,
      "request-size-limiting": true,
      "rate-limiting": true,
      "response-ratelimiting": true,
      "syslog": true,
      "loggly": true,
      "datadog": true,
      "ldap-auth": true,
      "statsd": true,
      "bot-detection": true,
      "aws-lambda": true,
      "request-termination": true,
      "prometheus": true,
      "proxy-cache": true,
      "session": true,
      "acme": true,
      "grpc-gateway": true,
      "grpc-web": true,
      "pre-function": true,
      "post-function": true,
      "azure-functions": true,
      "zipkin": true,
      "k-jwt-keycloak-introspection": true,
      "k-jwt-auth": true
    },
    "enabled_in_cluster": [
      "cors",
      "k-jwt-keycloak-introspection",
      "k-jwt-auth",
      "prometheus",
      "key-auth"
    ]
  },
  "configuration": {
    "ssl_cert": [
      "/usr/local/kong/ssl/kong-default.crt",
      "/usr/local/kong/ssl/kong-default-ecdsa.crt"
    ],
    "proxy_listeners": [
      {
        "ssl": false,
        "backlog=16384": true,
        "proxy_protocol": false,
        "port": 8000,
        "ip": "0.0.0.0",
        "http2": false,
        "bind": false,
        "deferred": false,
        "reuseport": true,
        "listener": "0.0.0.0:8000 reuseport backlog=16384"
      },
      {
        "ssl": true,
        "backlog=16384": true,
        "proxy_protocol": false,
        "port": 8443,
        "ip": "0.0.0.0",
        "http2": true,
        "bind": false,
        "deferred": false,
        "reuseport": true,
        "listener": "0.0.0.0:8443 ssl http2 reuseport backlog=16384"
      }
    ],
    "prefix": "/usr/local/kong",
    "stream_listeners": {
      
    },
    "lua_package_path": "./?.lua;./?/init.lua;",
    "nginx_err_logs": "/usr/local/kong/logs/error.log",
    "ssl_cipher_suite": "custom",
    "nginx_acc_logs": "/usr/local/kong/logs/access.log",
    "status_listeners": {
      
    },
    "admin_acc_logs": "/usr/local/kong/logs/admin_access.log",
    "cluster_listeners": [
      {
        "ssl": false,
        "ip": "0.0.0.0",
        "listener": "0.0.0.0:8005",
        "port": 8005,
        "proxy_protocol": false,
        "http2": false,
        "bind": false,
        "deferred": false,
        "reuseport": false,
        "backlog=%d+": false
      }
    ],
    "nginx_conf": "/usr/local/kong/nginx.conf",
    "nginx_kong_conf": "/usr/local/kong/nginx-kong.conf",
    "nginx_kong_stream_conf": "/usr/local/kong/nginx-kong-stream.conf",
    "kong_env": "/usr/local/kong/.kong_env",
    "ssl_cert_csr_default": "/usr/local/kong/ssl/kong-default.csr",
    "lua_package_cpath": "",
    "ssl_cert_default": "/usr/local/kong/ssl/kong-default.crt",
    "ssl_cert_key_default": "/usr/local/kong/ssl/kong-default.key",
    "ssl_cert_default_ecdsa": "/usr/local/kong/ssl/kong-default-ecdsa.crt",
    "worker_consistency": "strict",
    "ssl_cert_key_default_ecdsa": "/usr/local/kong/ssl/kong-default-ecdsa.key",
    "client_ssl_cert_default": "/usr/local/kong/ssl/kong-default.crt",
    "client_ssl_cert_key_default": "/usr/local/kong/ssl/kong-default.key",
    "admin_ssl_cert_default": "/usr/local/kong/ssl/admin-kong-default.crt",
    "admin_ssl_cert_key_default": "/usr/local/kong/ssl/admin-kong-default.key",
    "admin_ssl_cert_default_ecdsa": "/usr/local/kong/ssl/admin-kong-default-ecdsa.crt",
    "admin_ssl_cert_key_default_ecdsa": "/usr/local/kong/ssl/admin-kong-default-ecdsa.key",
    "status_ssl_cert_default": "/usr/local/kong/ssl/status-kong-default.crt",
    "status_ssl_cert_key_default": "/usr/local/kong/ssl/status-kong-default.key",
    "status_ssl_cert_default_ecdsa": "/usr/local/kong/ssl/status-kong-default-ecdsa.crt",
    "pluginserver_names": {
      
    },
    "status_ssl_cert_key_default_ecdsa": "/usr/local/kong/ssl/status-kong-default-ecdsa.key",
    "upstream_keepalive_pool_size": 60,
    "nginx_proxy_error_page": "495 496 497 /kong_error_handler",
    "error_default_type": "text/plain",
    "go_plugins_dir": "off",
    "port_maps": {
      
    },
    "nginx_proxy_large_client_header_buffers": "4 32k",
    "admin_listen": [
      "0.0.0.0:8001"
    ],
    "status_listen": [
      "off"
    ],
    "go_pluginserver_exe": "/usr/local/bin/go-pluginserver",
    "cluster_listen": [
      "0.0.0.0:8005"
    ],
    "admin_ssl_cert": {
      
    },
    "admin_ssl_cert_key": {
      
    },
    "db_update_frequency": 5,
    "status_ssl_cert_key": {
      
    },
    "db_resurrect_ttl": 30,
    "nginx_user": "kong kong",
    "db_cache_ttl": 0,
    "nginx_main_user": "kong kong",
    "nginx_daemon": "off",
    "nginx_main_daemon": "off",
    "nginx_worker_processes": "4",
    "nginx_main_worker_processes": "4",
    "trusted_ips": {
      
    },
    "real_ip_header": "X-Real-IP",
    "nginx_proxy_real_ip_header": "X-Real-IP",
    "real_ip_recursive": "off",
    "nginx_proxy_real_ip_recursive": "off",
    "client_max_body_size": "0",
    "nginx_http_client_max_body_size": "0",
    "admin_ssl_enabled": false,
    "status_ssl_enabled": false,
    "pg_port": 5432,
    "pg_password": "******",
    "nginx_events_multi_accept": "on",
    "worker_state_update_frequency": 5,
    "pg_max_concurrent_queries": 0,
    "lua_ssl_trusted_certificate_combined": "/usr/local/kong/.ca_combined",
    "pg_ro_ssl": false,
    "pg_ro_ssl_verify": false,
    "db_cache_warmup_entities": [
      "services"
    ],
    "cassandra_contact_points": [
      "127.0.0.1"
    ],
    "cassandra_port": 9042,
    "cassandra_ssl": false,
    "headers": [
      "off"
    ],
    "cassandra_write_consistency": "ONE",
    "nginx_admin_client_max_body_size": "10m",
    "client_ssl": false,
    "nginx_admin_client_body_buffer_size": "10m",
    "cassandra_read_consistency": "ONE",
    "cassandra_lb_policy": "RequestRoundRobin",
    "nginx_http_lua_regex_match_limit": "100000",
    "cassandra_refresh_frequency": 60,
    "cassandra_repl_strategy": "SimpleStrategy",
    "cassandra_repl_factor": 1,
    "cassandra_data_centers": [
      "dc1:2",
      "dc2:3"
    ],
    "cassandra_schema_consensus_timeout": 60000,
    "ssl_protocols": "TLSv1.1 TLSv1.2 TLSv1.3",
    "nginx_http_ssl_protocols": "TLSv1.1 TLSv1.2 TLSv1.3",
    "nginx_stream_ssl_protocols": "TLSv1.1 TLSv1.2 TLSv1.3",
    "ssl_prefer_server_ciphers": "on",
    "nginx_http_ssl_prefer_server_ciphers": "on",
    "nginx_stream_ssl_prefer_server_ciphers": "on",
    "ssl_session_tickets": "on",
    "nginx_http_ssl_session_tickets": "on",
    "nginx_stream_ssl_session_tickets": "on",
    "ssl_session_timeout": "1d",
    "nginx_http_ssl_session_timeout": "1d",
    "nginx_stream_ssl_session_timeout": "1d",
    "proxy_access_log": "/dev/stdout logfmt if=$loggable",
    "proxy_error_log": "/dev/stderr",
    "proxy_stream_access_log": "logs/access.log basic",
    "proxy_stream_error_log": "logs/error.log",
    "admin_access_log": "/dev/stdout logfmt if=$loggable",
    "admin_error_log": "/dev/stderr",
    "status_access_log": "off",
    "status_error_log": "logs/status_error.log",
    "log_level": "info",
    "vaults": [
      "off"
    ],
    "nginx_optimizations": true,
    "lua_ssl_trusted_certificate": [
      "/etc/ssl/certs/ca-certificates.pem"
    ],
    "lua_ssl_verify_depth": 2,
    "lua_ssl_protocols": "TLSv1.1 TLSv1.2 TLSv1.3",
    "nginx_http_lua_ssl_protocols": "TLSv1.1 TLSv1.2 TLSv1.3",
    "nginx_stream_lua_ssl_protocols": "TLSv1.1 TLSv1.2 TLSv1.3",
    "lua_socket_pool_size": 30,
    "kic": false,
    "cluster_control_plane": "127.0.0.1:8005",
    "ssl_cert_key": [
      "/usr/local/kong/ssl/kong-default.key",
      "/usr/local/kong/ssl/kong-default-ecdsa.key"
    ],
    "nginx_proxy_ssl_dhparam": "/usr/local/kong/ssl/dhparam.pem",
    "cluster_mtls": "shared",
    "pg_database": "kong",
    "nginx_http_client_body_buffer_size": "8k",
    "upstream_keepalive_idle_timeout": 60,
    "cluster_data_plane_purge_delay": 1209600,
    "cluster_ocsp": "off",
    "cluster_max_payload": 4194304,
    "untrusted_lua": "sandbox",
    "admin_listeners": [
      {
        "ssl": false,
        "ip": "0.0.0.0",
        "listener": "0.0.0.0:8001",
        "port": 8001,
        "proxy_protocol": false,
        "http2": false,
        "bind": false,
        "deferred": false,
        "reuseport": false,
        "backlog=%d+": false
      }
    ],
    "stream_proxy_ssl_enabled": false,
    "dns_resolver": [
      "198.51.100.1",
      "198.51.100.2",
      "198.51.100.3"
    ],
    "pg_ssl": false,
    "dns_hostsfile": "/etc/hosts",
    "mem_cache_size": "128m",
    "pg_ssl_verify": false,
    "cassandra_ssl_verify": false,
    "pg_user": "kong",
    "pg_timeout": 60000,
    "dns_error_ttl": 1,
    "database": "postgres",
    "dns_not_found_ttl": 30,
    "host_ports": {
      
    },
    "dns_stale_ttl": 4,
    "stream_listen": [
      "off"
    ],
    "dns_cache_size": 10000,
    "nginx_http_include": "/usr/local/kong/nginx-http.conf",
    "dns_order": [
      "LAST",
      "SRV",
      "A",
      "CNAME"
    ],
    "proxy_ssl_enabled": true,
    "dns_no_sync": false,
    "role": "traditional",
    "nginx_main_directives": [
      {
        "name": "daemon",
        "value": "off"
      },
      {
        "name": "user",
        "value": "kong kong"
      },
      {
        "name": "worker_processes",
        "value": "4"
      },
      {
        "name": "worker_rlimit_nofile",
        "value": "auto"
      }
    ],
    "nginx_events_directives": [
      {
        "name": "multi_accept",
        "value": "on"
      },
      {
        "name": "worker_connections",
        "value": "auto"
      }
    ],
    "upstream_keepalive_max_requests": 100,
    "untrusted_lua_sandbox_requires": {
      
    },
    "untrusted_lua_sandbox_environment": {
      
    },
    "cassandra_keyspace": "kong",
    "loaded_vaults": {
      
    },
    "plugins": [
      "bundled",
      "prometheus",
      "k-jwt-auth",
      "k-jwt-keycloak-introspection"
    ],
    "nginx_upstream_directives": {
      
    },
    "loaded_plugins": {
      "jwt": true,
      "acl": true,
      "correlation-id": true,
      "cors": true,
      "oauth2": true,
      "tcp-log": true,
      "udp-log": true,
      "file-log": true,
      "http-log": true,
      "key-auth": true,
      "hmac-auth": true,
      "basic-auth": true,
      "ip-restriction": true,
      "request-transformer": true,
      "response-transformer": true,
      "request-size-limiting": true,
      "rate-limiting": true,
      "response-ratelimiting": true,
      "syslog": true,
      "loggly": true,
      "datadog": true,
      "ldap-auth": true,
      "statsd": true,
      "bot-detection": true,
      "aws-lambda": true,
      "request-termination": true,
      "prometheus": true,
      "proxy-cache": true,
      "session": true,
      "acme": true,
      "grpc-gateway": true,
      "grpc-web": true,
      "pre-function": true,
      "post-function": true,
      "azure-functions": true,
      "zipkin": true,
      "k-jwt-auth": true,
      "k-jwt-keycloak-introspection": true
    },
    "nginx_proxy_directives": [
      {
        "name": "error_page",
        "value": "495 496 497 /kong_error_handler"
      },
      {
        "name": "large_client_header_buffers",
        "value": "4 32k"
      },
      {
        "name": "real_ip_header",
        "value": "X-Real-IP"
      },
      {
        "name": "real_ip_recursive",
        "value": "off"
      },
      {
        "name": "ssl_dhparam",
        "value": "/usr/local/kong/ssl/dhparam.pem"
      }
    ],
    "nginx_main_worker_rlimit_nofile": "auto",
    "nginx_status_directives": {
      
    },
    "nginx_events_worker_connections": "auto",
    "nginx_admin_directives": [
      {
        "name": "client_body_buffer_size",
        "value": "10m"
      },
      {
        "name": "client_max_body_size",
        "value": "10m"
      }
    ],
    "nginx_http_directives": [
      {
        "name": "client_body_buffer_size",
        "value": "8k"
      },
      {
        "name": "client_max_body_size",
        "value": "0"
      },
      {
        "name": "include",
        "value": "/usr/local/kong/nginx-http.conf"
      },
      {
        "name": "lua_regex_match_limit",
        "value": "100000"
      },
      {
        "name": "lua_shared_dict",
        "value": "prometheus_metrics 5m"
      },
      {
        "name": "lua_ssl_protocols",
        "value": "TLSv1.1 TLSv1.2 TLSv1.3"
      },
      {
        "name": "ssl_prefer_server_ciphers",
        "value": "on"
      },
      {
        "name": "ssl_protocols",
        "value": "TLSv1.1 TLSv1.2 TLSv1.3"
      },
      {
        "name": "ssl_session_tickets",
        "value": "on"
      },
      {
        "name": "ssl_session_timeout",
        "value": "1d"
      }
    ],
    "nginx_stream_directives": [
      {
        "name": "lua_shared_dict",
        "value": "stream_prometheus_metrics 5m"
      },
      {
        "name": "lua_ssl_protocols",
        "value": "TLSv1.1 TLSv1.2 TLSv1.3"
      },
      {
        "name": "ssl_prefer_server_ciphers",
        "value": "on"
      },
      {
        "name": "ssl_protocols",
        "value": "TLSv1.1 TLSv1.2 TLSv1.3"
      },
      {
        "name": "ssl_session_tickets",
        "value": "on"
      },
      {
        "name": "ssl_session_timeout",
        "value": "1d"
      }
    ],
    "cassandra_username": "kong",
    "nginx_supstream_directives": {
      
    },
    "cassandra_timeout": 60000,
    "nginx_sproxy_directives": {
      
    },
    "anonymous_reports": false,
    "db_update_propagation": 0,
    "status_ssl_cert": {
      
    },
    "pg_host": "stolon-proxy.apigateway.dsh.marathon.mesos",
    "nginx_http_upstream_directives": {
      
    },
    "nginx_http_status_directives": {
      
    },
    "pg_semaphore_timeout": 60000,
    "nginx_pid": "/usr/local/kong/pids/nginx.pid",
    "client_body_buffer_size": "8k",
    "proxy_listen": [
      "0.0.0.0:8000 reuseport backlog=16384",
      "0.0.0.0:8443 http2 ssl reuseport backlog=16384"
    ],
    "ssl_ciphers": "\"ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256\"",
    "enabled_headers": {
      "X-Kong-Proxy-Latency": false,
      "X-Kong-Response-Latency": false,
      "X-Kong-Admin-Latency": false,
      "server_tokens": false,
      "X-Kong-Upstream-Latency": false,
      "X-Kong-Upstream-Status": false,
      "Via": false,
      "Server": false,
      "latency_tokens": false
    }
  },
  "version": "2.8.1",
  "lua_version": "LuaJIT 2.1.0-beta3",
  "pids": {
    "workers": [
      40,
      4786,
      42,
      43
    ],
    "master": 35
  },
  "hostname": "0d9fe41ac71a"
}