We’re running 3 Kong instances in Mesos in front of Stolon. After upgrading to Kong 2.8.1 (from 1.5), we’re seeing a lot of 404’s from Kong, however not always: roughly 50% are good. I have no clue however as to why.
From the Kong logs it seems to start with a SIGKILL (due to memory limits) after which a new worker is launched and for whatever reason, this worker starts to return 404’s.
Aug 1, 2022 @ 12:57:21.118 2022/08/01 12:57:12 [info] 42#0: *312090 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:57:32.612 2022/08/01 12:57:29 [info] 33#0: *311135 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:57:35.076 2022/08/01 12:57:28 [info] 34#0: *311185 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:57:36.119 2022/08/01 12:57:32 [info] 42#0: *312392 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:57:50.076 2022/08/01 12:57:48 [info] 34#0: *311488 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:57:57.612 2022/08/01 12:57:49 [info] 33#0: *311444 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:58:01.119 2022/08/01 12:57:52 [info] 42#0: *312715 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:58:12.613 2022/08/01 12:58:09 [info] 33#0: *311742 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:58:15.077 2022/08/01 12:58:11 [notice] 29#0: signal 29 (SIGIO) received
Aug 1, 2022 @ 12:58:15.077 2022/08/01 12:58:11 [alert] 29#0: worker process 34 exited on signal 9
Aug 1, 2022 @ 12:58:15.077 2022/08/01 12:58:11 [info] 24161#0: *311846 [kong] handler.lua:57 [acme] acme renew timer started on worker 0, context: init_worker_by_lua*
Aug 1, 2022 @ 12:58:15.077 2022/08/01 12:58:11 [notice] 29#0: start worker process 24161
Aug 1, 2022 @ 12:58:15.077 2022/08/01 12:58:11 [notice] 24161#0: *311847 [lua] warmup.lua:70: finished warming up DNS entries' into the cache (in 19ms), context: ngx.timer
Aug 1, 2022 @ 12:58:15.077 2022/08/01 12:58:11 [notice] 29#0: signal 17 (SIGCHLD) received from 34
Aug 1, 2022 @ 12:58:15.077 2022/08/01 12:58:11 [notice] 24161#0: *311847 [lua] warmup.lua:36: warming up DNS entries ..., context: ngx.timer
Aug 1, 2022 @ 12:58:15.077 2022/08/01 12:58:11 [notice] 24161#0: *311846 [lua] warmup.lua:104: single_dao(): Preloading 'services' into the core_cache..., context: init_worker_by_lua*
Aug 1, 2022 @ 12:58:15.077 2022/08/01 12:58:08 [info] 34#0: *311796 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:58:15.077 2022/08/01 12:58:11 [notice] 24161#0: *311846 [lua] warmup.lua:145: single_dao(): finished preloading 'services' into the core_cache (in 0ms), context: init_worker_by_lua*
Aug 1, 2022 @ 12:58:16.120 2022/08/01 12:58:12 [info] 42#0: *313025 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:58:30.078 2022/08/01 12:58:28 [info] 36#0: *312148 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:58:37.614 2022/08/01 12:58:29 [info] 33#0: *312052 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:58:41.121 2022/08/01 12:58:32 [info] 42#0: *313326 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:58:52.614 2022/08/01 12:58:49 [info] 33#0: *312366 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:58:55.078 2022/08/01 12:58:48 [info] 36#0: *312454 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:58:56.122 2022/08/01 12:58:52 [info] 42#0: *313636 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:58:59.516 pname=kong timestamp=2022-08-01T12:58:58+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
Aug 1, 2022 @ 12:58:59.517 pname=kong timestamp=2022-08-01T12:58:59+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
Aug 1, 2022 @ 12:59:00.518 pname=kong timestamp=2022-08-01T12:58:59+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:00.519 pname=kong timestamp=2022-08-01T12:58:59+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
Aug 1, 2022 @ 12:59:00.519 pname=kong timestamp=2022-08-01T12:59:00+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
Aug 1, 2022 @ 12:59:00.520 pname=kong timestamp=2022-08-01T12:59:00+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
Aug 1, 2022 @ 12:59:03.520 pname=kong timestamp=2022-08-01T12:59:03+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:03.521 pname=kong timestamp=2022-08-01T12:59:03+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
Aug 1, 2022 @ 12:59:04.522 pname=kong timestamp=2022-08-01T12:59:03+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:04.522 pname=kong timestamp=2022-08-01T12:59:04+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
Aug 1, 2022 @ 12:59:05.523 pname=kong timestamp=2022-08-01T12:59:04+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
Aug 1, 2022 @ 12:59:05.524 pname=kong timestamp=2022-08-01T12:59:05+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:06.525 pname=kong timestamp=2022-08-01T12:59:05+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:09.525 pname=kong timestamp=2022-08-01T12:59:08+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:09.527 pname=kong timestamp=2022-08-01T12:59:09+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:09.528 pname=kong timestamp=2022-08-01T12:59:09+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
Aug 1, 2022 @ 12:59:10.079 2022/08/01 12:59:08 [info] 36#0: *312762 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:59:10.529 pname=kong timestamp=2022-08-01T12:59:09+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:11.529 pname=kong timestamp=2022-08-01T12:59:10+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:11.530 pname=kong timestamp=2022-08-01T12:59:11+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=Go-http-client/2.0 method=POST
Aug 1, 2022 @ 12:59:14.531 pname=kong timestamp=2022-08-01T12:59:13+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:14.531 pname=kong timestamp=2022-08-01T12:59:14+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:15.532 pname=kong timestamp=2022-08-01T12:59:14+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:16.533 pname=kong timestamp=2022-08-01T12:59:15+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:17.615 2022/08/01 12:59:09 [info] 33#0: *312666 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:59:19.534 pname=kong timestamp=2022-08-01T12:59:18+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
Aug 1, 2022 @ 12:59:19.534 pname=kong timestamp=2022-08-01T12:59:18+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:20.535 pname=kong timestamp=2022-08-01T12:59:19+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
Aug 1, 2022 @ 12:59:21.122 2022/08/01 12:59:12 [info] 42#0: *313956 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:59:23.536 pname=kong timestamp=2022-08-01T12:59:22+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:23.536 pname=kong timestamp=2022-08-01T12:59:21+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:23.537 pname=kong timestamp=2022-08-01T12:59:22+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
Aug 1, 2022 @ 12:59:24.538 pname=kong timestamp=2022-08-01T12:59:23+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:24.538 pname=kong timestamp=2022-08-01T12:59:24+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:24.539 pname=kong timestamp=2022-08-01T12:59:24+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
Aug 1, 2022 @ 12:59:27.539 pname=kong timestamp=2022-08-01T12:59:25+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:27.540 pname=kong timestamp=2022-08-01T12:59:25+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
Aug 1, 2022 @ 12:59:27.541 pname=kong timestamp=2022-08-01T12:59:26+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
Aug 1, 2022 @ 12:59:27.541 pname=kong timestamp=2022-08-01T12:59:26+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:27.542 pname=kong timestamp=2022-08-01T12:59:26+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
Aug 1, 2022 @ 12:59:27.542 pname=kong timestamp=2022-08-01T12:59:27+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
Aug 1, 2022 @ 12:59:30.543 pname=kong timestamp=2022-08-01T12:59:29+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:30.544 pname=kong timestamp=2022-08-01T12:59:30+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:32.616 2022/08/01 12:59:29 [info] 33#0: *312979 client 127.0.0.1 closed keepalive connection
Aug 1, 2022 @ 12:59:33.545 pname=kong timestamp=2022-08-01T12:59:31+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:33.545 pname=kong timestamp=2022-08-01T12:59:32+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:33.546 pname=kong timestamp=2022-08-01T12:59:32+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:33.546 pname=kong timestamp=2022-08-01T12:59:33+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:34.547 pname=kong timestamp=2022-08-01T12:59:33+00:00 status=404 request-uri=/auth/v0/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=GET
Aug 1, 2022 @ 12:59:34.547 pname=kong timestamp=2022-08-01T12:59:33+00:00 status=404 request-uri=/datastreams/v0/mqtt/token remote-user=- remote-address=10.0.3.81 user-agent=scalaj-http/1.0 method=POST
....
I see high latency metrics (see screenshot), and timeouts on the Kong routes are set on 60s, but if timeouts would be the case I would expect 5xx status codes. Also, the 404’s start well before hitting 60s latency.
We are not adding/deleting any routes or services. In fact, these remain static so it’s not due to strict consistency (In which case I would expect fix(router) detect changes on router/plugins iterator while rebuilding (backported) by bungle · Pull Request #5431 · Kong/kong · GitHub)
To be clear: no Kong instances crash, also requests return normal (2xx) in about 50% of the time, so this looks like a sync/cache problem but I’m very unsure where to look for now.
Any help is very much appreciated!
Kong configuration:
{
"tagline": "Welcome to kong",
"timers": {
"running": 0,
"pending": 8
},
"node_id": "96f5eca2-3fa1-4b31-9efb-bcd2cc8fb50f",
"plugins": {
"available_on_server": {
"jwt": true,
"acl": true,
"correlation-id": true,
"cors": true,
"oauth2": true,
"tcp-log": true,
"udp-log": true,
"file-log": true,
"http-log": true,
"key-auth": true,
"hmac-auth": true,
"basic-auth": true,
"ip-restriction": true,
"request-transformer": true,
"response-transformer": true,
"request-size-limiting": true,
"rate-limiting": true,
"response-ratelimiting": true,
"syslog": true,
"loggly": true,
"datadog": true,
"ldap-auth": true,
"statsd": true,
"bot-detection": true,
"aws-lambda": true,
"request-termination": true,
"prometheus": true,
"proxy-cache": true,
"session": true,
"acme": true,
"grpc-gateway": true,
"grpc-web": true,
"pre-function": true,
"post-function": true,
"azure-functions": true,
"zipkin": true,
"k-jwt-keycloak-introspection": true,
"k-jwt-auth": true
},
"enabled_in_cluster": [
"cors",
"k-jwt-keycloak-introspection",
"k-jwt-auth",
"prometheus",
"key-auth"
]
},
"configuration": {
"ssl_cert": [
"/usr/local/kong/ssl/kong-default.crt",
"/usr/local/kong/ssl/kong-default-ecdsa.crt"
],
"proxy_listeners": [
{
"ssl": false,
"backlog=16384": true,
"proxy_protocol": false,
"port": 8000,
"ip": "0.0.0.0",
"http2": false,
"bind": false,
"deferred": false,
"reuseport": true,
"listener": "0.0.0.0:8000 reuseport backlog=16384"
},
{
"ssl": true,
"backlog=16384": true,
"proxy_protocol": false,
"port": 8443,
"ip": "0.0.0.0",
"http2": true,
"bind": false,
"deferred": false,
"reuseport": true,
"listener": "0.0.0.0:8443 ssl http2 reuseport backlog=16384"
}
],
"prefix": "/usr/local/kong",
"stream_listeners": {
},
"lua_package_path": "./?.lua;./?/init.lua;",
"nginx_err_logs": "/usr/local/kong/logs/error.log",
"ssl_cipher_suite": "custom",
"nginx_acc_logs": "/usr/local/kong/logs/access.log",
"status_listeners": {
},
"admin_acc_logs": "/usr/local/kong/logs/admin_access.log",
"cluster_listeners": [
{
"ssl": false,
"ip": "0.0.0.0",
"listener": "0.0.0.0:8005",
"port": 8005,
"proxy_protocol": false,
"http2": false,
"bind": false,
"deferred": false,
"reuseport": false,
"backlog=%d+": false
}
],
"nginx_conf": "/usr/local/kong/nginx.conf",
"nginx_kong_conf": "/usr/local/kong/nginx-kong.conf",
"nginx_kong_stream_conf": "/usr/local/kong/nginx-kong-stream.conf",
"kong_env": "/usr/local/kong/.kong_env",
"ssl_cert_csr_default": "/usr/local/kong/ssl/kong-default.csr",
"lua_package_cpath": "",
"ssl_cert_default": "/usr/local/kong/ssl/kong-default.crt",
"ssl_cert_key_default": "/usr/local/kong/ssl/kong-default.key",
"ssl_cert_default_ecdsa": "/usr/local/kong/ssl/kong-default-ecdsa.crt",
"worker_consistency": "strict",
"ssl_cert_key_default_ecdsa": "/usr/local/kong/ssl/kong-default-ecdsa.key",
"client_ssl_cert_default": "/usr/local/kong/ssl/kong-default.crt",
"client_ssl_cert_key_default": "/usr/local/kong/ssl/kong-default.key",
"admin_ssl_cert_default": "/usr/local/kong/ssl/admin-kong-default.crt",
"admin_ssl_cert_key_default": "/usr/local/kong/ssl/admin-kong-default.key",
"admin_ssl_cert_default_ecdsa": "/usr/local/kong/ssl/admin-kong-default-ecdsa.crt",
"admin_ssl_cert_key_default_ecdsa": "/usr/local/kong/ssl/admin-kong-default-ecdsa.key",
"status_ssl_cert_default": "/usr/local/kong/ssl/status-kong-default.crt",
"status_ssl_cert_key_default": "/usr/local/kong/ssl/status-kong-default.key",
"status_ssl_cert_default_ecdsa": "/usr/local/kong/ssl/status-kong-default-ecdsa.crt",
"pluginserver_names": {
},
"status_ssl_cert_key_default_ecdsa": "/usr/local/kong/ssl/status-kong-default-ecdsa.key",
"upstream_keepalive_pool_size": 60,
"nginx_proxy_error_page": "495 496 497 /kong_error_handler",
"error_default_type": "text/plain",
"go_plugins_dir": "off",
"port_maps": {
},
"nginx_proxy_large_client_header_buffers": "4 32k",
"admin_listen": [
"0.0.0.0:8001"
],
"status_listen": [
"off"
],
"go_pluginserver_exe": "/usr/local/bin/go-pluginserver",
"cluster_listen": [
"0.0.0.0:8005"
],
"admin_ssl_cert": {
},
"admin_ssl_cert_key": {
},
"db_update_frequency": 5,
"status_ssl_cert_key": {
},
"db_resurrect_ttl": 30,
"nginx_user": "kong kong",
"db_cache_ttl": 0,
"nginx_main_user": "kong kong",
"nginx_daemon": "off",
"nginx_main_daemon": "off",
"nginx_worker_processes": "4",
"nginx_main_worker_processes": "4",
"trusted_ips": {
},
"real_ip_header": "X-Real-IP",
"nginx_proxy_real_ip_header": "X-Real-IP",
"real_ip_recursive": "off",
"nginx_proxy_real_ip_recursive": "off",
"client_max_body_size": "0",
"nginx_http_client_max_body_size": "0",
"admin_ssl_enabled": false,
"status_ssl_enabled": false,
"pg_port": 5432,
"pg_password": "******",
"nginx_events_multi_accept": "on",
"worker_state_update_frequency": 5,
"pg_max_concurrent_queries": 0,
"lua_ssl_trusted_certificate_combined": "/usr/local/kong/.ca_combined",
"pg_ro_ssl": false,
"pg_ro_ssl_verify": false,
"db_cache_warmup_entities": [
"services"
],
"cassandra_contact_points": [
"127.0.0.1"
],
"cassandra_port": 9042,
"cassandra_ssl": false,
"headers": [
"off"
],
"cassandra_write_consistency": "ONE",
"nginx_admin_client_max_body_size": "10m",
"client_ssl": false,
"nginx_admin_client_body_buffer_size": "10m",
"cassandra_read_consistency": "ONE",
"cassandra_lb_policy": "RequestRoundRobin",
"nginx_http_lua_regex_match_limit": "100000",
"cassandra_refresh_frequency": 60,
"cassandra_repl_strategy": "SimpleStrategy",
"cassandra_repl_factor": 1,
"cassandra_data_centers": [
"dc1:2",
"dc2:3"
],
"cassandra_schema_consensus_timeout": 60000,
"ssl_protocols": "TLSv1.1 TLSv1.2 TLSv1.3",
"nginx_http_ssl_protocols": "TLSv1.1 TLSv1.2 TLSv1.3",
"nginx_stream_ssl_protocols": "TLSv1.1 TLSv1.2 TLSv1.3",
"ssl_prefer_server_ciphers": "on",
"nginx_http_ssl_prefer_server_ciphers": "on",
"nginx_stream_ssl_prefer_server_ciphers": "on",
"ssl_session_tickets": "on",
"nginx_http_ssl_session_tickets": "on",
"nginx_stream_ssl_session_tickets": "on",
"ssl_session_timeout": "1d",
"nginx_http_ssl_session_timeout": "1d",
"nginx_stream_ssl_session_timeout": "1d",
"proxy_access_log": "/dev/stdout logfmt if=$loggable",
"proxy_error_log": "/dev/stderr",
"proxy_stream_access_log": "logs/access.log basic",
"proxy_stream_error_log": "logs/error.log",
"admin_access_log": "/dev/stdout logfmt if=$loggable",
"admin_error_log": "/dev/stderr",
"status_access_log": "off",
"status_error_log": "logs/status_error.log",
"log_level": "info",
"vaults": [
"off"
],
"nginx_optimizations": true,
"lua_ssl_trusted_certificate": [
"/etc/ssl/certs/ca-certificates.pem"
],
"lua_ssl_verify_depth": 2,
"lua_ssl_protocols": "TLSv1.1 TLSv1.2 TLSv1.3",
"nginx_http_lua_ssl_protocols": "TLSv1.1 TLSv1.2 TLSv1.3",
"nginx_stream_lua_ssl_protocols": "TLSv1.1 TLSv1.2 TLSv1.3",
"lua_socket_pool_size": 30,
"kic": false,
"cluster_control_plane": "127.0.0.1:8005",
"ssl_cert_key": [
"/usr/local/kong/ssl/kong-default.key",
"/usr/local/kong/ssl/kong-default-ecdsa.key"
],
"nginx_proxy_ssl_dhparam": "/usr/local/kong/ssl/dhparam.pem",
"cluster_mtls": "shared",
"pg_database": "kong",
"nginx_http_client_body_buffer_size": "8k",
"upstream_keepalive_idle_timeout": 60,
"cluster_data_plane_purge_delay": 1209600,
"cluster_ocsp": "off",
"cluster_max_payload": 4194304,
"untrusted_lua": "sandbox",
"admin_listeners": [
{
"ssl": false,
"ip": "0.0.0.0",
"listener": "0.0.0.0:8001",
"port": 8001,
"proxy_protocol": false,
"http2": false,
"bind": false,
"deferred": false,
"reuseport": false,
"backlog=%d+": false
}
],
"stream_proxy_ssl_enabled": false,
"dns_resolver": [
"198.51.100.1",
"198.51.100.2",
"198.51.100.3"
],
"pg_ssl": false,
"dns_hostsfile": "/etc/hosts",
"mem_cache_size": "128m",
"pg_ssl_verify": false,
"cassandra_ssl_verify": false,
"pg_user": "kong",
"pg_timeout": 60000,
"dns_error_ttl": 1,
"database": "postgres",
"dns_not_found_ttl": 30,
"host_ports": {
},
"dns_stale_ttl": 4,
"stream_listen": [
"off"
],
"dns_cache_size": 10000,
"nginx_http_include": "/usr/local/kong/nginx-http.conf",
"dns_order": [
"LAST",
"SRV",
"A",
"CNAME"
],
"proxy_ssl_enabled": true,
"dns_no_sync": false,
"role": "traditional",
"nginx_main_directives": [
{
"name": "daemon",
"value": "off"
},
{
"name": "user",
"value": "kong kong"
},
{
"name": "worker_processes",
"value": "4"
},
{
"name": "worker_rlimit_nofile",
"value": "auto"
}
],
"nginx_events_directives": [
{
"name": "multi_accept",
"value": "on"
},
{
"name": "worker_connections",
"value": "auto"
}
],
"upstream_keepalive_max_requests": 100,
"untrusted_lua_sandbox_requires": {
},
"untrusted_lua_sandbox_environment": {
},
"cassandra_keyspace": "kong",
"loaded_vaults": {
},
"plugins": [
"bundled",
"prometheus",
"k-jwt-auth",
"k-jwt-keycloak-introspection"
],
"nginx_upstream_directives": {
},
"loaded_plugins": {
"jwt": true,
"acl": true,
"correlation-id": true,
"cors": true,
"oauth2": true,
"tcp-log": true,
"udp-log": true,
"file-log": true,
"http-log": true,
"key-auth": true,
"hmac-auth": true,
"basic-auth": true,
"ip-restriction": true,
"request-transformer": true,
"response-transformer": true,
"request-size-limiting": true,
"rate-limiting": true,
"response-ratelimiting": true,
"syslog": true,
"loggly": true,
"datadog": true,
"ldap-auth": true,
"statsd": true,
"bot-detection": true,
"aws-lambda": true,
"request-termination": true,
"prometheus": true,
"proxy-cache": true,
"session": true,
"acme": true,
"grpc-gateway": true,
"grpc-web": true,
"pre-function": true,
"post-function": true,
"azure-functions": true,
"zipkin": true,
"k-jwt-auth": true,
"k-jwt-keycloak-introspection": true
},
"nginx_proxy_directives": [
{
"name": "error_page",
"value": "495 496 497 /kong_error_handler"
},
{
"name": "large_client_header_buffers",
"value": "4 32k"
},
{
"name": "real_ip_header",
"value": "X-Real-IP"
},
{
"name": "real_ip_recursive",
"value": "off"
},
{
"name": "ssl_dhparam",
"value": "/usr/local/kong/ssl/dhparam.pem"
}
],
"nginx_main_worker_rlimit_nofile": "auto",
"nginx_status_directives": {
},
"nginx_events_worker_connections": "auto",
"nginx_admin_directives": [
{
"name": "client_body_buffer_size",
"value": "10m"
},
{
"name": "client_max_body_size",
"value": "10m"
}
],
"nginx_http_directives": [
{
"name": "client_body_buffer_size",
"value": "8k"
},
{
"name": "client_max_body_size",
"value": "0"
},
{
"name": "include",
"value": "/usr/local/kong/nginx-http.conf"
},
{
"name": "lua_regex_match_limit",
"value": "100000"
},
{
"name": "lua_shared_dict",
"value": "prometheus_metrics 5m"
},
{
"name": "lua_ssl_protocols",
"value": "TLSv1.1 TLSv1.2 TLSv1.3"
},
{
"name": "ssl_prefer_server_ciphers",
"value": "on"
},
{
"name": "ssl_protocols",
"value": "TLSv1.1 TLSv1.2 TLSv1.3"
},
{
"name": "ssl_session_tickets",
"value": "on"
},
{
"name": "ssl_session_timeout",
"value": "1d"
}
],
"nginx_stream_directives": [
{
"name": "lua_shared_dict",
"value": "stream_prometheus_metrics 5m"
},
{
"name": "lua_ssl_protocols",
"value": "TLSv1.1 TLSv1.2 TLSv1.3"
},
{
"name": "ssl_prefer_server_ciphers",
"value": "on"
},
{
"name": "ssl_protocols",
"value": "TLSv1.1 TLSv1.2 TLSv1.3"
},
{
"name": "ssl_session_tickets",
"value": "on"
},
{
"name": "ssl_session_timeout",
"value": "1d"
}
],
"cassandra_username": "kong",
"nginx_supstream_directives": {
},
"cassandra_timeout": 60000,
"nginx_sproxy_directives": {
},
"anonymous_reports": false,
"db_update_propagation": 0,
"status_ssl_cert": {
},
"pg_host": "stolon-proxy.apigateway.dsh.marathon.mesos",
"nginx_http_upstream_directives": {
},
"nginx_http_status_directives": {
},
"pg_semaphore_timeout": 60000,
"nginx_pid": "/usr/local/kong/pids/nginx.pid",
"client_body_buffer_size": "8k",
"proxy_listen": [
"0.0.0.0:8000 reuseport backlog=16384",
"0.0.0.0:8443 http2 ssl reuseport backlog=16384"
],
"ssl_ciphers": "\"ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256\"",
"enabled_headers": {
"X-Kong-Proxy-Latency": false,
"X-Kong-Response-Latency": false,
"X-Kong-Admin-Latency": false,
"server_tokens": false,
"X-Kong-Upstream-Latency": false,
"X-Kong-Upstream-Status": false,
"Via": false,
"Server": false,
"latency_tokens": false
}
},
"version": "2.8.1",
"lua_version": "LuaJIT 2.1.0-beta3",
"pids": {
"workers": [
40,
4786,
42,
43
],
"master": 35
},
"hostname": "0d9fe41ac71a"
}