Hi everyone,
I have run and setup kong v2.x.x before and with that old version i haven’t any issue with ACME certificate renewal.
But now, since i recently install new one with v3.5.0, i just noticed that my ACME certificate is not renewed. I just setup a high number in renew_threshold_days to 89 but i cannot see any acme renew activity in /usr/local/kong/logs/error.log despite i already debug log level.
Please help me to figure out why my certificate is not renewed. thanks!
Below is my acme config
{
"created_at": 1703085411,
"updated_at": 1703929755,
"name": "acme",
"instance_name": "plugins-acme-global",
"enabled": true,
"service": null,
"route": null,
"id": "d8ecc715-e25f-4789-aa66-1ccf35fe355f",
"tags": null,
"protocols": [
"grpc",
"grpcs",
"http",
"https"
],
"config": {
"api_uri": "https://acme-v02.api.letsencrypt.org/directory",
"account_email": "*****",
"renew_threshold_days": 89,
"fail_backoff_minutes": 5,
"storage_config": {
"kong": {},
"vault": {
"token": null,
"tls_server_name": null,
"tls_verify": true,
"auth_path": null,
"https": false,
"jwt_path": null,
"auth_method": "token",
"host": null,
"auth_role": null,
"timeout": null,
"port": null,
"kv_path": null
},
"consul": {
"token": null,
"host": null,
"timeout": null,
"kv_path": null,
"port": null,
"https": false
},
"redis": {
"ssl_verify": false,
"ssl_server_name": null,
"namespace": "",
"scan_count": 10,
"database": null,
"port": null,
"host": null,
"ssl": false,
"auth": null
},
"shm": {
"shm_name": "kong"
}
},
"domains": [
],
"storage": "kong",
"account_key": null,
"eab_kid": null,
"eab_hmac_key": null,
"preferred_chain": null,
"cert_type": "rsa",
"rsa_key_size": 4096,
"tos_accepted": true,
"enable_ipv4_common_name": true,
"allow_any_domain": false
},
"consumer": null
}
and below is kong config
{
"plugins": {
"available_on_server": {},
"enabled_in_cluster": [
"prometheus",
"acme"
]
},
"node_id": "528a3e9e-538a-4969-8783-edce00c37eaf",
"version": "3.5.0",
"tagline": "Welcome to kong",
"hostname": "it-kong-cp",
"configuration": {
"nginx_inject_conf": "/usr/local/kong/nginx-inject.conf",
"stream_proxy_ssl_enabled": false,
"nginx_kong_inject_conf": "/usr/local/kong/nginx-kong-inject.conf",
"admin_gui_ssl_enabled": true,
"dns_resolver": {},
"enabled_headers_upstream": {
"X-Kong-Request-Id": true
},
"dns_hostsfile": "/etc/hosts",
"lua_package_path": "./?.lua;./?/init.lua;",
"ssl_cert_csr_default": "/usr/local/kong/ssl/kong-default.csr",
"ssl_cert_default": "/usr/local/kong/ssl/kong-default.crt",
"dns_error_ttl": 1,
"ssl_cert_key_default": "/usr/local/kong/ssl/kong-default.key",
"dns_not_found_ttl": 30,
"ssl_cert_default_ecdsa": "/usr/local/kong/ssl/kong-default-ecdsa.crt",
"dns_stale_ttl": 4,
"ssl_cert_key_default_ecdsa": "/usr/local/kong/ssl/kong-default-ecdsa.key",
"dns_cache_size": 10000,
"client_ssl_cert_default": "/usr/local/kong/ssl/kong-default.crt",
"dns_order": [
"LAST",
"SRV",
"A",
"CNAME"
],
"admin_ssl_cert_default": "/usr/local/kong/ssl/admin-kong-default.crt",
"dns_no_sync": true,
"admin_ssl_cert_key_default": "/usr/local/kong/ssl/admin-kong-default.key",
"admin_ssl_cert_default_ecdsa": "/usr/local/kong/ssl/admin-kong-default-ecdsa.crt",
"upstream_keepalive_pool_size": 512,
"admin_gui_ssl_cert_default": "/usr/local/kong/ssl/admin-gui-kong-default.crt",
"admin_gui_ssl_cert_key_default": "/usr/local/kong/ssl/admin-gui-kong-default.key",
"plugins": [
"bundled"
],
"admin_gui_ssl_cert_default_ecdsa": "/usr/local/kong/ssl/admin-gui-kong-default-ecdsa.crt",
"admin_gui_path": "/",
"pluginserver_names": {},
"status_ssl_cert_default": "/usr/local/kong/ssl/status-kong-default.crt",
"status_ssl_cert_key_default": "/usr/local/kong/ssl/status-kong-default.key",
"upstream_keepalive_idle_timeout": 60,
"upstream_keepalive_max_requests": 1000,
"status_ssl_cert_key_default_ecdsa": "/usr/local/kong/ssl/status-kong-default-ecdsa.key",
"port_maps": {},
"proxy_listen": [
"0.0.0.0:8000 reuseport backlog=16384",
"0.0.0.0:8443 http2 ssl reuseport backlog=16384"
],
"tracing_instrumentations": [
"off"
],
"admin_gui_listen": [
"172.19.x.x:8002",
"172.19.x.x:8445 ssl"
],
"status_listen": [
"off"
],
"stream_listen": [
"off"
],
"ssl_protocols": "TLSv1.1 TLSv1.2 TLSv1.3",
"admin_ssl_cert": [
"/usr/local/kong/ssl/admin-kong-default.crt",
"/usr/local/kong/ssl/admin-kong-default-ecdsa.crt"
],
"nginx_http_ssl_protocols": "TLSv1.2 TLSv1.3",
"nginx_stream_ssl_protocols": "TLSv1.2 TLSv1.3",
"ssl_prefer_server_ciphers": "on",
"nginx_http_ssl_prefer_server_ciphers": "off",
"nginx_stream_ssl_prefer_server_ciphers": "off",
"ssl_cipher_suite": "intermediate",
"nginx_http_ssl_dhparam": "ffdhe2048",
"nginx_stream_ssl_dhparam": "ffdhe2048",
"ssl_ciphers": "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384",
"nginx_http_ssl_session_tickets": "on",
"nginx_stream_ssl_session_tickets": "on",
"ssl_session_timeout": "1d",
"db_cache_warmup_entities": [
"services"
],
"nginx_stream_ssl_session_timeout": "1d",
"ssl_session_cache_size": "10m",
"proxy_access_log": "logs/access.log",
"proxy_error_log": "logs/error.log",
"proxy_stream_access_log": "logs/access.log basic",
"headers": [
"server_tokens",
"latency_tokens",
"x-kong-request-id"
],
"lua_package_cpath": "",
"status_ssl_cert_key": "******",
"db_resurrect_ttl": 30,
"nginx_user": "kong kong",
"status_access_log": "off",
"status_error_log": "logs/status_error.log",
"lua_ssl_trusted_certificate": [
"/etc/ssl/certs/ca-certificates.crt"
],
"lua_ssl_verify_depth": 1,
"ssl_cert_key": "******",
"nginx_stream_lua_ssl_protocols": "TLSv1.1 TLSv1.2 TLSv1.3",
"lua_socket_pool_size": 256,
"headers_upstream": [
"x-kong-request-id"
],
"trusted_ips": {},
"real_ip_header": "X-Real-IP",
"nginx_proxy_real_ip_header": "X-Real-IP",
"real_ip_recursive": "off",
"nginx_proxy_real_ip_recursive": "off",
"pg_port": 5432,
"pg_password": "******",
"pg_ssl": false,
"pg_ssl_verify": false,
"pg_max_concurrent_queries": 0,
"pg_semaphore_timeout": 60000,
"_debug_pg_ttl_cleanup_interval": 300,
"kic": false,
"pg_ro_ssl": false,
"pg_ro_ssl_verify": false,
"lua_max_req_headers": 100,
"lua_max_resp_headers": 100,
"worker_consistency": "eventual",
"lua_max_uri_args": 100,
"lua_max_post_args": 100,
"allow_debug_header": false,
"enabled_headers": {
"Server": true,
"X-Kong-Upstream-Latency": true,
"X-Kong-Upstream-Status": false,
"server_tokens": true,
"latency_tokens": true,
"X-Kong-Admin-Latency": true,
"X-Kong-Proxy-Latency": true,
"X-Kong-Request-Id": true,
"X-Kong-Response-Latency": true,
"Via": true
},
"cluster_listeners": [
{
"so_keepalive=on": false,
"so_keepalive=off": false,
"bind": false,
"listener": "172.19.x.x:8005",
"port": 8005,
"ip": "172.19.x.x",
"so_keepalive=%w*:%w*:%d*": false,
"ssl": false,
"http2": false,
"proxy_protocol": false,
"deferred": false,
"reuseport": false,
"backlog=%d+": false,
"ipv6only=on": false,
"ipv6only=off": false
}
],
"router_flavor": "traditional_compatible",
"status_ssl_enabled": false,
"status_listeners": {},
"admin_error_log": "logs/error.log",
"vaults": [
"bundled"
],
"admin_ssl_enabled": true,
"loaded_vaults": {
"env": true
},
"proxy_ssl_enabled": true,
"loaded_plugins": {
"jwt": true,
"acl": true,
"correlation-id": true,
"cors": true,
"oauth2": true,
"tcp-log": true,
"udp-log": true,
"file-log": true,
"http-log": true,
"key-auth": true,
"hmac-auth": true,
"basic-auth": true,
"ip-restriction": true,
"request-transformer": true,
"response-transformer": true,
"request-size-limiting": true,
"rate-limiting": true,
"response-ratelimiting": true,
"syslog": true,
"loggly": true,
"datadog": true,
"ldap-auth": true,
"statsd": true,
"bot-detection": true,
"aws-lambda": true,
"request-termination": true,
"prometheus": true,
"session": true,
"acme": true,
"grpc-gateway": true,
"grpc-web": true,
"pre-function": true,
"post-function": true,
"azure-functions": true,
"zipkin": true,
"opentelemetry": true,
"proxy-cache": true
},
"nginx_http_lua_regex_cache_max_entries": "8192",
"request_debug": true,
"proxy_listeners": [
{
"so_keepalive=on": false,
"so_keepalive=off": false,
"bind": false,
"listener": "0.0.0.0:8000 reuseport backlog=16384",
"port": 8000,
"ip": "0.0.0.0",
"so_keepalive=%w*:%w*:%d*": false,
"ipv6only=off": false,
"http2": false,
"proxy_protocol": false,
"deferred": false,
"reuseport": true,
"ssl": false,
"ipv6only=on": false,
"backlog=16384": true
},
{
"so_keepalive=on": false,
"so_keepalive=off": false,
"bind": false,
"listener": "0.0.0.0:8443 ssl http2 reuseport backlog=16384",
"port": 8443,
"ip": "0.0.0.0",
"so_keepalive=%w*:%w*:%d*": false,
"ipv6only=off": false,
"http2": true,
"proxy_protocol": false,
"deferred": false,
"reuseport": true,
"ssl": true,
"ipv6only=on": false,
"backlog=16384": true
}
],
"admin_listeners": [
{
"so_keepalive=on": false,
"so_keepalive=off": false,
"bind": false,
"listener": "172.19.x.x:8001 reuseport backlog=16384",
"port": 8001,
"ip": "172.19.x.x",
"so_keepalive=%w*:%w*:%d*": false,
"ipv6only=off": false,
"http2": false,
"proxy_protocol": false,
"deferred": false,
"reuseport": true,
"ssl": false,
"ipv6only=on": false,
"backlog=16384": true
},
{
"so_keepalive=on": false,
"so_keepalive=off": false,
"bind": false,
"listener": "172.19.x.x:8444 ssl http2 reuseport backlog=16384",
"port": 8444,
"ip": "172.19.x.x",
"so_keepalive=%w*:%w*:%d*": false,
"ipv6only=off": false,
"http2": true,
"proxy_protocol": false,
"deferred": false,
"reuseport": true,
"ssl": true,
"ipv6only=on": false,
"backlog=16384": true
}
],
"mem_cache_size": "128m",
"nginx_main_worker_rlimit_nofile": "auto",
"cluster_max_payload": 16777216,
"nginx_events_worker_connections": "auto",
"cluster_listen": [
"172.19.x.x:8005"
],
"nginx_events_multi_accept": "on",
"cluster_cert_key": "******",
"nginx_http_charset": "UTF-8",
"dedicated_config_processing": true,
"nginx_http_client_max_body_size": "0",
"admin_gui_ssl_cert": [
"/usr/local/kong/ssl/admin-gui-kong-default.crt",
"/usr/local/kong/ssl/admin-gui-kong-default-ecdsa.crt"
],
"log_level": "debug",
"status_ssl_cert": {},
"nginx_http_lua_ssl_protocols": "TLSv1.1 TLSv1.2 TLSv1.3",
"proxy_stream_error_log": "logs/error.log",
"lua_ssl_trusted_certificate_combined": "/usr/local/kong/.ca_combined",
"nginx_kong_stream_inject_conf": "/usr/local/kong/nginx-kong-stream-inject.conf",
"nginx_status_directives": {},
"worker_events_max_payload": 65535,
"nginx_http_lua_regex_match_limit": "100000",
"nginx_proxy_directives": [
{
"name": "real_ip_header",
"value": "X-Real-IP"
},
{
"name": "real_ip_recursive",
"value": "off"
}
],
"nginx_upstream_directives": {},
"nginx_http_directives": [
{
"name": "charset",
"value": "UTF-8"
},
{
"name": "client_body_buffer_size",
"value": "8k"
},
{
"name": "client_max_body_size",
"value": "0"
},
{
"name": "keepalive_requests",
"value": "1000"
},
{
"name": "lua_regex_cache_max_entries",
"value": "8192"
},
{
"name": "lua_regex_match_limit",
"value": "100000"
},
{
"name": "lua_shared_dict",
"value": "prometheus_metrics 5m"
},
{
"name": "ssl_dhparam",
"value": "/usr/local/kong/ssl/ffdhe2048.pem"
},
{
"name": "ssl_prefer_server_ciphers",
"value": "off"
},
{
"name": "ssl_protocols",
"value": "TLSv1.2 TLSv1.3"
},
{
"name": "ssl_session_tickets",
"value": "on"
},
{
"name": "ssl_session_timeout",
"value": "1d"
}
],
"nginx_http_keepalive_requests": "1000",
"nginx_events_directives": [
{
"name": "multi_accept",
"value": "on"
},
{
"name": "worker_connections",
"value": "auto"
}
],
"db_update_frequency": 5,
"role": "control_plane",
"db_update_propagation": 0,
"admin_gui_listeners": [
{
"so_keepalive=on": false,
"so_keepalive=off": false,
"bind": false,
"listener": "172.19.x.x:8002",
"port": 8002,
"ip": "172.19.x.x",
"so_keepalive=%w*:%w*:%d*": false,
"ssl": false,
"http2": false,
"proxy_protocol": false,
"deferred": false,
"reuseport": false,
"backlog=%d+": false,
"ipv6only=on": false,
"ipv6only=off": false
},
{
"so_keepalive=on": false,
"so_keepalive=off": false,
"bind": false,
"listener": "172.19.x.x:8445 ssl",
"port": 8445,
"ip": "172.19.x.x",
"so_keepalive=%w*:%w*:%d*": false,
"ssl": true,
"http2": false,
"proxy_protocol": false,
"deferred": false,
"reuseport": false,
"backlog=%d+": false,
"ipv6only=on": false,
"ipv6only=off": false
}
],
"database": "postgres",
"lua_ssl_protocols": "TLSv1.1 TLSv1.2 TLSv1.3",
"stream_listeners": {},
"db_cache_ttl": 0,
"nginx_admin_client_max_body_size": "10m",
"admin_ssl_cert_key_default_ecdsa": "/usr/local/kong/ssl/admin-kong-default-ecdsa.key",
"nginx_admin_client_body_buffer_size": "10m",
"nginx_kong_gui_include_conf": "/usr/local/kong/nginx-kong-gui-include.conf",
"client_body_buffer_size": "8k",
"nginx_main_daemon": "on",
"nginx_daemon": "on",
"host_ports": {},
"anonymous_reports": true,
"pg_host": "127.0.0.1",
"tracing_sampling_rate": 0.01,
"client_ssl_cert_key_default": "/usr/local/kong/ssl/kong-default.key",
"pg_timeout": 60000,
"pg_database": "kong",
"worker_state_update_frequency": 5,
"prefix": "/usr/local/kong",
"nginx_main_directives": [
{
"name": "daemon",
"value": "on"
},
{
"name": "user",
"value": "kong kong"
},
{
"name": "worker_processes",
"value": "auto"
},
{
"name": "worker_rlimit_nofile",
"value": "auto"
}
],
"cluster_data_plane_purge_delay": 1209600,
"cluster_ocsp": "off",
"pg_user": "kong",
"cluster_use_proxy": false,
"cluster_dp_labels": {},
"untrusted_lua": "sandbox",
"status_ssl_cert_default_ecdsa": "/usr/local/kong/ssl/status-kong-default-ecdsa.crt",
"untrusted_lua_sandbox_requires": {},
"untrusted_lua_sandbox_environment": {},
"lmdb_environment_path": "dbless.lmdb",
"lmdb_map_size": "2048m",
"opentelemetry_tracing": [
"off"
],
"opentelemetry_tracing_sampling_rate": 0.01,
"kong_process_secrets": "/usr/local/kong/.kong_process_secrets",
"proxy_server_ssl_verify": true,
"kong_env": "/usr/local/kong/.kong_env",
"nginx_admin_directives": [
{
"name": "client_body_buffer_size",
"value": "10m"
},
{
"name": "client_max_body_size",
"value": "10m"
}
],
"admin_gui_ssl_cert_key_default_ecdsa": "/usr/local/kong/ssl/admin-gui-kong-default-ecdsa.key",
"nginx_stream_directives": [
{
"name": "lua_shared_dict",
"value": "stream_prometheus_metrics 5m"
},
{
"name": "ssl_dhparam",
"value": "/usr/local/kong/ssl/ffdhe2048.pem"
},
{
"name": "ssl_prefer_server_ciphers",
"value": "off"
},
{
"name": "ssl_protocols",
"value": "TLSv1.2 TLSv1.3"
},
{
"name": "ssl_session_tickets",
"value": "on"
},
{
"name": "ssl_session_timeout",
"value": "1d"
}
],
"ssl_dhparam": "ffdhe2048",
"nginx_supstream_directives": {},
"nginx_main_user": "kong kong",
"nginx_sproxy_directives": {},
"admin_gui_access_log": "logs/admin_gui_access.log",
"cluster_cert": "/etc/kong/certificate/cluster.crt",
"nginx_wasm_wasmtime_directives": {},
"admin_gui_ssl_cert_key": "******",
"nginx_wasm_v8_directives": {},
"nginx_http_client_body_buffer_size": "8k",
"nginx_wasm_wasmer_directives": {},
"cluster_mtls": "shared",
"nginx_wasm_main_shm_directives": {},
"client_ssl": false,
"nginx_wasm_main_directives": {},
"ssl_cert": [
"/usr/local/kong/ssl/kong-default.crt",
"/usr/local/kong/ssl/kong-default-ecdsa.crt"
],
"ssl_session_tickets": "on",
"admin_gui_error_log": "logs/admin_gui_error.log",
"nginx_pid": "/usr/local/kong/pids/nginx.pid",
"admin_access_log": "logs/admin_access.log",
"nginx_err_logs": "/usr/local/kong/logs/error.log",
"wasm": false,
"admin_listen": [
"172.19.x.x:8001 reuseport backlog=16384",
"172.19.x.x:8444 http2 ssl reuseport backlog=16384"
],
"nginx_acc_logs": "/usr/local/kong/logs/access.log",
"cluster_control_plane": "127.0.0.1:8005",
"admin_acc_logs": "/usr/local/kong/logs/admin_access.log",
"nginx_main_worker_processes": "auto",
"nginx_conf": "/usr/local/kong/nginx.conf",
"nginx_worker_processes": "auto",
"error_default_type": "text/plain",
"nginx_http_ssl_session_timeout": "1d",
"nginx_kong_conf": "/usr/local/kong/nginx-kong.conf",
"admin_ssl_cert_key": "******",
"nginx_kong_stream_conf": "/usr/local/kong/nginx-kong-stream.conf"
},
"pids": {
"master": 129279,
"workers": [
129280,
129281,
129282,
129283
]
},
"lua_version": "LuaJIT 2.1.0-20230410",
"timers": {
"running": 33,
"pending": 1
}
}