Hello,
I’m receiving the error HTTP 499 when we make too many calls (this problem is intermittent).
I’m using kong with the Kubernetes in IBM Cloud.
Front-End → API KONG (on Kubernetes) → API.
I have many GET APIs, sometimes it works and sometimes it returns HTTP 499 error. If I make a simple call (in postman) I don’t get any errors.
Error Logs
kong-gateway XX.XXX.XXX.XXX - - [05/Oct/2022:11:48:44 +0000] "GET /MyPath/ HTTP/1.1" 499 0 "https://upstreamDNS/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36"
logs-application {'route': {'protocols': ['http', 'https'], 'request_buffering': True, 'response_buffering': True, 'created_at': 1662052043, 'updated_at': 1663615817, 'https_redirect_status_code': 426, 'preserve_host': False, 'ws_id': 'XXXXXXXXXXXXXXXXXX', 'regex_priority': 0, 'name': 'MyRoute', 'path_handling': 'v1', 'id': 'XXXXXXXXXXXXXXX', 'service': {'id': 'XXXXXXXXXXXXXXXXX'}, 'strip_path': True, 'methods': ['GET', 'PUT', 'DELETE', 'POST', 'OPTIONS'], 'paths': ['path']}, 'tries': [{'balancer_latency': 0, 'port': 443, 'balancer_start': 1664970520563, 'ip': 'XXX.XXX.XXX.XX'}], 'authenticated_entity': {'id': 'XXXXXXXXXXXXXXXXXXX'}, 'upstream_uri': '/MyPath/', 'response': {'headers': {'connection': 'keep-alive'}, 'size': 0, 'status': 499}, 'latencies': {'proxy': 3500, 'request': 3566, 'kong': 66}, 'consumer': {'username': 'Consumer', 'tags': [], 'created_at': 1657210292, 'id': 'XXXXXXXXXXXXXXXXXXX'}, 'service': {'protocol': 'https', 'host': 'UpstreanDNS', 'created_at': 1660136816, 'updated_at': 1663265214, 'path': '/', 'retries': 5, 'ws_id': 'XXXXXXXXXXXXXXXXXXXXX', 'port': 443, 'enabled': True, 'name': 'MyService', 'id': 'XXXXXXXXXXXXXXXXXX', 'connect_timeout': 60000, 'read_timeout': 60000, 'write_timeout': 60000, 'tags': []}, 'started_at': 1664970520497, 'client_ip': 'XX.XXX.XXX.XXX', 'request': {'headers': {'host': 'MyDNS', 'sec-ch-ua-mobile': '?0', 'x-consumer-groups': 'MyGroup', 'x-consumer-id': 'XXXXXXXXXXXXXXX', 'x-consumer-username': 'Consumer', 'x-credential-identifier': 'XXXXXXXXXXXXXXXXXXXXXXXXXXXX', 'authorization': 'REDACTED', 'content-type': 'application/json;charset=utf-8', 'x-forwarded-proto': 'https', 'accept-language': 'pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7,tr;q=0.6', 'x-real-ip': 'XX.XXX.XXX.XXX', 'x-forwarded-for': 'XX.XXX.XXX.XXX', 'x-forwarded-host': 'MyDNS', 'origin': 'https://UpstreamDNS', 'x-forwarded-scheme': 'https', 'x-request-id': '4ea9de0d1da7567edf37a3608d793e7f', 'referer': 'https://UpstreamDNS/', 'x-forwarded-port': '443', 'x-scheme': 'https', 'sec-fetch-mode': 'cors', 'sec-fetch-site': 'cross-site', 'sec-fetch-dest': 'empty', 'accept-encoding': 'gzip, deflate, br', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36', 'client_id': 'XXXXXXXXXXXXXXXXXXXXX', 'employeid': 'XXXX', 'sec-ch-ua': '"Chromium";v="106", "Google Chrome";v="106", "Not;A=Brand";v="99"', 'sec-ch-ua-platform': '"Windows"', 'accept': 'application/json, text/plain, */*'}, 'size': 1683, 'method': 'GET', 'url': 'http://MyDNS/MyPath', 'uri': '/MyPath/', 'querystring': {'employeeid': 'XXXXX'}}}
Configuration
Deployment
kind: Deployment
apiVersion: apps/v1
metadata:
name: kong-gateway
namespace: default
labels:
k8s-app: kong-gateway
annotations:
deployment.kubernetes.io/revision: '8'
spec:
replicas: 2
selector:
matchLabels:
k8s-app: kong-gateway
template:
metadata:
name: kong-gateway
creationTimestamp: null
labels:
k8s-app: kong-gateway
spec:
containers:
- name: kong-gateway
image: kong:latest
env:
- name: KONG_DATABASE
value: postgres
- name: KONG_PG_HOST
value: database
- name: KONG_PG_USER
value: user
- name: KONG_PG_PASSWORD
value: myPassword
- name: KONG_PROXY_ACCESS_LOG
value: /dev/stdout
- name: KONG_ADMIN_ACCESS_LOG
value: /dev/stdout
- name: KONG_PROXY_ERROR_LOG
value: /dev/stderr
- name: KONG_ADMIN_ERROR_LOG
value: /dev/stderr
- name: KONG_ADMIN_LISTEN
value: 0.0.0.0:8001
- name: KONG_ADMIN_LISTEN_SSL
value: 0.0.0.0:8444
- name: KONG_PROXY_LISTEN
value: 0.0.0.0:8000, 0.0.0.0:8443 ssl
- name: KONG_TRUSTED_IPS
value: 0.0.0.0/0
- name: KONG_NGINX_WORKER_PROCESSES
value: '2'
- name: KONG_KIC
value: 'on'
- name: KONG_ADMIN_ACCESS_LOG
value: /dev/stdout
- name: KONG_ADMIN_ERROR_LOG
value: /dev/stderr
- name: KONG_PROXY_ERROR_LOG
value: /dev/stderr
resources: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
imagePullPolicy: Always
restartPolicy: Always
terminationGracePeriodSeconds: 30
dnsPolicy: ClusterFirst
securityContext: {}
schedulerName: default-scheduler
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 25%
maxSurge: 25%
revisionHistoryLimit: 10
progressDeadlineSeconds: 600
Ingress:
kind: Ingress
apiVersion: networking.k8s.io/v1
metadata:
name: ingress-qas
namespace: default
annotations:
kubernetes.io/ingress.allow-http: 'false'
nginx.ingress.kubernetes.io/configuration-snippet: |
proxy_ignore_client_abort on;
nginx.ingress.kubernetes.io/proxy-connect-timeout: '180'
nginx.ingress.kubernetes.io/proxy-read-timeout: '180'
nginx.ingress.kubernetes.io/proxy-send-timeout: '180'
nginx.ingress.kubernetes.io/proxy_connect_timeout: 5s
managedFields:
- manager: nginx-ingress-controller
operation: Update
apiVersion: networking.k8s.io/v1
time: '2022-10-03T20:46:34Z'
fieldsType: FieldsV1
fieldsV1:
f:status:
f:loadBalancer:
f:ingress: {}
subresource: status
- manager: dashboard
operation: Update
apiVersion: networking.k8s.io/v1
time: '2022-10-05T10:42:16Z'
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.: {}
f:kubernetes.io/ingress.allow-http: {}
f:nginx.ingress.kubernetes.io/configuration-snippet: {}
f:nginx.ingress.kubernetes.io/proxy-connect-timeout: {}
f:nginx.ingress.kubernetes.io/proxy-read-timeout: {}
f:nginx.ingress.kubernetes.io/proxy-send-timeout: {}
f:nginx.ingress.kubernetes.io/proxy_connect_timeout: {}
f:spec:
f:rules: {}
f:tls: {}
spec:
ingressClassName: public-iks-k8s-nginx
tls:
- hosts:
- myhostname1
- myhostname2
- myhostname3
- myhostname4
secretName: server-certificate
rules:
- host: myhostname1
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: konga
port:
number: 80
- host: myhostname2
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: kong-gateway
port:
number: 8000
- host: myhostname3
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: grafana
port:
number: 3000
- host: myhostname4
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: api-doc
port:
number: 80
Service
{
protocol: "https",
tls_verify: null,
host: "UpstreamDNS",
tls_verify_depth: null,
ca_certificates: null,
created_at: 1660136816,
updated_at: 1663265214,
path: "/",
retries: 5,
port: 443,
enabled: true,
client_certificate: null,
name: "ServiceName",
id: "XXXXXXXXXXXXXXXXXXXXXXX",
connect_timeout: 60000,
read_timeout: 60000,
write_timeout: 60000,
tags: [ ]
}
Route
{
protocols: [
"http",
"https"
],
request_buffering: true,
response_buffering: true,
created_at: 1662052043,
updated_at: 1663615817,
https_redirect_status_code: 426,
preserve_host: false,
regex_priority: 0,
snis: null,
name: "RouteName",
path_handling: "v1",
tags: null,
id: "XXXXXXXXXXXXXXXXXXXXXXX",
headers: null,
destinations: null,
sources: null,
service: {
id: "XXXXXXXXXXXXXXXX"
},
strip_path: true,
hosts: null,
methods: [
"GET",
"PUT",
"DELETE",
"POST",
"OPTIONS"
],
paths: [
"/MyPath/"
]
}
Plugins
{
next: null,
data: [
{
name: "cors",
consumer: null,
route: null,
service: {
id: "XXXXXXXXXXXXXXXXXXXXXXXXX"
},
enabled: true,
tags: null,
protocols: [
"grpc",
"grpcs",
"http",
"https"
],
config: {
origins: [
"UpstreamDNS"
],
preflight_continue: false,
exposed_headers: null,
max_age: null,
headers: [
"*"
],
methods: [
"GET",
"HEAD",
"PUT",
"PATCH",
"POST",
"DELETE",
"OPTIONS",
"TRACE",
"CONNECT"
],
credentials: false
},
id: "XXXXXXXXXXXXXXXXXXXXXXXXX",
created_at: 1662058001
},
{
name: "acl",
consumer: null,
route: null,
service: {
id: "XXXXXXXXXXXXXXXXXXXXXXX"
},
enabled: true,
tags: null,
protocols: [
"grpc",
"grpcs",
"http",
"https"
],
config: {
allow: [
"MyGroup"
],
hide_groups_header: false,
deny: null
},
id: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
created_at: 1660140383
},
{
name: "key-auth",
consumer: null,
route: null,
service: {
id: "XXXXXXXXXXXXXXXXXXXXXX"
},
enabled: true,
tags: null,
protocols: [
"grpc",
"grpcs",
"http",
"https"
],
config: {
key_names: [
"client_id"
],
run_on_preflight: true,
key_in_header: true,
anonymous: null,
key_in_body: false,
hide_credentials: false,
key_in_query: true
},
id: "XXXXXXXXXXXXXXXXXXXXX",
created_at: 1660140372
}
]
}
I tryed use the annotations below, without success:
annotations:
kubernetes.io/ingress.allow-http: 'false'
nginx.ingress.kubernetes.io/configuration-snippet: |
proxy_ignore_client_abort on;
nginx.ingress.kubernetes.io/proxy-buffer-size: 160k
nginx.ingress.kubernetes.io/proxy-buffers-number: '64'
nginx.ingress.kubernetes.io/proxy-next-upstream: 'off'
nginx.ingress.kubernetes.io/proxy_buffers: 4 256k
nginx.ingress.kubernetes.io/proxy_busy_buffers_size: 256k
nginx.ingress.kubernetes.io/proxy_connect_timeout: 5s
nginx.ingress.kubernetes.io/reset_timedout_connection: 'on'
nginx.ingress.kubernetes.io/rewrite-target: /
nginx.ingress.kubernetes.io/secure-backends: 'true'
nginx.ingress.kubernetes.io/ssl-redirect: 'true'
Does anyone have any ideas?