Closed louis2557 closed 6 months ago
this is my crd config
Upstream config
Namespace: staging
Labels: <none>
Annotations: <none>
API Version: apisix.apache.org/v2
Kind: ApisixUpstream
Metadata:
Creation Timestamp: 2023-12-20T09:41:45Z
Generation: 1
Managed Fields:
API Version: apisix.apache.org/v2
Fields Type: FieldsV1
fieldsV1:
f:spec:
f:discovery:
f:serviceName:
f:type:
f:subsets:
Manager: Terraform
Operation: Apply
Time: 2023-12-20T09:41:45Z
API Version: apisix.apache.org/v2
Fields Type: FieldsV1
fieldsV1:
f:status:
.:
f:conditions:
Manager: apisix-ingress-controller
Operation: Update
Subresource: status
Time: 2023-12-20T09:41:45Z
Resource Version: 1096002412
UID: 0a11cfdf-9c9c-4677-8094-5699bac81f5b
Spec:
Discovery:
Service Name: staging/moovshop-api-writer-svc:http
Type: kubernetes
Subsets:
Labels:
Version: add-cart
Name: add-cart
Labels:
Version: active-cart
Name: active-cart
Route config
Namespace: staging
Labels: <none>
Annotations: <none>
API Version: apisix.apache.org/v2
Kind: ApisixRoute
Metadata:
Creation Timestamp: 2023-12-20T09:41:50Z
Generation: 1
Managed Fields:
API Version: apisix.apache.org/v2
Fields Type: FieldsV1
fieldsV1:
f:spec:
f:http:
Manager: Terraform
Operation: Apply
Time: 2023-12-20T09:41:50Z
API Version: apisix.apache.org/v2
Fields Type: FieldsV1
fieldsV1:
f:status:
.:
f:conditions:
Manager: apisix-ingress-controller
Operation: Update
Subresource: status
Time: 2023-12-20T09:41:50Z
Resource Version: 1096002517
UID: dad3cd39-a116-46f8-bef1-6007fa0e2245
Spec:
Http:
Backends:
Resolve Granularity: service
Service Name: moovshop-api-writer-svc
Service Port: 80
Subset: add-cart
Match:
Hosts:
uat-api.moovshop.com
Paths:
/api/cart/item/add
Name: add-cart
Plugins:
Config:
allow_credential: true
allow_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
allow_methods: GET,POST,PUT,DELETE,PATCH,HEAD,OPTION,CONNECT,TRACE,PURGE
allow_origins: https://uat.moovshop.com
expose_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
Enable: true
Name: cors
Config:
Functions:
return function(conf, ctx)
function string.starts(String, Start)
return string.sub(String,1,string.len(Start))==Start
end
-- Import neccessary libraries
local http = require("resty.http")
local core = require("apisix.core")
local path = ctx.curr_req_matched._path
local checkDatetime = "true"
if string.starts(path, "/api") and path ~= "/api/login" and string.starts(path, "/api/admin") ~= true then
local pageToken = core.request.header(ctx, "X-Page-Token")
local clientIp = core.request.header(ctx, "X-Forwarded-For")
if not pageToken or pageToken == "" then
core.response.exit(403, { error_msg = "pageToken is invalid" })
end
if not clientIp or clientIp == "" then
core.response.exit(403, { error_msg = "clientIp is invalid" })
end
core.log.info(core.request.headers(ctx))
core.log.info("check token: " .. pageToken .. ", client Ip: " .. clientIp)
-- Send request to extract page token
local httpc = http.new()
local res, err = httpc:request_uri("http://moovshop-extract-token.staging.svc.cluster.local", {
method = "GET",
path = "/getPageTokenPayload?clientIp=" .. clientIp .. "&checkDatetime=" .. checkDatetime,
headers = {
["X-Page-Token"] = pageToken
},
})
if not res then
core.response.exit(403, { error_msg = "extract page token failed" })
end
if res.status ~= 200 then
core.response.exit(403, { error_msg = "extract token failed, code: " .. res.status .. ", reason: " .. res.reason .. ", clientIp: " .. clientIp })
else
local sessionId = string.sub(res.body, 2, -2)
core.log.info("session id from extract token: " .. sessionId)
core.request.add_header(ctx, "X-Session-Id", sessionId)
core.request.set_header(ctx, "X-Page-Token", "")
end
end
end
Enable: true
Name: serverless-pre-function
Backends:
Resolve Granularity: service
Service Name: moovshop-api-writer-svc
Service Port: 80
Subset: active-cart
Match:
Hosts:
uat-api.moovshop.com
Paths:
/api/cart/active
Name: active-cart
Plugins:
Config:
allow_credential: true
allow_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
allow_methods: GET,POST,PUT,DELETE,PATCH,HEAD,OPTION,CONNECT,TRACE,PURGE
allow_origins: https://uat.moovshop.com
expose_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
Enable: true
Name: cors
Config:
Functions:
return function(conf, ctx)
function string.starts(String, Start)
return string.sub(String,1,string.len(Start))==Start
end
-- Import neccessary libraries
local http = require("resty.http")
local core = require("apisix.core")
local path = ctx.curr_req_matched._path
local checkDatetime = "true"
if string.starts(path, "/api") and path ~= "/api/login" and string.starts(path, "/api/admin") ~= true then
local pageToken = core.request.header(ctx, "X-Page-Token")
local clientIp = core.request.header(ctx, "X-Forwarded-For")
if not pageToken or pageToken == "" then
core.response.exit(403, { error_msg = "pageToken is invalid" })
end
if not clientIp or clientIp == "" then
core.response.exit(403, { error_msg = "clientIp is invalid" })
end
core.log.info(core.request.headers(ctx))
core.log.info("check token: " .. pageToken .. ", client Ip: " .. clientIp)
-- Send request to extract page token
local httpc = http.new()
local res, err = httpc:request_uri("http://moovshop-extract-token.staging.svc.cluster.local", {
method = "GET",
path = "/getPageTokenPayload?clientIp=" .. clientIp .. "&checkDatetime=" .. checkDatetime,
headers = {
["X-Page-Token"] = pageToken
},
})
if not res then
core.response.exit(403, { error_msg = "extract page token failed" })
end
if res.status ~= 200 then
core.response.exit(403, { error_msg = "extract token failed, code: " .. res.status .. ", reason: " .. res.reason .. ", clientIp: " .. clientIp })
else
local sessionId = string.sub(res.body, 2, -2)
core.log.info("session id from extract token: " .. sessionId)
core.request.add_header(ctx, "X-Session-Id", sessionId)
core.request.set_header(ctx, "X-Page-Token", "")
end
end
end
Enable: true
Name: serverless-pre-function
@louis2557 You dont need to add
Discovery:
Service Name: staging/moovshop-api-writer-svc:http
Type: kubernetes
in Upstream configuration. Can you try removing that?
@louis2557 You dont need to add
Discovery: Service Name: staging/moovshop-api-writer-svc:http Type: kubernetes
in Upstream configuration. Can you try removing that?
seems not working, I also make some changes. Since I used one service with multiple endpoints, I removed "Resolve Granularity: service" but it is not able to get the endpoints ip for the upstream. The config attribute "nodes" of all upstreams are empty array. dun know how to config
@louis2557 Can you show the output of
kubectl get service moovshop-api-writer-svc -n staging
kubectl get apisixupstream <name of your upstream> -n staging
kubectl get apisixroute <name of your route> -n staging
after you made the above change.
@louis2557 Can you show the output of
kubectl get service moovshop-api-writer-svc -n staging
kubectl get apisixupstream <name of your upstream> -n staging
kubectl get apisixroute <name of your route> -n staging
after you made the above change.
Oh, I found the problem, apisixupstream name must be the same as k8s service name. However, I still got the error in ingress controller, looks like the upstream name is automatically generated which were included subset name but the ingress connect to the upstream name without subset name, how can i to solve it?
the log like this
2023-12-27T16:00:03+08:00 error apisix/apisix_upstream.go:332 failed to get upstream staging_moovshop-api-writer-svc_80: not found 2023-12-27T16:00:03+08:00 warn apisix/apisix_upstream.go:488 sync ApisixUpstream failed, will retry {"object": {"Type":1,"Object":{"Key":"staging/moovshop-api-writer-svc","OldObject":null,"GroupVersion":"apisix.apache.org/v2"},"OldObject":null,"Tombstone":null}, "error": "not found"}
kubectl get service moovshop-api-writer-svc -n staging
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
moovshop-api-writer-svc ClusterIP 172.20.107.19
kubectl get service moovshop-api-writer-svc -n staging
NAME AGE moovshop-api-writer-svc 6m59s
kubectl get apisixroute moovshop-writer-route -n staging
NAME HOSTS URIS AGE moovshop-writer-route ["uat-api.moovshop.com"] ["/api/cart/item/remove"] 7m51s
@louis2557 Yes the upstream name should match the service name. The configuration you pasted didn't have upstream name so I couldn't see that. In my last message I forgot to mention -o yaml
. Can you show the output of the above kubectl commands that I asked but with -o yaml
. I want to confirm the state of these resources when you get this error.
@louis2557 Yes the upstream name should match the service name. The configuration you pasted didn't have upstream name so I couldn't see that. In my last message I forgot to mention
-o yaml
. Can you show the output of the above kubectl commands that I asked but with-o yaml
. I want to confirm the state of these resources when you get this error.
service
apiVersion: v1
kind: Service
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"argocd.argoproj.io/instance":"moovshop-api"},"name":"moovshop-api-writer-svc","namespace":"staging"},"spec":{"ports":[{"name":"http","port":80,"protocol":"TCP","targetPort":80},{"name":"https","port":443,"protocol":"TCP","targetPort":443}],"selector":{"app":"moovshop-api-writer"},"type":"ClusterIP"}}
creationTimestamp: "2023-10-19T07:31:24Z"
labels:
argocd.argoproj.io/instance: moovshop-api
name: moovshop-api-writer-svc
namespace: staging
resourceVersion: "990315141"
uid: 42b5910e-bf03-4340-9f89-edbac0d720eb
spec:
clusterIP: 172.20.107.19
clusterIPs:
- 172.20.107.19
internalTrafficPolicy: Cluster
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
ports:
- name: http
port: 80
protocol: TCP
targetPort: 80
- name: https
port: 443
protocol: TCP
targetPort: 443
selector:
app: moovshop-api-writer
sessionAffinity: None
type: ClusterIP
status:
loadBalancer: {}
apisixupstream
apiVersion: apisix.apache.org/v2
kind: ApisixUpstream
metadata:
creationTimestamp: "2023-12-27T07:55:58Z"
generation: 1
name: moovshop-api-writer-svc
namespace: staging
resourceVersion: "1107274261"
uid: 54510349-a106-479f-a806-3a260aebf478
spec:
subsets:
- labels:
version: add-cart
name: add-cart
- labels:
version: active-cart
name: active-cart
- labels:
version: checkout
name: checkout
- labels:
version: live
name: live
status:
conditions:
- message: not found
observedGeneration: 1
reason: ResourceSyncAborted
status: "False"
type: ResourcesAvailable
apisixroute
apiVersion: apisix.apache.org/v2
kind: ApisixRoute
metadata:
creationTimestamp: "2023-12-27T07:56:01Z"
generation: 2
name: moovshop-writer-route
namespace: staging
resourceVersion: "1107366295"
uid: 44ddcadf-8d51-47c7-ab85-f76b2551afe6
spec:
http:
- match:
hosts:
- uat-api.moovshop.com
paths:
- /api/cart/item/remove
name: remove-cart
plugins:
- config:
allow_credential: true
allow_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
allow_methods: GET,POST,PUT,DELETE,PATCH,HEAD,OPTION,CONNECT,TRACE,PURGE
allow_origins: https://uat.moovshop.com
expose_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
enable: true
name: cors
- config:
functions:
- |2-
...
enable: true
name: serverless-pre-function
upstreams:
- name: moovshop-api-writer-svc
- match:
hosts:
- uat-api.moovshop.com
paths:
- /api/cart/item/add
name: add-cart
plugins:
- config:
allow_credential: true
allow_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
allow_methods: GET,POST,PUT,DELETE,PATCH,HEAD,OPTION,CONNECT,TRACE,PURGE
allow_origins: https://uat.moovshop.com
expose_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
enable: true
name: cors
- config:
functions:
- |2-
...
enable: true
name: serverless-pre-function
upstreams:
- name: moovshop-api-writer-svc
- match:
hosts:
- uat-api.moovshop.com
paths:
- /api/cart/active
name: active-cart
plugins:
- config:
allow_credential: true
allow_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
allow_methods: GET,POST,PUT,DELETE,PATCH,HEAD,OPTION,CONNECT,TRACE,PURGE
allow_origins: https://uat.moovshop.com
expose_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
enable: true
name: cors
- config:
functions:
- |2-
...
enable: true
name: serverless-pre-function
upstreams:
- name: moovshop-api-writer-svc
- match:
hosts:
- uat-api.moovshop.com
paths:
- /api/cart/checkout
name: checkout
plugins:
- config:
allow_credential: true
allow_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
allow_methods: GET,POST,PUT,DELETE,PATCH,HEAD,OPTION,CONNECT,TRACE,PURGE
allow_origins: https://uat.moovshop.com
expose_headers: X-Forwarded-For,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization,X-Page-Token,Access-Control-Allow-Origin
enable: true
name: cors
- config:
functions:
- |2-
...
enable: true
name: serverless-pre-function
upstreams:
- name: moovshop-api-writer-svc
status:
conditions:
- message: "4 errors occurred:\n\t* unexpected status code 400; error message: {\"error_msg\":\"failed
to fetch upstream info by upstream id [fb04cf71], response code: 404\"}\n\n\t*
unexpected status code 400; error message: {\"error_msg\":\"failed to fetch
upstream info by upstream id [fb04cf71], response code: 404\"}\n\n\t* unexpected
status code 400; error message: {\"error_msg\":\"failed to fetch upstream info
by upstream id [fb04cf71], response code: 404\"}\n\n\t* unexpected status code
400; error message: {\"error_msg\":\"failed to fetch upstream info by upstream
id [fb04cf71], response code: 404\"}\n\n\n"
observedGeneration: 2
reason: ResourceSyncAborted
status: "False"
type: ResourcesAvailable
@louis2557 Did you remove
Discovery:
Service Name: staging/moovshop-api-writer-svc:http
Type: kubernetes
in Upstream configuration. Also did you change the name of ApisixUpstream? Can you delete and create the ApisixUpstream yaml again?
@louis2557 Can you show the output of
kubectl get service moovshop-api-writer-svc -n staging
kubectl get apisixupstream <name of your upstream> -n staging
kubectl get apisixroute <name of your route> -n staging
after you made the above change.Oh, I found the problem, apisixupstream name must be the same as k8s service name. However, I still got the error in ingress controller, looks like the upstream name is automatically generated which were included subset name but the ingress connect to the upstream name without subset name, how can i to solve it?
the log like this
2023-12-27T16:00:03+08:00 error apisix/apisix_upstream.go:332 failed to get upstream staging_moovshop-api-writer-svc_80: not found 2023-12-27T16:00:03+08:00 warn apisix/apisix_upstream.go:488 sync ApisixUpstream failed, will retry {"object": {"Type":1,"Object":{"Key":"staging/moovshop-api-writer-svc","OldObject":null,"GroupVersion":"apisix.apache.org/v2"},"OldObject":null,"Tombstone":null}, "error": "not found"}
kubectl get service moovshop-api-writer-svc -n staging
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE moovshop-api-writer-svc ClusterIP 172.20.107.19 80/TCP,443/TCP 69d
kubectl get service moovshop-api-writer-svc -n staging
NAME AGE moovshop-api-writer-svc 6m59s
kubectl get apisixroute moovshop-writer-route -n staging
NAME HOSTS URIS AGE moovshop-writer-route ["uat-api.moovshop.com"] ["/api/cart/item/remove"] 7m51s
@louis2557 Please also show the output of kubectl get apisixupstream -n staging -o yaml
This issue has been marked as stale due to 90 days of inactivity. It will be closed in 30 days if no further activity occurs. If this issue is still relevant, please simply write any comment. Even if closed, you can still revive the issue at any time or discuss it on the dev@apisix.apache.org list. Thank you for your contributions.
This issue has been closed due to lack of activity. If you think that is incorrect, or the issue requires additional review, you can revive the issue at any time.
Current Behavior
Hi, I used k8s crd to create apisixroute and apisixupstream, I also add subset and subsets config to both, but the taffic can't be routed to appropriate pods which contain corresponding labels
Expected Behavior
the route traffic can be routed to appropriate pods which contain corresponding labels
Error Logs
2023-12-27T14:50:01+08:00 error apisix/apisix_upstream.go:236 failed to get service staging/moovshop-cms-api-upstream: service "moovshop-cms-api-upstream" not found 2023-12-27T14:50:01+08:00 info apisix/apisix_upstream.go:481 sync ApisixUpstream but not found, ignore {"event_type": "sync", "ApisixUpstream": {"Key":"staging/moovshop-cms-api-upstream","OldObject":null,"GroupVersion":"apisix.apache.org/v2"}}
Steps to Reproduce