Send http Request by dispatch to remote cluster block

[ray1888]

i was using the proxywasm go sdk to write a wasm plugin , code are below

type Config struct {
    AclServiceHost string `json:"acl_service_host"`
    AclServicePort string `json:"acl_service_port"`
    ClusterName    string `json:"cluster_name"`
    RunMode        string `json:"run_mode"`
        Path                string `json:"path"`
}

func Load(data []byte, conf *Config) error {
    err := json.Unmarshal(data, conf)
    if err != nil {
        return err
    }
    return nil
}

type aclPluginContext struct {
    // Embed the default plugin context here,
    // so that we don't need to reimplement all the methods.
    types.DefaultPluginContext
    contextID uint32
    // callBack  func(numHeaders, bodySize, numTrailers int)
    // cnt       int
    conf  Config
}

// aclPluginContext implements types.VMContext.
type vmContext struct {
    // Embed the default VM context here,
    // so that we don't need to reimplement all the methods.
    types.DefaultVMContext
}

type httpContext struct {
    // Embed the default http context here,
    // so that we don't need to reimplement all the methods.
    types.DefaultHttpContext
    // contextID is the unique identifier assigned to each httpContext.
    contextID   uint32
    clusterName string
    runMode     string
        path           string 
    aclHost     string
    aclPort     string
}

func (ctx *httpContext) OnHttpResponseHeaders(int, bool) types.Action {
    return types.ActionContinue
}

// Bussine logic lies
func (ctx *httpContext) OnHttpRequestHeaders(int, bool) types.Action {
    proxywasm.LogDebugf("IN OnHttpRequestBody")
    hs, err := proxywasm.GetHttpRequestHeaders()
    if err != nil {
        proxywasm.LogErrorf("failed to get request headers: %v", err)
        return types.ActionContinue
    }

    var clientIp string

        ipByte, err := proxywasm.GetProperty([]string{"ngx", "remote_addr"})
        if err != nil {
        proxywasm.LogErrorf("get real ip error is %v", err)
    }
    if len(ipByte) > 0 {
        proxywasm.LogDebugf("set client ip from ngx get x-real-ip ")
        clientIp = string(ipByte)
    }

    if len(tenantId) == 0 {
        proxywasm.LogDebug("Tenant id is 0")
        return types.ActionContinue
    }

    if len(clientIp) == 0 {
        proxywasm.LogDebug("clientIP is empty")
        return types.ActionContinue
    }

    proxywasm.LogDebugf("clientip is %s, tenantId is %s", clientIp, tenantId)
    type reqs struct{
            A string `json:"a"`
        }
        req := reqs{A:"hello world"}
    body, err := json.Marshal(&req)
    if err != nil {
        proxywasm.LogErrorf("json encode body failed, err is %v", err)
        return types.ActionContinue
    }

    proxywasm.LogDebugf("req raw is %v", req)
    proxywasm.LogDebugf("body raw is %v", string(body))

    proxywasm.LogDebugf("http call dispatched to %s, contextId %d", ctx.clusterName, ctx.contextID)

    host := "kong-provy"

    _, err = proxywasm.DispatchHttpCall(ctx.clusterName, [][2]string{
        {":path", ctx.path},
        {":method", "POST"},
        {":scheme", "http"},
        {":authority", host},
    }, body, nil, 500, func(numHeaders, bodySize, numTrailers int) {
        proxywasm.LogDebug("httpCallResponseCallback callback!")

        _, err := proxywasm.GetHttpCallResponseBody(0, bodySize)
        if err != nil {
            proxywasm.LogErrorf("failed to get response body: %v", err)
            _ = proxywasm.ResumeHttpRequest()
            return
        }

        err = proxywasm.ResumeHttpRequest()
        proxywasm.LogDebugf("resume http header err is %v", err)
    })

    if err != nil {
        proxywasm.LogErrorf("get debug err is %v", err)
        return types.ActionContinue 
    }

    return types.ActionPause

}

// Plugin reation config and new httpContext
func (p *aclPluginContext) NewHttpContext(contextID uint32) types.HttpContext {
    proxywasm.LogInfof("plugin config fron plugin ctx : %v", p.conf.ClusterName)
    return &httpContext{
        contextID:   contextID,
        clusterName: p.conf.ClusterName,
        runMode:     p.conf.RunMode,
        aclHost:     p.conf.AclServiceHost,
        aclPort:     p.conf.AclServicePort,
                path:          p.conf.Path,
    }
}

// OnPluginStart implements types.PluginContext.
func (ctx *aclPluginContext) OnPluginStart(pluginConfigurationSize int) types.OnPluginStartStatus {
    data, err := proxywasm.GetPluginConfiguration()
    if err != nil {
        proxywasm.LogCriticalf("error reading plugin configuration: %v", err)
    }
    err = Load(data, &ctx.conf)
    if err != nil {
        proxywasm.LogCriticalf("error parsing plugin configuration: %v", err)
        return types.OnPluginStartStatusFailed
    }

    proxywasm.LogInfof("plugin config: %s", string(data))
    proxywasm.LogInfof("plugin config fron plugin ctx : %v", ctx.conf)
    return types.OnPluginStartStatusOK
}

// VM init
func (*vmContext) OnVMStart(vmConfigurationSize int) types.OnVMStartStatus {
    data, err := proxywasm.GetVMConfiguration()
    if err != nil {
        proxywasm.LogCriticalf("error reading vm configuration: %v", err)
    }

    proxywasm.LogInfof("vm config: %s", string(data))
    return types.OnVMStartStatusOK
}

func (*vmContext) NewPluginContext(uint32) types.PluginContext {
    return &aclPluginContext{}
}

func main() {
    proxywasm.SetVMContext(&vmContext{})
}

the process i support is after call a api( as long as the endpoint exist), the filter can goon to call the regular service, but is turn out, it just block there.

kong logs show like

after filters calling clusterremote , but not of the upstream is call .

how to reproduce

1. use tinygo to build the program above

   tinygo build -o filter.wasm  main.go

2. mkdir dir for config and wasm folder
3. save kong config , replace $some_backend1 and $some_backend2 with differnent backend( i think you can use two nginx with different port)

   
   _format_version: "1.1"
   _transform: true

services:

• name: reverse-proxy url: http://gitee routes:
  • name: root pass strip_path: false paths:
  • / filter_chains:
  • filters:
    • name: filter config: cluster_name: $some_backend run_mode: "kong" path: "/hello"

upstreams:

• name: gitee targets:
  • target: $some_backend2 weight: 100

5.  use docker with script like this 

set -x

DEMO_KONG_CONTAINER="${DEMO_KONG_CONTAINER:-kong-wasm}" DEMO_KONG_IMAGE="${DEMO_KONG_IMAGE:-kong/kong:nightly}"

script_dir=$(dirname $(realpath $0))

docker stop $DEMO_KONG_CONTAINER docker rm $DEMO_KONG_CONTAINER

access_localhost=""

docker run -d --name "$DEMO_KONG_CONTAINER" \ $access_localhost \ -v "$script_dir/config:/kong/config" \ -v "$script_dir/wasm:/wasm" \ -e "KONG_LOG_LEVEL=debug" \ -e "KONG_DATABASE=off" \ -e "KONG_DECLARATIVE_CONFIG=/kong/config/demo.yml" \ -e "KONG_NGINX_WASM_SHM_KV_KONG_WASM_RATE_LIMITING_COUNTERS=12m" \ -e "KONG_PROXY_ACCESS_LOG=/dev/stdout" \ -e "KONG_PROXY_ERROR_LOG=/dev/stderr" \ -e "KONG_WASM=on" \ -e "KONG_WASM_FILTERS_PATH=/wasm" \ --net=host \ "$DEMO_KONG_IMAGE"

6. curl to localhost:8000

[ray1888]

FYI: base environment is below

OS: centos 7.9 aarch64 (kernal_verison: 4.18.0-348.20.1.el7.aarch64 ) Docker verison: 20.10.24 Kong Version: 3.7.1

[thibaultcha]

While I didn't really understand what the issue was, I assume it is some issue with the external dispatch call. We are aware of some dispatch issues in the module in Kong 3.7.1. The latest release of the module has a number of fixes related to dispatch calls (see changelog). For now, I suggest waiting for a new Kong release that embeds a newer version of the module, which should come soon.

[thibaultcha]

I checked with my team and the timeline for a Kong release with this module's dispatch fixes is still several weeks away. If in the meantime you wish to keep developing your filter, I suggest working on a local build of ngx_wasm_module. It is very easy to work on the module locally and develop your filter against it. Then later when the new Kong release has the latest ngx_wasm_module, you can load your filter with it. To work with ngx_wasm_module locally consult DEVELOPER.md which has all the instructions needed (make setup and make are almost all that you need). Then you can write an nginx.conf that loads your filter, and keep running it and developing it efficiently. An FAQ section has a tutorial to have a quick nginx.conf file you can edit: see here.

[thibaultcha]

On second look this seems to be the same issue as the one you already opened last week, aka calling dispatch_http_call with ctx.clusterName instead of a real hostname. Closing this as duplicate.