search

alibaba / higress

Cloud Native API Gateway | 云原生API网关
https://higress.io
Apache License 2.0
2.5k stars 407 forks source link

feat: support baidu ernie bot ai model #1024

Closed hanxiantao closed 3 weeks ago

hanxiantao commented 4 weeks ago

Ⅰ. Describe what this PR did

Support baidu ernie bot ai model. API documentation: https://console.bce.baidu.com/tools/?_=1692863460488#/api?product=QIANFAN&project=%E5%8D%83%E5%B8%86%E5%A4%A7%E6%A8%A1%E5%9E%8B%E5%B9%B3%E5%8F%B0&parent=ERNIE%204.0&api=rpc%2F2.0%2Fai_custom%2Fv1%2Fwenxinworkshop%2Fchat%2Fcompletions_pro&method=post

Ⅱ. Does this pull request fix one issue?

fixes https://github.com/alibaba/higress/issues/941

Ⅲ. Why don't you add test cases (unit test/integration test)?

Ⅳ. Describe how to verify it

docker-compose.yaml

version: '3.7'
services:
  envoy:
    image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/gateway:1.4.0-rc.1
    entrypoint: /usr/local/bin/envoy
    # 注意这里对wasm开启了debug级别日志，正式部署时则默认info级别
    command: -c /etc/envoy/envoy.yaml --component-log-level wasm:debug
    depends_on:
      - httpbin
    networks:
      - wasmtest
    ports:
      - "10000:10000"
    volumes:
      - ./envoy.yaml:/etc/envoy/envoy.yaml
      - ./plugin.wasm:/etc/envoy/plugin.wasm
  httpbin:
    image: kennethreitz/httpbin:latest
    networks:
      - wasmtest
    ports:
      - "12345:80"
networks:
  wasmtest: {}

使用OpenAI协议

envoy.yaml

admin:
  address:
    socket_address:
      protocol: TCP
      address: 0.0.0.0
      port_value: 9901
static_resources:
  listeners:
    - name: listener_0
      address:
        socket_address:
          protocol: TCP
          address: 0.0.0.0
          port_value: 10000
      filter_chains:
        - filters:
            - name: envoy.filters.network.http_connection_manager
              typed_config:
                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
                scheme_header_transformation:
                  scheme_to_overwrite: https
                stat_prefix: ingress_http
                # Output envoy logs to stdout
                access_log:
                  - name: envoy.access_loggers.stdout
                    typed_config:
                      "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
                # Modify as required
                route_config:
                  name: local_route
                  virtual_hosts:
                    - name: local_service
                      domains: [ "*" ]
                      routes:
                        - match:
                            prefix: "/"
                          route:
                            cluster: baidu
                            timeout: 300s
                http_filters:
                  - name: wasmtest
                    typed_config:
                      "@type": type.googleapis.com/udpa.type.v1.TypedStruct
                      type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm
                      value:
                        config:
                          name: wasmtest
                          vm_config:
                            runtime: envoy.wasm.runtime.v8
                            code:
                              local:
                                filename: /etc/envoy/plugin.wasm
                          configuration:
                            "@type": "type.googleapis.com/google.protobuf.StringValue"
                            value: |
                              {
                                  "provider": {
                                    "type": "baidu",
                                    "apiTokens": [
                                      "YOUR_BAIDU_API_TOKEN"
                                    ],
                                    "modelMapping": {
                                      "gpt-3": "ERNIE-4.0",
                                      "*": "ERNIE-4.0"
                                    },
                                    "protocol": "openai"
                                  }
                              }
                  - name: envoy.filters.http.router
  clusters:
    - name: httpbin
      connect_timeout: 30s
      type: LOGICAL_DNS
      # Comment out the following line to test on v6 networks
      dns_lookup_family: V4_ONLY
      lb_policy: ROUND_ROBIN
      load_assignment:
        cluster_name: httpbin
        endpoints:
          - lb_endpoints:
              - endpoint:
                  address:
                    socket_address:
                      address: httpbin
                      port_value: 80
    - name: baidu
      connect_timeout: 30s
      type: LOGICAL_DNS
      dns_lookup_family: V4_ONLY
      lb_policy: ROUND_ROBIN
      load_assignment:
        cluster_name: baidu
        endpoints:
          - lb_endpoints:
              - endpoint:
                  address:
                    socket_address:
                      address: aip.baidubce.com
                      port_value: 443
      transport_socket:
        name: envoy.transport_sockets.tls
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
          "sni": "aip.baidubce.com"

非流式请求

curl -X POST 'http://localhost:10000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-d '{
    "model": "gpt-4-turbo",
    "messages": [
        {
            "role": "user",
            "content": "你好，你是谁？"
        }
    ],
    "stream": false
}'

响应：

{
    "id": "as-kf9d0sp13f",
    "choices": [
        {
            "index": 0,
            "message": {
                "role": "assistant",
                "content": "你好，我是文心一言，英文名是ERNIE Bot。我能够与人对话互动，回答问题，协助创作，高效便捷地帮助人们获取信息、知识和灵感。"
            },
            "finish_reason": "stop"
        }
    ],
    "created": 1717250605,
    "model": "ERNIE-4.0",
    "object": "chat.completion",
    "usage": {
        "prompt_tokens": 4,
        "completion_tokens": 33,
        "total_tokens": 37
    }
}

使用OpenAI协议非流式请求

流式请求

curl -X POST 'http://localhost:10000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-d '{
    "model": "gpt-4-turbo",
    "messages": [
        {
            "role": "user",
            "content": "你好，你是谁？"
        }
    ],
    "stream": true
}'

响应：

data:{"id":"as-f7p2skfqrk","choices":[{"index":0,"message":{"role":"assistant","content":"你好，"}}],"created":1717250972,"model":"ERNIE-4.0","object":"chat.completion","usage":{"prompt_tokens":4,"total_tokens":4}}

data:{"id":"as-f7p2skfqrk","choices":[{"index":0,"message":{"role":"assistant","content":"我是文心一言，英文名是ERNIE Bot。"}}],"created":1717250973,"model":"ERNIE-4.0","object":"chat.completion","usage":{"prompt_tokens":4,"total_tokens":4}}

data:{"id":"as-f7p2skfqrk","choices":[{"index":0,"message":{"role":"assistant","content":"我能够与人对话互动，回答问题，协助创作，高效便捷地帮助人们获取信息、知识和灵感。"}}],"created":1717250974,"model":"ERNIE-4.0","object":"chat.completion","usage":{"prompt_tokens":4,"total_tokens":4}}

data:{"id":"as-f7p2skfqrk","choices":[{"index":0,"message":{"role":"assistant"},"finish_reason":"stop"}],"created":1717250974,"model":"ERNIE-4.0","object":"chat.completion","usage":{"prompt_tokens":4,"completion_tokens":33,"total_tokens":37}}

使用OpenAI协议流式请求

使用文心一言协议

envoy.yaml

admin:
  address:
    socket_address:
      protocol: TCP
      address: 0.0.0.0
      port_value: 9901
static_resources:
  listeners:
    - name: listener_0
      address:
        socket_address:
          protocol: TCP
          address: 0.0.0.0
          port_value: 10000
      filter_chains:
        - filters:
            - name: envoy.filters.network.http_connection_manager
              typed_config:
                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
                scheme_header_transformation:
                  scheme_to_overwrite: https
                stat_prefix: ingress_http
                # Output envoy logs to stdout
                access_log:
                  - name: envoy.access_loggers.stdout
                    typed_config:
                      "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
                # Modify as required
                route_config:
                  name: local_route
                  virtual_hosts:
                    - name: local_service
                      domains: [ "*" ]
                      routes:
                        - match:
                            prefix: "/"
                          route:
                            cluster: baidu
                            timeout: 300s
                http_filters:
                  - name: wasmtest
                    typed_config:
                      "@type": type.googleapis.com/udpa.type.v1.TypedStruct
                      type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm
                      value:
                        config:
                          name: wasmtest
                          vm_config:
                            runtime: envoy.wasm.runtime.v8
                            code:
                              local:
                                filename: /etc/envoy/plugin.wasm
                          configuration:
                            "@type": "type.googleapis.com/google.protobuf.StringValue"
                            value: |
                              {
                                  "provider": {
                                    "type": "baidu",
                                    "apiTokens": [
                                      "YOUR_BAIDU_API_TOKEN"
                                    ],
                                    "modelMapping": {
                                      "gpt-3": "ERNIE-4.0",
                                      "*": "ERNIE-4.0"
                                    },
                                    "protocol": "original"
                                  }
                              }
                  - name: envoy.filters.http.router
  clusters:
    - name: httpbin
      connect_timeout: 30s
      type: LOGICAL_DNS
      # Comment out the following line to test on v6 networks
      dns_lookup_family: V4_ONLY
      lb_policy: ROUND_ROBIN
      load_assignment:
        cluster_name: httpbin
        endpoints:
          - lb_endpoints:
              - endpoint:
                  address:
                    socket_address:
                      address: httpbin
                      port_value: 80
    - name: baidu
      connect_timeout: 30s
      type: LOGICAL_DNS
      dns_lookup_family: V4_ONLY
      lb_policy: ROUND_ROBIN
      load_assignment:
        cluster_name: baidu
        endpoints:
          - lb_endpoints:
              - endpoint:
                  address:
                    socket_address:
                      address: aip.baidubce.com
                      port_value: 443
      transport_socket:
        name: envoy.transport_sockets.tls
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
          "sni": "aip.baidubce.com"

非流式请求

curl -X POST 'http://localhost:10000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-d '{
    "model": "ERNIE-4.0",
    "messages": [
        {
            "role": "user",
            "content": "你好，你是谁？"
        }
    ],
    "stream": false
}'

响应：

{
    "id": "as-6y6kdymmtb",
    "object": "chat.completion",
    "created": 1717251141,
    "result": "你好，我是文心一言，英文名是ERNIE Bot。我能够与人对话互动，回答问题，协助创作，高效便捷地帮助人们获取信息、知识和灵感。",
    "is_truncated": false,
    "need_clear_history": false,
    "finish_reason": "normal",
    "usage": {
        "prompt_tokens": 4,
        "completion_tokens": 33,
        "total_tokens": 37
    }
}

使用文心一言协议非流式请求

流式请求

curl -X POST 'http://localhost:10000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-d '{
    "model": "ERNIE-4.0",
    "messages": [
        {
            "role": "user",
            "content": "你好，你是谁？"
        }
    ],
    "stream": true
}'

响应：

data: {"id":"as-vg5n6iu9z7","object":"chat.completion","created":1717251199,"sentence_id":0,"is_end":false,"is_truncated":false,"result":"你好，","need_clear_history":false,"finish_reason":"normal","usage":{"prompt_tokens":4,"completion_tokens":0,"total_tokens":4}}

data: {"id":"as-vg5n6iu9z7","object":"chat.completion","created":1717251200,"sentence_id":1,"is_end":false,"is_truncated":false,"result":"我是文心一言，英文名是ERNIE Bot。","need_clear_history":false,"finish_reason":"normal","usage":{"prompt_tokens":4,"completion_tokens":0,"total_tokens":4}}

data: {"id":"as-vg5n6iu9z7","object":"chat.completion","created":1717251201,"sentence_id":2,"is_end":false,"is_truncated":false,"result":"我能够与人对话互动，回答问题，协助创作，高效便捷地帮助人们获取信息、知识和灵感。","need_clear_history":false,"finish_reason":"normal","usage":{"prompt_tokens":4,"completion_tokens":0,"total_tokens":4}}

data: {"id":"as-vg5n6iu9z7","object":"chat.completion","created":1717251202,"sentence_id":3,"is_end":true,"is_truncated":false,"result":"","need_clear_history":false,"finish_reason":"normal","usage":{"prompt_tokens":4,"completion_tokens":33,"total_tokens":37}}

使用文心一言协议流式请求

Ⅴ. Special notes for reviews

hanxiantao commented 3 weeks ago

@CH3CHO 代码已调整，麻烦再帮忙review下