test

[silenceper]

CNI

k8s 1.9

k8s cni

CNI 插件在k8s中使用：

• --network-plugin=cni : 指定使用cni插件
• --cni-bin-dir=/opt/cni/bin ：存放网络配置的可执行文件
• --cni-conf-dir=/etc/cni/net.d ：存放网络配置文件，如果存在多个，则取第一个（文件名排序）

server.go#L167

// ProbeNetworkPlugins collects all compiled-in plugins
func ProbeNetworkPlugins(cniConfDir, cniBinDir string) []network.NetworkPlugin {
    allPlugins := []network.NetworkPlugin{}

    // for each existing plugin, add to the list
    allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, cniBinDir)...)
    allPlugins = append(allPlugins, kubenet.NewPlugin(cniBinDir))

    return allPlugins
}

注册了两种网络插件：

• kubenet： 非常基本的网络插件，不提供跨节点网络。
• cni：符合cni规范的插件

CNI规范接口

type CNI interface {
    AddNetworkList(net *NetworkConfigList, rt *RuntimeConf) (types.Result, error)
    DelNetworkList(net *NetworkConfigList, rt *RuntimeConf) error

    AddNetwork(net *NetworkConfig, rt *RuntimeConf) (types.Result, error)
    DelNetwork(net *NetworkConfig, rt *RuntimeConf) error
}

参数NetworkConfigList结构：

struct {
    Name string                 
    CNIVersion string
    Plugins []*NetworkConfig
    Bytes []byte
}

参数RuntimeConf：

rt := &libcni.RuntimeConf{
        ContainerID: podSandboxID.ID,//容器ID
        NetNS:       podNetnsPath,  //namespace path
        IfName:      network.DefaultInterfaceName,
        Args: [][2]string{
            {"IgnoreUnknown", "1"},
            {"K8S_POD_NAMESPACE", podNs},
            {"K8S_POD_NAME", podName},
            {"K8S_POD_INFRA_CONTAINER_ID", podSandboxID.ID},
        },
    }

具体实现：

// AddNetworkList executes a sequence of plugins with the ADD command
func (c *CNIConfig) AddNetworkList(list *NetworkConfigList, rt *RuntimeConf) (types.Result, error) {
    var prevResult types.Result
    for _, net := range list.Plugins {
        //查找到配置文件
        pluginPath, err := invoke.FindInPath(net.Network.Type, c.Path)
        if err != nil {
            return nil, err
        }
        //将网络配置 注册到  *NetworkConfig
        newConf, err := buildOneConfig(list, net, prevResult, rt)
        if err != nil {
            return nil, err
        }

        //执行，并且将*NetworkConfig 作为输入，同事将`ADD`，以及RuntimeConf作为环境变量
        prevResult, err = invoke.ExecPluginWithResult(pluginPath, newConf.Bytes, c.args("ADD", rt))
        if err != nil {
            return nil, err
        }
    }

    return prevResult, nil
}

其中invoke.ExecPluginWithResult中调用追踪到：

stdoutBytes, err := e.RawExec.ExecPlugin(pluginPath, netconf, args.AsEnv())

其中AsEnv方法将引入如下环境变量：

    env = append([]string{
        "CNI_COMMAND=" + args.Command,
        "CNI_CONTAINERID=" + args.ContainerID,
        "CNI_NETNS=" + args.NetNS,
        "CNI_ARGS=" + pluginArgsStr,
        "CNI_IFNAME=" + args.IfName,
        "CNI_PATH=" + args.Path,
    }, env...)

说明如下：

• CNI_COMMAND: ADD or DEL
• CNI_CONTAINERID: 容器ID
• CNI_NETNS : network namespace path , ex:/proc/2378/ns/net
• CNI_ARGS : CNI参数，使用分号分开，只要是POD的一些相关信息，
• CNI_IFNAME: 容器内网卡名称
• CNI_PATH： CNI二进制文件路径

[silenceper]

CNI.md.zip

[silenceper]

    ## start server foo.bar.com
    server {
        server_name foo.bar.com ;

        listen 80;

        listen [::]:80;

        set $proxy_upstream_name "-";

        location ~* ^/foo\/?(?<baseuri>.*) {

            set $namespace      "default";
            set $ingress_name   "test";
            set $service_name   "s1";
            set $service_port   "80";
            set $location_path  "/foo";

            rewrite_by_lua_block {

                balancer.rewrite()

            }

            log_by_lua_block {

                balancer.log()

                monitor.call()
            }

            port_in_redirect off;

            set $proxy_upstream_name "";

            client_max_body_size                    "1m";

            proxy_set_header Host                   $best_http_host;

            # Pass the extracted client certificate to the backend

            # Allow websocket connections
            proxy_set_header                        Upgrade           $http_upgrade;

            proxy_set_header                        Connection        $connection_upgrade;

            proxy_set_header X-Request-ID           $req_id;
            proxy_set_header X-Real-IP              $the_real_ip;

            proxy_set_header X-Forwarded-For        $the_real_ip;

            proxy_set_header X-Forwarded-Host       $best_http_host;
            proxy_set_header X-Forwarded-Port       $pass_port;
            proxy_set_header X-Forwarded-Proto      $pass_access_scheme;

            proxy_set_header X-Original-URI         $request_uri;

            proxy_set_header X-Scheme               $pass_access_scheme;

            # Pass the original X-Forwarded-For
            proxy_set_header X-Original-Forwarded-For $http_x_forwarded_for;

            # mitigate HTTPoxy Vulnerability
            # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
            proxy_set_header Proxy                  "";

            # Custom headers to proxied server

            proxy_connect_timeout                   5s;
            proxy_send_timeout                      60s;
            proxy_read_timeout                      60s;

            proxy_buffering                         "off";
            proxy_buffer_size                       "4k";
            proxy_buffers                           4 "4k";
            proxy_request_buffering                 "on";

            proxy_http_version                      1.1;

            proxy_cookie_domain                     off;
            proxy_cookie_path                       off;

            # In case of errors try the next upstream server before returning an error
            proxy_next_upstream                     error timeout;
            proxy_next_upstream_tries               3;

            # No endpoints available for the request
            return 503;

        }

        location ~* ^/bar\/?(?<baseuri>.*) {

            set $namespace      "default";
            set $ingress_name   "test";
            set $service_name   "s2";
            set $service_port   "80";
            set $location_path  "/bar";

            rewrite_by_lua_block {

                balancer.rewrite()

            }

            log_by_lua_block {

                balancer.log()

                monitor.call()
            }

            port_in_redirect off;

            set $proxy_upstream_name "";

            client_max_body_size                    "1m";

            proxy_set_header Host                   $best_http_host;

            # Pass the extracted client certificate to the backend

            # Allow websocket connections
            proxy_set_header                        Upgrade           $http_upgrade;

            proxy_set_header                        Connection        $connection_upgrade;

            proxy_set_header X-Request-ID           $req_id;
            proxy_set_header X-Real-IP              $the_real_ip;

            proxy_set_header X-Forwarded-For        $the_real_ip;

            proxy_set_header X-Forwarded-Host       $best_http_host;
            proxy_set_header X-Forwarded-Port       $pass_port;
            proxy_set_header X-Forwarded-Proto      $pass_access_scheme;

            proxy_set_header X-Original-URI         $request_uri;

            proxy_set_header X-Scheme               $pass_access_scheme;

            # Pass the original X-Forwarded-For
            proxy_set_header X-Original-Forwarded-For $http_x_forwarded_for;

            # mitigate HTTPoxy Vulnerability
            # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
            proxy_set_header Proxy                  "";

            # Custom headers to proxied server

            proxy_connect_timeout                   5s;
            proxy_send_timeout                      60s;
            proxy_read_timeout                      60s;

            proxy_buffering                         "off";
            proxy_buffer_size                       "4k";
            proxy_buffers                           4 "4k";
            proxy_request_buffering                 "on";

            proxy_http_version                      1.1;

            proxy_cookie_domain                     off;
            proxy_cookie_path                       off;

            # In case of errors try the next upstream server before returning an error
            proxy_next_upstream                     error timeout;
            proxy_next_upstream_tries               3;

            # No endpoints available for the request
            return 503;

        }

        location / {

            set $namespace      "";
            set $ingress_name   "";
            set $service_name   "";
            set $service_port   "0";
            set $location_path  "/";

            rewrite_by_lua_block {

                balancer.rewrite()

            }

            log_by_lua_block {

                balancer.log()

                monitor.call()
            }

            access_log off;

            port_in_redirect off;

            set $proxy_upstream_name "upstream-default-backend";

            client_max_body_size                    "1m";

            proxy_set_header Host                   $best_http_host;

            # Pass the extracted client certificate to the backend

            # Allow websocket connections
            proxy_set_header                        Upgrade           $http_upgrade;

            proxy_set_header                        Connection        $connection_upgrade;

            proxy_set_header X-Request-ID           $req_id;
            proxy_set_header X-Real-IP              $the_real_ip;

            proxy_set_header X-Forwarded-For        $the_real_ip;

            proxy_set_header X-Forwarded-Host       $best_http_host;
            proxy_set_header X-Forwarded-Port       $pass_port;
            proxy_set_header X-Forwarded-Proto      $pass_access_scheme;

            proxy_set_header X-Original-URI         $request_uri;

            proxy_set_header X-Scheme               $pass_access_scheme;

            # Pass the original X-Forwarded-For
            proxy_set_header X-Original-Forwarded-For $http_x_forwarded_for;

            # mitigate HTTPoxy Vulnerability
            # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
            proxy_set_header Proxy                  "";

            # Custom headers to proxied server

            proxy_connect_timeout                   5s;
            proxy_send_timeout                      60s;
            proxy_read_timeout                      60s;

            proxy_buffering                         "off";
            proxy_buffer_size                       "4k";
            proxy_buffers                           4 "4k";
            proxy_request_buffering                 "on";

            proxy_http_version                      1.1;

            proxy_cookie_domain                     off;
            proxy_cookie_path                       off;

            # In case of errors try the next upstream server before returning an error
            proxy_next_upstream                     error timeout;
            proxy_next_upstream_tries               3;

            proxy_pass http://upstream_balancer;

            proxy_redirect                          off;

        }

    }
    ## end server foo.bar.com