tidb-incubator / weir

Apache License 2.0
97 stars 40 forks source link

NamespaceManager.PrepareReloadNamespace之后原有连接池未销毁,连接泄露 #81

Open joeytang opened 8 months ago

joeytang commented 8 months ago

NamespaceManager.PrepareReloadNamespace可以销毁原有连接吗

当我们变更一个namespace的对应的数据库实例列表时,通过管控接口,/reload/prepare/{namespace} 进行重新装载配置信息。通过数据库实例上tcp连接信息,能看到从weir还是会有连接与下掉的实例保持。代码跟踪发现在执行这个prepare管控接口的时候,会去重建连接池。重建之后,直接将原有连接池替换掉。原来的连接池没有销毁。

func (b *BackendImpl) initConnPools() error {
    connPools := make(map[string]*ConnPool)
    for addr := range b.cfg.Addrs {
        poolCfg := &ConnPoolConfig{
            Config:      Config{Addr: addr, UserName: b.cfg.UserName, Password: b.cfg.Password},
            Capacity:    b.cfg.Capacity,
            IdleTimeout: b.cfg.IdleTimeout,
        }
        connPool := NewConnPool(b.ns, poolCfg)
        connPools[addr] = connPool
    }
    successfulInitConnPoolAddrs := make(map[string]struct{})
    var initConnPoolErr error
    for addr, connPool := range connPools {
        if err := connPool.Init(); err != nil {
            initConnPoolErr = err
            break
        }
        successfulInitConnPoolAddrs[addr] = struct{}{}
    }
    if initConnPoolErr != nil {
        for addr := range successfulInitConnPoolAddrs {
            if err := connPools[addr].Close(); err != nil {
                logutil.BgLogger().Sugar().Error("close inited conn pool error, addr: %s, err: %v", addr, err)
            }
        }
        return initConnPoolErr
    }
    **b.connPools = connPools**
    return nil
}
joeytang commented 8 months ago

连接池自己会根据idletimeout时间自动关闭连接。因此理论上不会存在泄漏问题。但是现象看我们设置了idle为120秒,目前实例已经下线十几个小时了,连接依然在

joeytang commented 8 months ago

// closeIdleResources scans the pool for idle resources
func (rp *ResourcePool) closeIdleResources() {
    available := int(rp.Available())
    idleTimeout := rp.IdleTimeout()

    for i := 0; i < available; i++ {
        var wrapper resourceWrapper
        select {
        case wrapper = <-rp.resources:
        default:
            // stop early if we don't get anything new from the pool
            return
        }

        func() {
            defer func() { rp.resources <- wrapper }()

            if wrapper.resource != nil && idleTimeout > 0 && time.Until(wrapper.timeUsed.Add(idleTimeout)) < 0 {
                wrapper.resource.Close()
                rp.idleClosed.Add(1)
//此处能看到过期后会重新打开连接资源
                rp.reopenResource(&wrapper)
            }
        }()

    }
}
//连接资源重新打开的时候,会直接建立与数据库实例的tcp连接。这样这个连接会一直在。
//除非这个数据库实例网络不可用了
func (rp *ResourcePool) reopenResource(wrapper *resourceWrapper) {
    if r, err := rp.factory(context.TODO()); err == nil {
        wrapper.resource = r
        wrapper.timeUsed = time.Now()
    } else {
        wrapper.resource = nil
        rp.active.Add(-1)
    }
}

问题可能在这里