ListObjects() and race: limit on 8192 ... , dying

[runningmaster]

Please consider the example below as test case:

package main

import (
    "crypto/sha1"
    "fmt"
    "log"
    "runtime"
    "time"

    minio "github.com/minio/minio-go"
)

const (
    s3Address   = "127.0.0.1:9000"
    s3AccessKey = "YOUR-ACCESSKEYID"
    s3SecretKey = "YOUR-SECRETACCESSKEY"
    backetName  = "stream-in"
)

var s3cli *minio.Client

func main() {
    var err error
    s3cli, err = minio.New(s3Address, s3AccessKey, s3SecretKey, true)
    if err != nil {
        log.Fatal(err)
    }

    showRaceLimit()
}

// NOTE:
// if you will rename processing1() on processing2() here
// then all will be OK (NumGoroutine() will be constant).
func showRaceLimit() {
    c := time.Tick(1 * time.Millisecond)
    var err error
    for _ = range c {
        err = processing1(backetName, 1)
        if err != nil {
            log.Println(err)
        }
    }
}

func processing1(s string, n int) error {
    objs, err := listObjectsN(s, "", false, n)
    if err != nil {
        return err
    }

    for i := range objs {
        log.Println(objs[i].Key)
    }

    log.Println(runtime.NumGoroutine()) // NOTE: NumGoroutine() is increased
    return nil
}

func processing2(_ string, _ int) error {
    log.Println(fmt.Sprintf("%x", sha1.Sum([]byte(time.Now().String()))))
    log.Println(runtime.NumGoroutine()) // NOTE: NumGoroutine() is constant
    return nil
}

func listObjectsN(bucket, prefix string, recursive bool, n int) ([]minio.ObjectInfo, error) {
    doneCh := make(chan struct{}, 1)
    defer close(doneCh)

    i := 0
    objs := make([]minio.ObjectInfo, 0, n)
    for object := range s3cli.ListObjects(bucket, prefix, recursive, doneCh) {
        if object.Err != nil {
            return nil, object.Err
        }
        i++
        if i == n {
            doneCh <- struct{}{}
        }
        objs = append(objs, object)
    }

    return objs, nil
}

go run -race main.go

race: limit on 8192 simultaneously alive goroutines is exceeded, dying
exit status 66

[harshavardhana]

Thanks for this @runningmaster - verifying

[harshavardhana]

Looks like the problem is the retry go-routine which is leaking.

goroutine 19 [chan send]:
github.com/minio/minio-go.Client.newRetryTimer.func2(0xc8201341e0, 0x5, 0xc82013c0f0)
    /Users/harsha/mygo/src/github.com/minio/minio-go/retry.go:66 +0x7e
created by github.com/minio/minio-go.Client.newRetryTimer
    /Users/harsha/mygo/src/github.com/minio/minio-go/retry.go:69 +0x132

goroutine 38 [chan send]:
github.com/minio/minio-go.Client.newRetryTimer.func2(0xc820135740, 0x5, 0xc82039ae40)
    /Users/harsha/mygo/src/github.com/minio/minio-go/retry.go:66 +0x7e
created by github.com/minio/minio-go.Client.newRetryTimer
    /Users/harsha/mygo/src/github.com/minio/minio-go/retry.go:69 +0x132

goroutine 35 [chan send]:
github.com/minio/minio-go.Client.newRetryTimer.func2(0xc820134300, 0x5, 0xc82039a0f0)
    /Users/harsha/mygo/src/github.com/minio/minio-go/retry.go:66 +0x7e
created by github.com/minio/minio-go.Client.newRetryTimer
    /Users/harsha/mygo/src/github.com/minio/minio-go/retry.go:69 +0x132

goroutine 30 [chan send]:
github.com/minio/minio-go.Client.newRetryTimer.func2(0xc820365080, 0x5, 0xc8203faf00)
    /Users/harsha/mygo/src/github.com/minio/minio-go/retry.go:66 +0x7e
created by github.com/minio/minio-go.Client.newRetryTimer
    /Users/harsha/mygo/src/github.com/minio/minio-go/retry.go:69 +0x132

goroutine 41 [sleep]:
time.Sleep(0x8fe3333)
    /usr/local/opt/go/libexec/src/runtime/time.go:59 +0xf9
github.com/minio/minio-go.Client.newRetryTimer.func2(0xc8204f0720, 0x5, 0xc82039bb30)
    /Users/harsha/mygo/src/github.com/minio/minio-go/retry.go:67 +0x9f
created by github.com/minio/minio-go.Client.newRetryTimer
    /Users/harsha/mygo/src/github.com/minio/minio-go/retry.go:69 +0x132
exit status 2

[harshavardhana]

package main

import (
    "log"
    "runtime"
    "time"

    minio "github.com/minio/minio-go"
)

const (
    s3Address   = "play.minio.io:9000"
    s3AccessKey = "Q3AM3UQ867SPQQA43P2F"
    s3SecretKey = "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
    backetName  = "stream-in"
)

var s3cli *minio.Client

func main() {
    var err error
    s3cli, err = minio.New(s3Address, s3AccessKey, s3SecretKey, false)
    if err != nil {
        log.Fatal(err)
    }

    showRaceLimit()
}

// NOTE:
// if you will rename processing1() on processing2() here
// then all will be OK (NumGoroutine() will be constant).
func showRaceLimit() {
    c := time.Tick(1 * time.Millisecond)
    var err error
    for _ = range c {
        err = processing1(backetName, 1)
        if err != nil {
            log.Println(err)
        }
    }
}

func processing1(s string, n int) error {
    objs, err := listObjectsN(s, "", false, n)
    if err != nil {
        return err
    }

    for i := range objs {
        log.Println(objs[i].Key)
    }

    log.Println(runtime.NumGoroutine()) // NOTE: NumGoroutine() is
    // increased
    return nil
}

func processing2(_ string, _ int) error {
    // a channel to tell it to stop
    stopchan := make(chan struct{})
    // a channel to signal that it's stopped
    stoppedchan := make(chan struct{})
    go func() { // work in background
        // close the stoppedchan when this func
        // exits
        defer close(stoppedchan)
        for {
            select {
            default:
            case <-stopchan:
                // stop
                return
            }
        }
    }()

    log.Println("stopping...")
    close(stopchan) // tell it to stop
    <-stoppedchan   // wait for it to have stopped
    log.Println("Stopped.")

    log.Println(runtime.NumGoroutine()) // NOTE: NumGoroutine() is
    // constant
    return nil
}

func listObjectsN(bucket, prefix string, recursive bool, n int) ([]minio.ObjectInfo, error) {
    doneCh := make(chan struct{}, 1)
    defer close(doneCh)

    i := 0
    objs := make([]minio.ObjectInfo, 0, n)
    for object := range s3cli.ListObjects(bucket, prefix, recursive, doneCh) {
        if object.Err != nil {
            return nil, object.Err
        }
        i++
        if i == n {
            doneCh <- struct{}{}
        }
        objs = append(objs, object)
    }
    return objs, nil
}

A full case, a fix is on its way thanks for your report @runningmaster.