CommonPrefixes logic does not work on non-terminated prefix (without ending '/')

[DenKoren]

Instead of providing list of directory's contents, Fake S3 server just repeats the name of higher-level directory for the request with prefix.

This is true at least for s3afero backend in single bucket mode. The backend works well for 'root' level directories, when ListObjectsV2 is called with empty Prefix input option, but starts to just repeat the prefix, when it is not empty.

For example, if we have a bucket root with the following data inside:

level-1/level-2-1/emptyFile
level-1/level-2-2/emptyFile

the s3afero backend will return level-1/ CommonPrefixes for call with empty prefix (as it shuld), and level-1/level-1/ CommonPrefixes for call with prefix level-1 (without final /). Instead, it should provide just level-1, as real S3 API.

Here is the example of test, that shows the problem: (the test is written for s3afero package in backend/s3afero)

package s3afero

import (
    "context"
    "net/http/httptest"
    "os"
    "path/filepath"
    "testing"

    "github.com/aws/aws-sdk-go-v2/config"
    "github.com/aws/aws-sdk-go-v2/credentials"
    "github.com/aws/aws-sdk-go-v2/service/s3"
    "github.com/johannesboyne/gofakes3"
    "github.com/spf13/afero"
    "github.com/stretchr/testify/require"
)

func TestCommonPrefixes_Nested(t *testing.T) {
    bucketRoot := t.TempDir()
    bucketName := "my-test-bucket"
    delimiter := string(filepath.Separator)

    createFakeS3 := func() (endpoint string) {
        bucketFS := afero.NewBasePathFs(afero.NewOsFs(), bucketRoot)

        fsBackend, err := SingleBucket(bucketName, bucketFS, afero.NewMemMapFs())
        require.NoError(t, err, "failed to create S3 backend with FS data storage")

        fakeS3 := gofakes3.New(fsBackend)
        s3Server := httptest.NewServer(fakeS3.Server())
        t.Cleanup(s3Server.Close)

        return s3Server.URL
    }

    testDir := "level1"
    testRoot := filepath.Join(bucketRoot, testDir)
    require.NoError(t, os.Mkdir(testRoot, 0o750), "failed to create test dir inside bucket root")

    toCreate := []string{
        "level2-1" + delimiter,
        "level2-2" + delimiter,
    }

    for i := range toCreate {
        dirPath := filepath.Join(testRoot, toCreate[i])
        filePath := filepath.Join(dirPath, "emptyFile")

        require.NoError(t, os.Mkdir(dirPath, 0o750), "failed to create directory in test bucket root")
        require.NoError(t, os.WriteFile(filePath, nil, 0o640), "failed to create empty file in test bucket")
    }

    endpoint := createFakeS3()

    s3Cfg, err := config.LoadDefaultConfig(context.Background(),
        config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", "test")),
    )
    require.NoError(t, err, "cannot init S3 client")

    client := s3.NewFromConfig(s3Cfg, func(o *s3.Options) {
        o.BaseEndpoint = &endpoint
        o.UsePathStyle = true
    })

    resp, err := client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{
        Bucket:    &bucketName,
        Delimiter: &delimiter,
        Prefix:    &testDir,
    })
    require.NoError(t, err, "failed to list bucket objects")

    actualPrefixes := make([]string, 0, len(toCreate))
    for i := range resp.CommonPrefixes {
        actualPrefixes = append(actualPrefixes, *resp.CommonPrefixes[i].Prefix)
    }

    require.Equal(t, []string{"level1/"}, actualPrefixes, "wrong common prefixes returned from S3 service")

    prefix2 := testDir + "/"
    resp, err = client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{
        Bucket:    &bucketName,
        Delimiter: &delimiter,
        Prefix:    &prefix2,
    })
    require.NoError(t, err, "failed to list bucket objects")

    actualPrefixes = make([]string, 0, len(toCreate))
    for i := range resp.CommonPrefixes {
        actualPrefixes = append(actualPrefixes, *resp.CommonPrefixes[i].Prefix)
    }

    require.Equal(t, []string{"level1/level2-1/", "level1/level2-2/"}, actualPrefixes, "wrong common prefixes returned from S3 service")

    resp, err = client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{
        Bucket:    &bucketName,
        Delimiter: &delimiter,
        // Prefix:    new(string),
    })
    require.NoError(t, err, "failed to list bucket objects")

    actualPrefixes = make([]string, 0, len(toCreate))
    for i := range resp.CommonPrefixes {
        actualPrefixes = append(actualPrefixes, *resp.CommonPrefixes[i].Prefix)
    }

    require.Equal(t, []string{"level1/"}, actualPrefixes, "wrong common prefixes returned from S3 service")

    prefix3 := testDir + "/" + "level2"
    resp, err = client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{
        Bucket:    &bucketName,
        Delimiter: &delimiter,
        Prefix:    &prefix3,
    })
    require.NoError(t, err, "failed to list bucket objects")

    actualPrefixes = make([]string, 0, len(toCreate))
    for i := range resp.CommonPrefixes {
        actualPrefixes = append(actualPrefixes, *resp.CommonPrefixes[i].Prefix)
    }

    require.Equal(t, []string{"level1/level2-1/", "level1/level2-2/"}, actualPrefixes, "wrong common prefixes returned from S3 service")
}