test_versioning_concurrent_multi_object_delete creates the bucket twice and fails on BucketAlreadyExists

Hey! While running test_versioning_concurrent_multi_object_delete on NooBaa, I saw that the test is trying to create the bucket twice -

on get_new_bucket() - client.create_bucket(Bucket=name)
on _create_objects() -> get_new_bucket_resource() -> bucket.create()

See code references after the suggested fix.

According to my understanding, these are the needed changes for fixing the issue -

 def test_versioning_concurrent_multi_object_delete():
     num_objects = 5
     num_threads = 5
-    bucket_name = get_new_bucket()
+    bucket = get_new_bucket_resource()
+    bucket_name = bucket.name

     check_configure_versioning_retry(bucket_name, "Enabled", "Enabled")

     key_names = ["key_{:d}".format(x) for x in range(num_objects)]
-    bucket = _create_objects(bucket_name=bucket_name, keys=key_names)
+    _create_objects(bucket=bucket, bucket_name=bucket_name, keys=key_names)

     client = get_client()
     versions = client.list_object_versions(Bucket=bucket_name)['Versions']

It would be great if you would fix that, thanks a lot!

Code References -

get_new_bucket() -

def get_new_bucket(client=None, name=None):
"""
Get a bucket that exists and is empty.

Always recreates a bucket from scratch. This is useful to also
reset ACLs and such.
"""
if client is None:
    client = get_client()
if name is None:
    name = get_new_bucket_name()

client.create_bucket(Bucket=name) // create bucket first time
return name

get_new_bucket_resource() -

def get_new_bucket_resource(name=None):
"""
Get a bucket that exists and is empty.

Always recreates a bucket from scratch. This is useful to also
reset ACLs and such.
"""
s3 = boto3.resource('s3',
                    aws_access_key_id=config.main_access_key,
                    aws_secret_access_key=config.main_secret_key,
                    endpoint_url=config.default_endpoint,
                    use_ssl=config.default_is_secure,
                    verify=config.default_ssl_verify)
if name is None:
    name = get_new_bucket_name()
bucket = s3.Bucket(name)
bucket_location = bucket.create() // create bucket second time
return bucket

Test code -

def test_versioning_concurrent_multi_object_delete():
num_objects = 5
num_threads = 5
bucket_name = get_new_bucket() // 1

check_configure_versioning_retry(bucket_name, "Enabled", "Enabled")

key_names = ["key_{:d}".format(x) for x in range(num_objects)]
bucket = _create_objects(bucket_name=bucket_name, keys=key_names) // 2

client = get_client()
versions = client.list_object_versions(Bucket=bucket_name)['Versions']
assert len(versions) == num_objects
objs_dict = {'Objects': [dict((k, v[k]) for k in ["Key", "VersionId"]) for v in versions]}
results = [None] * num_threads

def do_request(n):
    results[n] = client.delete_objects(Bucket=bucket_name, Delete=objs_dict)

t = []
for i in range(num_threads):
    thr = threading.Thread(target = do_request, args=[i])
    thr.start()
    t.append(thr)
_do_wait_completion(t)

for response in results:
    assert len(response['Deleted']) == num_objects
    assert 'Errors' not in response

response = client.list_objects(Bucket=bucket_name)
assert 'Contents' not in response

ceph / s3-tests

test_versioning_concurrent_multi_object_delete creates the bucket twice and fails on BucketAlreadyExists #588