local_file_size = 0
# Calculate the starting position for resuming the download
range_start = local_file_size
# Set the Range header for resuming the download
range_header = f'bytes={range_start}-'
# Adjust total_length to only download remaining content
total_length -= range_start
with tqdm(total=total_length, desc=f'source: s3://{s3_bucket}/{s3_object_key}', bar_format="{percentage:.1f}%|{bar:25} | {rate_fmt} | {desc}", unit='B', unit_scale=True, unit_divisor=1024) as pbar:
with open(local_file_name, 'wb') as f:
response = s3client.get_object(Bucket=s3_bucket, Key=s3_object_key, Range=range_header)
for chunk in response['Body'].iter_chunks():
f.write(chunk)
pbar.update(len(chunk))
def s3_download(s3_bucket, s3_object_key, local_file_name): s3_object_key = S3_KEY_PREFIX + s3_object_key meta_data = s3client.head_object(Bucket=s3_bucket, Key=s3_object_key) total_length = int(meta_data.get('ContentLength', 0))
Check if the local file exists
if os.path.exists(local_file_name):
If file exists, get its size
else:
If file doesn't exist, set local_file_size to 0
range_start = local_file_size
range_header = f'bytes={range_start}-'
total_length -= range_start
with tqdm(total=total_length, desc=f'source: s3://{s3_bucket}/{s3_object_key}', bar_format="{percentage:.1f}%|{bar:25} | {rate_fmt} | {desc}", unit='B', unit_scale=True, unit_divisor=1024) as pbar: with open(local_file_name, 'wb') as f: response = s3client.get_object(Bucket=s3_bucket, Key=s3_object_key, Range=range_header) for chunk in response['Body'].iter_chunks(): f.write(chunk) pbar.update(len(chunk))