awslabs / aws-s3-transfer-manager-rs

Apache License 2.0
6 stars 1 forks source link

Transfer manager download support #8

Closed aajtodd closed 1 month ago

aajtodd commented 2 months ago

Add support for download entire bucket/prefix or single file to TransferManager

aajtodd commented 2 months ago

Strawman proposal for look and feel of the API (generally follows the AWS SDK for Rust):

use aws_s3_transfer_manager::Client;             
use aws_s3_transfer_manager::from_env;
use aws_s3_transfer_manager::config::{Throughput, MemoryLimit};
use aws_s3_transfer_manager::types::S3Object;

// simple, load from environment/auto detect
let tm = aws_s3_transfer_manager::from_env().load().await;

// load from env with overrides
let tm = aws_s3_transfer_manager::from_env()    // -> ConfigLoader
    .throughput(Throughput::Gbps(20.))
    .load()
    .await;

// download an object and write to file
let handle = tm.download()
    .bucket()
    .key()
    ...
    .send()
    .await?;

handle.write_to(self, impl AsRef<Path>).await?;

// download multiple objects to a directory
let handle = tm.download_objects()
    .bucket()
    .destination(Path)
    .key_prefix(String)
    .delimiter(String)
    .failure_policy(...)
    .request_transformer(Fn<>)
    .filter(DownloadFilter)   // type DownloadFilter = Fn<S3Object> -> bool
    .list_objects_transformer(Fn<>)
    .send()
    .await?;

// handle.pause() -> Result<ResumeHandle, ...>
// handle.abort() -> Result<AbortedDownload, ...>
// handle.progress() -> DownloadObjectsProgress
let resp = handle.join().await?;