search

awslabs / aws-sdk-rust

AWS SDK for the Rust Programming Language
https://awslabs.github.io/aws-sdk-rust/
Apache License 2.0
3k stars 249 forks source link

Unable to specify SHA256 checksum when creating presigned URL for PutObject action in S3 #1103

Open ciffelia opened 6 months ago

ciffelia commented 6 months ago

Describe the bug

I am trying to create presigned URL for PutObject action using aws-sdk-s3 crate. When I specify the value for x-amz-checksum-sha256 header using checksum_sha256 method, the specified checksum seems to be ignored.

use std::time::Duration;

use aws_sdk_s3::{presigning::PresigningConfig, types::ChecksumAlgorithm};

#[tokio::main]
async fn main() {
    let client = aws_sdk_s3::Client::new(&aws_config::load_from_env().await);

    // checksum for b"abc"
    let checksum = "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=";

    let presigned_request = client
        .put_object()
        .bucket("example-bucket")
        .key("foo")
        .checksum_algorithm(ChecksumAlgorithm::Sha256)
        .checksum_sha256(checksum)
        .presigned(PresigningConfig::expires_in(Duration::from_secs(360)).unwrap())
        .await
        .unwrap();

    dbg!(presigned_request.headers().collect::<Vec<_>>());

    assert!(presigned_request
        .headers()
        .any(|(k, v)| { k == "x-amz-checksum-sha256" && v == checksum }));
}

Expected Behavior

When I run the code above, the request header contains ("x-amz-checksum-sha256", "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=") and the assertion passes.

Current Behavior

The request header contains ("x-amz-checksum-sha256", "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=") and the assertion fails.

47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= is the checksum for empty data.

use base64::Engine;
use sha2::Digest;

fn main() {
    assert_eq!(
        base64::engine::general_purpose::STANDARD.encode(sha2::Sha256::digest(b"")),
        "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="
    );
}

https://play.rust-lang.org/?version=stable&mode=debug&edition=2021&gist=bf4910fbc283b478d9dd3d254f508392

Reproduction Steps

Full reproduction is available at the following repository.

https://github.com/ciffelia/aws-rust-sdk-repro-1

Possible Solution

No response

Additional Information/Context

No response

Version

aws-rust-sdk-repro-1 v0.1.0 (/home/...)
├── aws-config v1.1.8
│   ├── aws-credential-types v1.1.8
│   │   ├── aws-smithy-async v1.1.8
│   │   ├── aws-smithy-runtime-api v1.2.0
│   │   │   ├── aws-smithy-async v1.1.8 (*)
│   │   │   ├── aws-smithy-types v1.1.8
│   │   ├── aws-smithy-types v1.1.8 (*)
│   ├── aws-runtime v1.1.8
│   │   ├── aws-credential-types v1.1.8 (*)
│   │   ├── aws-sigv4 v1.2.0
│   │   │   ├── aws-credential-types v1.1.8 (*)
│   │   │   ├── aws-smithy-eventstream v0.60.4
│   │   │   │   ├── aws-smithy-types v1.1.8 (*)
│   │   │   ├── aws-smithy-http v0.60.7
│   │   │   │   ├── aws-smithy-eventstream v0.60.4 (*)
│   │   │   │   ├── aws-smithy-runtime-api v1.2.0 (*)
│   │   │   │   ├── aws-smithy-types v1.1.8 (*)
│   │   │   ├── aws-smithy-runtime-api v1.2.0 (*)
│   │   │   ├── aws-smithy-types v1.1.8 (*)
│   │   ├── aws-smithy-async v1.1.8 (*)
│   │   ├── aws-smithy-eventstream v0.60.4 (*)
│   │   ├── aws-smithy-http v0.60.7 (*)
│   │   ├── aws-smithy-runtime-api v1.2.0 (*)
│   │   ├── aws-smithy-types v1.1.8 (*)
│   │   ├── aws-types v1.1.8
│   │   │   ├── aws-credential-types v1.1.8 (*)
│   │   │   ├── aws-smithy-async v1.1.8 (*)
│   │   │   ├── aws-smithy-runtime-api v1.2.0 (*)
│   │   │   ├── aws-smithy-types v1.1.8 (*)
│   ├── aws-sdk-sso v1.17.0
│   │   ├── aws-credential-types v1.1.8 (*)
│   │   ├── aws-runtime v1.1.8 (*)
│   │   ├── aws-smithy-async v1.1.8 (*)
│   │   ├── aws-smithy-http v0.60.7 (*)
│   │   ├── aws-smithy-json v0.60.7
│   │   │   └── aws-smithy-types v1.1.8 (*)
│   │   ├── aws-smithy-runtime v1.1.8
│   │   │   ├── aws-smithy-async v1.1.8 (*)
│   │   │   ├── aws-smithy-http v0.60.7 (*)
│   │   │   ├── aws-smithy-runtime-api v1.2.0 (*)
│   │   │   ├── aws-smithy-types v1.1.8 (*)
│   │   ├── aws-smithy-runtime-api v1.2.0 (*)
│   │   ├── aws-smithy-types v1.1.8 (*)
│   │   ├── aws-types v1.1.8 (*)
│   ├── aws-sdk-ssooidc v1.17.0
│   │   ├── aws-credential-types v1.1.8 (*)
│   │   ├── aws-runtime v1.1.8 (*)
│   │   ├── aws-smithy-async v1.1.8 (*)
│   │   ├── aws-smithy-http v0.60.7 (*)
│   │   ├── aws-smithy-json v0.60.7 (*)
│   │   ├── aws-smithy-runtime v1.1.8 (*)
│   │   ├── aws-smithy-runtime-api v1.2.0 (*)
│   │   ├── aws-smithy-types v1.1.8 (*)
│   │   ├── aws-types v1.1.8 (*)
│   ├── aws-sdk-sts v1.17.0
│   │   ├── aws-credential-types v1.1.8 (*)
│   │   ├── aws-runtime v1.1.8 (*)
│   │   ├── aws-smithy-async v1.1.8 (*)
│   │   ├── aws-smithy-http v0.60.7 (*)
│   │   ├── aws-smithy-json v0.60.7 (*)
│   │   ├── aws-smithy-query v0.60.7
│   │   │   ├── aws-smithy-types v1.1.8 (*)
│   │   ├── aws-smithy-runtime v1.1.8 (*)
│   │   ├── aws-smithy-runtime-api v1.2.0 (*)
│   │   ├── aws-smithy-types v1.1.8 (*)
│   │   ├── aws-smithy-xml v0.60.7
│   │   ├── aws-types v1.1.8 (*)
│   ├── aws-smithy-async v1.1.8 (*)
│   ├── aws-smithy-http v0.60.7 (*)
│   ├── aws-smithy-json v0.60.7 (*)
│   ├── aws-smithy-runtime v1.1.8 (*)
│   ├── aws-smithy-runtime-api v1.2.0 (*)
│   ├── aws-smithy-types v1.1.8 (*)
│   ├── aws-types v1.1.8 (*)
├── aws-sdk-s3 v1.19.0
│   ├── aws-credential-types v1.1.8 (*)
│   ├── aws-runtime v1.1.8 (*)
│   ├── aws-sigv4 v1.2.0 (*)
│   ├── aws-smithy-async v1.1.8 (*)
│   ├── aws-smithy-checksums v0.60.7
│   │   ├── aws-smithy-http v0.60.7 (*)
│   │   ├── aws-smithy-types v1.1.8 (*)
│   ├── aws-smithy-eventstream v0.60.4 (*)
│   ├── aws-smithy-http v0.60.7 (*)
│   ├── aws-smithy-json v0.60.7 (*)
│   ├── aws-smithy-runtime v1.1.8 (*)
│   ├── aws-smithy-runtime-api v1.2.0 (*)
│   ├── aws-smithy-types v1.1.8 (*)
│   ├── aws-smithy-xml v0.60.7 (*)
│   ├── aws-types v1.1.8 (*)

Environment details (OS name and version, etc.)

Ubuntu 22.04 on WSL

Logs

No response

jdisanti commented 6 months ago

Thank you for the repro!

It looks like the assertion might have the wrong header name? It looks like it is in there:

[src/main.rs:22] presigned_request.headers().collect::<Vec<_>>() = [
    (
        "x-amz-sdk-checksum-algorithm",
        "SHA256",
    ),
    (
        "x-amz-checksum-sha256",
        "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=",
    ),
]

Are you saying that S3 is failing to validate the checksum when using this presigned request?

ciffelia commented 6 months ago

Sorry, I typed the wrong header name. I corrected it, but the assertion still fails because the checksum value do not match. It should be "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=", not "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=".

rcoh commented 6 months ago

yeah this seems like a bug—we're failing to detect the checksum is set already and we're recomputing it.

Seems related to #539 and #1085

CNorskov commented 6 months ago

I believe the issue is that you set the checksum algorithm as well as the sha256 checksum.

So with the following, it should work:

use std::time::Duration;

use aws_sdk_s3::{presigning::PresigningConfig, types::ChecksumAlgorithm};

#[tokio::main]
async fn main() {
    let client = aws_sdk_s3::Client::new(&aws_config::load_from_env().await);

    // checksum for b"abc"
    let checksum = "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=";

    let presigned_request = client
        .put_object()
        .bucket("example-bucket")
        .key("foo")
        .checksum_sha256(checksum)
        .presigned(PresigningConfig::expires_in(Duration::from_secs(360)).unwrap())
        .await
        .unwrap();

    dbg!(presigned_request.headers().collect::<Vec<_>>());

    assert!(presigned_request
        .headers()
        .any(|(k, v)| { k == "x-amz-checksum-sha256" && v == checksum }));
}