Issue generating pre-signed URL

[owbezick]

I have authenticated using the standard: Sys.setenv( AWS_ACCESS_KEY_ID = "abc", AWS_SECRET_ACCESS_KEY = "123", AWS_REGION = "us-east-1" ) This authentication has worked to list buckets and retrieve objects. However, when I try this code: bucket <- 'BUCKET_NAME' s3_key <- 'OBJECT_KEY s3()$generate_presigned_url( client_method='get_object', params=list(Bucket=bucket, Key=s3_key), expires_in=3600 ) A pre-signed URL is generated, but I am met with a: InvalidRequest The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256. VGDS2GCV99JAA9ED /Iq9V2rLJmvXGYLofe+1uMZz20O/u25GzyTcDQUWExntjB3FUoRrtSL81YMi/aGs9DEMTTwvOdM= I have checked and I am able to generate a pre-signed URL using the AWS console.

Any thoughts as to the error?

[DyfanJones]

Ah this make sense, currently generate_presigned_url is using sha1 algorithm. Which was developed from boto3. However I might of missed something that should change the hashing algorithm.

Are you able to get a working generate_presigned_url from boto3?

import boto3

s3 = boto3.client("s3")

bucket = "your-bucket"
key = "your-key"

s3.generate_presigned_url(
    "get_object",
    Params = {"Bucket": bucket, "Key": key}
)

[DyfanJones]

From further investigation boto3 allows you to change default signature_version.

By default boto3 will use sha1 and signature_version = "s3v4" will use sha256.

import boto3
from botocore.config import Config

s3_default = boto3.Session(profile_name = "paws").client("s3")
s3_custom = boto3.Session(profile_name = "paws").client(
    "s3", config=Config(signature_version="s3v4")
)

bucket = "my-bucket"

s3_default.generate_presigned_url(
    "list_objects", Params = {"Bucket": bucket}
)

#> https://my-bucket.s3.amazonaws.com/?encoding-type=url&AWSAccessKeyId=ABCDEFGHIJK123456789&Signature=BruHGTRQffmrsKtNkGR6bw62GTQ%3D&Expires=1689154977

s3_custom_v2.generate_presigned_url(
    "list_objects", Params={"Bucket":bucket}   
)
#> https://my-bucket.s3.amazonaws.com/?encoding-type=url&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ABCDEFGHIJK123456789%2F20230712%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230712T084559Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=ad6bc4af7955f38eff5956bf3a7b38a066d9ec283e53063e71f71865a718e682

[DyfanJones]

For demo purposes we can hack our current paws implementation to do the same thing.

library(paws.common)
s3 <- paws::s3()

bucket = "my-bucket"

s3$generate_presigned_url_v2 <- function (client_method, params = list(), expires_in = 3600, 
                                          http_method = NULL, signature_version="default") {
  stopifnot("\'client_method\' must to be a character" = is.character(client_method), 
            "\'params\' must be a list of parameters for client_method" = is.list(params), 
            "\'expires\' must be numeric" = is.numeric(expires_in), 
            "\'expires_in\' must be greater than 0" = expires_in > 0L,
            "\`http_method\` must be a character" = (is.character(http_method) || is.null(http_method)))
  pkg_name <- "paws.storage"
  pkg_api <- "s3"
  .pkg_api <- paste0(".", pkg_api)
  tryCatch({
    operation_fun <- get(sprintf("%s_%s", pkg_api, client_method), 
                         envir = getNamespace(pkg_name))
  }, error = function(err) {
    stop(sprintf("Client does not have method: %s", client_method), 
         call. = FALSE)
  })
  operation_body <- body(operation_fun)
  op <- eval(operation_body[[2]][[3]], envir = getNamespace("paws.common"))
  original_params <- formals(operation_fun)
  original_params <- if (!is.null(original_params)) 
    original_params
  else list()
  if (!is.null(original_params)) 
    original_params
  else list()
  param_check <- setdiff(names(params), names(original_params))
  if (!identical(param_check, character(0)) || is.null(param_check)) {
    stop(sprintf("Invalid parameter(s) [`%s`] for client method %s", 
                 paste(param_check, collapse = "`, `"), client_method), 
         call. = FALSE)
  }
  kwargs <- as.list(modifyList(original_params, params))
  input <- do.call(get(.pkg_api, envir = getNamespace(pkg_name))[[sprintf("%s_input", 
                                                                          client_method)]], kwargs)
  output <- get(.pkg_api, envir = getNamespace(pkg_name))[[sprintf("%s_input", 
                                                                   client_method)]]()
  config <- get_config()
  svc <- get(.pkg_api, envir = getNamespace(pkg_name))[["service"]](config)
  request <- new_request(svc, op, input, output)
  request$expire_time <- expires_in
  request <- do.call("build", list(request = request), envir = getNamespace("paws.common"))

  signer <- ifelse(signature_version=="s3v4", "s3v4_sign_request_handler", "sign_v1_auth_query")
  request <- do.call(signer, list(request = request), 
                     envir = getNamespace("paws.common"))
  if (!is.null(http_method)) {
    request$http_request$url$scheme <- http_method
  }
  return(do.call("build_url", list(url = request$http_request$url), 
                 envir = getNamespace("paws.common")))
}

s3$generate_presigned_url_v2(
  'list_objects',
  params=list(Bucket=bucket)
)
#> [1] "https://my-bucket.s3.eu-west-1.amazonaws.com/?AWSAccessKeyId= ABCDEFGHIJK123456789&Expires=1689155836&Signature=0Yz7Jtg2dCL%2F2MCSTRCdYtcc6IV9XkMbFQakAXk0h2g%3D"

s3$generate_presigned_url_v2(
  'list_objects',
  params=list(Bucket=bucket),
  signature_version = "s3v4"
)
#> [1] "https://my-bucket.s3.eu-west-1.amazonaws.com/?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential= ABCDEFGHIJK123456789%2F20230712%2Feu-west-1%2Fs3%2Faws4_request&X-Amz-Date=20230712T085717Z&X-Amz-Expires=3600&X-Amz-Signature=ff3763f382d749cfe87dcc56e0753dbbf4b42531881d47b8d9d71db5a1076ddf&X-Amz-SignedHeaders=host"

^{Created on 2023-07-12 with reprex v2.0.2}

[DyfanJones]

I believe I can have this in the next paws regen.

[DyfanJones]

Hi @owbezick ,

paws 0.4.0 will opt to setting the signature_version in the config of the client. This is align with boto3 implementation.

# paws 0.4.0
library(paws.common)

s3 <- paws::s3(config(signature_version = "s3v4"))

s3$generate_presigned_url("list_objects", params = list(Bucket = "my-bucket"))

import boto3
from botocore.config import Config

s3 = boto3.client("s3", config = Config(signature_version = "s3v4"))

s3.generate_presigned_url("list_objects", Params = {"Bucket" : "my-bucket"})

I will update ticket paws 0.4.0 ready on r-universe and finally cran

[owbezick]

This is great! Thank you so much.

On Wed, Jul 12, 2023 at 7:00 AM Larefly @.***> wrote:

Hi @owbezick https://github.com/owbezick ,

paws 0.4.0 will opt to setting the signature_version in the config of the client. This is align with boto3 implementation.

paws 0.4.0

library(paws.common) s3 <- paws::s3(config(signature_version = "s3v4")) s3$generate_presigned_url("list_objects", params = list(Bucket = "my-bucket"))

import boto3from botocore.config import Config s3 = boto3.client("s3", config = Config(signature_version = "s3v4")) s3.generate_presigned_url("list_objects", Params = {"Bucket" : "my-bucket"})

I will update ticket paws 0.4.0 ready on r-universe and finally cran

— Reply to this email directly, view it on GitHub https://github.com/paws-r/paws/issues/645#issuecomment-1632292442, or unsubscribe https://github.com/notifications/unsubscribe-auth/AMNDFXSUZVLREXEJJGP65STXPZ7UTANCNFSM6AAAAAA2GBQ2WQ . You are receiving this because you were mentioned.Message ID: @.***>

[DyfanJones]

paws v-0.4.0 has now been released to the cran. I will close this ticket for now.