v4 GetObject Signing

[schnecki]

The headers that are used to sign the GetObject do not work. See https://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html and https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html

The requests fails with (note the StringToSign section):

<Error>
  <Code>SignatureDoesNotMatch</Code>
  <Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message>
  <AWSAccessKeyId>AKIAQ7PJ5JUSSM2MYAKJ</AWSAccessKeyId>
  <StringToSign>AWS4-HMAC-SHA256
  20190908T172301Z
  20190908/eu-central-1/s3/aws4_request
  eaee0ef2ef563dcf782afab7b52755b5a86d0187d032fc3ee10cc61021823b0d</StringToSign>
  <SignatureProvided>5007613c3e8a46e38ee00b85d5051119a8029c0c98e4626f6d2c63119eeeb8c4</SignatureProvided>
  <StringToSignBytes>41 57 53 34 2d 48 4d 41 43 2d 53 48 41 32 35 36 0a 32 30 31 39 30 39 30 38 54 31 37 32 33 30 31 5a 0a 32 30 31 39 30 39 30 38 2f 65 75 2d 63 65 6e 74 72 61 6c 2d 31 2f 73 33 2f 61 77 73 34 5f 72 65 71 75 65 73 74 0a 65 61 65 65 30 65 66 32 65 66 35 36 33 64 63 66 37 38 32 61 66 61 62 37 62 35 32 37 35 35 62 35 61 38 36 64 30 31 38 37 64 30 33 32 66 63 33 65 65 31 30 63 63 36 31 30 32 31 38 32 33 62 30 64</StringToSignBytes>
  <CanonicalRequest>GET
  /Model_1_DevBoard_none_08.09.2019_12.31.58__98.jpg
  X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAQ7PJ5JUSSM2MYAKJ%2F20190908%2Feu-central-1%2Fs3%2Faws4_request&X-Amz-Date=20190908T172301Z&X-Amz-Expires=900&X-Amz-SignedHeaders=host%3Bx-amz-content-sha256%3Bx-amz-date
  host:devboardserverimages-testing.s3.eu-central-1.amazonaws.com
  x-amz-content-sha256:
  x-amz-date:

  host;x-amz-content-sha256;x-amz-date
  UNSIGNED-PAYLOAD</CanonicalRequest>
  <CanonicalRequestBytes>47 45 54 0a 2f 4d 6f 64 65 6c 5f 31 5f 44 65 76 42 6f 61 72 64 5f 6e 6f 6e 65 5f 30 38 2e 30 39 2e 32 30 31 39 5f 31 32 2e 33 31 2e 35 38 5f 5f 39 38 2e 6a 70 67 0a 58 2d 41 6d 7a 2d 41 6c 67 6f 72 69 74 68 6d 3d 41 57 53 34 2d 48 4d 41 43 2d 53 48 41 32 35 36 26 58 2d 41 6d 7a 2d 43 72 65 64 65 6e 74 69 61 6c 3d 41 4b 49 41 51 37 50 4a 35 4a 55 53 53 4d 32 4d 59 41 4b 4a 25 32 46 32 30 31 39 30 39 30 38 25 32 46 65 75 2d 63 65 6e 74 72 61 6c 2d 31 25 32 46 73 33 25 32 46 61 77 73 34 5f 72 65 71 75 65 73 74 26 58 2d 41 6d 7a 2d 44 61 74 65 3d 32 30 31 39 30 39 30 38 54 31 37 32 33 30 31 5a 26 58 2d 41 6d 7a 2d 45 78 70 69 72 65 73 3d 39 30 30 26 58 2d 41 6d 7a 2d 53 69 67 6e 65 64 48 65 61 64 65 72 73 3d 68 6f 73 74 25 33 42 78 2d 61 6d 7a 2d 63 6f 6e 74 65 6e 74 2d 73 68 61 32 35 36 25 33 42 78 2d 61 6d 7a 2d 64 61 74 65 0a 68 6f 73 74 3a 64 65 76 62 6f 61 72 64 73 65 72 76 65 72 69 6d 61 67 65 73 2d 74 65 73 74 69 6e 67 2e 73 33 2e 65 75 2d 63 65 6e 74 72 61 6c 2d 31 2e 61 6d 61 7a 6f 6e 61 77 73 2e 63 6f 6d 0a 78 2d 61 6d 7a 2d 63 6f 6e 74 65 6e 74 2d 73 68 61 32 35 36 3a 0a 78 2d 61 6d 7a 2d 64 61 74 65 3a 0a 0a 68 6f 73 74 3b 78 2d 61 6d 7a 2d 63 6f 6e 74 65 6e 74 2d 73 68 61 32 35 36 3b 78 2d 61 6d 7a 2d 64 61 74 65 0a 55 4e 53 49 47 4e 45 44 2d 50 41 59 4c 4f 41 44</CanonicalRequestBytes>
  <RequestId>A78356799D3BFFAA</RequestId>
  <HostId>s+T2sZeuf0ZsFWAdX2xEnN1LjmOeJZU789ITZQx+AvBBzFyZlfqUit25O+10h88SzoxWDOET+G4=</HostId>
</Error>

While the Debug mode gives:

Debug: String to sign: "GET\n/Model_1_DevBoard_none_08.09.2019_12.31.58__98.jpg\nX-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAQ7PJ5JUSSM2MYAKJ%2F20190908%2Feu-central-1%2Fs3%2Faws4_request&X-Amz-Date=20190908T172301Z&X-Amz-Expires=900&X-Amz-SignedHeaders=host%3Bx-amz-content-sha256%3Bx-amz-date\nhost:devboardserverimages-testing.s3.eu-central-1.amazonaws.com\nx-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nx-amz-date:20190908T172301Z\n\nhost;x-amz-content-sha256;x-amz-date\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"

Obviously this results in a missmatch and the reported SignatureDoesNotMatch error. I tried to build the headers myself, but couldn't get it to work, as I am not familiar with the library.

My test code gets the first 3 parameters correctly (see function stringToSign), but I couldn't get the hash to work. Can someone help me here?

awsUriGetObject :: (MonadIO io) => Configuration -> ServiceConfiguration GetObject UriOnlyQuery -> GetObject -> io B.ByteString
awsUriGetObject cfg info request =
  liftIO $ do
    let ti = timeInfo cfg
        cr = credentials cfg
    sd <- signatureData ti cr
    let q = signQuery' request info sd
    logger cfg Debug $ T.pack $ "String to sign: " ++ show (sqStringToSign q)
    return $ queryToUri q

signQuery' :: GetObject -> ServiceConfiguration GetObject queryType -> SignatureData -> SignedQuery
signQuery' GetObject {..} = s3SignQuery' S3Query {
                                   s3QMethod = Get
                                 , s3QBucket = Just $ T.encodeUtf8 goBucket
                                 , s3QObject = Just $ T.encodeUtf8 goObjectName
                                 , s3QSubresources = HTTP.toQuery [
                                                       ("versionId" :: B8.ByteString,) <$> goVersionId
                                                     , ("response-content-type" :: B8.ByteString,) <$> goResponseContentType
                                                     , ("response-content-language",) <$> goResponseContentLanguage
                                                     , ("response-expires",) <$> goResponseExpires
                                                     , ("response-cache-control",) <$> goResponseCacheControl
                                                     , ("response-content-disposition",) <$> goResponseContentDisposition
                                                     , ("response-content-encoding",) <$> goResponseContentEncoding
                                                     ]
                                 , s3QQuery = []
                                 , s3QContentType = Nothing
                                 , s3QContentMd5 = Nothing
                                 , s3QAmzHeaders = []
                                 , s3QOtherHeaders = catMaybes [
                                                       decodeRange <$> goResponseContentRange
                                                     , ("if-match",) . T.encodeUtf8 <$> goIfMatch
                                                     , ("if-none-match",) . T.encodeUtf8 <$> goIfNoneMatch
                                                     ]
                                 , s3QRequestBody = Nothing
                                 }
      where decodeRange (pos,len) = ("range",B8.concat $ ["bytes=", B8.pack (show pos), "-", B8.pack (show len)])

s3SignQuery' :: S3Query -> S3Configuration qt -> SignatureData -> SignedQuery
s3SignQuery' S3Query {..} S3Configuration {s3SignVersion = S3SignV2 {}, ..} _ = error "S3 V2 signing not allowed! Use V4 signing!"
s3SignQuery' S3Query{..} S3Configuration{ s3SignVersion = S3SignV4 signpayload, .. } sd@SignatureData{..}
    = SignedQuery
    { sqMethod = s3QMethod
    , sqProtocol = s3Protocol
    , sqHost = B.intercalate "." $ catMaybes host
    , sqPort = s3Port
    , sqPath = mconcat $ catMaybes path
    , sqQuery = queryString ++ signatureQuery :: HTTP.Query
    , sqDate = Just signatureTime
    , sqAuthorization = authorization
    , sqContentType = s3QContentType
    , sqContentMd5 = s3QContentMd5
    , sqAmzHeaders = Map.toList amzHeaders
    , sqOtherHeaders = s3QOtherHeaders
    , sqBody = s3QRequestBody
    , sqStringToSign = stringToSign
    }
    where
        -- V4 signing
        -- * <http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html>
        -- * <http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html>
        -- * <http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html>

        iamTok = maybe [] (\x -> [(hAmzSecurityToken, x)]) $ iamToken signatureCredentials

        amzHeaders = Map.fromList $ (hAmzDate, sigTime):(hAmzContentSha256, payloadHash):iamTok ++ s3QAmzHeaders
            where
                -- needs to match the one produces in the @authorizationV4@
                sigTime = fmtTime "%Y%m%dT%H%M%SZ" $ signatureTime
                payloadHash = case (signpayload, s3QRequestBody) of
                    (AlwaysUnsigned, _)                 -> "UNSIGNED-PAYLOAD"
                    (_, Nothing)                        -> emptyBodyHash
                    (_, Just (HTTP.RequestBodyLBS lbs)) -> Base16.encode $ ByteArray.convert (CH.hashlazy lbs :: CH.Digest CH.SHA256)
                    (_, Just (HTTP.RequestBodyBS bs))   -> Base16.encode $ ByteArray.convert (CH.hash bs :: CH.Digest CH.SHA256)
                    (SignWithEffort, _)                 -> "UNSIGNED-PAYLOAD"
                    (AlwaysSigned, _)                   -> error "aws: RequestBody must be a on-memory one when AlwaysSigned mode."
                emptyBodyHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"

        (host, path) = case s3RequestStyle of
            PathStyle   -> ([Just s3Endpoint], [Just "/", fmap (`B8.snoc` '/') s3QBucket, urlEncodedS3QObject])
            BucketStyle -> ([s3QBucket, Just s3Endpoint], [Just "/", urlEncodedS3QObject])
            VHostStyle  -> ([Just $ fromMaybe s3Endpoint s3QBucket], [Just "/", urlEncodedS3QObject])
            where
                urlEncodedS3QObject = s3UriEncode False <$> s3QObject

        -- must provide host in the canonical headers.
        canonicalHeaders = Map.union amzHeaders . Map.fromList $ catMaybes
            [ Just ("host", B.intercalate "." $ catMaybes host)
            , ("content-type",) <$> s3QContentType
            ]
        signedHeaders = B8.intercalate ";" (map CI.foldedCase $ Map.keys canonicalHeaders)
        stringToSign = B.intercalate "\n" $
          ["AWS4-HMAC-SHA256"
          , amzHeaders Map.! hAmzDate
          , B.tail $ B.dropWhile (/= W._slash) $ credentialV4    sd            region "s3"
          -- , payloadHash
          , B.intercalate "\n" headers ++ "\n"
          , Base16.encode $ ByteArray.convert (CH.hashlazy (L.fromStrict bs) :: CH.Digest CH.SHA256)

          ]

          where bs = B.intercalate "\n" headers <> "\n"
                region = s3ExtractRegion s3Endpoint
                headers =
                  catMaybes
                 [ ("content-type: " <>) <$> s3QContentType
                 , Just $ ("host: " <>) (B.intercalate "." $ catMaybes host)
                 -- , B8.intercalate ";" (map CI.foldedCase $ Map.keys canonicalHeaders)
                 , Just $ "x-amz-date: " <> (amzHeaders Map.! hAmzDate)
                 ]
        (authorization, signatureQuery, queryString) = case ti of
            AbsoluteTimestamp _  -> (Just auth, [], allQueries)
            AbsoluteExpires time ->
                ( Nothing
                , [(CI.original hAmzSignature, Just sig)]
                , (allQueries ++) . HTTP.toQuery . map (first CI.original) $
                    [ (hAmzAlgorithm, "AWS4-HMAC-SHA256")
                    , (hAmzCredential, cred)
                    , (hAmzDate, amzHeaders Map.! hAmzDate)
                    , (hAmzExpires, B8.pack . (show :: Integer -> String) . floor $ diffUTCTime time signatureTime)
                    , (hAmzSignedHeaders, signedHeaders)
                    ] ++ iamTok
                )
            where
                allQueries = s3QSubresources ++ s3QQuery
                region = s3ExtractRegion s3Endpoint
                auth = authorizationV4 sd HmacSHA256 region "s3" signedHeaders stringToSign
                sig  = signatureV4     sd HmacSHA256 region "s3"               stringToSign
                cred = credentialV4    sd            region "s3"
                ti = case (s3UseUri, signatureTimeInfo) of
                    (False, t) -> t
                    (True, AbsoluteTimestamp time) -> AbsoluteExpires $ s3DefaultExpiry `addUTCTime` time
                    (True, AbsoluteExpires time) -> AbsoluteExpires time

[andrewthad]

This is also a problem for me, and I do not know how to fix it either.

[joeyh]

I've had several users report S3 compatible services that seem to not work with V2 authorization and so I tried switching my program to use V4. GetObject from aws with V4 works for me. I wonder what I'm doing differently?

I tried both path-style and request-style, to us-eastern.

[maybeTomorrow]

i got same problem

[maybeTomorrow]

some body help?

[maybeTomorrow]

this work

`s3SignQuery S3Query{..} S3Configuration{ s3SignVersion = S3SignV4 signpayload,s3UseUri = True, .. } sd@SignatureData{..} = SignedQuery { sqMethod = s3QMethod , sqProtocol = s3Protocol , sqHost = B.intercalate "." $ catMaybes host , sqPort = s3Port , sqPath = mconcat $ catMaybes path , sqQuery = queryString ++ signatureQuery :: HTTP.Query , sqDate = Just signatureTime , sqAuthorization = authorization , sqContentType = s3QContentType , sqContentMd5 = s3QContentMd5 , sqAmzHeaders = Map.toList amzHeaders , sqOtherHeaders = s3QOtherHeaders , sqBody = s3QRequestBody , sqStringToSign = stringToSign } where -- V4 signing -- http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html -- http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html -- * http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html

    iamTok = maybe [] (\x -> [(hAmzSecurityToken, x)]) $ iamToken signatureCredentials

    amzHeaders = Map.fromList $ (hAmzDate, sigTime):(hAmzContentSha256, payloadHash):iamTok ++ s3QAmzHeaders
        where
            -- needs to match the one produces in the @authorizationV4@
            sigTime = fmtTime "%Y%m%dT%H%M%SZ" $ signatureTime
            payloadHash = case (signpayload, s3QRequestBody) of
                (AlwaysUnsigned, _)                 -> "UNSIGNED-PAYLOAD"
                (_, Nothing)                        -> emptyBodyHash
                (_, Just (HTTP.RequestBodyLBS lbs)) -> Base16.encode $ ByteArray.convert (CH.hashlazy lbs :: CH.Digest CH.SHA256)
                (_, Just (HTTP.RequestBodyBS bs))   -> Base16.encode $ ByteArray.convert (CH.hash bs :: CH.Digest CH.SHA256)
                (SignWithEffort, _)                 -> "UNSIGNED-PAYLOAD"
                (AlwaysSigned, _)                   -> error "aws: RequestBody must be a on-memory one when AlwaysSigned mode."
            emptyBodyHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"

    (host, path) = case s3RequestStyle of
        PathStyle   -> ([Just s3Endpoint], [Just "/", fmap (`B8.snoc` '/') s3QBucket, urlEncodedS3QObject])
        BucketStyle -> ([s3QBucket, Just s3Endpoint], [Just "/", urlEncodedS3QObject])
        VHostStyle  -> ([Just $ fromMaybe s3Endpoint s3QBucket], [Just "/", urlEncodedS3QObject])
        where
            urlEncodedS3QObject = s3UriEncode False <$> s3QObject

    -- must provide host in the canonical headers.
    -- Map.union amzHeaders .
    canonicalHeaders =  Map.fromList $ catMaybes
        [ Just ("host", B.intercalate "." $ catMaybes host)
        , ("content-type",) <$> s3QContentType
        ]
    signedHeaders = "host";-- B8.intercalate ";" (map CI.foldedCase $ Map.keys canonicalHeaders)
    stringToSign = B.intercalate "\n" $
        [ httpMethod s3QMethod                   -- method
        , mconcat . catMaybes $ path             -- path
        , s3RenderQuery False $ sort queryString -- query string
        ] ++
        Map.foldMapWithKey (\a b -> [CI.foldedCase a Sem.<> ":" Sem.<> b]) canonicalHeaders ++
        [ "" -- end headers
        , signedHeaders
        , amzHeaders Map.! hAmzContentSha256
        ]

    (authorization, signatureQuery, queryString) = case ti of
        AbsoluteTimestamp _  -> (Just auth, [], allQueries)
        AbsoluteExpires time ->
            ( Nothing
            , [(CI.original hAmzSignature, Just sig)]
            , (allQueries ++) . HTTP.toQuery . map (first CI.original) $
                [ (hAmzAlgorithm, "AWS4-HMAC-SHA256")
                , (hAmzCredential, cred)
                , (hAmzDate, amzHeaders Map.! hAmzDate)
                , (hAmzContentSha256, amzHeaders Map.! hAmzContentSha256)
                , (hAmzExpires, B8.pack . (show :: Integer -> String) . floor $ diffUTCTime time signatureTime)
                , (hAmzSignedHeaders, signedHeaders)
                ] ++ iamTok
            )
        where
            allQueries = s3QSubresources ++ s3QQuery
            region = s3ExtractRegion s3Endpoint
            auth = authorizationV4 sd HmacSHA256 region "s3" signedHeaders stringToSign
            sig  = signatureV4     sd HmacSHA256 region "s3"               stringToSign
            cred = credentialV4    sd            region "s3"
            ti = case ( signatureTimeInfo) of
                ( AbsoluteTimestamp time) -> AbsoluteExpires $ s3DefaultExpiry `addUTCTime` time
                ( AbsoluteExpires time) -> AbsoluteExpires time

`