search

paws-r / paws

Paws, a package for Amazon Web Services in R
https://www.paws-r-sdk.com
Other
315 stars 37 forks source link

s3$list_object_versions stopped working #783

Closed stevepowell99 closed 4 months ago

stevepowell99 commented 4 months ago

My code has included s3$list_object_versions("bucketname",Prefix = "filename",MaxKeys=10000)$Versions for a couple of years, all was working fine. Now I notice that this has started to return an empty list(). Everything else continues to work fine, I can save versions and restore versions if I know the ID of the version. Nothing has changed at my end except updated packages etc. If this isn't a known issue I will post a reprex.

DyfanJones commented 4 months ago

Sorry about that can you share the versions of paws.storage and paws.common.

stevepowell99 commented 4 months ago

paws.storage version 0.6.0 paws version 0.6.0 paws.common version 0.7.2 thanks

stevepowell99 commented 4 months ago

R version 4.3.3

DyfanJones commented 4 months ago

Hmm I am currently unable to replicate:

library(paws)

Bucket <- "paws-delete"
Key <- "demo-file.txt"

client <- s3(config(credentials(profile = "paws")))

client$create_bucket(
  Bucket = Bucket
)
#> $Location
#> [1] "/paws-delete"

resp <- client$put_bucket_versioning(
  Bucket = Bucket,
  VersioningConfiguration=list(
    Status = "Enabled"
  )
)

resp <- client$put_object(
  Bucket = Bucket,
  Key = Key,
  Body = charToRaw("helloworld")
)

client$list_object_versions(
  Bucket,
  Prefix = Key,
  MaxKeys=10
)$Versions
#> [[1]]
#> [[1]]$ETag
#> [1] "\"fc5e038d38a57032085441e7fe7010b0\""
#> 
#> [[1]]$ChecksumAlgorithm
#> list()
#> 
#> [[1]]$Size
#> [1] 10
#> 
#> [[1]]$StorageClass
#> [1] "STANDARD"
#> 
#> [[1]]$Key
#> [1] "demo-file.txt"
#> 
#> [[1]]$VersionId
#> [1] "tr254HqS3q0PvX7NHWhuIS.qhL3KMMPe"
#> 
#> [[1]]$IsLatest
#> [1] TRUE
#> 
#> [[1]]$LastModified
#> [1] "2024-05-13 14:55:47 GMT"
#> 
#> [[1]]$Owner
#> [[1]]$Owner$DisplayName
#> [1] "dyfan.r.jones"
#> 
#> [[1]]$Owner$ID
#> [1] "b3f6bed018b0c5c65f68de63469c0b80f9cde5fb347710d610f837a735773e0f"
#> 
#> 
#> [[1]]$RestoreStatus
#> [[1]]$RestoreStatus$IsRestoreInProgress
#> logical(0)
#> 
#> [[1]]$RestoreStatus$RestoreExpiryDate
#> POSIXct of length 0

resp <- client$list_object_versions(Bucket, Prefix = Key) |> 
  paginate_lapply(\(resp){
    list(
      Objects = lapply(resp$Versions, \(r) {
        list(
          Key = r$Key,
          VersionId = r$VersionId
        )
      })
    )
  },
  PageSize = 1000
)
for (i in resp) client$delete_objects(Bucket, Delete = i)

client$delete_bucket(Bucket)
#> list()

Created on 2024-05-13 with reprex v2.1.0

Session info ``` r sessioninfo::session_info() #> ─ Session info ─────────────────────────────────────────────────────────────── #> setting value #> version R version 4.4.0 (2024-04-24) #> os macOS Sonoma 14.4.1 #> system aarch64, darwin20 #> ui X11 #> language (EN) #> collate en_US.UTF-8 #> ctype en_US.UTF-8 #> tz Europe/London #> date 2024-05-13 #> pandoc 3.1.13 @ /opt/homebrew/bin/ (via rmarkdown) #> #> ─ Packages ─────────────────────────────────────────────────────────────────── #> package * version date (UTC) lib source #> base64enc 0.1-3 2015-07-28 [1] CRAN (R 4.4.0) #> cli 3.6.2 2023-12-11 [1] RSPM (R 4.4.0) #> crayon 1.5.2 2022-09-29 [1] CRAN (R 4.4.0) #> curl 5.2.1 2024-03-01 [1] RSPM (R 4.4.0) #> digest 0.6.35 2024-03-11 [1] RSPM (R 4.4.0) #> evaluate 0.23 2023-11-01 [1] RSPM (R 4.4.0) #> fastmap 1.1.1 2023-02-24 [1] CRAN (R 4.4.0) #> fs 1.6.4 2024-04-25 [1] RSPM (R 4.4.0) #> glue 1.7.0 2024-01-09 [1] RSPM (R 4.4.0) #> htmltools 0.5.8.1 2024-04-04 [1] RSPM (R 4.4.0) #> httr 1.4.7 2023-08-15 [1] CRAN (R 4.4.0) #> knitr 1.46 2024-04-06 [1] RSPM (R 4.4.0) #> lifecycle 1.0.4 2023-11-07 [1] RSPM (R 4.4.0) #> magrittr 2.0.3 2022-03-30 [1] CRAN (R 4.4.0) #> mime 0.12 2021-09-28 [1] CRAN (R 4.4.0) #> paws * 0.6.0 2024-05-01 [1] https://paws-r.r-universe.dev (R 4.4.0) #> paws.common 0.7.2 2024-04-09 [1] RSPM (R 4.4.0) #> paws.storage 0.6.0 2024-05-01 [1] https://paws-r.r-universe.dev (R 4.4.0) #> purrr 1.0.2 2023-08-10 [1] CRAN (R 4.4.0) #> R.cache 0.16.0 2022-07-21 [1] CRAN (R 4.4.0) #> R.methodsS3 1.8.2 2022-06-13 [1] CRAN (R 4.4.0) #> R.oo 1.26.0 2024-01-24 [1] CRAN (R 4.4.0) #> R.utils 2.12.3 2023-11-18 [1] CRAN (R 4.4.0) #> R6 2.5.1 2021-08-19 [1] CRAN (R 4.4.0) #> Rcpp 1.0.12 2024-01-09 [1] RSPM (R 4.4.0) #> reprex 2.1.0 2024-01-11 [1] RSPM (R 4.4.0) #> rlang 1.1.3 2024-01-10 [1] RSPM (R 4.4.0) #> rmarkdown 2.26 2024-03-05 [1] RSPM (R 4.4.0) #> rstudioapi 0.16.0 2024-03-24 [1] RSPM (R 4.4.0) #> sessioninfo 1.2.2 2021-12-06 [1] RSPM (R 4.4.0) #> styler 1.10.3 2024-04-07 [1] RSPM (R 4.4.0) #> vctrs 0.6.5 2023-12-01 [1] RSPM (R 4.4.0) #> withr 3.0.0 2024-01-16 [1] RSPM (R 4.4.0) #> xfun 0.43 2024-03-25 [1] RSPM (R 4.4.0) #> xml2 1.3.6 2023-12-04 [1] RSPM (R 4.4.0) #> yaml 2.3.8 2023-12-11 [1] RSPM (R 4.4.0) #> #> [1] /Users/dyfanjones/Library/R/arm64/4.4/library #> [2] /Library/Frameworks/R.framework/Versions/4.4-arm64/Resources/library #> #> ────────────────────────────────────────────────────────────────────────────── ```

I will down grade my R to see if it is a R version issue.

DyfanJones commented 4 months ago

It seems to work fine on R 4.3.3

library(paws)

Bucket <- "paws-delete"
Key <- "demo-file.txt"

client <- s3(config(credentials(profile = "paws")))

client$create_bucket(
  Bucket = Bucket
)
#> $Location
#> [1] "/paws-delete"

resp <- client$put_bucket_versioning(
  Bucket = Bucket,
  VersioningConfiguration=list(
    Status = "Enabled"
  )
)

for (i in 1:3) { 
  resp <- client$put_object(
    Bucket = Bucket,
    Key = Key,
    Body = charToRaw("helloworld")
  )
}

client$list_object_versions(
  Bucket,
  Prefix = Key,
  MaxKeys=10
)$Versions
#> [[1]]
#> [[1]]$ETag
#> [1] "\"fc5e038d38a57032085441e7fe7010b0\""
#> 
#> [[1]]$ChecksumAlgorithm
#> list()
#> 
#> [[1]]$Size
#> [1] 10
#> 
#> [[1]]$StorageClass
#> [1] "STANDARD"
#> 
#> [[1]]$Key
#> [1] "demo-file.txt"
#> 
#> [[1]]$VersionId
#> [1] "IxkANu3PhsWk7m3rxGprmLjN_gC1x6w7"
#> 
#> [[1]]$IsLatest
#> [1] TRUE
#> 
#> [[1]]$LastModified
#> [1] "2024-05-13 15:03:17 GMT"
#> 
#> [[1]]$Owner
#> [[1]]$Owner$DisplayName
#> [1] "dyfan.r.jones"
#> 
#> [[1]]$Owner$ID
#> [1] "b3f6bed018b0c5c65f68de63469c0b80f9cde5fb347710d610f837a735773e0f"
#> 
#> 
#> [[1]]$RestoreStatus
#> [[1]]$RestoreStatus$IsRestoreInProgress
#> logical(0)
#> 
#> [[1]]$RestoreStatus$RestoreExpiryDate
#> POSIXct of length 0
#> 
#> 
#> 
#> [[2]]
#> [[2]]$ETag
#> [1] "\"fc5e038d38a57032085441e7fe7010b0\""
#> 
#> [[2]]$ChecksumAlgorithm
#> list()
#> 
#> [[2]]$Size
#> [1] 10
#> 
#> [[2]]$StorageClass
#> [1] "STANDARD"
#> 
#> [[2]]$Key
#> [1] "demo-file.txt"
#> 
#> [[2]]$VersionId
#> [1] "K8v3VJHiPeOonYzU.yUf7h8Iq4cndsqo"
#> 
#> [[2]]$IsLatest
#> [1] FALSE
#> 
#> [[2]]$LastModified
#> [1] "2024-05-13 15:03:16 GMT"
#> 
#> [[2]]$Owner
#> [[2]]$Owner$DisplayName
#> [1] "dyfan.r.jones"
#> 
#> [[2]]$Owner$ID
#> [1] "b3f6bed018b0c5c65f68de63469c0b80f9cde5fb347710d610f837a735773e0f"
#> 
#> 
#> [[2]]$RestoreStatus
#> [[2]]$RestoreStatus$IsRestoreInProgress
#> logical(0)
#> 
#> [[2]]$RestoreStatus$RestoreExpiryDate
#> POSIXct of length 0
#> 
#> 
#> 
#> [[3]]
#> [[3]]$ETag
#> [1] "\"fc5e038d38a57032085441e7fe7010b0\""
#> 
#> [[3]]$ChecksumAlgorithm
#> list()
#> 
#> [[3]]$Size
#> [1] 10
#> 
#> [[3]]$StorageClass
#> [1] "STANDARD"
#> 
#> [[3]]$Key
#> [1] "demo-file.txt"
#> 
#> [[3]]$VersionId
#> [1] "CgcVhSCXmAKeH4kVjjOPltkCFUrn2gac"
#> 
#> [[3]]$IsLatest
#> [1] FALSE
#> 
#> [[3]]$LastModified
#> [1] "2024-05-13 15:03:16 GMT"
#> 
#> [[3]]$Owner
#> [[3]]$Owner$DisplayName
#> [1] "dyfan.r.jones"
#> 
#> [[3]]$Owner$ID
#> [1] "b3f6bed018b0c5c65f68de63469c0b80f9cde5fb347710d610f837a735773e0f"
#> 
#> 
#> [[3]]$RestoreStatus
#> [[3]]$RestoreStatus$IsRestoreInProgress
#> logical(0)
#> 
#> [[3]]$RestoreStatus$RestoreExpiryDate
#> POSIXct of length 0

resp <- client$list_object_versions(Bucket, Prefix = Key) |> 
  paginate_lapply(\(resp){
    list(
      Objects = lapply(resp$Versions, \(r) {
        list(
          Key = r$Key,
          VersionId = r$VersionId
        )
      })
    )
  },
  PageSize = 1000
)
for (i in resp) client$delete_objects(Bucket, Delete = i)

client$delete_bucket(Bucket)
#> list()

Created on 2024-05-13 with reprex v2.1.0

Session info ``` r sessioninfo::session_info() #> ─ Session info ─────────────────────────────────────────────────────────────── #> setting value #> version R version 4.3.3 (2024-02-29) #> os macOS Sonoma 14.4.1 #> system aarch64, darwin20 #> ui X11 #> language (EN) #> collate en_US.UTF-8 #> ctype en_US.UTF-8 #> tz Europe/London #> date 2024-05-13 #> pandoc 3.1.13 @ /opt/homebrew/bin/ (via rmarkdown) #> #> ─ Packages ─────────────────────────────────────────────────────────────────── #> package * version date (UTC) lib source #> base64enc 0.1-3 2015-07-28 [1] CRAN (R 4.3.0) #> cli 3.6.2 2023-12-11 [1] CRAN (R 4.3.1) #> crayon 1.5.2 2022-09-29 [1] CRAN (R 4.3.0) #> curl 5.2.1 2024-03-04 [1] https://r-releases.r-universe.dev (R 4.3.3) #> digest 0.6.35 2024-03-11 [1] RSPM (R 4.3.0) #> evaluate 0.23 2023-11-01 [1] CRAN (R 4.3.1) #> fastmap 1.1.1 2023-02-24 [1] CRAN (R 4.3.0) #> fs 1.6.4 2024-04-25 [1] RSPM (R 4.3.0) #> glue 1.7.0 2024-01-09 [1] CRAN (R 4.3.1) #> htmltools 0.5.8.1 2024-04-04 [1] RSPM (R 4.3.0) #> httr 1.4.7 2023-08-15 [1] CRAN (R 4.3.0) #> knitr 1.46 2024-04-06 [1] RSPM (R 4.3.0) #> lifecycle 1.0.4 2023-11-07 [1] RSPM (R 4.3.0) #> magrittr 2.0.3 2022-03-30 [1] CRAN (R 4.3.0) #> mime 0.12 2021-09-28 [1] CRAN (R 4.3.0) #> paws * 0.6.0 2024-05-13 [1] https://paws-r.r-universe.dev (R 4.3.3) #> paws.common 0.7.2 2024-04-09 [1] RSPM (R 4.3.0) #> paws.storage 0.6.0 2024-05-10 [1] RSPM (R 4.3.0) #> purrr 1.0.2 2023-08-10 [1] CRAN (R 4.3.0) #> R.cache 0.16.0 2022-07-21 [1] CRAN (R 4.3.0) #> R.methodsS3 1.8.2 2022-06-13 [1] CRAN (R 4.3.0) #> R.oo 1.26.0 2024-01-24 [1] CRAN (R 4.3.1) #> R.utils 2.12.3 2023-11-18 [1] CRAN (R 4.3.1) #> R6 2.5.1 2021-08-19 [1] CRAN (R 4.3.0) #> Rcpp 1.0.12 2024-01-09 [1] CRAN (R 4.3.1) #> reprex 2.1.0 2024-01-11 [1] CRAN (R 4.3.1) #> rlang 1.1.3 2024-01-10 [1] CRAN (R 4.3.2) #> rmarkdown 2.26 2024-03-05 [1] CRAN (R 4.3.1) #> rstudioapi 0.16.0 2024-03-24 [1] RSPM (R 4.3.0) #> sessioninfo 1.2.2 2021-12-06 [1] CRAN (R 4.3.0) #> styler 1.10.3 2024-04-07 [1] RSPM (R 4.3.0) #> vctrs 0.6.5 2023-12-01 [1] CRAN (R 4.3.1) #> withr 3.0.0 2024-01-16 [1] CRAN (R 4.3.1) #> xfun 0.43 2024-03-25 [1] RSPM (R 4.3.0) #> xml2 1.3.6 2023-12-04 [1] CRAN (R 4.3.1) #> yaml 2.3.8 2023-12-11 [1] CRAN (R 4.3.1) #> #> [1] /Users/dyfanjones/Library/R/arm64/4.3/library #> [2] /Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library #> #> ────────────────────────────────────────────────────────────────────────────── ```
DyfanJones commented 4 months ago

I wonder if it is an os issue 🤔 . @stevepowell99 what os are you using? A reprex would be perfect :)

stevepowell99 commented 4 months ago

yes I used your reprex and can save the file (versions) but get this:

> client$list_object_versions(
+     Bucket,
+     Prefix = Key,
+     MaxKeys=10
+ )$Versions
list()

(I wasn't able to create a bucket with the reprex, I got IllegalLocationConstraintException but used an existing bucket which has versioning enabled:

>   client$get_bucket_versioning(Bucket)
$Status
[1] "Enabled"

$MFADelete
character(0)

then ran the put_object loop and can see the versions in the AWS console. Just not with list_object_versions

stevepowell99 commented 4 months ago

I am on Windows 11 build 22621.3447

DyfanJones commented 4 months ago

Does the code work with paws.common 0.7.1?

DyfanJones commented 4 months ago

Hmm it is still working on my windows machine 🤔 (please see results below with session information). @stevepowell99 What AWS region are you working with? I am struggling to replicate your issue.

library(paws)

Bucket <- "paws-delete"
Key <- "demo-file.txt"

client <- s3()

client$create_bucket(
  Bucket = Bucket
)
#> $Location
#> [1] "/paws-delete"

resp <- client$put_bucket_versioning(
  Bucket = Bucket,
  VersioningConfiguration=list(
    Status = "Enabled"
  )
)

for (i in 1:2) {
  resp <- client$put_object(
    Bucket = Bucket,
    Key = Key,
    Body = charToRaw("helloworld")
  )
}

client$list_object_versions(
  Bucket,
  Prefix = Key,
  MaxKeys=10
)$Versions
#> [[1]]
#> [[1]]$ETag
#> [1] "\"fc5e038d38a57032085441e7fe7010b0\""
#> 
#> [[1]]$ChecksumAlgorithm
#> list()
#> 
#> [[1]]$Size
#> [1] 10
#> 
#> [[1]]$StorageClass
#> [1] "STANDARD"
#> 
#> [[1]]$Key
#> [1] "demo-file.txt"
#> 
#> [[1]]$VersionId
#> [1] "ZINUI6lTmZLPTmZqePIYyNdDuB7K9VyK"
#> 
#> [[1]]$IsLatest
#> [1] TRUE
#> 
#> [[1]]$LastModified
#> [1] "2024-05-13 21:41:47 GMT"
#> 
#> [[1]]$Owner
#> [[1]]$Owner$DisplayName
#> [1] "dyfan.r.jones"
#> 
#> [[1]]$Owner$ID
#> [1] "b3f6bed018b0c5c65f68de63469c0b80f9cde5fb347710d610f837a735773e0f"
#> 
#> 
#> [[1]]$RestoreStatus
#> [[1]]$RestoreStatus$IsRestoreInProgress
#> logical(0)
#> 
#> [[1]]$RestoreStatus$RestoreExpiryDate
#> POSIXct of length 0
#> 
#> 
#> 
#> [[2]]
#> [[2]]$ETag
#> [1] "\"fc5e038d38a57032085441e7fe7010b0\""
#> 
#> [[2]]$ChecksumAlgorithm
#> list()
#> 
#> [[2]]$Size
#> [1] 10
#> 
#> [[2]]$StorageClass
#> [1] "STANDARD"
#> 
#> [[2]]$Key
#> [1] "demo-file.txt"
#> 
#> [[2]]$VersionId
#> [1] "5kr9RxpImTrNtuE60Zo7cl3ai2bGXX.0"
#> 
#> [[2]]$IsLatest
#> [1] FALSE
#> 
#> [[2]]$LastModified
#> [1] "2024-05-13 21:41:47 GMT"
#> 
#> [[2]]$Owner
#> [[2]]$Owner$DisplayName
#> [1] "dyfan.r.jones"
#> 
#> [[2]]$Owner$ID
#> [1] "b3f6bed018b0c5c65f68de63469c0b80f9cde5fb347710d610f837a735773e0f"
#> 
#> 
#> [[2]]$RestoreStatus
#> [[2]]$RestoreStatus$IsRestoreInProgress
#> logical(0)
#> 
#> [[2]]$RestoreStatus$RestoreExpiryDate
#> POSIXct of length 0

resp <- client$list_object_versions(Bucket, Prefix = Key) |> 
  paginate_lapply(\(resp){
    list(
      Objects = lapply(resp$Versions, \(r) {
        list(
          Key = r$Key,
          VersionId = r$VersionId
        )
      })
    )
  },
  PageSize = 1000
  )

for (i in resp) client$delete_objects(Bucket, Delete = i)

client$delete_bucket(Bucket)
#> list()

Created on 2024-05-13 with reprex v2.1.0

Session info ``` r sessioninfo::session_info() #> ─ Session info ─────────────────────────────────────────────────────────────── #> setting value #> version R version 4.3.3 (2024-02-29 ucrt) #> os Windows 11 x64 (build 22631) #> system x86_64, mingw32 #> ui RTerm #> language (EN) #> collate English_United Kingdom.utf8 #> ctype English_United Kingdom.utf8 #> tz Europe/London #> date 2024-05-13 #> pandoc 3.1.2 @ C:/PROGRA~3/CHOCOL~1/bin/ (via rmarkdown) #> #> ─ Packages ─────────────────────────────────────────────────────────────────── #> package * version date (UTC) lib source #> base64enc 0.1-3 2015-07-28 [1] RSPM (R 4.3.0) #> cli 3.6.2 2023-12-11 [1] RSPM (R 4.3.0) #> crayon 1.5.2 2022-09-29 [1] RSPM (R 4.3.0) #> curl 5.2.0 2023-12-08 [1] RSPM (R 4.3.0) #> digest 0.6.35 2024-03-11 [1] RSPM (R 4.3.0) #> evaluate 0.23 2023-11-01 [1] CRAN (R 4.3.2) #> fastmap 1.1.1 2023-02-24 [1] RSPM (R 4.3.0) #> fs 1.6.4 2024-04-25 [1] RSPM (R 4.3.0) #> glue 1.7.0 2024-01-09 [1] CRAN (R 4.3.1) #> htmltools 0.5.8.1 2024-04-04 [1] RSPM (R 4.3.0) #> httr 1.4.7 2023-08-15 [1] RSPM (R 4.3.0) #> knitr 1.46 2024-04-06 [1] RSPM (R 4.3.0) #> lifecycle 1.0.4 2023-11-07 [1] RSPM (R 4.3.0) #> magrittr 2.0.3 2022-03-30 [1] RSPM (R 4.3.0) #> mime 0.12 2021-09-28 [1] RSPM (R 4.3.0) #> paws * 0.6.0 2024-05-13 [1] https://paws-r.r-universe.dev (R 4.3.3) #> paws.common 0.7.2 2024-04-09 [1] RSPM (R 4.3.0) #> paws.storage 0.6.0 2024-05-10 [1] RSPM (R 4.3.0) #> purrr 1.0.2 2023-08-10 [1] RSPM (R 4.3.0) #> R.cache 0.16.0 2022-07-21 [1] RSPM (R 4.3.0) #> R.methodsS3 1.8.2 2022-06-13 [1] RSPM (R 4.3.0) #> R.oo 1.26.0 2024-01-24 [1] RSPM (R 4.3.0) #> R.utils 2.12.3 2023-11-18 [1] RSPM (R 4.3.0) #> R6 2.5.1 2021-08-19 [1] RSPM (R 4.3.0) #> Rcpp 1.0.12 2024-01-09 [1] CRAN (R 4.3.1) #> reprex 2.1.0 2024-01-11 [1] RSPM (R 4.3.1) #> rlang 1.1.3 2024-01-10 [1] CRAN (R 4.3.1) #> rmarkdown 2.26 2024-03-05 [1] RSPM (R 4.3.0) #> rstudioapi 0.16.0 2024-03-24 [1] RSPM (R 4.3.0) #> sessioninfo 1.2.2 2021-12-06 [1] CRAN (R 4.3.2) #> styler 1.10.3 2024-04-07 [1] RSPM (R 4.3.0) #> vctrs 0.6.5 2023-12-01 [1] RSPM (R 4.3.0) #> withr 3.0.0 2024-01-16 [1] RSPM (R 4.3.0) #> xfun 0.43 2024-03-25 [1] RSPM (R 4.3.0) #> xml2 1.3.6 2023-12-04 [1] RSPM (R 4.3.0) #> yaml 2.3.8 2023-12-11 [1] RSPM (R 4.3.0) #> #> [1] C:/Users/dyfan/AppData/Local/R/win-library/4.3 #> [2] C:/Program Files/R/R-4.3.3/library #> #> ────────────────────────────────────────────────────────────────────────────── ```
stevepowell99 commented 4 months ago

tx I tried installing devtools::install_github("paws-r/paws@v0.7.1") but this did not work (got a 404). so I am using 0.7.2

my config.r has aws: AWS_ACCESS_KEY_ID: "xxx" AWS_SECRET_ACCESS_KEY: "xxx" AWS_REGION: "eu-west-2" AWS_DEFAULT_REGION: "eu-west-2"

so first part of my reprex:

library(paws)
  library(configr)
  config = configr::read.config("config.yml")$default
  Sys.setenv(AWS_ACCESS_KEY_ID = config$aws$AWS_ACCESS_KEY_ID)
  Sys.setenv(AWS_SECRET_ACCESS_KEY = config$aws$AWS_SECRET_ACCESS_KEY)
  Sys.setenv(AWS_REGION = "eu-west-2")
  Sys.setenv(AWS_DEFAULT_REGION = "eu-west-2")

  Key <- "demo-file.txt"
  Bucket = "deletebucketttt"

  client <- paws::s3()

  client$list_buckets()

# works fine, lists all buckets. I can also put and get objects
# however;

  client$create_bucket(
    Bucket = Bucket
  )
# Error: IllegalLocationConstraintException (HTTP 400). The unspecified location constraint is incompatible for the region specific endpoint this request was sent to.

maybe this is related?

Sess infosessioninfo::session_info() ─ Session info ────────────────────────────────────────────────── setting value version R version 4.3.3 (2024-02-29 ucrt) os Windows 11 x64 (build 22621) system x86_64, mingw32 ui RStudio language (EN) collate English_United Kingdom.utf8 ctype English_United Kingdom.utf8 tz Europe/London date 2024-05-14 rstudio 2024.04.0+735 Chocolate Cosmos (desktop) pandoc 3.1.11 @ C:/Program Files/RStudio/resources/app/bin/quarto/bin/tools/ (via rmarkdown) ─ Packages ────────────────────────────────────────────────────── package * version date (UTC) lib source bit 4.0.5 2022-11-15 [2] CRAN (R 4.3.3) bit64 4.0.5 2020-08-30 [2] CRAN (R 4.3.3) blob 1.2.4 2023-03-17 [2] CRAN (R 4.3.3) cachem 1.0.8 2023-05-01 [2] CRAN (R 4.3.3) cli 3.6.2 2023-12-11 [2] CRAN (R 4.3.3) config 0.3.2 2023-08-30 [2] CRAN (R 4.3.3) configr * 0.3.5 2020-07-17 [2] CRAN (R 4.3.3) crayon 1.5.2 2022-09-29 [2] CRAN (R 4.3.3) curl 5.2.1 2024-03-01 [2] CRAN (R 4.3.3) DBI 1.2.2 2024-02-16 [2] CRAN (R 4.3.3) devtools 2.4.5 2022-10-11 [1] CRAN (R 4.3.3) digest 0.6.35 2024-03-11 [2] CRAN (R 4.3.3) ellipsis 0.3.2 2021-04-29 [2] CRAN (R 4.3.3) evaluate 0.23 2023-11-01 [2] CRAN (R 4.3.3) fastmap 1.1.1 2023-02-24 [2] CRAN (R 4.3.3) fs 1.6.3 2023-07-20 [2] CRAN (R 4.3.3) gargle 1.5.2 2023-07-20 [2] CRAN (R 4.3.3) glue 1.7.0 2024-01-09 [2] CRAN (R 4.3.3) hms 1.1.3 2023-03-21 [2] CRAN (R 4.3.3) htmltools 0.5.8.1 2024-04-04 [2] CRAN (R 4.3.3) htmlwidgets 1.6.4 2023-12-06 [2] CRAN (R 4.3.3) httpuv 1.6.15 2024-03-26 [2] CRAN (R 4.3.3) httr 1.4.7 2023-08-15 [2] CRAN (R 4.3.3) ini 0.3.1 2018-05-20 [2] CRAN (R 4.3.3) jsonlite 1.8.8 2023-12-04 [2] CRAN (R 4.3.3) knitr 1.46 2024-04-06 [2] CRAN (R 4.3.3) later 1.3.2 2023-12-06 [2] CRAN (R 4.3.3) lifecycle 1.0.4 2023-11-07 [2] CRAN (R 4.3.3) magrittr 2.0.3 2022-03-30 [2] CRAN (R 4.3.3) memoise 2.0.1 2021-11-26 [2] CRAN (R 4.3.3) mime 0.12 2021-09-28 [2] CRAN (R 4.3.1) miniUI 0.1.1.1 2018-05-18 [1] CRAN (R 4.3.3) paws 0.6.0 2024-05-13 [1] https://paws-r.r-universe.dev (R 4.3.3) paws.common * 0.7.2 2024-04-09 [1] CRAN (R 4.3.3) paws.storage * 0.6.0 2024-05-13 [1] https://paws-r.r-universe.dev (R 4.3.3) pkgbuild 1.4.4 2024-03-17 [1] CRAN (R 4.3.3) pkgconfig 2.0.3 2019-09-22 [2] CRAN (R 4.3.3) pkgload 1.3.4 2024-01-16 [1] CRAN (R 4.3.3) profvis 0.3.8 2023-05-02 [1] CRAN (R 4.3.3) promises 1.3.0 2024-04-05 [2] CRAN (R 4.3.3) purrr 1.0.2 2023-08-10 [2] CRAN (R 4.3.3) R6 2.5.1 2021-08-19 [2] CRAN (R 4.3.3) Rcpp 1.0.12 2024-01-09 [2] CRAN (R 4.3.3) RcppTOML 0.2.2 2023-01-29 [2] CRAN (R 4.3.3) remotes 2.5.0 2024-03-17 [2] CRAN (R 4.3.3) rlang 1.1.3 2024-01-10 [2] CRAN (R 4.3.3) RMariaDB 1.3.1 2023-10-26 [2] CRAN (R 4.3.3) rmarkdown 2.26 2024-03-05 [2] CRAN (R 4.3.3) RSQLite 2.3.6 2024-03-31 [2] CRAN (R 4.3.3) rstudioapi 0.16.0 2024-03-24 [2] CRAN (R 4.3.3) sessioninfo 1.2.2 2021-12-06 [1] CRAN (R 4.3.3) shiny 1.8.1.1 2024-04-02 [2] CRAN (R 4.3.3) stringi 1.8.3 2023-12-11 [2] CRAN (R 4.3.2) stringr 1.5.1 2023-11-14 [2] CRAN (R 4.3.3) urlchecker 1.0.1 2021-11-30 [1] CRAN (R 4.3.3) usethis 2.2.3 2024-02-19 [1] CRAN (R 4.3.3) vctrs 0.6.5 2023-12-01 [2] CRAN (R 4.3.3) xfun 0.43 2024-03-25 [2] CRAN (R 4.3.3) xml2 1.3.6 2023-12-04 [2] CRAN (R 4.3.3) xtable 1.8-4 2019-04-21 [2] CRAN (R 4.3.3) yaml 2.3.8 2023-12-11 [2] CRAN (R 4.3.2) [1] C:/Users/Zoom/AppData/Local/R/win-library/4.3 [2] C:/Program Files/R/R-4.3.3/library
DyfanJones commented 4 months ago

Ah my apologises I didn't include method to install paws.common 0.7.1: 

remotes::install_version("paws.common", "0.7.1")
stevepowell99 commented 4 months ago

yes! It works with version 0.7.1: I can list versions. Thanks!

DyfanJones commented 4 months ago

hmm i will have to investigate why it has started to fail with v0.7.2 and why i can't replicate it

stevepowell99 commented 4 months ago

but this will explain why it suddenly started not working, I must have somewhere updated all packages without checking.

DyfanJones commented 4 months ago

It is interesting it didn't appear on my windows machine plus all the unit tests we are currently doing.

DyfanJones commented 4 months ago

@stevepowell99 Does this hold true in your R environment?

x <- list(list(a = 1, b = 2), list(a = 3, b = 4), list(a = 5, b = 6))

setNames(.mapply(list, x, NULL), names(x[[1]]))
#> $a
#> $a[[1]]
#> [1] 1
#> 
#> $a[[2]]
#> [1] 3
#> 
#> $a[[3]]
#> [1] 5
#> 
#> 
#> $b
#> $b[[1]]
#> [1] 2
#> 
#> $b[[2]]
#> [1] 4
#> 
#> $b[[3]]
#> [1] 6

identical(
  setNames(.mapply(list, x, NULL), names(x[[1]])),
  purrr::list_transpose(x, simplify = F)
)
#> [1] TRUE

bench::mark(
  base = setNames(.mapply(list, x, NULL), names(x[[1]])),
  purrr = purrr::list_transpose(x, simplify = F)
)
#> # A tibble: 2 × 6
#>   expression      min   median `itr/sec` mem_alloc `gc/sec`
#>   <bch:expr> <bch:tm> <bch:tm>     <dbl> <bch:byt>    <dbl>
#> 1 base         1.97µs   2.38µs   370007.        0B     37.0
#> 2 purrr       55.68µs  60.93µs    16004.        0B     33.6

Created on 2024-05-14 with reprex v2.1.0

stevepowell99 commented 4 months ago

yes it is TRUE image

stevepowell99 commented 4 months ago

unfortunately I now can't deploy my shiny app because I get Error building image: Error fetching paws.compute (0.6.0) source. <CRANPackageSource repo='http://cran.rstudio.org'> unable to satisfy package: paws.compute (0.6.0) Should I be using another version of paws.compute??

DyfanJones commented 4 months ago

I am currently in the process of releasing 0.6.0 https://github.com/paws-r/paws/issues/776. You can use r-universe https://paws-r.r-universe.dev/builds to get the latest version.

DyfanJones commented 4 months ago

Hmm I was thinking it would of been .mapply that had caused the issue (https://github.com/cran/paws.common/commit/cb6efad434fa4d48bcc875e395ec22796be4900d#diff-769cd5041a0c33c7590ddc41aca9f3e5237da7c1eaeedbdfb310698a43e43727) but it doesn't look like.

DyfanJones commented 4 months ago

I will finish release 0.6.0 and revisit this issue.

stevepowell99 commented 4 months ago

I tried uninstalling paws and paws.storage and paws.common and reinstalling from CRAN but again I can't list_versions

DyfanJones commented 4 months ago

I have managed to replicate this issue. I will see what has caused it to fail and get it in the next paws.common release :)

DyfanJones commented 4 months ago

I believe i have found the issue:

obj <- list(
  var1 = c(1, 2, 3),
  var2 = letters[1:3],
  var3 = list(),
  var4 = list()
)

.mapply(list, obj, NULL)
#> list()

purrr::list_transpose(obj, simplify = F)
#> [[1]]
#> [[1]]$var1
#> [1] 1
#> 
#> [[1]]$var2
#> [1] "a"
#> 
#> [[1]]$var3
#> NULL
#> 
#> [[1]]$var4
#> NULL
#> 
#> 
#> [[2]]
#> [[2]]$var1
#> [1] 2
#> 
#> [[2]]$var2
#> [1] "b"
#> 
#> [[2]]$var3
#> NULL
#> 
#> [[2]]$var4
#> NULL
#> 
#> 
#> [[3]]
#> [[3]]$var1
#> [1] 3
#> 
#> [[3]]$var2
#> [1] "c"
#> 
#> [[3]]$var3
#> NULL
#> 
#> [[3]]$var4
#> NULL

transpose <- function(x) {
  if (any(found <- lengths(x) == 0)) {
    n_row <- length(x[[1]])
    x[found] <- list(rep_len(x[found], n_row))
  }
  .mapply(list, x, NULL)
}

transpose(obj)
#> [[1]]
#> [[1]]$var1
#> [1] 1
#> 
#> [[1]]$var2
#> [1] "a"
#> 
#> [[1]]$var3
#> list()
#> 
#> [[1]]$var4
#> list()
#> 
#> 
#> [[2]]
#> [[2]]$var1
#> [1] 2
#> 
#> [[2]]$var2
#> [1] "b"
#> 
#> [[2]]$var3
#> list()
#> 
#> [[2]]$var4
#> list()
#> 
#> 
#> [[3]]
#> [[3]]$var1
#> [1] 3
#> 
#> [[3]]$var2
#> [1] "c"
#> 
#> [[3]]$var3
#> list()
#> 
#> [[3]]$var4
#> list()

bench::mark(
  purrr::list_transpose(obj, simplify = F),
  transpose(obj),
  check = F
)
#> # A tibble: 2 × 6
#>   expression                            min  median `itr/sec` mem_alloc `gc/sec`
#>   <bch:expr>                        <bch:t> <bch:t>     <dbl> <bch:byt>    <dbl>
#> 1 purrr::list_transpose(obj, simpl… 73.39µs  81.1µs    11853.        0B     36.2
#> 2 transpose(obj)                     4.14µs  4.71µs   198624.    4.41MB     39.7

Created on 2024-05-15 with reprex v2.1.0

stevepowell99 commented 4 months ago

thanks so what should I do? wait for the upcoming version?

DyfanJones commented 4 months ago

@stevepowell99 I have submitted paws.common 0.7.3 to the cran. Will let you know when it has been accepted

DyfanJones commented 4 months ago

paws.common 0.7.3 has been released to the cran. Also paws.compute has been released to the cran

stevepowell99 commented 4 months ago

thanks! please could you clarify what is the procedure to update now on Windows? Is it to install just paws and/or paws.common and/or paws.compute and/or paws.storage?

DyfanJones commented 4 months ago

If you update paws.common it will fix your issue. You can also update the entire SDK by update all paws packages.

Note: paws.common holds all the functionality i.e how it builds the API call (similar to botocore). The other packages create the client for each Aws service.

stevepowell99 commented 4 months ago

works perfectly 🥇 🥇 🥇