Open davidread opened 4 years ago
Workaround from another user:
There are a small number of people who experience this problem when they try to write to S3 due to issues in aws.s3. I have re-written the write function but I have not had time to integrate them properly into the package yet:
write_df_to_csv_in_s3_amended <- function(df, s3_path, overwrite=FALSE, multipart=TRUE, ...) { # write to an in-memory raw connection rc <- rawConnection(raw(0), "r+") write.csv(df, rc, ...) # upload the object to S3 # credentials <- suppressMessages(get_credentials()) # suppressMessages(refresh(credentials)) p <- separate_bucket_path(s3_path) if (overwrite || !(s3_file_exists(s3_path))) { rcv <- rawConnectionValue(rc) close(rc) return(aws.s3::put_object(file = rcv, bucket = p$bucket, object = p$object)) } else { close(rc) stop("File already exists and you haven't set overwrite = TRUE, stopping") } # close the connection close(rc) }
Equivalent function for general files:
write_file_to_s3_amended <- function(local_file_path, s3_path, overwrite=FALSE, multipart=TRUE) {
p <- separate_bucket_path(s3_path)
if (overwrite || !(s3_file_exists(s3_path))) {
return(aws.s3::put_object(file = local_file_path,
bucket = p$bucket,
object = p$object))
} else {
stop("File already exists and you haven't set overwrite = TRUE, stopping")
}
}
separate_bucket_path <- function(path) {
if (substring(path, 1, 1) == "/") {
path <- substring(path, 2)
}
parts <- strsplit(path, "/")[[1]]
bucket <- parts[1]
otherparts <- parts[2:length(parts)]
object <- paste(otherparts, collapse="/")
list("object" = object, "bucket" = bucket )
}
s3_file_exists <- function(s3_path) {
p <- separate_bucket_path(s3_path)
objs <- aws.s3::get_bucket(p$bucket, prefix = p$object, check_region=TRUE)
return(length(objs)>0)
}'