Open ORuessel opened 6 months ago
Maybe this code helps to support to get this fast as possible
package getter
import (
"fmt"
"net/url"
"strings"
)
// S3Detector implements Detector to detect S3 URLs and turn
// them into URLs that the S3 getter can understand.
type S3Detector struct{}
func (d *S3Detector) Detect(src, _ string) (string, bool, error) {
if len(src) == 0 {
return "", false, nil
}
if strings.Contains(src, ".amazonaws.com/") {
return d.detectHTTP(src)
}
return "", false, nil
}
func (d *S3Detector) detectHTTP(src string) (string, bool, error) {
parts := strings.Split(src, "/")
if len(parts) < 2 {
return "", false, fmt.Errorf("URL is not a valid S3 URL")
}
hostParts := strings.Split(parts[0], ".")
switch {
case len(hostParts) == 3:
return d.detectPathStyle(hostParts[0], parts[1:])
case len(hostParts) == 4:
return d.detectVhostStyle(hostParts[1], hostParts[0], parts[1:])
case len(hostParts) == 5 && hostParts[1] == "s3":
return d.detectNewVhostStyle(hostParts[2], hostParts[0], parts[1:])
case len(hostParts) > 5 && strings.Contains(hostParts[1], "vpce"): // New case for VPC endpoint URLs
return d.detectVPCEStyle(hostParts, parts[1:])
default:
return "", false, fmt.Errorf("URL is not a valid S3 URL")
}
}
func (d *S3Detector) detectPathStyle(region string, parts []string) (string, bool, error) {
urlStr := fmt.Sprintf("https://%s.amazonaws.com/%s", region, strings.Join(parts, "/"))
url, err := url.Parse(urlStr)
if err != nil {
return "", false, fmt.Errorf("error parsing S3 URL: %s", err)
}
return "s3::" + url.String(), true, nil
}
func (d *S3Detector) detectVhostStyle(region, bucket string, parts []string) (string, bool, error) {
urlStr := fmt.Sprintf("https://%s.amazonaws.com/%s/%s", region, bucket, strings.Join(parts, "/"))
url, err := url.Parse(urlStr)
if err != nil {
return "", false, fmt.Errorf("error parsing S3 URL: %s", err)
}
return "s3::" + url.String(), true, nil
}
func (d *S3Detector) detectNewVhostStyle(region, bucket string, parts []string) (string, bool, error) {
urlStr := fmt.Sprintf("https://s3.%s.amazonaws.com/%s/%s", region, bucket, strings.Join(parts, "/"))
url, err := url.Parse(urlStr)
if err != nil {
return "", false, fmt.Errorf("error parsing S3 URL: %s", err)
}
return "s3::" + url.String(), true, nil
}
// New function to handle the special VPC endpoint style URLs.
func (d *S3Detector) detectVPCEStyle(hostParts []string, parts []string) (string, bool, error) {
// Assuming the bucket name is the first part and the region is the fourth part
bucket := hostParts[0]
region := hostParts[3]
urlStr := fmt.Sprintf("https://s3.%s.amazonaws.com/%s/%s", region, bucket, strings.Join(parts, "/"))
url, err := url.Parse(urlStr)
if err != nil {
return "", false, fmt.Errorf("error parsing S3 URL: %s", err)
}
return "s3::" + url.String(), true, nil
}
Hi @ORuessel and thanks for raising this issue. Nomad imports the go-getter library, and therefore I think this issue should be moved to that repository as a feature enhancement. I've not taken a complete look through the code you've added, but if you want to raise a PR against the repository, please feel free to do so.
Hi @jrasell thanks you for you fast answer. Do you know any possibility to check the code with a own compiled nomad and go-getter version ? Then i want to raise a PR with a tested code. That it will do the expected thing.
Hi @ORuessel; if you have both sets of code checked out locally, you can edit Nomads go.mod file to include a replace statement. This statement would point the dependency to your local go-getter code, and would look similar to this current replace statement.
Hi there, we use in our AWS and Hybrid environment private Endpoints example URL : s3://artifact.vpce-0e10d123f4a5e8190-1rbv44qv.s3.eu-west-2.vpce.amazonaws.com/files/test.zip
I receive follow error message: ": MissingRegion: could not find region configuration
After check of the go-getter code maybe we receive this error message because no match of the url based on the logic of the s3_detect code https://github.com/hashicorp/go-getter/blob/main/detect_s3.go
Is it possible to add a match for the s3 detector to get the region information for this url ?
Thanks for your support
Proposal
Use-cases
Attempted Solutions