X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/927524f1be454de021180b74999d682780b8cb6b..b20fb44b419ba6dd800874552bcfb5b87479611d:/services/keepstore/s3aws_volume.go diff --git a/services/keepstore/s3aws_volume.go b/services/keepstore/s3aws_volume.go index 6205da5beb..8f2c275391 100644 --- a/services/keepstore/s3aws_volume.go +++ b/services/keepstore/s3aws_volume.go @@ -33,6 +33,21 @@ import ( "github.com/sirupsen/logrus" ) +func init() { + driver["S3"] = newS3AWSVolume +} + +const ( + s3DefaultReadTimeout = arvados.Duration(10 * time.Minute) + s3DefaultConnectTimeout = arvados.Duration(time.Minute) + maxClockSkew = 600 * time.Second + nearlyRFC1123 = "Mon, 2 Jan 2006 15:04:05 GMT" +) + +var ( + ErrS3TrashDisabled = fmt.Errorf("trash function is disabled because Collections.BlobTrashLifetime=0 and DriverParameters.UnsafeDelete=false") +) + // S3AWSVolume implements Volume using an S3 bucket. type S3AWSVolume struct { arvados.S3VolumeDriverParameters @@ -58,22 +73,6 @@ type s3AWSbucket struct { mu sync.Mutex } -// chooseS3VolumeDriver distinguishes between the old goamz driver and -// aws-sdk-go based on the UseAWSS3v2Driver feature flag -func chooseS3VolumeDriver(cluster *arvados.Cluster, volume arvados.Volume, logger logrus.FieldLogger, metrics *volumeMetricsVecs) (Volume, error) { - v := &S3Volume{cluster: cluster, volume: volume, metrics: metrics} - err := json.Unmarshal(volume.DriverParameters, v) - if err != nil { - return nil, err - } - if v.UseAWSS3v2Driver { - logger.Debugln("Using AWS S3 v2 driver") - return newS3AWSVolume(cluster, volume, logger, metrics) - } - logger.Debugln("Using goamz S3 driver") - return newS3Volume(cluster, volume, logger, metrics) -} - const ( PartSize = 5 * 1024 * 1024 ReadConcurrency = 13 @@ -182,19 +181,25 @@ func (v *S3AWSVolume) check(ec2metadataHostname string) error { if v.Endpoint != "" && service == "s3" { return aws.Endpoint{ URL: v.Endpoint, - SigningRegion: v.Region, + SigningRegion: region, }, nil } else if service == "ec2metadata" && ec2metadataHostname != "" { return aws.Endpoint{ URL: ec2metadataHostname, }, nil + } else { + return defaultResolver.ResolveEndpoint(service, region) } - - return defaultResolver.ResolveEndpoint(service, region) } cfg.EndpointResolver = aws.EndpointResolverFunc(myCustomResolver) } - + if v.Region == "" { + // Endpoint is already specified (otherwise we would + // have errored out above), but Region is also + // required by the aws sdk, in order to determine + // SignatureVersions. + v.Region = "us-east-1" + } cfg.Region = v.Region // Zero timeouts mean "wait forever", which is a bad