"Upgrading from 2.1.0":#v2_1_0
+h3. New spelling of S3 credential configs
+
+If you use the S3 driver for Keep volumes and specify credentials in your configuration file (as opposed to using an IAM role), you should change the spelling of the @AccessKey@ and @SecretKey@ config keys to @AccessKeyID@ and @SecretAccessKey@. If you don't update them, the previous spellings will still be accepted, but warnings will be logged at server startup.
+
h3. New proxy parameters for arvados-controller
In your Nginx configuration file (@/etc/nginx/conf.d/arvados-api-and-controller.conf@), add the following lines to the @location /@ block with @http://controller@ (see "Update nginx configuration":{{site.baseurl}}/install/install-api-server.html#update-nginx for an example) and reload/restart Nginx (@sudo nginx -s reload@).
# If you are not using an IAM role for authentication,
# specify access credentials here instead.
- AccessKey: <span class="userinput">""</span>
- SecretKey: <span class="userinput">""</span>
+ AccessKeyID: <span class="userinput">""</span>
+ SecretAccessKey: <span class="userinput">""</span>
# Storage provider region. For Google Cloud Storage, use ""
# or omit.
StorageClasses:
default: true
SAMPLE: true
- Driver: s3
+ Driver: S3
DriverParameters:
# for s3 driver -- see
# https://doc.arvados.org/install/configure-s3-object-storage.html
IAMRole: aaaaa
- AccessKey: aaaaa
- SecretKey: aaaaa
+ AccessKeyID: aaaaa
+ SecretAccessKey: aaaaa
Endpoint: ""
Region: us-east-1a
Bucket: aaaaa
package config
import (
+ "encoding/json"
"fmt"
"io/ioutil"
"net/url"
return nil
}
+func (ldr *Loader) applyDeprecatedVolumeDriverParameters(cfg *arvados.Config) error {
+ for clusterID, cluster := range cfg.Clusters {
+ for volID, vol := range cluster.Volumes {
+ if vol.Driver == "S3" {
+ var params struct {
+ AccessKey string `json:",omitempty"`
+ SecretKey string `json:",omitempty"`
+ AccessKeyID string
+ SecretAccessKey string
+ }
+ err := json.Unmarshal(vol.DriverParameters, ¶ms)
+ if err != nil {
+ return fmt.Errorf("error loading %s.Volumes.%s.DriverParameters: %w", clusterID, volID, err)
+ }
+ if params.AccessKey != "" || params.SecretKey != "" {
+ if params.AccessKeyID != "" || params.SecretAccessKey != "" {
+ return fmt.Errorf("cannot use old keys (AccessKey/SecretKey) and new keys (AccessKeyID/SecretAccessKey) at the same time in %s.Volumes.%s.DriverParameters -- you must remove the old config keys", clusterID, volID)
+ continue
+ }
+ var allparams map[string]interface{}
+ err = json.Unmarshal(vol.DriverParameters, &allparams)
+ if err != nil {
+ return fmt.Errorf("error loading %s.Volumes.%s.DriverParameters: %w", clusterID, volID, err)
+ }
+ for k := range allparams {
+ if lk := strings.ToLower(k); lk == "accesskey" || lk == "secretkey" {
+ delete(allparams, k)
+ }
+ }
+ ldr.Logger.Warnf("using your old config keys %s.Volumes.%s.DriverParameters.AccessKey/SecretKey -- but you should rename them to AccessKeyID/SecretAccessKey", clusterID, volID)
+ allparams["AccessKeyID"] = params.AccessKey
+ allparams["SecretAccessKey"] = params.SecretKey
+ vol.DriverParameters, err = json.Marshal(allparams)
+ if err != nil {
+ return err
+ }
+ cluster.Volumes[volID] = vol
+ }
+ }
+ }
+ }
+ return nil
+}
+
func applyDeprecatedNodeProfile(hostname string, ssi systemServiceInstance, svc *arvados.Service) {
scheme := "https"
if !ssi.TLS {
StorageClasses: array2boolmap(oldvol.StorageClasses),
}
params = arvados.S3VolumeDriverParameters{
- AccessKey: string(bytes.TrimSpace(accesskeydata)),
- SecretKey: string(bytes.TrimSpace(secretkeydata)),
+ AccessKeyID: string(bytes.TrimSpace(accesskeydata)),
+ SecretAccessKey: string(bytes.TrimSpace(secretkeydata)),
Endpoint: oldvol.Endpoint,
Region: oldvol.Region,
Bucket: oldvol.Bucket,
Driver: "S3",
Replication: 4,
}, &arvados.S3VolumeDriverParameters{
- AccessKey: "accesskeydata",
- SecretKey: "secretkeydata",
+ AccessKeyID: "accesskeydata",
+ SecretAccessKey: "secretkeydata",
Endpoint: "https://storage.googleapis.com",
Region: "us-east-1z",
Bucket: "testbucket",
return cluster, nil
}
+func (s *LoadSuite) TestLegacyVolumeDriverParameters(c *check.C) {
+ logs := checkEquivalent(c, `
+Clusters:
+ z1111:
+ Volumes:
+ z1111-nyw5e-aaaaaaaaaaaaaaa:
+ Driver: S3
+ DriverParameters:
+ AccessKey: exampleaccesskey
+ SecretKey: examplesecretkey
+ Region: foobar
+ ReadTimeout: 1200s
+`, `
+Clusters:
+ z1111:
+ Volumes:
+ z1111-nyw5e-aaaaaaaaaaaaaaa:
+ Driver: S3
+ DriverParameters:
+ AccessKeyID: exampleaccesskey
+ SecretAccessKey: examplesecretkey
+ Region: foobar
+ ReadTimeout: 1200s
+`)
+ c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*AccessKey.*`)
+ c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*SecretKey.*`)
+ c.Check(logs, check.Matches, `(?ms).*using your old config keys z1111\.Volumes\.z1111-nyw5e-aaaaaaaaaaaaaaa\.DriverParameters\.AccessKey/SecretKey -- but you should rename them to AccessKeyID/SecretAccessKey.*`)
+
+ _, err := testLoader(c, `
+Clusters:
+ z1111:
+ Volumes:
+ z1111-nyw5e-aaaaaaaaaaaaaaa:
+ Driver: S3
+ DriverParameters:
+ AccessKey: exampleaccesskey
+ SecretKey: examplesecretkey
+ AccessKeyID: exampleaccesskey
+`, nil).Load()
+ c.Check(err, check.ErrorMatches, `(?ms).*cannot use .*SecretKey.*and.*SecretAccessKey.*in z1111.Volumes.z1111-nyw5e-aaaaaaaaaaaaaaa.DriverParameters.*`)
+}
+
func (s *LoadSuite) TestDeprecatedNodeProfilesToServices(c *check.C) {
hostname, err := os.Hostname()
c.Assert(err, check.IsNil)
StorageClasses:
default: true
SAMPLE: true
- Driver: s3
+ Driver: S3
DriverParameters:
# for s3 driver -- see
# https://doc.arvados.org/install/configure-s3-object-storage.html
IAMRole: aaaaa
- AccessKey: aaaaa
- SecretKey: aaaaa
+ AccessKeyID: aaaaa
+ SecretAccessKey: aaaaa
Endpoint: ""
Region: us-east-1a
Bucket: aaaaa
return nil, fmt.Errorf("transcoding config data: %s", err)
}
+ var loadFuncs []func(*arvados.Config) error
if !ldr.SkipDeprecated {
- err = ldr.applyDeprecatedConfig(&cfg)
- if err != nil {
- return nil, err
- }
+ loadFuncs = append(loadFuncs,
+ ldr.applyDeprecatedConfig,
+ ldr.applyDeprecatedVolumeDriverParameters,
+ )
}
if !ldr.SkipLegacy {
// legacy file is required when either:
// * a non-default location was specified
// * no primary config was loaded, and this is the
// legacy config file for the current component
- for _, err := range []error{
- ldr.loadOldEnvironmentVariables(&cfg),
- ldr.loadOldKeepstoreConfig(&cfg),
- ldr.loadOldKeepWebConfig(&cfg),
- ldr.loadOldCrunchDispatchSlurmConfig(&cfg),
- ldr.loadOldWebsocketConfig(&cfg),
- ldr.loadOldKeepproxyConfig(&cfg),
- ldr.loadOldGitHttpdConfig(&cfg),
- ldr.loadOldKeepBalanceConfig(&cfg),
- } {
- if err != nil {
- return nil, err
- }
+ loadFuncs = append(loadFuncs,
+ ldr.loadOldEnvironmentVariables,
+ ldr.loadOldKeepstoreConfig,
+ ldr.loadOldKeepWebConfig,
+ ldr.loadOldCrunchDispatchSlurmConfig,
+ ldr.loadOldWebsocketConfig,
+ ldr.loadOldKeepproxyConfig,
+ ldr.loadOldGitHttpdConfig,
+ ldr.loadOldKeepBalanceConfig,
+ )
+ }
+ for _, f := range loadFuncs {
+ err = f(&cfg)
+ if err != nil {
+ return nil, err
}
}
`)
}
-func checkEquivalent(c *check.C, goty, expectedy string) {
- gotldr := testLoader(c, goty, nil)
+func checkEquivalent(c *check.C, goty, expectedy string) string {
+ var logbuf bytes.Buffer
+ gotldr := testLoader(c, goty, &logbuf)
expectedldr := testLoader(c, expectedy, nil)
checkEquivalentLoaders(c, gotldr, expectedldr)
+ return logbuf.String()
}
func checkEqualYAML(c *check.C, got, expected interface{}) {
type S3VolumeDriverParameters struct {
IAMRole string
- AccessKey string
- SecretKey string
+ AccessKeyID string
+ SecretAccessKey string
Endpoint string
Region string
Bucket string
flags.String("s3-bucket-volume", "", "Volumes.*.DriverParameters.Bucket")
flags.String("s3-region", "", "Volumes.*.DriverParameters.Region")
flags.String("s3-endpoint", "", "Volumes.*.DriverParameters.Endpoint")
- flags.String("s3-access-key-file", "", "Volumes.*.DriverParameters.AccessKey")
- flags.String("s3-secret-key-file", "", "Volumes.*.DriverParameters.SecretKey")
+ flags.String("s3-access-key-file", "", "Volumes.*.DriverParameters.AccessKeyID")
+ flags.String("s3-secret-key-file", "", "Volumes.*.DriverParameters.SecretAccessKey")
flags.String("s3-race-window", "", "Volumes.*.DriverParameters.RaceWindow")
flags.String("s3-replication", "", "Volumes.*.Replication")
flags.String("s3-unsafe-delete", "", "Volumes.*.DriverParameters.UnsafeDelete")
}
func (v *S3Volume) bootstrapIAMCredentials() error {
- if v.AccessKey != "" || v.SecretKey != "" {
+ if v.AccessKeyID != "" || v.SecretAccessKey != "" {
if v.IAMRole != "" {
- return errors.New("invalid DriverParameters: AccessKey and SecretKey must be blank if IAMRole is specified")
+ return errors.New("invalid DriverParameters: AccessKeyID and SecretAccessKey must be blank if IAMRole is specified")
}
return nil
}
}
func (v *S3Volume) newS3Client() *s3.S3 {
- auth := aws.NewAuth(v.AccessKey, v.SecretKey, v.AuthToken, v.AuthExpiration)
+ auth := aws.NewAuth(v.AccessKeyID, v.SecretAccessKey, v.AuthToken, v.AuthExpiration)
client := s3.New(*auth, v.region)
if !v.V2Signature {
client.Signature = aws.V4Signature
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
- return 0, fmt.Errorf("this instance does not have an IAM role assigned -- either assign a role, or configure AccessKey and SecretKey explicitly in DriverParameters (error getting %s: HTTP status %s)", url, resp.Status)
+ return 0, fmt.Errorf("this instance does not have an IAM role assigned -- either assign a role, or configure AccessKeyID and SecretAccessKey explicitly in DriverParameters (error getting %s: HTTP status %s)", url, resp.Status)
} else if resp.StatusCode != http.StatusOK {
return 0, fmt.Errorf("error getting %s: HTTP status %s", url, resp.Status)
}
if err != nil {
return 0, fmt.Errorf("error decoding credentials from %s: %s", url, err)
}
- v.AccessKey, v.SecretKey, v.AuthToken, v.AuthExpiration = cred.AccessKeyID, cred.SecretAccessKey, cred.Token, cred.Expiration
+ v.AccessKeyID, v.SecretAccessKey, v.AuthToken, v.AuthExpiration = cred.AccessKeyID, cred.SecretAccessKey, cred.Token, cred.Expiration
v.bucket.SetBucket(&s3.Bucket{
S3: v.newS3Client(),
Name: v.Bucket,
// Default V4 signature
vol := S3Volume{
S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
- AccessKey: "xxx",
- SecretKey: "xxx",
- Endpoint: stub.URL,
- Region: "test-region-1",
- Bucket: "test-bucket-name",
+ AccessKeyID: "xxx",
+ SecretAccessKey: "xxx",
+ Endpoint: stub.URL,
+ Region: "test-region-1",
+ Bucket: "test-bucket-name",
},
cluster: s.cluster,
logger: ctxlog.TestLogger(c),
// Force V2 signature
vol = S3Volume{
S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
- AccessKey: "xxx",
- SecretKey: "xxx",
- Endpoint: stub.URL,
- Region: "test-region-1",
- Bucket: "test-bucket-name",
- V2Signature: true,
+ AccessKeyID: "xxx",
+ SecretAccessKey: "xxx",
+ Endpoint: stub.URL,
+ Region: "test-region-1",
+ Bucket: "test-bucket-name",
+ V2Signature: true,
},
cluster: s.cluster,
logger: ctxlog.TestLogger(c),
defer s.metadata.Close()
v := s.newTestableVolume(c, s.cluster, arvados.Volume{Replication: 2}, newVolumeMetricsVecs(prometheus.NewRegistry()), 5*time.Minute)
- c.Check(v.AccessKey, check.Equals, "ASIAIOSFODNN7EXAMPLE")
- c.Check(v.SecretKey, check.Equals, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+ c.Check(v.AccessKeyID, check.Equals, "ASIAIOSFODNN7EXAMPLE")
+ c.Check(v.SecretAccessKey, check.Equals, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
c.Check(v.bucket.bucket.S3.Auth.AccessKey, check.Equals, "ASIAIOSFODNN7EXAMPLE")
c.Check(v.bucket.bucket.S3.Auth.SecretKey, check.Equals, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
S3Volume: &S3Volume{
S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
IAMRole: iamRole,
- AccessKey: accessKey,
- SecretKey: secretKey,
+ AccessKeyID: accessKey,
+ SecretAccessKey: secretKey,
Bucket: TestBucketName,
Endpoint: endpoint,
Region: "test-region-1",
creds := aws.NewChainProvider(
[]aws.CredentialsProvider{
- aws.NewStaticCredentialsProvider(v.AccessKey, v.SecretKey, v.AuthToken),
+ aws.NewStaticCredentialsProvider(v.AccessKeyID, v.SecretAccessKey, v.AuthToken),
ec2rolecreds.New(ec2metadata.New(cfg)),
})
// as of June 24, 2020. Cf. https://forums.aws.amazon.com/ann.jspa?annID=5816
vol := S3AWSVolume{
S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
- AccessKey: "xxx",
- SecretKey: "xxx",
- Endpoint: stub.URL,
- Region: "test-region-1",
- Bucket: "test-bucket-name",
+ AccessKeyID: "xxx",
+ SecretAccessKey: "xxx",
+ Endpoint: stub.URL,
+ Region: "test-region-1",
+ Bucket: "test-bucket-name",
},
cluster: s.cluster,
logger: ctxlog.TestLogger(c),
S3AWSVolume: &S3AWSVolume{
S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
IAMRole: iamRole,
- AccessKey: accessKey,
- SecretKey: secretKey,
+ AccessKeyID: accessKey,
+ SecretAccessKey: secretKey,
Bucket: S3AWSTestBucketName,
Endpoint: endpoint,
Region: "test-region-1",
projects / arvados.git / commitdiff