X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/8da1cf8c69337be51a7ace6923d8c0c0bc7d36e1..b541c9d898d3dde983de2e0ea80a40e17d4c9b9f:/lib/cloud/ec2/ec2_test.go

diff --git a/lib/cloud/ec2/ec2_test.go b/lib/cloud/ec2/ec2_test.go
index 6fde4bbbca..5e6cf2c82b 100644
--- a/lib/cloud/ec2/ec2_test.go
+++ b/lib/cloud/ec2/ec2_test.go
@@ -277,6 +277,12 @@ func (*EC2InstanceSetSuite) TestCreate(c *check.C) {
 	if *live == "" {
 		c.Check(ap.client.(*ec2stub).describeKeyPairsCalls, check.HasLen, 1)
 		c.Check(ap.client.(*ec2stub).importKeyPairCalls, check.HasLen, 1)
+
+		runcalls := ap.client.(*ec2stub).runInstancesCalls
+		if c.Check(runcalls, check.HasLen, 1) {
+			c.Check(runcalls[0].MetadataOptions.HttpEndpoint, check.DeepEquals, aws.String("enabled"))
+			c.Check(runcalls[0].MetadataOptions.HttpTokens, check.DeepEquals, aws.String("required"))
+		}
 	}
 }
 
@@ -357,6 +363,57 @@ func (*EC2InstanceSetSuite) TestCreateFailoverSecondSubnet(c *check.C) {
 		`.*`)
 }
 
+func (*EC2InstanceSetSuite) TestIsErrorSubnetSpecific(c *check.C) {
+	c.Check(isErrorSubnetSpecific(nil), check.Equals, false)
+	c.Check(isErrorSubnetSpecific(errors.New("misc error")), check.Equals, false)
+
+	c.Check(isErrorSubnetSpecific(&ec2stubError{
+		code: "InsufficientInstanceCapacity",
+	}), check.Equals, true)
+
+	c.Check(isErrorSubnetSpecific(&ec2stubError{
+		code: "InsufficientVolumeCapacity",
+	}), check.Equals, true)
+
+	c.Check(isErrorSubnetSpecific(&ec2stubError{
+		code:    "InsufficientFreeAddressesInSubnet",
+		message: "Not enough free addresses in subnet subnet-abcdefg\n\tstatus code: 400, request id: abcdef01-2345-6789-abcd-ef0123456789",
+	}), check.Equals, true)
+
+	// #21603: (Sometimes?) EC2 returns code InvalidParameterValue
+	// even though the code "InsufficientFreeAddressesInSubnet"
+	// seems like it must be meant for exactly this error.
+	c.Check(isErrorSubnetSpecific(&ec2stubError{
+		code:    "InvalidParameterValue",
+		message: "Not enough free addresses in subnet subnet-abcdefg\n\tstatus code: 400, request id: abcdef01-2345-6789-abcd-ef0123456789",
+	}), check.Equals, true)
+
+	// Similarly, AWS docs
+	// (https://repost.aws/knowledge-center/vpc-insufficient-ip-errors)
+	// suggest the following code/message combinations also exist.
+	c.Check(isErrorSubnetSpecific(&ec2stubError{
+		code:    "Client.InvalidParameterValue",
+		message: "There aren't sufficient free Ipv4 addresses or prefixes",
+	}), check.Equals, true)
+	c.Check(isErrorSubnetSpecific(&ec2stubError{
+		code:    "InvalidParameterValue",
+		message: "There aren't sufficient free Ipv4 addresses or prefixes",
+	}), check.Equals, true)
+	// Meanwhile, other AWS docs
+	// (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html)
+	// suggest Client.InvalidParameterValue is not a real code but
+	// ClientInvalidParameterValue is.
+	c.Check(isErrorSubnetSpecific(&ec2stubError{
+		code:    "ClientInvalidParameterValue",
+		message: "There aren't sufficient free Ipv4 addresses or prefixes",
+	}), check.Equals, true)
+
+	c.Check(isErrorSubnetSpecific(&ec2stubError{
+		code:    "InvalidParameterValue",
+		message: "Some other invalid parameter error",
+	}), check.Equals, false)
+}
+
 func (*EC2InstanceSetSuite) TestCreateAllSubnetsFailing(c *check.C) {
 	if *live != "" {
 		c.Skip("not applicable in live mode")
@@ -399,6 +456,37 @@ func (*EC2InstanceSetSuite) TestCreateAllSubnetsFailing(c *check.C) {
 		`.*`)
 }
 
+func (*EC2InstanceSetSuite) TestCreateOneSubnetFailingCapacity(c *check.C) {
+	if *live != "" {
+		c.Skip("not applicable in live mode")
+		return
+	}
+	ap, img, cluster, reg := GetInstanceSet(c, `{"SubnetID":["subnet-full","subnet-broken"]}`)
+	ap.client.(*ec2stub).subnetErrorOnRunInstances = map[string]error{
+		"subnet-full": &ec2stubError{
+			code:    "InsufficientFreeAddressesInSubnet",
+			message: "subnet is full",
+		},
+		"subnet-broken": &ec2stubError{
+			code:    "InsufficientInstanceCapacity",
+			message: "insufficient capacity",
+		},
+	}
+	for i := 0; i < 3; i++ {
+		_, err := ap.Create(cluster.InstanceTypes["tiny"], img, nil, "", nil)
+		c.Check(err, check.NotNil)
+		c.Check(err, check.ErrorMatches, `.*InsufficientInstanceCapacity.*`)
+	}
+	c.Check(ap.client.(*ec2stub).runInstancesCalls, check.HasLen, 6)
+	metrics := arvadostest.GatherMetricsAsString(reg)
+	c.Check(metrics, check.Matches, `(?ms).*`+
+		`arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-broken",success="0"} 3\n`+
+		`arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-broken",success="1"} 0\n`+
+		`arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-full",success="0"} 3\n`+
+		`arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-full",success="1"} 0\n`+
+		`.*`)
+}
+
 func (*EC2InstanceSetSuite) TestTagInstances(c *check.C) {
 	ap, _, _, _ := GetInstanceSet(c, "{}")
 	l, err := ap.Instances(nil)
@@ -508,8 +596,23 @@ func (*EC2InstanceSetSuite) TestWrapError(c *check.C) {
 	_, ok := wrapped.(cloud.RateLimitError)
 	c.Check(ok, check.Equals, true)
 
-	quotaError := awserr.New("InsufficientInstanceCapacity", "", nil)
+	quotaError := awserr.New("InstanceLimitExceeded", "", nil)
 	wrapped = wrapError(quotaError, nil)
 	_, ok = wrapped.(cloud.QuotaError)
 	c.Check(ok, check.Equals, true)
+
+	for _, trial := range []struct {
+		code string
+		msg  string
+	}{
+		{"InsufficientInstanceCapacity", ""},
+		{"Unsupported", "Your requested instance type (t3.micro) is not supported in your requested Availability Zone (us-east-1e). Please retry your request by not specifying an Availability Zone or choosing us-east-1a, us-east-1b, us-east-1c, us-east-1d, us-east-1f."},
+	} {
+		capacityError := awserr.New(trial.code, trial.msg, nil)
+		wrapped = wrapError(capacityError, nil)
+		caperr, ok := wrapped.(cloud.CapacityError)
+		c.Check(ok, check.Equals, true)
+		c.Check(caperr.IsCapacityError(), check.Equals, true)
+		c.Check(caperr.IsInstanceTypeSpecific(), check.Equals, true)
+	}
 }