21603: Recognize subnet error despite generic error code.

[arvados.git] / lib / cloud / ec2 / ec2_test.go

diff --git a/lib/cloud/ec2/ec2_test.go b/lib/cloud/ec2/ec2_test.go
index 6fde4bbbcafb9ba8e43f0ab55cbe9d141ecc2c79..5e6cf2c82b5caee81de1255936ee30a2edeb1024 100644 (file)
--- a/lib/cloud/ec2/ec2_test.go
+++ b/lib/cloud/ec2/ec2_test.go
@@ -277,6 +277,12 @@ func (*EC2InstanceSetSuite) TestCreate(c *check.C) {
        if *live == "" {
                c.Check(ap.client.(*ec2stub).describeKeyPairsCalls, check.HasLen, 1)
                c.Check(ap.client.(*ec2stub).importKeyPairCalls, check.HasLen, 1)
+
+               runcalls := ap.client.(*ec2stub).runInstancesCalls
+               if c.Check(runcalls, check.HasLen, 1) {
+                       c.Check(runcalls[0].MetadataOptions.HttpEndpoint, check.DeepEquals, aws.String("enabled"))
+                       c.Check(runcalls[0].MetadataOptions.HttpTokens, check.DeepEquals, aws.String("required"))
+               }
        }
 }
 
@@ -357,6 +363,57 @@ func (*EC2InstanceSetSuite) TestCreateFailoverSecondSubnet(c *check.C) {
                `.*`)
 }
 
+func (*EC2InstanceSetSuite) TestIsErrorSubnetSpecific(c *check.C) {
+       c.Check(isErrorSubnetSpecific(nil), check.Equals, false)
+       c.Check(isErrorSubnetSpecific(errors.New("misc error")), check.Equals, false)
+
+       c.Check(isErrorSubnetSpecific(&ec2stubError{
+               code: "InsufficientInstanceCapacity",
+       }), check.Equals, true)
+
+       c.Check(isErrorSubnetSpecific(&ec2stubError{
+               code: "InsufficientVolumeCapacity",
+       }), check.Equals, true)
+
+       c.Check(isErrorSubnetSpecific(&ec2stubError{
+               code:    "InsufficientFreeAddressesInSubnet",
+               message: "Not enough free addresses in subnet subnet-abcdefg\n\tstatus code: 400, request id: abcdef01-2345-6789-abcd-ef0123456789",
+       }), check.Equals, true)
+
+       // #21603: (Sometimes?) EC2 returns code InvalidParameterValue
+       // even though the code "InsufficientFreeAddressesInSubnet"
+       // seems like it must be meant for exactly this error.
+       c.Check(isErrorSubnetSpecific(&ec2stubError{
+               code:    "InvalidParameterValue",
+               message: "Not enough free addresses in subnet subnet-abcdefg\n\tstatus code: 400, request id: abcdef01-2345-6789-abcd-ef0123456789",
+       }), check.Equals, true)
+
+       // Similarly, AWS docs
+       // (https://repost.aws/knowledge-center/vpc-insufficient-ip-errors)
+       // suggest the following code/message combinations also exist.
+       c.Check(isErrorSubnetSpecific(&ec2stubError{
+               code:    "Client.InvalidParameterValue",
+               message: "There aren't sufficient free Ipv4 addresses or prefixes",
+       }), check.Equals, true)
+       c.Check(isErrorSubnetSpecific(&ec2stubError{
+               code:    "InvalidParameterValue",
+               message: "There aren't sufficient free Ipv4 addresses or prefixes",
+       }), check.Equals, true)
+       // Meanwhile, other AWS docs
+       // (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html)
+       // suggest Client.InvalidParameterValue is not a real code but
+       // ClientInvalidParameterValue is.
+       c.Check(isErrorSubnetSpecific(&ec2stubError{
+               code:    "ClientInvalidParameterValue",
+               message: "There aren't sufficient free Ipv4 addresses or prefixes",
+       }), check.Equals, true)
+
+       c.Check(isErrorSubnetSpecific(&ec2stubError{
+               code:    "InvalidParameterValue",
+               message: "Some other invalid parameter error",
+       }), check.Equals, false)
+}
+
 func (*EC2InstanceSetSuite) TestCreateAllSubnetsFailing(c *check.C) {
        if *live != "" {
                c.Skip("not applicable in live mode")
@@ -399,6 +456,37 @@ func (*EC2InstanceSetSuite) TestCreateAllSubnetsFailing(c *check.C) {
                `.*`)
 }
 
+func (*EC2InstanceSetSuite) TestCreateOneSubnetFailingCapacity(c *check.C) {
+       if *live != "" {
+               c.Skip("not applicable in live mode")
+               return
+       }
+       ap, img, cluster, reg := GetInstanceSet(c, `{"SubnetID":["subnet-full","subnet-broken"]}`)
+       ap.client.(*ec2stub).subnetErrorOnRunInstances = map[string]error{
+               "subnet-full": &ec2stubError{
+                       code:    "InsufficientFreeAddressesInSubnet",
+                       message: "subnet is full",
+               },
+               "subnet-broken": &ec2stubError{
+                       code:    "InsufficientInstanceCapacity",
+                       message: "insufficient capacity",
+               },
+       }
+       for i := 0; i < 3; i++ {
+               _, err := ap.Create(cluster.InstanceTypes["tiny"], img, nil, "", nil)
+               c.Check(err, check.NotNil)
+               c.Check(err, check.ErrorMatches, `.*InsufficientInstanceCapacity.*`)
+       }
+       c.Check(ap.client.(*ec2stub).runInstancesCalls, check.HasLen, 6)
+       metrics := arvadostest.GatherMetricsAsString(reg)
+       c.Check(metrics, check.Matches, `(?ms).*`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-broken",success="0"} 3\n`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-broken",success="1"} 0\n`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-full",success="0"} 3\n`+
+               `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-full",success="1"} 0\n`+
+               `.*`)
+}
+
 func (*EC2InstanceSetSuite) TestTagInstances(c *check.C) {
        ap, _, _, _ := GetInstanceSet(c, "{}")
        l, err := ap.Instances(nil)
@@ -508,8 +596,23 @@ func (*EC2InstanceSetSuite) TestWrapError(c *check.C) {
        _, ok := wrapped.(cloud.RateLimitError)
        c.Check(ok, check.Equals, true)
 
-       quotaError := awserr.New("InsufficientInstanceCapacity", "", nil)
+       quotaError := awserr.New("InstanceLimitExceeded", "", nil)
        wrapped = wrapError(quotaError, nil)
        _, ok = wrapped.(cloud.QuotaError)
        c.Check(ok, check.Equals, true)
+
+       for _, trial := range []struct {
+               code string
+               msg  string
+       }{
+               {"InsufficientInstanceCapacity", ""},
+               {"Unsupported", "Your requested instance type (t3.micro) is not supported in your requested Availability Zone (us-east-1e). Please retry your request by not specifying an Availability Zone or choosing us-east-1a, us-east-1b, us-east-1c, us-east-1d, us-east-1f."},
+       } {
+               capacityError := awserr.New(trial.code, trial.msg, nil)
+               wrapped = wrapError(capacityError, nil)
+               caperr, ok := wrapped.(cloud.CapacityError)
+               c.Check(ok, check.Equals, true)
+               c.Check(caperr.IsCapacityError(), check.Equals, true)
+               c.Check(caperr.IsInstanceTypeSpecific(), check.Equals, true)
+       }
 }