From 24f3823f6e96c60025cbde15c3cb94557f3d0bec Mon Sep 17 00:00:00 2001 From: Ward Vandewege Date: Sat, 15 Aug 2020 16:20:22 -0400 Subject: [PATCH] 16625: a-d-c: add support for Azure managed images. Update our packer file for Azure compute images to generate those instead of the old VHD images. Update documentation. Arvados-DCO-1.1-Signed-off-by: Ward Vandewege --- .../install-compute-node.html.textile.liquid | 5 +- go.mod | 14 +- go.sum | 40 ++++ lib/cloud/azure/azure.go | 210 +++++++++++++----- lib/cloud/azure/azure_test.go | 2 +- lib/config/config.default.yml | 10 +- lib/config/generated_config.go | 10 +- .../compute-images/arvados-images-azure.json | 8 +- tools/compute-images/build.sh | 11 +- 9 files changed, 229 insertions(+), 81 deletions(-) diff --git a/doc/install/crunch2-cloud/install-compute-node.html.textile.liquid b/doc/install/crunch2-cloud/install-compute-node.html.textile.liquid index 23da428b39..cdecc88152 100644 --- a/doc/install/crunch2-cloud/install-compute-node.html.textile.liquid +++ b/doc/install/crunch2-cloud/install-compute-node.html.textile.liquid @@ -92,8 +92,6 @@ Options: Azure secrets file which will be sourced from this script --azure-resource-group (default: false, required if building for Azure) Azure resource group - --azure-storage-account (default: false, required if building for Azure) - Azure storage account --azure-location (default: false, required if building for Azure) Azure location, e.g. centralus, eastus, westeurope --azure-sku (default: unset, required if building for Azure, e.g. 16.04-LTS) @@ -117,7 +115,6 @@ h2(#azure). Build an Azure image 
~$ ./build.sh --json-file arvados-images-azure.json \
            --arvados-cluster-id ClusterID \
            --azure-resource-group ResourceGroup \
-           --azure-storage-account StorageAccount \
            --azure-location AzureRegion \
            --azure-sku AzureSKU \
            --azure-secrets-file AzureSecretsFilePath \
@@ -126,7 +123,7 @@ h2(#azure). Build an Azure image
-For @ClusterID@, fill in your cluster ID. The @ResourceGroup@, @StorageAccount@ and @AzureRegion@ (e.g. 'eastus2') should be configured for where you want the compute image to be generated and stored. The @AzureSKU@ is the SKU of the base image to be used, e.g. '18.04-LTS' for Ubuntu 18.04. +For @ClusterID@, fill in your cluster ID. The @ResourceGroup@ and @AzureRegion@ (e.g. 'eastus2') should be configured for where you want the compute image to be generated and stored. The @AzureSKU@ is the SKU of the base image to be used, e.g. '18.04-LTS' for Ubuntu 18.04. @AzureSecretsFilePath@ should be replaced with the path to a shell script that loads the Azure secrets with sufficient permissions to create the image. The file would look like this: diff --git a/go.mod b/go.mod index 71052882ad..262978d912 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,12 @@ go 1.13 require ( github.com/AdRoll/goamz v0.0.0-20170825154802-2731d20f46f4 - github.com/Azure/azure-sdk-for-go v19.1.0+incompatible - github.com/Azure/go-autorest v10.15.2+incompatible + github.com/Azure/azure-sdk-for-go v45.1.0+incompatible + github.com/Azure/go-autorest v14.2.0+incompatible + github.com/Azure/go-autorest/autorest v0.11.3 + github.com/Azure/go-autorest/autorest/azure/auth v0.5.1 + github.com/Azure/go-autorest/autorest/to v0.4.0 + github.com/Azure/go-autorest/autorest/validation v0.3.0 // indirect github.com/Microsoft/go-winio v0.4.5 // indirect github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 // indirect github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect @@ -16,8 +20,6 @@ require ( github.com/bradleypeabody/godap v0.0.0-20170216002349-c249933bc092 github.com/coreos/go-oidc v2.1.0+incompatible github.com/coreos/go-systemd v0.0.0-20180108085132-cc4f39464dc7 - github.com/dgrijalva/jwt-go v3.1.0+incompatible // indirect - github.com/dimchansky/utfbom v1.0.0 // indirect github.com/dnaeon/go-vcr v1.0.1 // indirect github.com/docker/distribution v2.6.0-rc.1.0.20180105232752-277ed486c948+incompatible // indirect github.com/docker/docker v1.4.2-0.20180109013817-94b8a116fbf1 @@ -44,7 +46,6 @@ require ( github.com/kevinburke/ssh_config v0.0.0-20171013211458-802051befeb5 // indirect github.com/lib/pq v1.3.0 github.com/marstr/guid v1.1.1-0.20170427235115-8bdf7d1a087c // indirect - github.com/mitchellh/go-homedir v0.0.0-20161203194507-b8bc1bf76747 // indirect github.com/msteinert/pam v0.0.0-20190215180659-f29b9f28d6f9 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/opencontainers/image-spec v1.0.1-0.20171125024018-577479e4dc27 // indirect @@ -57,9 +58,8 @@ require ( github.com/sergi/go-diff v1.0.0 // indirect github.com/sirupsen/logrus v1.4.2 github.com/src-d/gcfg v1.3.0 // indirect - github.com/stretchr/testify v1.4.0 // indirect github.com/xanzy/ssh-agent v0.1.0 // indirect - golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 golang.org/x/net v0.0.0-20200202094626-16171245cfb2 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd diff --git a/go.sum b/go.sum index 2565964e7d..85d205112f 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,40 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +github.com/Azure/azure-sdk-for-go v0.2.0-beta h1:wYBqYNMWr0WL2lcEZi+dlK9n+N0wJ0Pjs4BKeOnDjfQ= github.com/Azure/azure-sdk-for-go v19.1.0+incompatible h1:ysqLW+tqZjJWOTE74heH/pDRbr4vlN3yV+dqQYgpyxw= github.com/Azure/azure-sdk-for-go v19.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v20.2.0+incompatible h1:La3ODnagAOf5ZFUepTfVftvNTdxkq06DNpgi1l0yaM0= +github.com/Azure/azure-sdk-for-go v20.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v45.1.0+incompatible h1:kxtaPD8n2z5Za+9e3sKsYG2IX6PG2R6VXtgS7gAbh3A= +github.com/Azure/azure-sdk-for-go v45.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-autorest v1.1.1 h1:4G9tVCqooRY3vDTB2bA1Z01PlSALtnUbji0AfzthUSs= github.com/Azure/go-autorest v10.15.2+incompatible h1:oZpnRzZie83xGV5txbT1aa/7zpCPvURGhV6ThJij2bs= github.com/Azure/go-autorest v10.15.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.0/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest v0.11.3 h1:fyYnmYujkIXUgv88D9/Wo2ybE4Zwd/TmQd5sSI5u2Ws= +github.com/Azure/go-autorest/autorest v0.11.3/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.2 h1:Aze/GQeAN1RRbGmnUJvUj+tFGBzFdIg3293/A9rbxC4= +github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.1 h1:bvUhZciHydpBxBmCheUgxxbSwJy7xcfjkUsjUcqSojc= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.1/go.mod h1:ea90/jvmnAwDrSooLH4sRIehEPtG/EPUXavDh31MnA4= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.0 h1:Ml+UCrnlKD+cJmSzrZ/RDcDw86NjkRUpnFh7V5JUhzU= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.0/go.mod h1:JljT387FplPzBA31vUcvsetLKF3pec5bdAxjVU4kI2s= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/autorest/validation v0.3.0 h1:3I9AAI63HfcLtphd9g39ruUwRI+Ca+z/f36KHPFRUss= +github.com/Azure/go-autorest/autorest/validation v0.3.0/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= +github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Microsoft/go-winio v0.4.5 h1:U2XsGR5dBg1yzwSEJoP2dE2/aAXpmad+CNG2hE9Pd5k= github.com/Microsoft/go-winio v0.4.5/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= @@ -48,8 +78,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.1.0+incompatible h1:FFziAwDQQ2dz1XClWMkwvukur3evtZx7x/wMHKM1i20= github.com/dgrijalva/jwt-go v3.1.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dimchansky/utfbom v1.0.0 h1:fGC2kkf4qOoKqZ4q7iIh+Vef4ubC1c38UDsEyZynZPc= github.com/dimchansky/utfbom v1.0.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= +github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/distribution v2.6.0-rc.1.0.20180105232752-277ed486c948+incompatible h1:PVtvnmmxSMUcT5AY6vG7sCCzRg3eyoW6vQvXtITC60c= @@ -78,6 +112,7 @@ github.com/go-ldap/ldap v3.0.3+incompatible h1:HTeSZO8hWMS1Rgb2Ziku6b8a7qRIZZMHj github.com/go-ldap/ldap v3.0.3+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= @@ -143,6 +178,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-homedir v0.0.0-20161203194507-b8bc1bf76747 h1:eQox4Rh4ewJF+mqYPxCkmBAirRnPaHEB26UkNuPyjlk= github.com/mitchellh/go-homedir v0.0.0-20161203194507-b8bc1bf76747/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -206,6 +243,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90Pveol golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -226,6 +265,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowK golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/lib/cloud/azure/azure.go b/lib/cloud/azure/azure.go index 6de367aa25..b448bddd70 100644 --- a/lib/cloud/azure/azure.go +++ b/lib/cloud/azure/azure.go @@ -18,7 +18,7 @@ import ( "git.arvados.org/arvados.git/lib/cloud" "git.arvados.org/arvados.git/sdk/go/arvados" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute" + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network" storageacct "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-02-01/storage" "github.com/Azure/azure-sdk-for-go/storage" @@ -41,6 +41,7 @@ type azureInstanceSetConfig struct { TenantID string CloudEnvironment string ResourceGroup string + ImageResourceGroup string Location string Network string NetworkResourceGroup string @@ -138,6 +139,25 @@ func (cl *interfacesClientImpl) listComplete(ctx context.Context, resourceGroupN return r, wrapAzureError(err) } +type disksClientWrapper interface { + listByResourceGroup(ctx context.Context, resourceGroupName string) (result compute.DiskListPage, err error) + delete(ctx context.Context, resourceGroupName string, diskName string) (result compute.DisksDeleteFuture, err error) +} + +type disksClientImpl struct { + inner compute.DisksClient +} + +func (cl *disksClientImpl) listByResourceGroup(ctx context.Context, resourceGroupName string) (result compute.DiskListPage, err error) { + r, err := cl.inner.ListByResourceGroup(ctx, resourceGroupName) + return r, wrapAzureError(err) +} + +func (cl *disksClientImpl) delete(ctx context.Context, resourceGroupName string, diskName string) (result compute.DisksDeleteFuture, err error) { + r, err := cl.inner.Delete(ctx, resourceGroupName, diskName) + return r, wrapAzureError(err) +} + var quotaRe = regexp.MustCompile(`(?i:exceed|quota|limit)`) type azureRateLimitError struct { @@ -196,20 +216,23 @@ func wrapAzureError(err error) error { } type azureInstanceSet struct { - azconfig azureInstanceSetConfig - vmClient virtualMachinesClientWrapper - netClient interfacesClientWrapper - blobcont containerWrapper - azureEnv azure.Environment - interfaces map[string]network.Interface - dispatcherID string - namePrefix string - ctx context.Context - stopFunc context.CancelFunc - stopWg sync.WaitGroup - deleteNIC chan string - deleteBlob chan storage.Blob - logger logrus.FieldLogger + azconfig azureInstanceSetConfig + vmClient virtualMachinesClientWrapper + netClient interfacesClientWrapper + disksClient disksClientWrapper + imageResourceGroup string + blobcont containerWrapper + azureEnv azure.Environment + interfaces map[string]network.Interface + dispatcherID string + namePrefix string + ctx context.Context + stopFunc context.CancelFunc + stopWg sync.WaitGroup + deleteNIC chan string + deleteBlob chan storage.Blob + deleteDisk chan compute.Disk + logger logrus.FieldLogger } func newAzureInstanceSet(config json.RawMessage, dispatcherID cloud.InstanceSetID, _ cloud.SharedResourceTags, logger logrus.FieldLogger) (prv cloud.InstanceSet, err error) { @@ -233,6 +256,7 @@ func (az *azureInstanceSet) setup(azcfg azureInstanceSetConfig, dispatcherID str az.azconfig = azcfg vmClient := compute.NewVirtualMachinesClient(az.azconfig.SubscriptionID) netClient := network.NewInterfacesClient(az.azconfig.SubscriptionID) + disksClient := compute.NewDisksClient(az.azconfig.SubscriptionID) storageAcctClient := storageacct.NewAccountsClient(az.azconfig.SubscriptionID) az.azureEnv, err = azure.EnvironmentFromName(az.azconfig.CloudEnvironment) @@ -253,26 +277,36 @@ func (az *azureInstanceSet) setup(azcfg azureInstanceSetConfig, dispatcherID str vmClient.Authorizer = authorizer netClient.Authorizer = authorizer + disksClient.Authorizer = authorizer storageAcctClient.Authorizer = authorizer az.vmClient = &virtualMachinesClientImpl{vmClient} az.netClient = &interfacesClientImpl{netClient} + az.disksClient = &disksClientImpl{disksClient} - result, err := storageAcctClient.ListKeys(az.ctx, az.azconfig.ResourceGroup, az.azconfig.StorageAccount) - if err != nil { - az.logger.WithError(err).Warn("Couldn't get account keys") - return err + az.imageResourceGroup = az.azconfig.ImageResourceGroup + if az.imageResourceGroup == "" { + az.imageResourceGroup = az.azconfig.ResourceGroup } - key1 := *(*result.Keys)[0].Value - client, err := storage.NewBasicClientOnSovereignCloud(az.azconfig.StorageAccount, key1, az.azureEnv) - if err != nil { - az.logger.WithError(err).Warn("Couldn't make client") - return err - } + var client storage.Client + if az.azconfig.StorageAccount != "" { + result, err := storageAcctClient.ListKeys(az.ctx, az.azconfig.ResourceGroup, az.azconfig.StorageAccount) + if err != nil { + az.logger.WithError(err).Warn("Couldn't get account keys") + return err + } - blobsvc := client.GetBlobService() - az.blobcont = blobsvc.GetContainerReference(az.azconfig.BlobContainer) + key1 := *(*result.Keys)[0].Value + client, err = storage.NewBasicClientOnSovereignCloud(az.azconfig.StorageAccount, key1, az.azureEnv) + if err != nil { + az.logger.WithError(err).Warn("Couldn't make client") + return err + } + + blobsvc := client.GetBlobService() + az.blobcont = blobsvc.GetContainerReference(az.azconfig.BlobContainer) + } az.dispatcherID = dispatcherID az.namePrefix = fmt.Sprintf("compute-%s-", az.dispatcherID) @@ -288,13 +322,17 @@ func (az *azureInstanceSet) setup(azcfg azureInstanceSetConfig, dispatcherID str tk.Stop() return case <-tk.C: - az.manageBlobs() + if az.blobcont != nil { + az.manageBlobs() + } + az.manageDisks() } } }() az.deleteNIC = make(chan string) az.deleteBlob = make(chan storage.Blob) + az.deleteDisk = make(chan compute.Disk) for i := 0; i < 4; i++ { go func() { @@ -325,6 +363,20 @@ func (az *azureInstanceSet) setup(azcfg azureInstanceSetConfig, dispatcherID str } } }() + go func() { + for { + disk, ok := <-az.deleteDisk + if !ok { + return + } + _, err := az.disksClient.delete(az.ctx, az.imageResourceGroup, *disk.Name) + if err != nil { + az.logger.WithError(err).Warnf("Error deleting disk %+v", *disk.Name) + } else { + az.logger.Printf("Deleted disk %v", *disk.Name) + } + } + }() } return nil @@ -390,13 +442,44 @@ func (az *azureInstanceSet) Create( } blobname := fmt.Sprintf("%s-os.vhd", name) - instanceVhd := fmt.Sprintf("https://%s.blob.%s/%s/%s", - az.azconfig.StorageAccount, - az.azureEnv.StorageEndpointSuffix, - az.azconfig.BlobContainer, - blobname) - customData := base64.StdEncoding.EncodeToString([]byte("#!/bin/sh\n" + initCommand + "\n")) + var storageProfile *compute.StorageProfile + + re := regexp.MustCompile(`^http(s?)://`) + if re.MatchString(string(imageID)) { + instanceVhd := fmt.Sprintf("https://%s.blob.%s/%s/%s", + az.azconfig.StorageAccount, + az.azureEnv.StorageEndpointSuffix, + az.azconfig.BlobContainer, + blobname) + az.logger.Info("using deprecated VHD image") + storageProfile = &compute.StorageProfile{ + OsDisk: &compute.OSDisk{ + OsType: compute.Linux, + Name: to.StringPtr(name + "-os"), + CreateOption: compute.DiskCreateOptionTypesFromImage, + Image: &compute.VirtualHardDisk{ + URI: to.StringPtr(string(imageID)), + }, + Vhd: &compute.VirtualHardDisk{ + URI: &instanceVhd, + }, + }, + } + } else { + az.logger.Info("using managed image") + + storageProfile = &compute.StorageProfile{ + ImageReference: &compute.ImageReference{ + ID: to.StringPtr("/subscriptions/" + az.azconfig.SubscriptionID + "/resourceGroups/" + az.imageResourceGroup + "/providers/Microsoft.Compute/images/" + string(imageID)), + }, + OsDisk: &compute.OSDisk{ + OsType: compute.Linux, + Name: to.StringPtr(name + "-os"), + CreateOption: compute.DiskCreateOptionTypesFromImage, + }, + } + } vmParameters := compute.VirtualMachine{ Location: &az.azconfig.Location, @@ -405,19 +488,7 @@ func (az *azureInstanceSet) Create( HardwareProfile: &compute.HardwareProfile{ VMSize: compute.VirtualMachineSizeTypes(instanceType.ProviderType), }, - StorageProfile: &compute.StorageProfile{ - OsDisk: &compute.OSDisk{ - OsType: compute.Linux, - Name: to.StringPtr(name + "-os"), - CreateOption: compute.FromImage, - Image: &compute.VirtualHardDisk{ - URI: to.StringPtr(string(imageID)), - }, - Vhd: &compute.VirtualHardDisk{ - URI: &instanceVhd, - }, - }, - }, + StorageProfile: storageProfile, NetworkProfile: &compute.NetworkProfile{ NetworkInterfaces: &[]compute.NetworkInterfaceReference{ compute.NetworkInterfaceReference{ @@ -449,12 +520,14 @@ func (az *azureInstanceSet) Create( vm, err := az.vmClient.createOrUpdate(az.ctx, az.azconfig.ResourceGroup, name, vmParameters) if err != nil { - _, delerr := az.blobcont.GetBlobReference(blobname).DeleteIfExists(nil) - if delerr != nil { - az.logger.WithError(delerr).Warnf("Error cleaning up vhd blob after failed create") + if az.blobcont != nil { + _, delerr := az.blobcont.GetBlobReference(blobname).DeleteIfExists(nil) + if delerr != nil { + az.logger.WithError(delerr).Warnf("Error cleaning up vhd blob after failed create") + } } - _, delerr = az.netClient.delete(context.Background(), az.azconfig.ResourceGroup, *nic.Name) + _, delerr := az.netClient.delete(context.Background(), az.azconfig.ResourceGroup, *nic.Name) if delerr != nil { az.logger.WithError(delerr).Warnf("Error cleaning up NIC after failed create") } @@ -573,6 +646,41 @@ func (az *azureInstanceSet) manageBlobs() { } } +// ManageDisks garbage collects managed compute disks (VM disk images) in the +// configured resource group. It will delete disks which +// have "namePrefix", are "available" (which means they are not +// leased to a VM) and were created more than DeleteDanglingResourcesAfter seconds ago. +// (Azure provides no modification timestamp on managed disks, there is only a +// creation timestamp) +func (az *azureInstanceSet) manageDisks() { + + re := regexp.MustCompile(`^` + az.namePrefix + `.*-os$`) + timestamp := time.Now() + + for { + response, err := az.disksClient.listByResourceGroup(az.ctx, az.imageResourceGroup) + if err != nil { + az.logger.WithError(err).Warn("Error listing disks") + return + } + for _, d := range response.Values() { + age := timestamp.Sub(d.DiskProperties.TimeCreated.ToTime()) + if d.DiskProperties.DiskState == "Unattached" && + re.MatchString(*d.Name) && + age.Seconds() > az.azconfig.DeleteDanglingResourcesAfter.Duration().Seconds() { + + az.logger.Printf("Disk %v is unlocked and not modified for %v seconds, will delete", *d.Name, age.Seconds()) + az.deleteDisk <- d + } + } + if response.Values() != nil { + response.Next() + } else { + break + } + } +} + func (az *azureInstanceSet) Stop() { az.stopFunc() az.stopWg.Wait() diff --git a/lib/cloud/azure/azure_test.go b/lib/cloud/azure/azure_test.go index 94af0b9a26..186966e19c 100644 --- a/lib/cloud/azure/azure_test.go +++ b/lib/cloud/azure/azure_test.go @@ -47,7 +47,7 @@ import ( "git.arvados.org/arvados.git/lib/dispatchcloud/test" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/config" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute" + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network" "github.com/Azure/azure-sdk-for-go/storage" "github.com/Azure/go-autorest/autorest" diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml index a2a34448f1..d5369d3ce6 100644 --- a/lib/config/config.default.yml +++ b/lib/config/config.default.yml @@ -952,6 +952,10 @@ Clusters: TimeoutShutdown: 10s # Worker VM image ID. + # (aws) AMI identifier + # (azure) managed disks: the name of the managed disk image + # (azure) unmanaged disks: the complete URI of the VHD, e.g. + # https://xxxxx.blob.core.windows.net/system/Microsoft.Compute/Images/images/xxxxx.vhd ImageID: "" # An executable file (located on the dispatcher host) to be @@ -1020,7 +1024,11 @@ Clusters: Network: "" Subnet: "" - # (azure) Where to store the VM VHD blobs + # (azure) managed disks: The resource group where the managed disk + # image can be found (if different from ResourceGroup). + ImageResourceGroup: "" + + # (azure) unmanaged disks: Where to store the VM VHD blobs StorageAccount: "" BlobContainer: "" diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go index bddb5cedb1..a14f18aaec 100644 --- a/lib/config/generated_config.go +++ b/lib/config/generated_config.go @@ -958,6 +958,10 @@ Clusters: TimeoutShutdown: 10s # Worker VM image ID. + # (aws) AMI identifier + # (azure) managed disks: the name of the managed disk image + # (azure) unmanaged disks: the complete URI of the VHD, e.g. + # https://xxxxx.blob.core.windows.net/system/Microsoft.Compute/Images/images/xxxxx.vhd ImageID: "" # An executable file (located on the dispatcher host) to be @@ -1026,7 +1030,11 @@ Clusters: Network: "" Subnet: "" - # (azure) Where to store the VM VHD blobs + # (azure) managed disks: The resource group where the managed disk + # image can be found (if different from ResourceGroup). + ImageResourceGroup: "" + + # (azure) unmanaged disks: Where to store the VM VHD blobs StorageAccount: "" BlobContainer: "" diff --git a/tools/compute-images/arvados-images-azure.json b/tools/compute-images/arvados-images-azure.json index f7fc1a07b4..c8db9499cd 100644 --- a/tools/compute-images/arvados-images-azure.json +++ b/tools/compute-images/arvados-images-azure.json @@ -1,6 +1,5 @@ { "variables": { - "storage_account": null, "resource_group": null, "client_id": "{{env `ARM_CLIENT_ID`}}", "client_secret": "{{env `ARM_CLIENT_SECRET`}}", @@ -30,11 +29,8 @@ "subscription_id": "{{user `subscription_id`}}", "tenant_id": "{{user `tenant_id`}}", - "resource_group_name": "{{user `resource_group`}}", - "storage_account": "{{user `storage_account`}}", - - "capture_container_name": "images", - "capture_name_prefix": "{{user `arvados_cluster`}}-compute", + "managed_image_resource_group_name": "{{user `resource_group`}}", + "managed_image_name": "{{user `arvados_cluster`}}-compute-v{{ timestamp }}", "ssh_username": "{{user `ssh_user`}}", "ssh_private_key_file": "{{user `ssh_private_key_file`}}", diff --git a/tools/compute-images/build.sh b/tools/compute-images/build.sh index e8265ae198..030eb410b8 100755 --- a/tools/compute-images/build.sh +++ b/tools/compute-images/build.sh @@ -43,8 +43,6 @@ Options: Azure secrets file which will be sourced from this script --azure-resource-group (default: false, required if building for Azure) Azure resource group - --azure-storage-account (default: false, required if building for Azure) - Azure storage account --azure-location (default: false, required if building for Azure) Azure location, e.g. centralus, eastus, westeurope --azure-sku (default: unset, required if building for Azure, e.g. 16.04-LTS) @@ -76,7 +74,6 @@ GCP_ACCOUNT_FILE= GCP_ZONE= AZURE_SECRETS_FILE= AZURE_RESOURCE_GROUP= -AZURE_STORAGE_ACCOUNT= AZURE_LOCATION= AZURE_CLOUD_ENVIRONMENT= DEBUG= @@ -86,7 +83,7 @@ AWS_DEFAULT_REGION=us-east-1 PUBLIC_KEY_FILE= PARSEDOPTS=$(getopt --name "$0" --longoptions \ - help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-storage-account:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,domain:,resolver:,reposuffix:,public-key-file:,debug \ + help,json-file:,arvados-cluster-id:,aws-source-ami:,aws-profile:,aws-secrets-file:,aws-region:,aws-vpc-id:,aws-subnet-id:,gcp-project-id:,gcp-account-file:,gcp-zone:,azure-secrets-file:,azure-resource-group:,azure-location:,azure-sku:,azure-cloud-environment:,ssh_user:,domain:,resolver:,reposuffix:,public-key-file:,debug \ -- "" "$@") if [ $? -ne 0 ]; then exit 1 @@ -139,9 +136,6 @@ while [ $# -gt 0 ]; do --azure-resource-group) AZURE_RESOURCE_GROUP="$2"; shift ;; - --azure-storage-account) - AZURE_STORAGE_ACCOUNT="$2"; shift - ;; --azure-location) AZURE_LOCATION="$2"; shift ;; @@ -248,9 +242,6 @@ fi if [[ "$AZURE_RESOURCE_GROUP" != "" ]]; then EXTRA2+=" -var resource_group=$AZURE_RESOURCE_GROUP" fi -if [[ "$AZURE_STORAGE_ACCOUNT" != "" ]]; then - EXTRA2+=" -var storage_account=$AZURE_STORAGE_ACCOUNT" -fi if [[ "$AZURE_LOCATION" != "" ]]; then EXTRA2+=" -var location=$AZURE_LOCATION" fi -- 2.30.2