From a2ce965f61f0650ca7eef4c41db5b0b3016f8861 Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Mon, 13 May 2024 17:27:20 -0400 Subject: [PATCH] 21705: Migrate keepstore s3 driver to latest aws-sdk-go-v2. Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- go.mod | 26 ++- go.sum | 41 ++++ services/keepstore/s3_volume.go | 225 ++++++++++++---------- services/keepstore/s3_volume_test.go | 182 +++++++++-------- services/keepstore/volume_generic_test.go | 2 +- 5 files changed, 284 insertions(+), 192 deletions(-) diff --git a/go.mod b/go.mod index bcbcb55abd..29b4a3a50d 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,13 @@ require ( github.com/Azure/go-autorest/autorest/azure/auth v0.5.9 github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/arvados/cgofuse v1.2.0-arvados1 - github.com/aws/aws-sdk-go v1.44.174 + github.com/aws/aws-sdk-go v1.44.256 github.com/aws/aws-sdk-go-v2 v1.26.1 + github.com/aws/aws-sdk-go-v2/config v1.27.12 + github.com/aws/aws-sdk-go-v2/credentials v1.17.12 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.16 + github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2 + github.com/aws/smithy-go v1.20.2 github.com/bmatcuk/doublestar/v4 v4.6.1 github.com/bradleypeabody/godap v0.0.0-20170216002349-c249933bc092 github.com/coreos/go-oidc/v3 v3.5.0 @@ -30,7 +35,7 @@ require ( github.com/imdario/mergo v0.3.12 github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff github.com/jmoiron/sqlx v1.2.0 - github.com/johannesboyne/gofakes3 v0.0.0-20200716060623-6b2b4cb092cc + github.com/johannesboyne/gofakes3 v0.0.0-20240513200200-99de01ee122d github.com/julienschmidt/httprouter v1.3.0 github.com/lib/pq v1.10.2 github.com/msteinert/pam v0.0.0-20190215180659-f29b9f28d6f9 @@ -64,19 +69,20 @@ require ( github.com/Microsoft/go-winio v0.5.2 // indirect github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 // indirect github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect - github.com/aws/aws-sdk-go-v2/config v1.27.11 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.11 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect - github.com/aws/smithy-go v1.20.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.28.7 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/speakeasy v0.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -114,13 +120,13 @@ require ( github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/satori/go.uuid v1.2.1-0.20180404165556-75cca531ea76 // indirect github.com/sergi/go-diff v1.0.0 // indirect - github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 // indirect + github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect github.com/src-d/gcfg v1.3.0 // indirect github.com/xanzy/ssh-agent v0.1.0 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect - golang.org/x/tools v0.6.0 // indirect + golang.org/x/tools v0.8.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/grpc v1.59.0 // indirect diff --git a/go.sum b/go.sum index 16e5a9cbab..46783b3324 100644 --- a/go.sum +++ b/go.sum @@ -48,39 +48,65 @@ github.com/arvados/cgofuse v1.2.0-arvados1 h1:4Q4vRJ4hbTCcI4gGEaa6hqwj3rqlUuzeFQ github.com/arvados/cgofuse v1.2.0-arvados1/go.mod h1:79WFV98hrkRHK9XPhh2IGGOwpFSjocsWubgxAs2KhRc= github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef h1:cl7DIRbiAYNqaVxg3CZY8qfZoBOKrj06H/x9SPGaxas= github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef/go.mod h1:rCtgyMmBGEbjTm37fCuBYbNL0IhztiALzo3OB9HyiOM= +github.com/arvados/gofakes3 v0.0.0-20240513190428-7dd242f94aa2 h1:q9L4OHOFwd3zNFlEADucYTBfKasWpz4NwjaGjJlYhqs= +github.com/arvados/gofakes3 v0.0.0-20240513190428-7dd242f94aa2/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g= github.com/arvados/yaml v0.0.0-20210427145106-92a1cab0904b h1:hK0t0aJTTXI64lpXln2A1SripqOym+GVNTnwsLes39Y= github.com/arvados/yaml v0.0.0-20210427145106-92a1cab0904b/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.44.174 h1:9lR4a6MKQW/t6YCG0ZKAt1GAkjdEPP8sWch/pfcuR0c= github.com/aws/aws-sdk-go v1.44.174/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVWY4= +github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v0.23.0 h1:+E1q1LLSfHSDn/DzOtdJOX+pLZE2HiNV2yO5AjZINwM= github.com/aws/aws-sdk-go-v2 v0.23.0/go.mod h1:2LhT7UgHOXK3UXONKI5OMgIyoQL6zTAw/jwIeX6yqzw= github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg= github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA= github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE= +github.com/aws/aws-sdk-go-v2/config v1.27.12 h1:vq88mBaZI4NGLXk8ierArwSILmYHDJZGJOeAc/pzEVQ= +github.com/aws/aws-sdk-go-v2/config v1.27.12/go.mod h1:IOrsf4IiN68+CgzyuyGUYTpCrtUQTbbMEAtR/MR/4ZU= github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs= github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo= +github.com/aws/aws-sdk-go-v2/credentials v1.17.12 h1:PVbKQ0KjDosI5+nEdRMU8ygEQDmkJTSHBqPjEX30lqc= +github.com/aws/aws-sdk-go-v2/credentials v1.17.12/go.mod h1:jlWtGFRtKsqc5zqerHZYmKmRkUXo3KPM14YJ13ZEjwE= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.16 h1:n4k5rgvy0M748NadpDlGLOQ/KCVbNHCQsfI895wLrt0= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.16/go.mod h1:Rd5rkCdq44ZC5rS4CbF3Wc8FiWo7f9brp7qeIwWsJaU= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0= github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0 h1:ooy0OFbrdSwgk32OFGPnvBwry5ySYCKkgTEbQ2hejs8= github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0/go.mod h1:xejKuuRDjz6z5OqyeLsz01MlOqqW7CqpAB4PabNvpu8= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2 h1:rq2hglTQM3yHZvOPVMtNvLS5x6hijx7JvRDgKiTNDGQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o= github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w= github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.6 h1:o5cTaeunSpfXiLTIBx5xo2enQmiChtu1IBbzXnfU9Hs= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.6/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.5 h1:Ciiz/plN+Z+pPO1G0W2zJoYIIl0KtKzY0LJ78NXYTws= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.5/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak= github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU= github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.7 h1:et3Ta53gotFR4ERLXXHIHl/Uuk1qYpP5uU7cvNql8ns= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.7/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw= github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -229,6 +255,10 @@ github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/johannesboyne/gofakes3 v0.0.0-20200716060623-6b2b4cb092cc h1:JJPhSHowepOF2+ElJVyb9jgt5ZyBkPMkPuhS0uODSFs= github.com/johannesboyne/gofakes3 v0.0.0-20200716060623-6b2b4cb092cc/go.mod h1:fNiSoOiEI5KlkWXn26OwKnNe58ilTIkpBlgOrt7Olu8= +github.com/johannesboyne/gofakes3 v0.0.0-20240501064628-9cf5179a8df0 h1:+XMEfFS2W506TvrEY5ViKNEIqmX6mxLmBmx98/dBNAg= +github.com/johannesboyne/gofakes3 v0.0.0-20240501064628-9cf5179a8df0/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g= +github.com/johannesboyne/gofakes3 v0.0.0-20240513200200-99de01ee122d h1:9dIJ/sx3yapvuq3kvTSVQ6UVS2HxfOB4MCwWiH8JcvQ= +github.com/johannesboyne/gofakes3 v0.0.0-20240513200200-99de01ee122d/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kevinburke/ssh_config v0.0.0-20171013211458-802051befeb5 h1:xXn0nBttYwok7DhU4RxqaADEpQn7fEMt5kKc3yoj/n0= @@ -286,6 +316,8 @@ github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 h1:J6qvD6rbmOil46orKqJaRPG+zTpoGlBTUdyv8ki63L0= github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63/go.mod h1:n+VKSARF5y/tS9XFSP7vWDfS+GUC5vs/YT7M5XDTUEM= +github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI= +github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -311,6 +343,7 @@ github.com/xanzy/ssh-agent v0.1.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnW github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -332,6 +365,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -353,6 +387,7 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= @@ -376,6 +411,7 @@ golang.org/x/sys v0.0.0-20190310054646-10058d7d4faa/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -388,6 +424,7 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= @@ -399,6 +436,7 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= @@ -423,12 +461,15 @@ golang.org/x/tools v0.0.0-20190308174544-00c44ba9c14f/go.mod h1:25r3+/G6/xytQM8i golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go index 2e2e97a974..49b94a72a1 100644 --- a/services/keepstore/s3_volume.go +++ b/services/keepstore/s3_volume.go @@ -13,6 +13,7 @@ import ( "errors" "fmt" "io" + "net/url" "os" "regexp" "strings" @@ -22,13 +23,13 @@ import ( "git.arvados.org/arvados.git/sdk/go/arvados" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/aws/awserr" - "github.com/aws/aws-sdk-go-v2/aws/defaults" - "github.com/aws/aws-sdk-go-v2/aws/ec2metadata" - "github.com/aws/aws-sdk-go-v2/aws/ec2rolecreds" - "github.com/aws/aws-sdk-go-v2/aws/endpoints" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/s3manager" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" ) @@ -49,9 +50,14 @@ const ( ) var ( - errS3TrashDisabled = fmt.Errorf("trash function is disabled because Collections.BlobTrashLifetime=0 and DriverParameters.UnsafeDelete=false") - s3AWSKeepBlockRegexp = regexp.MustCompile(`^[0-9a-f]{32}$`) - s3AWSZeroTime time.Time + errS3TrashDisabled = fmt.Errorf("trash function is disabled because Collections.BlobTrashLifetime=0 and DriverParameters.UnsafeDelete=false") + s3AWSKeepBlockRegexp = regexp.MustCompile(`^[0-9a-f]{32}$`) + s3AWSZeroTime time.Time + defaultEndpointResolverV2 = s3.NewDefaultEndpointResolverV2() + + // Returned by an aws.EndpointResolverWithOptions to indicate + // that the default resolver should be used. + errEndpointNotOverridden = &aws.EndpointNotFoundError{Err: errors.New("endpoint not overridden")} ) // s3Volume implements Volume using an S3 bucket. @@ -68,6 +74,9 @@ type s3Volume struct { bucket *s3Bucket region string startOnce sync.Once + + overrideEndpoint *aws.Endpoint + usePathStyle bool // used by test suite } // s3bucket wraps s3.bucket and counts I/O and API usage stats. The @@ -114,12 +123,15 @@ func news3Volume(params newVolumeParams) (volume, error) { } func (v *s3Volume) translateError(err error) error { - if _, ok := err.(*aws.RequestCanceledError); ok { + if cerr := (interface{ CanceledError() bool })(nil); errors.As(err, &cerr) && cerr.CanceledError() { + // *aws.RequestCanceledError and *smithy.CanceledError + // implement this interface. return context.Canceled - } else if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() == "NotFound" { - return os.ErrNotExist - } else if aerr.Code() == "NoSuchKey" { + } + var aerr smithy.APIError + if errors.As(err, &aerr) { + switch aerr.ErrorCode() { + case "NotFound", "NoSuchKey": return os.ErrNotExist } } @@ -140,20 +152,17 @@ func (v *s3Volume) safeCopy(dst, src string) error { Key: aws.String(dst), } - req := v.bucket.svc.CopyObjectRequest(input) - resp, err := req.Send(context.Background()) + resp, err := v.bucket.svc.CopyObject(context.Background(), input) err = v.translateError(err) if os.IsNotExist(err) { return err } else if err != nil { return fmt.Errorf("PutCopy(%q ← %q): %s", dst, v.bucket.bucket+"/"+src, err) - } - - if resp.CopyObjectResult.LastModified == nil { - return fmt.Errorf("PutCopy succeeded but did not return a timestamp: %q: %s", resp.CopyObjectResult.LastModified, err) - } else if time.Now().Sub(*resp.CopyObjectResult.LastModified) > maxClockSkew { - return fmt.Errorf("PutCopy succeeded but returned an old timestamp: %q: %s", resp.CopyObjectResult.LastModified, resp.CopyObjectResult.LastModified) + } else if resp.CopyObjectResult.LastModified == nil { + return fmt.Errorf("PutCopy(%q ← %q): succeeded but did not return a timestamp", dst, v.bucket.bucket+"/"+src) + } else if skew := time.Now().UTC().Sub(*resp.CopyObjectResult.LastModified); skew > maxClockSkew { + return fmt.Errorf("PutCopy succeeded but returned old timestamp %s (skew %v > max %v, now %s)", resp.CopyObjectResult.LastModified, skew, maxClockSkew, time.Now()) } return nil } @@ -173,28 +182,18 @@ func (v *s3Volume) check(ec2metadataHostname string) error { return errors.New("DriverParameters: V2Signature is not supported") } - defaultResolver := endpoints.NewDefaultResolver() - - cfg := defaults.Config() - if v.Endpoint == "" && v.Region == "" { return fmt.Errorf("AWS region or endpoint must be specified") - } else if v.Endpoint != "" || ec2metadataHostname != "" { - myCustomResolver := func(service, region string) (aws.Endpoint, error) { - if v.Endpoint != "" && service == "s3" { - return aws.Endpoint{ - URL: v.Endpoint, - SigningRegion: region, - }, nil - } else if service == "ec2metadata" && ec2metadataHostname != "" { - return aws.Endpoint{ - URL: ec2metadataHostname, - }, nil - } else { - return defaultResolver.ResolveEndpoint(service, region) - } + } else if v.Endpoint != "" { + _, err := url.Parse(v.Endpoint) + if err != nil { + return fmt.Errorf("error parsing custom S3 endpoint %q: %w", v.Endpoint, err) + } + v.overrideEndpoint = &aws.Endpoint{ + URL: v.Endpoint, + HostnameImmutable: true, + Source: aws.EndpointSourceCustom, } - cfg.EndpointResolver = aws.EndpointResolverFunc(myCustomResolver) } if v.Region == "" { // Endpoint is already specified (otherwise we would @@ -203,7 +202,6 @@ func (v *s3Volume) check(ec2metadataHostname string) error { // SignatureVersions. v.Region = "us-east-1" } - cfg.Region = v.Region // Zero timeouts mean "wait forever", which is a bad // default. Default to long timeouts instead. @@ -214,33 +212,65 @@ func (v *s3Volume) check(ec2metadataHostname string) error { v.ReadTimeout = s3DefaultReadTimeout } - creds := aws.NewChainProvider( - []aws.CredentialsProvider{ - aws.NewStaticCredentialsProvider(v.AccessKeyID, v.SecretAccessKey, v.AuthToken), - ec2rolecreds.New(ec2metadata.New(cfg), func(opts *ec2rolecreds.ProviderOptions) { - // (from aws-sdk-go-v2 comments) - // "allow the credentials to trigger - // refreshing prior to the credentials - // actually expiring. This is - // beneficial so race conditions with - // expiring credentials do not cause - // request to fail unexpectedly due to - // ExpiredTokenException exceptions." - // - // (from - // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) - // "We make new credentials available - // at least five minutes before the - // expiration of the old credentials." - opts.ExpiryWindow = 5 * time.Minute - }), - }) - - cfg.Credentials = creds + cfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithRegion(v.Region), + config.WithCredentialsCacheOptions(func(o *aws.CredentialsCacheOptions) { + // (from aws-sdk-go-v2 comments) "allow the + // credentials to trigger refreshing prior to + // the credentials actually expiring. This is + // beneficial so race conditions with expiring + // credentials do not cause request to fail + // unexpectedly due to ExpiredTokenException + // exceptions." + // + // (from + // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) + // "We make new credentials available at least + // five minutes before the expiration of the + // old credentials." + o.ExpiryWindow = 5 * time.Minute + }), + func(o *config.LoadOptions) error { + if v.AccessKeyID == "" && v.SecretAccessKey == "" { + // Use default sdk behavior (IAM / IMDS) + return nil + } + v.logger.Debug("using static credentials") + o.Credentials = credentials.StaticCredentialsProvider{ + Value: aws.Credentials{ + AccessKeyID: v.AccessKeyID, + SecretAccessKey: v.SecretAccessKey, + Source: "Arvados configuration", + }, + } + return nil + }, + func(o *config.LoadOptions) error { + if ec2metadataHostname != "" { + o.EC2IMDSEndpoint = ec2metadataHostname + } + if v.overrideEndpoint != nil { + o.EndpointResolverWithOptions = aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + if service == "S3" { + return *v.overrideEndpoint, nil + } + return aws.Endpoint{}, errEndpointNotOverridden // use default resolver + }) + } + return nil + }, + ) + if err != nil { + return fmt.Errorf("error loading aws client config: %w", err) + } v.bucket = &s3Bucket{ bucket: v.Bucket, - svc: s3.New(cfg), + svc: s3.NewFromConfig(cfg, func(o *s3.Options) { + if v.usePathStyle { + o.UsePathStyle = true + } + }), } // Set up prometheus metrics @@ -263,7 +293,7 @@ func (v *s3Volume) EmptyTrash() { // Define "ready to delete" as "...when EmptyTrash started". startT := time.Now() - emptyOneKey := func(trash *s3.Object) { + emptyOneKey := func(trash *types.Object) { key := strings.TrimPrefix(*trash.Key, "trash/") loc, isblk := v.isKeepBlock(key) if !isblk { @@ -337,7 +367,7 @@ func (v *s3Volume) EmptyTrash() { } var wg sync.WaitGroup - todo := make(chan *s3.Object, v.cluster.Collections.BlobDeleteConcurrency) + todo := make(chan *types.Object, v.cluster.Collections.BlobDeleteConcurrency) for i := 0; i < v.cluster.Collections.BlobDeleteConcurrency; i++ { wg.Add(1) go func() { @@ -411,8 +441,7 @@ func (v *s3Volume) head(key string) (result *s3.HeadObjectOutput, err error) { Key: aws.String(key), } - req := v.bucket.svc.HeadObjectRequest(input) - res, err := req.Send(context.TODO()) + res, err := v.bucket.svc.HeadObject(context.Background(), input) v.bucket.stats.TickOps("head") v.bucket.stats.Tick(&v.bucket.stats.Ops, &v.bucket.stats.HeadOps) @@ -421,8 +450,7 @@ func (v *s3Volume) head(key string) (result *s3.HeadObjectOutput, err error) { if err != nil { return nil, v.translateError(err) } - result = res.HeadObjectOutput - return + return res, nil } // BlockRead reads a Keep block that has been stored as a block blob @@ -459,11 +487,11 @@ func (v *s3Volume) BlockRead(ctx context.Context, hash string, w io.WriterAt) er } func (v *s3Volume) readWorker(ctx context.Context, key string, dst io.WriterAt) error { - downloader := s3manager.NewDownloaderWithClient(v.bucket.svc, func(u *s3manager.Downloader) { + downloader := manager.NewDownloader(v.bucket.svc, func(u *manager.Downloader) { u.PartSize = s3downloaderPartSize u.Concurrency = s3downloaderReadConcurrency }) - count, err := downloader.DownloadWithContext(ctx, dst, &s3.GetObjectInput{ + count, err := downloader.Download(ctx, dst, &s3.GetObjectInput{ Bucket: aws.String(v.bucket.bucket), Key: aws.String(key), }) @@ -481,7 +509,7 @@ func (v *s3Volume) writeObject(ctx context.Context, key string, r io.Reader) err r = bytes.NewReader(nil) } - uploadInput := s3manager.UploadInput{ + uploadInput := s3.PutObjectInput{ Bucket: aws.String(v.bucket.bucket), Key: aws.String(key), Body: r, @@ -501,20 +529,15 @@ func (v *s3Volume) writeObject(ctx context.Context, key string, r io.Reader) err // throughput, better than higher concurrency (10 or 13) by ~5%. // Defining u.BufferProvider = s3manager.NewBufferedReadSeekerWriteToPool(64 * 1024 * 1024) // is detrimental to throughput (minus ~15%). - uploader := s3manager.NewUploaderWithClient(v.bucket.svc, func(u *s3manager.Uploader) { + uploader := manager.NewUploader(v.bucket.svc, func(u *manager.Uploader) { u.PartSize = s3uploaderPartSize u.Concurrency = s3uploaderWriteConcurrency }) - // Unlike the goamz S3 driver, we don't need to precompute ContentSHA256: - // the aws-sdk-go v2 SDK uses a ReadSeeker to avoid having to copy the - // block, so there is no extra memory use to be concerned about. See - // makeSha256Reader in aws/signer/v4/v4.go. In fact, we explicitly disable - // calculating the Sha-256 because we don't need it; we already use md5sum - // hashes that match the name of the block. - _, err := uploader.UploadWithContext(ctx, &uploadInput, s3manager.WithUploaderRequestOptions(func(r *aws.Request) { - r.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD") - })) + _, err := uploader.Upload(ctx, &uploadInput, + // Avoid precomputing SHA256 before sending. + manager.WithUploaderRequestOptions(s3.WithAPIOptions(v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware)), + ) v.bucket.stats.TickOps("put") v.bucket.stats.Tick(&v.bucket.stats.Ops, &v.bucket.stats.PutOps) @@ -544,13 +567,13 @@ type s3awsLister struct { PageSize int Stats *s3awsbucketStats ContinuationToken string - buf []s3.Object + buf []types.Object err error } // First fetches the first page and returns the first item. It returns // nil if the response is the empty set or an error occurs. -func (lister *s3awsLister) First() *s3.Object { +func (lister *s3awsLister) First() *types.Object { lister.getPage() return lister.pop() } @@ -558,7 +581,7 @@ func (lister *s3awsLister) First() *s3.Object { // Next returns the next item, fetching the next page if necessary. It // returns nil if the last available item has already been fetched, or // an error occurs. -func (lister *s3awsLister) Next() *s3.Object { +func (lister *s3awsLister) Next() *types.Object { if len(lister.buf) == 0 && lister.ContinuationToken != "" { lister.getPage() } @@ -578,22 +601,22 @@ func (lister *s3awsLister) getPage() { if lister.ContinuationToken == "" { input = &s3.ListObjectsV2Input{ Bucket: aws.String(lister.Bucket.bucket), - MaxKeys: aws.Int64(int64(lister.PageSize)), + MaxKeys: aws.Int32(int32(lister.PageSize)), Prefix: aws.String(lister.Prefix), } } else { input = &s3.ListObjectsV2Input{ Bucket: aws.String(lister.Bucket.bucket), - MaxKeys: aws.Int64(int64(lister.PageSize)), + MaxKeys: aws.Int32(int32(lister.PageSize)), Prefix: aws.String(lister.Prefix), ContinuationToken: &lister.ContinuationToken, } } - req := lister.Bucket.svc.ListObjectsV2Request(input) - resp, err := req.Send(context.Background()) + resp, err := lister.Bucket.svc.ListObjectsV2(context.Background(), input) if err != nil { - if aerr, ok := err.(awserr.Error); ok { + var aerr smithy.APIError + if errors.As(err, &aerr) { lister.err = aerr } else { lister.err = err @@ -606,7 +629,7 @@ func (lister *s3awsLister) getPage() { } else { lister.ContinuationToken = "" } - lister.buf = make([]s3.Object, 0, len(resp.Contents)) + lister.buf = make([]types.Object, 0, len(resp.Contents)) for _, key := range resp.Contents { if !strings.HasPrefix(*key.Key, lister.Prefix) { lister.Logger.Warnf("s3awsLister: S3 Bucket.List(prefix=%q) returned key %q", lister.Prefix, *key.Key) @@ -616,7 +639,7 @@ func (lister *s3awsLister) getPage() { } } -func (lister *s3awsLister) pop() (k *s3.Object) { +func (lister *s3awsLister) pop() (k *types.Object) { if len(lister.buf) > 0 { k = &lister.buf[0] lister.buf = lister.buf[1:] @@ -774,8 +797,7 @@ func (b *s3Bucket) Del(path string) error { Bucket: aws.String(b.bucket), Key: aws.String(path), } - req := b.svc.DeleteObjectRequest(input) - _, err := req.Send(context.Background()) + _, err := b.svc.DeleteObject(context.Background(), input) b.stats.TickOps("delete") b.stats.Tick(&b.stats.Ops, &b.stats.DelOps) b.stats.TickErr(err) @@ -833,12 +855,11 @@ func (s *s3awsbucketStats) TickErr(err error) { return } errType := fmt.Sprintf("%T", err) - if aerr, ok := err.(awserr.Error); ok { - if reqErr, ok := err.(awserr.RequestFailure); ok { - // A service error occurred - errType = errType + fmt.Sprintf(" %d %s", reqErr.StatusCode(), aerr.Code()) + if aerr := smithy.APIError(nil); errors.As(err, &aerr) { + if rerr := interface{ HTTPStatusCode() int }(nil); errors.As(err, &rerr) { + errType = errType + fmt.Sprintf(" %d %s", rerr.HTTPStatusCode(), aerr.ErrorCode()) } else { - errType = errType + fmt.Sprintf(" 000 %s", aerr.Code()) + errType = errType + fmt.Sprintf(" 000 %s", aerr.ErrorCode()) } } s.statsTicker.TickErr(err, errType) diff --git a/services/keepstore/s3_volume_test.go b/services/keepstore/s3_volume_test.go index fb68e1c057..50010b3bef 100644 --- a/services/keepstore/s3_volume_test.go +++ b/services/keepstore/s3_volume_test.go @@ -15,14 +15,15 @@ import ( "net/http/httptest" "os" "strings" + "sync/atomic" "time" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/ctxlog" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/s3manager" "github.com/johannesboyne/gofakes3" "github.com/johannesboyne/gofakes3/backend/s3mem" @@ -31,22 +32,18 @@ import ( check "gopkg.in/check.v1" ) -const ( - s3TestBucketName = "testbucket" -) - -type s3AWSFakeClock struct { +type s3fakeClock struct { now *time.Time } -func (c *s3AWSFakeClock) Now() time.Time { +func (c *s3fakeClock) Now() time.Time { if c.now == nil { return time.Now().UTC() } return c.now.UTC() } -func (c *s3AWSFakeClock) Since(t time.Time) time.Duration { +func (c *s3fakeClock) Since(t time.Time) time.Duration { return c.Now().Sub(t) } @@ -55,14 +52,16 @@ var _ = check.Suite(&stubbedS3Suite{}) var srv httptest.Server type stubbedS3Suite struct { - s3server *httptest.Server - metadata *httptest.Server - cluster *arvados.Cluster - volumes []*testableS3Volume + s3server *httptest.Server + s3fakeClock *s3fakeClock + metadata *httptest.Server + cluster *arvados.Cluster + volumes []*testableS3Volume } func (s *stubbedS3Suite) SetUpTest(c *check.C) { s.s3server = nil + s.s3fakeClock = &s3fakeClock{} s.metadata = nil s.cluster = testCluster(c) s.cluster.Volumes = map[string]arvados.Volume{ @@ -71,6 +70,12 @@ func (s *stubbedS3Suite) SetUpTest(c *check.C) { } } +func (s *stubbedS3Suite) TearDownTest(c *check.C) { + if s.s3server != nil { + s.s3server.Close() + } +} + func (s *stubbedS3Suite) TestGeneric(c *check.C) { DoGenericVolumeTests(c, false, func(t TB, params newVolumeParams) TestableVolume { // Use a negative raceWindow so s3test's 1-second @@ -145,9 +150,9 @@ func (s *stubbedS3Suite) TestSignature(c *check.C) { logger: ctxlog.TestLogger(c), metrics: newVolumeMetricsVecs(prometheus.NewRegistry()), } - err := vol.check("") // Our test S3 server uses the older 'Path Style' - vol.bucket.svc.ForcePathStyle = true + vol.usePathStyle = true + err := vol.check("") c.Check(err, check.IsNil) err = vol.BlockWrite(context.Background(), "acbd18db4cc2f85cedef654fccc4a4d8", []byte("foo")) @@ -156,20 +161,37 @@ func (s *stubbedS3Suite) TestSignature(c *check.C) { } func (s *stubbedS3Suite) TestIAMRoleCredentials(c *check.C) { + var reqHeader http.Header + stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reqHeader = r.Header + })) + defer stub.Close() + + retrievedMetadata := false s.metadata = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + retrievedMetadata = true upd := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339) exp := time.Now().UTC().Add(time.Hour).Format(time.RFC3339) - // Literal example from - // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials - // but with updated timestamps - io.WriteString(w, `{"Code":"Success","LastUpdated":"`+upd+`","Type":"AWS-HMAC","AccessKeyId":"ASIAIOSFODNN7EXAMPLE","SecretAccessKey":"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY","Token":"token","Expiration":"`+exp+`"}`) + c.Logf("metadata stub received request: %s %s", r.Method, r.URL.Path) + switch { + case r.URL.Path == "/latest/meta-data/iam/security-credentials/": + io.WriteString(w, "testcredential\n") + case r.URL.Path == "/latest/api/token", + r.URL.Path == "/latest/meta-data/iam/security-credentials/testcredential": + // Literal example from + // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials + // but with updated timestamps + io.WriteString(w, `{"Code":"Success","LastUpdated":"`+upd+`","Type":"AWS-HMAC","AccessKeyId":"ASIAIOSFODNN7EXAMPLE","SecretAccessKey":"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY","Token":"token","Expiration":"`+exp+`"}`) + default: + w.WriteHeader(http.StatusNotFound) + } })) defer s.metadata.Close() v := &s3Volume{ S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{ IAMRole: s.metadata.URL + "/latest/api/token", - Endpoint: "http://localhost:12345", + Endpoint: stub.URL, Region: "test-region-1", Bucket: "test-bucket-name", }, @@ -179,18 +201,21 @@ func (s *stubbedS3Suite) TestIAMRoleCredentials(c *check.C) { } err := v.check(s.metadata.URL + "/latest") c.Check(err, check.IsNil) - creds, err := v.bucket.svc.Client.Config.Credentials.Retrieve(context.Background()) + resp, err := v.bucket.svc.ListBuckets(context.Background(), &s3.ListBucketsInput{}) c.Check(err, check.IsNil) - c.Check(creds.AccessKeyID, check.Equals, "ASIAIOSFODNN7EXAMPLE") - c.Check(creds.SecretAccessKey, check.Equals, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY") + c.Check(resp.Buckets, check.HasLen, 0) + c.Check(retrievedMetadata, check.Equals, true) + c.Check(reqHeader.Get("Authorization"), check.Matches, `AWS4-HMAC-SHA256 Credential=ASIAIOSFODNN7EXAMPLE/\d+/test-region-1/s3/aws4_request, SignedHeaders=.*`) + retrievedMetadata = false s.metadata = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + retrievedMetadata = true + c.Logf("metadata stub received request: %s %s", r.Method, r.URL.Path) w.WriteHeader(http.StatusNotFound) })) deadv := &s3Volume{ S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{ - IAMRole: s.metadata.URL + "/fake-metadata/test-role", - Endpoint: "http://localhost:12345", + Endpoint: "http://localhost:9", Region: "test-region-1", Bucket: "test-bucket-name", }, @@ -200,9 +225,10 @@ func (s *stubbedS3Suite) TestIAMRoleCredentials(c *check.C) { } err = deadv.check(s.metadata.URL + "/latest") c.Check(err, check.IsNil) - _, err = deadv.bucket.svc.Client.Config.Credentials.Retrieve(context.Background()) - c.Check(err, check.ErrorMatches, `(?s).*EC2RoleRequestError: no EC2 instance role found.*`) + _, err = deadv.bucket.svc.ListBuckets(context.Background(), &s3.ListBucketsInput{}) + c.Check(err, check.ErrorMatches, `(?s).*failed to refresh cached credentials, no EC2 IMDS role found.*`) c.Check(err, check.ErrorMatches, `(?s).*404.*`) + c.Check(retrievedMetadata, check.Equals, true) } func (s *stubbedS3Suite) TestStats(c *check.C) { @@ -224,7 +250,7 @@ func (s *stubbedS3Suite) TestStats(c *check.C) { err := v.BlockRead(context.Background(), loc, brdiscard) c.Check(err, check.NotNil) c.Check(stats(), check.Matches, `.*"Ops":[^0],.*`) - c.Check(stats(), check.Matches, `.*"s3.requestFailure 404 NoSuchKey[^"]*":[^0].*`) + c.Check(stats(), check.Matches, `.*"\*smithy.OperationError 404 NoSuchKey":[^0].*`) c.Check(stats(), check.Matches, `.*"InBytes":0,.*`) err = v.BlockWrite(context.Background(), loc, []byte("foo")) @@ -334,9 +360,9 @@ func (s *stubbedS3Suite) TestBackendStates(c *check.C) { if t == none { return } - v.serverClock.now = &t - uploader := s3manager.NewUploaderWithClient(v.bucket.svc) - _, err := uploader.UploadWithContext(context.Background(), &s3manager.UploadInput{ + s.s3fakeClock.now = &t + uploader := manager.NewUploader(v.bucket.svc) + _, err := uploader.Upload(context.Background(), &s3.PutObjectInput{ Bucket: aws.String(v.bucket.bucket), Key: aws.String(key), Body: bytes.NewReader(data), @@ -344,7 +370,7 @@ func (s *stubbedS3Suite) TestBackendStates(c *check.C) { if err != nil { panic(err) } - v.serverClock.now = nil + s.s3fakeClock.now = nil _, err = v.head(key) if err != nil { panic(err) @@ -473,14 +499,14 @@ func (s *stubbedS3Suite) TestBackendStates(c *check.C) { putS3Obj(scenario.dataT, key, blk) putS3Obj(scenario.recentT, "recent/"+key, nil) putS3Obj(scenario.trashT, "trash/"+key, blk) - v.serverClock.now = &t0 + v.s3fakeClock.now = &t0 return loc, blk } // Check canGet loc, blk := setupScenario() err := v.BlockRead(context.Background(), loc, brdiscard) - c.Check(err == nil, check.Equals, scenario.canGet) + c.Check(err == nil, check.Equals, scenario.canGet, check.Commentf("err was %+v", err)) if err != nil { c.Check(os.IsNotExist(err), check.Equals, true) } @@ -538,47 +564,49 @@ type testableS3Volume struct { *s3Volume server *httptest.Server c *check.C - serverClock *s3AWSFakeClock + s3fakeClock *s3fakeClock } -type LogrusLog struct { - log *logrus.FieldLogger +type gofakes3logger struct { + logrus.FieldLogger } -func (l LogrusLog) Print(level gofakes3.LogLevel, v ...interface{}) { +func (l gofakes3logger) Print(level gofakes3.LogLevel, v ...interface{}) { switch level { case gofakes3.LogErr: - (*l.log).Errorln(v...) + l.Errorln(v...) case gofakes3.LogWarn: - (*l.log).Warnln(v...) + l.Warnln(v...) case gofakes3.LogInfo: - (*l.log).Infoln(v...) + l.Infoln(v...) default: panic("unknown level") } } -func (s *stubbedS3Suite) newTestableVolume(c *check.C, params newVolumeParams, raceWindow time.Duration) *testableS3Volume { - - clock := &s3AWSFakeClock{} - // fake s3 - backend := s3mem.New(s3mem.WithTimeSource(clock)) +var testBucketSerial atomic.Int64 - // To enable GoFakeS3 debug logging, pass logger to gofakes3.WithLogger() - /* logger := new(LogrusLog) - ctxLogger := ctxlog.FromContext(context.Background()) - logger.log = &ctxLogger */ - faker := gofakes3.New(backend, gofakes3.WithTimeSource(clock), gofakes3.WithLogger(nil), gofakes3.WithTimeSkewLimit(0)) - srv := httptest.NewServer(faker.Server()) - - endpoint := srv.URL - if s.s3server != nil { - endpoint = s.s3server.URL +func (s *stubbedS3Suite) newTestableVolume(c *check.C, params newVolumeParams, raceWindow time.Duration) *testableS3Volume { + if params.Logger == nil { + params.Logger = ctxlog.TestLogger(c) } + if s.s3server == nil { + backend := s3mem.New(s3mem.WithTimeSource(s.s3fakeClock)) + logger := ctxlog.TestLogger(c) + faker := gofakes3.New(backend, + gofakes3.WithTimeSource(s.s3fakeClock), + gofakes3.WithLogger(gofakes3logger{FieldLogger: logger}), + gofakes3.WithTimeSkewLimit(0)) + s.s3server = httptest.NewServer(faker.Server()) + } + endpoint := s.s3server.URL + bucketName := fmt.Sprintf("testbucket%d", testBucketSerial.Add(1)) - iamRole, accessKey, secretKey := "", "xxx", "xxx" + var metadataURL, iamRole, accessKey, secretKey string if s.metadata != nil { - iamRole, accessKey, secretKey = s.metadata.URL+"/fake-metadata/test-role", "", "" + metadataURL, iamRole = s.metadata.URL, s.metadata.URL+"/fake-metadata/test-role" + } else { + accessKey, secretKey = "xxx", "xxx" } v := &testableS3Volume{ @@ -587,32 +615,29 @@ func (s *stubbedS3Suite) newTestableVolume(c *check.C, params newVolumeParams, r IAMRole: iamRole, AccessKeyID: accessKey, SecretAccessKey: secretKey, - Bucket: s3TestBucketName, + Bucket: bucketName, Endpoint: endpoint, Region: "test-region-1", LocationConstraint: true, UnsafeDelete: true, IndexPageSize: 1000, }, - cluster: params.Cluster, - volume: params.ConfigVolume, - logger: params.Logger, - metrics: params.MetricsVecs, - bufferPool: params.BufferPool, + cluster: params.Cluster, + volume: params.ConfigVolume, + logger: params.Logger, + metrics: params.MetricsVecs, + bufferPool: params.BufferPool, + usePathStyle: true, }, c: c, - server: srv, - serverClock: clock, + s3fakeClock: s.s3fakeClock, } - c.Assert(v.s3Volume.check(""), check.IsNil) - // Our test S3 server uses the older 'Path Style' - v.s3Volume.bucket.svc.ForcePathStyle = true + c.Assert(v.s3Volume.check(metadataURL), check.IsNil) // Create the testbucket input := &s3.CreateBucketInput{ - Bucket: aws.String(s3TestBucketName), + Bucket: aws.String(bucketName), } - req := v.s3Volume.bucket.svc.CreateBucketRequest(input) - _, err := req.Send(context.Background()) + _, err := v.s3Volume.bucket.svc.CreateBucket(context.Background(), input) c.Assert(err, check.IsNil) // We couldn't set RaceWindow until now because check() // rejects negative values. @@ -624,12 +649,12 @@ func (v *testableS3Volume) blockWriteWithoutMD5Check(loc string, block []byte) e key := v.key(loc) r := newCountingReader(bytes.NewReader(block), v.bucket.stats.TickOutBytes) - uploader := s3manager.NewUploaderWithClient(v.bucket.svc, func(u *s3manager.Uploader) { + uploader := manager.NewUploader(v.bucket.svc, func(u *manager.Uploader) { u.PartSize = 5 * 1024 * 1024 u.Concurrency = 13 }) - _, err := uploader.Upload(&s3manager.UploadInput{ + _, err := uploader.Upload(context.Background(), &s3.PutObjectInput{ Bucket: aws.String(v.bucket.bucket), Key: aws.String(key), Body: r, @@ -639,7 +664,7 @@ func (v *testableS3Volume) blockWriteWithoutMD5Check(loc string, block []byte) e } empty := bytes.NewReader([]byte{}) - _, err = uploader.Upload(&s3manager.UploadInput{ + _, err = uploader.Upload(context.Background(), &s3.PutObjectInput{ Bucket: aws.String(v.bucket.bucket), Key: aws.String("recent/" + key), Body: empty, @@ -651,11 +676,11 @@ func (v *testableS3Volume) blockWriteWithoutMD5Check(loc string, block []byte) e // there are no other operations happening on the same s3test server // while we do this. func (v *testableS3Volume) TouchWithDate(loc string, lastPut time.Time) { - v.serverClock.now = &lastPut + v.s3fakeClock.now = &lastPut - uploader := s3manager.NewUploaderWithClient(v.bucket.svc) + uploader := manager.NewUploader(v.bucket.svc) empty := bytes.NewReader([]byte{}) - _, err := uploader.UploadWithContext(context.Background(), &s3manager.UploadInput{ + _, err := uploader.Upload(context.Background(), &s3.PutObjectInput{ Bucket: aws.String(v.bucket.bucket), Key: aws.String("recent/" + v.key(loc)), Body: empty, @@ -664,11 +689,10 @@ func (v *testableS3Volume) TouchWithDate(loc string, lastPut time.Time) { panic(err) } - v.serverClock.now = nil + v.s3fakeClock.now = nil } func (v *testableS3Volume) Teardown() { - v.server.Close() } func (v *testableS3Volume) ReadWriteOperationLabelValues() (r, w string) { diff --git a/services/keepstore/volume_generic_test.go b/services/keepstore/volume_generic_test.go index 16084058b7..08911f3848 100644 --- a/services/keepstore/volume_generic_test.go +++ b/services/keepstore/volume_generic_test.go @@ -271,7 +271,7 @@ func (s *genericVolumeSuite) testPutAndTouch(t TB, factory TestableVolumeFactory // Make sure v.Mtime() agrees the above Utime really worked. if t0, err := v.Mtime(TestHash); err != nil || t0.IsZero() || !t0.Before(threshold) { - t.Errorf("Setting mtime failed: %v, %v", t0, err) + t.Errorf("Setting mtime failed: threshold %v, t0 %v, err %v", threshold.UTC(), t0.UTC(), err) } // Write the same block again. -- 2.30.2