github.com/Azure/go-autorest/autorest/azure/auth v0.5.9
github.com/Azure/go-autorest/autorest/to v0.4.0
github.com/arvados/cgofuse v1.2.0-arvados1
- github.com/aws/aws-sdk-go v1.44.174
+ github.com/aws/aws-sdk-go v1.44.256
github.com/aws/aws-sdk-go-v2 v1.26.1
+ github.com/aws/aws-sdk-go-v2/config v1.27.12
+ github.com/aws/aws-sdk-go-v2/credentials v1.17.12
+ github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.16
+ github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2
+ github.com/aws/smithy-go v1.20.2
github.com/bmatcuk/doublestar/v4 v4.6.1
github.com/bradleypeabody/godap v0.0.0-20170216002349-c249933bc092
github.com/coreos/go-oidc/v3 v3.5.0
github.com/imdario/mergo v0.3.12
github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff
github.com/jmoiron/sqlx v1.2.0
- github.com/johannesboyne/gofakes3 v0.0.0-20200716060623-6b2b4cb092cc
+ github.com/johannesboyne/gofakes3 v0.0.0-20240513200200-99de01ee122d
github.com/julienschmidt/httprouter v1.3.0
github.com/lib/pq v1.10.2
github.com/msteinert/pam v0.0.0-20190215180659-f29b9f28d6f9
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 // indirect
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
- github.com/aws/aws-sdk-go-v2/config v1.27.11 // indirect
- github.com/aws/aws-sdk-go-v2/credentials v1.17.11 // indirect
+ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
- github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
- github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
- github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
- github.com/aws/smithy-go v1.20.2 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
+ github.com/aws/aws-sdk-go-v2/service/sso v1.20.6 // indirect
+ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.5 // indirect
+ github.com/aws/aws-sdk-go-v2/service/sts v1.28.7 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bgentry/speakeasy v0.1.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect
github.com/satori/go.uuid v1.2.1-0.20180404165556-75cca531ea76 // indirect
github.com/sergi/go-diff v1.0.0 // indirect
- github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 // indirect
+ github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect
github.com/src-d/gcfg v1.3.0 // indirect
github.com/xanzy/ssh-agent v0.1.0 // indirect
go.opencensus.io v0.24.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
- golang.org/x/tools v0.6.0 // indirect
+ golang.org/x/tools v0.8.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/grpc v1.59.0 // indirect
github.com/arvados/cgofuse v1.2.0-arvados1/go.mod h1:79WFV98hrkRHK9XPhh2IGGOwpFSjocsWubgxAs2KhRc=
github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef h1:cl7DIRbiAYNqaVxg3CZY8qfZoBOKrj06H/x9SPGaxas=
github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef/go.mod h1:rCtgyMmBGEbjTm37fCuBYbNL0IhztiALzo3OB9HyiOM=
+github.com/arvados/gofakes3 v0.0.0-20240513190428-7dd242f94aa2 h1:q9L4OHOFwd3zNFlEADucYTBfKasWpz4NwjaGjJlYhqs=
+github.com/arvados/gofakes3 v0.0.0-20240513190428-7dd242f94aa2/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g=
github.com/arvados/yaml v0.0.0-20210427145106-92a1cab0904b h1:hK0t0aJTTXI64lpXln2A1SripqOym+GVNTnwsLes39Y=
github.com/arvados/yaml v0.0.0-20210427145106-92a1cab0904b/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.44.174 h1:9lR4a6MKQW/t6YCG0ZKAt1GAkjdEPP8sWch/pfcuR0c=
github.com/aws/aws-sdk-go v1.44.174/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVWY4=
+github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go-v2 v0.23.0 h1:+E1q1LLSfHSDn/DzOtdJOX+pLZE2HiNV2yO5AjZINwM=
github.com/aws/aws-sdk-go-v2 v0.23.0/go.mod h1:2LhT7UgHOXK3UXONKI5OMgIyoQL6zTAw/jwIeX6yqzw=
github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA=
github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE=
+github.com/aws/aws-sdk-go-v2/config v1.27.12 h1:vq88mBaZI4NGLXk8ierArwSILmYHDJZGJOeAc/pzEVQ=
+github.com/aws/aws-sdk-go-v2/config v1.27.12/go.mod h1:IOrsf4IiN68+CgzyuyGUYTpCrtUQTbbMEAtR/MR/4ZU=
github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs=
github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.12 h1:PVbKQ0KjDosI5+nEdRMU8ygEQDmkJTSHBqPjEX30lqc=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.12/go.mod h1:jlWtGFRtKsqc5zqerHZYmKmRkUXo3KPM14YJ13ZEjwE=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.16 h1:n4k5rgvy0M748NadpDlGLOQ/KCVbNHCQsfI895wLrt0=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.16/go.mod h1:Rd5rkCdq44ZC5rS4CbF3Wc8FiWo7f9brp7qeIwWsJaU=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0 h1:ooy0OFbrdSwgk32OFGPnvBwry5ySYCKkgTEbQ2hejs8=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0/go.mod h1:xejKuuRDjz6z5OqyeLsz01MlOqqW7CqpAB4PabNvpu8=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2 h1:rq2hglTQM3yHZvOPVMtNvLS5x6hijx7JvRDgKiTNDGQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w=
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.6 h1:o5cTaeunSpfXiLTIBx5xo2enQmiChtu1IBbzXnfU9Hs=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.6/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.5 h1:Ciiz/plN+Z+pPO1G0W2zJoYIIl0KtKzY0LJ78NXYTws=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.5/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU=
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.7 h1:et3Ta53gotFR4ERLXXHIHl/Uuk1qYpP5uU7cvNql8ns=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.7/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/johannesboyne/gofakes3 v0.0.0-20200716060623-6b2b4cb092cc h1:JJPhSHowepOF2+ElJVyb9jgt5ZyBkPMkPuhS0uODSFs=
github.com/johannesboyne/gofakes3 v0.0.0-20200716060623-6b2b4cb092cc/go.mod h1:fNiSoOiEI5KlkWXn26OwKnNe58ilTIkpBlgOrt7Olu8=
+github.com/johannesboyne/gofakes3 v0.0.0-20240501064628-9cf5179a8df0 h1:+XMEfFS2W506TvrEY5ViKNEIqmX6mxLmBmx98/dBNAg=
+github.com/johannesboyne/gofakes3 v0.0.0-20240501064628-9cf5179a8df0/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g=
+github.com/johannesboyne/gofakes3 v0.0.0-20240513200200-99de01ee122d h1:9dIJ/sx3yapvuq3kvTSVQ6UVS2HxfOB4MCwWiH8JcvQ=
+github.com/johannesboyne/gofakes3 v0.0.0-20240513200200-99de01ee122d/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kevinburke/ssh_config v0.0.0-20171013211458-802051befeb5 h1:xXn0nBttYwok7DhU4RxqaADEpQn7fEMt5kKc3yoj/n0=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63 h1:J6qvD6rbmOil46orKqJaRPG+zTpoGlBTUdyv8ki63L0=
github.com/shabbyrobe/gocovmerge v0.0.0-20180507124511-f6ea450bfb63/go.mod h1:n+VKSARF5y/tS9XFSP7vWDfS+GUC5vs/YT7M5XDTUEM=
+github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI=
+github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
+golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
"errors"
"fmt"
"io"
+ "net/url"
"os"
"regexp"
"strings"
"git.arvados.org/arvados.git/sdk/go/arvados"
"github.com/aws/aws-sdk-go-v2/aws"
- "github.com/aws/aws-sdk-go-v2/aws/awserr"
- "github.com/aws/aws-sdk-go-v2/aws/defaults"
- "github.com/aws/aws-sdk-go-v2/aws/ec2metadata"
- "github.com/aws/aws-sdk-go-v2/aws/ec2rolecreds"
- "github.com/aws/aws-sdk-go-v2/aws/endpoints"
+ v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+ "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
- "github.com/aws/aws-sdk-go-v2/service/s3/s3manager"
+ "github.com/aws/aws-sdk-go-v2/service/s3/types"
+ "github.com/aws/smithy-go"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
)
)
var (
- errS3TrashDisabled = fmt.Errorf("trash function is disabled because Collections.BlobTrashLifetime=0 and DriverParameters.UnsafeDelete=false")
- s3AWSKeepBlockRegexp = regexp.MustCompile(`^[0-9a-f]{32}$`)
- s3AWSZeroTime time.Time
+ errS3TrashDisabled = fmt.Errorf("trash function is disabled because Collections.BlobTrashLifetime=0 and DriverParameters.UnsafeDelete=false")
+ s3AWSKeepBlockRegexp = regexp.MustCompile(`^[0-9a-f]{32}$`)
+ s3AWSZeroTime time.Time
+ defaultEndpointResolverV2 = s3.NewDefaultEndpointResolverV2()
+
+ // Returned by an aws.EndpointResolverWithOptions to indicate
+ // that the default resolver should be used.
+ errEndpointNotOverridden = &aws.EndpointNotFoundError{Err: errors.New("endpoint not overridden")}
)
// s3Volume implements Volume using an S3 bucket.
bucket *s3Bucket
region string
startOnce sync.Once
+
+ overrideEndpoint *aws.Endpoint
+ usePathStyle bool // used by test suite
}
// s3bucket wraps s3.bucket and counts I/O and API usage stats. The
}
func (v *s3Volume) translateError(err error) error {
- if _, ok := err.(*aws.RequestCanceledError); ok {
+ if cerr := (interface{ CanceledError() bool })(nil); errors.As(err, &cerr) && cerr.CanceledError() {
+ // *aws.RequestCanceledError and *smithy.CanceledError
+ // implement this interface.
return context.Canceled
- } else if aerr, ok := err.(awserr.Error); ok {
- if aerr.Code() == "NotFound" {
- return os.ErrNotExist
- } else if aerr.Code() == "NoSuchKey" {
+ }
+ var aerr smithy.APIError
+ if errors.As(err, &aerr) {
+ switch aerr.ErrorCode() {
+ case "NotFound", "NoSuchKey":
return os.ErrNotExist
}
}
Key: aws.String(dst),
}
- req := v.bucket.svc.CopyObjectRequest(input)
- resp, err := req.Send(context.Background())
+ resp, err := v.bucket.svc.CopyObject(context.Background(), input)
err = v.translateError(err)
if os.IsNotExist(err) {
return err
} else if err != nil {
return fmt.Errorf("PutCopy(%q ← %q): %s", dst, v.bucket.bucket+"/"+src, err)
- }
-
- if resp.CopyObjectResult.LastModified == nil {
- return fmt.Errorf("PutCopy succeeded but did not return a timestamp: %q: %s", resp.CopyObjectResult.LastModified, err)
- } else if time.Now().Sub(*resp.CopyObjectResult.LastModified) > maxClockSkew {
- return fmt.Errorf("PutCopy succeeded but returned an old timestamp: %q: %s", resp.CopyObjectResult.LastModified, resp.CopyObjectResult.LastModified)
+ } else if resp.CopyObjectResult.LastModified == nil {
+ return fmt.Errorf("PutCopy(%q ← %q): succeeded but did not return a timestamp", dst, v.bucket.bucket+"/"+src)
+ } else if skew := time.Now().UTC().Sub(*resp.CopyObjectResult.LastModified); skew > maxClockSkew {
+ return fmt.Errorf("PutCopy succeeded but returned old timestamp %s (skew %v > max %v, now %s)", resp.CopyObjectResult.LastModified, skew, maxClockSkew, time.Now())
}
return nil
}
return errors.New("DriverParameters: V2Signature is not supported")
}
- defaultResolver := endpoints.NewDefaultResolver()
-
- cfg := defaults.Config()
-
if v.Endpoint == "" && v.Region == "" {
return fmt.Errorf("AWS region or endpoint must be specified")
- } else if v.Endpoint != "" || ec2metadataHostname != "" {
- myCustomResolver := func(service, region string) (aws.Endpoint, error) {
- if v.Endpoint != "" && service == "s3" {
- return aws.Endpoint{
- URL: v.Endpoint,
- SigningRegion: region,
- }, nil
- } else if service == "ec2metadata" && ec2metadataHostname != "" {
- return aws.Endpoint{
- URL: ec2metadataHostname,
- }, nil
- } else {
- return defaultResolver.ResolveEndpoint(service, region)
- }
+ } else if v.Endpoint != "" {
+ _, err := url.Parse(v.Endpoint)
+ if err != nil {
+ return fmt.Errorf("error parsing custom S3 endpoint %q: %w", v.Endpoint, err)
+ }
+ v.overrideEndpoint = &aws.Endpoint{
+ URL: v.Endpoint,
+ HostnameImmutable: true,
+ Source: aws.EndpointSourceCustom,
}
- cfg.EndpointResolver = aws.EndpointResolverFunc(myCustomResolver)
}
if v.Region == "" {
// Endpoint is already specified (otherwise we would
// SignatureVersions.
v.Region = "us-east-1"
}
- cfg.Region = v.Region
// Zero timeouts mean "wait forever", which is a bad
// default. Default to long timeouts instead.
v.ReadTimeout = s3DefaultReadTimeout
}
- creds := aws.NewChainProvider(
- []aws.CredentialsProvider{
- aws.NewStaticCredentialsProvider(v.AccessKeyID, v.SecretAccessKey, v.AuthToken),
- ec2rolecreds.New(ec2metadata.New(cfg), func(opts *ec2rolecreds.ProviderOptions) {
- // (from aws-sdk-go-v2 comments)
- // "allow the credentials to trigger
- // refreshing prior to the credentials
- // actually expiring. This is
- // beneficial so race conditions with
- // expiring credentials do not cause
- // request to fail unexpectedly due to
- // ExpiredTokenException exceptions."
- //
- // (from
- // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html)
- // "We make new credentials available
- // at least five minutes before the
- // expiration of the old credentials."
- opts.ExpiryWindow = 5 * time.Minute
- }),
- })
-
- cfg.Credentials = creds
+ cfg, err := config.LoadDefaultConfig(context.TODO(),
+ config.WithRegion(v.Region),
+ config.WithCredentialsCacheOptions(func(o *aws.CredentialsCacheOptions) {
+ // (from aws-sdk-go-v2 comments) "allow the
+ // credentials to trigger refreshing prior to
+ // the credentials actually expiring. This is
+ // beneficial so race conditions with expiring
+ // credentials do not cause request to fail
+ // unexpectedly due to ExpiredTokenException
+ // exceptions."
+ //
+ // (from
+ // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html)
+ // "We make new credentials available at least
+ // five minutes before the expiration of the
+ // old credentials."
+ o.ExpiryWindow = 5 * time.Minute
+ }),
+ func(o *config.LoadOptions) error {
+ if v.AccessKeyID == "" && v.SecretAccessKey == "" {
+ // Use default sdk behavior (IAM / IMDS)
+ return nil
+ }
+ v.logger.Debug("using static credentials")
+ o.Credentials = credentials.StaticCredentialsProvider{
+ Value: aws.Credentials{
+ AccessKeyID: v.AccessKeyID,
+ SecretAccessKey: v.SecretAccessKey,
+ Source: "Arvados configuration",
+ },
+ }
+ return nil
+ },
+ func(o *config.LoadOptions) error {
+ if ec2metadataHostname != "" {
+ o.EC2IMDSEndpoint = ec2metadataHostname
+ }
+ if v.overrideEndpoint != nil {
+ o.EndpointResolverWithOptions = aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+ if service == "S3" {
+ return *v.overrideEndpoint, nil
+ }
+ return aws.Endpoint{}, errEndpointNotOverridden // use default resolver
+ })
+ }
+ return nil
+ },
+ )
+ if err != nil {
+ return fmt.Errorf("error loading aws client config: %w", err)
+ }
v.bucket = &s3Bucket{
bucket: v.Bucket,
- svc: s3.New(cfg),
+ svc: s3.NewFromConfig(cfg, func(o *s3.Options) {
+ if v.usePathStyle {
+ o.UsePathStyle = true
+ }
+ }),
}
// Set up prometheus metrics
// Define "ready to delete" as "...when EmptyTrash started".
startT := time.Now()
- emptyOneKey := func(trash *s3.Object) {
+ emptyOneKey := func(trash *types.Object) {
key := strings.TrimPrefix(*trash.Key, "trash/")
loc, isblk := v.isKeepBlock(key)
if !isblk {
}
var wg sync.WaitGroup
- todo := make(chan *s3.Object, v.cluster.Collections.BlobDeleteConcurrency)
+ todo := make(chan *types.Object, v.cluster.Collections.BlobDeleteConcurrency)
for i := 0; i < v.cluster.Collections.BlobDeleteConcurrency; i++ {
wg.Add(1)
go func() {
Key: aws.String(key),
}
- req := v.bucket.svc.HeadObjectRequest(input)
- res, err := req.Send(context.TODO())
+ res, err := v.bucket.svc.HeadObject(context.Background(), input)
v.bucket.stats.TickOps("head")
v.bucket.stats.Tick(&v.bucket.stats.Ops, &v.bucket.stats.HeadOps)
if err != nil {
return nil, v.translateError(err)
}
- result = res.HeadObjectOutput
- return
+ return res, nil
}
// BlockRead reads a Keep block that has been stored as a block blob
}
func (v *s3Volume) readWorker(ctx context.Context, key string, dst io.WriterAt) error {
- downloader := s3manager.NewDownloaderWithClient(v.bucket.svc, func(u *s3manager.Downloader) {
+ downloader := manager.NewDownloader(v.bucket.svc, func(u *manager.Downloader) {
u.PartSize = s3downloaderPartSize
u.Concurrency = s3downloaderReadConcurrency
})
- count, err := downloader.DownloadWithContext(ctx, dst, &s3.GetObjectInput{
+ count, err := downloader.Download(ctx, dst, &s3.GetObjectInput{
Bucket: aws.String(v.bucket.bucket),
Key: aws.String(key),
})
r = bytes.NewReader(nil)
}
- uploadInput := s3manager.UploadInput{
+ uploadInput := s3.PutObjectInput{
Bucket: aws.String(v.bucket.bucket),
Key: aws.String(key),
Body: r,
// throughput, better than higher concurrency (10 or 13) by ~5%.
// Defining u.BufferProvider = s3manager.NewBufferedReadSeekerWriteToPool(64 * 1024 * 1024)
// is detrimental to throughput (minus ~15%).
- uploader := s3manager.NewUploaderWithClient(v.bucket.svc, func(u *s3manager.Uploader) {
+ uploader := manager.NewUploader(v.bucket.svc, func(u *manager.Uploader) {
u.PartSize = s3uploaderPartSize
u.Concurrency = s3uploaderWriteConcurrency
})
- // Unlike the goamz S3 driver, we don't need to precompute ContentSHA256:
- // the aws-sdk-go v2 SDK uses a ReadSeeker to avoid having to copy the
- // block, so there is no extra memory use to be concerned about. See
- // makeSha256Reader in aws/signer/v4/v4.go. In fact, we explicitly disable
- // calculating the Sha-256 because we don't need it; we already use md5sum
- // hashes that match the name of the block.
- _, err := uploader.UploadWithContext(ctx, &uploadInput, s3manager.WithUploaderRequestOptions(func(r *aws.Request) {
- r.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
- }))
+ _, err := uploader.Upload(ctx, &uploadInput,
+ // Avoid precomputing SHA256 before sending.
+ manager.WithUploaderRequestOptions(s3.WithAPIOptions(v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware)),
+ )
v.bucket.stats.TickOps("put")
v.bucket.stats.Tick(&v.bucket.stats.Ops, &v.bucket.stats.PutOps)
PageSize int
Stats *s3awsbucketStats
ContinuationToken string
- buf []s3.Object
+ buf []types.Object
err error
}
// First fetches the first page and returns the first item. It returns
// nil if the response is the empty set or an error occurs.
-func (lister *s3awsLister) First() *s3.Object {
+func (lister *s3awsLister) First() *types.Object {
lister.getPage()
return lister.pop()
}
// Next returns the next item, fetching the next page if necessary. It
// returns nil if the last available item has already been fetched, or
// an error occurs.
-func (lister *s3awsLister) Next() *s3.Object {
+func (lister *s3awsLister) Next() *types.Object {
if len(lister.buf) == 0 && lister.ContinuationToken != "" {
lister.getPage()
}
if lister.ContinuationToken == "" {
input = &s3.ListObjectsV2Input{
Bucket: aws.String(lister.Bucket.bucket),
- MaxKeys: aws.Int64(int64(lister.PageSize)),
+ MaxKeys: aws.Int32(int32(lister.PageSize)),
Prefix: aws.String(lister.Prefix),
}
} else {
input = &s3.ListObjectsV2Input{
Bucket: aws.String(lister.Bucket.bucket),
- MaxKeys: aws.Int64(int64(lister.PageSize)),
+ MaxKeys: aws.Int32(int32(lister.PageSize)),
Prefix: aws.String(lister.Prefix),
ContinuationToken: &lister.ContinuationToken,
}
}
- req := lister.Bucket.svc.ListObjectsV2Request(input)
- resp, err := req.Send(context.Background())
+ resp, err := lister.Bucket.svc.ListObjectsV2(context.Background(), input)
if err != nil {
- if aerr, ok := err.(awserr.Error); ok {
+ var aerr smithy.APIError
+ if errors.As(err, &aerr) {
lister.err = aerr
} else {
lister.err = err
} else {
lister.ContinuationToken = ""
}
- lister.buf = make([]s3.Object, 0, len(resp.Contents))
+ lister.buf = make([]types.Object, 0, len(resp.Contents))
for _, key := range resp.Contents {
if !strings.HasPrefix(*key.Key, lister.Prefix) {
lister.Logger.Warnf("s3awsLister: S3 Bucket.List(prefix=%q) returned key %q", lister.Prefix, *key.Key)
}
}
-func (lister *s3awsLister) pop() (k *s3.Object) {
+func (lister *s3awsLister) pop() (k *types.Object) {
if len(lister.buf) > 0 {
k = &lister.buf[0]
lister.buf = lister.buf[1:]
Bucket: aws.String(b.bucket),
Key: aws.String(path),
}
- req := b.svc.DeleteObjectRequest(input)
- _, err := req.Send(context.Background())
+ _, err := b.svc.DeleteObject(context.Background(), input)
b.stats.TickOps("delete")
b.stats.Tick(&b.stats.Ops, &b.stats.DelOps)
b.stats.TickErr(err)
return
}
errType := fmt.Sprintf("%T", err)
- if aerr, ok := err.(awserr.Error); ok {
- if reqErr, ok := err.(awserr.RequestFailure); ok {
- // A service error occurred
- errType = errType + fmt.Sprintf(" %d %s", reqErr.StatusCode(), aerr.Code())
+ if aerr := smithy.APIError(nil); errors.As(err, &aerr) {
+ if rerr := interface{ HTTPStatusCode() int }(nil); errors.As(err, &rerr) {
+ errType = errType + fmt.Sprintf(" %d %s", rerr.HTTPStatusCode(), aerr.ErrorCode())
} else {
- errType = errType + fmt.Sprintf(" 000 %s", aerr.Code())
+ errType = errType + fmt.Sprintf(" 000 %s", aerr.ErrorCode())
}
}
s.statsTicker.TickErr(err, errType)
"net/http/httptest"
"os"
"strings"
+ "sync/atomic"
"time"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
- "github.com/aws/aws-sdk-go-v2/service/s3/s3manager"
"github.com/johannesboyne/gofakes3"
"github.com/johannesboyne/gofakes3/backend/s3mem"
check "gopkg.in/check.v1"
)
-const (
- s3TestBucketName = "testbucket"
-)
-
-type s3AWSFakeClock struct {
+type s3fakeClock struct {
now *time.Time
}
-func (c *s3AWSFakeClock) Now() time.Time {
+func (c *s3fakeClock) Now() time.Time {
if c.now == nil {
return time.Now().UTC()
}
return c.now.UTC()
}
-func (c *s3AWSFakeClock) Since(t time.Time) time.Duration {
+func (c *s3fakeClock) Since(t time.Time) time.Duration {
return c.Now().Sub(t)
}
var srv httptest.Server
type stubbedS3Suite struct {
- s3server *httptest.Server
- metadata *httptest.Server
- cluster *arvados.Cluster
- volumes []*testableS3Volume
+ s3server *httptest.Server
+ s3fakeClock *s3fakeClock
+ metadata *httptest.Server
+ cluster *arvados.Cluster
+ volumes []*testableS3Volume
}
func (s *stubbedS3Suite) SetUpTest(c *check.C) {
s.s3server = nil
+ s.s3fakeClock = &s3fakeClock{}
s.metadata = nil
s.cluster = testCluster(c)
s.cluster.Volumes = map[string]arvados.Volume{
}
}
+func (s *stubbedS3Suite) TearDownTest(c *check.C) {
+ if s.s3server != nil {
+ s.s3server.Close()
+ }
+}
+
func (s *stubbedS3Suite) TestGeneric(c *check.C) {
DoGenericVolumeTests(c, false, func(t TB, params newVolumeParams) TestableVolume {
// Use a negative raceWindow so s3test's 1-second
logger: ctxlog.TestLogger(c),
metrics: newVolumeMetricsVecs(prometheus.NewRegistry()),
}
- err := vol.check("")
// Our test S3 server uses the older 'Path Style'
- vol.bucket.svc.ForcePathStyle = true
+ vol.usePathStyle = true
+ err := vol.check("")
c.Check(err, check.IsNil)
err = vol.BlockWrite(context.Background(), "acbd18db4cc2f85cedef654fccc4a4d8", []byte("foo"))
}
func (s *stubbedS3Suite) TestIAMRoleCredentials(c *check.C) {
+ var reqHeader http.Header
+ stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ reqHeader = r.Header
+ }))
+ defer stub.Close()
+
+ retrievedMetadata := false
s.metadata = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ retrievedMetadata = true
upd := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339)
exp := time.Now().UTC().Add(time.Hour).Format(time.RFC3339)
- // Literal example from
- // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials
- // but with updated timestamps
- io.WriteString(w, `{"Code":"Success","LastUpdated":"`+upd+`","Type":"AWS-HMAC","AccessKeyId":"ASIAIOSFODNN7EXAMPLE","SecretAccessKey":"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY","Token":"token","Expiration":"`+exp+`"}`)
+ c.Logf("metadata stub received request: %s %s", r.Method, r.URL.Path)
+ switch {
+ case r.URL.Path == "/latest/meta-data/iam/security-credentials/":
+ io.WriteString(w, "testcredential\n")
+ case r.URL.Path == "/latest/api/token",
+ r.URL.Path == "/latest/meta-data/iam/security-credentials/testcredential":
+ // Literal example from
+ // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials
+ // but with updated timestamps
+ io.WriteString(w, `{"Code":"Success","LastUpdated":"`+upd+`","Type":"AWS-HMAC","AccessKeyId":"ASIAIOSFODNN7EXAMPLE","SecretAccessKey":"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY","Token":"token","Expiration":"`+exp+`"}`)
+ default:
+ w.WriteHeader(http.StatusNotFound)
+ }
}))
defer s.metadata.Close()
v := &s3Volume{
S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
IAMRole: s.metadata.URL + "/latest/api/token",
- Endpoint: "http://localhost:12345",
+ Endpoint: stub.URL,
Region: "test-region-1",
Bucket: "test-bucket-name",
},
}
err := v.check(s.metadata.URL + "/latest")
c.Check(err, check.IsNil)
- creds, err := v.bucket.svc.Client.Config.Credentials.Retrieve(context.Background())
+ resp, err := v.bucket.svc.ListBuckets(context.Background(), &s3.ListBucketsInput{})
c.Check(err, check.IsNil)
- c.Check(creds.AccessKeyID, check.Equals, "ASIAIOSFODNN7EXAMPLE")
- c.Check(creds.SecretAccessKey, check.Equals, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+ c.Check(resp.Buckets, check.HasLen, 0)
+ c.Check(retrievedMetadata, check.Equals, true)
+ c.Check(reqHeader.Get("Authorization"), check.Matches, `AWS4-HMAC-SHA256 Credential=ASIAIOSFODNN7EXAMPLE/\d+/test-region-1/s3/aws4_request, SignedHeaders=.*`)
+ retrievedMetadata = false
s.metadata = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ retrievedMetadata = true
+ c.Logf("metadata stub received request: %s %s", r.Method, r.URL.Path)
w.WriteHeader(http.StatusNotFound)
}))
deadv := &s3Volume{
S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
- IAMRole: s.metadata.URL + "/fake-metadata/test-role",
- Endpoint: "http://localhost:12345",
+ Endpoint: "http://localhost:9",
Region: "test-region-1",
Bucket: "test-bucket-name",
},
}
err = deadv.check(s.metadata.URL + "/latest")
c.Check(err, check.IsNil)
- _, err = deadv.bucket.svc.Client.Config.Credentials.Retrieve(context.Background())
- c.Check(err, check.ErrorMatches, `(?s).*EC2RoleRequestError: no EC2 instance role found.*`)
+ _, err = deadv.bucket.svc.ListBuckets(context.Background(), &s3.ListBucketsInput{})
+ c.Check(err, check.ErrorMatches, `(?s).*failed to refresh cached credentials, no EC2 IMDS role found.*`)
c.Check(err, check.ErrorMatches, `(?s).*404.*`)
+ c.Check(retrievedMetadata, check.Equals, true)
}
func (s *stubbedS3Suite) TestStats(c *check.C) {
err := v.BlockRead(context.Background(), loc, brdiscard)
c.Check(err, check.NotNil)
c.Check(stats(), check.Matches, `.*"Ops":[^0],.*`)
- c.Check(stats(), check.Matches, `.*"s3.requestFailure 404 NoSuchKey[^"]*":[^0].*`)
+ c.Check(stats(), check.Matches, `.*"\*smithy.OperationError 404 NoSuchKey":[^0].*`)
c.Check(stats(), check.Matches, `.*"InBytes":0,.*`)
err = v.BlockWrite(context.Background(), loc, []byte("foo"))
if t == none {
return
}
- v.serverClock.now = &t
- uploader := s3manager.NewUploaderWithClient(v.bucket.svc)
- _, err := uploader.UploadWithContext(context.Background(), &s3manager.UploadInput{
+ s.s3fakeClock.now = &t
+ uploader := manager.NewUploader(v.bucket.svc)
+ _, err := uploader.Upload(context.Background(), &s3.PutObjectInput{
Bucket: aws.String(v.bucket.bucket),
Key: aws.String(key),
Body: bytes.NewReader(data),
if err != nil {
panic(err)
}
- v.serverClock.now = nil
+ s.s3fakeClock.now = nil
_, err = v.head(key)
if err != nil {
panic(err)
putS3Obj(scenario.dataT, key, blk)
putS3Obj(scenario.recentT, "recent/"+key, nil)
putS3Obj(scenario.trashT, "trash/"+key, blk)
- v.serverClock.now = &t0
+ v.s3fakeClock.now = &t0
return loc, blk
}
// Check canGet
loc, blk := setupScenario()
err := v.BlockRead(context.Background(), loc, brdiscard)
- c.Check(err == nil, check.Equals, scenario.canGet)
+ c.Check(err == nil, check.Equals, scenario.canGet, check.Commentf("err was %+v", err))
if err != nil {
c.Check(os.IsNotExist(err), check.Equals, true)
}
*s3Volume
server *httptest.Server
c *check.C
- serverClock *s3AWSFakeClock
+ s3fakeClock *s3fakeClock
}
-type LogrusLog struct {
- log *logrus.FieldLogger
+type gofakes3logger struct {
+ logrus.FieldLogger
}
-func (l LogrusLog) Print(level gofakes3.LogLevel, v ...interface{}) {
+func (l gofakes3logger) Print(level gofakes3.LogLevel, v ...interface{}) {
switch level {
case gofakes3.LogErr:
- (*l.log).Errorln(v...)
+ l.Errorln(v...)
case gofakes3.LogWarn:
- (*l.log).Warnln(v...)
+ l.Warnln(v...)
case gofakes3.LogInfo:
- (*l.log).Infoln(v...)
+ l.Infoln(v...)
default:
panic("unknown level")
}
}
-func (s *stubbedS3Suite) newTestableVolume(c *check.C, params newVolumeParams, raceWindow time.Duration) *testableS3Volume {
-
- clock := &s3AWSFakeClock{}
- // fake s3
- backend := s3mem.New(s3mem.WithTimeSource(clock))
+var testBucketSerial atomic.Int64
- // To enable GoFakeS3 debug logging, pass logger to gofakes3.WithLogger()
- /* logger := new(LogrusLog)
- ctxLogger := ctxlog.FromContext(context.Background())
- logger.log = &ctxLogger */
- faker := gofakes3.New(backend, gofakes3.WithTimeSource(clock), gofakes3.WithLogger(nil), gofakes3.WithTimeSkewLimit(0))
- srv := httptest.NewServer(faker.Server())
-
- endpoint := srv.URL
- if s.s3server != nil {
- endpoint = s.s3server.URL
+func (s *stubbedS3Suite) newTestableVolume(c *check.C, params newVolumeParams, raceWindow time.Duration) *testableS3Volume {
+ if params.Logger == nil {
+ params.Logger = ctxlog.TestLogger(c)
}
+ if s.s3server == nil {
+ backend := s3mem.New(s3mem.WithTimeSource(s.s3fakeClock))
+ logger := ctxlog.TestLogger(c)
+ faker := gofakes3.New(backend,
+ gofakes3.WithTimeSource(s.s3fakeClock),
+ gofakes3.WithLogger(gofakes3logger{FieldLogger: logger}),
+ gofakes3.WithTimeSkewLimit(0))
+ s.s3server = httptest.NewServer(faker.Server())
+ }
+ endpoint := s.s3server.URL
+ bucketName := fmt.Sprintf("testbucket%d", testBucketSerial.Add(1))
- iamRole, accessKey, secretKey := "", "xxx", "xxx"
+ var metadataURL, iamRole, accessKey, secretKey string
if s.metadata != nil {
- iamRole, accessKey, secretKey = s.metadata.URL+"/fake-metadata/test-role", "", ""
+ metadataURL, iamRole = s.metadata.URL, s.metadata.URL+"/fake-metadata/test-role"
+ } else {
+ accessKey, secretKey = "xxx", "xxx"
}
v := &testableS3Volume{
IAMRole: iamRole,
AccessKeyID: accessKey,
SecretAccessKey: secretKey,
- Bucket: s3TestBucketName,
+ Bucket: bucketName,
Endpoint: endpoint,
Region: "test-region-1",
LocationConstraint: true,
UnsafeDelete: true,
IndexPageSize: 1000,
},
- cluster: params.Cluster,
- volume: params.ConfigVolume,
- logger: params.Logger,
- metrics: params.MetricsVecs,
- bufferPool: params.BufferPool,
+ cluster: params.Cluster,
+ volume: params.ConfigVolume,
+ logger: params.Logger,
+ metrics: params.MetricsVecs,
+ bufferPool: params.BufferPool,
+ usePathStyle: true,
},
c: c,
- server: srv,
- serverClock: clock,
+ s3fakeClock: s.s3fakeClock,
}
- c.Assert(v.s3Volume.check(""), check.IsNil)
- // Our test S3 server uses the older 'Path Style'
- v.s3Volume.bucket.svc.ForcePathStyle = true
+ c.Assert(v.s3Volume.check(metadataURL), check.IsNil)
// Create the testbucket
input := &s3.CreateBucketInput{
- Bucket: aws.String(s3TestBucketName),
+ Bucket: aws.String(bucketName),
}
- req := v.s3Volume.bucket.svc.CreateBucketRequest(input)
- _, err := req.Send(context.Background())
+ _, err := v.s3Volume.bucket.svc.CreateBucket(context.Background(), input)
c.Assert(err, check.IsNil)
// We couldn't set RaceWindow until now because check()
// rejects negative values.
key := v.key(loc)
r := newCountingReader(bytes.NewReader(block), v.bucket.stats.TickOutBytes)
- uploader := s3manager.NewUploaderWithClient(v.bucket.svc, func(u *s3manager.Uploader) {
+ uploader := manager.NewUploader(v.bucket.svc, func(u *manager.Uploader) {
u.PartSize = 5 * 1024 * 1024
u.Concurrency = 13
})
- _, err := uploader.Upload(&s3manager.UploadInput{
+ _, err := uploader.Upload(context.Background(), &s3.PutObjectInput{
Bucket: aws.String(v.bucket.bucket),
Key: aws.String(key),
Body: r,
}
empty := bytes.NewReader([]byte{})
- _, err = uploader.Upload(&s3manager.UploadInput{
+ _, err = uploader.Upload(context.Background(), &s3.PutObjectInput{
Bucket: aws.String(v.bucket.bucket),
Key: aws.String("recent/" + key),
Body: empty,
// there are no other operations happening on the same s3test server
// while we do this.
func (v *testableS3Volume) TouchWithDate(loc string, lastPut time.Time) {
- v.serverClock.now = &lastPut
+ v.s3fakeClock.now = &lastPut
- uploader := s3manager.NewUploaderWithClient(v.bucket.svc)
+ uploader := manager.NewUploader(v.bucket.svc)
empty := bytes.NewReader([]byte{})
- _, err := uploader.UploadWithContext(context.Background(), &s3manager.UploadInput{
+ _, err := uploader.Upload(context.Background(), &s3.PutObjectInput{
Bucket: aws.String(v.bucket.bucket),
Key: aws.String("recent/" + v.key(loc)),
Body: empty,
panic(err)
}
- v.serverClock.now = nil
+ v.s3fakeClock.now = nil
}
func (v *testableS3Volume) Teardown() {
- v.server.Close()
}
func (v *testableS3Volume) ReadWriteOperationLabelValues() (r, w string) {
// Make sure v.Mtime() agrees the above Utime really worked.
if t0, err := v.Mtime(TestHash); err != nil || t0.IsZero() || !t0.Before(threshold) {
- t.Errorf("Setting mtime failed: %v, %v", t0, err)
+ t.Errorf("Setting mtime failed: threshold %v, t0 %v, err %v", threshold.UTC(), t0.UTC(), err)
}
// Write the same block again.
projects / arvados.git / commitdiff