Merge branch '17118-arvput-fix'

Closes #17118

Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas@di-pentima.com.ar>

diff --git a/build/run-library.sh b/build/run-library.sh
index 6f95a8f4bfd8cb9736a5b9fba6c8076005ce2de3..9efc8028b51f395d4e344bcd34dfb6489cb1374c 100755 (executable)
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -347,11 +347,11 @@ test_package_presence() {
         repo_subdir=${pkgname:0:1}
       fi
 
-      repo_pkg_list=$(curl -s -o - http://apt.arvados.org/pool/${D}-dev/main/${repo_subdir}/${pkgname}/)
+      repo_pkg_list=$(curl -s -o - http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/)
       echo "${repo_pkg_list}" |grep -q ${full_pkgname}
       if [ $? -eq 0 ] ; then
         echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
-        curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/pool/${D}-dev/main/${repo_subdir}/${pkgname}/${full_pkgname}
+        curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/${full_pkgname}
         return 1
       elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then
         echo "Package $full_pkgname exists, not rebuilding!"

diff --git a/doc/install/packages.html.textile.liquid b/doc/install/packages.html.textile.liquid
index cb7102bb3770ebaa74fba78317446c78c7c215a1..81866507bedb8ee870c112d2e3e262df709437e9 100644 (file)
--- a/doc/install/packages.html.textile.liquid
+++ b/doc/install/packages.html.textile.liquid
@@ -41,9 +41,9 @@ As root, add the Arvados package repository to your sources.  This command depen
 
 table(table table-bordered table-condensed).
 |_. OS version|_. Command|
-|Debian 10 ("buster")|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ buster main" &#x7c; tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
-|Ubuntu 18.04 ("bionic")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ bionic main" &#x7c; tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
-|Ubuntu 16.04 ("xenial")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ xenial main" &#x7c; tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
+|Debian 10 ("buster")|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/buster buster main" &#x7c; tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
+|Ubuntu 18.04 ("bionic")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/bionic bionic main" &#x7c; tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
+|Ubuntu 16.04 ("xenial")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/xenial xenial main" &#x7c; tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
 
 
 {% include 'notebox_begin' %}

diff --git a/docker/jobs/apt.arvados.org-dev.list b/docker/jobs/apt.arvados.org-dev.list
index 4de87397bca754a57e384c3155d88b82a30983fc..210f5d55119da35ff6e2060fa6dfddeb8099a54d 100644 (file)
--- a/docker/jobs/apt.arvados.org-dev.list
+++ b/docker/jobs/apt.arvados.org-dev.list
@@ -1,2 +1,2 @@
 # apt.arvados.org
-deb http://apt.arvados.org/ buster-dev main
+deb http://apt.arvados.org/buster buster-dev main

diff --git a/docker/jobs/apt.arvados.org-stable.list b/docker/jobs/apt.arvados.org-stable.list
index 7882afd01c96235b1fde32767d56a68aeada8d03..153e7298057eff34ccd2e4e8f39e2bcda978adf2 100644 (file)
--- a/docker/jobs/apt.arvados.org-stable.list
+++ b/docker/jobs/apt.arvados.org-stable.list
@@ -1,2 +1,2 @@
 # apt.arvados.org
-deb http://apt.arvados.org/ buster main
+deb http://apt.arvados.org/buster buster main

diff --git a/docker/jobs/apt.arvados.org-testing.list b/docker/jobs/apt.arvados.org-testing.list
index 3bb599087eaf513bb5c3f6dc2e32d54108d3db53..d5f458168585ada0d74aa36afa79d5a2bdbf9f31 100644 (file)
--- a/docker/jobs/apt.arvados.org-testing.list
+++ b/docker/jobs/apt.arvados.org-testing.list
@@ -1,2 +1,2 @@
 # apt.arvados.org
-deb http://apt.arvados.org/ buster-testing main
+deb http://apt.arvados.org/buster buster-testing main

diff --git a/docker/migrate-docker19/Dockerfile b/docker/migrate-docker19/Dockerfile
index a515ec45cbe7c4b538f44ef7144f730bd86e23df..23bb63547b404301367a98531dd7a19b1d0c6d25 100644 (file)
--- a/docker/migrate-docker19/Dockerfile
+++ b/docker/migrate-docker19/Dockerfile
@@ -14,7 +14,7 @@ RUN apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7 && \
 VOLUME /var/lib/docker
 
 RUN mkdir -p /etc/apt/sources.list.d && \
-    echo deb http://apt.arvados.org/ jessie main > /etc/apt/sources.list.d/apt.arvados.org.list && \
+    echo deb http://apt.arvados.org/jessie jessie main > /etc/apt/sources.list.d/apt.arvados.org.list && \
     apt-get clean && \
     apt-get update && \
     apt-get install -yq --no-install-recommends -o Acquire::Retries=6 \

diff --git a/lib/controller/integration_test.go b/lib/controller/integration_test.go
index 3418c1f81a1f9d7dfc4881fe300fd94fdaefc984..0388f21bee916ff4a0046971edfa9908553d00b1 100644 (file)
--- a/lib/controller/integration_test.go
+++ b/lib/controller/integration_test.go
@@ -353,7 +353,7 @@ func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) {
                buf, _ = exec.Command("s3cmd", append(s3args, "get", "s3://"+coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput()
                // Command fails because we don't return Etag header.
                flen := strconv.Itoa(len(testText))
-               c.Check(string(buf), check.Matches, `(?ms).*`+flen+` of `+flen+`.*`)
+               c.Check(string(buf), check.Matches, `(?ms).*`+flen+` (bytes in|of `+flen+`).*`)
        }
 }
 

diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index 46ff655dd6afc274d7172c8ae0d79cc675288177..e8dd186050fb200ca575a480ff6d15db1d88adf0 100644 (file)
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -120,7 +120,7 @@ Options:
 
        if len(uuids) < 1 {
                flags.Usage()
-               err = fmt.Errorf("Error: no uuid(s) provided")
+               err = fmt.Errorf("error: no uuid(s) provided")
                exitCode = 2
                return
        }
@@ -285,7 +285,7 @@ func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid str
 
 func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, cr arvados.ContainerRequest) (node nodeInfo, err error) {
        if cr.LogUUID == "" {
-               err = errors.New("No log collection")
+               err = errors.New("no log collection")
                return
        }
 
@@ -509,7 +509,7 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
                        var crCsv map[string]float64
                        crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
                        if err != nil {
-                               err = fmt.Errorf("Error generating CSV for uuid %s: %s", uuid, err.Error())
+                               err = fmt.Errorf("error generating CSV for uuid %s: %s", uuid, err.Error())
                                exitcode = 2
                                return
                        }
@@ -521,9 +521,9 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
                        // It is identified by the user uuid. As such, cost analysis for the
                        // "Home" project is not supported by this program. Skip this uuid, but
                        // keep going.
-                       logger.Errorf("Cost analysis is not supported for the 'Home' project: %s", uuid)
+                       logger.Errorf("cost analysis is not supported for the 'Home' project: %s", uuid)
                } else {
-                       logger.Errorf("This argument does not look like a uuid: %s\n", uuid)
+                       logger.Errorf("this argument does not look like a uuid: %s\n", uuid)
                        exitcode = 3
                        return
                }
@@ -554,7 +554,7 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
                aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
                err = ioutil.WriteFile(aFile, []byte(csv), 0644)
                if err != nil {
-                       err = fmt.Errorf("Error writing file with path %s: %s", aFile, err.Error())
+                       err = fmt.Errorf("error writing file with path %s: %s", aFile, err.Error())
                        exitcode = 1
                        return
                }

diff --git a/lib/dispatchcloud/test/ssh_service.go b/lib/dispatchcloud/test/ssh_service.go
index f1fde4f422ce55198742871883f8a0bbd7c682d3..31919b566df81769f791782b4cb3709b468122e8 100644 (file)
--- a/lib/dispatchcloud/test/ssh_service.go
+++ b/lib/dispatchcloud/test/ssh_service.go
@@ -18,6 +18,8 @@ import (
        check "gopkg.in/check.v1"
 )
 
+// LoadTestKey returns a public/private ssh keypair, read from the files
+// identified by the path of the private key.
 func LoadTestKey(c *check.C, fnm string) (ssh.PublicKey, ssh.Signer) {
        rawpubkey, err := ioutil.ReadFile(fnm + ".pub")
        c.Assert(err, check.IsNil)

diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index a81cf505294e0ce1bbc442da3b844d4cb5c5b073..9ca24312582060d42f6ed878f600c780ae9d5872 100644 (file)
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -165,8 +165,6 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
        return 0
 }
 
-const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
-
 func getListenAddr(svcs arvados.Services, prog arvados.ServiceName, log logrus.FieldLogger) (arvados.URL, error) {
        svc, ok := svcs.Map()[prog]
        if !ok {

diff --git a/lib/service/tls.go b/lib/service/tls.go
index db3c567eec5ed3da1e95efbe962e162dad573067..c6307b76ab02b79342cfa3395899c5f27ffd5f57 100644 (file)
--- a/lib/service/tls.go
+++ b/lib/service/tls.go
@@ -23,7 +23,7 @@ func tlsConfigWithCertUpdater(cluster *arvados.Cluster, logger logrus.FieldLogge
 
        key, cert := cluster.TLS.Key, cluster.TLS.Certificate
        if !strings.HasPrefix(key, "file://") || !strings.HasPrefix(cert, "file://") {
-               return nil, errors.New("cannot use TLS certificate: TLS.Key and TLS.Certificate must be specified as file://...")
+               return nil, errors.New("cannot use TLS certificate: TLS.Key and TLS.Certificate must be specified with a 'file://' prefix")
        }
        key, cert = key[7:], cert[7:]
 

diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index 54602fb54e4b9bee7a167b80127f1eba18064056..d90c618f7a1effa8c3da8031cb98909a1b37df3f 100644 (file)
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -210,7 +210,7 @@ func (c *ArvadosClient) CallRaw(method string, resourceType string, uuid string,
                Scheme: scheme,
                Host:   c.ApiServer}
 
-       if resourceType != API_DISCOVERY_RESOURCE {
+       if resourceType != ApiDiscoveryResource {
                u.Path = "/arvados/v1"
        }
 
@@ -400,7 +400,7 @@ func (c *ArvadosClient) List(resource string, parameters Dict, output interface{
        return c.Call("GET", resource, "", "", parameters, output)
 }
 
-const API_DISCOVERY_RESOURCE = "discovery/v1/apis/arvados/v1/rest"
+const ApiDiscoveryResource = "discovery/v1/apis/arvados/v1/rest"
 
 // Discovery returns the value of the given parameter in the discovery
 // document. Returns a non-nil error if the discovery document cannot
@@ -409,7 +409,7 @@ const API_DISCOVERY_RESOURCE = "discovery/v1/apis/arvados/v1/rest"
 func (c *ArvadosClient) Discovery(parameter string) (value interface{}, err error) {
        if len(c.DiscoveryDoc) == 0 {
                c.DiscoveryDoc = make(Dict)
-               err = c.Call("GET", API_DISCOVERY_RESOURCE, "", "", nil, &c.DiscoveryDoc)
+               err = c.Call("GET", ApiDiscoveryResource, "", "", nil, &c.DiscoveryDoc)
                if err != nil {
                        return nil, err
                }

diff --git a/sdk/go/manifest/manifest.go b/sdk/go/manifest/manifest.go
index 7a705eb8c2f6ac9c6a7fe71687f209e6f67fdc1a..954fb710c0596a4580f76bf2a78945e39203f2f6 100644 (file)
--- a/sdk/go/manifest/manifest.go
+++ b/sdk/go/manifest/manifest.go
@@ -529,6 +529,8 @@ func (m *Manifest) FileSegmentIterByName(filepath string) <-chan *FileSegment {
        return ch
 }
 
+// BlockIterWithDuplicates iterates over the block locators of a manifest.
+//
 // Blocks may appear multiple times within the same manifest if they
 // are used by multiple files. In that case this Iterator will output
 // the same block multiple times.

diff --git a/services/fuse/README.rst b/services/fuse/README.rst
index d91ae05a1158fc3f327162d4507914fb8cebed20..25f1cda3b25d17d812c33471967573dab68e4d56 100644 (file)
--- a/services/fuse/README.rst
+++ b/services/fuse/README.rst
@@ -38,7 +38,7 @@ Installing on Debian systems
 
 1. Add this Arvados repository to your sources list::
 
-     deb http://apt.arvados.org/ jessie main
+     deb http://apt.arvados.org/buster buster main
 
 2. Update your package list.
 

diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 97201d2922116bf7db8cd8368557b4247093a074..1c84976d2b1bf26622ba67619438f4536e6551f0 100644 (file)
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -75,6 +75,8 @@ func s3querystring(u *url.URL) string {
        return strings.Join(keys, "&")
 }
 
+var reMultipleSlashChars = regexp.MustCompile(`//+`)
+
 func s3stringToSign(alg, scope, signedHeaders string, r *http.Request) (string, error) {
        timefmt, timestr := "20060102T150405Z", r.Header.Get("X-Amz-Date")
        if timestr == "" {
@@ -97,7 +99,10 @@ func s3stringToSign(alg, scope, signedHeaders string, r *http.Request) (string,
                }
        }
 
-       canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", r.Method, r.URL.EscapedPath(), s3querystring(r.URL), canonicalHeaders, signedHeaders, r.Header.Get("X-Amz-Content-Sha256"))
+       normalizedURL := *r.URL
+       normalizedURL.RawPath = ""
+       normalizedURL.Path = reMultipleSlashChars.ReplaceAllString(normalizedURL.Path, "/")
+       canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", r.Method, normalizedURL.EscapedPath(), s3querystring(r.URL), canonicalHeaders, signedHeaders, r.Header.Get("X-Amz-Content-Sha256"))
        ctxlog.FromContext(r.Context()).Debugf("s3stringToSign: canonicalRequest %s", canonicalRequest)
        return fmt.Sprintf("%s\n%s\n%s\n%s", alg, r.Header.Get("X-Amz-Date"), scope, hashdigest(sha256.New(), canonicalRequest)), nil
 }
@@ -259,7 +264,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
                bucketName = strings.SplitN(strings.TrimPrefix(r.URL.Path, "/"), "/", 2)[0]
                objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
        }
-       fspath += r.URL.Path
+       fspath += reMultipleSlashChars.ReplaceAllString(r.URL.Path, "/")
 
        switch {
        case r.Method == http.MethodGet && !objectNameGiven:

diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index a6aab357e301e4b4703f2c72ef4a1a490ab65766..52ef79509759ecbafe57b75f085d5d3f57c7940e 100644 (file)
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -7,6 +7,7 @@ package main
 import (
        "bytes"
        "crypto/rand"
+       "crypto/sha256"
        "fmt"
        "io/ioutil"
        "net/http"
@@ -202,6 +203,11 @@ func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix
        c.Check(err, check.IsNil)
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        c.Check(resp.ContentLength, check.Equals, int64(4))
+
+       // HeadObject with superfluous leading slashes
+       exists, err = bucket.Exists(prefix + "//sailboat.txt")
+       c.Check(err, check.IsNil)
+       c.Check(exists, check.Equals, true)
 }
 
 func (s *IntegrationSuite) TestS3CollectionPutObjectSuccess(c *check.C) {
@@ -228,6 +234,18 @@ func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket,
                        path:        "newdir/newfile",
                        size:        1 << 26,
                        contentType: "application/octet-stream",
+               }, {
+                       path:        "/aaa",
+                       size:        2,
+                       contentType: "application/octet-stream",
+               }, {
+                       path:        "//bbb",
+                       size:        2,
+                       contentType: "application/octet-stream",
+               }, {
+                       path:        "ccc//",
+                       size:        0,
+                       contentType: "application/x-directory",
                }, {
                        path:        "newdir1/newdir2/newfile",
                        size:        0,
@@ -243,9 +261,14 @@ func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket,
                objname := prefix + trial.path
 
                _, err := bucket.GetReader(objname)
+               if !c.Check(err, check.NotNil) {
+                       continue
+               }
                c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
                c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
-               c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
+               if !c.Check(err, check.ErrorMatches, `The specified key does not exist.`) {
+                       continue
+               }
 
                buf := make([]byte, trial.size)
                rand.Read(buf)
@@ -363,14 +386,6 @@ func (s *IntegrationSuite) TestS3ProjectPutObjectFailure(c *check.C) {
 func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) {
        s.testServer.Config.cluster.Collections.S3FolderObjects = false
 
-       // Can't use V4 signature for these tests, because
-       // double-slash is incorrectly cleaned by the aws.V4Signature,
-       // resulting in a "bad signature" error. (Cleaning the path is
-       // appropriate for other services, but not in S3 where object
-       // names "foo//bar" and "foo/bar" are semantically different.)
-       bucket.S3.Auth = *(aws.NewAuth(arvadostest.ActiveToken, "none", "", time.Now().Add(time.Hour)))
-       bucket.S3.Signature = aws.V2Signature
-
        var wg sync.WaitGroup
        for _, trial := range []struct {
                path string
@@ -393,8 +408,6 @@ func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket,
                        path: "/",
                }, {
                        path: "//",
-               }, {
-                       path: "foo//bar",
                }, {
                        path: "",
                },
@@ -441,6 +454,17 @@ func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) {
        c.Assert(fs.Sync(), check.IsNil)
 }
 
+func (s *IntegrationSuite) sign(c *check.C, req *http.Request, key, secret string) {
+       scope := "20200202/region/service/aws4_request"
+       signedHeaders := "date"
+       req.Header.Set("Date", time.Now().UTC().Format(time.RFC1123))
+       stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, req)
+       c.Assert(err, check.IsNil)
+       sig, err := s3signature(secret, scope, signedHeaders, stringToSign)
+       c.Assert(err, check.IsNil)
+       req.Header.Set("Authorization", s3SignAlgorithm+" Credential="+key+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig)
+}
+
 func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
        stage := s.s3setup(c)
        defer stage.teardown(c)
@@ -487,12 +511,29 @@ func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
                        responseCode:   http.StatusOK,
                        responseRegexp: []string{`boop`},
                },
+               {
+                       url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
+                       method:       "GET",
+                       responseCode: http.StatusNotFound,
+               },
+               {
+                       url:          "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
+                       method:       "PUT",
+                       body:         "boop",
+                       responseCode: http.StatusOK,
+               },
+               {
+                       url:            "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
+                       method:         "GET",
+                       responseCode:   http.StatusOK,
+                       responseRegexp: []string{`boop`},
+               },
        } {
                url, err := url.Parse(trial.url)
                c.Assert(err, check.IsNil)
                req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body)))
                c.Assert(err, check.IsNil)
-               req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
+               s.sign(c, req, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken)
                rr := httptest.NewRecorder()
                s.testServer.Server.Handler.ServeHTTP(rr, req)
                resp := rr.Result()
@@ -505,6 +546,38 @@ func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
        }
 }
 
+func (s *IntegrationSuite) TestS3NormalizeURIForSignature(c *check.C) {
+       stage := s.s3setup(c)
+       defer stage.teardown(c)
+       for _, trial := range []struct {
+               rawPath        string
+               normalizedPath string
+       }{
+               {"/foo", "/foo"},             // boring case
+               {"/foo%5fbar", "/foo_bar"},   // _ must not be escaped
+               {"/foo%2fbar", "/foo/bar"},   // / must not be escaped
+               {"/(foo)", "/%28foo%29"},     // () must be escaped
+               {"/foo%5bbar", "/foo%5Bbar"}, // %XX must be uppercase
+       } {
+               date := time.Now().UTC().Format("20060102T150405Z")
+               scope := "20200202/fakeregion/S3/aws4_request"
+               canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", "GET", trial.normalizedPath, "", "host:host.example.com\n", "host", "")
+               c.Logf("canonicalRequest %q", canonicalRequest)
+               expect := fmt.Sprintf("%s\n%s\n%s\n%s", s3SignAlgorithm, date, scope, hashdigest(sha256.New(), canonicalRequest))
+               c.Logf("expected stringToSign %q", expect)
+
+               req, err := http.NewRequest("GET", "https://host.example.com"+trial.rawPath, nil)
+               req.Header.Set("X-Amz-Date", date)
+               req.Host = "host.example.com"
+
+               obtained, err := s3stringToSign(s3SignAlgorithm, scope, "host", req)
+               if !c.Check(err, check.IsNil) {
+                       continue
+               }
+               c.Check(obtained, check.Equals, expect)
+       }
+}
+
 func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
        stage := s.s3setup(c)
        defer stage.teardown(c)

diff --git a/tools/compute-images/scripts/base.sh b/tools/compute-images/scripts/base.sh
index 73c7b9dac05d73144a668d1d5b7c8e538ffcc1af..78cbccdb98fa6856b464c0411e086d11f94d2c54 100644 (file)
--- a/tools/compute-images/scripts/base.sh
+++ b/tools/compute-images/scripts/base.sh
@@ -29,7 +29,7 @@ LSB_RELEASE_CODENAME=${TMP_LSB//[$'\t\r\n ']}
 
 # Add the arvados apt repository
 echo "# apt.arvados.org" |$SUDO tee --append /etc/apt/sources.list.d/apt.arvados.org.list
-echo "deb http://apt.arvados.org/ $LSB_RELEASE_CODENAME${REPOSUFFIX} main" |$SUDO tee --append /etc/apt/sources.list.d/apt.arvados.org.list
+echo "deb http://apt.arvados.org/$LSB_RELEASE_CODENAME $LSB_RELEASE_CODENAME${REPOSUFFIX} main" |$SUDO tee --append /etc/apt/sources.list.d/apt.arvados.org.list
 
 # Add the arvados signing key
 cat /tmp/1078ECD7.asc | $SUDO apt-key add -