esac
done
-start_api() {
- echo 'Starting API server...'
+start_services() {
+ echo 'Starting API, keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
mkdir -p "$WORKSPACE/services/api/log"
fi
&& eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
&& export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
&& export ARVADOS_TEST_API_INSTALLED="$$" \
- && python sdk/python/tests/run_test_server.py start_ws \
- && python sdk/python/tests/run_test_server.py start_nginx \
- && (env | egrep ^ARVADOS)
-}
-
-start_nginx_proxy_services() {
- echo 'Starting keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
- cd "$WORKSPACE" \
&& python sdk/python/tests/run_test_server.py start_keep_proxy \
&& python sdk/python/tests/run_test_server.py start_keep-web \
&& python sdk/python/tests/run_test_server.py start_arv-git-httpd \
&& python sdk/python/tests/run_test_server.py start_ws \
&& python sdk/python/tests/run_test_server.py start_nginx \
- && export ARVADOS_TEST_PROXY_SERVICES=1
+ && (env | egrep ^ARVADOS)
}
stop_services() {
- if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
- unset ARVADOS_TEST_PROXY_SERVICES
- cd "$WORKSPACE" \
- && python sdk/python/tests/run_test_server.py stop_nginx \
- && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
- && python sdk/python/tests/run_test_server.py stop_ws \
- && python sdk/python/tests/run_test_server.py stop_keep-web \
- && python sdk/python/tests/run_test_server.py stop_keep_proxy
- fi
- if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
- unset ARVADOS_TEST_API_HOST
- cd "$WORKSPACE" \
- && python sdk/python/tests/run_test_server.py stop_nginx \
- && python sdk/python/tests/run_test_server.py stop_ws \
- && python sdk/python/tests/run_test_server.py stop
+ if [[ -z "$ARVADOS_TEST_API_HOST" ]]; then
+ return
fi
+ unset ARVADOS_TEST_API_HOST
+ cd "$WORKSPACE" \
+ && python sdk/python/tests/run_test_server.py stop_nginx \
+ && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
+ && python sdk/python/tests/run_test_server.py stop_ws \
+ && python sdk/python/tests/run_test_server.py stop_keep-web \
+ && python sdk/python/tests/run_test_server.py stop_keep_proxy \
+ && python sdk/python/tests/run_test_server.py stop
}
interrupt() {
) || return 1
fi
+ cd "$WORKSPACE/services/api" \
+ && rm -rf tmp/git \
+ && mkdir -p tmp/git \
+ && cd tmp/git \
+ && tar xf ../../test/test.git.tar \
+ && mkdir -p internal.git \
+ && git --git-dir internal.git init \
+ || return 1
+
cd "$WORKSPACE/services/api" \
&& RAILS_ENV=test bundle exec rake db:drop \
&& RAILS_ENV=test bundle exec rake db:setup \
exit_cleanly
fi
-start_api || { stop_services; fatal "start_api"; }
+start_services || { stop_services; fatal "start_services"; }
test_ruby_sdk() {
cd "$WORKSPACE/sdk/ruby" \
done
test_workbench_units() {
- start_nginx_proxy_services \
- && cd "$WORKSPACE/apps/workbench" \
+ cd "$WORKSPACE/apps/workbench" \
&& env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS=-v ${testargs[apps/workbench]}
}
do_test apps/workbench_units workbench_units
test_workbench_functionals() {
- start_nginx_proxy_services \
- && cd "$WORKSPACE/apps/workbench" \
+ cd "$WORKSPACE/apps/workbench" \
&& env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS=-v ${testargs[apps/workbench]}
}
do_test apps/workbench_functionals workbench_functionals
test_workbench_integration() {
- start_nginx_proxy_services \
- && cd "$WORKSPACE/apps/workbench" \
+ cd "$WORKSPACE/apps/workbench" \
&& env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS=-v ${testargs[apps/workbench]}
}
do_test apps/workbench_integration workbench_integration
test_workbench_benchmark() {
- start_nginx_proxy_services \
- && cd "$WORKSPACE/apps/workbench" \
+ cd "$WORKSPACE/apps/workbench" \
&& env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]}
}
do_test apps/workbench_benchmark workbench_benchmark
test_workbench_profile() {
- start_nginx_proxy_services \
- && cd "$WORKSPACE/apps/workbench" \
+ cd "$WORKSPACE/apps/workbench" \
&& env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:profile ${testargs[apps/workbench_profile]}
}
do_test apps/workbench_profile workbench_profile
"kind":"collection",
"uuid":"..."
}</code></pre>|
-|Git tree|@git_tree@|One of { @"git-url"@, @"repository_name"@, @"uuid"@ } must be provided.
-One of { @"commit"@, @"revisions"@ } must be provided.
-"path" may be provided. The default path is "/".
-At container startup, the target path will have the source tree indicated by the given revision. The @.git@ metadata directory _will not_ be available: typically the system will use @git-archive@ rather than @git-checkout@ to prepare the target directory.
-- If a value is given for @"revisions"@, it will be resolved to a set of commits (as desribed in the "ranges" section of git-revisions(1)) and the container request will be satisfiable by any commit in that set.
-- If a value is given for @"commit"@, it will be resolved to a single commit, and the tree resulting from that commit will be used.
-- @"path"@ can be used to select a subdirectory or a single file from the tree indicated by the selected commit.
-- Multiple commits can resolve to the same tree: for example, the file/directory given in @"path"@ might not have changed between commits A and B.
-- The resolved mount (found in the Container record) will have only the "kind" key and a "blob" or "tree" key indicating the 40-character hash of the git tree/blob used.|<pre><code>{
+|Git tree|@git_tree@|@"uuid"@ must be the UUID of an Arvados-hosted git repository.
+@"commit"@ must be a full 40-character commit hash.
+@"path"@, if provided, must be "/".
+At container startup, the target path will have the source tree indicated by the given commit. The @.git@ metadata directory _will not_ be available.|<pre><code>{
"kind":"git_tree",
"uuid":"zzzzz-s0uqq-xxxxxxxxxxxxxxx",
- "commit":"master"
+ "commit":"f315c59f90934cccae6381e72bba59d27ba42099"
}
-{
- "kind":"git_tree",
- "uuid":"zzzzz-s0uqq-xxxxxxxxxxxxxxx",
- "commit_range":"bugfix^..master",
- "path":"/crunch_scripts/grep"
-}</code></pre>|
+</code></pre>|
|Temporary directory|@tmp@|@"capacity"@: capacity (in bytes) of the storage device.
@"device_type"@ (optional, default "network"): one of @{"ram", "ssd", "disk", "network"}@ indicating the acceptable level of performance.
At container startup, the target path will be empty. When the container finishes, the content will be discarded. This will be backed by a storage mechanism no slower than the specified type.|<pre><code>{
BasePath string `json:"basePath"`
DefaultCollectionReplication int `json:"defaultCollectionReplication"`
BlobSignatureTTL int64 `json:"blobSignatureTtl"`
+ GitURL string `json:"gitUrl"`
Schemas map[string]Schema `json:"schemas"`
Resources map[string]Resource `json:"resources"`
}
Content interface{} `json:"content"`
ExcludeFromOutput bool `json:"exclude_from_output"`
Capacity int64 `json:"capacity"`
+ Commit string `json:"commit"` // only if kind=="git_tree"
+ RepositoryName string `json:"repository_name"` // only if kind=="git_tree"
+ GitURL string `json:"git_url"` // only if kind=="git_tree"
}
// RuntimeConstraints specify a container's compute resources (RAM,
Dispatch1AuthUUID = "zzzzz-gj3su-k9dvestay1plssr"
QueuedContainerUUID = "zzzzz-dz642-queuedcontainer"
+
+ ArvadosRepoUUID = "zzzzz-s0uqq-arvadosrepo0123"
+ ArvadosRepoName = "arvados"
+ FooRepoUUID = "zzzzz-s0uqq-382brsig8rp3666"
+ FooRepoName = "active/foo"
+ Repository2UUID = "zzzzz-s0uqq-382brsig8rp3667"
+ Repository2Name = "active/foo2"
)
// PathologicalManifest : A valid manifest designed to test
websocketUrl: Rails.application.config.websocket_address,
workbenchUrl: Rails.application.config.workbench_address,
keepWebServiceUrl: Rails.application.config.keep_web_service_url,
+ gitUrl: case Rails.application.config.git_repo_https_base
+ when false
+ ''
+ when true
+ 'https://git.%s.arvadosapi.com/' % Rails.configuration.uuid_prefix
+ else
+ Rails.application.config.git_repo_https_base
+ end,
parameters: {
alt: {
type: "string",
end
def self.cache_dir_base
- Rails.root.join 'tmp', 'git'
+ Rails.root.join 'tmp', 'git-cache'
end
def self.fetch_remote_repository gitdir, git_url
FileUtils.mkdir_p @tmpdir
system("tar", "-xC", @tmpdir.to_s, "-f", "test/test.git.tar")
Rails.configuration.git_repositories_dir = "#{@tmpdir}/test"
-
- # Initialize an empty internal git repo.
- intdir =
- Rails.configuration.git_internal_dir =
- Rails.root.join(@tmpdir, 'internal.git').to_s
- FileUtils.mkdir_p intdir
- IO.read("|git --git-dir #{intdir.shellescape} init")
- assert $?.success?
+ Rails.configuration.git_internal_dir = "#{@tmpdir}/internal.git"
end
base.teardown do
- FileUtils.remove_entry @tmpdir, true
FileUtils.remove_entry Commit.cache_dir_base, true
+ FileUtils.mkdir_p @tmpdir
+ system("tar", "-xC", @tmpdir.to_s, "-f", "test/test.git.tar")
end
end
package main
import (
+ "errors"
"log"
"net/http"
"os"
+ "regexp"
"strings"
"sync"
"time"
log.Fatal(err)
}
h.clientPool = &arvadosclient.ClientPool{Prototype: ac}
- log.Printf("%+v", h.clientPool.Prototype)
}
func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// Nobody has called WriteHeader yet: that
// must be our job.
w.WriteHeader(statusCode)
- w.Write([]byte(statusText))
+ if statusCode >= 400 {
+ w.Write([]byte(statusText))
+ }
}
// If the given password is a valid token, log the first 10 characters of the token.
// Ask API server whether the repository is readable using
// this token (by trying to read it!)
arv.ApiToken = apiToken
- reposFound := arvadosclient.Dict{}
- if err := arv.List("repositories", arvadosclient.Dict{
- "filters": [][]string{{"name", "=", repoName}},
- }, &reposFound); err != nil {
+ repoUUID, err := h.lookupRepo(arv, repoName)
+ if err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
validApiToken = true
- if avail, ok := reposFound["items_available"].(float64); !ok {
- statusCode, statusText = http.StatusInternalServerError, "bad list response from API"
- return
- } else if avail < 1 {
+ if repoUUID == "" {
statusCode, statusText = http.StatusNotFound, "not found"
return
- } else if avail > 1 {
- statusCode, statusText = http.StatusInternalServerError, "name collision"
- return
}
- repoUUID := reposFound["items"].([]interface{})[0].(map[string]interface{})["uuid"].(string)
-
isWrite := strings.HasSuffix(r.URL.Path, "/git-receive-pack")
if !isWrite {
statusText = "read"
h.handler.ServeHTTP(w, r)
}
+
+var uuidRegexp = regexp.MustCompile(`^[0-9a-z]{5}-s0uqq-[0-9a-z]{15}$`)
+
+func (h *authHandler) lookupRepo(arv *arvadosclient.ArvadosClient, repoName string) (string, error) {
+ reposFound := arvadosclient.Dict{}
+ var column string
+ if uuidRegexp.MatchString(repoName) {
+ column = "uuid"
+ } else {
+ column = "name"
+ }
+ err := arv.List("repositories", arvadosclient.Dict{
+ "filters": [][]string{{column, "=", repoName}},
+ }, &reposFound)
+ if err != nil {
+ return "", err
+ } else if avail, ok := reposFound["items_available"].(float64); !ok {
+ return "", errors.New("bad list response from API")
+ } else if avail < 1 {
+ return "", nil
+ } else if avail > 1 {
+ return "", errors.New("name collision")
+ }
+ return reposFound["items"].([]interface{})[0].(map[string]interface{})["uuid"].(string), nil
+}
package main
import (
+ "io"
+ "log"
"net/http"
"net/http/httptest"
"net/url"
+ "path/filepath"
+ "strings"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
)
type AuthHandlerSuite struct{}
+func (s *AuthHandlerSuite) SetUpSuite(c *check.C) {
+ arvadostest.StartAPI()
+}
+
+func (s *AuthHandlerSuite) TearDownSuite(c *check.C) {
+ arvadostest.StopAPI()
+}
+
+func (s *AuthHandlerSuite) SetUpTest(c *check.C) {
+ arvadostest.ResetEnv()
+ repoRoot, err := filepath.Abs("../api/tmp/git/test")
+ c.Assert(err, check.IsNil)
+ theConfig = &Config{
+ Client: arvados.Client{
+ APIHost: arvadostest.APIHost(),
+ Insecure: true,
+ },
+ Listen: ":0",
+ GitCommand: "/usr/bin/git",
+ RepoRoot: repoRoot,
+ ManagementToken: arvadostest.ManagementToken,
+ }
+}
+
+func (s *AuthHandlerSuite) TestPermission(c *check.C) {
+ h := &authHandler{handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ log.Printf("%v", r.URL)
+ io.WriteString(w, r.URL.Path)
+ })}
+ baseURL, err := url.Parse("http://git.example/")
+ c.Assert(err, check.IsNil)
+ for _, trial := range []struct {
+ label string
+ token string
+ pathIn string
+ pathOut string
+ status int
+ }{
+ {
+ label: "read repo by name",
+ token: arvadostest.ActiveToken,
+ pathIn: arvadostest.Repository2Name + ".git/git-upload-pack",
+ pathOut: arvadostest.Repository2UUID + ".git/git-upload-pack",
+ },
+ {
+ label: "read repo by uuid",
+ token: arvadostest.ActiveToken,
+ pathIn: arvadostest.Repository2UUID + ".git/git-upload-pack",
+ pathOut: arvadostest.Repository2UUID + ".git/git-upload-pack",
+ },
+ {
+ label: "write repo by name",
+ token: arvadostest.ActiveToken,
+ pathIn: arvadostest.Repository2Name + ".git/git-receive-pack",
+ pathOut: arvadostest.Repository2UUID + ".git/git-receive-pack",
+ },
+ {
+ label: "write repo by uuid",
+ token: arvadostest.ActiveToken,
+ pathIn: arvadostest.Repository2UUID + ".git/git-receive-pack",
+ pathOut: arvadostest.Repository2UUID + ".git/git-receive-pack",
+ },
+ {
+ label: "uuid not found",
+ token: arvadostest.ActiveToken,
+ pathIn: strings.Replace(arvadostest.Repository2UUID, "6", "z", -1) + ".git/git-upload-pack",
+ status: http.StatusNotFound,
+ },
+ {
+ label: "name not found",
+ token: arvadostest.ActiveToken,
+ pathIn: "nonexistent-bogus.git/git-upload-pack",
+ status: http.StatusNotFound,
+ },
+ {
+ label: "read read-only repo",
+ token: arvadostest.SpectatorToken,
+ pathIn: arvadostest.FooRepoName + ".git/git-upload-pack",
+ pathOut: arvadostest.FooRepoUUID + "/.git/git-upload-pack",
+ },
+ {
+ label: "write read-only repo",
+ token: arvadostest.SpectatorToken,
+ pathIn: arvadostest.FooRepoName + ".git/git-receive-pack",
+ status: http.StatusForbidden,
+ },
+ } {
+ c.Logf("trial label: %q", trial.label)
+ u, err := baseURL.Parse(trial.pathIn)
+ c.Assert(err, check.IsNil)
+ resp := httptest.NewRecorder()
+ req := &http.Request{
+ Method: "POST",
+ URL: u,
+ Header: http.Header{
+ "Authorization": {"Bearer " + trial.token}}}
+ h.ServeHTTP(resp, req)
+ if trial.status == 0 {
+ trial.status = http.StatusOK
+ }
+ c.Check(resp.Code, check.Equals, trial.status)
+ if trial.status < 400 {
+ if trial.pathOut != "" && !strings.HasPrefix(trial.pathOut, "/") {
+ trial.pathOut = "/" + trial.pathOut
+ }
+ c.Check(resp.Body.String(), check.Equals, trial.pathOut)
+ }
+ }
+}
+
func (s *AuthHandlerSuite) TestCORS(c *check.C) {
h := &authHandler{}
return fmt.Errorf("While creating keep mount temp dir: %v", err)
}
+ token, err := runner.ContainerToken()
+ if err != nil {
+ return fmt.Errorf("could not get container token: %s", err)
+ }
+
pdhOnly := true
tmpcount := 0
arvMountCmd := []string{
return fmt.Errorf("writing temp file: %v", err)
}
runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s:ro", tmpfn, bind))
+
+ case mnt.Kind == "git_tree":
+ tmpdir, err := runner.MkTempDir("", "")
+ if err != nil {
+ return fmt.Errorf("creating temp dir: %v", err)
+ }
+ runner.CleanupTempDir = append(runner.CleanupTempDir, tmpdir)
+ err = gitMount(mnt).extractTree(runner.ArvClient, tmpdir, token)
+ if err != nil {
+ return err
+ }
+ runner.Binds = append(runner.Binds, tmpdir+":"+bind+":ro")
}
}
}
arvMountCmd = append(arvMountCmd, runner.ArvMountPoint)
- token, err := runner.ContainerToken()
- if err != nil {
- return fmt.Errorf("could not get container token: %s", err)
- }
-
runner.ArvMount, err = runner.RunArvMount(arvMountCmd, token)
if err != nil {
return fmt.Errorf("While trying to start arv-mount: %v", err)
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/manifest"
dockertypes "github.com/docker/docker/api/types"
cr.CleanupDirs()
checkEmpty()
}
+
+ // git_tree mounts
+ {
+ i = 0
+ cr.ArvMountPoint = ""
+ (*GitMountSuite)(nil).useTestGitServer(c)
+ cr.token = arvadostest.ActiveToken
+ cr.Container.Mounts = make(map[string]arvados.Mount)
+ cr.Container.Mounts = map[string]arvados.Mount{
+ "/tip": {
+ Kind: "git_tree",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
+ Path: "/",
+ },
+ "/non-tip": {
+ Kind: "git_tree",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ Path: "/",
+ },
+ }
+ cr.OutputPath = "/tmp"
+
+ err := cr.SetupMounts()
+ c.Check(err, IsNil)
+
+ // dirMap[mountpoint] == tmpdir
+ dirMap := make(map[string]string)
+ for _, bind := range cr.Binds {
+ tokens := strings.Split(bind, ":")
+ dirMap[tokens[1]] = tokens[0]
+
+ if cr.Container.Mounts[tokens[1]].Writable {
+ c.Check(len(tokens), Equals, 2)
+ } else {
+ c.Check(len(tokens), Equals, 3)
+ c.Check(tokens[2], Equals, "ro")
+ }
+ }
+
+ data, err := ioutil.ReadFile(dirMap["/tip"] + "/dir1/dir2/file with mode 0644")
+ c.Check(err, IsNil)
+ c.Check(string(data), Equals, "\000\001\002\003")
+ _, err = ioutil.ReadFile(dirMap["/tip"] + "/file only on testbranch")
+ c.Check(err, FitsTypeOf, &os.PathError{})
+ c.Check(os.IsNotExist(err), Equals, true)
+
+ data, err = ioutil.ReadFile(dirMap["/non-tip"] + "/dir1/dir2/file with mode 0644")
+ c.Check(err, IsNil)
+ c.Check(string(data), Equals, "\000\001\002\003")
+ data, err = ioutil.ReadFile(dirMap["/non-tip"] + "/file only on testbranch")
+ c.Check(err, IsNil)
+ c.Check(string(data), Equals, "testfile\n")
+
+ cr.CleanupDirs()
+ checkEmpty()
+ }
}
func (s *TestSuite) TestStdout(c *C) {
--- /dev/null
+package main
+
+import (
+ "fmt"
+ "net/url"
+ "os"
+ "path/filepath"
+ "regexp"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "gopkg.in/src-d/go-billy.v3/osfs"
+ git "gopkg.in/src-d/go-git.v4"
+ git_config "gopkg.in/src-d/go-git.v4/config"
+ git_plumbing "gopkg.in/src-d/go-git.v4/plumbing"
+ git_http "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
+ "gopkg.in/src-d/go-git.v4/storage/memory"
+)
+
+type gitMount arvados.Mount
+
+var (
+ sha1re = regexp.MustCompile(`^[0-9a-f]{40}$`)
+ repoUUIDre = regexp.MustCompile(`^[0-9a-z]{5}-s0uqq-[0-9a-z]{15}$`)
+)
+
+func (gm gitMount) validate() error {
+ if gm.Path != "" && gm.Path != "/" {
+ return fmt.Errorf("cannot mount git_tree with path %q -- only \"/\" is supported", gm.Path)
+ }
+ if !sha1re.MatchString(gm.Commit) {
+ return fmt.Errorf("cannot mount git_tree with commit %q -- must be a 40-char SHA1", gm.Commit)
+ }
+ if gm.RepositoryName != "" || gm.GitURL != "" {
+ return fmt.Errorf("cannot mount git_tree -- repository_name and git_url must be empty")
+ }
+ if !repoUUIDre.MatchString(gm.UUID) {
+ return fmt.Errorf("cannot mount git_tree with uuid %q -- must be a repository UUID", gm.UUID)
+ }
+ if gm.Writable {
+ return fmt.Errorf("writable git_tree mount is not supported")
+ }
+ return nil
+}
+
+// ExtractTree extracts the specified tree into dir, which is an
+// existing empty local directory.
+func (gm gitMount) extractTree(ac IArvadosClient, dir string, token string) error {
+ err := gm.validate()
+ if err != nil {
+ return err
+ }
+ baseURL, err := ac.Discovery("gitUrl")
+ if err != nil {
+ return fmt.Errorf("discover gitUrl from API: %s", err)
+ } else if _, ok := baseURL.(string); !ok {
+ return fmt.Errorf("discover gitUrl from API: expected string, found %T", baseURL)
+ }
+
+ u, err := url.Parse(baseURL.(string))
+ if err != nil {
+ return fmt.Errorf("parse gitUrl %q: %s", baseURL, err)
+ }
+ u, err = u.Parse("/" + gm.UUID + ".git")
+ if err != nil {
+ return fmt.Errorf("build git url from %q, %q: %s", baseURL, gm.UUID, err)
+ }
+ store := memory.NewStorage()
+ repo, err := git.Init(store, osfs.New(dir))
+ if err != nil {
+ return fmt.Errorf("init repo: %s", err)
+ }
+ _, err = repo.CreateRemote(&git_config.RemoteConfig{
+ Name: "origin",
+ URLs: []string{u.String()},
+ })
+ if err != nil {
+ return fmt.Errorf("create remote %q: %s", u.String(), err)
+ }
+ err = repo.Fetch(&git.FetchOptions{
+ RemoteName: "origin",
+ Auth: git_http.NewBasicAuth("none", token),
+ })
+ if err != nil {
+ return fmt.Errorf("git fetch %q: %s", u.String(), err)
+ }
+ wt, err := repo.Worktree()
+ if err != nil {
+ return fmt.Errorf("worktree failed: %s", err)
+ }
+ err = wt.Checkout(&git.CheckoutOptions{
+ Hash: git_plumbing.NewHash(gm.Commit),
+ })
+ if err != nil {
+ return fmt.Errorf("checkout failed: %s", err)
+ }
+ err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ // copy user rx bits to group and other, in case
+ // prevailing umask is more restrictive than 022
+ mode := info.Mode()
+ mode = mode | ((mode >> 3) & 050) | ((mode >> 6) & 5)
+ return os.Chmod(path, mode)
+ })
+ if err != nil {
+ return fmt.Errorf("chmod -R %q: %s", dir, err)
+ }
+ return nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "io/ioutil"
+ "os"
+ "path/filepath"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+ git_client "gopkg.in/src-d/go-git.v4/plumbing/transport/client"
+ git_http "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
+)
+
+type GitMountSuite struct {
+ tmpdir string
+}
+
+var _ = check.Suite(&GitMountSuite{})
+
+func (s *GitMountSuite) SetUpTest(c *check.C) {
+ s.useTestGitServer(c)
+
+ var err error
+ s.tmpdir, err = ioutil.TempDir("", "")
+ c.Assert(err, check.IsNil)
+}
+
+func (s *GitMountSuite) TearDownTest(c *check.C) {
+ err := os.RemoveAll(s.tmpdir)
+ c.Check(err, check.IsNil)
+}
+
+// Commit fd3531f is crunch-run-tree-test
+func (s *GitMountSuite) TestextractTree(c *check.C) {
+ gm := gitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
+ }
+ err := gm.extractTree(&ArvTestClient{}, s.tmpdir, arvadostest.ActiveToken)
+ c.Check(err, check.IsNil)
+
+ fnm := filepath.Join(s.tmpdir, "dir1/dir2/file with mode 0644")
+ data, err := ioutil.ReadFile(fnm)
+ c.Check(err, check.IsNil)
+ c.Check(data, check.DeepEquals, []byte{0, 1, 2, 3})
+ fi, err := os.Stat(fnm)
+ c.Check(err, check.IsNil)
+ if err == nil {
+ c.Check(fi.Mode(), check.Equals, os.FileMode(0644))
+ }
+
+ fnm = filepath.Join(s.tmpdir, "dir1/dir2/file with mode 0755")
+ data, err = ioutil.ReadFile(fnm)
+ c.Check(err, check.IsNil)
+ c.Check(string(data), check.DeepEquals, "#!/bin/sh\nexec echo OK\n")
+ fi, err = os.Stat(fnm)
+ c.Check(err, check.IsNil)
+ if err == nil {
+ c.Check(fi.Mode(), check.Equals, os.FileMode(0755))
+ }
+
+ // Ensure there's no extra stuff like a ".git" dir
+ s.checkTmpdirContents(c, []string{"dir1"})
+
+ // Ensure tmpdir is world-readable and world-executable so the
+ // UID inside the container can use it.
+ fi, err = os.Stat(s.tmpdir)
+ c.Check(err, check.IsNil)
+ c.Check(fi.Mode()&os.ModePerm, check.Equals, os.FileMode(0755))
+}
+
+// Commit 5ebfab0 is not the tip of any branch or tag, but is
+// reachable in branch "crunch-run-non-tip-test".
+func (s *GitMountSuite) TestExtractNonTipCommit(c *check.C) {
+ gm := gitMount{
+ UUID: arvadostest.Repository2UUID,
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ }
+ err := gm.extractTree(&ArvTestClient{}, s.tmpdir, arvadostest.ActiveToken)
+ c.Check(err, check.IsNil)
+
+ fnm := filepath.Join(s.tmpdir, "file only on testbranch")
+ data, err := ioutil.ReadFile(fnm)
+ c.Check(err, check.IsNil)
+ c.Check(string(data), check.DeepEquals, "testfile\n")
+}
+
+func (s *GitMountSuite) TestNonexistentRepository(c *check.C) {
+ gm := gitMount{
+ Path: "/",
+ UUID: "zzzzz-s0uqq-nonexistentrepo",
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ }
+ err := gm.extractTree(&ArvTestClient{}, s.tmpdir, arvadostest.ActiveToken)
+ c.Check(err, check.NotNil)
+ c.Check(err, check.ErrorMatches, ".*repository not found.*")
+
+ s.checkTmpdirContents(c, []string{})
+}
+
+func (s *GitMountSuite) TestNonexistentCommit(c *check.C) {
+ gm := gitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "bb66b6bb6b6bbb6b6b6b66b6b6b6b6b6b6b6b66b",
+ }
+ err := gm.extractTree(&ArvTestClient{}, s.tmpdir, arvadostest.ActiveToken)
+ c.Check(err, check.NotNil)
+ c.Check(err, check.ErrorMatches, ".*object not found.*")
+
+ s.checkTmpdirContents(c, []string{})
+}
+
+func (s *GitMountSuite) TestGitUrlDiscoveryFails(c *check.C) {
+ delete(discoveryMap, "gitUrl")
+ gm := gitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ }
+ err := gm.extractTree(&ArvTestClient{}, s.tmpdir, arvadostest.ActiveToken)
+ c.Check(err, check.ErrorMatches, ".*gitUrl.*")
+}
+
+func (s *GitMountSuite) TestInvalid(c *check.C) {
+ for _, trial := range []struct {
+ gm gitMount
+ matcher string
+ }{
+ {
+ gm: gitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "abc123",
+ },
+ matcher: ".*SHA1.*",
+ },
+ {
+ gm: gitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ RepositoryName: arvadostest.Repository2Name,
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ },
+ matcher: ".*repository_name.*",
+ },
+ {
+ gm: gitMount{
+ Path: "/",
+ GitURL: "https://localhost:0/" + arvadostest.Repository2Name + ".git",
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ },
+ matcher: ".*git_url.*",
+ },
+ {
+ gm: gitMount{
+ Path: "/dir1/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ },
+ matcher: ".*path.*",
+ },
+ {
+ gm: gitMount{
+ Path: "/",
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ },
+ matcher: ".*UUID.*",
+ },
+ {
+ gm: gitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ Writable: true,
+ },
+ matcher: ".*writable.*",
+ },
+ } {
+ err := trial.gm.extractTree(&ArvTestClient{}, s.tmpdir, arvadostest.ActiveToken)
+ c.Check(err, check.NotNil)
+ s.checkTmpdirContents(c, []string{})
+
+ err = trial.gm.validate()
+ c.Check(err, check.ErrorMatches, trial.matcher)
+ }
+}
+
+func (s *GitMountSuite) checkTmpdirContents(c *check.C, expect []string) {
+ f, err := os.Open(s.tmpdir)
+ c.Check(err, check.IsNil)
+ names, err := f.Readdirnames(-1)
+ c.Check(err, check.IsNil)
+ c.Check(names, check.DeepEquals, expect)
+}
+
+func (*GitMountSuite) useTestGitServer(c *check.C) {
+ git_client.InstallProtocol("https", git_http.NewClient(arvados.InsecureHTTPClient))
+
+ port, err := ioutil.ReadFile("../../tmp/arv-git-httpd-ssl.port")
+ c.Assert(err, check.IsNil)
+ discoveryMap["gitUrl"] = "https://localhost:" + string(port)
+}
projects / arvados.git / commitdiff