keepstore
keep-web
libarvados-perl
+ libpam-arvados
+ libpam-arvados-go
python-arvados-fuse
python-arvados-python-client
python-arvados-cwl-runner"
"Copy all data from one set of Keep servers to another"
package_go_binary tools/keep-exercise keep-exercise \
"Performance testing tool for Arvados Keep"
+package_go_so lib/pam pam_arvados.so libpam-arvados-go \
+ "Arvados PAM authentication module"
# The Python SDK - Should be built first because it's needed by others
fpm_build_virtualenv "arvados-python-client" "sdk/python"
__returnvar="$version"
}
-# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision"
+# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision" [apache-2.0.txt]
package_go_binary() {
local src_path="$1"; shift
local prog="$1"; shift
fpm_build "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
}
+# Usage: package_go_so lib/foo arvados_foo.so arvados-foo "Arvados foo library"
+package_go_so() {
+ local src_path="$1"; shift
+ local sofile="$1"; shift
+ local pkg="$1"; shift
+ local description="$1"; shift
+
+ debug_echo "package_go_so $src_path as $pkg"
+
+ calculate_go_package_version go_package_version $src_path
+ cd $WORKSPACE/packages/$TARGET
+ test_package_presence $pkg $go_package_version go || return 1
+ cd $WORKSPACE/$src_path
+ go build -buildmode=c-shared -o ${GOPATH}/bin/${sofile}
+ cd $WORKSPACE/packages/$TARGET
+ local -a fpmargs=(
+ "--url=https://arvados.org"
+ "--license=Apache License, Version 2.0"
+ "--description=${description}"
+ "$WORKSPACE/apache-2.0.txt=/usr/share/doc/$pkg/apache-2.0.txt"
+ )
+ if [[ -e "$WORKSPACE/$src_path/pam-configs-arvados" ]]; then
+ fpmargs+=("$WORKSPACE/$src_path/pam-configs-arvados=/usr/share/pam-configs/arvados-go")
+ fi
+ fpm_build "$GOPATH/bin/${sofile}=/usr/lib/${sofile}" "${pkg}" dir "${go_package_version}" "${fpmargs[@]}"
+}
+
default_iteration() {
if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
echo "$ARVADOS_BUILDING_ITERATION"
--- /dev/null
+pam_arvados.h
+pam_arvados.so
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "bytes"
+ "crypto/tls"
+ "fmt"
+ "io/ioutil"
+ "net"
+ "net/http"
+ "net/http/httputil"
+ "net/url"
+ "os"
+ "os/exec"
+ "strings"
+ "testing"
+
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "gopkg.in/check.v1"
+)
+
+type DockerSuite struct {
+ tmpdir string
+ hostip string
+ proxyln net.Listener
+ proxysrv *http.Server
+}
+
+var _ = check.Suite(&DockerSuite{})
+
+func Test(t *testing.T) { check.TestingT(t) }
+
+func (s *DockerSuite) SetUpSuite(c *check.C) {
+ if testing.Short() {
+ c.Skip("skipping docker tests in short mode")
+ } else if _, err := exec.Command("docker", "info").CombinedOutput(); err != nil {
+ c.Skip("skipping docker tests because docker is not available")
+ }
+
+ s.tmpdir = c.MkDir()
+
+ // The integration-testing controller listens on the loopback
+ // interface, so it won't be reachable directly from the
+ // docker container -- so here we run a proxy on 0.0.0.0 for
+ // the duration of the test.
+ hostips, err := exec.Command("hostname", "-I").Output()
+ c.Assert(err, check.IsNil)
+ s.hostip = strings.Split(strings.Trim(string(hostips), "\n"), " ")[0]
+ ln, err := net.Listen("tcp", s.hostip+":0")
+ c.Assert(err, check.IsNil)
+ s.proxyln = ln
+ proxy := httputil.NewSingleHostReverseProxy(&url.URL{Scheme: "https", Host: os.Getenv("ARVADOS_API_HOST")})
+ proxy.Transport = &http.Transport{
+ TLSClientConfig: &tls.Config{
+ InsecureSkipVerify: true,
+ },
+ }
+ s.proxysrv = &http.Server{Handler: proxy}
+ go s.proxysrv.ServeTLS(ln, "../../services/api/tmp/self-signed.pem", "../../services/api/tmp/self-signed.key")
+ proxyhost := ln.Addr().String()
+
+ // Build a pam module to install & configure in the docker
+ // container.
+ cmd := exec.Command("go", "build", "-buildmode=c-shared", "-o", s.tmpdir+"/pam_arvados.so")
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ err = cmd.Run()
+ c.Assert(err, check.IsNil)
+
+ // Write a PAM config file that uses our proxy as
+ // ARVADOS_API_HOST.
+ confdata := fmt.Sprintf(`Name: Arvados authentication
+Default: yes
+Priority: 256
+Auth-Type: Primary
+Auth:
+ [success=end default=ignore] /usr/lib/pam_arvados.so %s testvm2.shell insecure
+Auth-Initial:
+ [success=end default=ignore] /usr/lib/pam_arvados.so %s testvm2.shell insecure
+`, proxyhost, proxyhost)
+ err = ioutil.WriteFile(s.tmpdir+"/conffile", []byte(confdata), 0755)
+ c.Assert(err, check.IsNil)
+
+ // Build the testclient program that will (from inside the
+ // docker container) configure the system to use the above PAM
+ // config, and then try authentication.
+ cmd = exec.Command("go", "build", "-o", s.tmpdir+"/testclient", "./testclient.go")
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ err = cmd.Run()
+ c.Assert(err, check.IsNil)
+}
+
+func (s *DockerSuite) TearDownSuite(c *check.C) {
+ if s.proxysrv != nil {
+ s.proxysrv.Close()
+ }
+ if s.proxyln != nil {
+ s.proxyln.Close()
+ }
+}
+
+func (s *DockerSuite) runTestClient(c *check.C, args ...string) (stdout, stderr *bytes.Buffer, err error) {
+ cmd := exec.Command("docker", append([]string{
+ "run", "--rm",
+ "--add-host", "zzzzz.arvadosapi.com:" + s.hostip,
+ "-v", s.tmpdir + "/pam_arvados.so:/usr/lib/pam_arvados.so:ro",
+ "-v", s.tmpdir + "/conffile:/usr/share/pam-configs/arvados:ro",
+ "-v", s.tmpdir + "/testclient:/testclient:ro",
+ "debian:buster",
+ "/testclient"}, args...)...)
+ stdout = &bytes.Buffer{}
+ stderr = &bytes.Buffer{}
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ return
+}
+
+func (s *DockerSuite) TestSuccess(c *check.C) {
+ stdout, stderr, err := s.runTestClient(c, "try", "active", arvadostest.ActiveTokenV2)
+ c.Check(err, check.IsNil)
+ c.Logf("%s", stderr.String())
+ c.Check(stdout.String(), check.Equals, "")
+ c.Check(stderr.String(), check.Matches, `(?ms).*authentication succeeded.*`)
+}
+
+func (s *DockerSuite) TestFailure(c *check.C) {
+ for _, trial := range []struct {
+ label string
+ username string
+ token string
+ }{
+ {"bad token", "active", arvadostest.ActiveTokenV2 + "badtoken"},
+ {"empty token", "active", ""},
+ {"empty username", "", arvadostest.ActiveTokenV2},
+ {"wrong username", "wrongusername", arvadostest.ActiveTokenV2},
+ } {
+ c.Logf("trial: %s", trial.label)
+ stdout, stderr, err := s.runTestClient(c, "try", trial.username, trial.token)
+ c.Logf("%s", stderr.String())
+ c.Check(err, check.NotNil)
+ c.Check(stdout.String(), check.Equals, "")
+ c.Check(stderr.String(), check.Matches, `(?ms).*authentication failed.*`)
+ }
+}
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+fpm_depends+=(ca-certificates)
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# This file is packaged as /usr/share/pam-configs/arvados-go; see build/run-library.sh
+
+# 1. Run `pam-auth-update` and choose Arvados authentication
+# 2. In /etc/pam.d/common-auth, change "api.example" to your ARVADOS_API_HOST
+# 3. In /etc/pam.d/common-auth, change "shell.example" to this host's hostname
+# (as it appears in the Arvados virtual_machines list)
+
+Name: Arvados authentication
+Default: yes
+Priority: 256
+Auth-Type: Primary
+Auth:
+ [success=end default=ignore] /usr/lib/pam_arvados.so api.example shell.example
+Auth-Initial:
+ [success=end default=ignore] /usr/lib/pam_arvados.so api.example shell.example
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// To enable, add an entry in /etc/pam.d/common-auth where pam_unix.so
+// would normally be. Examples:
+//
+// auth [success=1 default=ignore] /usr/lib/pam_arvados.so zzzzz.arvadosapi.com vmhostname.example
+// auth [success=1 default=ignore] /usr/lib/pam_arvados.so zzzzz.arvadosapi.com vmhostname.example insecure debug
+//
+// Replace zzzzz.arvadosapi.com with your controller host or
+// host:port.
+//
+// Replace vmhostname.example with the VM's name as it appears in the
+// Arvados virtual_machine object.
+//
+// Use "insecure" if your API server certificate does not pass name
+// verification.
+//
+// Use "debug" to enable debug log messages.
+
+package main
+
+import (
+ "io/ioutil"
+ "log/syslog"
+
+ "context"
+ "errors"
+ "fmt"
+ "runtime"
+ "syscall"
+ "time"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "github.com/sirupsen/logrus"
+ lSyslog "github.com/sirupsen/logrus/hooks/syslog"
+ "golang.org/x/sys/unix"
+)
+
+/*
+#cgo LDFLAGS: -lpam -fPIC
+#include <security/pam_ext.h>
+char *stringindex(char** a, int i);
+const char *get_user(pam_handle_t *pamh);
+const char *get_authtoken(pam_handle_t *pamh);
+*/
+import "C"
+
+func main() {}
+
+func init() {
+ if err := unix.Prctl(syscall.PR_SET_DUMPABLE, 0, 0, 0, 0); err != nil {
+ newLogger(false).WithError(err).Warn("unable to disable ptrace")
+ }
+}
+
+//export pam_sm_setcred
+func pam_sm_setcred(pamh *C.pam_handle_t, flags, cArgc C.int, cArgv **C.char) C.int {
+ return C.PAM_IGNORE
+}
+
+//export pam_sm_authenticate
+func pam_sm_authenticate(pamh *C.pam_handle_t, flags, cArgc C.int, cArgv **C.char) C.int {
+ runtime.GOMAXPROCS(1)
+ logger := newLogger(flags&C.PAM_SILENT == 0)
+ cUsername := C.get_user(pamh)
+ if cUsername == nil {
+ return C.PAM_USER_UNKNOWN
+ }
+
+ cToken := C.get_authtoken(pamh)
+ if cToken == nil {
+ return C.PAM_AUTH_ERR
+ }
+
+ argv := make([]string, cArgc)
+ for i := 0; i < int(cArgc); i++ {
+ argv[i] = C.GoString(C.stringindex(cArgv, C.int(i)))
+ }
+
+ err := authenticate(logger, C.GoString(cUsername), C.GoString(cToken), argv)
+ if err != nil {
+ logger.WithError(err).Error("authentication failed")
+ return C.PAM_AUTH_ERR
+ }
+ return C.PAM_SUCCESS
+}
+
+func authenticate(logger *logrus.Logger, username, token string, argv []string) error {
+ hostname := ""
+ apiHost := ""
+ insecure := false
+ for idx, arg := range argv {
+ if idx == 0 {
+ apiHost = arg
+ } else if idx == 1 {
+ hostname = arg
+ } else if arg == "insecure" {
+ insecure = true
+ } else if arg == "debug" {
+ logger.SetLevel(logrus.DebugLevel)
+ } else {
+ logger.Warnf("unkown option: %s\n", arg)
+ }
+ }
+ logger.Debugf("username=%q arvados_api_host=%q hostname=%q insecure=%t", username, apiHost, hostname, insecure)
+ if apiHost == "" || hostname == "" {
+ logger.Warnf("cannot authenticate: config error: arvados_api_host and hostname must be non-empty")
+ return errors.New("config error")
+ }
+ arv := &arvados.Client{
+ Scheme: "https",
+ APIHost: apiHost,
+ AuthToken: token,
+ Insecure: insecure,
+ }
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute))
+ defer cancel()
+ var vms arvados.VirtualMachineList
+ err := arv.RequestAndDecodeContext(ctx, &vms, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{
+ Limit: 2,
+ Filters: []arvados.Filter{
+ {"hostname", "=", hostname},
+ },
+ })
+ if err != nil {
+ return err
+ }
+ if len(vms.Items) == 0 {
+ // It's possible there is no VM entry for the
+ // configured hostname, but typically this just means
+ // the user does not have permission to see (let alone
+ // log in to) this VM.
+ return errors.New("permission denied")
+ } else if len(vms.Items) > 1 {
+ return fmt.Errorf("multiple results for hostname %q", hostname)
+ } else if vms.Items[0].Hostname != hostname {
+ return fmt.Errorf("looked up hostname %q but controller returned record with hostname %q", hostname, vms.Items[0].Hostname)
+ }
+ var user arvados.User
+ err = arv.RequestAndDecodeContext(ctx, &user, "GET", "arvados/v1/users/current", nil, nil)
+ if err != nil {
+ return err
+ }
+ var links arvados.LinkList
+ err = arv.RequestAndDecodeContext(ctx, &links, "GET", "arvados/v1/links", nil, arvados.ListOptions{
+ Limit: 1,
+ Filters: []arvados.Filter{
+ {"link_class", "=", "permission"},
+ {"name", "=", "can_login"},
+ {"tail_uuid", "=", user.UUID},
+ {"head_uuid", "=", vms.Items[0].UUID},
+ {"properties.username", "=", username},
+ },
+ })
+ if err != nil {
+ return err
+ }
+ if len(links.Items) < 1 || links.Items[0].Properties["username"] != username {
+ return errors.New("permission denied")
+ }
+ logger.Debugf("permission granted based on link with UUID %s", links.Items[0].UUID)
+ return nil
+}
+
+func newLogger(stderr bool) *logrus.Logger {
+ logger := logrus.New()
+ if !stderr {
+ logger.Out = ioutil.Discard
+ }
+ if hook, err := lSyslog.NewSyslogHook("udp", "localhost:514", syslog.LOG_AUTH|syslog.LOG_INFO, "pam_arvados"); err != nil {
+ logger.Hooks.Add(hook)
+ }
+ return logger
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+/*
+#cgo LDFLAGS: -lpam -fPIC
+#include <security/pam_ext.h>
+char *stringindex(char** a, int i) { return a[i]; }
+const char *get_user(pam_handle_t *pamh) {
+ const char *user;
+ if (pam_get_item(pamh, PAM_USER, (const void**)&user) != PAM_SUCCESS)
+ return NULL;
+ return user;
+}
+const char *get_authtoken(pam_handle_t *pamh) {
+ const char *token;
+ if (pam_get_authtok(pamh, PAM_AUTHTOK, &token, NULL) != PAM_SUCCESS)
+ return NULL;
+ return token;
+}
+*/
+import "C"
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// +build never
+
+// This file is compiled by docker_test.go to build a test client.
+// It's not part of the pam module itself.
+
+package main
+
+import (
+ "fmt"
+ "os"
+ "os/exec"
+
+ "github.com/msteinert/pam"
+ "github.com/sirupsen/logrus"
+)
+
+func main() {
+ if len(os.Args) != 4 || os.Args[1] != "try" {
+ logrus.Print("usage: testclient try 'username' 'password'")
+ os.Exit(1)
+ }
+ username := os.Args[2]
+ password := os.Args[3]
+
+ // Configure PAM to use arvados token auth by default.
+ cmd := exec.Command("pam-auth-update", "--force", "arvados", "--remove", "unix")
+ cmd.Env = append([]string{"DEBIAN_FRONTEND=noninteractive"}, os.Environ()...)
+ cmd.Stdin = nil
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ err := cmd.Run()
+ if err != nil {
+ logrus.WithError(err).Error("pam-auth-update failed")
+ os.Exit(1)
+ }
+
+ // Check that pam-auth-update actually added arvados config.
+ cmd = exec.Command("grep", "-Hn", "arvados", "/etc/pam.d/common-auth")
+ cmd.Stdout = os.Stderr
+ cmd.Stderr = os.Stderr
+ err = cmd.Run()
+ if err != nil {
+ panic(err)
+ }
+
+ logrus.Debugf("starting pam: username=%q password=%q", username, password)
+
+ sentPassword := false
+ errorMessage := ""
+ tx, err := pam.StartFunc("default", username, func(style pam.Style, message string) (string, error) {
+ logrus.Debugf("pam conversation: style=%v message=%q", style, message)
+ switch style {
+ case pam.ErrorMsg:
+ logrus.WithField("Message", message).Info("pam.ErrorMsg")
+ errorMessage = message
+ return "", nil
+ case pam.TextInfo:
+ logrus.WithField("Message", message).Info("pam.TextInfo")
+ errorMessage = message
+ return "", nil
+ case pam.PromptEchoOn, pam.PromptEchoOff:
+ sentPassword = true
+ return password, nil
+ default:
+ return "", fmt.Errorf("unrecognized message style %d", style)
+ }
+ })
+ if err != nil {
+ logrus.WithError(err).Print("StartFunc failed")
+ os.Exit(1)
+ }
+ err = tx.Authenticate(pam.DisallowNullAuthtok)
+ if err != nil {
+ err = fmt.Errorf("PAM: %s (message = %q)", err, errorMessage)
+ logrus.WithError(err).Print("authentication failed")
+ os.Exit(1)
+ }
+ logrus.Print("authentication succeeded")
+}
// Link is an arvados#link record
type Link struct {
- UUID string `json:"uuid,omiempty"`
- OwnerUUID string `json:"owner_uuid"`
- Name string `json:"name"`
- LinkClass string `json:"link_class"`
- HeadUUID string `json:"head_uuid"`
- HeadKind string `json:"head_kind"`
- TailUUID string `json:"tail_uuid"`
- TailKind string `json:"tail_kind"`
+ UUID string `json:"uuid,omiempty"`
+ OwnerUUID string `json:"owner_uuid"`
+ Name string `json:"name"`
+ LinkClass string `json:"link_class"`
+ HeadUUID string `json:"head_uuid"`
+ HeadKind string `json:"head_kind"`
+ TailUUID string `json:"tail_uuid"`
+ TailKind string `json:"tail_kind"`
+ Properties map[string]interface{} `json:"properties"`
}
// UserList is an arvados#userList resource.
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import "time"
+
+// VirtualMachine is an arvados#virtualMachine resource.
+type VirtualMachine struct {
+ UUID string `json:"uuid"`
+ OwnerUUID string `json:"owner_uuid"`
+ Hostname string `json:"hostname"`
+ CreatedAt *time.Time `json:"created_at"`
+ ModifiedAt *time.Time `json:"modified_at"`
+ ModifiedByUserUUID string `json:"modified_by_user_uuid"`
+}
+
+// VirtualMachineList is an arvados#virtualMachineList resource.
+type VirtualMachineList struct {
+ Items []VirtualMachine `json:"items"`
+ ItemsAvailable int `json:"items_available"`
+ Offset int `json:"offset"`
+ Limit int `json:"limit"`
+}
projects / arvados.git / commitdiff