sdk/perl/Makefile
sdk/perl/blib
sdk/perl/pm_to_blib
-*/vendor/bundle
+*/vendor
+*/*/vendor
sdk/java/target
*.class
-apps/workbench/vendor/bundle
-services/api/vendor/bundle
sdk/java/log
-sdk/cli/vendor
+/tmp
# Ignore all logfiles and tempfiles.
/log/*.log
+/log/*.log.gz
/tmp
/config/.secret_token
Thread.current[:arvados_api_token] = auth['api_token']
end
- teardown do
+ setup do
Thread.current[:arvados_api_token] = nil
Thread.current[:user] = nil
Thread.current[:reader_tokens] = nil
# Diagnostics suite doesn't run a server, so there's no cache to clear.
Rails.cache.clear unless (Rails.env == "diagnostics")
# Restore configuration settings changed during tests
+ self.class.reset_application_config
+ end
+
+ def self.reset_application_config
$application_config.each do |k,v|
if k.match /^[^.]*$/
Rails.configuration.send (k + '='), v
end
class ApiServerForTests
+ PYTHON_TESTS_DIR = File.expand_path('../../../../sdk/python/tests', __FILE__)
ARV_API_SERVER_DIR = File.expand_path('../../../../services/api', __FILE__)
- SERVER_PID_PATH = File.expand_path('tmp/pids/wbtest-server.pid', ARV_API_SERVER_DIR)
- WEBSOCKET_PID_PATH = File.expand_path('tmp/pids/wstest-server.pid', ARV_API_SERVER_DIR)
+ SERVER_PID_PATH = File.expand_path('tmp/pids/test-server.pid', ARV_API_SERVER_DIR)
+ WEBSOCKET_PID_PATH = File.expand_path('tmp/pids/test-server.pid', ARV_API_SERVER_DIR)
@main_process_pid = $$
+ @@server_is_running = false
- def _system(*cmd)
- $stderr.puts "_system #{cmd.inspect}"
+ def check_call *args
+ output = nil
Bundler.with_clean_env do
- if not system({'RAILS_ENV' => 'test', "ARVADOS_WEBSOCKETS" => (if @websocket then "ws-only" end)}, *cmd)
- raise RuntimeError, "#{cmd[0]} returned exit code #{$?.exitstatus}"
+ output = IO.popen *args do |io|
+ io.read
+ end
+ if not $?.success?
+ raise RuntimeError, "Command failed (#{$?}): #{args.inspect}"
end
end
+ output
end
- def make_ssl_cert
- unless File.exists? './self-signed.key'
- _system('openssl', 'req', '-new', '-x509', '-nodes',
- '-out', './self-signed.pem',
- '-keyout', './self-signed.key',
- '-days', '3650',
- '-subj', '/CN=localhost')
+ def run_test_server
+ env_script = nil
+ Dir.chdir PYTHON_TESTS_DIR do
+ env_script = check_call %w(python ./run_test_server.py start --auth admin)
end
- end
-
- def kill_server
- if (pid = find_server_pid)
- $stderr.puts "Sending TERM to API server, pid #{pid}"
- Process.kill 'TERM', pid
+ test_env = {}
+ env_script.each_line do |line|
+ line = line.chomp
+ if 0 == line.index('export ')
+ toks = line.sub('export ', '').split '=', 2
+ $stderr.puts "run_test_server.py: #{toks[0]}=#{toks[1]}"
+ test_env[toks[0]] = toks[1]
+ end
end
+ test_env
end
- def find_server_pid
- pid = nil
- begin
- pid = IO.read(@pidfile).to_i
- $stderr.puts "API server is running, pid #{pid.inspect}"
- rescue Errno::ENOENT
+ def stop_test_server
+ Dir.chdir PYTHON_TESTS_DIR do
+ # This is a no-op if we're running within run-tests.sh
+ check_call %w(python ./run_test_server.py stop)
end
- return pid
+ @@server_is_running = false
end
- def run(args=[])
+ def run args=[]
+ return if @@server_is_running
+
+ # Stop server left over from interrupted previous run
+ stop_test_server
+
::MiniTest.after_run do
- self.kill_server
+ stop_test_server
end
- @websocket = args.include?("--websockets")
-
- @pidfile = if @websocket
- WEBSOCKET_PID_PATH
- else
- SERVER_PID_PATH
- end
-
- # Kill server left over from previous test run
- self.kill_server
-
- Capybara.javascript_driver = :poltergeist
- Dir.chdir(ARV_API_SERVER_DIR) do |apidir|
- ENV["NO_COVERAGE_TEST"] = "1"
- if @websocket
- _system('bundle', 'exec', 'passenger', 'start', '-d', '-p3333',
- '--pid-file', @pidfile)
- else
- make_ssl_cert
- if ENV['ARVADOS_TEST_API_INSTALLED'].blank?
- _system('bundle', 'exec', 'rake', 'db:test:load')
- _system('bundle', 'exec', 'rake', 'db:fixtures:load')
- end
- _system('bundle', 'exec', 'passenger', 'start', '-d', '-p3000',
- '--pid-file', @pidfile,
- '--ssl',
- '--ssl-certificate', 'self-signed.pem',
- '--ssl-certificate-key', 'self-signed.key')
- end
- timeout = Time.now.tv_sec + 10
- good_pid = false
- while (not good_pid) and (Time.now.tv_sec < timeout)
- sleep 0.2
- server_pid = find_server_pid
- good_pid = (server_pid and
- (server_pid > 0) and
- (Process.kill(0, server_pid) rescue false))
- end
- if not good_pid
- raise RuntimeError, "could not find API server Rails pid"
- end
- end
+ test_env = run_test_server
+ $application_config['arvados_login_base'] = "https://#{test_env['ARVADOS_API_HOST']}/login"
+ $application_config['arvados_v1_base'] = "https://#{test_env['ARVADOS_API_HOST']}/arvados/v1"
+ $application_config['arvados_insecure_host'] = true
+ ActiveSupport::TestCase.reset_application_config
+
+ @@server_is_running = true
end
- def run_rake_task(task_name, arg_string)
- Dir.chdir(ARV_API_SERVER_DIR) do
- _system('bundle', 'exec', 'rake', "#{task_name}[#{arg_string}]")
+ def run_rake_task task_name, arg_string
+ Dir.chdir ARV_API_SERVER_DIR do
+ check_call ['bundle', 'exec', 'rake', "#{task_name}[#{arg_string}]"]
end
end
end
TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}}},
External: external}
- if os.Getenv("ARVADOS_API_HOST") == "" {
+ if kc.ApiServer == "" {
return kc, MissingArvadosApiHost
}
- if os.Getenv("ARVADOS_API_TOKEN") == "" {
+ if kc.ApiToken == "" {
return kc, MissingArvadosApiToken
}
package arvadosclient
import (
- "fmt"
. "gopkg.in/check.v1"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
"net/http"
"os"
- "os/exec"
"testing"
)
// Tests that require the Keep server running
type ServerRequiredSuite struct{}
-func pythonDir() string {
- cwd, _ := os.Getwd()
- return fmt.Sprintf("%s/../../python/tests", cwd)
+func (s *ServerRequiredSuite) SetUpSuite(c *C) {
+ arvadostest.StartAPI()
+ arvadostest.StartKeep()
}
-func (s *ServerRequiredSuite) SetUpSuite(c *C) {
- os.Chdir(pythonDir())
- if err := exec.Command("python", "run_test_server.py", "start").Run(); err != nil {
- panic("'python run_test_server.py start' returned error")
- }
- if err := exec.Command("python", "run_test_server.py", "start_keep").Run(); err != nil {
- panic("'python run_test_server.py start_keep' returned error")
- }
+func (s *ServerRequiredSuite) SetUpTest(c *C) {
+ arvadostest.ResetEnv()
}
-func (s *ServerRequiredSuite) TestMakeArvadosClient(c *C) {
- os.Setenv("ARVADOS_API_HOST", "localhost:3000")
- os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
+func (s *ServerRequiredSuite) TestMakeArvadosClientSecure(c *C) {
os.Setenv("ARVADOS_API_HOST_INSECURE", "")
-
kc, err := MakeArvadosClient()
- c.Check(kc.ApiServer, Equals, "localhost:3000")
- c.Check(kc.ApiToken, Equals, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
+ c.Assert(err, Equals, nil)
+ c.Check(kc.ApiServer, Equals, os.Getenv("ARVADOS_API_HOST"))
+ c.Check(kc.ApiToken, Equals, os.Getenv("ARVADOS_API_TOKEN"))
c.Check(kc.ApiInsecure, Equals, false)
+}
+func (s *ServerRequiredSuite) TestMakeArvadosClientInsecure(c *C) {
os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
-
- kc, err = MakeArvadosClient()
- c.Check(kc.ApiServer, Equals, "localhost:3000")
- c.Check(kc.ApiToken, Equals, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
+ kc, err := MakeArvadosClient()
+ c.Assert(err, Equals, nil)
c.Check(kc.ApiInsecure, Equals, true)
+ c.Check(kc.ApiServer, Equals, os.Getenv("ARVADOS_API_HOST"))
+ c.Check(kc.ApiToken, Equals, os.Getenv("ARVADOS_API_TOKEN"))
c.Check(kc.Client.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify, Equals, true)
-
- c.Assert(err, Equals, nil)
}
func (s *ServerRequiredSuite) TestCreatePipelineTemplate(c *C) {
- os.Setenv("ARVADOS_API_HOST", "localhost:3000")
- os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
- os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
-
arv, err := MakeArvadosClient()
getback := make(Dict)
}
func (s *ServerRequiredSuite) TestErrorResponse(c *C) {
- os.Setenv("ARVADOS_API_HOST", "localhost:3000")
- os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
- os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
-
arv, _ := MakeArvadosClient()
getback := make(Dict)
--- /dev/null
+package arvadostest
+
+import (
+ "bufio"
+ "bytes"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "os"
+ "os/exec"
+ "strings"
+)
+
+var authSettings = make(map[string]string)
+
+func ResetEnv() {
+ for k, v := range authSettings {
+ os.Setenv(k, v)
+ }
+}
+
+func ParseAuthSettings(authScript []byte) {
+ scanner := bufio.NewScanner(bytes.NewReader(authScript))
+ for scanner.Scan() {
+ line := scanner.Text()
+ if 0 != strings.Index(line, "export ") {
+ log.Printf("Ignoring: %v", line)
+ continue
+ }
+ toks := strings.SplitN(strings.Replace(line, "export ", "", 1), "=", 2)
+ if len(toks) == 2 {
+ authSettings[toks[0]] = toks[1]
+ } else {
+ log.Fatalf("Could not parse: %v", line)
+ }
+ }
+ log.Printf("authSettings: %v", authSettings)
+}
+
+var pythonTestDir string = ""
+
+func chdirToPythonTests() {
+ if pythonTestDir != "" {
+ if err := os.Chdir(pythonTestDir); err != nil {
+ log.Fatalf("chdir %s: %s", pythonTestDir, err)
+ }
+ return
+ }
+ for {
+ if err := os.Chdir("sdk/python/tests"); err == nil {
+ pythonTestDir, err = os.Getwd()
+ return
+ }
+ if parent, err := os.Getwd(); err != nil || parent == "/" {
+ log.Fatalf("sdk/python/tests/ not found in any ancestor")
+ }
+ if err := os.Chdir(".."); err != nil {
+ log.Fatal(err)
+ }
+ }
+}
+
+func StartAPI() {
+ cwd, _ := os.Getwd()
+ defer os.Chdir(cwd)
+ chdirToPythonTests()
+
+ cmd := exec.Command("python", "run_test_server.py", "start", "--auth", "admin")
+ stderr, err := cmd.StderrPipe()
+ if err != nil {
+ log.Fatal(err)
+ }
+ go io.Copy(os.Stderr, stderr)
+ stdout, err := cmd.StdoutPipe()
+ if err != nil {
+ log.Fatal(err)
+ }
+ if err = cmd.Start(); err != nil {
+ log.Fatal(err)
+ }
+ var authScript []byte
+ if authScript, err = ioutil.ReadAll(stdout); err != nil {
+ log.Fatal(err)
+ }
+ if err = cmd.Wait(); err != nil {
+ log.Fatal(err)
+ }
+ ParseAuthSettings(authScript)
+ ResetEnv()
+}
+
+func StopAPI() {
+ cwd, _ := os.Getwd()
+ defer os.Chdir(cwd)
+ chdirToPythonTests()
+
+ exec.Command("python", "run_test_server.py", "stop").Run()
+}
+
+func StartKeep() {
+ cwd, _ := os.Getwd()
+ defer os.Chdir(cwd)
+ chdirToPythonTests()
+
+ cmd := exec.Command("python", "run_test_server.py", "start_keep")
+ stderr, err := cmd.StderrPipe()
+ if err != nil {
+ log.Fatalf("Setting up stderr pipe: %s", err)
+ }
+ go io.Copy(os.Stderr, stderr)
+ if err := cmd.Run(); err != nil {
+ panic(fmt.Sprintf("'python run_test_server.py start_keep' returned error %s", err))
+ }
+}
+
+func StopKeep() {
+ cwd, _ := os.Getwd()
+ defer os.Chdir(cwd)
+ chdirToPythonTests()
+
+ exec.Command("python", "run_test_server.py", "stop_keep").Run()
+}
"flag"
"fmt"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/streamer"
. "gopkg.in/check.v1"
"io"
"net"
"net/http"
"os"
- "os/exec"
"testing"
)
c.Skip("Skipping tests that require server")
return
}
- os.Chdir(pythonDir())
- {
- cmd := exec.Command("python", "run_test_server.py", "start")
- stderr, err := cmd.StderrPipe()
- if err != nil {
- log.Fatalf("Setting up stderr pipe: %s", err)
- }
- go io.Copy(os.Stderr, stderr)
- if err := cmd.Run(); err != nil {
- panic(fmt.Sprintf("'python run_test_server.py start' returned error %s", err))
- }
- }
- {
- cmd := exec.Command("python", "run_test_server.py", "start_keep")
- stderr, err := cmd.StderrPipe()
- if err != nil {
- log.Fatalf("Setting up stderr pipe: %s", err)
- }
- go io.Copy(os.Stderr, stderr)
- if err := cmd.Run(); err != nil {
- panic(fmt.Sprintf("'python run_test_server.py start_keep' returned error %s", err))
- }
- }
+ arvadostest.StartAPI()
+ arvadostest.StartKeep()
}
func (s *ServerRequiredSuite) TearDownSuite(c *C) {
- os.Chdir(pythonDir())
- exec.Command("python", "run_test_server.py", "stop_keep").Run()
- exec.Command("python", "run_test_server.py", "stop").Run()
+ if *no_server {
+ return
+ }
+ arvadostest.StopKeep()
+ arvadostest.StopAPI()
}
func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
- os.Setenv("ARVADOS_API_HOST", "localhost:3000")
- os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
- os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
-
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, Equals, nil)
c.Assert(err, Equals, nil)
c.Check(len(kc.ServiceRoots()), Equals, 2)
for _, root := range kc.ServiceRoots() {
- c.Check(root, Matches, "http://localhost:2510[\\d]")
+ c.Check(root, Matches, "http://localhost:\\d+")
}
}
}
func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
- os.Setenv("ARVADOS_API_HOST", "localhost:3000")
- os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
- os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
content := []byte("TestPutGetHead")
arv, err := arvadosclient.MakeArvadosClient()
r, n, url2, err := kc.Get(hash)
c.Check(err, Equals, nil)
c.Check(n, Equals, int64(len(content)))
- c.Check(url2, Equals, fmt.Sprintf("http://localhost:25108/%s", hash))
+ c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
read_content, err2 := ioutil.ReadAll(r)
c.Check(err2, Equals, nil)
n, url2, err := kc.Ask(hash)
c.Check(err, Equals, nil)
c.Check(n, Equals, int64(len(content)))
- c.Check(url2, Equals, fmt.Sprintf("http://localhost:25108/%s", hash))
+ c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
}
}
"log"
"net"
"net/http"
- "os"
"strings"
"time"
)
}
func (this *KeepClient) DiscoverKeepServers() error {
- if prx := os.Getenv("ARVADOS_KEEP_PROXY"); prx != "" {
- sr := map[string]string{"proxy": prx}
- this.SetServiceRoots(sr)
- this.Using_proxy = true
- this.setClientSettingsProxy()
- return nil
- }
-
type svcList struct {
Items []keepDisk `json:"items"`
}
pass
class StaleWriterStateError(Exception):
pass
+class FeatureNotEnabledError(Exception):
+ pass
-from ws4py.client.threadedclient import WebSocketClient
-import threading
+import arvados
+import config
+import errors
+
+import logging
import json
-import os
+import threading
import time
-import ssl
+import os
import re
-import config
-import logging
-import arvados
+import ssl
+from ws4py.client.threadedclient import WebSocketClient
_logger = logging.getLogger('arvados.events')
ssl_options['cert_reqs'] = ssl.CERT_NONE
else:
ssl_options['cert_reqs'] = ssl.CERT_REQUIRED
+
+ # Warning: If the host part of url resolves to both IPv6 and
+ # IPv4 addresses (common with "localhost"), only one of them
+ # will be attempted -- and it might not be the right one. See
+ # ws4py's WebSocketBaseClient.__init__.
super(EventClient, self).__init__(url, ssl_options=ssl_options)
self.filters = filters
self.on_event = on_event
del self.filters[self.filters.index(filters)]
+def _subscribe_websocket(api, filters, on_event):
+ endpoint = api._rootDesc.get('websocketUrl', None)
+ if not endpoint:
+ raise errors.FeatureNotEnabledError(
+ "Server does not advertise a websocket endpoint")
+ uri_with_token = "{}?api_token={}".format(endpoint, api.api_token)
+ client = EventClient(uri_with_token, filters, on_event)
+ ok = False
+ try:
+ client.connect()
+ ok = True
+ return client
+ finally:
+ if not ok:
+ client.close_connection()
+
def subscribe(api, filters, on_event, poll_fallback=15):
'''
api: a client object retrieved from arvados.api(). The caller should not use this client object for anything else after calling subscribe().
on_event: The callback when a message is received.
poll_fallback: If websockets are not available, fall back to polling every N seconds. If poll_fallback=False, this will return None if websockets are not available.
'''
- ws = None
- if 'websocketUrl' in api._rootDesc:
- try:
- url = "{}?api_token={}".format(api._rootDesc['websocketUrl'], api.api_token)
- ws = EventClient(url, filters, on_event)
- ws.connect()
- return ws
- except Exception as e:
- _logger.warn("Got exception %s trying to connect to websockets at %s" % (e, api._rootDesc['websocketUrl']))
- if ws:
- ws.close_connection()
- if poll_fallback:
- _logger.warn("Websockets not available, falling back to log table polling")
- p = PollClient(api, filters, on_event, poll_fallback)
- p.start()
- return p
- else:
- _logger.error("Websockets not available")
- return None
+ if not poll_fallback:
+ return _subscribe_websocket(api, filters, on_event)
+
+ try:
+ return _subscribe_websocket(api, filters, on_event)
+ except Exception as e:
+ _logger.warn("Falling back to polling after websocket error: %s" % e)
+ p = PollClient(api, filters, on_event, poll_fallback)
+ p.start()
+ return p
#!/usr/bin/env python
import argparse
+import atexit
+import httplib2
import os
+import pipes
+import random
+import re
import shutil
import signal
+import socket
import subprocess
+import string
import sys
import tempfile
import time
import arvados.api
import arvados.config
-SERVICES_SRC_DIR = os.path.join(MY_DIRNAME, '../../../services')
-SERVER_PID_PATH = 'tmp/pids/webrick-test.pid'
-WEBSOCKETS_SERVER_PID_PATH = 'tmp/pids/passenger-test.pid'
+ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
+SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
+SERVER_PID_PATH = 'tmp/pids/test-server.pid'
if 'GOPATH' in os.environ:
gopaths = os.environ['GOPATH'].split(':')
gobins = [os.path.join(path, 'bin') for path in gopaths]
os.environ['PATH'] = ':'.join(gobins) + ':' + os.environ['PATH']
-if os.path.isdir('tests'):
- TEST_TMPDIR = 'tests/tmp'
-else:
- TEST_TMPDIR = 'tmp'
+TEST_TMPDIR = os.path.join(ARVADOS_DIR, 'tmp')
+if not os.path.exists(TEST_TMPDIR):
+ os.mkdir(TEST_TMPDIR)
+
+my_api_host = None
def find_server_pid(PID_PATH, wait=10):
now = time.time()
return server_pid
-def kill_server_pid(PID_PATH, wait=10):
+def kill_server_pid(pidfile, wait=10, passenger_root=False):
+ # Must re-import modules in order to work during atexit
+ import os
+ import signal
+ import subprocess
+ import time
try:
+ if passenger_root:
+ # First try to shut down nicely
+ restore_cwd = os.getcwd()
+ os.chdir(passenger_root)
+ subprocess.call([
+ 'bundle', 'exec', 'passenger', 'stop', '--pid-file', pidfile])
+ os.chdir(restore_cwd)
now = time.time()
timeout = now + wait
- with open(PID_PATH, 'r') as f:
+ with open(pidfile, 'r') as f:
server_pid = int(f.read())
while now <= timeout:
- os.kill(server_pid, signal.SIGTERM)
- os.getpgid(server_pid) # throw OSError if no such pid
- now = time.time()
+ if not passenger_root or timeout - now < wait / 2:
+ # Half timeout has elapsed. Start sending SIGTERM
+ os.kill(server_pid, signal.SIGTERM)
+ # Raise OSError if process has disappeared
+ os.getpgid(server_pid)
time.sleep(0.1)
+ now = time.time()
except IOError:
- good_pid = False
+ pass
except OSError:
- good_pid = False
-
-def run(websockets=False, reuse_server=False):
- cwd = os.getcwd()
- os.chdir(os.path.join(SERVICES_SRC_DIR, 'api'))
-
- if websockets:
- pid_file = WEBSOCKETS_SERVER_PID_PATH
- else:
- pid_file = SERVER_PID_PATH
-
- test_pid = find_server_pid(pid_file, 0)
-
- if test_pid is None or not reuse_server:
- # do not try to run both server variants at once
- stop()
-
- # delete cached discovery document
- shutil.rmtree(arvados.http_cache('discovery'))
-
- # Setup database
- os.environ["RAILS_ENV"] = "test"
- subprocess.call(['bundle', 'exec', 'rake', 'tmp:cache:clear'])
- subprocess.call(['bundle', 'exec', 'rake', 'db:test:load'])
- subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load'])
+ pass
- subprocess.call(['bundle', 'exec', 'rails', 'server', '-d',
- '--pid',
- os.path.join(os.getcwd(), SERVER_PID_PATH),
- '-p3000'])
- os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
+def find_available_port():
+ """Return an IPv4 port number that is not in use right now.
- if websockets:
- os.environ["ARVADOS_WEBSOCKETS"] = "ws-only"
- subprocess.call(['bundle', 'exec',
- 'passenger', 'start', '-d', '-p3333',
- '--pid-file',
- os.path.join(os.getcwd(), WEBSOCKETS_SERVER_PID_PATH)
- ])
+ We assume whoever needs to use the returned port is able to reuse
+ a recently used port without waiting for TIME_WAIT (see
+ SO_REUSEADDR / SO_REUSEPORT).
- pid = find_server_pid(SERVER_PID_PATH)
+ Some opportunity for races here, but it's better than choosing
+ something at random and not checking at all. If all of our servers
+ (hey Passenger) knew that listening on port 0 was a thing, the OS
+ would take care of the races, and this wouldn't be needed at all.
+ """
- os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
- os.environ["ARVADOS_API_TOKEN"] = ""
- os.chdir(cwd)
+ sock = socket.socket()
+ sock.bind(('0.0.0.0', 0))
+ port = sock.getsockname()[1]
+ sock.close()
+ return port
-def stop():
- cwd = os.getcwd()
- os.chdir(os.path.join(SERVICES_SRC_DIR, 'api'))
+def run(leave_running_atexit=False):
+ """Ensure an API server is running, and ARVADOS_API_* env vars have
+ admin credentials for it.
- kill_server_pid(WEBSOCKETS_SERVER_PID_PATH, 0)
- kill_server_pid(SERVER_PID_PATH, 0)
+ If ARVADOS_TEST_API_HOST is set, a parent process has started a
+ test server for us to use: we just need to reset() it using the
+ admin token fixture.
- try:
- os.unlink('self-signed.pem')
- except:
- pass
-
- try:
- os.unlink('self-signed.key')
- except:
- pass
+ If a previous call to run() started a new server process, and it
+ is still running, we just need to reset() it to fixture state and
+ return.
- os.chdir(cwd)
+ If neither of those options work out, we'll really start a new
+ server.
+ """
+ global my_api_host
+
+ # Delete cached discovery document.
+ shutil.rmtree(arvados.http_cache('discovery'))
+
+ pid_file = os.path.join(SERVICES_SRC_DIR, 'api', SERVER_PID_PATH)
+ pid_file_ok = find_server_pid(pid_file, 0)
+
+ existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
+ if existing_api_host and pid_file_ok:
+ if existing_api_host == my_api_host:
+ try:
+ return reset()
+ except:
+ # Fall through to shutdown-and-start case.
+ pass
+ else:
+ # Server was provided by parent. Can't recover if it's
+ # unresettable.
+ return reset()
+
+ # Before trying to start up our own server, call stop() to avoid
+ # "Phusion Passenger Standalone is already running on PID 12345".
+ # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
+ # we know the server is ours to kill.)
+ stop(force=True)
+
+ restore_cwd = os.getcwd()
+ api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
+ os.chdir(api_src_dir)
+
+ # Either we haven't started a server of our own yet, or it has
+ # died, or we have lost our credentials, or something else is
+ # preventing us from calling reset(). Start a new one.
+
+ if not os.path.exists('tmp/self-signed.pem'):
+ # We assume here that either passenger reports its listening
+ # address as https:/0.0.0.0:port/. If it reports "127.0.0.1"
+ # then the certificate won't match the host and reset() will
+ # fail certificate verification. If it reports "localhost",
+ # clients (notably Python SDK's websocket client) might
+ # resolve localhost as ::1 and then fail to connect.
+ subprocess.check_call([
+ 'openssl', 'req', '-new', '-x509', '-nodes',
+ '-out', 'tmp/self-signed.pem',
+ '-keyout', 'tmp/self-signed.key',
+ '-days', '3650',
+ '-subj', '/CN=0.0.0.0'],
+ stdout=sys.stderr)
+
+ port = find_available_port()
+ env = os.environ.copy()
+ env['RAILS_ENV'] = 'test'
+ env['ARVADOS_WEBSOCKETS'] = 'yes'
+ env.pop('ARVADOS_TEST_API_HOST', None)
+ env.pop('ARVADOS_API_HOST', None)
+ env.pop('ARVADOS_API_HOST_INSECURE', None)
+ env.pop('ARVADOS_API_TOKEN', None)
+ start_msg = subprocess.check_output(
+ ['bundle', 'exec',
+ 'passenger', 'start', '-d', '-p{}'.format(port),
+ '--pid-file', os.path.join(os.getcwd(), pid_file),
+ '--log-file', os.path.join(os.getcwd(), 'log/test.log'),
+ '--ssl',
+ '--ssl-certificate', 'tmp/self-signed.pem',
+ '--ssl-certificate-key', 'tmp/self-signed.key'],
+ env=env)
+
+ if not leave_running_atexit:
+ atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
+
+ match = re.search(r'Accessible via: https://(.*?)/', start_msg)
+ if not match:
+ raise Exception(
+ "Passenger did not report endpoint: {}".format(start_msg))
+ my_api_host = match.group(1)
+ os.environ['ARVADOS_API_HOST'] = my_api_host
+
+ # Make sure the server has written its pid file before continuing
+ find_server_pid(pid_file)
+
+ reset()
+ os.chdir(restore_cwd)
+
+def reset():
+ """Reset the test server to fixture state.
+
+ This resets the ARVADOS_TEST_API_HOST provided by a parent process
+ if any, otherwise the server started by run().
+
+ It also resets ARVADOS_* environment vars to point to the test
+ server with admin credentials.
+ """
+ existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
+ token = auth_token('admin')
+ httpclient = httplib2.Http(ca_certs=os.path.join(
+ SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem'))
+ httpclient.request(
+ 'https://{}/database/reset'.format(existing_api_host),
+ 'POST',
+ headers={'Authorization': 'OAuth2 {}'.format(token)})
+ os.environ['ARVADOS_API_HOST_INSECURE'] = 'true'
+ os.environ['ARVADOS_API_HOST'] = existing_api_host
+ os.environ['ARVADOS_API_TOKEN'] = token
+
+def stop(force=False):
+ """Stop the API server, if one is running.
+
+ If force==False, kill it only if we started it ourselves. (This
+ supports the use case where a Python test suite calls run(), but
+ run() just uses the ARVADOS_TEST_API_HOST provided by the parent
+ process, and the test suite cleans up after itself by calling
+ stop(). In this case the test server provided by the parent
+ process should be left alone.)
+
+ If force==True, kill it even if we didn't start it
+ ourselves. (This supports the use case in __main__, where "run"
+ and "stop" happen in different processes.)
+ """
+ global my_api_host
+ if force or my_api_host is not None:
+ kill_server_pid(os.path.join(SERVICES_SRC_DIR, 'api', SERVER_PID_PATH))
+ my_api_host = None
def _start_keep(n, keep_args):
keep0 = tempfile.mkdtemp()
+ port = find_available_port()
keep_cmd = ["keepstore",
"-volumes={}".format(keep0),
- "-listen=:{}".format(25107+n),
+ "-listen=:{}".format(port),
"-pid={}".format("{}/keep{}.pid".format(TEST_TMPDIR, n))]
for arg, val in keep_args.iteritems():
with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
f.write(keep0)
+ return port
+
def run_keep(blob_signing_key=None, enforce_permissions=False):
stop_keep()
- if not os.path.exists(TEST_TMPDIR):
- os.mkdir(TEST_TMPDIR)
-
keep_args = {}
if blob_signing_key:
with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
if enforce_permissions:
keep_args['--enforce-permissions'] = 'true'
- _start_keep(0, keep_args)
- _start_keep(1, keep_args)
-
- os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
- os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
-
- authorize_with("admin")
- api = arvados.api('v1', cache=False)
+ api = arvados.api(
+ 'v1', cache=False,
+ host=os.environ['ARVADOS_API_HOST'],
+ token=os.environ['ARVADOS_API_TOKEN'],
+ insecure=True)
for d in api.keep_services().list().execute()['items']:
api.keep_services().delete(uuid=d['uuid']).execute()
for d in api.keep_disks().list().execute()['items']:
api.keep_disks().delete(uuid=d['uuid']).execute()
- s1 = api.keep_services().create(body={"keep_service": {
- "uuid": "zzzzz-bi6l4-5bo5n1iekkjyz6b",
- "service_host": "localhost",
- "service_port": 25107,
- "service_type": "disk"
- }}).execute()
- s2 = api.keep_services().create(body={"keep_service": {
- "uuid": "zzzzz-bi6l4-2nz60e0ksj7vr3s",
- "service_host": "localhost",
- "service_port": 25108,
- "service_type": "disk"
- }}).execute()
- api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s1["uuid"] } }).execute()
- api.keep_disks().create(body={"keep_disk": {"keep_service_uuid": s2["uuid"] } }).execute()
+ for d in range(0, 2):
+ port = _start_keep(d, keep_args)
+ svc = api.keep_services().create(body={'keep_service': {
+ 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d),
+ 'service_host': 'localhost',
+ 'service_port': port,
+ 'service_type': 'disk',
+ 'service_ssl_flag': False,
+ }}).execute()
+ api.keep_disks().create(body={
+ 'keep_disk': {'keep_service_uuid': svc['uuid'] }
+ }).execute()
def _stop_keep(n):
kill_server_pid("{}/keep{}.pid".format(TEST_TMPDIR, n), 0)
_stop_keep(0)
_stop_keep(1)
-def run_keep_proxy(auth):
+def run_keep_proxy():
stop_keep_proxy()
- if not os.path.exists(TEST_TMPDIR):
- os.mkdir(TEST_TMPDIR)
-
- os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
- os.environ["ARVADOS_API_HOST_INSECURE"] = "true"
- os.environ["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[auth]["api_token"]
-
- kp0 = subprocess.Popen(["keepproxy",
- "-pid={}/keepproxy.pid".format(TEST_TMPDIR),
- "-listen=:{}".format(25101)])
-
- authorize_with("admin")
- api = arvados.api('v1', cache=False)
- api.keep_services().create(body={"keep_service": {"service_host": "localhost", "service_port": 25101, "service_type": "proxy"} }).execute()
-
- os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:25101"
+ admin_token = auth_token('admin')
+ port = find_available_port()
+ env = os.environ.copy()
+ env['ARVADOS_API_TOKEN'] = admin_token
+ kp = subprocess.Popen(
+ ['keepproxy',
+ '-pid={}/keepproxy.pid'.format(TEST_TMPDIR),
+ '-listen=:{}'.format(port)],
+ env=env)
+
+ api = arvados.api(
+ 'v1', cache=False,
+ host=os.environ['ARVADOS_API_HOST'],
+ token=admin_token,
+ insecure=True)
+ for d in api.keep_services().list(
+ filters=[['service_type','=','proxy']]).execute()['items']:
+ api.keep_services().delete(uuid=d['uuid']).execute()
+ api.keep_services().create(body={'keep_service': {
+ 'service_host': 'localhost',
+ 'service_port': port,
+ 'service_type': 'proxy',
+ 'service_ssl_flag': False,
+ }}).execute()
+ os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:{}".format(port)
def stop_keep_proxy():
kill_server_pid(os.path.join(TEST_TMPDIR, "keepproxy.pid"), 0)
pass
return yaml.load(yaml_file)
-def authorize_with(token):
- '''token is the symbolic name of the token from the api_client_authorizations fixture'''
- arvados.config.settings()["ARVADOS_API_TOKEN"] = fixture("api_client_authorizations")[token]["api_token"]
+def auth_token(token_name):
+ return fixture("api_client_authorizations")[token_name]["api_token"]
+
+def authorize_with(token_name):
+ '''token_name is the symbolic name of the token from the api_client_authorizations fixture'''
+ arvados.config.settings()["ARVADOS_API_TOKEN"] = auth_token(token_name)
arvados.config.settings()["ARVADOS_API_HOST"] = os.environ.get("ARVADOS_API_HOST")
arvados.config.settings()["ARVADOS_API_HOST_INSECURE"] = "true"
cls._orig_environ = os.environ.copy()
cls._orig_config = arvados.config.settings().copy()
cls._cleanup_funcs = []
+ os.environ.pop('ARVADOS_KEEP_PROXY', None)
+ os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
for server_kwargs, start_func, stop_func in (
- (cls.MAIN_SERVER, run, stop),
- (cls.KEEP_SERVER, run_keep, stop_keep),
- (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy)):
+ (cls.MAIN_SERVER, run, reset),
+ (cls.KEEP_SERVER, run_keep, stop_keep),
+ (cls.KEEP_PROXY_SERVER, run_keep_proxy, stop_keep_proxy)):
if server_kwargs is not None:
start_func(**server_kwargs)
cls._cleanup_funcs.append(stop_func)
- os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
- if cls.KEEP_PROXY_SERVER is None:
- os.environ.pop('ARVADOS_KEEP_PROXY', None)
if (cls.KEEP_SERVER is None) and (cls.KEEP_PROXY_SERVER is None):
cls.local_store = tempfile.mkdtemp()
os.environ['KEEP_LOCAL_STORE'] = cls.local_store
if __name__ == "__main__":
+ actions = ['start', 'stop',
+ 'start_keep', 'stop_keep',
+ 'start_keep_proxy', 'stop_keep_proxy']
parser = argparse.ArgumentParser()
- parser.add_argument('action', type=str, help='''one of "start", "stop", "start_keep", "stop_keep"''')
- parser.add_argument('--websockets', action='store_true', default=False)
- parser.add_argument('--reuse', action='store_true', default=False)
- parser.add_argument('--auth', type=str, help='Print authorization info for given api_client_authorizations fixture')
+ parser.add_argument('action', type=str, help="one of {}".format(actions))
+ parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture')
args = parser.parse_args()
if args.action == 'start':
- run(websockets=args.websockets, reuse_server=args.reuse)
+ stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
+ run(leave_running_atexit=True)
+ host = os.environ['ARVADOS_API_HOST']
if args.auth is not None:
- authorize_with(args.auth)
- print("export ARVADOS_API_HOST={}".format(arvados.config.settings()["ARVADOS_API_HOST"]))
- print("export ARVADOS_API_TOKEN={}".format(arvados.config.settings()["ARVADOS_API_TOKEN"]))
- print("export ARVADOS_API_HOST_INSECURE={}".format(arvados.config.settings()["ARVADOS_API_HOST_INSECURE"]))
+ token = auth_token(args.auth)
+ print("export ARVADOS_API_TOKEN={}".format(pipes.quote(token)))
+ print("export ARVADOS_API_HOST={}".format(pipes.quote(host)))
+ print("export ARVADOS_API_HOST_INSECURE=true")
+ else:
+ print(host)
elif args.action == 'stop':
- stop()
+ stop(force=('ARVADOS_TEST_API_HOST' not in os.environ))
elif args.action == 'start_keep':
run_keep()
elif args.action == 'stop_keep':
stop_keep()
elif args.action == 'start_keep_proxy':
- run_keep_proxy("admin")
+ run_keep_proxy()
elif args.action == 'stop_keep_proxy':
stop_keep_proxy()
else:
- print('Unrecognized action "{}", actions are "start", "stop", "start_keep", "stop_keep"'.format(args.action))
+ print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions))
insecure=True,
requestBuilder=req_builder)
- @classmethod
- def tearDownClass(cls):
- run_test_server.stop()
+ def tearDown(cls):
+ run_test_server.reset()
def test_new_api_objects_with_cache(self):
clients = [arvados.api('v1', cache=True,
class KeepProxyTestCase(run_test_server.TestCaseWithServers):
MAIN_SERVER = {}
KEEP_SERVER = {}
- KEEP_PROXY_SERVER = {'auth': 'admin'}
+ KEEP_PROXY_SERVER = {}
@classmethod
def setUpClass(cls):
super(KeepProxyTestCase, cls).setUpClass()
+ run_test_server.authorize_with('active')
cls.api_client = arvados.api('v1')
def tearDown(self):
import apiclient
import run_test_server
-class PipelineTemplateTest(unittest.TestCase):
- def setUp(self):
- run_test_server.run()
+class PipelineTemplateTest(run_test_server.TestCaseWithServers):
+ MAIN_SERVER = {}
+ KEEP_SERVER = {}
def runTest(self):
run_test_server.authorize_with("admin")
geterror_response = arvados.api('v1').pipeline_templates().get(
uuid=pt_uuid
).execute()
-
- def tearDown(self):
- run_test_server.stop()
import unittest
import arvados
import arvados.events
+import mock
import threading
-class EventTestBase(object):
- def runTest(self):
- run_test_server.authorize_with("admin")
+class WebsocketTest(run_test_server.TestCaseWithServers):
+ MAIN_SERVER = {}
+
+ def setUp(self):
+ self.ws = None
+
+ def tearDown(self):
+ if self.ws:
+ self.ws.close()
+ super(WebsocketTest, self).tearDown()
+
+ def _test_subscribe(self, poll_fallback, expect_type):
+ run_test_server.authorize_with('active')
events = Queue.Queue(3)
self.ws = arvados.events.subscribe(
arvados.api('v1'), [['object_uuid', 'is_a', 'arvados#human']],
- events.put, poll_fallback=2)
- self.assertIsInstance(self.ws, self.WS_TYPE)
+ events.put, poll_fallback=poll_fallback)
+ self.assertIsInstance(self.ws, expect_type)
self.assertEqual(200, events.get(True, 10)['status'])
human = arvados.api('v1').humans().create(body={}).execute()
self.assertEqual(human['uuid'], events.get(True, 10)['object_uuid'])
self.assertTrue(events.empty(), "got more events than expected")
- def tearDown(self):
- try:
- self.ws.close()
- except AttributeError:
- pass
- super(EventTestBase, self).tearDown()
-
+ def test_subscribe_websocket(self):
+ self._test_subscribe(
+ poll_fallback=False, expect_type=arvados.events.EventClient)
-class WebsocketTest(EventTestBase, run_test_server.TestCaseWithServers):
- MAIN_SERVER = {'websockets': True}
- WS_TYPE = arvados.events.EventClient
-
-
-class PollClientTest(EventTestBase, run_test_server.TestCaseWithServers):
- MAIN_SERVER = {}
- WS_TYPE = arvados.events.PollClient
+ @mock.patch('arvados.events.EventClient.__init__')
+ def test_subscribe_poll(self, event_client_constr):
+ event_client_constr.side_effect = Exception('All is well')
+ self._test_subscribe(
+ poll_fallback=1, expect_type=arvados.events.PollClient)
# Ignore all logfiles and tempfiles.
/log/*.log
+/log/*.log.gz
/tmp
# Sensitive files and local configuration
gem 'acts_as_api'
-gem 'passenger', :group => :production
+gem 'passenger'
gem 'omniauth', '1.1.1'
gem 'omniauth-oauth2', '1.1.1'
blob_signing_key: zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc
user_profile_notification_address: arvados@example.com
workbench_address: https://localhost:3001/
- websocket_address: ws://127.0.0.1:3333/websocket
common:
uuid_prefix: <%= Digest::MD5.hexdigest(`hostname`).to_i(16).to_s(36)[0..4] %>
def _system(*cmd)
Bundler.with_clean_env do
if not system({'ARVADOS_WEBSOCKETS' => 'ws-only', 'RAILS_ENV' => 'test'}, *cmd)
- raise RuntimeError, "#{cmd[0]} returned exit code #{$?.exitstatus}"
+ raise RuntimeError, "Command failed with exit status #{$?}: #{cmd.inspect}"
end
end
end
begin
super(args)
ensure
- Process.kill('TERM', server_pid)
+ Dir.chdir($ARV_API_SERVER_DIR) do
+ _system('passenger', 'stop', '-p3002')
+ end
+ # DatabaseCleaner leaves the database empty. Prefer to leave it full.
+ dc = DatabaseController.new
+ dc.define_singleton_method :render do |*args| end
+ dc.reset
end
end
end
).execute(num_retries=self.num_retries)
if "items" in tags:
self.merge(tags['items'],
- lambda i: i['name'] if 'name' in i else i['uuid'],
- lambda a, i: a.tag == i,
+ lambda i: i['name'],
+ lambda a, i: a.tag == i['name'],
lambda i: TagDirectory(self.inode, self.inodes, self.api, self.num_retries, i['name'], poll=self._poll, poll_time=self._poll_time))
self.keeptmp = tempfile.mkdtemp()
os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
self.mounttmp = tempfile.mkdtemp()
- run_test_server.run(False)
+ run_test_server.run()
run_test_server.authorize_with("admin")
- self.api = api = fuse.SafeApi(arvados.config)
+ self.api = fuse.SafeApi(arvados.config)
- def make_mount(self, root_class, *root_args):
+ def make_mount(self, root_class, **root_kwargs):
operations = fuse.Operations(os.getuid(), os.getgid())
operations.inodes.add_entry(root_class(
- llfuse.ROOT_INODE, operations.inodes, self.api, 0, *root_args))
+ llfuse.ROOT_INODE, operations.inodes, self.api, 0, **root_kwargs))
llfuse.init(operations, self.mounttmp, [])
threading.Thread(None, llfuse.main).start()
# wait until the driver is finished initializing
operations.initlock.wait()
def tearDown(self):
- run_test_server.stop()
-
# llfuse.close is buggy, so use fusermount instead.
#llfuse.close(unmount=True)
count = 0
os.rmdir(self.mounttmp)
shutil.rmtree(self.keeptmp)
+ run_test_server.reset()
def assertDirContents(self, subdir, expect_content):
path = self.mounttmp
self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
def runTest(self):
- self.make_mount(fuse.CollectionDirectory, self.testcollection)
+ self.make_mount(fuse.CollectionDirectory, collection=self.testcollection)
self.assertDirContents(None, ['thing1.txt', 'thing2.txt',
'edgecases', 'dir1', 'dir2'])
}}).execute()
def runTest(self):
- operations = fuse.Operations(os.getuid(), os.getgid())
- e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, self.api, 0, poll_time=1))
-
- llfuse.init(operations, self.mounttmp, [])
- t = threading.Thread(None, lambda: llfuse.main())
- t.start()
+ self.make_mount(fuse.TagsDirectory, poll_time=1)
- # wait until the driver is finished initializing
- operations.initlock.wait()
self.assertIn('foo_tag', os.listdir(self.mounttmp))
bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
class FuseSharedTest(MountTestBase):
def runTest(self):
self.make_mount(fuse.SharedDirectory,
- self.api.users().current().execute()['uuid'])
+ exclude=self.api.users().current().execute()['uuid'])
# shared_dirs is a list of the directories exposed
# by fuse.SharedDirectory (i.e. any object visible
class FuseHomeTest(MountTestBase):
def runTest(self):
self.make_mount(fuse.ProjectDirectory,
- self.api.users().current().execute())
+ project_object=self.api.users().current().execute())
d1 = os.listdir(self.mounttmp)
self.assertIn('Unrestricted public data', d1)
"crypto/tls"
"fmt"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
. "gopkg.in/check.v1"
"io"
"net/http"
"net/url"
"os"
- "os/exec"
"strings"
"testing"
"time"
// Tests that require the Keep server running
type ServerRequiredSuite struct{}
-func pythonDir() string {
- cwd, _ := os.Getwd()
- return fmt.Sprintf("%s/../../sdk/python/tests", cwd)
-}
-
// Wait (up to 1 second) for keepproxy to listen on a port. This
// avoids a race condition where we hit a "connection refused" error
// because we start testing the proxy too soon.
}
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
- cwd, _ := os.Getwd()
- defer os.Chdir(cwd)
-
- os.Chdir(pythonDir())
- {
- cmd := exec.Command("python", "run_test_server.py", "start")
- stderr, err := cmd.StderrPipe()
- if err != nil {
- log.Fatalf("Setting up stderr pipe: %s", err)
- }
- go io.Copy(os.Stderr, stderr)
- if err := cmd.Run(); err != nil {
- panic(fmt.Sprintf("'python run_test_server.py start' returned error %s", err))
- }
- }
- {
- cmd := exec.Command("python", "run_test_server.py", "start_keep")
- stderr, err := cmd.StderrPipe()
- if err != nil {
- log.Fatalf("Setting up stderr pipe: %s", err)
- }
- go io.Copy(os.Stderr, stderr)
- if err := cmd.Run(); err != nil {
- panic(fmt.Sprintf("'python run_test_server.py start_keep' returned error %s", err))
- }
- }
+ arvadostest.StartAPI()
+ arvadostest.StartKeep()
+}
- os.Setenv("ARVADOS_API_HOST", "localhost:3000")
- os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
- os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
+func (s *ServerRequiredSuite) SetUpTest(c *C) {
+ arvadostest.ResetEnv()
}
func (s *ServerRequiredSuite) TearDownSuite(c *C) {
- cwd, _ := os.Getwd()
- defer os.Chdir(cwd)
-
- os.Chdir(pythonDir())
- exec.Command("python", "run_test_server.py", "stop_keep").Run()
- exec.Command("python", "run_test_server.py", "stop").Run()
+ arvadostest.StopKeep()
+ arvadostest.StopAPI()
}
func setupProxyService() {
}
}
-func runProxy(c *C, args []string, token string, port int) keepclient.KeepClient {
- os.Args = append(args, fmt.Sprintf("-listen=:%v", port))
- os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
-
- listener = nil
- go main()
- time.Sleep(100 * time.Millisecond)
-
- os.Setenv("ARVADOS_KEEP_PROXY", fmt.Sprintf("http://localhost:%v", port))
- os.Setenv("ARVADOS_API_TOKEN", token)
+func runProxy(c *C, args []string, port int, bogusClientToken bool) keepclient.KeepClient {
+ if bogusClientToken {
+ os.Setenv("ARVADOS_API_TOKEN", "bogus-token")
+ }
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, Equals, nil)
- kc, err := keepclient.MakeKeepClient(&arv)
- c.Assert(err, Equals, nil)
+ kc := keepclient.KeepClient{
+ Arvados: &arv,
+ Want_replicas: 2,
+ Using_proxy: true,
+ Client: &http.Client{},
+ }
+ kc.SetServiceRoots(map[string]string{
+ "proxy": fmt.Sprintf("http://localhost:%v", port),
+ })
c.Check(kc.Using_proxy, Equals, true)
c.Check(len(kc.ServiceRoots()), Equals, 1)
for _, root := range kc.ServiceRoots() {
c.Check(root, Equals, fmt.Sprintf("http://localhost:%v", port))
}
- os.Setenv("ARVADOS_KEEP_PROXY", "")
log.Print("keepclient created")
+ if bogusClientToken {
+ arvadostest.ResetEnv()
+ }
+
+ {
+ os.Args = append(args, fmt.Sprintf("-listen=:%v", port))
+ listener = nil
+ go main()
+ }
+
return kc
}
log.Print("TestPutAndGet start")
os.Args = []string{"keepproxy", "-listen=:29950"}
- os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
listener = nil
go main()
time.Sleep(100 * time.Millisecond)
c.Check(root, Equals, "http://localhost:29950")
}
os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
- log.Print("keepclient created")
waitForListener()
defer closeListener()
func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
log.Print("TestPutAskGetForbidden start")
- kc := runProxy(c, []string{"keepproxy"}, "123abc", 29951)
+ kc := runProxy(c, []string{"keepproxy"}, 29951, true)
waitForListener()
defer closeListener()
- log.Print("keepclient created")
-
hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
{
func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
log.Print("TestGetDisabled start")
- kc := runProxy(c, []string{"keepproxy", "-no-get"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29952)
+ kc := runProxy(c, []string{"keepproxy", "-no-get"}, 29952, false)
waitForListener()
defer closeListener()
func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
log.Print("TestPutDisabled start")
- kc := runProxy(c, []string{"keepproxy", "-no-put"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29953)
+ kc := runProxy(c, []string{"keepproxy", "-no-put"}, 29953, false)
waitForListener()
defer closeListener()
}
func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
- runProxy(c, []string{"keepproxy"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29954)
+ runProxy(c, []string{"keepproxy"}, 29954, false)
waitForListener()
defer closeListener()
}
func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
- runProxy(c, []string{"keepproxy"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29955)
+ runProxy(c, []string{"keepproxy"}, 29955, false)
waitForListener()
defer closeListener()
projects / arvados.git / commitdiff