ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
+
+# Work around https://bugs.python.org/issue27805, should be no longer
+# necessary from sometime in Python 3.8.x
+if not os.environ.get('ARVADOS_DEBUG', ''):
+ WRITE_MODE = 'a'
+else:
+ WRITE_MODE = 'w'
+
if 'GOPATH' in os.environ:
# Add all GOPATH bin dirs to PATH -- but insert them after the
# ruby gems bin dir, to ensure "bundle" runs the Ruby bundler
my_api_host = None
_cached_config = {}
_cached_db_config = {}
+_already_used_port = {}
def find_server_pid(PID_PATH, wait=10):
now = time.time()
would take care of the races, and this wouldn't be needed at all.
"""
- sock = socket.socket()
- sock.bind(('0.0.0.0', 0))
- port = sock.getsockname()[1]
- sock.close()
- return port
+ global _already_used_port
+ while True:
+ sock = socket.socket()
+ sock.bind(('0.0.0.0', 0))
+ port = sock.getsockname()[1]
+ sock.close()
+ if port not in _already_used_port:
+ _already_used_port[port] = True
+ return port
def _wait_until_port_listens(port, timeout=10, warn=True):
"""Wait for a process to start listening on the given port.
port = internal_port_from_config("RailsAPI")
env = os.environ.copy()
env['RAILS_ENV'] = 'test'
+ env['ARVADOS_RAILS_LOG_TO_STDOUT'] = '1'
env.pop('ARVADOS_WEBSOCKETS', None)
env.pop('ARVADOS_TEST_API_HOST', None)
env.pop('ARVADOS_API_HOST', None)
env.pop('ARVADOS_API_HOST_INSECURE', None)
env.pop('ARVADOS_API_TOKEN', None)
- start_msg = subprocess.check_output(
+ logf = open(_logfilename('railsapi'), WRITE_MODE)
+ railsapi = subprocess.Popen(
['bundle', 'exec',
- 'passenger', 'start', '-d', '-p{}'.format(port),
+ 'passenger', 'start', '-p{}'.format(port),
'--pid-file', pid_file,
- '--log-file', os.path.join(os.getcwd(), 'log/test.log'),
+ '--log-file', '/dev/stdout',
'--ssl',
'--ssl-certificate', 'tmp/self-signed.pem',
'--ssl-certificate-key', 'tmp/self-signed.key'],
- env=env)
+ env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
if not leave_running_atexit:
atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
- match = re.search(r'Accessible via: https://(.*?)/', start_msg)
- if not match:
- raise Exception(
- "Passenger did not report endpoint: {}".format(start_msg))
- my_api_host = match.group(1)
+ my_api_host = "127.0.0.1:"+str(port)
os.environ['ARVADOS_API_HOST'] = my_api_host
# Make sure the server has written its pid file and started
# listening on its TCP port
- find_server_pid(pid_file)
_wait_until_port_listens(port)
+ find_server_pid(pid_file)
reset()
os.chdir(restore_cwd)
if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
return
stop_controller()
- logf = open(_logfilename('controller'), 'a')
+ logf = open(_logfilename('controller'), WRITE_MODE)
port = internal_port_from_config("Controller")
controller = subprocess.Popen(
["arvados-server", "controller"],
return
stop_ws()
port = internal_port_from_config("Websocket")
- logf = open(_logfilename('ws'), 'a')
- ws = subprocess.Popen(["ws"],
+ logf = open(_logfilename('ws'), WRITE_MODE)
+ ws = subprocess.Popen(
+ ["arvados-server", "ws"],
stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
with open(_pidfile('ws'), 'w') as f:
f.write(str(ws.pid))
yaml.safe_dump(confdata, f)
keep_cmd = ["keepstore", "-config", conf]
- with open(_logfilename('keep{}'.format(n)), 'a') as logf:
+ with open(_logfilename('keep{}'.format(n)), WRITE_MODE) as logf:
with open('/dev/null') as _stdin:
child = subprocess.Popen(
keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True)
port = internal_port_from_config("Keepproxy")
env = os.environ.copy()
env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
- logf = open(_logfilename('keepproxy'), 'a')
+ logf = open(_logfilename('keepproxy'), WRITE_MODE)
kp = subprocess.Popen(
['keepproxy'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
gitport = internal_port_from_config("GitHTTP")
env = os.environ.copy()
env.pop('ARVADOS_API_TOKEN', None)
- logf = open(_logfilename('arv-git-httpd'), 'a')
+ logf = open(_logfilename('arv-git-httpd'), WRITE_MODE)
agh = subprocess.Popen(['arv-git-httpd'],
env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
with open(_pidfile('arv-git-httpd'), 'w') as f:
keepwebport = internal_port_from_config("WebDAV")
env = os.environ.copy()
- logf = open(_logfilename('keep-web'), 'a')
+ logf = open(_logfilename('keep-web'), WRITE_MODE)
keepweb = subprocess.Popen(
['keep-web'],
env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
return
stop_nginx()
nginxconf = {}
+ nginxconf['LISTENHOST'] = 'localhost'
nginxconf['CONTROLLERPORT'] = internal_port_from_config("Controller")
nginxconf['CONTROLLERSSLPORT'] = external_port_from_config("Controller")
nginxconf['KEEPWEBPORT'] = internal_port_from_config("WebDAV")
nginxconf['KEEPPROXYSSLPORT'] = external_port_from_config("Keepproxy")
nginxconf['GITPORT'] = internal_port_from_config("GitHTTP")
nginxconf['GITSSLPORT'] = external_port_from_config("GitHTTP")
+ nginxconf['HEALTHPORT'] = internal_port_from_config("Health")
+ nginxconf['HEALTHSSLPORT'] = external_port_from_config("Health")
nginxconf['WSPORT'] = internal_port_from_config("Websocket")
nginxconf['WSSSLPORT'] = external_port_from_config("Websocket")
nginxconf['WORKBENCH1PORT'] = internal_port_from_config("Workbench1")
conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
with open(conffile, 'w') as f:
f.write(re.sub(
- r'{{([A-Z]+)}}',
+ r'{{([A-Z]+[A-Z0-9]+)}}',
lambda match: str(nginxconf.get(match.group(1))),
open(conftemplatefile).read()))
workbench1_external_port = find_available_port()
git_httpd_port = find_available_port()
git_httpd_external_port = find_available_port()
+ health_httpd_port = find_available_port()
+ health_httpd_external_port = find_available_port()
keepproxy_port = find_available_port()
keepproxy_external_port = find_available_port()
- keepstore_ports = sorted([str(find_available_port()) for _ in xrange(0,4)])
+ keepstore_ports = sorted([str(find_available_port()) for _ in range(0,4)])
keep_web_port = find_available_port()
keep_web_external_port = find_available_port()
keep_web_dl_port = find_available_port()
keep_web_dl_external_port = find_available_port()
- dbconf = os.path.join(os.environ["CONFIGSRC"], "config.yml")
-
- print("Getting config from %s" % dbconf, file=sys.stderr)
-
- pgconnection = yaml.safe_load(open(dbconf))["Clusters"]["zzzzz"]["PostgreSQL"]["Connection"]
+ configsrc = os.environ.get("CONFIGSRC", None)
+ if configsrc:
+ clusterconf = os.path.join(configsrc, "config.yml")
+ print("Getting config from %s" % clusterconf, file=sys.stderr)
+ pgconnection = yaml.safe_load(open(clusterconf))["Clusters"]["zzzzz"]["PostgreSQL"]["Connection"]
+ else:
+ # assume "arvados-server install -type test" has set up the
+ # conventional db credentials
+ pgconnection = {
+ "client_encoding": "utf8",
+ "host": "localhost",
+ "dbname": "arvados_test",
+ "user": "arvados",
+ "password": "insecure_arvados_test",
+ }
localhost = "127.0.0.1"
services = {
"http://%s:%s"%(localhost, git_httpd_port): {}
},
},
+ "Health": {
+ "ExternalURL": "https://%s:%s" % (localhost, health_httpd_external_port),
+ "InternalURLs": {
+ "http://%s:%s"%(localhost, health_httpd_port): {}
+ },
+ },
"Keepstore": {
"InternalURLs": {
"http://%s:%s"%(localhost, port): {} for port in keepstore_ports
"http://%s:%s"%(localhost, keep_web_dl_port): {},
},
},
+ "SSO": {
+ "ExternalURL": "http://localhost:3002",
+ },
}
config = {
"SystemRootToken": auth_token('system_user'),
"API": {
"RequestTimeout": "30s",
+ "RailsSessionSecretToken": "e24205c490ac07e028fd5f8a692dcb398bcd654eff1aef5f9fe6891994b18483",
+ },
+ "Login": {
+ "SSO": {
+ "ProviderAppID": "arvados-server",
+ "ProviderAppSecret": "608dbf356a327e2d0d4932b60161e212c2d8d8f5e25690d7b622f850a990cd33",
+ },
},
"SystemLogs": {
"LogLevel": ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
"Services": services,
"Users": {
"AnonymousUserToken": auth_token('anonymous'),
+ "UserProfileNotificationAddress": "arvados@example.com",
},
"Collections": {
"BlobSigningKey": "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc",
- "TrustAllContent": True,
+ "TrustAllContent": False,
"ForwardSlashNameSubstitution": "/",
+ "TrashSweepInterval": "-1s",
},
"Git": {
- "Repositories": "%s/test" % os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git'),
+ "Repositories": os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git', 'test'),
+ },
+ "Containers": {
+ "JobsAPI": {
+ "GitInternalDir": os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'internal.git'),
+ },
+ "SupportedDockerImageFormats": {"v1": {}},
},
"Volumes": {
"zzzzz-nyw5e-%015d"%n: {
projects / arvados.git / blobdiff