13108: Refactor signal handling

a-c-r handles SystemExit the same way as KeyboardInterrupt.

Bump version of cwltest (to get the terminate-on-timeout bugfix).

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/build/build.list b/build/build.list
index da7c0305c77bcdba3493dae43d0b3081fd8fd417..5ef3899cbe86063d714296e31900119f3514aa2f 100644 (file)
--- a/build/build.list
+++ b/build/build.list
@@ -43,7 +43,7 @@ centos7|keepalive|0.5|2|python|all
 debian8,debian9,ubuntu1204,ubuntu1404,ubuntu1604,centos7|lockfile|0.12.2|2|python|all|--epoch 1
 debian8,ubuntu1404,centos7|subprocess32|3.2.7|2|python|all
 all|ruamel.yaml|0.13.7|2|python|amd64|--python-setup-py-arguments --single-version-externally-managed
-all|cwltest|1.0.20180209171722|4|python|all|--depends 'python-futures >= 3.0.5' --depends 'python-subprocess32'
+all|cwltest|1.0.20180413145017|4|python|all|--depends 'python-futures >= 3.0.5' --depends 'python-subprocess32'
 all|junit-xml|1.7|3|python|all
 all|rdflib-jsonld|0.4.0|2|python|all
 all|futures|3.0.5|2|python|all

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 2fa6da703464fcb74be35ccc9b42dbc5c65508fe..c74fa50bad6f2907cdf652c7c2761e86078eb3c9 100644 (file)
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -33,6 +33,7 @@ import arvados
 import arvados.config
 from arvados.keep import KeepClient
 from arvados.errors import ApiError
+import arvados.commands._util as arv_cmd
 
 from .arvcontainer import ArvadosContainer, RunnerContainer
 from .arvjob import ArvadosJob, RunnerJob, RunnerTemplate
@@ -253,7 +254,7 @@ class ArvCwlRunner(object):
                 self.api.collections().delete(uuid=i).execute(num_retries=self.num_retries)
             except:
                 logger.warn("Failed to delete intermediate output: %s", sys.exc_info()[1], exc_info=(sys.exc_info()[1] if self.debug else False))
-            if sys.exc_info()[0] is KeyboardInterrupt:
+            if sys.exc_info()[0] is KeyboardInterrupt or sys.exc_info()[0] is SystemExit:
                 break
 
     def check_features(self, obj):
@@ -565,7 +566,7 @@ class ArvCwlRunner(object):
         except UnsupportedRequirement:
             raise
         except:
-            if sys.exc_info()[0] is KeyboardInterrupt:
+            if sys.exc_info()[0] is KeyboardInterrupt or sys.exc_info()[0] is SystemExit:
                 logger.error("Interrupted, workflow will be cancelled")
             else:
                 logger.error("Execution failed: %s", sys.exc_info()[1], exc_info=(sys.exc_info()[1] if self.debug else False))
@@ -763,6 +764,10 @@ def add_arv_hints():
         "http://arvados.org/cwl#ReuseRequirement"
     ])
 
+def exit_signal_handler(sigcode, frame):
+    logger.error("Caught signal {}, exiting.".format(sigcode))
+    sys.exit(-sigcode)
+
 def main(args, stdout, stderr, api_client=None, keep_client=None,
          install_sig_handlers=True):
     parser = arg_parser()
@@ -771,7 +776,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
     arvargs = parser.parse_args(args)
 
     if install_sig_handlers:
-        signal.signal(signal.SIGTERM, lambda x, y: thread.interrupt_main())
+        arv_cmd.install_signal_handlers()
 
     if arvargs.update_workflow:
         if arvargs.update_workflow.find('-7fd4e-') == 5:

diff --git a/sdk/python/arvados/commands/_util.py b/sdk/python/arvados/commands/_util.py
index d4d9497d3e298b0c8312243d9684c3119d79a9c0..df7fdba5d9653055f5e34f346bac5ebf867d0ffb 100644 (file)
--- a/sdk/python/arvados/commands/_util.py
+++ b/sdk/python/arvados/commands/_util.py
@@ -5,6 +5,9 @@
 import argparse
 import errno
 import os
+import logging
+import signal
+from future.utils import listitems, listvalues
 
 def _pos_int(s):
     num = int(s)
@@ -44,3 +47,18 @@ def make_home_conf_dir(path, mode=None, errors='ignore'):
         if mode is not None:
             os.chmod(abs_path, mode)
     return abs_path
+
+CAUGHT_SIGNALS = [signal.SIGINT, signal.SIGQUIT, signal.SIGTERM]
+
+def exit_signal_handler(sigcode, frame):
+    logging.getLogger('arvados').error("Caught signal {}, exiting.".format(sigcode))
+    sys.exit(-sigcode)
+
+def install_signal_handlers():
+    global orig_signal_handlers
+    orig_signal_handlers = {sigcode: signal.signal(sigcode, exit_signal_handler)
+                            for sigcode in CAUGHT_SIGNALS}
+
+def restore_signal_handlers():
+    for sigcode, orig_handler in listitems(orig_signal_handlers):
+        signal.signal(sigcode, orig_handler)

diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index af8e243b849f10458cfe0ff058c9890649e53888..6e7e023a78f008c9b1333398a3a64fe219b30b09 100644 (file)
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -34,7 +34,6 @@ from arvados._version import __version__
 
 import arvados.commands._util as arv_cmd
 
-CAUGHT_SIGNALS = [signal.SIGINT, signal.SIGQUIT, signal.SIGTERM]
 api_client = None
 
 upload_opts = argparse.ArgumentParser(add_help=False)
@@ -978,10 +977,6 @@ def progress_writer(progress_func, outfile=sys.stderr):
         outfile.write(progress_func(bytes_written, bytes_expected))
     return write_progress
 
-def exit_signal_handler(sigcode, frame):
-    logging.getLogger('arvados.arv_put').error("Caught signal {}, exiting.".format(sigcode))
-    sys.exit(-sigcode)
-
 def desired_project_uuid(api_client, project_uuid, num_retries):
     if not project_uuid:
         query = api_client.users().current()
@@ -1013,12 +1008,8 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr,
     if api_client is None:
         api_client = arvados.api('v1', request_id=request_id)
 
-    # Install our signal handler for each code in CAUGHT_SIGNALS, and save
-    # the originals.
-    orig_signal_handlers = {}
     if install_sig_handlers:
-        orig_signal_handlers = {sigcode: signal.signal(sigcode, exit_signal_handler)
-                                for sigcode in CAUGHT_SIGNALS}
+        arv_cmd.install_signal_handlers()
 
     # Determine the name to use
     if args.name:
@@ -1185,8 +1176,7 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr,
         if not output.endswith('\n'):
             stdout.write('\n')
 
-    for sigcode, orig_handler in listitems(orig_signal_handlers):
-        signal.signal(sigcode, orig_handler)
+    arv_cmd.restore_signal_handlers()
 
     if status != 0:
         sys.exit(status)