import arvados
import json
from arvados.events import subscribe
+import signal
+import subprocess
def main(arguments=None):
logger = logging.getLogger('arvados.arv-ws')
parser = argparse.ArgumentParser()
- parser.add_argument('-u', '--uuid', type=str, default="")
- parser.add_argument('-f', '--filters', type=str, default="")
- parser.add_argument('-p', '--pipeline', type=str, default="", help="Print log output from a pipeline and its jobs")
- parser.add_argument('-j', '--job', type=str, default="", help="Print log output from a job")
+ parser.add_argument('-u', '--uuid', type=str, default="", help="Filter events on object_uuid")
+ parser.add_argument('-f', '--filters', type=str, default="", help="Arvados query filter to apply to log events (JSON encoded)")
+
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument('--poll-interval', default=15, type=int, help="If websockets is not available, specify the polling interval, default is every 15 seconds")
+ group.add_argument('--no-poll', action='store_false', dest='poll_interval', help="Do not poll if websockets are not available, just fail")
+
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument('-p', '--pipeline', type=str, default="", help="Supply pipeline uuid, print log output from pipeline and its jobs")
+ group.add_argument('-j', '--job', type=str, default="", help="Supply job uuid, print log output from jobs")
+ group.add_argument('--project', type=str, default="", help="Monitor change events for a specific project")
+
+ parser.add_argument('--command', type=str, default="", help="Command to run when project event occurs")
+
args = parser.parse_args(arguments)
+ global filters
+ global known_component_jobs
+ global ws
+
filters = []
+ known_component_jobs = set()
+ ws = None
+
+ def update_subscribed_components(components):
+ global known_component_jobs
+ global filters
+ pipeline_jobs = set()
+ for c in components:
+ if "job" in components[c]:
+ pipeline_jobs.add(components[c]["job"]["uuid"])
+ if known_component_jobs != pipeline_jobs:
+ ws.unsubscribe(filters)
+ filters = [['object_uuid', 'in', [args.pipeline] + list(pipeline_jobs)]]
+ ws.subscribe([['object_uuid', 'in', [args.pipeline] + list(pipeline_jobs)]])
+ known_component_jobs = pipeline_jobs
+
+ api = arvados.api('v1', cache=False)
+
if args.uuid:
filters += [ ['object_uuid', '=', args.uuid] ]
if args.filters:
filters += json.loads(args.filters)
- if args.pipeline:
- filters += [ ['object_uuid', '=', args.pipeline] ]
-
if args.job:
filters += [ ['object_uuid', '=', args.job] ]
- api = arvados.api('v1', cache=False)
+ if args.pipeline:
+ filters += [ ['object_uuid', '=', args.pipeline] ]
def on_message(ev):
- print json.dumps(ev)
+ global filters
+ global ws
+
+ logger.debug(ev)
+ if 'event_type' in ev and (args.pipeline or args.job):
+ if ev['event_type'] in ('stderr', 'stdout'):
+ sys.stdout.write(ev["properties"]["text"])
+ elif ev["event_type"] in ("create", "update"):
+ if ev["object_kind"] == "arvados#pipelineInstance":
+ update_subscribed_components(ev["properties"]["new_attributes"]["components"])
+
+ if ev["object_kind"] == "arvados#pipelineInstance" and args.pipeline:
+ if ev["properties"]["new_attributes"]["state"] in ("Complete", "Failed", "Paused"):
+ ws.close()
+
+ if ev["object_kind"] == "arvados#job" and args.job:
+ if ev["properties"]["new_attributes"]["state"] in ("Complete", "Failed", "Cancelled"):
+ ws.close()
+ return
+
+ if 'status' in ev and ev['status'] == 200:
+ return
+
+ if args.project:
+ old_attr = None
+ if 'old_attributes' in ev['properties'] and ev['properties']['old_attributes']:
+ old_attr = ev['properties']['old_attributes']
+ if args.project not in (ev['properties']['new_attributes']['owner_uuid'],
+ old_attr['owner_uuid'] if old_attr else None):
+ return
+
+ if args.command:
+ et = ev['event_type']
+ if ev['event_type'] == 'update' and ev['properties']['new_attributes']['owner_uuid'] != ev['properties']['old_attributes']['owner_uuid']:
+ if args.project == ev['properties']['new_attributes']['owner_uuid']:
+ et = 'add'
+ else:
+ et = 'remove'
+ subprocess.call([args.command, args.project, et, ev['object_uuid']])
+ return
+
+ print json.dumps(ev)
- ws = None
try:
- ws = subscribe(api, filters, lambda ev: on_message(ev))
- ws.run_forever()
+ ws = subscribe(arvados.api('v1', cache=False), filters, on_message, poll_fallback=args.poll_interval)
+ if ws:
+ if args.pipeline:
+ c = api.pipeline_instances().get(uuid=args.pipeline).execute()
+ update_subscribed_components(c["components"])
+ if c["state"] in ("Complete", "Failed", "Paused"):
+ ws.close()
+ ws.run_forever()
except KeyboardInterrupt:
pass
- except Exception:
- logger.exception('')
+ except Exception as e:
+ logger.error(e)
finally:
if ws:
- ws.close_connection()
+ ws.close()
projects / arvados.git / blobdiff