X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/758d59141102b29371f44512671b9efe0cf62821..bdb92619b5f6d920119b8c32c3027cf4b751ed16:/sdk/python/arvados/commands/ws.py diff --git a/sdk/python/arvados/commands/ws.py b/sdk/python/arvados/commands/ws.py index c27503cc1f..72ef1befed 100644 --- a/sdk/python/arvados/commands/ws.py +++ b/sdk/python/arvados/commands/ws.py @@ -6,47 +6,76 @@ import argparse import arvados import json from arvados.events import subscribe +from arvados._version import __version__ +import signal def main(arguments=None): logger = logging.getLogger('arvados.arv-ws') parser = argparse.ArgumentParser() + parser.add_argument('--version', action='version', + version="%s %s" % (sys.argv[0], __version__), + help='Print version and exit.') parser.add_argument('-u', '--uuid', type=str, default="", help="Filter events on object_uuid") parser.add_argument('-f', '--filters', type=str, default="", help="Arvados query filter to apply to log events (JSON encoded)") + parser.add_argument('-s', '--start-time', type=str, default="", help="Arvados query filter to fetch log events created at or after this time. This will be server time in UTC. Allowed format: YYYY-MM-DD or YYYY-MM-DD hh:mm:ss") + parser.add_argument('-i', '--id', type=int, default=None, help="Start from given log id.") - group = parser.add_argument_group('Polling fallback') - group.add_argument('--poll-interval', default=15, help="If websockets is not available, specify the polling interval, default is every 15 seconds") - group.add_argument('--no-poll', action='store_false', dest='poll_fallback', help="Do not poll if websockets are not available, just fail") + group = parser.add_mutually_exclusive_group() + group.add_argument('--poll-interval', default=15, type=int, help="If websockets is not available, specify the polling interval, default is every 15 seconds") + group.add_argument('--no-poll', action='store_false', dest='poll_interval', help="Do not poll if websockets are not available, just fail") - group = parser.add_argument_group('Jobs and Pipelines') + group = parser.add_mutually_exclusive_group() group.add_argument('-p', '--pipeline', type=str, default="", help="Supply pipeline uuid, print log output from pipeline and its jobs") group.add_argument('-j', '--job', type=str, default="", help="Supply job uuid, print log output from jobs") args = parser.parse_args(arguments) global filters + global known_component_jobs + global ws + filters = [] + known_component_jobs = set() + ws = None + + def update_subscribed_components(components): + global known_component_jobs + global filters + pipeline_jobs = set() + for c in components: + if "job" in components[c]: + pipeline_jobs.add(components[c]["job"]["uuid"]) + if known_component_jobs != pipeline_jobs: + ws.unsubscribe(filters) + filters = [['object_uuid', 'in', [args.pipeline] + list(pipeline_jobs)]] + ws.subscribe([['object_uuid', 'in', [args.pipeline] + list(pipeline_jobs)]]) + known_component_jobs = pipeline_jobs + + api = arvados.api('v1') + if args.uuid: filters += [ ['object_uuid', '=', args.uuid] ] if args.filters: filters += json.loads(args.filters) - if args.pipeline: - filters += [ ['object_uuid', '=', args.pipeline] ] - if args.job: filters += [ ['object_uuid', '=', args.job] ] - api = arvados.api('v1', cache=False) + if args.pipeline: + filters += [ ['object_uuid', '=', args.pipeline] ] - global known_component_jobs - global ws + if args.start_time: + last_log_id = 1 + filters += [ ['created_at', '>=', args.start_time] ] + else: + last_log_id = None + + if args.id: + last_log_id = args.id-1 - known_component_jobs = set() - ws = None def on_message(ev): - global known_component_jobs global filters global ws @@ -56,25 +85,33 @@ def main(arguments=None): sys.stdout.write(ev["properties"]["text"]) elif ev["event_type"] in ("create", "update"): if ev["object_kind"] == "arvados#pipelineInstance": - pipeline_jobs = set() - for c in ev["properties"]["new_attributes"]["components"]: - if "job" in ev["properties"]["new_attributes"]["components"][c]: - pipeline_jobs.add(ev["properties"]["new_attributes"]["components"][c]["job"]["uuid"]) - if known_component_jobs != pipeline_jobs: - ws.unsubscribe(filters) - filters = [['object_uuid', 'in', [args.pipeline] + list(pipeline_jobs)]] - ws.subscribe([['object_uuid', 'in', [args.pipeline] + list(pipeline_jobs)]]) - known_component_jobs = pipeline_jobs + update_subscribed_components(ev["properties"]["new_attributes"]["components"]) + + if ev["object_kind"] == "arvados#pipelineInstance" and args.pipeline: + if ev["properties"]["new_attributes"]["state"] in ("Complete", "Failed", "Paused"): + ws.close() + + if ev["object_kind"] == "arvados#job" and args.job: + if ev["properties"]["new_attributes"]["state"] in ("Complete", "Failed", "Cancelled"): + ws.close() + elif 'status' in ev and ev['status'] == 200: + pass else: print json.dumps(ev) try: - ws = subscribe(api, filters, on_message, poll_fallback=args.poll_fallback) - ws.run_forever() + ws = subscribe(arvados.api('v1'), filters, on_message, poll_fallback=args.poll_interval, last_log_id=last_log_id) + if ws: + if args.pipeline: + c = api.pipeline_instances().get(uuid=args.pipeline).execute() + update_subscribed_components(c["components"]) + if c["state"] in ("Complete", "Failed", "Paused"): + ws.close() + ws.run_forever() except KeyboardInterrupt: pass - except Exception: - logger.exception('') + except Exception as e: + logger.error(e) finally: if ws: ws.close()