From 51d712fe2f3e46fe6b2358bb20196c6d4e4e26a3 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Tue, 24 Nov 2020 17:03:32 -0500 Subject: [PATCH] 17022: Improve output. Added documentation page. Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- build/run-build-packages-python-and-ruby.sh | 2 + build/run-build-packages.sh | 2 +- doc/_config.yml | 1 + doc/admin/user-activity.html.textile.liquid | 95 +++++++++++++++++++ .../arvados_user_activity/main.py | 27 ++++-- tools/user-activity/bin/arv-user-activity | 8 ++ 6 files changed, 127 insertions(+), 8 deletions(-) create mode 100644 doc/admin/user-activity.html.textile.liquid create mode 100755 tools/user-activity/bin/arv-user-activity diff --git a/build/run-build-packages-python-and-ruby.sh b/build/run-build-packages-python-and-ruby.sh index f3b7564d71..871641fcdb 100755 --- a/build/run-build-packages-python-and-ruby.sh +++ b/build/run-build-packages-python-and-ruby.sh @@ -194,6 +194,8 @@ if [ $PYTHON -eq 1 ]; then python_wrapper arvados-python-client "$WORKSPACE/sdk/python" python_wrapper arvados-cwl-runner "$WORKSPACE/sdk/cwl" python_wrapper arvados_fuse "$WORKSPACE/services/fuse" + python_wrapper crunchstat_summary "$WORKSPACE/tools/crunchstat-summary" + python_wrapper arvados-user-activity "$WORKSPACE/tools/user-activity" if [ $((${#failures[@]} - $GEM_BUILD_FAILURES)) -ne 0 ]; then PYTHON_BUILD_FAILURES=$((${#failures[@]} - $GEM_BUILD_FAILURES)) diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index ddb21c4cca..42c5f3d094 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -327,7 +327,7 @@ fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "python3" # The Docker image cleaner fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3" -# The Arvados crunchstat-summary tool +# The Arvados user activity tool fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "python3" # The cwltest package, which lives out of tree diff --git a/doc/_config.yml b/doc/_config.yml index d56a95c1e2..75a55b469d 100644 --- a/doc/_config.yml +++ b/doc/_config.yml @@ -173,6 +173,7 @@ navbar: - user/topics/arvados-sync-groups.html.textile.liquid - admin/scoped-tokens.html.textile.liquid - admin/token-expiration-policy.html.textile.liquid + - admin/user-activity.html.textile.liquid - Monitoring: - admin/logging.html.textile.liquid - admin/metrics.html.textile.liquid diff --git a/doc/admin/user-activity.html.textile.liquid b/doc/admin/user-activity.html.textile.liquid new file mode 100644 index 0000000000..08d4877720 --- /dev/null +++ b/doc/admin/user-activity.html.textile.liquid @@ -0,0 +1,95 @@ +--- +layout: default +navsection: admin +title: "User activity report" +... +{% comment %} +Copyright (C) The Arvados Authors. All rights reserved. + +SPDX-License-Identifier: CC-BY-SA-3.0 +{% endcomment %} + +The @arv-user-activity@ tool generates a summary report of user activity on an Arvados instance based on the audit logs (the @logs@ table). + +h2. Installation + +h2. Option 1: Install from a distribution package + +This installation method is recommended to make the CLI tools available system-wide. It can coexist with the installation method described in option 2, below. + +First, configure the "Arvados package repositories":../../install/packages.html + +{% assign arvados_component = 'python3-arvados-user-activity' %} + +{% include 'install_packages' %} + +h2. Option 2: Install with pip + +Run @pip install arvados-user-activity'@ in an appropriate installation environment, such as a @virtualenv@. + +Note: depends on the "Arvados Python SDK":../sdk/python/sdk-python.html and its associated build prerequisites (e.g. @pycurl@). + +h2. Usage + +Set your Arvados environment, then run the tool giving it the number of days to report for. It will query the logs and generate a summary report on standard output. + +Example run: + +
+$ bin/arv-user-activity --days 14
+User activity on pirca between 2020-11-10 16:42 and 2020-11-24 16:42
+
+Peter Amstutz  (https://workbench.pirca.arvadosapi.com/users/jutro-tpzed-a4qnxq3pcfcgtkz)
+  organization: "Curii"
+  role: "Software Developer"
+
+  2020-11-10 16:51-05:00 to 2020-11-11 13:51-05:00 (21:00) Account activity
+  2020-11-13 13:47-05:00 to 2020-11-14 03:32-05:00 (13:45) Account activity
+  2020-11-14 04:33-05:00 to 2020-11-15 20:33-05:00 (40:00) Account activity
+  2020-11-15 21:34-05:00 to 2020-11-16 13:34-05:00 (16:00) Account activity
+  2020-11-16 16:21-05:00 to 2020-11-16 16:28-05:00 (00:07) Account activity
+  2020-11-17 15:49-05:00 to 2020-11-17 15:49-05:00 (00:00) Account activity
+  2020-11-17 15:51-05:00 Created project "New project" (pirca-j7d0g-7bxvkyr4khfa1a4)
+  2020-11-17 15:51-05:00 Updated project "Test run" (pirca-j7d0g-7bxvkyr4khfa1a4)
+  2020-11-17 15:51-05:00 Ran container "bwa-mem.cwl container" (pirca-xvhdp-xf2w8dkk17jkk5r)
+  2020-11-17 15:51-05:00 to 2020-11-17 15:51-05:00 (0:00) Account activity
+  2020-11-17 15:53-05:00 Ran container "WGS processing workflow scattered over samples container" (pirca-xvhdp-u7bm0wdy6lq4r8k)
+  2020-11-17 15:53-05:00 to 2020-11-17 15:54-05:00 (00:01) Account activity
+  2020-11-17 15:55-05:00 Created collection "output for pirca-dz642-36ffk81c8zzopxz" (pirca-4zz18-np35gw690ndzzk7)
+  2020-11-17 15:55-05:00 to 2020-11-17 15:55-05:00 (0:00) Account activity
+  2020-11-17 15:55-05:00 Created collection "Output of main" (pirca-4zz18-oiiymetwhnnhhwc)
+  2020-11-17 15:55-05:00 Tagged pirca-4zz18-oiiymetwhnnhhwc
+  2020-11-17 15:55-05:00 Updated collection "Output of main" (pirca-4zz18-oiiymetwhnnhhwc)
+  2020-11-17 15:55-05:00 to 2020-11-17 16:04-05:00 (00:09) Account activity
+  2020-11-17 16:04-05:00 Created collection "Output of main" (pirca-4zz18-f6n9n89e3dhtwvl)
+  2020-11-17 16:04-05:00 Tagged pirca-4zz18-f6n9n89e3dhtwvl
+  2020-11-17 16:04-05:00 Updated collection "Output of main" (pirca-4zz18-f6n9n89e3dhtwvl)
+  2020-11-17 16:04-05:00 to 2020-11-17 17:55-05:00 (01:51) Account activity
+  2020-11-17 20:09-05:00 to 2020-11-17 20:09-05:00 (00:00) Account activity
+  2020-11-17 21:35-05:00 to 2020-11-17 21:35-05:00 (00:00) Account activity
+  2020-11-18 10:09-05:00 to 2020-11-18 11:00-05:00 (00:51) Account activity
+  2020-11-18 14:37-05:00 Untagged pirca-4zz18-st8yzjan1nhxo1a
+  2020-11-18 14:37-05:00 Deleted collection "Output of main" (pirca-4zz18-st8yzjan1nhxo1a)
+  2020-11-18 17:44-05:00 to 2020-11-18 17:44-05:00 (00:00) Account activity
+  2020-11-19 12:18-05:00 to 2020-11-19 12:19-05:00 (00:01) Account activity
+  2020-11-19 13:57-05:00 to 2020-11-19 14:21-05:00 (00:24) Account activity
+  2020-11-20 09:48-05:00 to 2020-11-20 22:51-05:00 (13:03) Account activity
+  2020-11-20 23:52-05:00 to 2020-11-22 22:32-05:00 (46:40) Account activity
+  2020-11-22 23:37-05:00 to 2020-11-23 13:52-05:00 (14:15) Account activity
+  2020-11-23 14:53-05:00 to 2020-11-24 11:58-05:00 (21:05) Account activity
+  2020-11-24 15:06-05:00 to 2020-11-24 16:38-05:00 (01:32) Account activity
+
+Marc Rubenfield  (https://workbench.pirca.arvadosapi.com/users/jutro-tpzed-v9s9q97pgydh1yf)
+  2020-11-11 12:27-05:00 Untagged pirca-4zz18-xmq257bsla4kdco
+  2020-11-11 12:27-05:00 Deleted collection "Output of main" (pirca-4zz18-xmq257bsla4kdco)
+
+Ward Vandewege  (https://workbench.pirca.arvadosapi.com/users/jutro-tpzed-9z6foyez9ydn2hl)
+  organization: "Curii Corporation, Inc."
+  organization_email: "ward@curii.com"
+  role: "System Administrator"
+  website_url: "https://curii.com"
+
+  2020-11-19 19:30-05:00 to 2020-11-19 19:46-05:00 (00:16) Account activity
+  2020-11-20 10:51-05:00 to 2020-11-20 11:26-05:00 (00:35) Account activity
+  2020-11-24 12:01-05:00 to 2020-11-24 13:01-05:00 (01:00) Account activity
+
diff --git a/tools/user-activity/arvados_user_activity/main.py b/tools/user-activity/arvados_user_activity/main.py index d1635c6872..959f16d898 100755 --- a/tools/user-activity/arvados_user_activity/main.py +++ b/tools/user-activity/arvados_user_activity/main.py @@ -32,9 +32,14 @@ def getowner(arv, uuid, owners): return getowner(arv, owners[uuid], owners) -def getusername(arv, uuid): +def getuserinfo(arv, uuid): u = arv.users().get(uuid=uuid).execute() - return "%s %s <%s> (%s)" % (u["first_name"], u["last_name"], u["email"], uuid) + prof = "\n".join(" %s: \"%s\"" % (k, v) for k, v in u["prefs"].get("profile", {}).items() if v) + if prof: + prof = "\n"+prof+"\n" + return "%s %s <%s> (%susers/%s)%s" % (u["first_name"], u["last_name"], u["email"], + arv.config()["Services"]["Workbench1"]["ExternalURL"], + uuid, prof) def getname(u): return "\"%s\" (%s)" % (u["name"], u["uuid"]) @@ -49,7 +54,9 @@ def main(arguments=None): since = datetime.datetime.utcnow() - datetime.timedelta(days=args.days) - print("Activity since %s\n" % (datetime.datetime.now() - datetime.timedelta(days=args.days)).isoformat()) + print("User activity on %s between %s and %s\n" % (arv.config()["ClusterID"], + (datetime.datetime.now() - datetime.timedelta(days=args.days)).isoformat(sep=" ", timespec="minutes"), + datetime.datetime.now().isoformat(sep=" ", timespec="minutes"))) events = arvados.util.keyset_list_all(arv.logs().list, filters=[["created_at", ">=", since.isoformat()]]) @@ -59,7 +66,7 @@ def main(arguments=None): for e in events: owner = getowner(arv, e["object_owner_uuid"], owners) users.setdefault(owner, []) - event_at = ciso8601.parse_datetime(e["event_at"]).astimezone().isoformat() + event_at = ciso8601.parse_datetime(e["event_at"]).astimezone().isoformat(sep=" ", timespec="minutes") # loguuid = e["uuid"] loguuid = "" @@ -87,11 +94,17 @@ def main(arguments=None): users[owner].append("%s Updated project %s" % (event_at, getname(e["properties"]["new_attributes"]))) elif e["event_type"] in ("create", "update") and e["object_uuid"][6:11] == "gj3su": + since_last = None if len(users[owner]) > 0 and users[owner][-1].endswith("activity"): sp = users[owner][-1].split(" ") - users[owner][-1] = "%s to %s Account activity" % (sp[0], event_at) + start = sp[0]+" "+sp[1] + since_last = ciso8601.parse_datetime(event_at) - ciso8601.parse_datetime(sp[3]+" "+sp[4]) + span = ciso8601.parse_datetime(event_at) - ciso8601.parse_datetime(start) + + if since_last is not None and since_last < datetime.timedelta(minutes=61): + users[owner][-1] = "%s to %s (%02d:%02d) Account activity" % (start, event_at, span.days*24 + int(span.seconds/3600), int((span.seconds % 3600)/60)) else: - users[owner].append("%s Account activity" % (event_at)) + users[owner].append("%s to %s (0:00) Account activity" % (event_at, event_at)) elif e["event_type"] == "create" and e["object_uuid"][6:11] == "o0j2j": if e["properties"]["new_attributes"]["link_class"] == "tag": @@ -130,7 +143,7 @@ def main(arguments=None): for k,v in users.items(): if k is None or k.endswith("-tpzed-000000000000000"): continue - print("%s:" % getusername(arv, k)) + print(getuserinfo(arv, k)) for ev in v: print(" %s" % ev) print("") diff --git a/tools/user-activity/bin/arv-user-activity b/tools/user-activity/bin/arv-user-activity new file mode 100755 index 0000000000..bc73f84afc --- /dev/null +++ b/tools/user-activity/bin/arv-user-activity @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +import arvados_user_activity.main + +arvados_user_activity.main.main() -- 2.30.2