"errors"
"fmt"
"io"
+ "io/fs"
"io/ioutil"
"log"
"net/http"
}
// NewClientFromEnv creates a new Client that uses the default HTTP
-// client with the API endpoint and credentials given by the
-// ARVADOS_API_* environment variables.
+// client, and loads API endpoint and credentials from ARVADOS_*
+// environment variables (if set) and
+// $HOME/.config/arvados/settings.conf (if readable).
+//
+// If a config exists in both locations, the environment variable is
+// used.
+//
+// If there is an error (other than ENOENT) reading settings.conf,
+// NewClientFromEnv logs the error to log.Default(), then proceeds as
+// if settings.conf did not exist.
+//
+// Space characters are trimmed when reading the settings file, so
+// these are equivalent:
+//
+// ARVADOS_API_HOST=localhost\n
+// ARVADOS_API_HOST=localhost\r\n
+// ARVADOS_API_HOST = localhost \n
+// \tARVADOS_API_HOST = localhost\n
func NewClientFromEnv() *Client {
+ vars := map[string]string{}
+ home := os.Getenv("HOME")
+ conffile := home + "/.config/arvados/settings.conf"
+ if home == "" {
+ // no $HOME => just use env vars
+ } else if settings, err := os.ReadFile(conffile); errors.Is(err, fs.ErrNotExist) {
+ // no config file => just use env vars
+ } else if err != nil {
+ // config file unreadable => log message, then use env vars
+ log.Printf("continuing without loading %s: %s", conffile, err)
+ } else {
+ for _, line := range bytes.Split(settings, []byte{'\n'}) {
+ kv := bytes.SplitN(line, []byte{'='}, 2)
+ k := string(bytes.TrimSpace(kv[0]))
+ if len(kv) != 2 || !strings.HasPrefix(k, "ARVADOS_") {
+ // Same behavior as python sdk:
+ // silently skip leading # (comments),
+ // blank lines, typos, and non-Arvados
+ // vars.
+ continue
+ }
+ vars[k] = string(bytes.TrimSpace(kv[1]))
+ }
+ }
+ for _, env := range os.Environ() {
+ if !strings.HasPrefix(env, "ARVADOS_") {
+ continue
+ }
+ kv := strings.SplitN(env, "=", 2)
+ if len(kv) == 2 {
+ vars[kv[0]] = kv[1]
+ }
+ }
var svcs []string
- for _, s := range strings.Split(os.Getenv("ARVADOS_KEEP_SERVICES"), " ") {
+ for _, s := range strings.Split(vars["ARVADOS_KEEP_SERVICES"], " ") {
if s == "" {
continue
} else if u, err := url.Parse(s); err != nil {
}
}
var insecure bool
- if s := strings.ToLower(os.Getenv("ARVADOS_API_HOST_INSECURE")); s == "1" || s == "yes" || s == "true" {
+ if s := strings.ToLower(vars["ARVADOS_API_HOST_INSECURE"]); s == "1" || s == "yes" || s == "true" {
insecure = true
}
return &Client{
Scheme: "https",
- APIHost: os.Getenv("ARVADOS_API_HOST"),
- AuthToken: os.Getenv("ARVADOS_API_TOKEN"),
+ APIHost: vars["ARVADOS_API_HOST"],
+ AuthToken: vars["ARVADOS_API_TOKEN"],
Insecure: insecure,
KeepServiceURIs: svcs,
Timeout: 5 * time.Minute,
"io/ioutil"
"net/http"
"net/url"
+ "os"
+ "strings"
"sync"
- "testing"
"testing/iotest"
+
+ check "gopkg.in/check.v1"
)
type stubTransport struct {
}, nil
}
-func TestCurrentUser(t *testing.T) {
- t.Parallel()
+var _ = check.Suite(&clientSuite{})
+
+type clientSuite struct{}
+
+func (*clientSuite) TestCurrentUser(c *check.C) {
stub := &stubTransport{
Responses: map[string]string{
"/arvados/v1/users/current": `{"uuid":"zzzzz-abcde-012340123401234"}`,
},
}
- c := &Client{
+ client := &Client{
Client: &http.Client{
Transport: stub,
},
APIHost: "zzzzz.arvadosapi.com",
AuthToken: "xyzzy",
}
- u, err := c.CurrentUser()
- if err != nil {
- t.Fatal(err)
- }
- if x := "zzzzz-abcde-012340123401234"; u.UUID != x {
- t.Errorf("got uuid %q, expected %q", u.UUID, x)
- }
- if len(stub.Requests) < 1 {
- t.Fatal("empty stub.Requests")
- }
+ u, err := client.CurrentUser()
+ c.Check(err, check.IsNil)
+ c.Check(u.UUID, check.Equals, "zzzzz-abcde-012340123401234")
+ c.Check(stub.Requests, check.Not(check.HasLen), 0)
hdr := stub.Requests[len(stub.Requests)-1].Header
- if hdr.Get("Authorization") != "OAuth2 xyzzy" {
- t.Errorf("got headers %+q, expected Authorization header", hdr)
- }
+ c.Check(hdr.Get("Authorization"), check.Equals, "OAuth2 xyzzy")
- c.Client.Transport = &errorTransport{}
- u, err = c.CurrentUser()
- if err == nil {
- t.Errorf("got nil error, expected something awful")
- }
+ client.Client.Transport = &errorTransport{}
+ u, err = client.CurrentUser()
+ c.Check(err, check.NotNil)
}
-func TestAnythingToValues(t *testing.T) {
+func (*clientSuite) TestAnythingToValues(c *check.C) {
type testCase struct {
in interface{}
// ok==nil means anythingToValues should return an
ok: nil,
},
} {
- t.Logf("%#v", tc.in)
+ c.Logf("%#v", tc.in)
out, err := anythingToValues(tc.in)
- switch {
- case tc.ok == nil:
- if err == nil {
- t.Errorf("got %#v, expected error", out)
- }
- case err != nil:
- t.Errorf("got err %#v, expected nil", err)
- case !tc.ok(out):
- t.Errorf("got %#v but tc.ok() says that is wrong", out)
+ if tc.ok == nil {
+ c.Check(err, check.NotNil)
+ continue
}
+ c.Check(err, check.IsNil)
+ c.Check(tc.ok(out), check.Equals, true)
}
}
+
+func (*clientSuite) TestLoadConfig(c *check.C) {
+ oldenv := os.Environ()
+ defer func() {
+ os.Clearenv()
+ for _, s := range oldenv {
+ i := strings.IndexRune(s, '=')
+ os.Setenv(s[:i], s[i+1:])
+ }
+ }()
+
+ tmp := c.MkDir()
+ os.Setenv("HOME", tmp)
+ for _, s := range os.Environ() {
+ if strings.HasPrefix(s, "ARVADOS_") {
+ i := strings.IndexRune(s, '=')
+ os.Unsetenv(s[:i])
+ }
+ }
+ os.Mkdir(tmp+"/.config", 0777)
+ os.Mkdir(tmp+"/.config/arvados", 0777)
+
+ // Use $HOME/.config/arvados/settings.conf if no env vars are
+ // set
+ os.WriteFile(tmp+"/.config/arvados/settings.conf", []byte(`
+ ARVADOS_API_HOST = localhost:1
+ ARVADOS_API_TOKEN = token_from_settings_file1
+ `), 0777)
+ client := NewClientFromEnv()
+ c.Check(client.AuthToken, check.Equals, "token_from_settings_file1")
+ c.Check(client.APIHost, check.Equals, "localhost:1")
+ c.Check(client.Insecure, check.Equals, false)
+
+ // ..._INSECURE=true, comments, ignored lines in settings.conf
+ os.WriteFile(tmp+"/.config/arvados/settings.conf", []byte(`
+ (ignored) = (ignored)
+ #ARVADOS_API_HOST = localhost:2
+ ARVADOS_API_TOKEN = token_from_settings_file2
+ ARVADOS_API_HOST_INSECURE = true
+ `), 0777)
+ client = NewClientFromEnv()
+ c.Check(client.AuthToken, check.Equals, "token_from_settings_file2")
+ c.Check(client.APIHost, check.Equals, "")
+ c.Check(client.Insecure, check.Equals, true)
+
+ // Environment variables override settings.conf
+ os.Setenv("ARVADOS_API_HOST", "[::]:3")
+ os.Setenv("ARVADOS_API_HOST_INSECURE", "0")
+ client = NewClientFromEnv()
+ c.Check(client.AuthToken, check.Equals, "token_from_settings_file2")
+ c.Check(client.APIHost, check.Equals, "[::]:3")
+ c.Check(client.Insecure, check.Equals, false)
+}
import arvados.util
import datetime
import ciso8601
+import csv
def parse_arguments(arguments):
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('--start', help='Start date for the report in YYYY-MM-DD format (UTC)')
arg_parser.add_argument('--end', help='End date for the report in YYYY-MM-DD format (UTC)')
arg_parser.add_argument('--days', type=int, help='Number of days before now() to start the report')
+ arg_parser.add_argument('--csv', action='store_true', help='Output in csv format (default: false)')
args = arg_parser.parse_args(arguments)
if args.days and (args.start or args.end):
return "%s %s <%s> (%susers/%s)%s" % (u["first_name"], u["last_name"], u["email"],
arv.config()["Services"]["Workbench1"]["ExternalURL"],
uuid, prof)
+def getuserinfocsv(arv, uuid):
+ try:
+ u = arv.users().get(uuid=uuid).execute()
+ except:
+ return [uuid,"deleted","user",""]
+ return [uuid, u["first_name"], u["last_name"], u["email"]]
+
collectionNameCache = {}
def getCollectionName(arv, uuid, pdh):
arv = arvados.api()
- print("User activity on %s between %s and %s\n" % (arv.config()["ClusterID"],
+ prefix = ''
+ suffix = "\n"
+ if args.csv:
+ prefix = '# '
+ suffix = ''
+ print("%sUser activity on %s between %s and %s%s" % (prefix, arv.config()["ClusterID"],
since.isoformat(sep=" ", timespec="minutes"),
- to.isoformat(sep=" ", timespec="minutes")))
+ to.isoformat(sep=" ", timespec="minutes"), suffix))
events = arvados.util.keyset_list_all(arv.logs().list, filters=[["created_at", ">=", since.isoformat()],["created_at", "<", to.isoformat()]])
owner = getowner(arv, e["object_owner_uuid"], owners)
users.setdefault(owner, [])
event_at = ciso8601.parse_datetime(e["event_at"]).astimezone().isoformat(sep=" ", timespec="minutes")
- # loguuid = e["uuid"]
- loguuid = ""
+ loguuid = e["uuid"]
if e["event_type"] == "create" and e["object_uuid"][6:11] == "tpzed":
users.setdefault(e["object_uuid"], [])
- users[e["object_uuid"]].append("%s User account created" % event_at)
+ users[e["object_uuid"]].append([loguuid, event_at, "User account created"])
elif e["event_type"] == "update" and e["object_uuid"][6:11] == "tpzed":
pass
elif e["event_type"] == "create" and e["object_uuid"][6:11] == "xvhdp":
if e["properties"]["new_attributes"]["requesting_container_uuid"] is None:
- users[owner].append("%s Ran container %s %s" % (event_at, getname(e["properties"]["new_attributes"]), loguuid))
+ users[owner].append([loguuid, event_at, "Ran container %s" % (getname(e["properties"]["new_attributes"]))])
elif e["event_type"] == "update" and e["object_uuid"][6:11] == "xvhdp":
pass
elif e["event_type"] == "create" and e["object_uuid"][6:11] == "j7d0g":
- users[owner].append("%s Created project %s" % (event_at, getname(e["properties"]["new_attributes"])))
+ users[owner].append([loguuid, event_at,"Created project %s" % (getname(e["properties"]["new_attributes"]))])
elif e["event_type"] == "delete" and e["object_uuid"][6:11] == "j7d0g":
- users[owner].append("%s Deleted project %s" % (event_at, getname(e["properties"]["old_attributes"])))
+ users[owner].append([loguuid, event_at,"Deleted project %s" % (getname(e["properties"]["old_attributes"]))])
elif e["event_type"] == "update" and e["object_uuid"][6:11] == "j7d0g":
- users[owner].append("%s Updated project %s" % (event_at, getname(e["properties"]["new_attributes"])))
+ users[owner].append([loguuid, event_at,"Updated project %s" % (getname(e["properties"]["new_attributes"]))])
elif e["event_type"] in ("create", "update") and e["object_uuid"][6:11] == "gj3su":
since_last = None
- if len(users[owner]) > 0 and users[owner][-1].endswith("activity"):
- sp = users[owner][-1].split(" ")
- start = sp[0]+" "+sp[1]
- since_last = ciso8601.parse_datetime(event_at) - ciso8601.parse_datetime(sp[3]+" "+sp[4])
+ if len(users[owner]) > 0 and users[owner][-1][-1].endswith("activity"):
+ sp = users[owner][-1][-1].split(" ")
+ start = users[owner][-1][1]
+ since_last = ciso8601.parse_datetime(event_at) - ciso8601.parse_datetime(sp[1]+" "+sp[2])
span = ciso8601.parse_datetime(event_at) - ciso8601.parse_datetime(start)
if since_last is not None and since_last < datetime.timedelta(minutes=61):
- users[owner][-1] = "%s to %s (%02d:%02d) Account activity" % (start, event_at, span.days*24 + int(span.seconds/3600), int((span.seconds % 3600)/60))
+ users[owner][-1] = [loguuid, start,"to %s (%02d:%02d) Account activity" % (event_at, span.days*24 + int(span.seconds/3600), int((span.seconds % 3600)/60))]
else:
- users[owner].append("%s to %s (0:00) Account activity" % (event_at, event_at))
+ users[owner].append([loguuid, event_at,"to %s (0:00) Account activity" % (event_at)])
elif e["event_type"] == "create" and e["object_uuid"][6:11] == "o0j2j":
if e["properties"]["new_attributes"]["link_class"] == "tag":
- users[owner].append("%s Tagged %s" % (event_at, e["properties"]["new_attributes"]["head_uuid"]))
+ users[owner].append([event_at,"Tagged %s" % (e["properties"]["new_attributes"]["head_uuid"])])
elif e["properties"]["new_attributes"]["link_class"] == "permission":
- users[owner].append("%s Shared %s with %s" % (event_at, e["properties"]["new_attributes"]["tail_uuid"], e["properties"]["new_attributes"]["head_uuid"]))
+ users[owner].append([loguuid, event_at,"Shared %s with %s" % (e["properties"]["new_attributes"]["tail_uuid"], e["properties"]["new_attributes"]["head_uuid"])])
else:
- users[owner].append("%s %s %s %s %s" % (event_at, e["event_type"], e["object_kind"], e["object_uuid"], loguuid))
+ users[owner].append([loguuid, event_at,"%s %s %s" % (e["event_type"], e["object_kind"], e["object_uuid"])])
elif e["event_type"] == "delete" and e["object_uuid"][6:11] == "o0j2j":
if e["properties"]["old_attributes"]["link_class"] == "tag":
- users[owner].append("%s Untagged %s" % (event_at, e["properties"]["old_attributes"]["head_uuid"]))
+ users[owner].append([loguuid, event_at,"Untagged %s" % (e["properties"]["old_attributes"]["head_uuid"])])
elif e["properties"]["old_attributes"]["link_class"] == "permission":
- users[owner].append("%s Unshared %s with %s" % (event_at, e["properties"]["old_attributes"]["tail_uuid"], e["properties"]["old_attributes"]["head_uuid"]))
+ users[owner].append([loguuid, event_at,"Unshared %s with %s" % (e["properties"]["old_attributes"]["tail_uuid"], e["properties"]["old_attributes"]["head_uuid"])])
else:
- users[owner].append("%s %s %s %s %s" % (event_at, e["event_type"], e["object_kind"], e["object_uuid"], loguuid))
+ users[owner].append([loguuid, event_at,"%s %s %s" % (e["event_type"], e["object_kind"], e["object_uuid"])])
elif e["event_type"] == "create" and e["object_uuid"][6:11] == "4zz18":
if e["properties"]["new_attributes"]["properties"].get("type") in ("log", "output", "intermediate"):
pass
else:
- users[owner].append("%s Created collection %s %s" % (event_at, getname(e["properties"]["new_attributes"]), loguuid))
+ users[owner].append([loguuid, event_at,"Created collection %s" % (getname(e["properties"]["new_attributes"]))])
elif e["event_type"] == "update" and e["object_uuid"][6:11] == "4zz18":
- users[owner].append("%s Updated collection %s %s" % (event_at, getname(e["properties"]["new_attributes"]), loguuid))
+ users[owner].append([loguuid, event_at,"Updated collection %s" % (getname(e["properties"]["new_attributes"]))])
elif e["event_type"] == "delete" and e["object_uuid"][6:11] == "4zz18":
if e["properties"]["old_attributes"]["properties"].get("type") in ("log", "output", "intermediate"):
pass
else:
- users[owner].append("%s Deleted collection %s %s" % (event_at, getname(e["properties"]["old_attributes"]), loguuid))
+ users[owner].append([loguuid, event_at, "Deleted collection %s" % (getname(e["properties"]["old_attributes"]))])
elif e["event_type"] == "file_download":
users.setdefault(e["object_uuid"], [])
- users[e["object_uuid"]].append("%s Downloaded file \"%s\" from \"%s\" (%s) (%s)" % (event_at,
+ users[e["object_uuid"]].append([loguuid, event_at, "Downloaded file \"%s\" from \"%s\" (%s) (%s)" % (
e["properties"].get("collection_file_path") or e["properties"].get("reqPath"),
getCollectionName(arv, e["properties"].get("collection_uuid"), e["properties"].get("portable_data_hash")),
e["properties"].get("collection_uuid"),
- e["properties"].get("portable_data_hash")))
+ e["properties"].get("portable_data_hash"))])
+
elif e["event_type"] == "file_upload":
users.setdefault(e["object_uuid"], [])
- users[e["object_uuid"]].append("%s Uploaded file \"%s\" to \"%s\" (%s)" % (event_at,
+ users[e["object_uuid"]].append([loguuid, event_at, "Uploaded file \"%s\" to \"%s\" (%s)" % (
e["properties"].get("collection_file_path") or e["properties"].get("reqPath"),
getCollectionName(arv, e["properties"].get("collection_uuid"), e["properties"].get("portable_data_hash")),
- e["properties"].get("collection_uuid")))
+ e["properties"].get("collection_uuid"))])
else:
- users[owner].append("%s %s %s %s %s" % (event_at, e["event_type"], e["object_kind"], e["object_uuid"], loguuid))
+ users[owner].append([loguuid, event_at, "%s %s %s" % (e["event_type"], e["object_kind"], e["object_uuid"])])
+
+ if args.csv:
+ csvwriter = csv.writer(sys.stdout, dialect='unix')
for k,v in users.items():
if k is None or k.endswith("-tpzed-000000000000000"):
continue
- print(getuserinfo(arv, k))
- for ev in v:
- print(" %s" % ev)
- print("")
+ if not args.csv:
+ print(getuserinfo(arv, k))
+ for ev in v:
+ # Remove the log entry uuid, this report is intended for human consumption
+ ev.pop(0)
+ print(" %s" % ' '.join(ev))
+ print("")
+ else:
+ user = getuserinfocsv(arv, k)
+ for ev in v:
+ ev = user + ev
+ csvwriter.writerow(ev)
if __name__ == "__main__":
main()
projects / arvados.git / commitdiff