X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/e8d1a643cdbc3a5f4c0e5c745da58d9f7e1248d8..HEAD:/sdk/go/arvados/fs_project.go diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go index bf6391a74e..df1d06e753 100644 --- a/sdk/go/arvados/fs_project.go +++ b/sdk/go/arvados/fs_project.go @@ -6,7 +6,9 @@ package arvados import ( "log" + "os" "strings" + "time" ) func (fs *customFileSystem) defaultUUID(uuid string) (string, error) { @@ -33,11 +35,13 @@ func (fs *customFileSystem) projectsLoadOne(parent inode, uuid, name string) (in contents = CollectionList{} err = fs.RequestAndDecode(&contents, "GET", "arvados/v1/groups/"+uuid+"/contents", nil, ResourceListParams{ Count: "none", + Order: "uuid", Filters: []Filter{ {"name", "=", strings.Replace(name, subst, "/", -1)}, {"uuid", "is_a", []string{"arvados#collection", "arvados#group"}}, - {"groups.group_class", "=", "project"}, + {"groups.group_class", "in", []string{"project", "filter"}}, }, + Select: []string{"uuid", "name", "modified_at", "properties"}, }) if err != nil { return nil, err @@ -63,11 +67,20 @@ func (fs *customFileSystem) projectsLoadOne(parent inode, uuid, name string) (in if strings.Contains(coll.UUID, "-j7d0g-") { // Group item was loaded into a Collection var -- but // we only need the Name and UUID anyway, so it's OK. - return fs.newProjectNode(parent, coll.Name, coll.UUID), nil + return &hardlink{ + inode: fs.projectSingleton(coll.UUID, &Group{ + UUID: coll.UUID, + Name: coll.Name, + ModifiedAt: coll.ModifiedAt, + Properties: coll.Properties, + }), + parent: parent, + name: coll.Name, + }, nil } else if strings.Contains(coll.UUID, "-4zz18-") { - return deferredCollectionFS(fs, parent, coll), nil + return fs.newDeferredCollectionDir(parent, name, coll.UUID, coll.ModifiedAt, coll.Properties), nil } else { - log.Printf("projectnode: unrecognized UUID in response: %q", coll.UUID) + log.Printf("group contents: unrecognized UUID in response: %q", coll.UUID) return nil, ErrInvalidArgument } } @@ -78,60 +91,100 @@ func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode, return nil, err } + pagesize := 100000 var inodes []inode - // Note: the "filters" slice's backing array might be reused - // by append(filters,...) below. This isn't goroutine safe, - // but all accesses are in the same goroutine, so it's OK. - filters := []Filter{{"owner_uuid", "=", uuid}} - params := ResourceListParams{ - Count: "none", - Filters: filters, - Order: "uuid", - } - for { - var resp CollectionList - err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params) - if err != nil { - return nil, err + // When #17424 is resolved, remove the outer loop here and use + // []string{"arvados#collection", "arvados#group"} directly as the uuid + // filter. + for _, class := range []string{"arvados#collection", "arvados#group"} { + // Note: the "filters" slice's backing array might be reused + // by append(filters,...) below. This isn't goroutine safe, + // but all accesses are in the same goroutine, so it's OK. + filters := []Filter{ + {"uuid", "is_a", class}, } - if len(resp.Items) == 0 { - break + if class == "arvados#group" { + filters = append(filters, Filter{"groups.group_class", "in", []string{"project", "filter"}}) } - for _, i := range resp.Items { - coll := i - if fs.forwardSlashNameSubstitution != "" { - coll.Name = strings.Replace(coll.Name, "/", fs.forwardSlashNameSubstitution, -1) - } - if !permittedName(coll.Name) { - continue - } - inodes = append(inodes, deferredCollectionFS(fs, parent, coll)) - } - params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}) - } - filters = append(filters, Filter{"group_class", "=", "project"}) - params.Filters = filters - for { - var resp GroupList - err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params) - if err != nil { - return nil, err - } - if len(resp.Items) == 0 { - break + params := ResourceListParams{ + Count: "none", + Filters: filters, + Order: "uuid", + Select: []string{"uuid", "name", "modified_at", "properties"}, + Limit: &pagesize, } - for _, group := range resp.Items { - if fs.forwardSlashNameSubstitution != "" { - group.Name = strings.Replace(group.Name, "/", fs.forwardSlashNameSubstitution, -1) + + for { + // The groups content endpoint returns + // Collection and Group (project) + // objects. This function only accesses the + // UUID, Name, and ModifiedAt fields. Both + // collections and groups have those fields, + // so it is easier to just treat the + // ObjectList that comes back as a + // CollectionList. + var resp CollectionList + err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups/"+uuid+"/contents", nil, params) + if err != nil { + return nil, err + } + if len(resp.Items) == 0 { + break } - if !permittedName(group.Name) { - continue + for _, i := range resp.Items { + if fs.forwardSlashNameSubstitution != "" { + i.Name = strings.Replace(i.Name, "/", fs.forwardSlashNameSubstitution, -1) + } + if !permittedName(i.Name) { + continue + } + if strings.Contains(i.UUID, "-j7d0g-") { + inodes = append(inodes, fs.newProjectDir(parent, i.Name, i.UUID, &Group{ + UUID: i.UUID, + Name: i.Name, + ModifiedAt: i.ModifiedAt, + Properties: i.Properties, + })) + } else if strings.Contains(i.UUID, "-4zz18-") { + inodes = append(inodes, fs.newDeferredCollectionDir(parent, i.Name, i.UUID, i.ModifiedAt, i.Properties)) + } else { + log.Printf("group contents: unrecognized UUID in response: %q", i.UUID) + return nil, ErrInvalidArgument + } } - inodes = append(inodes, fs.newProjectNode(parent, group.Name, group.UUID)) + params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}) } - params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}) } return inodes, nil } + +func (fs *customFileSystem) newProjectDir(parent inode, name, uuid string, proj *Group) inode { + return &hardlink{inode: fs.projectSingleton(uuid, proj), parent: parent, name: name} +} + +func (fs *customFileSystem) newDeferredCollectionDir(parent inode, name, uuid string, modTime time.Time, props map[string]interface{}) inode { + if modTime.IsZero() { + modTime = time.Now() + } + placeholder := &treenode{ + fs: fs, + parent: parent, + inodes: nil, + fileinfo: fileinfo{ + name: name, + modTime: modTime, + mode: 0755 | os.ModeDir, + sys: func() interface{} { return &Collection{UUID: uuid, Name: name, ModifiedAt: modTime, Properties: props} }, + }, + } + return &deferrednode{wrapped: placeholder, create: func() inode { + node, err := fs.collectionSingleton(uuid) + if err != nil { + log.Printf("BUG: unhandled error: %s", err) + return placeholder + } + return &hardlink{inode: node, parent: parent, name: name} + }} +}