X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/4237a24fcc5e0ff8cf6429b04844ca5f8e5b48c5..0542eb85c46bf04c4139d6a976d49860f5f7d5e6:/sdk/go/arvados/fs_project.go diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go index c5eb03360a..ac1c01b591 100644 --- a/sdk/go/arvados/fs_project.go +++ b/sdk/go/arvados/fs_project.go @@ -8,6 +8,7 @@ import ( "log" "os" "strings" + "time" ) func (fs *customFileSystem) defaultUUID(uuid string) (string, error) { @@ -22,6 +23,20 @@ func (fs *customFileSystem) defaultUUID(uuid string) (string, error) { return resp.UUID, nil } +// The groups content endpoint returns Collection and Group (project) +// objects. This struct lets us load the common Items fields for both +// types (UUID, Name, ModifiedAt, and Properties), and GroupClass for +// groups, into one struct. +type groupContentsResponse struct { + Items []struct { + UUID string `json:"uuid"` + Name string `json:"name"` + ModifiedAt time.Time `json:"modified_at"` + GroupClass string `json:"group_class"` + Properties map[string]interface{} `json:"properties"` + } +} + // loadOneChild loads only the named child, if it exists. func (fs *customFileSystem) projectsLoadOne(parent inode, uuid, name string) (inode, error) { uuid, err := fs.defaultUUID(uuid) @@ -29,21 +44,22 @@ func (fs *customFileSystem) projectsLoadOne(parent inode, uuid, name string) (in return nil, err } - var contents CollectionList + var resp groupContentsResponse for _, subst := range []string{"/", fs.forwardSlashNameSubstitution} { - contents = CollectionList{} - err = fs.RequestAndDecode(&contents, "GET", "arvados/v1/groups/"+uuid+"/contents", nil, ResourceListParams{ + resp = groupContentsResponse{} + err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups/"+uuid+"/contents", nil, ResourceListParams{ Count: "none", Filters: []Filter{ {"name", "=", strings.Replace(name, subst, "/", -1)}, {"uuid", "is_a", []string{"arvados#collection", "arvados#group"}}, - {"groups.group_class", "=", "project"}, + {"groups.group_class", "in", []string{"project", "filter"}}, }, + Select: []string{"uuid", "name", "modified_at", "properties", "group_class"}, }) if err != nil { return nil, err } - if len(contents.Items) > 0 || fs.forwardSlashNameSubstitution == "/" || fs.forwardSlashNameSubstitution == "" || !strings.Contains(name, fs.forwardSlashNameSubstitution) { + if len(resp.Items) > 0 || fs.forwardSlashNameSubstitution == "/" || fs.forwardSlashNameSubstitution == "" || !strings.Contains(name, fs.forwardSlashNameSubstitution) { break } // If the requested name contains the configured "/" @@ -56,19 +72,28 @@ func (fs *customFileSystem) projectsLoadOne(parent inode, uuid, name string) (in // Note this doesn't handle items whose names contain // both "/" and the substitution string. } - if len(contents.Items) == 0 { - return nil, os.ErrNotExist + if len(resp.Items) == 0 { + return nil, nil } - coll := contents.Items[0] - - if strings.Contains(coll.UUID, "-j7d0g-") { - // Group item was loaded into a Collection var -- but - // we only need the Name and UUID anyway, so it's OK. - return fs.newProjectNode(parent, coll.Name, coll.UUID), nil - } else if strings.Contains(coll.UUID, "-4zz18-") { - return deferredCollectionFS(fs, parent, coll), nil + item := resp.Items[0] + isGroup := strings.Contains(item.UUID, "-j7d0g-") + if strings.Contains(item.UUID, "-4zz18-") { + return fs.newDeferredCollectionDir(parent, name, item.UUID, item.ModifiedAt, item.Properties), nil + } else if isGroup && item.GroupClass == "filter" { + return fs.newCollectionOrProjectSymlink(parent, name, item.UUID, item.ModifiedAt, item.Properties) + } else if isGroup && item.GroupClass == "project" { + return &hardlink{ + inode: fs.projectSingleton(item.UUID, &Group{ + UUID: item.UUID, + Name: item.Name, + ModifiedAt: item.ModifiedAt, + Properties: item.Properties, + }), + parent: parent, + name: item.Name, + }, nil } else { - log.Printf("projectnode: unrecognized UUID in response: %q", coll.UUID) + log.Printf("group contents: unrecognized UUID in response: %q", item.UUID) return nil, ErrInvalidArgument } } @@ -79,60 +104,124 @@ func (fs *customFileSystem) projectsLoadAll(parent inode, uuid string) ([]inode, return nil, err } + pagesize := 100000 var inodes []inode - // Note: the "filters" slice's backing array might be reused - // by append(filters,...) below. This isn't goroutine safe, - // but all accesses are in the same goroutine, so it's OK. - filters := []Filter{{"owner_uuid", "=", uuid}} - params := ResourceListParams{ - Count: "none", - Filters: filters, - Order: "uuid", - } - for { - var resp CollectionList - err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params) - if err != nil { - return nil, err + // When #17424 is resolved, remove the outer loop here and use + // []string{"arvados#collection", "arvados#group"} directly as the uuid + // filter. + for _, class := range []string{"arvados#collection", "arvados#group"} { + // Note: the "filters" slice's backing array might be reused + // by append(filters,...) below. This isn't goroutine safe, + // but all accesses are in the same goroutine, so it's OK. + filters := []Filter{ + {"uuid", "is_a", class}, } - if len(resp.Items) == 0 { - break + if class == "arvados#group" { + filters = append(filters, Filter{"groups.group_class", "in", []string{"project", "filter"}}) } - for _, i := range resp.Items { - coll := i - if fs.forwardSlashNameSubstitution != "" { - coll.Name = strings.Replace(coll.Name, "/", fs.forwardSlashNameSubstitution, -1) - } - if !permittedName(coll.Name) { - continue - } - inodes = append(inodes, deferredCollectionFS(fs, parent, coll)) - } - params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}) - } - filters = append(filters, Filter{"group_class", "=", "project"}) - params.Filters = filters - for { - var resp GroupList - err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params) - if err != nil { - return nil, err - } - if len(resp.Items) == 0 { - break + params := ResourceListParams{ + Count: "none", + Filters: filters, + Order: "uuid", + Select: []string{"uuid", "name", "modified_at", "properties", "group_class"}, + Limit: &pagesize, } - for _, group := range resp.Items { - if fs.forwardSlashNameSubstitution != "" { - group.Name = strings.Replace(group.Name, "/", fs.forwardSlashNameSubstitution, -1) + + for { + var resp groupContentsResponse + err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups/"+uuid+"/contents", nil, params) + if err != nil { + return nil, err } - if !permittedName(group.Name) { - continue + if len(resp.Items) == 0 { + break } - inodes = append(inodes, fs.newProjectNode(parent, group.Name, group.UUID)) + for _, i := range resp.Items { + if fs.forwardSlashNameSubstitution != "" { + i.Name = strings.Replace(i.Name, "/", fs.forwardSlashNameSubstitution, -1) + } + if !permittedName(i.Name) { + continue + } + isGroup := strings.Contains(i.UUID, "-j7d0g-") + if strings.Contains(i.UUID, "-4zz18-") { + inodes = append(inodes, fs.newDeferredCollectionDir(parent, i.Name, i.UUID, i.ModifiedAt, i.Properties)) + } else if isGroup && i.GroupClass == "filter" { + inode, err := fs.newCollectionOrProjectSymlink(parent, i.Name, i.UUID, i.ModifiedAt, i.Properties) + if err != nil { + return nil, err + } + if inode != nil { + inodes = append(inodes, inode) + } + } else if isGroup && i.GroupClass == "project" { + inodes = append(inodes, fs.newProjectDir(parent, i.Name, i.UUID, &Group{ + UUID: i.UUID, + Name: i.Name, + ModifiedAt: i.ModifiedAt, + Properties: i.Properties, + })) + } else { + log.Printf("group contents: unrecognized UUID in response: %q", i.UUID) + return nil, ErrInvalidArgument + } + } + params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}) } - params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID}) } return inodes, nil } + +// create a symlink to the given collection or project. If it's not +// possible to create a symlink because the filesystem does not have a +// "by_id" mount point to put the target in, return (nil, nil). +func (fs *customFileSystem) newCollectionOrProjectSymlink(parent inode, name, targetUUID string, modTime time.Time, props map[string]interface{}) (inode, error) { + fs.root.treenode.Lock() + byIDPath := fs.byIDPath + fs.root.treenode.Unlock() + if byIDPath == "" { + return nil, nil + } + targetPath := []byte("/" + byIDPath + "/" + targetUUID) + return &getternode{ + Getter: func() ([]byte, error) { return targetPath, nil }, + treenode: treenode{ + fileinfo: fileinfo{ + name: name, + modTime: modTime, + mode: os.ModeSymlink, + }, + }, + }, nil +} + +func (fs *customFileSystem) newProjectDir(parent inode, name, uuid string, proj *Group) inode { + return &hardlink{inode: fs.projectSingleton(uuid, proj), parent: parent, name: name} +} + +func (fs *customFileSystem) newDeferredCollectionDir(parent inode, name, uuid string, modTime time.Time, props map[string]interface{}) inode { + if modTime.IsZero() { + modTime = time.Now() + } + placeholder := &treenode{ + fs: fs, + parent: parent, + inodes: nil, + fileinfo: fileinfo{ + name: name, + modTime: modTime, + mode: 0755 | os.ModeDir, + sys: func() interface{} { return &Collection{UUID: uuid, Name: name, ModifiedAt: modTime, Properties: props} }, + }, + } + return &deferrednode{wrapped: placeholder, create: func() inode { + node, err := fs.collectionSingleton(uuid) + if err != nil { + log.Printf("BUG: unhandled error: %s", err) + return placeholder + } + return &hardlink{inode: node, parent: parent, name: name} + }} +}