+
+func walkFS(fs arvados.CustomFileSystem, path string, fn func(path string, fi os.FileInfo) error) error {
+ f, err := fs.Open(path)
+ if err != nil {
+ return fmt.Errorf("open %q: %w", path, err)
+ }
+ defer f.Close()
+ if path == "/" {
+ path = ""
+ }
+ fis, err := f.Readdir(-1)
+ if err != nil {
+ return err
+ }
+ sort.Slice(fis, func(i, j int) bool { return fis[i].Name() < fis[j].Name() })
+ for _, fi := range fis {
+ err = fn(path+"/"+fi.Name(), fi)
+ if err == filepath.SkipDir {
+ continue
+ } else if err != nil {
+ return err
+ }
+ if fi.IsDir() {
+ err = walkFS(fs, path+"/"+fi.Name(), fn)
+ if err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+var errDone = errors.New("done")
+
+func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.CustomFileSystem) {
+ var params struct {
+ bucket string
+ delimiter string
+ marker string
+ maxKeys int
+ prefix string
+ }
+ params.bucket = strings.SplitN(r.URL.Path[1:], "/", 2)[0]
+ params.delimiter = r.FormValue("delimiter")
+ params.marker = r.FormValue("marker")
+ if mk, _ := strconv.ParseInt(r.FormValue("max-keys"), 10, 64); mk > 0 {
+ params.maxKeys = int(mk)
+ } else {
+ params.maxKeys = 100
+ }
+ params.prefix = r.FormValue("prefix")
+
+ bucketdir := "by_id/" + params.bucket
+ // walkpath is the directory (relative to bucketdir) we need
+ // to walk: the innermost directory that is guaranteed to
+ // contain all paths that have the requested prefix. Examples:
+ // prefix "foo/bar" => walkpath "foo"
+ // prefix "foo/bar/" => walkpath "foo/bar"
+ // prefix "foo" => walkpath ""
+ // prefix "" => walkpath ""
+ walkpath := params.prefix
+ if !strings.HasSuffix(walkpath, "/") {
+ walkpath, _ = filepath.Split(walkpath)
+ }
+ walkpath = strings.TrimSuffix(walkpath, "/")
+
+ type commonPrefix struct {
+ Prefix string
+ }
+ type serverListResponse struct {
+ s3.ListResp
+ CommonPrefixes []commonPrefix
+ }
+ resp := serverListResponse{ListResp: s3.ListResp{
+ Name: strings.SplitN(r.URL.Path[1:], "/", 2)[0],
+ Prefix: params.prefix,
+ Delimiter: params.delimiter,
+ Marker: params.marker,
+ MaxKeys: params.maxKeys,
+ }}
+ err := walkFS(fs, strings.TrimSuffix(bucketdir+"/"+walkpath, "/"), func(path string, fi os.FileInfo) error {
+ path = path[len(bucketdir)+1:]
+ if !strings.HasPrefix(path, params.prefix) {
+ return filepath.SkipDir
+ }
+ if fi.IsDir() {
+ return nil
+ }
+ if path < params.marker {
+ return nil
+ }
+ // TODO: check delimiter, roll up common prefixes
+ if len(resp.Contents)+len(resp.CommonPrefixes) >= params.maxKeys {
+ resp.IsTruncated = true
+ if params.delimiter == "" {
+ resp.NextMarker = path
+ }
+ return errDone
+ }
+ resp.ListResp.Contents = append(resp.ListResp.Contents, s3.Key{
+ Key: path,
+ })
+ return nil
+ })
+ if err != nil && err != errDone {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ if err := xml.NewEncoder(w).Encode(resp); err != nil {
+ ctxlog.FromContext(r.Context()).WithError(err).Error("error writing xml response")
+ }
+}