1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
19 "git.arvados.org/arvados.git/sdk/go/arvados"
20 "git.arvados.org/arvados.git/sdk/go/ctxlog"
21 "github.com/AdRoll/goamz/s3"
24 const s3MaxKeys = 1000
26 // serveS3 handles r and returns true if r is a request from an S3
27 // client, otherwise it returns false.
28 func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
30 if auth := r.Header.Get("Authorization"); strings.HasPrefix(auth, "AWS ") {
31 split := strings.SplitN(auth[4:], ":", 2)
33 w.WriteHeader(http.StatusUnauthorized)
37 } else if strings.HasPrefix(auth, "AWS4-HMAC-SHA256 ") {
38 for _, cmpt := range strings.Split(auth[17:], ",") {
39 cmpt = strings.TrimSpace(cmpt)
40 split := strings.SplitN(cmpt, "=", 2)
41 if len(split) == 2 && split[0] == "Credential" {
42 keyandscope := strings.Split(split[1], "/")
43 if len(keyandscope[0]) > 0 {
44 token = keyandscope[0]
50 w.WriteHeader(http.StatusBadRequest)
51 fmt.Println(w, "invalid V4 signature")
58 _, kc, client, release, err := h.getClients(r.Header.Get("X-Request-Id"), token)
60 http.Error(w, "Pool failed: "+h.clientPool.Err().Error(), http.StatusInternalServerError)
65 fs := client.SiteFileSystem(kc)
66 fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
68 objectNameGiven := strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
71 case r.Method == http.MethodGet && !objectNameGiven:
72 // Path is "/{uuid}" or "/{uuid}/", has no object name
73 if _, ok := r.URL.Query()["versioning"]; ok {
74 // GetBucketVersioning
75 w.Header().Set("Content-Type", "application/xml")
76 io.WriteString(w, xml.Header)
77 fmt.Fprintln(w, `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`)
83 case r.Method == http.MethodGet || r.Method == http.MethodHead:
84 fspath := "/by_id" + r.URL.Path
85 fi, err := fs.Stat(fspath)
86 if r.Method == "HEAD" && !objectNameGiven {
88 if err == nil && fi.IsDir() {
89 w.WriteHeader(http.StatusOK)
90 } else if os.IsNotExist(err) {
91 w.WriteHeader(http.StatusNotFound)
93 http.Error(w, err.Error(), http.StatusBadGateway)
97 if err == nil && fi.IsDir() && objectNameGiven && strings.HasSuffix(fspath, "/") && h.Config.cluster.Collections.S3FolderObjects {
98 w.Header().Set("Content-Type", "application/x-directory")
99 w.WriteHeader(http.StatusOK)
102 if os.IsNotExist(err) ||
103 (err != nil && err.Error() == "not a directory") ||
104 (fi != nil && fi.IsDir()) {
105 http.Error(w, "not found", http.StatusNotFound)
108 // shallow copy r, and change URL path
111 http.FileServer(fs).ServeHTTP(w, &r)
113 case r.Method == http.MethodPut:
114 if !objectNameGiven {
115 http.Error(w, "missing object name in PUT request", http.StatusBadRequest)
118 fspath := "by_id" + r.URL.Path
120 if strings.HasSuffix(fspath, "/") {
121 if !h.Config.cluster.Collections.S3FolderObjects {
122 http.Error(w, "invalid object name: trailing slash", http.StatusBadRequest)
125 n, err := r.Body.Read(make([]byte, 1))
126 if err != nil && err != io.EOF {
127 http.Error(w, fmt.Sprintf("error reading request body: %s", err), http.StatusInternalServerError)
130 http.Error(w, "cannot create object with trailing '/' char unless content is empty", http.StatusBadRequest)
132 } else if strings.SplitN(r.Header.Get("Content-Type"), ";", 2)[0] != "application/x-directory" {
133 http.Error(w, "cannot create object with trailing '/' char unless Content-Type is 'application/x-directory'", http.StatusBadRequest)
136 // Given PUT "foo/bar/", we'll use "foo/bar/."
137 // in the "ensure parents exist" block below,
138 // and then we'll be done.
142 fi, err := fs.Stat(fspath)
143 if err != nil && err.Error() == "not a directory" {
144 // requested foo/bar, but foo is a file
145 http.Error(w, "object name conflicts with existing object", http.StatusBadRequest)
148 if strings.HasSuffix(r.URL.Path, "/") && err == nil && !fi.IsDir() {
149 // requested foo/bar/, but foo/bar is a file
150 http.Error(w, "object name conflicts with existing object", http.StatusBadRequest)
153 // create missing parent/intermediate directories, if any
154 for i, c := range fspath {
155 if i > 0 && c == '/' {
157 if strings.HasSuffix(dir, "/") {
158 err = errors.New("invalid object name (consecutive '/' chars)")
159 http.Error(w, err.Error(), http.StatusBadRequest)
162 err = fs.Mkdir(dir, 0755)
163 if err == arvados.ErrInvalidArgument {
164 // Cannot create a directory
166 err = fmt.Errorf("mkdir %q failed: %w", dir, err)
167 http.Error(w, err.Error(), http.StatusBadRequest)
169 } else if err != nil && !os.IsExist(err) {
170 err = fmt.Errorf("mkdir %q failed: %w", dir, err)
171 http.Error(w, err.Error(), http.StatusInternalServerError)
177 f, err := fs.OpenFile(fspath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644)
178 if os.IsNotExist(err) {
179 f, err = fs.OpenFile(fspath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644)
182 err = fmt.Errorf("open %q failed: %w", r.URL.Path, err)
183 http.Error(w, err.Error(), http.StatusBadRequest)
187 _, err = io.Copy(f, r.Body)
189 err = fmt.Errorf("write to %q failed: %w", r.URL.Path, err)
190 http.Error(w, err.Error(), http.StatusBadGateway)
195 err = fmt.Errorf("write to %q failed: close: %w", r.URL.Path, err)
196 http.Error(w, err.Error(), http.StatusBadGateway)
202 err = fmt.Errorf("sync failed: %w", err)
203 http.Error(w, err.Error(), http.StatusInternalServerError)
206 w.WriteHeader(http.StatusOK)
208 case r.Method == http.MethodDelete:
209 if !objectNameGiven || r.URL.Path == "/" {
210 http.Error(w, "missing object name in DELETE request", http.StatusBadRequest)
213 fspath := "by_id" + r.URL.Path
214 if strings.HasSuffix(fspath, "/") {
215 fspath = strings.TrimSuffix(fspath, "/")
216 fi, err := fs.Stat(fspath)
217 if os.IsNotExist(err) {
218 w.WriteHeader(http.StatusNoContent)
220 } else if err != nil {
221 http.Error(w, err.Error(), http.StatusInternalServerError)
223 } else if !fi.IsDir() {
224 // if "foo" exists and is a file, then
225 // "foo/" doesn't exist, so we say
226 // delete was successful.
227 w.WriteHeader(http.StatusNoContent)
230 } else if fi, err := fs.Stat(fspath); err == nil && fi.IsDir() {
231 // if "foo" is a dir, it is visible via S3
232 // only as "foo/", not "foo" -- so we leave
233 // the dir alone and return 204 to indicate
234 // that "foo" does not exist.
235 w.WriteHeader(http.StatusNoContent)
238 err = fs.Remove(fspath)
239 if os.IsNotExist(err) {
240 w.WriteHeader(http.StatusNoContent)
244 err = fmt.Errorf("rm failed: %w", err)
245 http.Error(w, err.Error(), http.StatusBadRequest)
250 err = fmt.Errorf("sync failed: %w", err)
251 http.Error(w, err.Error(), http.StatusInternalServerError)
254 w.WriteHeader(http.StatusNoContent)
257 http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
262 // Call fn on the given path (directory) and its contents, in
263 // lexicographic order.
265 // If isRoot==true and path is not a directory, return nil.
267 // If fn returns filepath.SkipDir when called on a directory, don't
268 // descend into that directory.
269 func walkFS(fs arvados.CustomFileSystem, path string, isRoot bool, fn func(path string, fi os.FileInfo) error) error {
271 fi, err := fs.Stat(path)
272 if os.IsNotExist(err) || (err == nil && !fi.IsDir()) {
274 } else if err != nil {
278 if err == filepath.SkipDir {
280 } else if err != nil {
284 f, err := fs.Open(path)
285 if os.IsNotExist(err) && isRoot {
287 } else if err != nil {
288 return fmt.Errorf("open %q: %w", path, err)
294 fis, err := f.Readdir(-1)
298 sort.Slice(fis, func(i, j int) bool { return fis[i].Name() < fis[j].Name() })
299 for _, fi := range fis {
300 err = fn(path+"/"+fi.Name(), fi)
301 if err == filepath.SkipDir {
303 } else if err != nil {
307 err = walkFS(fs, path+"/"+fi.Name(), false, fn)
316 var errDone = errors.New("done")
318 func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.CustomFileSystem) {
326 params.bucket = strings.SplitN(r.URL.Path[1:], "/", 2)[0]
327 params.delimiter = r.FormValue("delimiter")
328 params.marker = r.FormValue("marker")
329 if mk, _ := strconv.ParseInt(r.FormValue("max-keys"), 10, 64); mk > 0 && mk < s3MaxKeys {
330 params.maxKeys = int(mk)
332 params.maxKeys = s3MaxKeys
334 params.prefix = r.FormValue("prefix")
336 bucketdir := "by_id/" + params.bucket
337 // walkpath is the directory (relative to bucketdir) we need
338 // to walk: the innermost directory that is guaranteed to
339 // contain all paths that have the requested prefix. Examples:
340 // prefix "foo/bar" => walkpath "foo"
341 // prefix "foo/bar/" => walkpath "foo/bar"
342 // prefix "foo" => walkpath ""
343 // prefix "" => walkpath ""
344 walkpath := params.prefix
345 if cut := strings.LastIndex(walkpath, "/"); cut >= 0 {
346 walkpath = walkpath[:cut]
351 type commonPrefix struct {
354 type listResp struct {
355 XMLName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
357 // s3.ListResp marshals an empty tag when
358 // CommonPrefixes is nil, which confuses some clients.
359 // Fix by using this nested struct instead.
360 CommonPrefixes []commonPrefix
361 // Similarly, we need omitempty here, because an empty
362 // tag confuses some clients (e.g.,
363 // github.com/aws/aws-sdk-net never terminates its
365 NextMarker string `xml:"NextMarker,omitempty"`
368 ListResp: s3.ListResp{
369 Name: strings.SplitN(r.URL.Path[1:], "/", 2)[0],
370 Prefix: params.prefix,
371 Delimiter: params.delimiter,
372 Marker: params.marker,
373 MaxKeys: params.maxKeys,
376 commonPrefixes := map[string]bool{}
377 err := walkFS(fs, strings.TrimSuffix(bucketdir+"/"+walkpath, "/"), true, func(path string, fi os.FileInfo) error {
378 if path == bucketdir {
381 path = path[len(bucketdir)+1:]
382 filesize := fi.Size()
387 if len(path) <= len(params.prefix) {
388 if path > params.prefix[:len(path)] {
389 // with prefix "foobar", walking "fooz" means we're done
392 if path < params.prefix[:len(path)] {
393 // with prefix "foobar", walking "foobag" is pointless
394 return filepath.SkipDir
396 if fi.IsDir() && !strings.HasPrefix(params.prefix+"/", path) {
397 // with prefix "foo/bar", walking "fo"
398 // is pointless (but walking "foo" or
399 // "foo/bar" is necessary)
400 return filepath.SkipDir
402 if len(path) < len(params.prefix) {
403 // can't skip anything, and this entry
404 // isn't in the results, so just
409 if path[:len(params.prefix)] > params.prefix {
410 // with prefix "foobar", nothing we
411 // see after "foozzz" is relevant
415 if path < params.marker || path < params.prefix {
418 if fi.IsDir() && !h.Config.cluster.Collections.S3FolderObjects {
419 // Note we don't add anything to
420 // commonPrefixes here even if delimiter is
421 // "/". We descend into the directory, and
422 // return a commonPrefix only if we end up
423 // finding a regular file inside it.
426 if params.delimiter != "" {
427 idx := strings.Index(path[len(params.prefix):], params.delimiter)
429 // with prefix "foobar" and delimiter
430 // "z", when we hit "foobar/baz", we
431 // add "/baz" to commonPrefixes and
433 commonPrefixes[path[:len(params.prefix)+idx+1]] = true
434 return filepath.SkipDir
437 if len(resp.Contents)+len(commonPrefixes) >= params.maxKeys {
438 resp.IsTruncated = true
439 if params.delimiter != "" {
440 resp.NextMarker = path
444 resp.Contents = append(resp.Contents, s3.Key{
446 LastModified: fi.ModTime().UTC().Format("2006-01-02T15:04:05.999") + "Z",
451 if err != nil && err != errDone {
452 http.Error(w, err.Error(), http.StatusInternalServerError)
455 if params.delimiter != "" {
456 resp.CommonPrefixes = make([]commonPrefix, 0, len(commonPrefixes))
457 for prefix := range commonPrefixes {
458 resp.CommonPrefixes = append(resp.CommonPrefixes, commonPrefix{prefix})
460 sort.Slice(resp.CommonPrefixes, func(i, j int) bool { return resp.CommonPrefixes[i].Prefix < resp.CommonPrefixes[j].Prefix })
462 w.Header().Set("Content-Type", "application/xml")
463 io.WriteString(w, xml.Header)
464 if err := xml.NewEncoder(w).Encode(resp); err != nil {
465 ctxlog.FromContext(r.Context()).WithError(err).Error("error writing xml response")