1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
19 "git.arvados.org/arvados.git/sdk/go/arvados"
20 "git.arvados.org/arvados.git/sdk/go/ctxlog"
21 "github.com/AdRoll/goamz/s3"
24 // serveS3 handles r and returns true if r is a request from an S3
25 // client, otherwise it returns false.
26 func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
28 if auth := r.Header.Get("Authorization"); strings.HasPrefix(auth, "AWS ") {
29 split := strings.SplitN(auth[4:], ":", 2)
31 w.WriteHeader(http.StatusUnauthorized)
35 } else if strings.HasPrefix(auth, "AWS4-HMAC-SHA256 ") {
36 for _, cmpt := range strings.Split(auth[17:], ",") {
37 cmpt = strings.TrimSpace(cmpt)
38 split := strings.SplitN(cmpt, "=", 2)
39 if len(split) == 2 && split[0] == "Credential" {
40 keyandscope := strings.Split(split[1], "/")
41 if len(keyandscope[0]) > 0 {
42 token = keyandscope[0]
48 w.WriteHeader(http.StatusBadRequest)
49 fmt.Println(w, "invalid V4 signature")
56 _, kc, client, release, err := h.getClients(r.Header.Get("X-Request-Id"), token)
58 http.Error(w, "Pool failed: "+h.clientPool.Err().Error(), http.StatusInternalServerError)
63 fs := client.SiteFileSystem(kc)
64 fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
67 case r.Method == "GET" && strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") == 1:
68 // Path is "/{uuid}" or "/{uuid}/", has no object name
71 case r.Method == "GET":
72 fspath := "/by_id" + r.URL.Path
73 fi, err := fs.Stat(fspath)
74 if os.IsNotExist(err) ||
75 (err != nil && err.Error() == "not a directory") ||
76 (fi != nil && fi.IsDir()) {
77 http.Error(w, "not found", http.StatusNotFound)
80 // shallow copy r, and change URL path
83 http.FileServer(fs).ServeHTTP(w, &r)
85 case r.Method == "PUT":
86 if strings.HasSuffix(r.URL.Path, "/") {
87 http.Error(w, "invalid object name (trailing '/' char)", http.StatusBadRequest)
90 fspath := "by_id" + r.URL.Path
91 _, err = fs.Stat(fspath)
92 if err != nil && err.Error() == "not a directory" {
93 // requested foo/bar, but foo is a file
94 http.Error(w, "object name conflicts with existing object", http.StatusBadRequest)
97 f, err := fs.OpenFile(fspath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644)
98 if os.IsNotExist(err) {
99 // create missing intermediate directories, then try again
100 for i, c := range fspath {
101 if i > 0 && c == '/' {
103 if strings.HasSuffix(dir, "/") {
104 err = errors.New("invalid object name (consecutive '/' chars)")
105 http.Error(w, err.Error(), http.StatusBadRequest)
108 err := fs.Mkdir(dir, 0755)
109 if err != nil && err != os.ErrExist {
110 err = fmt.Errorf("mkdir %q failed: %w", dir, err)
111 http.Error(w, err.Error(), http.StatusInternalServerError)
116 f, err = fs.OpenFile(fspath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644)
119 err = fmt.Errorf("open %q failed: %w", r.URL.Path, err)
120 http.Error(w, err.Error(), http.StatusBadRequest)
124 _, err = io.Copy(f, r.Body)
126 err = fmt.Errorf("write to %q failed: %w", r.URL.Path, err)
127 http.Error(w, err.Error(), http.StatusBadGateway)
132 err = fmt.Errorf("write to %q failed: %w", r.URL.Path, err)
133 http.Error(w, err.Error(), http.StatusBadGateway)
138 err = fmt.Errorf("sync failed: %w", err)
139 http.Error(w, err.Error(), http.StatusInternalServerError)
142 w.WriteHeader(http.StatusOK)
145 http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
150 func walkFS(fs arvados.CustomFileSystem, path string, ignoreNotFound bool, fn func(path string, fi os.FileInfo) error) error {
151 f, err := fs.Open(path)
152 if os.IsNotExist(err) && ignoreNotFound {
154 } else if err != nil {
155 return fmt.Errorf("open %q: %w", path, err)
161 fis, err := f.Readdir(-1)
165 sort.Slice(fis, func(i, j int) bool { return fis[i].Name() < fis[j].Name() })
166 for _, fi := range fis {
167 err = fn(path+"/"+fi.Name(), fi)
168 if err == filepath.SkipDir {
170 } else if err != nil {
174 err = walkFS(fs, path+"/"+fi.Name(), false, fn)
183 var errDone = errors.New("done")
185 func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.CustomFileSystem) {
193 params.bucket = strings.SplitN(r.URL.Path[1:], "/", 2)[0]
194 params.delimiter = r.FormValue("delimiter")
195 params.marker = r.FormValue("marker")
196 if mk, _ := strconv.ParseInt(r.FormValue("max-keys"), 10, 64); mk > 0 {
197 params.maxKeys = int(mk)
201 params.prefix = r.FormValue("prefix")
203 bucketdir := "by_id/" + params.bucket
204 // walkpath is the directory (relative to bucketdir) we need
205 // to walk: the innermost directory that is guaranteed to
206 // contain all paths that have the requested prefix. Examples:
207 // prefix "foo/bar" => walkpath "foo"
208 // prefix "foo/bar/" => walkpath "foo/bar"
209 // prefix "foo" => walkpath ""
210 // prefix "" => walkpath ""
211 walkpath := params.prefix
212 if cut := strings.LastIndex(walkpath, "/"); cut >= 0 {
213 walkpath = walkpath[:cut]
219 Name: strings.SplitN(r.URL.Path[1:], "/", 2)[0],
220 Prefix: params.prefix,
221 Delimiter: params.delimiter,
222 Marker: params.marker,
223 MaxKeys: params.maxKeys,
225 commonPrefixes := map[string]bool{}
226 err := walkFS(fs, strings.TrimSuffix(bucketdir+"/"+walkpath, "/"), true, func(path string, fi os.FileInfo) error {
227 path = path[len(bucketdir)+1:]
228 if len(path) <= len(params.prefix) {
229 if path > params.prefix[:len(path)] {
230 // with prefix "foobar", walking "fooz" means we're done
233 if path < params.prefix[:len(path)] {
234 // with prefix "foobar", walking "foobag" is pointless
235 return filepath.SkipDir
237 if fi.IsDir() && !strings.HasPrefix(params.prefix+"/", path+"/") {
238 // with prefix "foo/bar", walking "fo"
239 // is pointless (but walking "foo" or
240 // "foo/bar" is necessary)
241 return filepath.SkipDir
243 if len(path) < len(params.prefix) {
244 // can't skip anything, and this entry
245 // isn't in the results, so just
250 if path[:len(params.prefix)] > params.prefix {
251 // with prefix "foobar", nothing we
252 // see after "foozzz" is relevant
256 if path < params.marker || path < params.prefix {
262 if params.delimiter != "" {
263 idx := strings.Index(path[len(params.prefix):], params.delimiter)
265 // with prefix "foobar" and delimiter
266 // "z", when we hit "foobar/baz", we
267 // add "/baz" to commonPrefixes and
268 // stop descending (note that even if
269 // delimiter is "/" we don't add
270 // anything to commonPrefixes when
271 // seeing a dir: we wait until we see
272 // a file, so we don't incorrectly
273 // return results for empty dirs)
274 commonPrefixes[path[:len(params.prefix)+idx+1]] = true
275 return filepath.SkipDir
278 if len(resp.Contents)+len(commonPrefixes) >= params.maxKeys {
279 resp.IsTruncated = true
280 if params.delimiter != "" {
281 resp.NextMarker = path
285 resp.Contents = append(resp.Contents, s3.Key{
290 if err != nil && err != errDone {
291 http.Error(w, err.Error(), http.StatusInternalServerError)
294 if params.delimiter != "" {
295 for prefix := range commonPrefixes {
296 resp.CommonPrefixes = append(resp.CommonPrefixes, prefix)
297 sort.Strings(resp.CommonPrefixes)
300 if err := xml.NewEncoder(w).Encode(resp); err != nil {
301 ctxlog.FromContext(r.Context()).WithError(err).Error("error writing xml response")