X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/1e3f8ceebd90058e902494fae84b1fd57ac6693b..HEAD:/services/fuse/arvados_fuse/unmount.py diff --git a/services/fuse/arvados_fuse/unmount.py b/services/fuse/arvados_fuse/unmount.py index 442213d301..144c582ddc 100644 --- a/services/fuse/arvados_fuse/unmount.py +++ b/services/fuse/arvados_fuse/unmount.py @@ -6,6 +6,7 @@ import collections import errno import os import subprocess +import sys import time @@ -30,6 +31,54 @@ def mountinfo(): return mi +def paths_to_unmount(path, mnttype): + paths = [] + for m in mountinfo(): + if m.path == path or m.path.startswith(path+"/"): + paths.append(m.path) + if not (m.is_fuse and (mnttype is None or + mnttype == m.mnttype)): + raise Exception( + "cannot unmount {}: mount type is {}".format( + path, m.mnttype)) + return paths + + +def safer_realpath(path, loop=True): + """Similar to os.path.realpath(), but avoids calling lstat(). + + Leaves some symlinks unresolved.""" + if path == '/': + return path, True + elif not path.startswith('/'): + path = os.path.abspath(path) + while True: + path = path.rstrip('/') + dirname, basename = os.path.split(path) + try: + path, resolved = safer_realpath(os.path.join(dirname, os.readlink(path)), loop=False) + except OSError as e: + # Path is not a symlink (EINVAL), or is unreadable, or + # doesn't exist. If the error was EINVAL and dirname can + # be resolved, we will have eliminated all symlinks and it + # will be safe to call normpath(). + dirname, resolved = safer_realpath(dirname, loop=loop) + path = os.path.join(dirname, basename) + if resolved and e.errno == errno.EINVAL: + return os.path.normpath(path), True + else: + return path, False + except RuntimeError: + if not loop: + # Unwind to the point where we first started following + # symlinks. + raise + # Resolving the whole path landed in a symlink cycle, but + # we might still be able to resolve dirname. + dirname, _ = safer_realpath(dirname, loop=loop) + return os.path.join(dirname, basename), False + + def unmount(path, subtype=None, timeout=10, recursive=False): """Unmount the fuse mount at path. @@ -45,7 +94,7 @@ def unmount(path, subtype=None, timeout=10, recursive=False): fuse mount at all. Raises an exception if it cannot be unmounted. """ - path = os.path.realpath(path) + path, _ = safer_realpath(path) if subtype is None: mnttype = None @@ -55,21 +104,20 @@ def unmount(path, subtype=None, timeout=10, recursive=False): mnttype = 'fuse.' + subtype if recursive: - paths = [] - for m in mountinfo(): - if m.path == path or m.path.startswith(path+"/"): - paths.append(m.path) - if not (m.is_fuse and (mnttype is None or - mnttype == m.mnttype)): - raise Exception( - "cannot unmount {}: mount type is {}".format( - path, m.mnttype)) + paths = paths_to_unmount(path, mnttype) + if not paths: + # We might not have found any mounts merely because path + # contains symlinks, so we should resolve them and try + # again. We didn't do this from the outset because + # realpath() can hang (see explanation below). + paths = paths_to_unmount(os.path.realpath(path), mnttype) for path in sorted(paths, key=len, reverse=True): unmount(path, timeout=timeout, recursive=False) return len(paths) > 0 was_mounted = False attempted = False + fusermount_output = b'' if timeout is None: deadline = None else: @@ -80,16 +128,49 @@ def unmount(path, subtype=None, timeout=10, recursive=False): for m in mountinfo(): if m.is_fuse and (mnttype is None or mnttype == m.mnttype): try: - if os.path.realpath(m.path) == path: + if m.path == path: was_mounted = True mounted = True break except OSError: continue - if not mounted: + if not was_mounted and path != os.path.realpath(path): + # If the specified path contains symlinks, it won't appear + # verbatim in mountinfo. + # + # It might seem like we should have called realpath() from + # the outset. But we can't: realpath() hangs (in lstat()) + # if we call it on an unresponsive mount point, and this + # is an important and common scenario. + # + # By waiting until now to try realpath(), we avoid this + # problem in the most common cases, which are: (1) the + # specified path has no symlinks and is a mount point, in + # which case was_mounted==True and we can proceed without + # calling realpath(); and (2) the specified path is not a + # mount point (e.g., it was already unmounted by someone + # else, or it's a typo), and realpath() can determine that + # without hitting any other unresponsive mounts. + path = os.path.realpath(path) + continue + elif not mounted: + if was_mounted: + # This appears to avoid a race condition where we + # return control to the caller after running + # "fusermount -u -z" (see below), the caller (e.g., + # arv-mount --replace) immediately tries to attach a + # new fuse mount at the same mount point, the + # lazy-unmount process unmounts that _new_ mount while + # it is being initialized, and the setup code waits + # forever for the new mount to be initialized. + time.sleep(1) return was_mounted if attempted: + # Report buffered stderr from previous call to fusermount, + # now that we know it didn't succeed. + sys.stderr.buffer.write(fusermount_output) + delay = 1 if deadline: delay = min(delay, deadline - time.time()) @@ -107,6 +188,10 @@ def unmount(path, subtype=None, timeout=10, recursive=False): attempted = True try: - subprocess.check_call(["fusermount", "-u", "-z", path]) - except subprocess.CalledProcessError: - pass + subprocess.check_output( + ["fusermount", "-u", "-z", path], + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + fusermount_output = e.output + else: + fusermount_output = b''