#
# SPDX-License-Identifier: AGPL-3.0
+from future.utils import native_str
+from builtins import range
+from builtins import object
import argparse
import arvados
import daemon
with "--".
""")
self.add_argument('--version', action='version',
- version="%s %s" % (sys.argv[0], __version__),
+ version=u"%s %s" % (sys.argv[0], __version__),
help='Print version and exit.')
self.add_argument('mountpoint', type=str, help="""Mount point.""")
self.add_argument('--allow-other', action='store_true',
type=str, metavar='PATH', action='append', default=[],
help="Create a new collection, mount it in read/write mode at mountpoint/PATH, and delete it when unmounting.")
+
self.add_argument('--debug', action='store_true', help="""Debug mode""")
self.add_argument('--logfile', help="""Write debug logs and errors to the specified file (default stderr).""")
self.add_argument('--foreground', action='store_true', help="""Run in foreground (default is to daemonize unless --exec specified)""", default=False)
self.add_argument('--encoding', type=str, help="Character encoding to use for filesystem, default is utf-8 (see Python codec registry for list of available encodings)", default="utf-8")
- self.add_argument('--file-cache', type=int, help="File data cache size, in bytes (default 256MiB)", default=256*1024*1024)
- self.add_argument('--directory-cache', type=int, help="Directory data cache size, in bytes (default 128MiB)", default=128*1024*1024)
+ self.add_argument('--file-cache', type=int, help="File data cache size, in bytes (default 8 GiB for disk-based cache or 256 MiB with RAM-only cache)", default=0)
+ self.add_argument('--directory-cache', type=int, help="Directory data cache size, in bytes (default 128 MiB)", default=128*1024*1024)
+
+ cachetype = self.add_mutually_exclusive_group()
+ cachetype.add_argument('--ram-cache', action='store_false', dest='disk_cache', help="Use in-memory caching only", default=True)
+ cachetype.add_argument('--disk-cache', action='store_true', dest='disk_cache', help="Use disk based caching (default)", default=True)
+
+ self.add_argument('--disk-cache-dir', type=str, help="Disk cache location (default ~/.cache/arvados/keep)", default=None)
self.add_argument('--disable-event-listening', action='store_true', help="Don't subscribe to events on the API server", dest="disable_event_listening", default=False)
self.add_argument('--read-only', action='store_false', help="Mount will be read only (default)", dest="enable_write", default=False)
self.add_argument('--read-write', action='store_true', help="Mount will be read-write", dest="enable_write", default=False)
+ self.add_argument('--storage-classes', type=str, metavar='CLASSES', help="Specify comma separated list of storage classes to be used when saving data of new collections", default=None)
self.add_argument('--crunchstat-interval', type=float, help="Write stats to stderr every N seconds (default disabled)", default=0)
if self.args.replace:
unmount(path=self.args.mountpoint,
timeout=self.args.unmount_timeout)
- llfuse.init(self.operations, self.args.mountpoint, self._fuse_options())
+ llfuse.init(self.operations, native_str(self.args.mountpoint), self._fuse_options())
if self.daemon:
daemon.DaemonContext(
working_directory=os.path.dirname(self.args.mountpoint),
- files_preserve=range(
- 3, resource.getrlimit(resource.RLIMIT_NOFILE)[1])
+ files_preserve=list(range(
+ 3, resource.getrlimit(resource.RLIMIT_NOFILE)[1]))
).open()
if self.listen_for_events and not self.args.disable_event_listening:
self.operations.listen_for_events()
try:
self.api = arvados.safeapi.ThreadSafeApiCache(
apiconfig=arvados.config.settings(),
+ # default value of file_cache is 0, this tells KeepBlockCache to
+ # choose a default based on whether disk_cache is enabled or not.
keep_params={
- 'block_cache': arvados.keep.KeepBlockCache(self.args.file_cache),
+ 'block_cache': arvados.keep.KeepBlockCache(cache_max=self.args.file_cache,
+ disk_cache=self.args.disk_cache,
+ disk_cache_dir=self.args.disk_cache_dir),
'num_retries': self.args.retries,
- })
+ },
+ version='v1',
+ )
except KeyError as e:
self.logger.error("Missing environment: %s", e)
exit(1)
usr = self.api.users().current().execute(num_retries=self.args.retries)
now = time.time()
dir_class = None
- dir_args = [llfuse.ROOT_INODE, self.operations.inodes, self.api, self.args.retries]
+ dir_args = [llfuse.ROOT_INODE, self.operations.inodes, self.api, self.args.retries, self.args.enable_write]
mount_readme = False
+ storage_classes = None
+ if self.args.storage_classes is not None:
+ storage_classes = self.args.storage_classes.replace(' ', '').split(',')
+ self.logger.info("Storage classes requested for new collections: {}".format(', '.join(storage_classes)))
+
if self.args.collection is not None:
# Set up the request handler with the collection at the root
# First check that the collection is readable
mount_readme = True
if dir_class is not None:
- ent = dir_class(*dir_args)
+ if dir_class in [TagsDirectory, CollectionDirectory]:
+ ent = dir_class(*dir_args)
+ else:
+ ent = dir_class(*dir_args, storage_classes=storage_classes)
self.operations.inodes.add_entry(ent)
self.listen_for_events = ent.want_event_subscribe()
return
e = self.operations.inodes.add_entry(Directory(
- llfuse.ROOT_INODE, self.operations.inodes))
+ llfuse.ROOT_INODE, self.operations.inodes, self.api.config, self.args.enable_write))
dir_args[0] = e.inode
for name in self.args.mount_by_id:
- self._add_mount(e, name, MagicDirectory(*dir_args, pdh_only=False))
+ self._add_mount(e, name, MagicDirectory(*dir_args, pdh_only=False, storage_classes=storage_classes))
for name in self.args.mount_by_pdh:
self._add_mount(e, name, MagicDirectory(*dir_args, pdh_only=True))
for name in self.args.mount_by_tag:
self._add_mount(e, name, TagsDirectory(*dir_args))
for name in self.args.mount_home:
- self._add_mount(e, name, ProjectDirectory(*dir_args, project_object=usr, poll=True))
+ self._add_mount(e, name, ProjectDirectory(*dir_args, project_object=usr, poll=True, storage_classes=storage_classes))
for name in self.args.mount_shared:
- self._add_mount(e, name, SharedDirectory(*dir_args, exclude=usr, poll=True))
+ self._add_mount(e, name, SharedDirectory(*dir_args, exclude=usr, poll=True, storage_classes=storage_classes))
for name in self.args.mount_tmp:
- self._add_mount(e, name, TmpCollectionDirectory(*dir_args))
+ self._add_mount(e, name, TmpCollectionDirectory(*dir_args, storage_classes=storage_classes))
if mount_readme:
text = self._readme_text(