ADD = "add"
DEL = "del"
MOD = "mod"
+TOK = "tok"
FILE = "file"
COLLECTION = "collection"
def __init__(self, parent=None):
self.parent = parent
- self._modified = True
+ self._committed = False
self._callback = None
self._items = {}
else:
item = ArvadosFile(self, pathcomponents[0])
self._items[pathcomponents[0]] = item
- self._modified = True
+ self._committed = False
self.notify(ADD, self, pathcomponents[0], item)
return item
else:
# create new collection
item = Subcollection(self, pathcomponents[0])
self._items[pathcomponents[0]] = item
- self._modified = True
+ self._committed = False
self.notify(ADD, self, pathcomponents[0], item)
if isinstance(item, RichCollectionBase):
return item.find_or_create(pathcomponents[1], create_type)
else:
- raise IOError(errno.ENOTDIR, "Interior path components must be subcollection")
+ raise IOError(errno.ENOTDIR, "Not a directory: '%s'" % pathcomponents[0])
else:
return self
"""
if not path:
- raise errors.ArgumentError("Parameter 'path' must not be empty.")
+ raise errors.ArgumentError("Parameter 'path' is empty.")
pathcomponents = path.split("/", 1)
item = self._items.get(pathcomponents[0])
else:
return item
else:
- raise IOError(errno.ENOTDIR, "Interior path components must be subcollection")
+ raise IOError(errno.ENOTDIR, "Is not a directory: %s" % pathcomponents[0])
+ @synchronized
def mkdirs(self, path):
"""Recursive subcollection create.
- Like `os.mkdirs()`. Will create intermediate subcollections needed to
- contain the leaf subcollection path.
+ Like `os.makedirs()`. Will create intermediate subcollections needed
+ to contain the leaf subcollection path.
"""
+
+ if self.find(path) != None:
+ raise IOError(errno.EEXIST, "Directory or file exists: '%s'" % path)
+
return self.find_or_create(path, COLLECTION)
def open(self, path, mode="r"):
if arvfile is None:
raise IOError(errno.ENOENT, "File not found")
if not isinstance(arvfile, ArvadosFile):
- raise IOError(errno.EISDIR, "Path must refer to a file.")
+ raise IOError(errno.EISDIR, "Is a directory: %s" % path)
if mode[0] == "w":
arvfile.truncate(0)
name = os.path.basename(path)
if mode == "r":
- return ArvadosFileReader(arvfile, mode, num_retries=self.num_retries)
+ return ArvadosFileReader(arvfile, num_retries=self.num_retries)
else:
return ArvadosFileWriter(arvfile, mode, num_retries=self.num_retries)
- @synchronized
def modified(self):
- """Test if the collection (or any subcollection or file) has been modified."""
- if self._modified:
- return True
- for k,v in self._items.items():
- if v.modified():
- return True
- return False
+ """Determine if the collection has been modified since last commited."""
+ return not self.committed()
@synchronized
- def set_unmodified(self):
- """Recursively clear modified flag."""
- self._modified = False
+ def committed(self):
+ """Determine if the collection has been committed to the API server."""
+
+ if self._committed is False:
+ return False
+ for v in self._items.values():
+ if v.committed() is False:
+ return False
+ return True
+
+ @synchronized
+ def set_committed(self):
+ """Recursively set committed flag to True."""
+ self._committed = True
for k,v in self._items.items():
- v.set_unmodified()
+ v.set_committed()
@synchronized
def __iter__(self):
def __delitem__(self, p):
"""Delete an item by name which is directly contained by this collection."""
del self._items[p]
- self._modified = True
+ self._committed = False
self.notify(DEL, self, p, None)
@synchronized
"""
if not path:
- raise errors.ArgumentError("Parameter 'path' must not be empty.")
+ raise errors.ArgumentError("Parameter 'path' is empty.")
pathcomponents = path.split("/", 1)
item = self._items.get(pathcomponents[0])
raise IOError(errno.ENOTEMPTY, "Subcollection not empty")
deleteditem = self._items[pathcomponents[0]]
del self._items[pathcomponents[0]]
- self._modified = True
+ self._committed = False
self.notify(DEL, self, pathcomponents[0], deleteditem)
else:
item.remove(pathcomponents[1])
# Actually make the move or copy.
if reparent:
- source_obj.reparent(self, target_name)
+ source_obj._reparent(self, target_name)
item = source_obj
else:
item = source_obj.clone(self, target_name)
self._items[target_name] = item
- self._modified = True
+ self._committed = False
if modified_from:
self.notify(MOD, self, target_name, (modified_from, item))
source_obj, target_dir, target_name = self._get_src_target(source, target_path, source_collection, False)
if not source_obj.writable():
- raise IOError(errno.EROFS, "Source collection must be writable.")
+ raise IOError(errno.EROFS, "Source collection is read only.")
target_dir.add(source_obj, target_name, overwrite, True)
+ def portable_manifest_text(self, stream_name="."):
+ """Get the manifest text for this collection, sub collections and files.
+
+ This method does not flush outstanding blocks to Keep. It will return
+ a normalized manifest with access tokens stripped.
+
+ :stream_name:
+ Name to use for this stream (directory)
+
+ """
+ return self._get_manifest_text(stream_name, True, True)
+
@synchronized
def manifest_text(self, stream_name=".", strip=False, normalize=False):
"""Get the manifest text for this collection, sub collections and files.
+ This method will flush outstanding blocks to Keep. By default, it will
+ not normalize an unmodified manifest or strip access tokens.
+
:stream_name:
- Name of the stream (directory)
+ Name to use for this stream (directory)
:strip:
If True, remove signing tokens from block locators if present.
"""
- if self.modified() or self._manifest_text is None or normalize:
+ self._my_block_manager().commit_all()
+ return self._get_manifest_text(stream_name, strip, normalize)
+
+ @synchronized
+ def _get_manifest_text(self, stream_name, strip, normalize):
+ """Get the manifest text for this collection, sub collections and files.
+
+ :stream_name:
+ Name to use for this stream (directory)
+
+ :strip:
+ If True, remove signing tokens from block locators if present.
+ If False (default), block locators are left unchanged.
+
+ :normalize:
+ If True, always export the manifest text in normalized form
+ even if the Collection is not modified. If False (default) and the collection
+ is not modified, return the original manifest text even if it is not
+ in normalized form.
+
+ """
+
+ if not self.committed() or self._manifest_text is None or normalize:
stream = {}
buf = []
sorted_keys = sorted(self.keys())
if stream:
buf.append(" ".join(normalize_stream(stream_name, stream)) + "\n")
for dirname in [s for s in sorted_keys if isinstance(self[s], RichCollectionBase)]:
- buf.append(self[dirname].manifest_text(stream_name=os.path.join(stream_name, dirname), strip=strip))
+ buf.append(self[dirname].manifest_text(stream_name=os.path.join(stream_name, dirname), strip=strip, normalize=True))
return "".join(buf)
else:
if strip:
changes.extend(self[k].diff(end_collection[k], os.path.join(prefix, k), holding_collection))
elif end_collection[k] != self[k]:
changes.append((MOD, os.path.join(prefix, k), self[k].clone(holding_collection, ""), end_collection[k].clone(holding_collection, "")))
+ else:
+ changes.append((TOK, os.path.join(prefix, k), self[k].clone(holding_collection, ""), end_collection[k].clone(holding_collection, "")))
else:
changes.append((ADD, os.path.join(prefix, k), end_collection[k].clone(holding_collection, "")))
return changes
alternate path indicating the conflict.
"""
+ if changes:
+ self._committed = False
for change in changes:
event_type = change[0]
path = change[1]
initial = change[2]
local = self.find(path)
- conflictpath = "%s~conflict-%s~" % (path, time.strftime("%Y-%m-%d-%H:%M:%S",
+ conflictpath = "%s~%s~conflict~" % (path, time.strftime("%Y%m%d-%H%M%S",
time.gmtime()))
if event_type == ADD:
if local is None:
# There is already local file and it is different:
# save change to conflict file.
self.copy(initial, conflictpath)
- elif event_type == MOD:
+ elif event_type == MOD or event_type == TOK:
final = change[3]
if local == initial:
# Local matches the "initial" item so it has not
def portable_data_hash(self):
"""Get the portable data hash for this collection's manifest."""
- stripped = self.manifest_text(strip=True)
+ stripped = self.portable_manifest_text()
return hashlib.md5(stripped).hexdigest() + '+' + str(len(stripped))
@synchronized
def __ne__(self, other):
return not self.__eq__(other)
+ @synchronized
+ def flush(self):
+ """Flush bufferblocks to Keep."""
+ for e in self.values():
+ e.flush()
+
class Collection(RichCollectionBase):
"""Represents the root of an Arvados Collection.
self._manifest_locator = None
self._manifest_text = None
self._api_response = None
+ self._past_versions = set()
self.lock = threading.RLock()
self.events = None
self._manifest_text = manifest_locator_or_text
else:
raise errors.ArgumentError(
- "Argument to CollectionReader must be a manifest or a collection UUID")
+ "Argument to CollectionReader is not a manifest or a collection UUID")
try:
self._populate()
def writable(self):
return True
+ @synchronized
+ def known_past_version(self, modified_at_and_portable_data_hash):
+ return modified_at_and_portable_data_hash in self._past_versions
+
@synchronized
@retry_method
def update(self, other=None, num_retries=None):
if self._manifest_locator is None:
raise errors.ArgumentError("`other` is None but collection does not have a manifest_locator uuid")
response = self._my_api().collections().get(uuid=self._manifest_locator).execute(num_retries=num_retries)
+ if (self.known_past_version((response.get("modified_at"), response.get("portable_data_hash"))) and
+ response.get("portable_data_hash") != self.portable_data_hash()):
+ # The record on the server is different from our current one, but we've seen it before,
+ # so ignore it because it's already been merged.
+ # However, if it's the same as our current record, proceed with the update, because we want to update
+ # our tokens.
+ return
+ else:
+ self._past_versions.add((response.get("modified_at"), response.get("portable_data_hash")))
other = CollectionReader(response["manifest_text"])
baseline = CollectionReader(self._manifest_text)
self.apply(baseline.diff(other))
+ self._manifest_text = self.manifest_text()
@synchronized
def _my_api(self):
self._block_manager = _BlockManager(self._my_keep())
return self._block_manager
+ def _remember_api_response(self, response):
+ self._api_response = response
+ self._past_versions.add((response.get("modified_at"), response.get("portable_data_hash")))
+
def _populate_from_api_server(self):
# As in KeepClient itself, we must wait until the last
# possible moment to instantiate an API client, in order to
# clause, just like any other Collection lookup
# failure. Return an exception, or None if successful.
try:
- self._api_response = self._my_api().collections().get(
+ self._remember_api_response(self._my_api().collections().get(
uuid=self._manifest_locator).execute(
- num_retries=self.num_retries)
+ num_retries=self.num_retries))
self._manifest_text = self._api_response['manifest_text']
return None
except Exception as e:
def __exit__(self, exc_type, exc_value, traceback):
"""Support scoped auto-commit in a with: block."""
- if exc_type is not None:
+ if exc_type is None:
if self.writable() and self._has_collection_uuid():
self.save()
+ self.stop_threads()
+
+ def stop_threads(self):
if self._block_manager is not None:
self._block_manager.stop_threads()
"""Save collection to an existing collection record.
Commit pending buffer blocks to Keep, merge with remote record (if
- merge=True, the default), write the manifest to Keep, and update the
- collection record.
+ merge=True, the default), and update the collection record. Returns
+ the current manifest text.
Will raise AssertionError if not associated with a collection record on
the API server. If you want to save a manifest to Keep only, see
Retry count on API calls (if None, use the collection default)
"""
- if self.modified():
+ if not self.committed():
if not self._has_collection_uuid():
- raise AssertionError("Collection manifest_locator must be a collection uuid. Use save_new() for new collections.")
+ raise AssertionError("Collection manifest_locator is not a collection uuid. Use save_new() for new collections.")
+
self._my_block_manager().commit_all()
+
if merge:
self.update()
- self._my_keep().put(self.manifest_text(strip=True), num_retries=num_retries)
text = self.manifest_text(strip=False)
- self._api_response = self._my_api().collections().update(
+ self._remember_api_response(self._my_api().collections().update(
uuid=self._manifest_locator,
body={'manifest_text': text}
).execute(
- num_retries=num_retries)
+ num_retries=num_retries))
self._manifest_text = self._api_response["manifest_text"]
- self.set_unmodified()
+ self.set_committed()
+
+ return self._manifest_text
@must_be_writable
@synchronized
@retry_method
- def save_new(self, name=None, create_collection_record=True, owner_uuid=None, ensure_unique_name=False, num_retries=None):
+ def save_new(self, name=None,
+ create_collection_record=True,
+ owner_uuid=None,
+ ensure_unique_name=False,
+ num_retries=None):
"""Save collection to a new collection record.
- Commit pending buffer blocks to Keep, write the manifest to Keep, and
- create a new collection record (if create_collection_record True).
- After creating a new collection record, this Collection object will be
- associated with the new record used by `save()`.
+ Commit pending buffer blocks to Keep and, when create_collection_record
+ is True (default), create a new collection record. After creating a
+ new collection record, this Collection object will be associated with
+ the new record used by `save()`. Returns the current manifest text.
:name:
The collection name.
:create_collection_record:
- If True, create a collection record. If False, only save the manifest to keep.
+ If True, create a collection record on the API server.
+ If False, only commit blocks to Keep and return the manifest text.
:owner_uuid:
the user, or project uuid that will own this collection.
"""
self._my_block_manager().commit_all()
- self._my_keep().put(self.manifest_text(strip=True), num_retries=num_retries)
text = self.manifest_text(strip=False)
if create_collection_record:
if name is None:
- name = "Collection created %s" % (time.strftime("%Y-%m-%d %H:%M:%S %Z", time.localtime()))
+ name = "New collection"
ensure_unique_name = True
body = {"manifest_text": text,
if owner_uuid:
body["owner_uuid"] = owner_uuid
- self._api_response = self._my_api().collections().create(ensure_unique_name=ensure_unique_name, body=body).execute(num_retries=num_retries)
+ self._remember_api_response(self._my_api().collections().create(ensure_unique_name=ensure_unique_name, body=body).execute(num_retries=num_retries))
text = self._api_response["manifest_text"]
self._manifest_locator = self._api_response["uuid"]
- self._manifest_text = text
- self.set_unmodified()
+ self._manifest_text = text
+ self.set_committed()
+
+ return text
@synchronized
def _import_manifest(self, manifest_text):
segments = []
streamoffset = 0L
state = BLOCKS
- self.mkdirs(stream_name)
+ self.find_or_create(stream_name, COLLECTION)
continue
if state == BLOCKS:
raise errors.SyntaxError("File %s conflicts with stream of the same name.", filepath)
else:
# error!
- raise errors.SyntaxError("Invalid manifest format")
+ raise errors.SyntaxError("Invalid manifest format, expected file segment but did not match format: '%s'" % tok)
if sep == "\n":
stream_name = None
state = STREAM_NAME
- self.set_unmodified()
+ self.set_committed()
@synchronized
def notify(self, event, collection, name, item):
"""This is a subdirectory within a collection that doesn't have its own API
server record.
- It falls under the umbrella of the root collection.
+ Subcollection locking falls under the umbrella lock of its root collection.
"""
c._clonefrom(self)
return c
+ @must_be_writable
+ @synchronized
+ def _reparent(self, newparent, newname):
+ self._committed = False
+ self.flush()
+ self.parent.remove(self.name, recursive=True)
+ self.parent = newparent
+ self.name = newname
+ self.lock = self.parent.root_collection().lock
+
class CollectionReader(Collection):
"""A read-only collection object.