target_dir.add(source_obj, target_name, overwrite)
- @synchronized
+ def portable_manifest_text(self, stream_name="."):
+ """Get the manifest text for this collection, sub collections and files.
+
+ This method does not flush outstanding blocks to Keep. It will return
+ a normalized manifest with access tokens stripped.
+
+ :stream_name:
+ Name to use for this stream (directory)
+
+ """
+ return self._get_manifest_text(stream_name, True, True)
+
def manifest_text(self, stream_name=".", strip=False, normalize=False):
"""Get the manifest text for this collection, sub collections and files.
+ This method will flush outstanding blocks to Keep. By default, it will
+ not normalize an unmodified manifest or strip access tokens.
+
:stream_name:
- Name of the stream (directory)
+ Name to use for this stream (directory)
:strip:
If True, remove signing tokens from block locators if present.
- If False (default), block locators are left unchanged. Note: if
- strip is False, this will also flush any open files (committing blocks to Keep).
+ If False (default), block locators are left unchanged.
:normalize:
If True, always export the manifest text in normalized form
"""
- if self.modified() or self._manifest_text is None or normalize:
- if strip is False:
- self._my_block_manager().commit_all()
+ self._my_block_manager().commit_all()
+ return self._get_manifest_text(stream_name, strip, normalize)
+
+ @synchronized
+ def _get_manifest_text(self, stream_name, strip, normalize):
+ """Get the manifest text for this collection, sub collections and files.
+ :stream_name:
+ Name to use for this stream (directory)
+
+ :strip:
+ If True, remove signing tokens from block locators if present.
+ If False (default), block locators are left unchanged.
+
+ :normalize:
+ If True, always export the manifest text in normalized form
+ even if the Collection is not modified. If False (default) and the collection
+ is not modified, return the original manifest text even if it is not
+ in normalized form.
+
+ """
+
+ if self.modified() or self._manifest_text is None or normalize:
stream = {}
buf = []
sorted_keys = sorted(self.keys())
if stream:
buf.append(" ".join(normalize_stream(stream_name, stream)) + "\n")
for dirname in [s for s in sorted_keys if isinstance(self[s], RichCollectionBase)]:
- buf.append(self[dirname].manifest_text(stream_name=os.path.join(stream_name, dirname), strip=strip))
+ buf.append(self[dirname].manifest_text(stream_name=os.path.join(stream_name, dirname), strip=strip, normalize=True))
return "".join(buf)
else:
if strip:
def portable_data_hash(self):
"""Get the portable data hash for this collection's manifest."""
- stripped = self.manifest_text(strip=True)
+ stripped = self.portable_manifest_text()
return hashlib.md5(stripped).hexdigest() + '+' + str(len(stripped))
@synchronized
"""Save collection to an existing collection record.
Commit pending buffer blocks to Keep, merge with remote record (if
- merge=True, the default), and update the collection record.
+ merge=True, the default), and update the collection record. Returns
+ the current manifest text.
Will raise AssertionError if not associated with a collection record on
the API server. If you want to save a manifest to Keep only, see
if not self._has_collection_uuid():
raise AssertionError("Collection manifest_locator must be a collection uuid. Use save_new() for new collections.")
+ self._my_block_manager().commit_all()
+
if merge:
self.update()
self._manifest_text = self._api_response["manifest_text"]
self.set_unmodified()
+ return self._manifest_text
+
@must_be_writable
@synchronized
@retry_method
- def save_new(self, name=None, owner_uuid=None, ensure_unique_name=False, num_retries=None):
+ def save_new(self, name=None,
+ create_collection_record=True,
+ owner_uuid=None,
+ ensure_unique_name=False,
+ num_retries=None):
"""Save collection to a new collection record.
- Commit pending buffer blocks to Keep and create a new collection record
- (if create_collection_record True). After creating a new collection
- record, this Collection object will be associated with the new record
- used by `save()`.
+ Commit pending buffer blocks to Keep and, when create_collection_record
+ is True (default), create a new collection record. After creating a
+ new collection record, this Collection object will be associated with
+ the new record used by `save()`. Returns the current manifest text.
:name:
The collection name.
+ :create_collection_record:
+ If True, create a collection record on the API server.
+ If False, only commit blocks to Keep and return the manifest text.
+
:owner_uuid:
the user, or project uuid that will own this collection.
If None, defaults to the current user.
Retry count on API calls (if None, use the collection default)
"""
+ self._my_block_manager().commit_all()
text = self.manifest_text(strip=False)
- if name is None:
- name = "Collection created %s" % (time.strftime("%Y-%m-%d %H:%M:%S %Z", time.localtime()))
+ if create_collection_record:
+ if name is None:
+ name = "Collection created %s" % (time.strftime("%Y-%m-%d %H:%M:%S %Z", time.localtime()))
- body = {"manifest_text": text,
- "name": name}
- if owner_uuid:
- body["owner_uuid"] = owner_uuid
+ body = {"manifest_text": text,
+ "name": name}
+ if owner_uuid:
+ body["owner_uuid"] = owner_uuid
- self._api_response = self._my_api().collections().create(ensure_unique_name=ensure_unique_name, body=body).execute(num_retries=num_retries)
- text = self._api_response["manifest_text"]
+ self._api_response = self._my_api().collections().create(ensure_unique_name=ensure_unique_name, body=body).execute(num_retries=num_retries)
+ text = self._api_response["manifest_text"]
- self._manifest_locator = self._api_response["uuid"]
+ self._manifest_locator = self._api_response["uuid"]
- self._manifest_text = text
- self.set_unmodified()
+ self._manifest_text = text
+ self.set_unmodified()
+
+ return text
@synchronized
def subscribe(self, callback):
block_locator = re.match(r'[0-9a-f]{32}\+(\d+)(\+\S+)*', tok)
if block_locator:
blocksize = long(block_locator.group(1))
- blocks.append(Range(tok, streamoffset, blocksize))
+ blocks.append(Range(tok, streamoffset, blocksize, 0))
streamoffset += blocksize
else:
state = SEGMENTS
projects / arvados.git / blobdiff