path = None
return path
-def api(version=None, cache=True, host=None, token=None, insecure=False, **kwargs):
+def api(version=None, cache=True, host=None, token=None, insecure=False, apiconfig=None, **kwargs):
"""Return an apiclient Resources object for an Arvados instance.
Arguments:
* host: The Arvados API server host (and optional :port) to connect to.
* token: The authentication token to send with each API call.
* insecure: If True, ignore SSL certificate validation errors.
+ * apiconfig: If provided, this should be a dict containing with entries for ARVADOS_API_HOST, ARVADOS_API_TOKEN, and optionally ARVADOS_API_HOST_INSECURE
Additional keyword arguments will be passed directly to
`apiclient_discovery.build` if a new Resource object is created.
pass
elif not host and not token:
# Load from user configuration or environment
+ if apiconfig is None:
+ apiconfig = config.settings()
for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
- if x not in config.settings():
+ if x not in apiconfig:
raise ValueError("%s is not set. Aborting." % x)
- host = config.get('ARVADOS_API_HOST')
- token = config.get('ARVADOS_API_TOKEN')
- insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE')
+ host = apiconfig.get('ARVADOS_API_HOST')
+ token = apiconfig.get('ARVADOS_API_TOKEN')
+ insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE', apiconfig)
else:
# Caller provided one but not the other
if not host:
MOD = "mod"
class SynchronizedCollectionBase(CollectionBase):
+ """Base class for Collections and Subcollections. Implements the majority of
+ functionality relating to accessing items in the Collection."""
+
def __init__(self, parent=None):
self.parent = parent
self._modified = True
"""
changes = []
if holding_collection is None:
- holding_collection = Collection()
+ holding_collection = CollectionRoot(api_client=self._my_api(), keep_client=self._my_keep(), sync=SYNC_READONLY)
for k in self:
if k not in end_collection:
changes.append((DEL, os.path.join(prefix, k), self[k].clone(holding_collection)))
def __ne__(self, other):
return not self.__eq__(other)
-class Collection(SynchronizedCollectionBase):
- """Store an Arvados collection, consisting of a set of files and
- sub-collections. This object
+class CollectionRoot(SynchronizedCollectionBase):
+ """Represents the root of an Arvados Collection, which may be associated with
+ an API server Collection record.
+
+ Brief summary of useful methods:
+
+ :To read an existing file:
+ `c.open("myfile", "r")`
+
+ :To write a new file:
+ `c.open("myfile", "w")`
+
+ :To determine if a file exists:
+ `c.find("myfile") is not None`
+
+ :To copy a file:
+ `c.copy("source", "dest")`
+
+ :To delete a file:
+ `c.remove("myfile")`
+
+ :To save to an existing collection record:
+ `c.save()`
+
+ :To save a new collection record:
+ `c.save_new()`
+
+ :To merge remote changes into this object:
+ `c.update()`
+
+ This class is threadsafe. The root collection object, all subcollections
+ and files are protected by a single lock (i.e. each access locks the entire
+ collection).
+
"""
def __init__(self, manifest_locator_or_text=None,
parent=None,
- config=None,
+ apiconfig=None,
api_client=None,
keep_client=None,
num_retries=None,
block_manager=None,
- sync=SYNC_READONLY):
+ sync=None):
""":manifest_locator_or_text:
One of Arvados collection UUID, block locator of
a manifest, raw manifest text, or None (to create an empty collection).
:parent:
the parent Collection, may be None.
- :config:
- the arvados configuration to get the hostname and api token.
+ :apiconfig:
+ A dict containing keys for ARVADOS_API_HOST and ARVADOS_API_TOKEN.
Prefer this over supplying your own api_client and keep_client (except in testing).
Will use default config settings if not specified.
:api_client:
- The API client object to use for requests. If not specified, create one using `config`.
+ The API client object to use for requests. If not specified, create one using `apiconfig`.
:keep_client:
- the Keep client to use for requests. If not specified, create one using `config`.
+ the Keep client to use for requests. If not specified, create one using `apiconfig`.
:num_retries:
the number of retries for API and Keep requests.
:block_manager:
background websocket events, on block write, or on file close.
"""
- super(Collection, self).__init__(parent)
+ super(CollectionRoot, self).__init__(parent)
self._api_client = api_client
self._keep_client = keep_client
self._block_manager = block_manager
- self._config = config
+
+ if apiconfig:
+ self._config = apiconfig
+ else:
+ self._config = config.settings()
+
self.num_retries = num_retries
self._manifest_locator = None
self._manifest_text = None
self._api_response = None
+
+ if sync is None:
+ raise errors.ArgumentError("Must specify sync mode")
+
self._sync = sync
self.lock = threading.RLock()
self.callbacks = []
if self._sync == SYNC_LIVE:
if not self._has_collection_uuid():
raise errors.ArgumentError("Cannot SYNC_LIVE associated with a collection uuid")
- self.events = events.subscribe(arvados.api(), [["object_uuid", "=", self._manifest_locator]], self.on_message)
+ self.events = events.subscribe(arvados.api(apiconfig=self._config),
+ [["object_uuid", "=", self._manifest_locator]],
+ self.on_message)
- @staticmethod
- def create(name, owner_uuid=None, sync=SYNC_EXPLICIT):
- c = Collection(sync=sync)
- c.save_as(name, owner_uuid=owner_uuid, ensure_unique_name=True)
- return c
def root_collection(self):
return self
def sync_mode(self):
return self._sync
- def on_message(self):
- self.update()
+ def on_message(self, event):
+ if event.get("object_uuid") == self._manifest_locator:
+ self.update()
+
+ @staticmethod
+ def create(name, owner_uuid=None, sync=SYNC_EXPLICIT, apiconfig=None):
+ """Create a new empty Collection with associated collection record."""
+ c = Collection(sync=SYNC_EXPLICIT, apiconfig=apiconfig)
+ c.save_new(name, owner_uuid=owner_uuid, ensure_unique_name=True)
+ if sync == SYNC_LIVE:
+ c.events = events.subscribe(arvados.api(apiconfig=self._config), [["object_uuid", "=", c._manifest_locator]], c.on_message)
+ return c
@synchronized
@retry_method
def clone(self, new_parent=None, new_sync=SYNC_READONLY, new_config=None):
if new_config is None:
new_config = self._config
- c = Collection(parent=new_parent, config=new_config, sync=new_sync)
+ c = CollectionRoot(parent=new_parent, apiconfig=new_config, sync=new_sync)
if new_sync == SYNC_READONLY:
c.lock = NoopLock()
c._items = {}
for c in self.callbacks:
c(event, collection, name, item)
+def ReadOnlyCollection(*args, **kwargs):
+ kwargs["sync"] = SYNC_READONLY
+ return CollectionRoot(*args, **kwargs)
+
+def WritableCollection(*args, **kwargs):
+ kwargs["sync"] = SYNC_EXPLICIT
+ return CollectionRoot(*args, **kwargs)
+
+def LiveCollection(*args, **kwargs):
+ kwargs["sync"] = SYNC_LIVE
+ return CollectionRoot(*args, **kwargs)
+
+
class Subcollection(SynchronizedCollectionBase):
"""This is a subdirectory within a collection that doesn't have its own API
server record. It falls under the umbrella of the root collection."""
cfg[var] = val
return cfg
-def flag_is_true(key):
- return get(key, '').lower() in set(['1', 't', 'true', 'y', 'yes'])
+def flag_is_true(key, d=None):
+ if d is None:
+ d = settings()
+ return d.get(key, '').lower() in set(['1', 't', 'true', 'y', 'yes'])
def get(key, default_val=None):
return settings().get(key, default_val)
import api
import keep
import config
+import copy
class SafeApi(object):
"""Threadsafe wrapper for API object. This stores and returns a different api
"""
def __init__(self, apiconfig=None, keep_params={}):
- if not apiconfig:
- apiconfig = config
- self.host = apiconfig.get('ARVADOS_API_HOST')
- self.api_token = apiconfig.get('ARVADOS_API_TOKEN')
- self.insecure = apiconfig.flag_is_true('ARVADOS_API_HOST_INSECURE')
+ if apiconfig is None:
+ apiconfig = config.settings()
+ self.apiconfig = copy.copy(apiconfig)
self.local = threading.local()
self.keep = keep.KeepClient(api_client=self, **keep_params)
def localapi(self):
if 'api' not in self.local.__dict__:
- self.local.api = api.api('v1', False, self.host,
- self.api_token, self.insecure)
+ self.local.api = api.api('v1', False, apiconfig=self.apiconfig)
return self.local.api
def __getattr__(self, name):
# Proxy nonexistent attributes to the thread-local API client.
+ if name == "api_token":
+ return self.apiconfig['ARVADOS_API_TOKEN']
try:
return getattr(self.localapi(), name)
except AttributeError:
api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate",
"manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
- api_client=api, keep=keep, sync=SYNC_EXPLICIT) as c:
+ with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual(writer.size(), 10)
writer.seek(5)
api = ArvadosFileWriterTestCase.MockApi({"name":"test_append",
"manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
- api_client=api, keep=keep, sync=SYNC_EXPLICIT) as c:
+ with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
writer.seek(5, os.SEEK_SET)
self.assertEqual("56789", writer.read(8))
def test_write0(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
- keep=keep, sync=SYNC_EXPLICIT) as c:
+ with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("0123456789", writer.readfrom(0, 13))
writer.seek(0, os.SEEK_SET)
def test_write1(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
- keep=keep, sync=SYNC_EXPLICIT) as c:
+ with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("0123456789", writer.readfrom(0, 13))
writer.seek(3, os.SEEK_SET)
def test_write2(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
- keep=keep, sync=SYNC_EXPLICIT) as c:
+ with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("0123456789", writer.readfrom(0, 13))
writer.seek(7, os.SEEK_SET)
def test_write3(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
- keep=keep, sync=SYNC_EXPLICIT) as c:
+ with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
+ keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("012345678901234", writer.readfrom(0, 15))
writer.seek(7, os.SEEK_SET)
def test_write4(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
- keep=keep, sync=SYNC_EXPLICIT) as c:
+ with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
+ keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("012301230123", writer.readfrom(0, 15))
writer.seek(2, os.SEEK_SET)
api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
"manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with import_manifest('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
- api_client=api, keep=keep, sync=SYNC_EXPLICIT) as c:
+ with arvados.WritableCollection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
+ api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
text = ''.join(["0123456789" for a in xrange(0, 100)])
for b in xrange(0, 100000):
def test_write_rewrite0(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
- with import_manifest('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
- keep=keep, sync=SYNC_EXPLICIT) as c:
+ with arvados.WritableCollection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
+ keep_client=keep) as c:
writer = c.open("count.txt", "r+")
for b in xrange(0, 10):
writer.seek(0, os.SEEK_SET)
def test_write_rewrite1(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
- keep=keep, sync=SYNC_EXPLICIT) as c:
+ with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
+ keep_client=keep) as c:
writer = c.open("count.txt", "r+")
for b in xrange(0, 10):
writer.seek(10, os.SEEK_SET)
def test_write_rewrite2(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
- keep=keep, sync=SYNC_EXPLICIT) as c:
+ with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
+ keep_client=keep) as c:
writer = c.open("count.txt", "r+")
for b in xrange(0, 10):
writer.seek(5, os.SEEK_SET)
api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
"manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with import_manifest('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
- api_client=api, keep=keep, sync=SYNC_EXPLICIT) as c:
+ with arvados.WritableCollection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
+ api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
text = ''.join(["0123456789" for a in xrange(0, 100)])
for b in xrange(0, 100000):
api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with arvados.Collection(api_client=api, keep_client=keep, sync=SYNC_EXPLICIT) as c:
+ with arvados.WritableCollection(api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "w+")
self.assertEqual(writer.size(), 0)
writer.write("01234567")
api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
"manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with arvados.Collection(api_client=api, keep_client=keep, sync=SYNC_EXPLICIT) as c:
+ with arvados.WritableCollection(api_client=api, keep_client=keep) as c:
writer = c.open("foo/bar/count.txt", "w+")
writer.write("01234567")
c.save_new("test_create")
api = ArvadosFileWriterTestCase.MockApi({"name":"test_overwrite",
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
- api_client=api, keep=keep, sync=SYNC_EXPLICIT) as c:
+ with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "w+")
self.assertEqual(writer.size(), 0)
writer.write("01234567")
self.assertEqual(False, c.modified())
def test_file_not_found(self):
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
+ with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
with self.assertRaises(IOError):
writer = c.open("nocount.txt", "r")
def test_cannot_open_directory(self):
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
+ with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
with self.assertRaises(IOError):
writer = c.open(".", "r")
api = ArvadosFileWriterTestCase.MockApi({"name":"test_create_multiple",
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with arvados.Collection(api_client=api, keep_client=keep, sync=SYNC_EXPLICIT) as c:
+ with arvados.WritableCollection(api_client=api, keep_client=keep) as c:
w1 = c.open("count1.txt", "w")
w2 = c.open("count2.txt", "w")
w1.write("01234567")
def test_prefetch(self):
keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"})
- with import_manifest(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep=keep) as c:
+ with arvados.WritableCollection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
r = c.open("count.txt", "r")
self.assertEqual("0123", r.read(4))
self.assertIn("2e9ec317e197819358fbc43afca7d837+8", keep.requests)
self.assertIn("e8dc4081b13434b45189a720b77b6818+8", keep.requests)
def test__eq__(self):
- with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
- with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
+ with arvados.arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
+ with arvados.arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
n += k.size
except ValueError:
pass
- col = arvados.Collection(keep_client=self.keep_client())
+ col = arvados.ReadOnlyCollection(keep_client=self.keep_client())
col._my_block_manager().prefetch_enabled = False
af = ArvadosFile(col,
stream=stream,
from arvados.ranges import Range, LocatorAndRange
from arvados import import_manifest, export_manifest
from arvados.arvfile import SYNC_EXPLICIT
-from arvados.collection import Collection
class TestResumableWriter(arvados.ResumableCollectionWriter):
KEEP_BLOCK_SIZE = 1024 # PUT to Keep every 1K.
. 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
. 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt
"""
- self.assertEqual(". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt\n", arvados.export_manifest(Collection(m1)))
+ self.assertEqual(". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt\n", arvados.export_manifest(arvados.ReadOnlyCollection(m1)))
def test_init_manifest(self):
m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
. 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
. 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt
"""
- self.assertEqual(". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt\n", arvados.export_manifest(arvados.Collection(m1)))
+ self.assertEqual(". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt\n", arvados.export_manifest(arvados.ReadOnlyCollection(m1)))
def test_remove(self):
- c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n', sync=SYNC_EXPLICIT)
+ c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n')
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", export_manifest(c))
self.assertIn("count1.txt", c)
c.remove("count1.txt")
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(c))
def test_remove_in_subdir(self):
- c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT)
+ c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
c.remove("foo/count2.txt")
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
def test_remove_empty_subdir(self):
- c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT)
+ c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
c.remove("foo/count2.txt")
c.remove("foo")
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
def test_remove_nonempty_subdir(self):
- c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT)
+ c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
with self.assertRaises(IOError):
c.remove("foo")
c.remove("foo", rm_r=True)
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
def test_copy_to_dir1(self):
- c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
+ c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
c.copy("count1.txt", "foo/count2.txt")
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(c))
def test_copy_to_dir2(self):
- c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
+ c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
c.copy("count1.txt", "foo")
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
def test_copy_to_dir2(self):
- c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
+ c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
c.copy("count1.txt", "foo/")
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
def test_copy_file(self):
- c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
+ c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
c.copy("count1.txt", "count2.txt")
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", c.manifest_text())
def test_clone(self):
- c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT)
+ c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
cl = c.clone()
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(cl))
def test_diff1(self):
- c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
- c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = arvados.WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"]),
('add', './count1.txt', c1["count1.txt"])])
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff2(self):
- c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
- c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [])
d = c1.diff(c2)
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff3(self):
- c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
- c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
+ c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = arvados.WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('mod', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
d = c1.diff(c2)
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff4(self):
- c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
- c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt 10:20:count2.txt\n')
+ c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt 10:20:count2.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"])])
d = c1.diff(c2)
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff5(self):
- c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
- c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './foo', c2["foo"])])
d = c1.diff(c2)
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff6(self):
- c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n', sync=SYNC_EXPLICIT)
- c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:3:count3.txt\n')
+ c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c2 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:3:count3.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './foo/count3.txt', c2.find("foo/count3.txt")),
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff7(self):
- c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n', sync=SYNC_EXPLICIT)
- c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:3:foo\n')
+ c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c2 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:3:foo\n')
d = c2.diff(c1)
self.assertEqual(d, [('mod', './foo', c2["foo"], c1["foo"])])
d = c1.diff(c2)
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_conflict1(self):
- c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
- c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = arvados.WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
d = c1.diff(c2)
self.assertEqual(d, [('del', './count1.txt', c1["count1.txt"]),
('add', './count2.txt', c2["count2.txt"])])
self.assertEqual(c1.manifest_text(), ". 95ebc3c7b3b9f1d2c40fec14415d3cb8+5 5348b82a029fd9e971a811ce1f71360b+43 0:5:count1.txt 5:10:count2.txt\n")
def test_conflict2(self):
- c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt', sync=SYNC_EXPLICIT)
- c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
+ c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
+ c2 = arvados.WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
d = c1.diff(c2)
self.assertEqual(d, [('mod', './count1.txt', c1["count1.txt"], c2["count1.txt"])])
with c1.open("count1.txt", "w") as f:
c1.manifest_text()))
def test_conflict3(self):
- c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT)
- c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
+ c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
+ c2 = arvados.WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
d = c1.diff(c2)
self.assertEqual(d, [('del', './count2.txt', c1["count2.txt"]),
('add', './count1.txt', c2["count1.txt"])])
c1.manifest_text()))
def test_conflict4(self):
- c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt', sync=SYNC_EXPLICIT)
- c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
+ c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
+ c2 = arvados.WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
d = c1.diff(c2)
self.assertEqual(d, [('mod', './count1.txt', c1["count1.txt"], c2["count1.txt"])])
c1.remove("count1.txt")
c1.manifest_text()))
def test_notify1(self):
- c1 = Collection(sync=SYNC_EXPLICIT)
+ c1 = arvados.WritableCollection(sync=SYNC_EXPLICIT)
events = []
c1.subscribe(lambda event, collection, name, item: events.append((event, collection, name, item)))
c1.find("")
projects / arvados.git / commitdiff