so it wouldn't be awkwardly shadowed by the 'api' method.
import time
import threading
-from api import *
+from apisetup import api, http_cache
from collection import CollectionReader, CollectionWriter, ResumableCollectionWriter
from keep import *
from stream import *
"""
+ stream_name = stream_name.replace(' ', '\\040')
stream_tokens = [stream_name]
sortedfiles = list(stream.keys())
sortedfiles.sort()
def api(version=None, cache=True, host=None, token=None, insecure=False, **kwargs):
"""Return an apiclient Resources object for an Arvados instance.
- Arguments:
- * version: A string naming the version of the Arvados API to use (for
+ :version:
+ A string naming the version of the Arvados API to use (for
example, 'v1').
- * cache: Use a cache (~/.cache/arvados/discovery) for the discovery
+
+ :cache:
+ Use a cache (~/.cache/arvados/discovery) for the discovery
document.
- * host: The Arvados API server host (and optional :port) to connect to.
- * token: The authentication token to send with each API call.
- * insecure: If True, ignore SSL certificate validation errors.
+
+ :host:
+ The Arvados API server host (and optional :port) to connect to.
+
+ :token:
+ The authentication token to send with each API call.
+
+ :insecure:
+ If True, ignore SSL certificate validation errors.
Additional keyword arguments will be passed directly to
`apiclient_discovery.build` if a new Resource object is created.
import os
import zlib
import bz2
-from ._ranges import locators_and_ranges, replace_range
+from ._ranges import locators_and_ranges, replace_range, Range
from arvados.retry import retry_method
import config
import hashlib
def decompressed_name(self):
return re.sub('\.(bz2|gz)$', '', self.name)
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
def seek(self, pos, whence=os.SEEK_CUR):
if whence == os.SEEK_CUR:
pos += self._filepos
def tell(self):
return self._filepos
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def readall(self, size=2**20, num_retries=None):
while True:
break
yield data
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def readline(self, size=float('inf'), num_retries=None):
cache_pos, cache_data = self._readline_cache
self._readline_cache = (self.tell(), data[nextline_index:])
return data[:nextline_index]
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def decompress(self, decompress, size, num_retries=None):
for segment in self.readall(size, num_retries):
if data:
yield data
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def readall_decompressed(self, size=2**20, num_retries=None):
self.seek(0)
else:
return self.readall(size, num_retries=num_retries)
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def readlines(self, sizehint=float('inf'), num_retries=None):
data = []
n = self.segments[-1]
return n.range_start + n.range_size
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def read(self, size, num_retries=None):
"""Read up to 'size' bytes from the stream, starting at the current file position"""
self._filepos += len(data)
return data
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def readfrom(self, start, size, num_retries=None):
"""Read up to 'size' bytes from the stream, starting at 'start'"""
(self._state == _BufferBlock.PENDING and nextstate == _BufferBlock.COMMITTED)):
self._state = nextstate
if self._state == _BufferBlock.COMMITTED:
- bufferblock._locator = loc
- bufferblock.buffer_view = None
- bufferblock.buffer_block = None
+ self._locator = loc
+ self.buffer_view = None
+ self.buffer_block = None
else:
raise AssertionError("Invalid state change from %s to %s" % (self.state, state))
@synchronized
def manifest_text(self, stream_name=".", portable_locators=False, normalize=False):
+ buf = ""
item = self
filestream = []
for segment in item.segments:
def size(self):
return self.arvadosfile.size()
- @FileLikeObjectBase._before_close
+ def stream_name(self):
+ return self.arvadosfile.parent.stream_name()
+
+ @_FileLikeObjectBase._before_close
@retry_method
def read(self, size, num_retries=None):
"""Read up to `size` bytes from the stream, starting at the current file position."""
self._filepos += len(data)
return data
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def readfrom(self, offset, size, num_retries=None):
"""Read up to `size` bytes from the stream, starting at the current file position."""
def __init__(self, arvadosfile, name, mode, num_retries=None):
super(ArvadosFileWriter, self).__init__(arvadosfile, name, mode, num_retries=num_retries)
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def write(self, data, num_retries=None):
if self.mode[0] == "a":
self.arvadosfile.writeto(self._filepos, data, num_retries)
self._filepos += len(data)
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def writelines(self, seq, num_retries=None):
for s in seq:
from collections import deque
from stat import *
-from .arvfile import split, FileLikeObjectBase, ArvadosFile, ArvadosFileWriter, ArvadosFileReader, _BlockManager, synchronized, must_be_writable, SYNC_READONLY, SYNC_EXPLICIT, NoopLock
+from .arvfile import split, _FileLikeObjectBase, ArvadosFile, ArvadosFileWriter, ArvadosFileReader, _BlockManager, synchronized, must_be_writable, SYNC_READONLY, SYNC_EXPLICIT, NoopLock
from keep import *
-from .stream import StreamReader, normalize_stream, locator_block_size
-from .ranges import Range, LocatorAndRange
+from .stream import StreamReader, normalize_stream
+from ._ranges import Range, LocatorAndRange
from .safeapi import ThreadSafeApiCache
import config
import errors
return ''.join(clean)
-class _WriterFile(FileLikeObjectBase):
+class _WriterFile(_FileLikeObjectBase):
def __init__(self, coll_writer, name):
super(_WriterFile, self).__init__(name, 'wb')
self.dest = coll_writer
super(_WriterFile, self).close()
self.dest.finish_current_file()
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
def write(self, data):
self.dest.write(data)
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
def writelines(self, seq):
for data in seq:
self.write(data)
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
def flush(self):
self.dest.flush_data()
def _my_block_manager(self):
raise NotImplementedError()
- def _populate(self):
- raise NotImplementedError()
-
def sync_mode(self):
raise NotImplementedError()
def notify(self, event, collection, name, item):
raise NotImplementedError()
+ def stream_name(self):
+ raise NotImplementedError()
+
@must_be_writable
@synchronized
def find_or_create(self, path, create_type):
if isinstance(item, SynchronizedCollectionBase):
return item.find_or_create("/".join(pathcomponents), create_type)
else:
- raise errors.ArgumentError("Interior path components must be subcollection")
+ raise IOError((errno.ENOTDIR, "Interior path components must be subcollection"))
else:
return self
if isinstance(item, SynchronizedCollectionBase):
return item.find("/".join(pathcomponents))
else:
- raise errors.ArgumentError("Interior path components must be subcollection")
+ raise IOError((errno.ENOTDIR, "Interior path components must be subcollection"))
else:
return self
"""
return self.find_or_create(path, COLLECTION)
- def open(self, path, mode):
+ def open(self, path, mode="r"):
"""Open a file-like object for access.
:path:
"""
mode = mode.replace("b", "")
if len(mode) == 0 or mode[0] not in ("r", "w", "a"):
- raise ArgumentError("Bad mode '%s'" % mode)
+ raise errors.ArgumentError("Bad mode '%s'" % mode)
create = (mode != "r")
if create and self.sync_mode() == SYNC_READONLY:
if mode[0] == "w":
arvfile.truncate(0)
+ name = os.path.basename(path)
+
if mode == "r":
- return ArvadosFileReader(arvfile, path, mode, num_retries=self.num_retries)
+ return ArvadosFileReader(arvfile, name, mode, num_retries=self.num_retries)
else:
- return ArvadosFileWriter(arvfile, path, mode, num_retries=self.num_retries)
+ return ArvadosFileWriter(arvfile, name, mode, num_retries=self.num_retries)
@synchronized
def modified(self):
"""
- portable_locators = strip
if self.modified() or self._manifest_text is None or normalize:
item = self
stream = {}
+ buf = ""
sorted_keys = sorted(item.keys())
for filename in [s for s in sorted_keys if isinstance(item[s], ArvadosFile)]:
# Create a stream per file `k`
loc = segment.locator
if arvfile.parent._my_block_manager().is_bufferblock(loc):
loc = arvfile.parent._my_block_manager().get_bufferblock(loc).locator()
- if portable_locators:
+ if strip:
loc = KeepLocator(loc).stripped()
- filestream.append(LocatorAndRange(loc, locator_block_size(loc),
+ filestream.append(LocatorAndRange(loc, KeepLocator(loc).size,
segment.segment_offset, segment.range_size))
stream[filename] = filestream
if stream:
buf += ' '.join(normalize_stream(stream_name, stream))
buf += "\n"
for dirname in [s for s in sorted_keys if isinstance(item[s], SynchronizedCollectionBase)]:
- buf += item[dirname].manifest_text(stream_name=os.path.join(stream_name, dirname), portable_locators=portable_locators)
+ buf += item[dirname].manifest_text(stream_name=os.path.join(stream_name, dirname), strip=strip)
return buf
else:
- if portable_locators:
+ if strip:
return self.stripped_manifest()
else:
return self._manifest_text
"""
changes = []
if holding_collection is None:
- holding_collection = Collection(api_client=self._my_api(), keep_client=self._my_keep(), sync=SYNC_EXPLICIT)
+ holding_collection = Collection(api_client=self._my_api(), keep_client=self._my_keep())
for k in self:
if k not in end_collection:
changes.append((DEL, os.path.join(prefix, k), self[k].clone(holding_collection)))
else:
self._config = config.settings()
- self.num_retries = num_retries if num_retries is not None else 2
+ self.num_retries = num_retries if num_retries is not None else 0
self._manifest_locator = None
self._manifest_text = None
self._api_response = None
self._sync = SYNC_EXPLICIT
- if not self.lock:
- self.lock = threading.RLock()
+ self.lock = threading.RLock()
self.callbacks = []
self.events = None
def root_collection(self):
return self
+ def stream_name(self):
+ return "."
+
def sync_mode(self):
return self._sync
if self._manifest_locator is None:
raise errors.ArgumentError("`other` is None but collection does not have a manifest_locator uuid")
response = self._my_api().collections().get(uuid=self._manifest_locator).execute(num_retries=num_retries)
- other = import_manifest(response["manifest_text"])
- baseline = import_manifest(self._manifest_text)
+ other = CollectionReader(response["manifest_text"])
+ baseline = CollectionReader(self._manifest_text)
self.apply(baseline.diff(other))
@synchronized
if self._api_client is None:
self._my_api()
else:
- self._keep_client = KeepClient(api=self._api_client)
+ self._keep_client = KeepClient(api_client=self._api_client)
return self._keep_client
@synchronized
error_via_keep))
# populate
self._baseline_manifest = self._manifest_text
- import_manifest(self._manifest_text, self)
+ self._import_manifest(self._manifest_text)
def _has_collection_uuid(self):
self._block_manager.stop_threads()
@synchronized
- def clone(self, new_parent=None, new_sync=SYNC_READONLY, new_config=None):
+ def clone(self, new_parent=None, readonly=False, new_config=None):
if new_config is None:
new_config = self._config
- newcollection = Collection(parent=new_parent, apiconfig=new_config, sync=SYNC_EXPLICIT)
- if new_sync == SYNC_READONLY:
- newcollection.lock = NoopLock()
+ if readonly:
+ newcollection = CollectionReader(parent=new_parent, apiconfig=new_config)
+ else:
+ newcollection = Collection(parent=new_parent, apiconfig=new_config)
+
+ newcollection._sync = None
self._cloneinto(newcollection)
- newcollection._sync = new_sync
+ newcollection._sync = SYNC_READONLY if readonly else SYNC_EXPLICIT
return newcollection
@synchronized
def __init__(self, parent):
super(Subcollection, self).__init__(parent)
self.lock = self.root_collection().lock
+ self._manifest_text = None
def root_collection(self):
return self.parent.root_collection()
def _my_block_manager(self):
return self.root_collection()._my_block_manager()
- def _populate(self):
- self.root_collection()._populate()
-
def notify(self, event, collection, name, item):
return self.root_collection().notify(event, collection, name, item)
+ def stream_name(self):
+ for k, v in self.parent.items():
+ if v is self:
+ return os.path.join(self.parent.stream_name(), k)
+ return '.'
+
@synchronized
def clone(self, new_parent):
c = Subcollection(new_parent)
if not args and not kwargs.get("manifest_locator_or_text"):
raise errors.ArgumentError("Must provide manifest locator or text to initialize ReadOnlyCollection")
+ super(CollectionReader, self).__init__(*args, **kwargs)
+
# Forego any locking since it should never change once initialized.
self.lock = NoopLock()
-
- super(ReadOnlyCollection, self).__init__(*args, **kwargs)
-
self._sync = SYNC_READONLY
- self._streams = [sline.split()
- for sline in self._manifest_text.split("\n")
- if sline]
+ # Backwards compatability with old CollectionReader
+ # all_streams() and all_files()
+ if self._manifest_text:
+ self._streams = [sline.split()
+ for sline in self._manifest_text.split("\n")
+ if sline]
+ else:
+ self._streams = []
def normalize(self):
# Rearrange streams
import threading
-import api
+import apisetup
import keep
import config
import copy
def localapi(self):
if 'api' not in self.local.__dict__:
- self.local.api = api.api_from_config('v1', apiconfig=self.apiconfig)
+ self.local.api = apisetup.api_from_config('v1', apiconfig=self.apiconfig)
return self.local.api
def __getattr__(self, name):
import functools
import copy
-from ._ranges import locators_and_ranges
+from ._ranges import locators_and_ranges, Range
from .arvfile import StreamFileReader
from arvados.retry import retry_method
from keep import *
# Add the Python SDK source to the library path.
sys.path.insert(1, os.path.dirname(MY_DIRNAME))
-import arvados.api
+import arvados
import arvados.config
ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
import hashlib
import arvados
-from arvados import Range, KeepLocator
-from arvados.collection import import_manifest, export_manifest, ReadOnlyCollection, WritableCollection
+from arvados._ranges import Range
+from arvados.keep import KeepLocator
+from arvados.collection import Collection, CollectionReader
from arvados.arvfile import ArvadosFile, ArvadosFileReader, SYNC_READONLY, SYNC_EXPLICIT
import arvados_testutil as tutil
"manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0",
"manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual(writer.size(), 10)
"manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0",
"manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual(writer.size(), 10)
def test_append(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
writer = c.open("count.txt", "a+")
self.assertEqual(writer.read(20), "0123456789")
writer.seek(0, os.SEEK_SET)
writer.write("world")
self.assertEqual(writer.read(20), "0123456789helloworld")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.manifest_text())
def test_write_at_beginning(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("0123456789", writer.readfrom(0, 13))
writer.write("foo")
self.assertEqual(writer.size(), 10)
self.assertEqual("foo3456789", writer.readfrom(0, 13))
- self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", export_manifest(c))
+ self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", c.manifest_text())
def test_write_in_middle(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("0123456789", writer.readfrom(0, 13))
writer.write("foo")
self.assertEqual(writer.size(), 10)
self.assertEqual("012foo6789", writer.readfrom(0, 13))
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", c.manifest_text())
def test_write_at_end(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("0123456789", writer.readfrom(0, 13))
writer.write("foo")
self.assertEqual(writer.size(), 10)
self.assertEqual("0123456foo", writer.readfrom(0, 13))
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", c.manifest_text())
def test_write_across_segment_boundary(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("012345678901234", writer.readfrom(0, 15))
writer.write("foobar")
self.assertEqual(writer.size(), 20)
self.assertEqual("0123456foobar34", writer.readfrom(0, 15))
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", c.manifest_text())
def test_write_across_several_segments(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("012301230123", writer.readfrom(0, 15))
writer.write("abcdefg")
self.assertEqual(writer.size(), 12)
self.assertEqual("01abcdefg123", writer.readfrom(0, 15))
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", c.manifest_text())
def test_write_large(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
"manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0",
"manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n"})
- with WritableCollection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
+ with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
text = ''.join(["0123456789" for a in xrange(0, 100)])
def test_rewrite_on_empty_file(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
- with WritableCollection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
+ with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
for b in xrange(0, 10):
writer.arvadosfile._repack_writes()
self.assertEqual(writer.size(), 10)
self.assertEqual("0123456789", writer.readfrom(0, 20))
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.manifest_text())
def test_rewrite_append_existing_file(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
for b in xrange(0, 10):
writer.arvadosfile._repack_writes()
self.assertEqual(writer.size(), 20)
self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.manifest_text())
def test_rewrite_over_existing_file(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
for b in xrange(0, 10):
writer.arvadosfile._repack_writes()
self.assertEqual(writer.size(), 15)
self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.manifest_text())
def test_write_large_rewrite(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
"manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0",
"manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n"})
- with WritableCollection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
+ with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
text = ''.join(["0123456789" for a in xrange(0, 100)])
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0",
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
- with WritableCollection(api_client=api, keep_client=keep) as c:
+ with Collection(api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "w+")
self.assertEqual(writer.size(), 0)
writer.write("01234567")
"manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0",
"manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
- with WritableCollection(api_client=api, keep_client=keep) as c:
+ with Collection(api_client=api, keep_client=keep) as c:
self.assertIsNone(c.api_response())
writer = c.open("foo/bar/count.txt", "w+")
writer.write("01234567")
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0",
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "w+")
self.assertEqual(writer.size(), 0)
self.assertEqual(False, c.modified())
def test_file_not_found(self):
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
with self.assertRaises(IOError):
writer = c.open("nocount.txt", "r")
def test_cannot_open_directory(self):
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
with self.assertRaises(IOError):
writer = c.open(".", "r")
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0",
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n"})
- with WritableCollection(api_client=api, keep_client=keep) as c:
+ with Collection(api_client=api, keep_client=keep) as c:
w1 = c.open("count1.txt", "w")
w2 = c.open("count2.txt", "w")
w1.write("01234567")
def test_prefetch(self):
keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"})
- with WritableCollection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
+ with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
r = c.open("count.txt", "r")
self.assertEqual("0123", r.read(4))
self.assertIn("2e9ec317e197819358fbc43afca7d837+8", keep.requests)
self.assertIn("e8dc4081b13434b45189a720b77b6818+8", keep.requests)
def test__eq__from_manifest(self):
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
def test__eq__from_writes(self):
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
- with WritableCollection() as c2:
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
+ with Collection() as c2:
with c2.open("count1.txt", "w") as f:
f.write("0123456789")
self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
def test__ne__(self):
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
- with WritableCollection() as c2:
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
+ with Collection() as c2:
with c2.open("count1.txt", "w") as f:
f.write("1234567890")
n += k.size
except ValueError:
pass
- col = ReadOnlyCollection(keep_client=self.keep_client())
+ col = Collection(keep_client=self.keep_client())
col._my_block_manager().prefetch_enabled = False
af = ArvadosFile(col,
stream=stream,
class BlockManagerTest(unittest.TestCase):
def test_bufferblock_append(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
- blockmanager = arvados.arvfile.BlockManager(keep)
+ blockmanager = arvados.arvfile._BlockManager(keep)
bufferblock = blockmanager.alloc_bufferblock()
bufferblock.append("foo")
self.assertEqual(bufferblock.buffer_view[0:6], "foobar")
self.assertEqual(bufferblock.locator(), "3858f62230ac3c915f300c664312c63f+6")
- bufferblock.set_state(arvados.arvfile.BufferBlock.PENDING)
+ bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
with self.assertRaises(arvados.errors.AssertionError):
bufferblock.append("bar")
def test_bufferblock_dup(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
- blockmanager = arvados.arvfile.BlockManager(keep)
+ blockmanager = arvados.arvfile._BlockManager(keep)
bufferblock = blockmanager.alloc_bufferblock()
bufferblock.append("foo")
self.assertEqual(bufferblock.size(), 3)
self.assertEqual(bufferblock.buffer_view[0:3], "foo")
self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
- bufferblock.set_state(arvados.arvfile.BufferBlock.PENDING)
+ bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
bufferblock2 = blockmanager.dup_block(bufferblock, None)
self.assertNotEqual(bufferblock.blockid, bufferblock2.blockid)
def test_bufferblock_get(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- blockmanager = arvados.arvfile.BlockManager(keep)
+ blockmanager = arvados.arvfile._BlockManager(keep)
bufferblock = blockmanager.alloc_bufferblock()
bufferblock.append("foo")
def test_bufferblock_commit(self):
mockkeep = mock.MagicMock()
- blockmanager = arvados.arvfile.BlockManager(mockkeep)
+ blockmanager = arvados.arvfile._BlockManager(mockkeep)
bufferblock = blockmanager.alloc_bufferblock()
bufferblock.append("foo")
blockmanager.commit_all()
self.assertTrue(mockkeep.put.called)
- self.assertEqual(bufferblock.state(), arvados.arvfile.BufferBlock.COMMITTED)
+ self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
self.assertIsNone(bufferblock.buffer_view)
def test_bufferblock_commit_with_error(self):
mockkeep = mock.MagicMock()
mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail")
- blockmanager = arvados.arvfile.BlockManager(mockkeep)
+ blockmanager = arvados.arvfile._BlockManager(mockkeep)
bufferblock = blockmanager.alloc_bufferblock()
bufferblock.append("foo")
with self.assertRaises(arvados.errors.KeepWriteError) as err:
blockmanager.commit_all()
self.assertEquals(str(err.exception), "Error writing some blocks: acbd18db4cc2f85cedef654fccc4a4d8+3 raised KeepWriteError (fail)")
- self.assertEqual(bufferblock.state(), arvados.arvfile.BufferBlock.PENDING)
+ self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.PENDING)
import unittest
import run_test_server
-import arvados_testutil as tutil
-from arvados.ranges import Range, LocatorAndRange
-from arvados.collection import import_manifest, export_manifest, ReadOnlyCollection, WritableCollection
+from arvados._ranges import Range, LocatorAndRange
+from arvados.collection import Collection, CollectionReader
from arvados.arvfile import SYNC_EXPLICIT
+import arvados_testutil as tutil
class TestResumableWriter(arvados.ResumableCollectionWriter):
KEEP_BLOCK_SIZE = 1024 # PUT to Keep every 1K.
def test_uuid_init_failure_raises_api_error(self):
client = self.api_client_mock(500)
- reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
with self.assertRaises(arvados.errors.ApiError):
- reader.manifest_text()
+ reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
def test_locator_init(self):
client = self.api_client_mock(200)
def test_uuid_init_no_fallback_to_keep(self):
# Do not look up a collection UUID in Keep.
client = self.api_client_mock(404)
- reader = arvados.CollectionReader(self.DEFAULT_UUID,
- api_client=client)
with tutil.mock_get_responses(self.DEFAULT_MANIFEST, 200):
with self.assertRaises(arvados.errors.ApiError):
- reader.manifest_text()
+ reader = arvados.CollectionReader(self.DEFAULT_UUID,
+ api_client=client)
def test_try_keep_first_if_permission_hint(self):
# To verify that CollectionReader tries Keep first here, we
cfile = reader.open('./foo')
self.check_open_file(cfile, '.', 'foo', 3)
- def test_open_collection_file_two_arguments(self):
- client = self.api_client_mock(200)
- reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
- cfile = reader.open('.', 'foo')
- self.check_open_file(cfile, '.', 'foo', 3)
-
def test_open_deep_file(self):
coll_name = 'collection_with_files_in_subdir'
client = self.api_client_mock(200)
def test_open_nonexistent_stream(self):
client = self.api_client_mock(200)
reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
- self.assertRaises(ValueError, reader.open, './nonexistent', 'foo')
+ self.assertRaises(IOError, reader.open, './nonexistent/foo')
def test_open_nonexistent_file(self):
client = self.api_client_mock(200)
reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
- self.assertRaises(ValueError, reader.open, '.', 'nonexistent')
+ self.assertRaises(IOError, reader.open, 'nonexistent')
@tutil.skip_sleep
class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
- def test_import_export_manifest(self):
- m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
-. 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
-. 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt
-"""
- self.assertEqual(". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt\n", export_manifest(import_manifest(m1)))
def test_init_manifest(self):
m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
. 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
. 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt
"""
- self.assertEqual(". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt\n", export_manifest(ReadOnlyCollection(m1)))
+ self.assertEqual(m1, CollectionReader(m1).manifest_text(normalize=False))
+ self.assertEqual(". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt\n", CollectionReader(m1).manifest_text(normalize=True))
def test_remove(self):
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n')
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", export_manifest(c))
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n')
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", c.manifest_text())
self.assertIn("count1.txt", c)
c.remove("count1.txt")
self.assertNotIn("count1.txt", c)
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", c.manifest_text())
def test_remove_in_subdir(self):
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
c.remove("foo/count2.txt")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", c.manifest_text())
def test_remove_empty_subdir(self):
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
c.remove("foo/count2.txt")
c.remove("foo")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", c.manifest_text())
def test_remove_nonempty_subdir(self):
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
with self.assertRaises(IOError):
c.remove("foo")
c.remove("foo", recursive=True)
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", c.manifest_text())
def test_copy_to_file_in_dir(self):
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
c.copy("count1.txt", "foo/count2.txt")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", c.manifest_text())
def test_copy_file(self):
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
c.copy("count1.txt", "count2.txt")
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", c.manifest_text())
def test_copy_to_existing_dir(self):
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
c.copy("count1.txt", "foo")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", c.manifest_text())
def test_copy_to_new_dir(self):
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
c.copy("count1.txt", "foo/")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", c.manifest_text())
def test_clone(self):
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
cl = c.clone()
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(cl))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", cl.manifest_text())
def test_diff_del_add(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
- c2 = WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"]),
('add', './count1.txt', c1["count1.txt"])])
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff_same(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
- c2 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [])
d = c1.diff(c2)
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff_mod(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
- c2 = WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('mod', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
d = c1.diff(c2)
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff_add(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
- c2 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt 10:20:count2.txt\n')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt 10:20:count2.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"])])
d = c1.diff(c2)
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff_add_in_subcollection(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
- c2 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './foo', c2["foo"])])
d = c1.diff(c2)
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff_del_add_in_subcollection(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
- c2 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:3:count3.txt\n')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:3:count3.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './foo/count3.txt', c2.find("foo/count3.txt")),
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff_mod_in_subcollection(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
- c2 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:3:foo\n')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:3:foo\n')
d = c2.diff(c1)
self.assertEqual(d, [('mod', './foo', c2["foo"], c1["foo"])])
d = c1.diff(c2)
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_conflict_keep_local_change(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
- c2 = WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
d = c1.diff(c2)
self.assertEqual(d, [('del', './count1.txt', c1["count1.txt"]),
('add', './count2.txt', c2["count2.txt"])])
self.assertEqual(c1.manifest_text(), ". 95ebc3c7b3b9f1d2c40fec14415d3cb8+5 5348b82a029fd9e971a811ce1f71360b+43 0:5:count1.txt 5:10:count2.txt\n")
def test_conflict_mod(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
- c2 = WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
+ c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
d = c1.diff(c2)
self.assertEqual(d, [('mod', './count1.txt', c1["count1.txt"], c2["count1.txt"])])
with c1.open("count1.txt", "w") as f:
c1.manifest_text()))
def test_conflict_add(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
- c2 = WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
+ c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
d = c1.diff(c2)
self.assertEqual(d, [('del', './count2.txt', c1["count2.txt"]),
('add', './count1.txt', c2["count1.txt"])])
c1.manifest_text()))
def test_conflict_del(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
- c2 = WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
+ c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
d = c1.diff(c2)
self.assertEqual(d, [('mod', './count1.txt', c1["count1.txt"], c2["count1.txt"])])
c1.remove("count1.txt")
c1.manifest_text()))
def test_notify(self):
- c1 = WritableCollection()
+ c1 = Collection()
events = []
c1.subscribe(lambda event, collection, name, item: events.append((event, collection, name, item)))
f = c1.open("foo.txt", "w")
self.assertEqual(events[0], (arvados.collection.ADD, c1, "foo.txt", f.arvadosfile))
def test_open_w(self):
- c1 = WritableCollection(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n")
+ c1 = Collection(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n")
self.assertEqual(c1["count1.txt"].size(), 10)
c1.open("count1.txt", "w").close()
self.assertEqual(c1["count1.txt"].size(), 0)
MAIN_SERVER = {}
KEEP_SERVER = {}
- def test_create_and_save(self):
- c = arvados.collection.createWritableCollection("hello world")
+ def create_count_txt(self):
+ # Create an empty collection, save it to the API server, then write a
+ # file, but don't save it.
+
+ c = Collection()
+ c.save_new("CollectionCreateUpdateTest", ensure_unique_name=True)
self.assertEquals(c.portable_data_hash(), "d41d8cd98f00b204e9800998ecf8427e+0")
self.assertEquals(c.api_response()["portable_data_hash"], "d41d8cd98f00b204e9800998ecf8427e+0" )
self.assertEquals(c.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
- c.save()
+ return c
+ def test_create_and_save(self):
+ c = self.create_count_txt()
+ c.save()
self.assertTrue(re.match(r"^\. 781e5e245d69b566979b86e28d23f2c7\+10\+A[a-f0-9]{40}@[a-f0-9]{8} 0:10:count.txt$",
c.manifest_text()))
def test_create_and_save_new(self):
- c = arvados.collection.createWritableCollection("hello world")
- self.assertEquals(c.portable_data_hash(), "d41d8cd98f00b204e9800998ecf8427e+0")
- self.assertEquals(c.api_response()["portable_data_hash"], "d41d8cd98f00b204e9800998ecf8427e+0" )
-
- with c.open("count.txt", "w") as f:
- f.write("0123456789")
-
- self.assertEquals(c.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
-
+ c = self.create_count_txt()
c.save_new()
-
self.assertTrue(re.match(r"^\. 781e5e245d69b566979b86e28d23f2c7\+10\+A[a-f0-9]{40}@[a-f0-9]{8} 0:10:count.txt$",
c.manifest_text()))
def test_create_diff_apply(self):
- c1 = arvados.collection.createWritableCollection("hello world")
- self.assertEquals(c1.portable_data_hash(), "d41d8cd98f00b204e9800998ecf8427e+0")
- self.assertEquals(c1.api_response()["portable_data_hash"], "d41d8cd98f00b204e9800998ecf8427e+0" )
- with c1.open("count.txt", "w") as f:
- f.write("0123456789")
-
- self.assertEquals(c1.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
-
+ c1 = self.create_count_txt()
c1.save()
- c2 = WritableCollection(c1._manifest_locator)
+ c2 = Collection(c1._manifest_locator)
with c2.open("count.txt", "w") as f:
f.write("abcdefg")
self.assertEqual(c1.portable_data_hash(), c2.portable_data_hash())
def test_diff_apply_with_token(self):
- baseline = ReadOnlyCollection(". 781e5e245d69b566979b86e28d23f2c7+10+A715fd31f8111894f717eb1003c1b0216799dd9ec@54f5dd1a 0:10:count.txt\n")
- c = WritableCollection(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
- other = ReadOnlyCollection(". 7ac66c0f148de9519b8bd264312c4d64+7+A715fd31f8111894f717eb1003c1b0216799dd9ec@54f5dd1a 0:7:count.txt\n")
+ baseline = CollectionReader(". 781e5e245d69b566979b86e28d23f2c7+10+A715fd31f8111894f717eb1003c1b0216799dd9ec@54f5dd1a 0:10:count.txt\n")
+ c = Collection(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
+ other = CollectionReader(". 7ac66c0f148de9519b8bd264312c4d64+7+A715fd31f8111894f717eb1003c1b0216799dd9ec@54f5dd1a 0:7:count.txt\n")
diff = baseline.diff(other)
self.assertEqual(diff, [('mod', u'./count.txt', c["count.txt"], other["count.txt"])])
def test_create_and_update(self):
- c1 = arvados.collection.createWritableCollection("hello world")
- self.assertEquals(c1.portable_data_hash(), "d41d8cd98f00b204e9800998ecf8427e+0")
- self.assertEquals(c1.api_response()["portable_data_hash"], "d41d8cd98f00b204e9800998ecf8427e+0" )
- with c1.open("count.txt", "w") as f:
- f.write("0123456789")
-
- self.assertEquals(c1.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
-
+ c1 = self.create_count_txt()
c1.save()
- c2 = arvados.collection.WritableCollection(c1._manifest_locator)
+ c2 = arvados.collection.Collection(c1._manifest_locator)
with c2.open("count.txt", "w") as f:
f.write("abcdefg")
def test_create_and_update_with_conflict(self):
- c1 = arvados.collection.createWritableCollection("hello world")
- self.assertEquals(c1.portable_data_hash(), "d41d8cd98f00b204e9800998ecf8427e+0")
- self.assertEquals(c1.api_response()["portable_data_hash"], "d41d8cd98f00b204e9800998ecf8427e+0" )
- with c1.open("count.txt", "w") as f:
- f.write("0123456789")
-
- self.assertEquals(c1.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
-
+ c1 = self.create_count_txt()
c1.save()
+
with c1.open("count.txt", "w") as f:
f.write("XYZ")
- c2 = arvados.collection.WritableCollection(c1._manifest_locator)
+ c2 = arvados.collection.Collection(c1._manifest_locator)
with c2.open("count.txt", "w") as f:
f.write("abcdefg")
class TestSDK(unittest.TestCase):
- @mock.patch('arvados.api')
+ @mock.patch('arvados.apisetup.api_from_config')
@mock.patch('arvados.current_task')
@mock.patch('arvados.current_job')
def test_one_task_per_input_file_normalize(self, mock_job, mock_task, mock_api):
# it should now create only one job task and not three.
arvados.job_setup.one_task_per_input_file(and_end_task=False)
mock_api('v1').job_tasks().create().execute.assert_called_once_with()
-
import hashlib
import arvados
-from arvados import StreamReader, StreamFileReader, Range
+from arvados import StreamReader, StreamFileReader
+from arvados._ranges import Range
import arvados_testutil as tutil
import run_test_server
projects / arvados.git / commitdiff