from __future__ import print_function
from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import object
import gflags
-import httplib
+import http.client
import httplib2
import logging
import os
def __init__(self, parameters=dict(), runtime_constraints=dict()):
print("init jobtask %s %s" % (parameters, runtime_constraints))
-class job_setup:
+class job_setup(object):
@staticmethod
def one_task_per_input_file(if_sequence=0, and_end_task=True, input_as_path=False, api_client=None):
if if_sequence != current_task()['sequence']:
+from __future__ import division
+from past.utils import old_div
+from builtins import object
import logging
_logger = logging.getLogger('arvados.ranges')
hi = len(data_locators)
lo = 0
- i = int((hi + lo) / 2)
+ i = int(old_div((hi + lo), 2))
block_size = data_locators[i].range_size
block_start = data_locators[i].range_start
block_end = block_start + block_size
lo = i
else:
hi = i
- i = int((hi + lo) / 2)
+ i = int(old_div((hi + lo), 2))
block_size = data_locators[i].range_size
block_start = data_locators[i].range_start
block_end = block_start + block_size
from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
import collections
-import httplib
+import http.client
import httplib2
import json
import logging
# High probability of failure due to connection atrophy. Make
# sure this request [re]opens a new connection by closing and
# forgetting all cached connections first.
- for conn in self.connections.itervalues():
+ for conn in self.connections.values():
conn.close()
self.connections.clear()
self._last_request_time = time.time()
try:
return self.orig_http_request(uri, **kwargs)
- except httplib.HTTPException:
+ except http.client.HTTPException:
_logger.debug("Retrying API request in %d s after HTTP error",
delay, exc_info=True)
except socket.error:
# httplib2 reopens connections when needed.
_logger.debug("Retrying API request in %d s after socket error",
delay, exc_info=True)
- for conn in self.connections.itervalues():
+ for conn in self.connections.values():
conn.close()
time.sleep(delay)
delay = delay * self._retry_delay_backoff
from __future__ import absolute_import
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
+from past.utils import old_div
+from builtins import object
import functools
import os
import zlib
from . import config
import hashlib
import threading
-import Queue
+import queue
import copy
import errno
import re
# blocks pending. If they are full 64 MiB blocks, that means up to
# 256 MiB of internal buffering, which is the same size as the
# default download block cache in KeepClient.
- self._put_queue = Queue.Queue(maxsize=2)
+ self._put_queue = queue.Queue(maxsize=2)
self._put_threads = []
- for i in xrange(0, self.num_put_threads):
+ for i in range(0, self.num_put_threads):
thread = threading.Thread(target=self._commit_bufferblock_worker)
self._put_threads.append(thread)
thread.daemon = True
@synchronized
def start_get_threads(self):
if self._prefetch_threads is None:
- self._prefetch_queue = Queue.Queue()
+ self._prefetch_queue = queue.Queue()
self._prefetch_threads = []
- for i in xrange(0, self.num_get_threads):
+ for i in range(0, self.num_get_threads):
thread = threading.Thread(target=self._block_prefetch_worker)
self._prefetch_threads.append(thread)
thread.daemon = True
# A WRITABLE block with its owner.closed() implies that it's
# size is <= KEEP_BLOCK_SIZE/2.
try:
- small_blocks = [b for b in self._bufferblocks.values() if b.state() == _BufferBlock.WRITABLE and b.owner.closed()]
+ small_blocks = [b for b in list(self._bufferblocks.values()) if b.state() == _BufferBlock.WRITABLE and b.owner.closed()]
except AttributeError:
# Writable blocks without owner shouldn't exist.
raise UnownedBlockError()
self.repack_small_blocks(force=True, sync=True)
with self.lock:
- items = self._bufferblocks.items()
+ items = list(self._bufferblocks.items())
for k,v in items:
if v.state() != _BufferBlock.COMMITTED and v.owner:
with self.lock:
if len(self._segments) != len(othersegs):
return False
- for i in xrange(0, len(othersegs)):
+ for i in range(0, len(othersegs)):
seg1 = self._segments[i]
seg2 = othersegs[i]
loc1 = seg1.locator
"""
self._writers.remove(writer)
- if flush or self.size() > config.KEEP_BLOCK_SIZE / 2:
+ if flush or self.size() > old_div(config.KEEP_BLOCK_SIZE, 2):
# File writer closed, not small enough for repacking
self.flush()
elif self.closed():
+from builtins import object
import errno
import md5
import os
from __future__ import absolute_import
+from builtins import str
+from past.builtins import basestring
+from builtins import object
import functools
import logging
import os
return writer
def check_dependencies(self):
- for path, orig_stat in self._dependencies.items():
+ for path, orig_stat in list(self._dependencies.items()):
if not S_ISREG(orig_stat[ST_MODE]):
raise errors.StaleWriterStateError("{} not file".format(path))
try:
if value == self._committed:
return
if value:
- for k,v in self._items.items():
+ for k,v in list(self._items.items()):
v.set_committed(True)
self._committed = True
else:
@synchronized
def __iter__(self):
"""Iterate over names of files and collections contained in this collection."""
- return iter(self._items.keys())
+ return iter(list(self._items.keys()))
@synchronized
def __getitem__(self, k):
@synchronized
def keys(self):
"""Get a list of names of files and collections directly contained in this collection."""
- return self._items.keys()
+ return list(self._items.keys())
@synchronized
def values(self):
"""Get a list of files and collection objects directly contained in this collection."""
- return self._items.values()
+ return list(self._items.values())
@synchronized
def items(self):
"""Get a list of (name, object) tuples directly contained in this collection."""
- return self._items.items()
+ return list(self._items.items())
def exists(self, path):
"""Test if there is a file or collection at `path`."""
item.remove(pathcomponents[1])
def _clonefrom(self, source):
- for k,v in source.items():
+ for k,v in list(source.items()):
self._items[k] = v.clone(self, k)
def clone(self):
@synchronized
def flush(self):
"""Flush bufferblocks to Keep."""
- for e in self.values():
+ for e in list(self.values()):
e.flush()
if state == BLOCKS:
block_locator = re.match(r'[0-9a-f]{32}\+(\d+)(\+\S+)*', tok)
if block_locator:
- blocksize = long(block_locator.group(1))
+ blocksize = int(block_locator.group(1))
blocks.append(Range(tok, streamoffset, blocksize, 0))
streamoffset += blocksize
else:
if state == SEGMENTS:
file_segment = re.search(r'^(\d+):(\d+):(\S+)', tok)
if file_segment:
- pos = long(file_segment.group(1))
- size = long(file_segment.group(2))
+ pos = int(file_segment.group(1))
+ size = int(file_segment.group(2))
name = file_segment.group(3).replace('\\040', ' ')
filepath = os.path.join(stream_name, name)
afile = self.find_or_create(filepath, FILE)
# instances src and dst. If either of these files is not found,
# arv-copy will issue an error.
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from past.builtins import basestring
+from builtins import object
+from past.utils import old_div
import argparse
import contextlib
import getpass
import sys
import logging
import tempfile
-import urlparse
+import urllib.parse
import arvados
import arvados.config
abort("cannot copy object {} of type {}".format(args.object_uuid, t))
# Clean up any outstanding temp git repositories.
- for d in local_repo_dir.values():
+ for d in list(local_repo_dir.values()):
shutil.rmtree(d, ignore_errors=True)
# If no exception was thrown and the response does not have an
be None if that is not known.
"""
errors = []
- for cname, cspec in template_components.iteritems():
+ for cname, cspec in template_components.items():
def add_error(errmsg):
errors.append("{}: {}".format(cname, errmsg))
if not isinstance(cspec, dict):
# names. The return value is undefined.
#
def copy_git_repos(p, src, dst, dst_repo, args):
- for component in p['components'].itervalues():
+ for component in p['components'].values():
migrate_jobspec(component, src, dst, dst_repo, args)
if 'job' in component:
migrate_jobspec(component['job'], src, dst, dst_repo, args)
git_url = None
for url in priority:
if url.startswith("http"):
- u = urlparse.urlsplit(url)
- baseurl = urlparse.urlunsplit((u.scheme, u.netloc, "", "", ""))
+ u = urllib.parse.urlsplit(url)
+ baseurl = urllib.parse.urlunsplit((u.scheme, u.netloc, "", "", ""))
git_config = ["-c", "credential.%s/.username=none" % baseurl,
"-c", "credential.%s/.helper=!cred(){ cat >/dev/null; if [ \"$1\" = get ]; then echo password=$ARVADOS_API_TOKEN; fi; };cred" % baseurl]
else:
runtime_constraints field from src to dst."""
logger.debug('copy_docker_images: {}'.format(pipeline['uuid']))
- for c_name, c_info in pipeline['components'].iteritems():
+ for c_name, c_info in pipeline['components'].items():
if ('runtime_constraints' in c_info and
'docker_image' in c_info['runtime_constraints']):
copy_docker_image(
return "\r{}: {}M / {}M {:.1%} ".format(
obj_uuid,
bytes_written >> 20, bytes_expected >> 20,
- float(bytes_written) / bytes_expected)
+ old_div(float(bytes_written), bytes_expected))
else:
return "\r{}: {} ".format(obj_uuid, bytes_written)
#!/usr/bin/env python
+from builtins import next
import argparse
import collections
import datetime
# and add image listings for them, retaining the API server preference
# sorting.
images_start_size = len(images)
- for collection_uuid, link in hash_link_map.iteritems():
+ for collection_uuid, link in hash_link_map.items():
if not seen_image_names[collection_uuid]:
images.append(_new_image_listing(link, link['name']))
if len(images) > images_start_size:
#!/usr/bin/env python
from __future__ import print_function
+from __future__ import division
+from past.utils import old_div
import argparse
import sys
return parser.parse_args(args)
def size_formatter(coll_file):
- return "{:>10}".format((coll_file.size() + 1023) / 1024)
+ return "{:>10}".format(old_div((coll_file.size() + 1023), 1024))
def name_formatter(coll_file):
return "{}/{}".format(coll_file.stream_name(), coll_file.name)
from __future__ import print_function
+from __future__ import division
+from past.utils import old_div
import argparse
import time
import sys
if pdh not in already_migrated and (only_migrate is None or pdh in only_migrate):
need_migrate[pdh] = img
with CollectionReader(i["manifest_text"]) as c:
- if c.values()[0].size() > biggest:
- biggest = c.values()[0].size()
+ if list(c.values())[0].size() > biggest:
+ biggest = list(c.values())[0].size()
if args.print_unmigrated:
only_migrate = set()
logger.info("Already migrated %i images", len(already_migrated))
logger.info("Need to migrate %i images", len(need_migrate))
logger.info("Using tempdir %s", tempfile.gettempdir())
- logger.info("Biggest image is about %i MiB, tempdir needs at least %i MiB free", biggest/(2**20), (biggest*2)/(2**20))
+ logger.info("Biggest image is about %i MiB, tempdir needs at least %i MiB free", old_div(biggest,(2**20)), old_div((biggest*2),(2**20)))
if args.dry_run:
return
success = []
failures = []
count = 1
- for old_image in need_migrate.values():
+ for old_image in list(need_migrate.values()):
if uuid_to_collection[old_image["collection"]]["portable_data_hash"] in already_migrated:
continue
oldcol = CollectionReader(uuid_to_collection[old_image["collection"]]["manifest_text"])
- tarfile = oldcol.keys()[0]
+ tarfile = list(oldcol.keys())[0]
logger.info("[%i/%i] Migrating %s:%s (%s) (%i MiB)", count, len(need_migrate), old_image["repo"],
- old_image["tag"], old_image["collection"], oldcol.values()[0].size()/(2**20))
+ old_image["tag"], old_image["collection"], old_div(list(oldcol.values())[0].size(),(2**20)))
count += 1
start = time.time()
# TODO:
# --md5sum - display md5 of each file as read from disk
+from __future__ import division
+from builtins import str
+from past.utils import old_div
+from builtins import object
import argparse
import arvados
import arvados.collection
if len(args.paths) == 0:
args.paths = ['-']
- args.paths = map(lambda x: "-" if x == "/dev/stdin" else x, args.paths)
+ args.paths = ["-" if x == "/dev/stdin" else x for x in args.paths]
if len(args.paths) != 1 or os.path.isdir(args.paths[0]):
if args.filename:
Recursively get the total size of the collection
"""
size = 0
- for item in collection.values():
+ for item in list(collection.values()):
if isinstance(item, arvados.collection.Collection) or isinstance(item, arvados.collection.Subcollection):
size += self._collection_size(item)
else:
def collection_file_paths(self, col, path_prefix='.'):
"""Return a list of file paths by recursively go through the entire collection `col`"""
file_paths = []
- for name, item in col.items():
+ for name, item in list(col.items()):
if isinstance(item, arvados.arvfile.ArvadosFile):
file_paths.append(os.path.join(path_prefix, name))
elif isinstance(item, arvados.collection.Subcollection):
locators.append(loc)
return locators
elif isinstance(item, arvados.collection.Collection):
- l = [self._datablocks_on_item(x) for x in item.values()]
+ l = [self._datablocks_on_item(x) for x in list(item.values())]
# Fast list flattener method taken from:
# http://stackoverflow.com/questions/952914/making-a-flat-list-out-of-list-of-lists-in-python
return [loc for sublist in l for loc in sublist]
if bytes_expected:
return "\r{}M / {}M {:.1%} ".format(
bytes_written >> 20, bytes_expected >> 20,
- float(bytes_written) / bytes_expected)
+ old_div(float(bytes_written), bytes_expected))
else:
return "\r{} ".format(bytes_written)
if not output.endswith('\n'):
stdout.write('\n')
- for sigcode, orig_handler in orig_signal_handlers.items():
+ for sigcode, orig_handler in list(orig_signal_handlers.items()):
signal.signal(sigcode, orig_handler)
if status != 0:
from __future__ import print_function
from __future__ import absolute_import
+from builtins import range
+from past.builtins import basestring
+from builtins import object
import arvados
import arvados.commands.ws as ws
import argparse
if files:
uploadfiles(files, api, dry_run=args.dry_run, num_retries=args.retries, project=project)
- for i in xrange(1, len(slots)):
+ for i in range(1, len(slots)):
slots[i] = [("%s%s" % (c.prefix, c.fn)) if isinstance(c, ArvFile) else c for c in slots[i]]
component = {
group_parser.add_argument('-b', '--batch-size', type=int)
group_parser.add_argument('args', nargs=argparse.REMAINDER)
- for s in xrange(2, len(slots)):
- for i in xrange(0, len(slots[s])):
+ for s in range(2, len(slots)):
+ for i in range(0, len(slots[s])):
if slots[s][i] == '--':
inp = "input%i" % (s-2)
groupargs = group_parser.parse_args(slots[2][i+1:])
+from builtins import object
import json
import os
self._request_errors = OrderedDict(request_errors)
if self._request_errors:
exc_reports = [self._format_error(*err_pair)
- for err_pair in self._request_errors.iteritems()]
+ for err_pair in self._request_errors.items()]
base_msg = "{}: {}".format(message, "; ".join(exc_reports))
else:
base_msg = message
from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import object
import arvados
from . import config
from . import errors
import logging
import json
-import thread
+import _thread
import threading
import time
import os
self.on_event_cb(m)
except Exception as e:
_logger.exception("Unexpected exception from event callback.")
- thread.interrupt_main()
+ _thread.interrupt_main()
def on_closed(self):
if not self.is_closed.is_set():
if tries_left == 0:
_logger.exception("EventClient thread could not contact websocket server.")
self.is_closed.set()
- thread.interrupt_main()
+ _thread.interrupt_main()
return
def run_forever(self):
_logger.exception("PollClient thread could not contact API server.")
with self._closing_lock:
self._closing.set()
- thread.interrupt_main()
+ _thread.interrupt_main()
return
for i in items["items"]:
skip_old_events = [["id", ">", str(i["id"])]]
self.on_event(i)
except Exception as e:
_logger.exception("Unexpected exception from event callback.")
- thread.interrupt_main()
+ _thread.interrupt_main()
if items["items_available"] > len(items["items"]):
moreitems = True
if not moreitems:
from __future__ import absolute_import
-import cStringIO
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import next
+from builtins import str
+from builtins import range
+from past.utils import old_div
+from builtins import object
+import io
import datetime
import hashlib
import logging
import math
import os
import pycurl
-import Queue
+import queue
import re
import socket
import ssl
import sys
import threading
from . import timer
-import urlparse
+import urllib.parse
import arvados
import arvados.config as config
self._cache = [c for c in self._cache if not (c.ready.is_set() and c.content is None)]
sm = sum([slot.size() for slot in self._cache])
while len(self._cache) > 0 and sm > self.cache_max:
- for i in xrange(len(self._cache)-1, -1, -1):
+ for i in range(len(self._cache)-1, -1, -1):
if self._cache[i].ready.is_set():
del self._cache[i]
break
def _get(self, locator):
# Test if the locator is already in the cache
- for i in xrange(0, len(self._cache)):
+ for i in range(0, len(self._cache)):
if self._cache[i].locator == locator:
n = self._cache[i]
if i != 0:
arvados.errors.HttpError,
)
- def __init__(self, root, user_agent_pool=Queue.LifoQueue(),
+ def __init__(self, root, user_agent_pool=queue.LifoQueue(),
upload_counter=None,
download_counter=None, **headers):
self.root = root
def _get_user_agent(self):
try:
return self._user_agent_pool.get(block=False)
- except Queue.Empty:
+ except queue.Empty:
return pycurl.Curl()
def _put_user_agent(self, ua):
try:
with timer.Timer() as t:
self._headers = {}
- response_body = cStringIO.StringIO()
+ response_body = io.StringIO()
curl.setopt(pycurl.NOSIGNAL, 1)
curl.setopt(pycurl.OPENSOCKETFUNCTION, self._socket_open)
curl.setopt(pycurl.URL, url.encode('utf-8'))
curl.setopt(pycurl.HTTPHEADER, [
- '{}: {}'.format(k,v) for k,v in self.get_headers.iteritems()])
+ '{}: {}'.format(k,v) for k,v in self.get_headers.items()])
curl.setopt(pycurl.WRITEFUNCTION, response_body.write)
curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
if method == "HEAD":
self._result['status_code'],
len(self._result['body']),
t.msecs,
- (len(self._result['body'])/(1024.0*1024))/t.secs if t.secs > 0 else 0)
+ old_div((old_div(len(self._result['body']),(1024.0*1024))),t.secs) if t.secs > 0 else 0)
if self.download_counter:
self.download_counter.add(len(self._result['body']))
try:
with timer.Timer() as t:
self._headers = {}
- body_reader = cStringIO.StringIO(body)
- response_body = cStringIO.StringIO()
+ body_reader = io.StringIO(body)
+ response_body = io.StringIO()
curl.setopt(pycurl.NOSIGNAL, 1)
curl.setopt(pycurl.OPENSOCKETFUNCTION, self._socket_open)
curl.setopt(pycurl.URL, url.encode('utf-8'))
curl.setopt(pycurl.INFILESIZE, len(body))
curl.setopt(pycurl.READFUNCTION, body_reader.read)
curl.setopt(pycurl.HTTPHEADER, [
- '{}: {}'.format(k,v) for k,v in self.put_headers.iteritems()])
+ '{}: {}'.format(k,v) for k,v in self.put_headers.items()])
curl.setopt(pycurl.WRITEFUNCTION, response_body.write)
curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
self._setcurltimeouts(curl, timeout)
self._result['status_code'],
len(body),
t.msecs,
- (len(body)/(1024.0*1024))/t.secs if t.secs > 0 else 0)
+ old_div((old_div(len(body),(1024.0*1024))),t.secs) if t.secs > 0 else 0)
if self.upload_counter:
self.upload_counter.add(len(body))
return True
# Returning None implies all bytes were written
- class KeepWriterQueue(Queue.Queue):
+ class KeepWriterQueue(queue.Queue):
def __init__(self, copies):
- Queue.Queue.__init__(self) # Old-style superclass
+ queue.Queue.__init__(self) # Old-style superclass
self.wanted_copies = copies
self.successful_copies = 0
self.response = None
return service, service_root
elif self.empty():
self.pending_tries_notification.notify_all()
- raise Queue.Empty
+ raise queue.Empty
else:
self.pending_tries_notification.wait()
if (not max_service_replicas) or (max_service_replicas >= copies):
num_threads = 1
else:
- num_threads = int(math.ceil(float(copies) / max_service_replicas))
+ num_threads = int(math.ceil(old_div(float(copies), max_service_replicas)))
_logger.debug("Pool max threads is %d", num_threads)
self.workers = []
self.queue = KeepClient.KeepWriterQueue(copies)
while True:
try:
service, service_root = self.queue.get_next_task()
- except Queue.Empty:
+ except queue.Empty:
return
try:
locator, copies = self.do_task(service, service_root)
self.block_cache = block_cache if block_cache else KeepBlockCache()
self.timeout = timeout
self.proxy_timeout = proxy_timeout
- self._user_agent_pool = Queue.LifoQueue()
+ self._user_agent_pool = queue.LifoQueue()
self.upload_counter = Counter()
self.download_counter = Counter()
self.put_counter = Counter()
if not proxy_uris[i].endswith('/'):
proxy_uris[i] += '/'
# URL validation
- url = urlparse.urlparse(proxy_uris[i])
+ url = urllib.parse.urlparse(proxy_uris[i])
if not (url.scheme and url.netloc):
raise arvados.errors.ArgumentError("Invalid proxy URI: {}".format(proxy_uris[i]))
self.api_token = api_token
raise arvados.errors.NoKeepServersError()
# Precompute the base URI for each service.
- for r in self._gateway_services.itervalues():
+ for r in self._gateway_services.values():
host = r['service_host']
if not host.startswith('[') and host.find(':') >= 0:
# IPv6 URIs must be formatted like http://[::1]:80/...
_logger.debug(str(self._gateway_services))
self._keep_services = [
- ks for ks in self._gateway_services.itervalues()
+ ks for ks in self._gateway_services.values()
if not ks.get('service_type', '').startswith('gateway:')]
self._writable_services = [ks for ks in self._keep_services
if not ks.get('read_only')]
KeepClient is initialized.
"""
- if isinstance(data, unicode):
+ if isinstance(data, str):
data = data.encode("ascii")
elif not isinstance(data, str):
raise arvados.errors.ArgumentError("Argument 'data' to KeepClient.put is not type 'str'")
#!/usr/bin/env python
+from builtins import range
+from builtins import object
import functools
import inspect
import pycurl
import arvados.errors
-_HTTP_SUCCESSES = set(xrange(200, 300))
+_HTTP_SUCCESSES = set(range(200, 300))
_HTTP_CAN_RETRY = set([408, 409, 422, 423, 500, 502, 503, 504])
class RetryLoop(object):
def running(self):
return self._running and (self._success is None)
- def next(self):
+ def __next__(self):
if self._running is None:
self._running = True
if (self.tries_left < 1) or not self.running():
from __future__ import absolute_import
+from builtins import object
import copy
import threading
from __future__ import print_function
from __future__ import absolute_import
+from builtins import object
import collections
import hashlib
import os
s = re.match(r'^[0-9a-f]{32}\+(\d+)(\+\S+)*$', tok)
if s:
- blocksize = long(s.group(1))
+ blocksize = int(s.group(1))
self._data_locators.append(Range(tok, streamoffset, blocksize, 0))
streamoffset += blocksize
continue
s = re.search(r'^(\d+):(\d+):(\S+)', tok)
if s:
- pos = long(s.group(1))
- size = long(s.group(2))
+ pos = int(s.group(1))
+ size = int(s.group(2))
name = s.group(3).replace('\\040', ' ')
if name not in self._files:
self._files[name] = StreamFileReader(self, [Range(pos, 0, size, 0)], name)
return self._files
def all_files(self):
- return self._files.values()
+ return list(self._files.values())
def size(self):
n = self._data_locators[-1]
manifest_text.extend([d.locator for d in self._data_locators])
manifest_text.extend([' '.join(["{}:{}:{}".format(seg.locator, seg.range_size, f.name.replace(' ', '\\040'))
for seg in f.segments])
- for f in self._files.values()])
+ for f in list(self._files.values())])
return ' '.join(manifest_text) + '\n'
from __future__ import print_function
+from builtins import object
import time
class Timer(object):
raise arvados.errors.CommandFailedError(
"tar exited %d" % p.returncode)
os.symlink(tarball, os.path.join(path, '.locator'))
- tld_extracts = filter(lambda f: f != '.locator', os.listdir(path))
+ tld_extracts = [f for f in os.listdir(path) if f != '.locator']
lockfile.close()
if len(tld_extracts) == 1:
return os.path.join(path, tld_extracts[0])
"unzip exited %d" % p.returncode)
os.unlink(zip_filename)
os.symlink(zipball, os.path.join(path, '.locator'))
- tld_extracts = filter(lambda f: f != '.locator', os.listdir(path))
+ tld_extracts = [f for f in os.listdir(path) if f != '.locator']
lockfile.close()
if len(tld_extracts) == 1:
return os.path.join(path, tld_extracts[0])
#!/usr/bin/env python
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import range
+from builtins import object
import arvados
import contextlib
import errno
import hashlib
-import httplib
+import http.client
import httplib2
import io
import mock
import os
import pycurl
-import Queue
+import queue
import shutil
import sys
import tempfile
given, it will be consumed to fill the queue before queue_with()
returns.
"""
- queue = Queue.Queue()
+ queue = queue.Queue()
for val in items:
queue.put(val)
return lambda *args, **kwargs: queue.get(block=False)
# mock calls to httplib2.Http.request()
def fake_httplib2_response(code, **headers):
headers.update(status=str(code),
- reason=httplib.responses.get(code, "Unknown Response"))
+ reason=http.client.responses.get(code, "Unknown Response"))
return httplib2.Response(headers)
def mock_responses(body, *codes, **headers):
sys.stderr = orig_stderr
-class FakeCurl:
+class FakeCurl(object):
@classmethod
def make(cls, code, body='', headers={}):
return mock.Mock(spec=cls, wraps=cls(code, body, headers))
raise ValueError
if self._headerfunction:
self._headerfunction("HTTP/1.1 {} Status".format(self._resp_code))
- for k, v in self._resp_headers.iteritems():
+ for k, v in self._resp_headers.items():
self._headerfunction(k + ': ' + str(v))
if type(self._resp_body) is not bool:
self._writer(self._resp_body)
-import BaseHTTPServer
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from past.utils import old_div
+import http.server
import hashlib
import os
import re
-import SocketServer
+import socketserver
import time
-class Server(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer, object):
+class Server(socketserver.ThreadingMixIn, http.server.HTTPServer, object):
allow_reuse_address = 1
def setdelays(self, **kwargs):
"""In future requests, induce delays at the given checkpoints."""
- for (k, v) in kwargs.iteritems():
+ for (k, v) in kwargs.items():
self.delays.get(k) # NameError if unknown key
self.delays[k] = v
self._sleep_at_least(self.delays[k])
-class Handler(BaseHTTPServer.BaseHTTPRequestHandler, object):
+class Handler(http.server.BaseHTTPRequestHandler, object):
def wfile_bandwidth_write(self, data_to_write):
if self.server.bandwidth == None and self.server.delays['mid_write'] == 0:
self.wfile.write(data_to_write)
else:
- BYTES_PER_WRITE = int(self.server.bandwidth/4.0) or 32768
+ BYTES_PER_WRITE = int(old_div(self.server.bandwidth,4.0)) or 32768
outage_happened = False
num_bytes = len(data_to_write)
num_sent_bytes = 0
num_sent_bytes:num_sent_bytes+num_write_bytes])
num_sent_bytes += num_write_bytes
if self.server.bandwidth is not None:
- target_time += num_write_bytes / self.server.bandwidth
+ target_time += old_div(num_write_bytes, self.server.bandwidth)
self.server._sleep_at_least(target_time - time.time())
return None
if self.server.bandwidth == None and self.server.delays['mid_read'] == 0:
return self.rfile.read(bytes_to_read)
else:
- BYTES_PER_READ = int(self.server.bandwidth/4.0) or 32768
+ BYTES_PER_READ = int(old_div(self.server.bandwidth,4.0)) or 32768
data = ''
outage_happened = False
bytes_read = 0
data += self.rfile.read(next_bytes_to_read)
bytes_read += next_bytes_to_read
if self.server.bandwidth is not None:
- target_time += next_bytes_to_read / self.server.bandwidth
+ target_time += old_div(next_bytes_to_read, self.server.bandwidth)
self.server._sleep_at_least(target_time - time.time())
return data
from __future__ import absolute_import
+from builtins import range
+from builtins import object
import arvados
from . import arvados_testutil as tutil
from __future__ import print_function
from __future__ import absolute_import
+from builtins import range
import unittest
from .performance_profiler import profiled
#!/usr/bin/env python
from __future__ import print_function
+from __future__ import division
+from builtins import str
+from builtins import range
+from past.utils import old_div
import argparse
import atexit
import errno
# Use up to half of the +wait+ period waiting for "passenger
# stop" to work. If the process hasn't exited by then, start
# sending TERM signals.
- startTERM += wait/2
+ startTERM += old_div(wait,2)
server_pid = None
while now <= deadline and server_pid is None:
"-listen=:{}".format(port),
"-pid="+_pidfile('keep{}'.format(n))]
- for arg, val in keep_args.iteritems():
+ for arg, val in keep_args.items():
keep_cmd.append("{}={}".format(arg, val))
logf = open(_fifo2stderr('keep{}'.format(n)), 'w')
@staticmethod
def _restore_dict(src, dest):
- for key in dest.keys():
+ for key in list(dest.keys()):
if key not in src:
del dest[key]
dest.update(src)
#!/usr/bin/env python
from __future__ import absolute_import
+from builtins import str
+from builtins import range
import arvados
import collections
import httplib2
api = arvados.api('v1',
requestBuilder=req_builder, model=OrderedJsonModel())
result = api.humans().get(uuid='test').execute()
- self.assertEqual(string.hexdigits, ''.join(result.keys()))
+ self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
class RetryREST(unittest.TestCase):
mock_conns = {str(i): mock.MagicMock() for i in range(2)}
self.api._http.connections = mock_conns.copy()
self.api.users().create(body={}).execute()
- for c in mock_conns.itervalues():
+ for c in mock_conns.values():
self.assertEqual(c.close.call_count, expect)
@mock.patch('time.sleep')
# -*- coding: utf-8 -*-
from __future__ import absolute_import
+from builtins import str
+from builtins import range
import io
import os
import random
# -*- coding: utf-8 -*-
from __future__ import absolute_import
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import range
+from past.utils import old_div
import apiclient
import io
import mock
import hashlib
import random
-from cStringIO import StringIO
+from io import StringIO
import arvados
import arvados.commands.put as arv_put
_, self.large_file_name = tempfile.mkstemp()
fileobj = open(self.large_file_name, 'w')
# Make sure to write just a little more than one block
- for _ in range((arvados.config.KEEP_BLOCK_SIZE/(1024*1024))+1):
+ for _ in range((old_div(arvados.config.KEEP_BLOCK_SIZE,(1024*1024)))+1):
data = random.choice(['x', 'y', 'z']) * 1024 * 1024 # 1 MB
fileobj.write(data)
fileobj.close()
def test_known_human_progress(self):
for count, total in [(0, 1), (2, 4), (45, 60)]:
- expect = '{:.1%}'.format(float(count) / total)
+ expect = '{:.1%}'.format(old_div(float(count), total))
actual = arv_put.human_progress(count, total)
self.assertTrue(actual.startswith('\r'))
self.assertIn(expect, actual)
#!/usr/bin/env python
from __future__ import absolute_import
+from builtins import hex
+from builtins import str
+from builtins import range
+from builtins import object
import bz2
import datetime
import gzip
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
text = "0123456789" * 100
- for b in xrange(0, 100000):
+ for b in range(0, 100000):
writer.write(text)
self.assertEqual(writer.size(), 100000000)
with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- for b in xrange(0, 10):
+ for b in range(0, 10):
writer.seek(0, os.SEEK_SET)
writer.write("0123456789")
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- for b in xrange(0, 10):
+ for b in range(0, 10):
writer.seek(10, os.SEEK_SET)
writer.write("abcdefghij")
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- for b in xrange(0, 10):
+ for b in range(0, 10):
writer.seek(5, os.SEEK_SET)
writer.write("abcdefghij")
with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
- text = ''.join(["0123456789" for a in xrange(0, 100)])
- for b in xrange(0, 100000):
+ text = ''.join(["0123456789" for a in range(0, 100)])
+ for b in range(0, 100000):
writer.write(text)
writer.seek(0, os.SEEK_SET)
writer.write("foo")
from __future__ import print_function
from __future__ import absolute_import
+from builtins import str
+from builtins import range
import md5
import mock
import os
def _random(n):
- return bytearray(random.getrandbits(8) for _ in xrange(n))
+ return bytearray(random.getrandbits(8) for _ in range(n))
class CacheTestThread(threading.Thread):
#
# ARVADOS_API_TOKEN=abc ARVADOS_API_HOST=arvados.local python -m unittest discover
+from builtins import object
import arvados
import copy
import mock
from __future__ import print_function
from __future__ import absolute_import
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
+from builtins import object
+from past.utils import old_div
import arvados
import io
import logging
import mock
-import Queue
+import queue
from . import run_test_server
import threading
import time
def _test_subscribe(self, poll_fallback, expect_type, start_time=None, expected=1):
run_test_server.authorize_with('active')
- events = Queue.Queue(100)
+ events = queue.Queue(100)
# Create ancestor before subscribing.
# When listening with start_time in the past, this should also be retrieved.
log_object_uuids.append(events.get(True, 5)['object_uuid'])
if expected < 2:
- with self.assertRaises(Queue.Empty):
+ with self.assertRaises(queue.Empty):
# assertEqual just serves to show us what unexpected
# thing comes out of the queue when the assertRaises
# fails; when the test passes, this assertEqual
return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(t))
def localiso(self, t):
- return time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime(t)) + self.isotz(-time.timezone/60)
+ return time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime(t)) + self.isotz(old_div(-time.timezone,60))
def isotz(self, offset):
"""Convert minutes-east-of-UTC to RFC3339- and ISO-compatible time zone designator"""
- return '{:+03d}:{:02d}'.format(offset/60, offset%60)
+ return '{:+03d}:{:02d}'.format(old_div(offset,60), offset%60)
# Test websocket reconnection on (un)execpted close
def _test_websocket_reconnect(self, close_unexpected):
run_test_server.authorize_with('active')
- events = Queue.Queue(100)
+ events = queue.Queue(100)
logstream = io.BytesIO()
rootLogger = logging.getLogger()
# expect an event
self.assertIn(human['uuid'], events.get(True, 5)['object_uuid'])
- with self.assertRaises(Queue.Empty):
+ with self.assertRaises(queue.Empty):
self.assertEqual(events.get(True, 2), None)
# close (im)properly
event = events.get(True, 5)
if event.get('object_uuid') != None:
log_object_uuids.append(event['object_uuid'])
- with self.assertRaises(Queue.Empty):
+ with self.assertRaises(queue.Empty):
self.assertEqual(events.get(True, 2), None)
self.assertNotIn(human['uuid'], log_object_uuids)
self.assertIn(human2['uuid'], log_object_uuids)
else:
- with self.assertRaises(Queue.Empty):
+ with self.assertRaises(queue.Empty):
self.assertEqual(events.get(True, 2), None)
# verify log message to ensure that an (un)expected close
rootLogger.addHandler(streamHandler)
run_test_server.authorize_with('active')
- events = Queue.Queue(100)
+ events = queue.Queue(100)
filters = [['object_uuid', 'is_a', 'arvados#human']]
self.ws = arvados.events.subscribe(
from __future__ import absolute_import
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import range
+from past.utils import old_div
+from builtins import object
import hashlib
import mock
import os
import threading
import time
import unittest
-import urlparse
+import urllib.parse
import arvados
import arvados.retry
def get_service_roots(self, api_client):
keep_client = arvados.KeepClient(api_client=api_client)
services = keep_client.weighted_service_roots(arvados.KeepLocator('0'*32))
- return [urlparse.urlparse(url) for url in sorted(services)]
+ return [urllib.parse.urlparse(url) for url in sorted(services)]
def test_ssl_flag_respected_in_roots(self):
for ssl_flag in [False, True]:
num_retries=3)
self.assertEqual([403, 403], [
getattr(error, 'status_code', None)
- for error in err_check.exception.request_errors().itervalues()])
+ for error in err_check.exception.request_errors().values()])
def test_get_error_reflects_last_retry(self):
self.check_errors_from_last_retry('get', arvados.errors.KeepReadError)
self.assertRaises(exc_class) as err_check:
curl_mock.return_value.side_effect = socket.timeout
getattr(keep_client, verb)(data)
- urls = [urlparse.urlparse(url)
+ urls = [urllib.parse.urlparse(url)
for url in err_check.exception.request_errors()]
self.assertEqual([('keep0x' + c, aport) for c in '3eab2d5fc9681074'],
[(url.hostname, url.port) for url in urls])
def test_only_write_enough_on_success(self):
for i in range(10):
- ks = self.FakeKeepService(delay=i/10.0, will_succeed=True)
+ ks = self.FakeKeepService(delay=old_div(i,10.0), will_succeed=True)
self.pool.add_task(ks, None)
self.pool.join()
self.assertEqual(self.pool.done(), self.copies)
def test_only_write_enough_on_partial_success(self):
for i in range(5):
- ks = self.FakeKeepService(delay=i/10.0, will_succeed=False)
+ ks = self.FakeKeepService(delay=old_div(i,10.0), will_succeed=False)
self.pool.add_task(ks, None)
- ks = self.FakeKeepService(delay=i/10.0, will_succeed=True)
+ ks = self.FakeKeepService(delay=old_div(i,10.0), will_succeed=True)
self.pool.add_task(ks, None)
self.pool.join()
self.assertEqual(self.pool.done(), self.copies)
def test_only_write_enough_when_some_crash(self):
for i in range(5):
- ks = self.FakeKeepService(delay=i/10.0, will_raise=Exception())
+ ks = self.FakeKeepService(delay=old_div(i,10.0), will_raise=Exception())
self.pool.add_task(ks, None)
- ks = self.FakeKeepService(delay=i/10.0, will_succeed=True)
+ ks = self.FakeKeepService(delay=old_div(i,10.0), will_succeed=True)
self.pool.add_task(ks, None)
self.pool.join()
self.assertEqual(self.pool.done(), self.copies)
def test_fail_when_too_many_crash(self):
for i in range(self.copies+1):
- ks = self.FakeKeepService(delay=i/10.0, will_raise=Exception())
+ ks = self.FakeKeepService(delay=old_div(i,10.0), will_raise=Exception())
self.pool.add_task(ks, None)
for i in range(self.copies-1):
- ks = self.FakeKeepService(delay=i/10.0, will_succeed=True)
+ ks = self.FakeKeepService(delay=old_div(i,10.0), will_succeed=True)
self.pool.add_task(ks, None)
self.pool.join()
self.assertEqual(self.pool.done(), self.copies-1)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+from builtins import next
+from builtins import zip
+from builtins import str
+from builtins import range
import datetime
import itertools
import random
def numstrs(fmtstr, base, exponent):
def genstrs(self, count=None):
return (fmtstr.format(random.randint(0, base ** exponent))
- for c in xrange(count or self.DEFAULT_TEST_COUNT))
+ for c in range(count or self.DEFAULT_TEST_COUNT))
return genstrs
checksums = numstrs('{:032x}', 16, 32)
def base_locators(self, count=DEFAULT_TEST_COUNT):
return ('+'.join(pair) for pair in
- itertools.izip(self.checksums(count), self.sizes(count)))
+ zip(self.checksums(count), self.sizes(count)))
def perm_hints(self, count=DEFAULT_TEST_COUNT):
- for sig, ts in itertools.izip(self.signatures(count),
+ for sig, ts in zip(self.signatures(count),
self.timestamps(count)):
yield 'A{}@{}'.format(sig, ts)
def test_good_locators_returned(self):
for hint_gens in [(), (self.sizes(),),
(self.sizes(), self.perm_hints())]:
- for loc_data in itertools.izip(self.checksums(), *hint_gens):
+ for loc_data in zip(self.checksums(), *hint_gens):
locator = '+'.join(loc_data)
self.assertEqual(locator, str(KeepLocator(locator)))
#!/usr/bin/env python
+from builtins import zip
+from builtins import range
+from builtins import object
import itertools
import unittest
responses = itertools.chain(results, itertools.repeat(None))
retrier = arv_retry.RetryLoop(num_retries, self.loop_success,
**kwargs)
- for tries_left, response in itertools.izip(retrier, responses):
+ for tries_left, response in zip(retrier, responses):
retrier.save_result(response)
return retrier
check_is_not = check('assertIsNot')
def test_obvious_successes(self):
- self.check_is(True, *range(200, 207))
+ self.check_is(True, *list(range(200, 207)))
def test_obvious_stops(self):
self.check_is(False, 424, 426, 428, 431,
- *range(400, 408) + range(410, 420))
+ *list(range(400, 408)) + list(range(410, 420)))
def test_obvious_retries(self):
self.check_is(None, 500, 502, 503, 504)
self.check_is(None, 408, 409, 422, 423)
def test_5xx_failures(self):
- self.check_is(False, 501, *range(505, 512))
+ self.check_is(False, 501, *list(range(505, 512)))
def test_1xx_not_retried(self):
self.check_is_not(None, 100, 101)
def test_redirects_not_retried(self):
- self.check_is_not(None, *range(300, 309))
+ self.check_is_not(None, *list(range(300, 309)))
def test_wacky_code_retries(self):
self.check_is(None, 0, 99, 600, -200)
#!/usr/bin/env python
from __future__ import absolute_import
+from builtins import object
import mock
import os
import unittest
#!/usr/bin/env python
from __future__ import absolute_import
+from builtins import object
import bz2
import gzip
import io
projects / arvados.git / commitdiff