Merge branch 'master' into 1646-arv-put

Conflicts:
	sdk/python/arvados.py

diff --cc sdk/python/arvados.py
index 6aa7dc9a7198291b9d182d07552c043678e5668d,7af06096702ea41a3a0109325e1ab968b9bc9271..8208e836672461aeadfc31d7788ab2e735c8a2c0
--- 1/sdk/python/arvados.py
--- 2/sdk/python/arvados.py
+++ b/sdk/python/arvados.py
@@@ -163,21 -152,33 +166,34 @@@ class job_setup
              exit(0)
  
  class util:
+     @staticmethod
+     def clear_tmpdir(path=None):
+         """
+         Ensure the given directory (or TASK_TMPDIR if none given)
+         exists and is empty.
+         """
+         if path == None:
+             path = current_task().tmpdir
+         if os.path.exists(path):
+             p = subprocess.Popen(['rm', '-rf', path])
+             stdout, stderr = p.communicate(None)
+             if p.returncode != 0:
+                 raise Exception('rm -rf %s: %s' % (path, stderr))
+         os.mkdir(path)
+ 
      @staticmethod
      def run_command(execargs, **kwargs):
-         if 'stdin' not in kwargs:
-             kwargs['stdin'] = subprocess.PIPE
-         if 'stdout' not in kwargs:
-             kwargs['stdout'] = subprocess.PIPE
-         if 'stderr' not in kwargs:
-             kwargs['stderr'] = subprocess.PIPE
-         p = subprocess.Popen(execargs, close_fds=True, shell=False,
-                              **kwargs)
+         kwargs.setdefault('stdin', subprocess.PIPE)
+         kwargs.setdefault('stdout', subprocess.PIPE)
+         kwargs.setdefault('stderr', sys.stderr)
+         kwargs.setdefault('close_fds', True)
+         kwargs.setdefault('shell', False)
+         p = subprocess.Popen(execargs, **kwargs)
          stdoutdata, stderrdata = p.communicate(None)
          if p.returncode != 0:
 -            raise Exception("run_command %s exit %d:\n%s" %
 -                            (execargs, p.returncode, stderrdata))
 +            raise errors.CommandFailedError(
 +                "run_command %s exit %d:\n%s" %
 +                (execargs, p.returncode, stderrdata))
          return stdoutdata, stderrdata
  
      @staticmethod
@@@ -380,11 -384,8 +401,11 @@@
                          outfile.write(buf)
                      outfile.close()
          if len(files_got) < len(files):
 -            raise Exception("Wanted files %s but only got %s from %s" % (files, files_got, map(lambda z: z.name(), list(CollectionReader(collection).all_files()))))
 +            raise errors.AssertionError(
 +                "Wanted files %s but only got %s from %s" %
 +                (files, files_got,
 +                 [z.name() for z in CollectionReader(collection).all_files()]))
-         os.symlink(collection, os.path.join(path, '.locator'))
+         os.symlink(collection_hash, os.path.join(path, '.locator'))
  
          lockfile.close()
          return path
@@@ -757,9 -755,8 +782,9 @@@ class CollectionWriter(object)
          self.set_current_stream_name(newstreamname)
      def set_current_stream_name(self, newstreamname):
          if re.search(r'[ \t\n]', newstreamname):
 -            raise AssertionError("Manifest stream names cannot contain whitespace")
 +            raise errors.AssertionError(
 +                "Manifest stream names cannot contain whitespace")
-         self._current_stream_name = newstreamname
+         self._current_stream_name = '.' if newstreamname=='' else newstreamname
      def current_stream_name(self):
          return self._current_stream_name
      def finish_current_stream(self):