Merge branch '1885-keep-proxy' into 2751-python-sdk-keep-proxy-support

diff --cc sdk/python/arvados/keep.py
index d1e2255481574087326bd27b824534d46f89ad29,e414d267a1347a51c9ae48354082e14bf48da29d..7618cff18f5844ba088855810f638429c61d5e0d
--- 1/sdk/python/arvados/keep.py
--- 2/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@@ -166,50 -147,60 +166,80 @@@ class KeepClient(object)
      def shuffled_service_roots(self, hash):
          if self.service_roots == None:
              self.lock.acquire()
 -            try:
 -                keep_disks = arvados.api().keep_disks().list().execute()['items']
 -                roots = (("http%s://%s:%d/" %
 -                          ('s' if f['service_ssl_flag'] else '',
 -                           f['service_host'],
 -                           f['service_port']))
 -                         for f in keep_disks)
 -                self.service_roots = sorted(set(roots))
 -                logging.debug(str(self.service_roots))
 -            finally:
 -                self.lock.release()
 +
 +            # Override normal keep disk lookup with an explict proxy
 +            # configuration.
 +            keep_proxy_env = config.get("ARVADOS_KEEP_PROXY")
 +            if keep_proxy_env != None:
 +                if keep_proxy_env[-1:] != '/':
 +                    keep_proxy_env += "/"
 +                self.service_roots = [keep_proxy_env]
 +                self.using_proxy = True
 +            else:
 +                try:
 +                    try:
 +                        keep_services = arvados.api().keep_services().accessible().execute()['items']
 +                    except:
 +                        keep_services = arvados.api().keep_disks().list().execute()['items']
 +
 +                    if len(keep_services) == 0:
 +                        raise arvados.errors.NoKeepServersError()
 +
 +                    if 'service_type' in keep_services[0] and keep_services[0]['service_type'] == 'proxy':
 +                        self.using_proxy = True
 +
 +                    roots = (("http%s://%s:%d/" %
 +                              ('s' if f['service_ssl_flag'] else '',
 +                               f['service_host'],
 +                               f['service_port']))
 +                             for f in keep_services)
 +                    self.service_roots = sorted(set(roots))
 +                    logging.debug(str(self.service_roots))
 +                finally:
 +                    self.lock.release()
  
+         # Build an ordering with which to query the Keep servers based on the
+         # contents of the hash.
+         # "hash" is a hex-encoded number at least 8 digits
+         # (32 bits) long
+ 
+         # seed used to calculate the next keep server from 'pool'
+         # to be added to 'pseq'
          seed = hash
+ 
+         # Keep servers still to be added to the ordering
          pool = self.service_roots[:]
+ 
+         # output probe sequence
          pseq = []
+ 
+         # iterate while there are servers left to be assigned
          while len(pool) > 0:
              if len(seed) < 8:
-                 if len(pseq) < len(hash) / 4: # first time around
+                 # ran out of digits in the seed
+                 if len(pseq) < len(hash) / 4:
+                     # the number of servers added to the probe sequence is less
+                     # than the number of 4-digit slices in 'hash' so refill the
+                     # seed with the last 4 digits and then append the contents
+                     # of 'hash'.
                      seed = hash[-4:] + hash
                  else:
+                     # refill the seed with the contents of 'hash'
                      seed += hash
+ 
+             # Take the next 8 digits (32 bytes) and interpret as an integer,
+             # then modulus with the size of the remaining pool to get the next
+             # selected server.
              probe = int(seed[0:8], 16) % len(pool)
+ 
+             print seed[0:8], int(seed[0:8], 16), len(pool), probe
+ 
+             # Append the selected server to the probe sequence and remove it
+             # from the pool.
              pseq += [pool[probe]]
              pool = pool[:probe] + pool[probe+1:]
+ 
+             # Remove the digits just used from the seed
              seed = seed[8:]
          logging.debug(str(pseq))
          return pseq