apps/workbench/app/assets/javascripts/upload_to_collection.js

   1 var app = angular.module('Workbench', ['Arvados']);
   2 app.controller('UploadToCollection', UploadToCollection);
   3 app.directive('arvUuid', arvUuid);
   4
   5 function arvUuid() {
   6     // Copy the given uuid into the current $scope.
   7     return {
   8         restrict: 'A',
   9         link: function(scope, element, attributes) {
  10             scope.uuid = attributes.arvUuid;
  11         }
  12     };
  13 }
  14
  15 UploadToCollection.$inject = ['$scope', '$filter', '$q', '$timeout',
  16                               'ArvadosClient', 'arvadosApiToken'];
  17 function UploadToCollection($scope, $filter, $q, $timeout,
  18                             ArvadosClient, arvadosApiToken) {
  19     $.extend($scope, {
  20         uploadQueue: [],
  21         uploader: new QueueUploader(),
  22         addFilesToQueue: function(files) {
  23             // Angular binding doesn't work its usual magic for file
  24             // inputs, so we need to $scope.$apply() this update.
  25             $scope.$apply(function(){
  26                 var i, nItemsTodo;
  27                 // Add these new files after the items already waiting
  28                 // in the queue -- but before the items that are
  29                 // 'Done' and have therefore been pushed to the
  30                 // bottom.
  31                 for (nItemsTodo = 0;
  32                      (nItemsTodo < $scope.uploadQueue.length &&
  33                       $scope.uploadQueue[nItemsTodo].state !== 'Done'); ) {
  34                     nItemsTodo++;
  35                 }
  36                 for (i=0; i<files.length; i++) {
  37                     $scope.uploadQueue.splice(nItemsTodo+i, 0,
  38                         new FileUploader(files[i]));
  39                 }
  40             });
  41         },
  42         go: function() {
  43             $scope.uploader.go();
  44         },
  45         stop: function() {
  46             $scope.uploader.stop();
  47         },
  48         removeFileFromQueue: function(index) {
  49             var wasRunning = $scope.uploader.running;
  50             $scope.uploadQueue[index].stop();
  51             $scope.uploadQueue.splice(index, 1);
  52             if (wasRunning)
  53                 $scope.go();
  54         },
  55         countInStates: function(want_states) {
  56             var found = 0;
  57             $.each($scope.uploadQueue, function() {
  58                 if (want_states.indexOf(this.state) >= 0) {
  59                     ++found;
  60                 }
  61             });
  62             return found;
  63         }
  64     });
  65     ////////////////////////////////
  66
  67     var keepProxy;
  68
  69     function SliceReader(_slice) {
  70         var that = this;
  71         $.extend(this, {
  72             go: go
  73         });
  74         ////////////////////////////////
  75         var _deferred;
  76         var _reader;
  77         function go() {
  78             // Return a promise, which will be resolved with the
  79             // requested slice data.
  80             _deferred = $.Deferred();
  81             _reader = new FileReader();
  82             _reader.onload = resolve;
  83             _reader.onerror = _deferred.reject;
  84             _reader.onprogress = _deferred.notify;
  85             _reader.readAsArrayBuffer(_slice.blob);
  86             return _deferred.promise();
  87         }
  88         function resolve() {
  89             if (that._reader.result.length !== that._slice.size) {
  90                 // Sometimes we get an onload event even if the read
  91                 // did not return the desired number of bytes. We
  92                 // treat that as a fail.
  93                 _deferred.reject(
  94                     null, "Read error",
  95                     "Short read: wanted " + _slice.size +
  96                         ", received " + _reader.result.length);
  97                 return;
  98             }
  99             return _deferred.resolve(_reader.result);
 100         }
 101     }
 102
 103     function SliceUploader(_label, _data, _dataSize) {
 104         $.extend(this, {
 105             go: go,
 106             stop: stop
 107         });
 108         ////////////////////////////////
 109         var that = this;
 110         var _deferred;
 111         var _failCount = 0;
 112         var _failMax = 3;
 113         var _jqxhr;
 114         function go() {
 115             // Send data to the Keep proxy. Retry a few times on
 116             // fail. Return a promise that will get resolved with
 117             // resolve(locator) when the block is accepted by the
 118             // proxy.
 119             _deferred = $.Deferred();
 120             goSend();
 121             return _deferred.promise();
 122         }
 123         function stop() {
 124             _failMax = 0;
 125             _jqxhr.abort();
 126             _deferred.reject({
 127                 textStatus: 'stopped',
 128                 err: 'interrupted at slice '+_label
 129             });
 130         }
 131         function goSend() {
 132             _jqxhr = $.ajax({
 133                 url: proxyUriBase(),
 134                 type: 'POST',
 135                 crossDomain: true,
 136                 headers: {
 137                     'Authorization': 'OAuth2 '+arvadosApiToken,
 138                     'Content-Type': 'application/octet-stream',
 139                     'X-Keep-Desired-Replicas': '2'
 140                 },
 141                 xhr: function() {
 142                     // Make an xhr that reports upload progress
 143                     var xhr = $.ajaxSettings.xhr();
 144                     if (xhr.upload) {
 145                         xhr.upload.onprogress = onSendProgress;
 146                     }
 147                     return xhr;
 148                 },
 149                 processData: false,
 150                 data: _data
 151             });
 152             _jqxhr.then(onSendResolve, onSendReject);
 153         }
 154         function onSendProgress(xhrProgressEvent) {
 155             _deferred.notify(xhrProgressEvent.loaded, _dataSize);
 156         }
 157         function onSendResolve(data, textStatus, jqxhr) {
 158             _deferred.resolve(data, _dataSize);
 159         }
 160         function onSendReject(xhr, textStatus, err) {
 161             if (++_failCount < _failMax) {
 162                 // TODO: nice to tell the user that retry is happening.
 163                 console.log('slice ' + _label + ': ' +
 164                             textStatus + ', retry ' + _failCount);
 165                 goSend();
 166             } else {
 167                 _deferred.reject(
 168                     {xhr: xhr, textStatus: textStatus, err: err});
 169             }
 170         }
 171         function proxyUriBase() {
 172             return ((keepProxy.service_ssl_flag ? 'https' : 'http') +
 173                     '://' + keepProxy.service_host + ':' +
 174                     keepProxy.service_port + '/');
 175         }
 176     }
 177
 178     function FileUploader(file) {
 179         $.extend(this, {
 180             file: file,
 181             locators: [],
 182             progress: 0.0,
 183             state: 'Queued',    // Queued, Uploading, Paused, Uploaded, Done
 184             statistics: null,
 185             go: go,
 186             stop: stop          // User wants to stop.
 187         });
 188         ////////////////////////////////
 189         var that = this;
 190         var _currentUploader;
 191         var _currentSlice;
 192         var _deferred;
 193         var _maxBlobSize = Math.pow(2,26);
 194         var _bytesDone = 0;
 195         var _queueTime = Date.now();
 196         var _startTime;
 197         var _startByte;
 198         var _finishTime;
 199         var _readPos = 0;       // number of bytes confirmed uploaded
 200         function go() {
 201             if (_deferred)
 202                 _deferred.reject({textStatus: 'restarted'});
 203             _deferred = $.Deferred();
 204             that.state = 'Uploading';
 205             _startTime = Date.now();
 206             _startByte = _readPos;
 207             setProgress();
 208             goSlice();
 209             return _deferred.promise().always(function() { _deferred = null; });
 210         }
 211         function stop() {
 212             if (_deferred) {
 213                 that.state = 'Paused';
 214                 _deferred.reject({textStatus: 'stopped', err: 'interrupted'});
 215             }
 216             if (_currentUploader) {
 217                 _currentUploader.stop();
 218                 _currentUploader = null;
 219             }
 220         }
 221         function goSlice() {
 222             // Ensure this._deferred gets resolved or rejected --
 223             // either right here, or when a new promise arranged right
 224             // here is fulfilled.
 225             _currentSlice = nextSlice();
 226             if (!_currentSlice) {
 227                 // All slices have been uploaded, but the work won't
 228                 // be truly Done until the target collection has been
 229                 // updated by the QueueUploader. This state is called:
 230                 that.state = 'Uploaded';
 231                 setProgress(_readPos);
 232                 _currentUploader = null;
 233                 _deferred.resolve([that]);
 234                 return;
 235             }
 236             _currentUploader = new SliceUploader(
 237                 _readPos.toString(),
 238                 _currentSlice.blob,
 239                 _currentSlice.size);
 240             _currentUploader.go().then(
 241                 onUploaderResolve,
 242                 onUploaderReject,
 243                 onUploaderProgress);
 244         }
 245         function onUploaderResolve(locator, dataSize) {
 246             var sizeHint = (''+locator).split('+')[1];
 247             if (!locator || parseInt(sizeHint) !== dataSize) {
 248                 console.log("onUploaderResolve, but locator '" + locator +
 249                             "' with size hint '" + sizeHint +
 250                             "' does not look right for dataSize=" + dataSize);
 251                 return onUploaderReject({
 252                     textStatus: "error",
 253                     err: "Bad response from slice upload"
 254                 });
 255             }
 256             that.locators.push(locator);
 257             _readPos += dataSize;
 258             _currentUploader = null;
 259             goSlice();
 260         }
 261         function onUploaderReject(reason) {
 262             that.state = 'Paused';
 263             setProgress(_readPos);
 264             _currentUploader = null;
 265             if (_deferred)
 266                 _deferred.reject(reason);
 267         }
 268         function onUploaderProgress(sliceDone, sliceSize) {
 269             setProgress(_readPos + sliceDone);
 270         }
 271         function nextSlice() {
 272             var size = Math.min(
 273                 _maxBlobSize,
 274                 that.file.size - _readPos);
 275             setProgress(_readPos);
 276             if (size === 0) {
 277                 return false;
 278             }
 279             var blob = that.file.slice(
 280                 _readPos, _readPos+size,
 281                 'application/octet-stream; charset=x-user-defined');
 282             return {blob: blob, size: size};
 283         }
 284         function setProgress(bytesDone) {
 285             var kBps;
 286             if (that.file.size == 0)
 287                 that.progress = 100;
 288             else
 289                 that.progress = Math.min(100, 100 * bytesDone / that.file.size);
 290             if (bytesDone > _startByte) {
 291                 kBps = (bytesDone - _startByte) /
 292                     (Date.now() - _startTime);
 293                 that.statistics = (
 294                     '' + $filter('number')(bytesDone/1024, '0') + ' KiB ' +
 295                         'at ~' + $filter('number')(kBps, '0') + ' KiB/s')
 296                 if (that.state === 'Paused') {
 297                     that.statistics += ', paused';
 298                 } else if (that.state === 'Uploading') {
 299                     that.statistics += ', ETA ' +
 300                         $filter('date')(
 301                             new Date(
 302                                 Date.now() + (that.file.size - bytesDone) / kBps),
 303                             'shortTime')
 304                 }
 305             } else {
 306                 that.statistics = that.state;
 307             }
 308             if (that.state === 'Uploaded') {
 309                 // 'Uploaded' gets reported as 'finished', which is a
 310                 // little misleading because the collection hasn't
 311                 // been updated yet. But FileUploader's portion of the
 312                 // work (and the time when it makes sense to show
 313                 // speed and ETA) is finished.
 314                 that.statistics += ', finished ' +
 315                     $filter('date')(Date.now(), 'shortTime');
 316                 _finishTime = Date.now();
 317             }
 318             if (_deferred)
 319                 _deferred.notify();
 320         }
 321     }
 322
 323     function QueueUploader() {
 324         $.extend(this, {
 325             state: 'Idle',      // Idle, Running, Stopped, Failed
 326             stateReason: null,
 327             statusSuccess: null,
 328             go: go,
 329             stop: stop
 330         });
 331         ////////////////////////////////
 332         var that = this;
 333         var _deferred;          // the one we promise to go()'s caller
 334         var _deferredAppend;    // tracks current appendToCollection
 335         function go() {
 336             if (_deferred) return _deferred.promise();
 337             if (_deferredAppend) return _deferredAppend.promise();
 338             _deferred = $.Deferred();
 339             that.state = 'Running';
 340             ArvadosClient.apiPromise(
 341                 'keep_services', 'list',
 342                 {filters: [['service_type','=','proxy']]}).
 343                 then(doQueueWithProxy);
 344             onQueueProgress();
 345             return _deferred.promise().always(function() { _deferred = null; });
 346         }
 347         function stop() {
 348             that.state = 'Stopped';
 349             if (_deferred) {
 350                 _deferred.reject({});
 351             }
 352             for (var i=0; i<$scope.uploadQueue.length; i++)
 353                 $scope.uploadQueue[i].stop();
 354             onQueueProgress();
 355         }
 356         function doQueueWithProxy(data) {
 357             keepProxy = data.items[0];
 358             if (!keepProxy) {
 359                 that.state = 'Failed';
 360                 that.stateReason =
 361                     'There seems to be no Keep proxy service available.';
 362                 _deferred.reject(null, 'error', that.stateReason);
 363                 return;
 364             }
 365             return doQueueWork();
 366         }
 367         function doQueueWork() {
 368             // If anything is not Done, do it.
 369             if ($scope.uploadQueue.length > 0 &&
 370                 $scope.uploadQueue[0].state !== 'Done') {
 371                 if (_deferred) {
 372                     that.stateReason = null;
 373                     return $scope.uploadQueue[0].go().
 374                         then(appendToCollection, null, onQueueProgress).
 375                         then(doQueueWork, onQueueReject);
 376                 } else {
 377                     // Queue work has been stopped. Just update the
 378                     // view.
 379                     onQueueProgress();
 380                     return;
 381                 }
 382             }
 383             // If everything is Done, resolve the promise and clean
 384             // up. Note this can happen even after the _deferred
 385             // promise has been rejected: specifically, when stop() is
 386             // called too late to prevent completion of the last
 387             // upload. In that case we want to update state to "Idle",
 388             // rather than leave it at "Stopped".
 389             onQueueResolve();
 390         }
 391         function onQueueReject(reason) {
 392             if (!_deferred) {
 393                 // Outcome has already been decided (by stop()).
 394                 return;
 395             }
 396
 397             that.state = 'Failed';
 398             that.stateReason = (
 399                 (reason.textStatus || 'Error') +
 400                     (reason.xhr && reason.xhr.options
 401                      ? (' (from ' + reason.xhr.options.url + ')')
 402                      : '') +
 403                     ': ' +
 404                     (reason.err || ''));
 405             if (reason.xhr && reason.xhr.responseText)
 406                 that.stateReason += ' -- ' + reason.xhr.responseText;
 407             _deferred.reject(reason);
 408             onQueueProgress();
 409         }
 410         function onQueueResolve() {
 411             that.state = 'Idle';
 412             that.stateReason = 'Done!';
 413             if (_deferred)
 414                 _deferred.resolve();
 415             onQueueProgress();
 416         }
 417         function onQueueProgress() {
 418             // Ensure updates happen after FileUpload promise callbacks.
 419             $timeout(function(){$scope.$apply();});
 420         }
 421         function appendToCollection(uploads) {
 422             _deferredAppend = $.Deferred();
 423             ArvadosClient.apiPromise(
 424                 'collections', 'get',
 425                 { uuid: $scope.uuid }).
 426                 then(function(collection) {
 427                     var manifestText = '';
 428                     $.each(uploads, function(_, upload) {
 429                         var locators = upload.locators;
 430                         if (locators.length === 0) {
 431                             // Every stream must have at least one
 432                             // data locator, even if it is zero bytes
 433                             // long:
 434                             locators = ['d41d8cd98f00b204e9800998ecf8427e+0'];
 435                         }
 436                         filename = ArvadosClient.uniqueNameForManifest(
 437                             collection.manifest_text,
 438                             '.', upload.file.name);
 439                         collection.manifest_text += '. ' +
 440                             locators.join(' ') +
 441                             ' 0:' + upload.file.size.toString() + ':' +
 442                             filename +
 443                             '\n';
 444                     });
 445                     return ArvadosClient.apiPromise(
 446                         'collections', 'update',
 447                         { uuid: $scope.uuid,
 448                           collection:
 449                           { manifest_text:
 450                             collection.manifest_text }
 451                         });
 452                 }).
 453                 then(function() {
 454                     // Mark the completed upload(s) as Done and push
 455                     // them to the bottom of the queue.
 456                     var i, qLen = $scope.uploadQueue.length;
 457                     for (i=0; i<qLen; i++) {
 458                         if (uploads.indexOf($scope.uploadQueue[i]) >= 0) {
 459                             $scope.uploadQueue[i].state = 'Done';
 460                             $scope.uploadQueue.push.apply(
 461                                 $scope.uploadQueue,
 462                                 $scope.uploadQueue.splice(i, 1));
 463                             --i;
 464                             --qLen;
 465                         }
 466                     }
 467                 }).
 468                 then(_deferredAppend.resolve,
 469                      _deferredAppend.reject);
 470             return _deferredAppend.promise().
 471                 always(function() {
 472                     _deferredAppend = null;
 473                 });
 474         }
 475     }
 476 }