apps/workbench/app/assets/javascripts/upload_to_collection.js

   1 var app = angular.module('Workbench', ['Arvados']);
   2 app.controller('UploadToCollection', UploadToCollection);
   3 app.directive('arvUuid', arvUuid);
   4
   5 function arvUuid() {
   6     // Copy the given uuid into the current $scope.
   7     return {
   8         restrict: 'A',
   9         link: function(scope, element, attributes) {
  10             scope.uuid = attributes.arvUuid;
  11         }
  12     };
  13 }
  14
  15 UploadToCollection.$inject = ['$scope', '$filter', '$q', '$timeout',
  16                               'ArvadosClient', 'arvadosApiToken'];
  17 function UploadToCollection($scope, $filter, $q, $timeout,
  18                             ArvadosClient, arvadosApiToken) {
  19     $.extend($scope, {
  20         uploadQueue: [],
  21         uploader: new QueueUploader(),
  22         addFilesToQueue: function(files) {
  23             // Angular binding doesn't work its usual magic for file
  24             // inputs, so we need to $scope.$apply() this update.
  25             $scope.$apply(function(){
  26                 var i, nItemsTodo;
  27                 // Add these new files after the items already waiting
  28                 // in the queue -- but before the items that are
  29                 // 'Done' and have therefore been pushed to the
  30                 // bottom.
  31                 for (nItemsTodo = 0;
  32                      (nItemsTodo < $scope.uploadQueue.length &&
  33                       $scope.uploadQueue[nItemsTodo].state !== 'Done'); ) {
  34                     nItemsTodo++;
  35                 }
  36                 for (i=0; i<files.length; i++) {
  37                     $scope.uploadQueue.splice(nItemsTodo+i, 0,
  38                         new FileUploader(files[i]));
  39                 }
  40             });
  41         },
  42         go: function() {
  43             $scope.uploader.go();
  44         },
  45         stop: function() {
  46             $scope.uploader.stop();
  47         },
  48         removeFileFromQueue: function(index) {
  49             var wasRunning = $scope.uploader.running;
  50             $scope.uploadQueue[index].stop();
  51             $scope.uploadQueue.splice(index, 1);
  52             if (wasRunning)
  53                 $scope.go();
  54         },
  55         countInStates: function(want_states) {
  56             var found = 0;
  57             $.each($scope.uploadQueue, function() {
  58                 if (want_states.indexOf(this.state) >= 0) {
  59                     ++found;
  60                 }
  61             });
  62             return found;
  63         }
  64     });
  65     ////////////////////////////////
  66
  67     var keepProxy;
  68
  69     function SliceReader(_slice) {
  70         var that = this;
  71         $.extend(this, {
  72             go: go
  73         });
  74         ////////////////////////////////
  75         var _deferred;
  76         var _reader;
  77         function go() {
  78             // Return a promise, which will be resolved with the
  79             // requested slice data.
  80             _deferred = $.Deferred();
  81             _reader = new FileReader();
  82             _reader.onload = resolve;
  83             _reader.onerror = _deferred.reject;
  84             _reader.onprogress = _deferred.notify;
  85             _reader.readAsArrayBuffer(_slice.blob);
  86             return _deferred.promise();
  87         }
  88         function resolve() {
  89             if (that._reader.result.length !== that._slice.size) {
  90                 // Sometimes we get an onload event even if the read
  91                 // did not return the desired number of bytes. We
  92                 // treat that as a fail.
  93                 _deferred.reject(
  94                     null, "Read error",
  95                     "Short read: wanted " + _slice.size +
  96                         ", received " + _reader.result.length);
  97                 return;
  98             }
  99             return _deferred.resolve(_reader.result);
 100         }
 101     }
 102
 103     function SliceUploader(_label, _data, _dataSize) {
 104         $.extend(this, {
 105             go: go,
 106             stop: stop
 107         });
 108         ////////////////////////////////
 109         var that = this;
 110         var _deferred;
 111         var _failCount = 0;
 112         var _failMax = 3;
 113         var _jqxhr;
 114         function go() {
 115             // Send data to the Keep proxy. Retry a few times on
 116             // fail. Return a promise that will get resolved with
 117             // resolve(locator) when the block is accepted by the
 118             // proxy.
 119             _deferred = $.Deferred();
 120             goSend();
 121             return _deferred.promise();
 122         }
 123         function stop() {
 124             _failMax = 0;
 125             _jqxhr.abort();
 126             _deferred.reject({
 127                 textStatus: 'stopped',
 128                 err: 'interrupted at slice '+_label
 129             });
 130         }
 131         function goSend() {
 132             _jqxhr = $.ajax({
 133                 url: proxyUriBase(),
 134                 type: 'POST',
 135                 crossDomain: true,
 136                 headers: {
 137                     'Authorization': 'OAuth2 '+arvadosApiToken,
 138                     'Content-Type': 'application/octet-stream',
 139                     'X-Keep-Desired-Replicas': '2'
 140                 },
 141                 xhr: function() {
 142                     // Make an xhr that reports upload progress
 143                     var xhr = $.ajaxSettings.xhr();
 144                     if (xhr.upload) {
 145                         xhr.upload.onprogress = onSendProgress;
 146                     }
 147                     return xhr;
 148                 },
 149                 processData: false,
 150                 data: _data
 151             });
 152             _jqxhr.then(onSendResolve, onSendReject);
 153         }
 154         function onSendProgress(xhrProgressEvent) {
 155             _deferred.notify(xhrProgressEvent.loaded, _dataSize);
 156         }
 157         function onSendResolve(data, textStatus, jqxhr) {
 158             _deferred.resolve(data, _dataSize);
 159         }
 160         function onSendReject(xhr, textStatus, err) {
 161             if (++_failCount < _failMax) {
 162                 // TODO: nice to tell the user that retry is happening.
 163                 console.log('slice ' + _label + ': ' +
 164                             textStatus + ', retry ' + _failCount);
 165                 goSend();
 166             } else {
 167                 _deferred.reject(
 168                     {xhr: xhr, textStatus: textStatus, err: err});
 169             }
 170         }
 171         function proxyUriBase() {
 172             return ((keepProxy.service_ssl_flag ? 'https' : 'http') +
 173                     '://' + keepProxy.service_host + ':' +
 174                     keepProxy.service_port + '/');
 175         }
 176     }
 177
 178     function FileUploader(file) {
 179         $.extend(this, {
 180             file: file,
 181             locators: [],
 182             progress: 0.0,
 183             state: 'Queued',    // Queued, Uploading, Paused, Uploaded, Done
 184             statistics: null,
 185             go: go,
 186             stop: stop          // User wants to stop.
 187         });
 188         ////////////////////////////////
 189         var that = this;
 190         var _currentUploader;
 191         var _currentSlice;
 192         var _deferred;
 193         var _maxBlobSize = Math.pow(2,26);
 194         var _bytesDone = 0;
 195         var _queueTime = Date.now();
 196         var _startTime;
 197         var _startByte;
 198         var _finishTime;
 199         var _readPos = 0;       // number of bytes confirmed uploaded
 200         function go() {
 201             if (_deferred)
 202                 _deferred.reject({textStatus: 'restarted'});
 203             _deferred = $.Deferred();
 204             that.state = 'Uploading';
 205             _startTime = Date.now();
 206             _startByte = _readPos;
 207             setProgress();
 208             goSlice();
 209             return _deferred.promise().always(function() { _deferred = null; });
 210         }
 211         function stop() {
 212             if (_deferred) {
 213                 that.state = 'Paused';
 214                 _deferred.reject({textStatus: 'stopped', err: 'interrupted'});
 215             }
 216             if (_currentUploader) {
 217                 _currentUploader.stop();
 218                 _currentUploader = null;
 219             }
 220         }
 221         function goSlice() {
 222             // Ensure this._deferred gets resolved or rejected --
 223             // either right here, or when a new promise arranged right
 224             // here is fulfilled.
 225             _currentSlice = nextSlice();
 226             if (!_currentSlice) {
 227                 // All slices have been uploaded, but the work won't
 228                 // be truly Done until the target collection has been
 229                 // updated by the QueueUploader. This state is called:
 230                 that.state = 'Uploaded';
 231                 setProgress(_readPos);
 232                 _currentUploader = null;
 233                 _deferred.resolve([that]);
 234                 return;
 235             }
 236             _currentUploader = new SliceUploader(
 237                 _readPos.toString(),
 238                 _currentSlice.blob,
 239                 _currentSlice.size);
 240             _currentUploader.go().then(
 241                 onUploaderResolve,
 242                 onUploaderReject,
 243                 onUploaderProgress);
 244         }
 245         function onUploaderResolve(locator, dataSize) {
 246             var sizeHint = (''+locator).split('+')[1];
 247             if (!locator || parseInt(sizeHint) !== dataSize) {
 248                 console.log("onUploaderResolve, but locator '" + locator +
 249                             "' with size hint '" + sizeHint +
 250                             "' does not look right for dataSize=" + dataSize);
 251                 return onUploaderReject({
 252                     textStatus: "error",
 253                     err: "Bad response from slice upload"
 254                 });
 255             }
 256             that.locators.push(locator);
 257             _readPos += dataSize;
 258             _currentUploader = null;
 259             goSlice();
 260         }
 261         function onUploaderReject(reason) {
 262             that.state = 'Paused';
 263             setProgress(_readPos);
 264             _currentUploader = null;
 265             _deferred.reject(reason);
 266         }
 267         function onUploaderProgress(sliceDone, sliceSize) {
 268             setProgress(_readPos + sliceDone);
 269         }
 270         function nextSlice() {
 271             var size = Math.min(
 272                 _maxBlobSize,
 273                 that.file.size - _readPos);
 274             setProgress(_readPos);
 275             if (size === 0) {
 276                 return false;
 277             }
 278             var blob = that.file.slice(
 279                 _readPos, _readPos+size,
 280                 'application/octet-stream; charset=x-user-defined');
 281             return {blob: blob, size: size};
 282         }
 283         function setProgress(bytesDone) {
 284             var kBps;
 285             that.progress = Math.min(100, 100 * bytesDone / that.file.size)
 286             if (bytesDone > _startByte) {
 287                 kBps = (bytesDone - _startByte) /
 288                     (Date.now() - _startTime);
 289                 that.statistics = (
 290                     '' + $filter('number')(bytesDone/1024, '0') + ' KiB ' +
 291                         'at ~' + $filter('number')(kBps, '0') + ' KiB/s')
 292                 if (that.state === 'Paused') {
 293                     that.statistics += ', paused';
 294                 } else if (that.state === 'Uploading') {
 295                     that.statistics += ', ETA ' +
 296                         $filter('date')(
 297                             new Date(
 298                                 Date.now() + (that.file.size - bytesDone) / kBps),
 299                             'shortTime')
 300                 }
 301             } else {
 302                 that.statistics = that.state;
 303             }
 304             if (that.state === 'Uploading' || that.state === 'Uploaded') {
 305                 // 'Uploaded' gets reported as 'finished', which is a
 306                 // little misleading because the collection hasn't
 307                 // been updated yet. But FileUploader's portion of the
 308                 // work (and the time when it makes sense to show
 309                 // speed and ETA) is finished.
 310                 that.statistics += ', finished ' +
 311                     $filter('date')(Date.now(), 'shortTime');
 312                 _finishTime = Date.now();
 313             }
 314             _deferred.notify();
 315         }
 316     }
 317
 318     function QueueUploader() {
 319         $.extend(this, {
 320             state: 'Idle',
 321             stateReason: null,
 322             statusSuccess: null,
 323             go: go,
 324             stop: stop
 325         });
 326         ////////////////////////////////
 327         var that = this;
 328         var _deferred;          // the one we promise to go()'s caller
 329         var _deferredAppend;    // tracks current appendToCollection
 330         function go() {
 331             if (_deferred) return _deferred.promise();
 332             if (_deferredAppend) return _deferredAppend.promise();
 333             _deferred = $.Deferred();
 334             that.state = 'Running';
 335             ArvadosClient.apiPromise(
 336                 'keep_services', 'list',
 337                 {filters: [['service_type','=','proxy']]}).
 338                 then(doQueueWithProxy);
 339             onQueueProgress();
 340             return _deferred.promise().always(function() { _deferred = null; });
 341         }
 342         function stop() {
 343             that.state = 'Stopped';
 344             if (_deferred) {
 345                 _deferred.reject({});
 346             }
 347             if (_deferredAppend) {
 348                 _deferredAppend.reject({});
 349             }
 350             for (var i=0; i<$scope.uploadQueue.length; i++)
 351                 $scope.uploadQueue[i].stop();
 352             onQueueProgress();
 353         }
 354         function doQueueWithProxy(data) {
 355             keepProxy = data.items[0];
 356             if (!keepProxy) {
 357                 that.state = 'Failed';
 358                 that.stateReason =
 359                     'There seems to be no Keep proxy service available.';
 360                 _deferred.reject(null, 'error', that.stateReason);
 361                 return;
 362             }
 363             return doQueueWork();
 364         }
 365         function doQueueWork() {
 366             if (!_deferred) {
 367                 // Queue work has been stopped.
 368                 return;
 369             }
 370             that.stateReason = null;
 371             // If anything is not Done, do it.
 372             if ($scope.uploadQueue.length > 0 &&
 373                 $scope.uploadQueue[0].state !== 'Done') {
 374                 return $scope.uploadQueue[0].go().
 375                     then(appendToCollection, null, onQueueProgress).
 376                     then(doQueueWork, onQueueReject);
 377             }
 378             // If everything is Done, resolve the promise and clean up.
 379             return onQueueResolve();
 380         }
 381         function onQueueReject(reason) {
 382             if (!_deferred) {
 383                 // Outcome has already been decided (by stop()).
 384                 return;
 385             }
 386
 387             that.stateReason = (
 388                 (reason.textStatus || 'Error') +
 389                     (reason.xhr && reason.xhr.options
 390                      ? (' (from ' + reason.xhr.options.url + ')')
 391                      : '') +
 392                     ': ' +
 393                     (reason.err || ''));
 394             if (reason.xhr && reason.xhr.responseText)
 395                 that.stateReason += ' -- ' + reason.xhr.responseText;
 396             _deferred.reject(reason);
 397             onQueueProgress();
 398         }
 399         function onQueueResolve() {
 400             that.state = 'Idle';
 401             that.stateReason = 'Done!';
 402             _deferred.resolve();
 403             onQueueProgress();
 404         }
 405         function onQueueProgress() {
 406             // Ensure updates happen after FileUpload promise callbacks.
 407             $timeout(function(){$scope.$apply();});
 408         }
 409         function appendToCollection(uploads) {
 410             _deferredAppend = $.Deferred();
 411             ArvadosClient.apiPromise(
 412                 'collections', 'get',
 413                 { uuid: $scope.uuid }).
 414                 then(function(collection) {
 415                     var manifestText = '';
 416                     $.each(uploads, function(_, upload) {
 417                         filename = ArvadosClient.uniqueNameForManifest(
 418                             collection.manifest_text,
 419                             '.', upload.file.name);
 420                         collection.manifest_text += '. ' +
 421                             upload.locators.join(' ') +
 422                             ' 0:' + upload.file.size.toString() + ':' +
 423                             filename +
 424                             '\n';
 425                     });
 426                     return ArvadosClient.apiPromise(
 427                         'collections', 'update',
 428                         { uuid: $scope.uuid,
 429                           collection:
 430                           { manifest_text:
 431                             collection.manifest_text }
 432                         });
 433                 }).
 434                 then(function() {
 435                     // Mark the completed upload(s) as Done and push
 436                     // them to the bottom of the queue.
 437                     var i, qLen = $scope.uploadQueue.length;
 438                     for (i=0; i<qLen; i++) {
 439                         if (uploads.indexOf($scope.uploadQueue[i]) >= 0) {
 440                             $scope.uploadQueue[i].state = 'Done';
 441                             $scope.uploadQueue.push.apply(
 442                                 $scope.uploadQueue,
 443                                 $scope.uploadQueue.splice(i, 1));
 444                             --i;
 445                             --qLen;
 446                         }
 447                     }
 448                 }).
 449                 then(_deferredAppend.resolve,
 450                      _deferredAppend.reject).
 451                 always(function() {
 452                     _deferredAppend = null;
 453                 });
 454             return _deferredAppend.promise();
 455         }
 456     }
 457 }