apps/workbench/app/assets/javascripts/upload_to_collection.js

   1 var app = angular.module('Workbench', ['Arvados']);
   2 app.controller('UploadToCollection', UploadToCollection);
   3 app.directive('arvUuid', arvUuid);
   4
   5 function arvUuid() {
   6     // Copy the given uuid into the current $scope.
   7     return {
   8         restrict: 'A',
   9         link: function(scope, element, attributes) {
  10             scope.uuid = attributes.arvUuid;
  11         }
  12     };
  13 }
  14
  15 UploadToCollection.$inject = ['$scope', '$filter', '$q', '$timeout',
  16                               'ArvadosClient', 'arvadosApiToken'];
  17 function UploadToCollection($scope, $filter, $q, $timeout,
  18                             ArvadosClient, arvadosApiToken) {
  19     $.extend($scope, {
  20         uploadQueue: [],
  21         uploader: new QueueUploader(),
  22         addFilesToQueue: function(files) {
  23             // Angular binding doesn't work its usual magic for file
  24             // inputs, so we need to $scope.$apply() this update.
  25             $scope.$apply(function(){
  26                 var i, nItemsTodo;
  27                 // Add these new files after the items already waiting
  28                 // in the queue -- but before the items that are
  29                 // 'Done' and have therefore been pushed to the
  30                 // bottom.
  31                 for (nItemsTodo = 0;
  32                      (nItemsTodo < $scope.uploadQueue.length &&
  33                       $scope.uploadQueue[nItemsTodo].state !== 'Done'); ) {
  34                     nItemsTodo++;
  35                 }
  36                 for (i=0; i<files.length; i++) {
  37                     $scope.uploadQueue.splice(nItemsTodo+i, 0,
  38                         new FileUploader(files[i]));
  39                 }
  40             });
  41         },
  42         go: function() {
  43             $scope.uploader.go();
  44         },
  45         stop: function() {
  46             $scope.uploader.stop();
  47         },
  48         removeFileFromQueue: function(index) {
  49             var wasRunning = $scope.uploader.running;
  50             $scope.uploadQueue[index].stop();
  51             $scope.uploadQueue.splice(index, 1);
  52             if (wasRunning)
  53                 $scope.go();
  54         },
  55         countInStates: function(want_states) {
  56             var found = 0;
  57             $.each($scope.uploadQueue, function() {
  58                 if (want_states.indexOf(this.state) >= 0) {
  59                     ++found;
  60                 }
  61             });
  62             return found;
  63         }
  64     });
  65     ////////////////////////////////
  66
  67     var keepProxy;
  68     var defaultErrorMessage = 'A network error occurred: either the server was unreachable, or there is a server configuration problem. Please check your browser debug console for a more specific error message (browser security features prevent us from showing the details here).';
  69
  70     function SliceReader(_slice) {
  71         var that = this;
  72         $.extend(this, {
  73             go: go
  74         });
  75         ////////////////////////////////
  76         var _deferred;
  77         var _reader;
  78         function go() {
  79             // Return a promise, which will be resolved with the
  80             // requested slice data.
  81             _deferred = $.Deferred();
  82             _reader = new FileReader();
  83             _reader.onload = resolve;
  84             _reader.onerror = _deferred.reject;
  85             _reader.onprogress = _deferred.notify;
  86             _reader.readAsArrayBuffer(_slice.blob);
  87             return _deferred.promise();
  88         }
  89         function resolve() {
  90             if (that._reader.result.length !== that._slice.size) {
  91                 // Sometimes we get an onload event even if the read
  92                 // did not return the desired number of bytes. We
  93                 // treat that as a fail.
  94                 _deferred.reject(
  95                     null, "Read error",
  96                     "Short read: wanted " + _slice.size +
  97                         ", received " + _reader.result.length);
  98                 return;
  99             }
 100             return _deferred.resolve(_reader.result);
 101         }
 102     }
 103
 104     function SliceUploader(_label, _data, _dataSize) {
 105         $.extend(this, {
 106             go: go,
 107             stop: stop
 108         });
 109         ////////////////////////////////
 110         var that = this;
 111         var _deferred;
 112         var _failCount = 0;
 113         var _failMax = 3;
 114         var _jqxhr;
 115         function go() {
 116             // Send data to the Keep proxy. Retry a few times on
 117             // fail. Return a promise that will get resolved with
 118             // resolve(locator) when the block is accepted by the
 119             // proxy.
 120             _deferred = $.Deferred();
 121             if (proxyUriBase().match(/^http:/) &&
 122                 window.location.origin.match(/^https:/)) {
 123                 // In this case, requests will fail, and no ajax
 124                 // success/fail handlers will be called (!), which
 125                 // will leave our status saying "uploading" and the
 126                 // user waiting for something to happen. Better to
 127                 // give up now.
 128                 _deferred.reject({
 129                     textStatus: 'error',
 130                     err: 'There is a server configuration problem. Proxy ' + proxyUriBase() + ' cannot be used from origin ' + window.location.origin + ' due to the browser\'s mixed-content (https/http) policy.'
 131                 });
 132             } else {
 133                 goSend();
 134             }
 135             return _deferred.promise();
 136         }
 137         function stop() {
 138             _failMax = 0;
 139             _jqxhr.abort();
 140             _deferred.reject({
 141                 textStatus: 'stopped',
 142                 err: 'interrupted at slice '+_label
 143             });
 144         }
 145         function goSend() {
 146             _jqxhr = $.ajax({
 147                 url: proxyUriBase(),
 148                 type: 'POST',
 149                 crossDomain: true,
 150                 headers: {
 151                     'Authorization': 'OAuth2 '+arvadosApiToken,
 152                     'Content-Type': 'application/octet-stream',
 153                     'X-Keep-Desired-Replicas': '2'
 154                 },
 155                 xhr: function() {
 156                     // Make an xhr that reports upload progress
 157                     var xhr = $.ajaxSettings.xhr();
 158                     if (xhr.upload) {
 159                         xhr.upload.onprogress = onSendProgress;
 160                     }
 161                     return xhr;
 162                 },
 163                 processData: false,
 164                 data: _data
 165             });
 166             _jqxhr.then(onSendResolve, onSendReject);
 167         }
 168         function onSendProgress(xhrProgressEvent) {
 169             _deferred.notify(xhrProgressEvent.loaded, _dataSize);
 170         }
 171         function onSendResolve(data, textStatus, jqxhr) {
 172             _deferred.resolve(data, _dataSize);
 173         }
 174         function onSendReject(xhr, textStatus, err) {
 175             if (++_failCount < _failMax) {
 176                 // TODO: nice to tell the user that retry is happening.
 177                 console.log('slice ' + _label + ': ' +
 178                             textStatus + ', retry ' + _failCount);
 179                 goSend();
 180             } else {
 181                 _deferred.reject(
 182                     {xhr: xhr, textStatus: textStatus, err: err});
 183             }
 184         }
 185         function proxyUriBase() {
 186             return ((keepProxy.service_ssl_flag ? 'https' : 'http') +
 187                     '://' + keepProxy.service_host + ':' +
 188                     keepProxy.service_port + '/');
 189         }
 190     }
 191
 192     function FileUploader(file) {
 193         $.extend(this, {
 194             file: file,
 195             locators: [],
 196             progress: 0.0,
 197             state: 'Queued',    // Queued, Uploading, Paused, Uploaded, Done
 198             statistics: null,
 199             go: go,
 200             stop: stop          // User wants to stop.
 201         });
 202         ////////////////////////////////
 203         var that = this;
 204         var _currentUploader;
 205         var _currentSlice;
 206         var _deferred;
 207         var _maxBlobSize = Math.pow(2,26);
 208         var _bytesDone = 0;
 209         var _queueTime = Date.now();
 210         var _startTime;
 211         var _startByte;
 212         var _finishTime;
 213         var _readPos = 0;       // number of bytes confirmed uploaded
 214         function go() {
 215             if (_deferred)
 216                 _deferred.reject({textStatus: 'restarted'});
 217             _deferred = $.Deferred();
 218             that.state = 'Uploading';
 219             _startTime = Date.now();
 220             _startByte = _readPos;
 221             setProgress();
 222             goSlice();
 223             return _deferred.promise().always(function() { _deferred = null; });
 224         }
 225         function stop() {
 226             if (_deferred) {
 227                 that.state = 'Paused';
 228                 _deferred.reject({textStatus: 'stopped', err: 'interrupted'});
 229             }
 230             if (_currentUploader) {
 231                 _currentUploader.stop();
 232                 _currentUploader = null;
 233             }
 234         }
 235         function goSlice() {
 236             // Ensure this._deferred gets resolved or rejected --
 237             // either right here, or when a new promise arranged right
 238             // here is fulfilled.
 239             _currentSlice = nextSlice();
 240             if (!_currentSlice) {
 241                 // All slices have been uploaded, but the work won't
 242                 // be truly Done until the target collection has been
 243                 // updated by the QueueUploader. This state is called:
 244                 that.state = 'Uploaded';
 245                 setProgress(_readPos);
 246                 _currentUploader = null;
 247                 _deferred.resolve([that]);
 248                 return;
 249             }
 250             _currentUploader = new SliceUploader(
 251                 _readPos.toString(),
 252                 _currentSlice.blob,
 253                 _currentSlice.size);
 254             _currentUploader.go().then(
 255                 onUploaderResolve,
 256                 onUploaderReject,
 257                 onUploaderProgress);
 258         }
 259         function onUploaderResolve(locator, dataSize) {
 260             var sizeHint = (''+locator).split('+')[1];
 261             if (!locator || parseInt(sizeHint) !== dataSize) {
 262                 console.log("onUploaderResolve, but locator '" + locator +
 263                             "' with size hint '" + sizeHint +
 264                             "' does not look right for dataSize=" + dataSize);
 265                 return onUploaderReject({
 266                     textStatus: "error",
 267                     err: "Bad response from slice upload"
 268                 });
 269             }
 270             that.locators.push(locator);
 271             _readPos += dataSize;
 272             _currentUploader = null;
 273             goSlice();
 274         }
 275         function onUploaderReject(reason) {
 276             that.state = 'Paused';
 277             setProgress(_readPos);
 278             _currentUploader = null;
 279             if (_deferred)
 280                 _deferred.reject(reason);
 281         }
 282         function onUploaderProgress(sliceDone, sliceSize) {
 283             setProgress(_readPos + sliceDone);
 284         }
 285         function nextSlice() {
 286             var size = Math.min(
 287                 _maxBlobSize,
 288                 that.file.size - _readPos);
 289             setProgress(_readPos);
 290             if (size === 0) {
 291                 return false;
 292             }
 293             var blob = that.file.slice(
 294                 _readPos, _readPos+size,
 295                 'application/octet-stream; charset=x-user-defined');
 296             return {blob: blob, size: size};
 297         }
 298         function setProgress(bytesDone) {
 299             var kBps;
 300             if (that.file.size == 0)
 301                 that.progress = 100;
 302             else
 303                 that.progress = Math.min(100, 100 * bytesDone / that.file.size);
 304             if (bytesDone > _startByte) {
 305                 kBps = (bytesDone - _startByte) /
 306                     (Date.now() - _startTime);
 307                 that.statistics = (
 308                     '' + $filter('number')(bytesDone/1024, '0') + ' KiB ' +
 309                         'at ~' + $filter('number')(kBps, '0') + ' KiB/s')
 310                 if (that.state === 'Paused') {
 311                     that.statistics += ', paused';
 312                 } else if (that.state === 'Uploading') {
 313                     that.statistics += ', ETA ' +
 314                         $filter('date')(
 315                             new Date(
 316                                 Date.now() + (that.file.size - bytesDone) / kBps),
 317                             'shortTime')
 318                 }
 319             } else {
 320                 that.statistics = that.state;
 321             }
 322             if (that.state === 'Uploaded') {
 323                 // 'Uploaded' gets reported as 'finished', which is a
 324                 // little misleading because the collection hasn't
 325                 // been updated yet. But FileUploader's portion of the
 326                 // work (and the time when it makes sense to show
 327                 // speed and ETA) is finished.
 328                 that.statistics += ', finished ' +
 329                     $filter('date')(Date.now(), 'shortTime');
 330                 _finishTime = Date.now();
 331             }
 332             if (_deferred)
 333                 _deferred.notify();
 334         }
 335     }
 336
 337     function QueueUploader() {
 338         $.extend(this, {
 339             state: 'Idle',      // Idle, Running, Stopped, Failed
 340             stateReason: null,
 341             statusSuccess: null,
 342             go: go,
 343             stop: stop
 344         });
 345         ////////////////////////////////
 346         var that = this;
 347         var _deferred;          // the one we promise to go()'s caller
 348         var _deferredAppend;    // tracks current appendToCollection
 349         function go() {
 350             if (_deferred) return _deferred.promise();
 351             if (_deferredAppend) return _deferredAppend.promise();
 352             _deferred = $.Deferred();
 353             that.state = 'Running';
 354             ArvadosClient.apiPromise(
 355                 'keep_services', 'list',
 356                 {filters: [['service_type','=','proxy']]}).
 357                 then(doQueueWithProxy);
 358             onQueueProgress();
 359             return _deferred.promise().always(function() { _deferred = null; });
 360         }
 361         function stop() {
 362             that.state = 'Stopped';
 363             if (_deferred) {
 364                 _deferred.reject({});
 365             }
 366             for (var i=0; i<$scope.uploadQueue.length; i++)
 367                 $scope.uploadQueue[i].stop();
 368             onQueueProgress();
 369         }
 370         function doQueueWithProxy(data) {
 371             keepProxy = data.items[0];
 372             if (!keepProxy) {
 373                 that.state = 'Failed';
 374                 that.stateReason =
 375                     'There seems to be no Keep proxy service available.';
 376                 _deferred.reject(null, 'error', that.stateReason);
 377                 return;
 378             }
 379             return doQueueWork();
 380         }
 381         function doQueueWork() {
 382             // If anything is not Done, do it.
 383             if ($scope.uploadQueue.length > 0 &&
 384                 $scope.uploadQueue[0].state !== 'Done') {
 385                 if (_deferred) {
 386                     that.stateReason = null;
 387                     return $scope.uploadQueue[0].go().
 388                         then(appendToCollection, null, onQueueProgress).
 389                         then(doQueueWork, onQueueReject);
 390                 } else {
 391                     // Queue work has been stopped. Just update the
 392                     // view.
 393                     onQueueProgress();
 394                     return;
 395                 }
 396             }
 397             // If everything is Done, resolve the promise and clean
 398             // up. Note this can happen even after the _deferred
 399             // promise has been rejected: specifically, when stop() is
 400             // called too late to prevent completion of the last
 401             // upload. In that case we want to update state to "Idle",
 402             // rather than leave it at "Stopped".
 403             onQueueResolve();
 404         }
 405         function onQueueReject(reason) {
 406             if (!_deferred) {
 407                 // Outcome has already been decided (by stop()).
 408                 return;
 409             }
 410
 411             that.state = 'Failed';
 412             that.stateReason = (
 413                 (reason.textStatus || 'Error') +
 414                     (reason.xhr && reason.xhr.options
 415                      ? (' (from ' + reason.xhr.options.url + ')')
 416                      : '') +
 417                     ': ' +
 418                     (reason.err || defaultErrorMessage));
 419             if (reason.xhr && reason.xhr.responseText)
 420                 that.stateReason += ' -- ' + reason.xhr.responseText;
 421             _deferred.reject(reason);
 422             onQueueProgress();
 423         }
 424         function onQueueResolve() {
 425             that.state = 'Idle';
 426             that.stateReason = 'Done!';
 427             if (_deferred)
 428                 _deferred.resolve();
 429             onQueueProgress();
 430         }
 431         function onQueueProgress() {
 432             // Ensure updates happen after FileUpload promise callbacks.
 433             $timeout(function(){$scope.$apply();});
 434         }
 435         function appendToCollection(uploads) {
 436             _deferredAppend = $.Deferred();
 437             ArvadosClient.apiPromise(
 438                 'collections', 'get',
 439                 { uuid: $scope.uuid }).
 440                 then(function(collection) {
 441                     var manifestText = '';
 442                     $.each(uploads, function(_, upload) {
 443                         var locators = upload.locators;
 444                         if (locators.length === 0) {
 445                             // Every stream must have at least one
 446                             // data locator, even if it is zero bytes
 447                             // long:
 448                             locators = ['d41d8cd98f00b204e9800998ecf8427e+0'];
 449                         }
 450                         filename = ArvadosClient.uniqueNameForManifest(
 451                             collection.manifest_text,
 452                             '.', upload.file.name);
 453                         collection.manifest_text += '. ' +
 454                             locators.join(' ') +
 455                             ' 0:' + upload.file.size.toString() + ':' +
 456                             filename +
 457                             '\n';
 458                     });
 459                     return ArvadosClient.apiPromise(
 460                         'collections', 'update',
 461                         { uuid: $scope.uuid,
 462                           collection:
 463                           { manifest_text:
 464                             collection.manifest_text }
 465                         });
 466                 }).
 467                 then(function() {
 468                     // Mark the completed upload(s) as Done and push
 469                     // them to the bottom of the queue.
 470                     var i, qLen = $scope.uploadQueue.length;
 471                     for (i=0; i<qLen; i++) {
 472                         if (uploads.indexOf($scope.uploadQueue[i]) >= 0) {
 473                             $scope.uploadQueue[i].state = 'Done';
 474                             $scope.uploadQueue.push.apply(
 475                                 $scope.uploadQueue,
 476                                 $scope.uploadQueue.splice(i, 1));
 477                             --i;
 478                             --qLen;
 479                         }
 480                     }
 481                 }).
 482                 then(_deferredAppend.resolve,
 483                      _deferredAppend.reject);
 484             return _deferredAppend.promise().
 485                 always(function() {
 486                     _deferredAppend = null;
 487                 });
 488         }
 489     }
 490 }