apps/workbench/app/assets/javascripts/upload_to_collection.js

   1 var app = angular.module('Workbench', ['Arvados']);
   2 app.controller('UploadToCollection', UploadToCollection);
   3 app.directive('arvUuid', arvUuid);
   4
   5 function arvUuid() {
   6     // Copy the given uuid into the current $scope.
   7     return {
   8         restrict: 'A',
   9         link: function(scope, element, attributes) {
  10             scope.uuid = attributes.arvUuid;
  11         }
  12     };
  13 }
  14
  15 UploadToCollection.$inject = ['$scope', '$filter', '$q', '$timeout',
  16                               'ArvadosClient', 'arvadosApiToken'];
  17 function UploadToCollection($scope, $filter, $q, $timeout,
  18                             ArvadosClient, arvadosApiToken) {
  19     $.extend($scope, {
  20         uploadQueue: [],
  21         uploader: new QueueUploader(),
  22         addFilesToQueue: function(files) {
  23             // Angular binding doesn't work its usual magic for file
  24             // inputs, so we need to $scope.$apply() this update.
  25             $scope.$apply(function(){
  26                 var i;
  27                 var insertAt;
  28                 for (insertAt=0; (insertAt<$scope.uploadQueue.length &&
  29                                   $scope.uploadQueue[insertAt].state != 'Done');
  30                      insertAt++);
  31                 for (i=0; i<files.length; i++) {
  32                     $scope.uploadQueue.splice(insertAt+i, 0,
  33                         new FileUploader(files[i]));
  34                 }
  35             });
  36         },
  37         go: function() {
  38             $scope.uploader.go();
  39         },
  40         stop: function() {
  41             $scope.uploader.stop();
  42         },
  43         removeFileFromQueue: function(index) {
  44             var wasRunning = $scope.uploader.running;
  45             $scope.uploadQueue[index].stop();
  46             $scope.uploadQueue.splice(index, 1);
  47             if (wasRunning)
  48                 $scope.go();
  49         },
  50         countDone: function() {
  51             var done=0;
  52             for (var i=0; i<$scope.uploadQueue.length; i++) {
  53                 if ($scope.uploadQueue[i].state == 'Done') {
  54                     ++done;
  55                 }
  56             }
  57             return done;
  58         }
  59     });
  60     // TODO: watch uploadQueue, abort uploads if entries disappear
  61
  62     var keepProxy;
  63
  64     function SliceReader(_slice) {
  65         var that = this;
  66         $.extend(this, {
  67             go: go
  68         });
  69         ////////////////////////////////
  70         var _deferred;
  71         var _reader;
  72         function go() {
  73             // Return a promise, which will be resolved with the
  74             // requested slice data.
  75             _deferred = $.Deferred();
  76             _reader = new FileReader();
  77             _reader.onload = resolve;
  78             _reader.onerror = _deferred.reject;
  79             _reader.onprogress = _deferred.notify;
  80             _reader.readAsArrayBuffer(_slice.blob);
  81             return _deferred.promise();
  82         }
  83         function resolve() {
  84             if (that._reader.result.length != that._slice.size) {
  85                 // Sometimes we get an onload event even if the read
  86                 // did not return the desired number of bytes. We
  87                 // treat that as a fail.
  88                 _deferred.reject(
  89                     null, "Read error",
  90                     "Short read: wanted " + _slice.size +
  91                         ", received " + _reader.result.length);
  92                 return;
  93             }
  94             return _deferred.resolve(_reader.result);
  95         }
  96     }
  97
  98     function SliceUploader(_label, _data, _dataSize) {
  99         $.extend(this, {
 100             go: go,
 101             stop: stop
 102         });
 103         ////////////////////////////////
 104         var that = this;
 105         var _deferred;
 106         var _failCount = 0;
 107         var _failMax = 3;
 108         var _jqxhr;
 109         function go() {
 110             // Send data to the Keep proxy. Retry a few times on
 111             // fail. Return a promise that will get resolved with
 112             // resolve(locator) when the block is accepted by the
 113             // proxy.
 114             _deferred = $.Deferred();
 115             goSend();
 116             return _deferred.promise();
 117         }
 118         function stop() {
 119             _failMax = 0;
 120             _jqxhr.abort();
 121             _deferred.reject({
 122                 textStatus: 'stopped',
 123                 err: 'interrupted at slice '+_label
 124             });
 125         }
 126         function goSend() {
 127             _jqxhr = $.ajax({
 128                 url: proxyUriBase(),
 129                 type: 'POST',
 130                 crossDomain: true,
 131                 headers: {
 132                     'Authorization': 'OAuth2 '+arvadosApiToken,
 133                     'Content-Type': 'application/octet-stream',
 134                     'X-Keep-Desired-Replicas': '2'
 135                 },
 136                 xhr: function() {
 137                     // Make an xhr that reports upload progress
 138                     var xhr = $.ajaxSettings.xhr();
 139                     if (xhr.upload) {
 140                         xhr.upload.onprogress = onSendProgress;
 141                     }
 142                     return xhr;
 143                 },
 144                 processData: false,
 145                 data: _data
 146             });
 147             _jqxhr.then(onSendResolve, onSendReject);
 148         }
 149         function onSendProgress(xhrProgressEvent) {
 150             _deferred.notify(xhrProgressEvent.loaded, _dataSize);
 151         }
 152         function onSendResolve(data, textStatus, jqxhr) {
 153             _deferred.resolve(data, _dataSize);
 154         }
 155         function onSendReject(xhr, textStatus, err) {
 156             if (++_failCount < _failMax) {
 157                 // TODO: nice to tell the user that retry is happening.
 158                 console.log('slice ' + _label + ': ' +
 159                             textStatus + ', retry ' + _failCount);
 160                 goSend();
 161             } else {
 162                 _deferred.reject(
 163                     {xhr: xhr, textStatus: textStatus, err: err});
 164             }
 165         }
 166         function proxyUriBase() {
 167             return ((keepProxy.service_ssl_flag ? 'https' : 'http') +
 168                     '://' + keepProxy.service_host + ':' +
 169                     keepProxy.service_port + '/');
 170         }
 171     }
 172
 173     function FileUploader(file) {
 174         $.extend(this, {
 175             committed: false,
 176             file: file,
 177             locators: [],
 178             progress: 0.0,
 179             state: 'Queued',    // Queued, Uploading, Paused, Done
 180             statistics: null,
 181             go: go,
 182             stop: stop          // User wants to stop.
 183         });
 184         ////////////////////////////////
 185         var that = this;
 186         var _currentUploader;
 187         var _currentSlice;
 188         var _deferred;
 189         var _maxBlobSize = Math.pow(2,26);
 190         var _bytesDone = 0;
 191         var _queueTime = Date.now();
 192         var _startTime;
 193         var _startByte;
 194         var _finishTime;
 195         var _readPos = 0;       // number of bytes confirmed uploaded
 196         function go() {
 197             if (_deferred)
 198                 _deferred.reject({textStatus: 'restarted'});
 199             _deferred = $q.defer();
 200             that.state = 'Uploading';
 201             _startTime = Date.now();
 202             _startByte = _readPos;
 203             setProgress();
 204             goSlice();
 205             return _deferred.promise;
 206         }
 207         function stop() {
 208             if (_deferred) {
 209                 that.state = 'Paused';
 210                 _deferred.reject({textStatus: 'stopped', err: 'interrupted'});
 211             }
 212             if (_currentUploader) {
 213                 _currentUploader.stop();
 214                 _currentUploader = null;
 215             }
 216         }
 217         function goSlice() {
 218             // Ensure this._deferred gets resolved or rejected --
 219             // either right here, or when a new promise arranged right
 220             // here is fulfilled.
 221             _currentSlice = nextSlice();
 222             if (!_currentSlice) {
 223                 that.state = 'Done';
 224                 setProgress(_readPos);
 225                 _currentUploader = null;
 226                 _deferred.resolve([that]);
 227                 return;
 228             }
 229             _currentUploader = new SliceUploader(
 230                 _readPos.toString(),
 231                 _currentSlice.blob,
 232                 _currentSlice.size);
 233             _currentUploader.go().then(
 234                 onUploaderResolve,
 235                 onUploaderReject,
 236                 onUploaderProgress);
 237         }
 238         function onUploaderResolve(locator, dataSize) {
 239             if (!locator || _currentSlice.size != dataSize) {
 240                 console.log("onUploaderResolve but locator=" + locator +
 241                             " and " + _currentSlice.size + " != " + dataSize);
 242                 return onUploaderReject({
 243                     textStatus: "error",
 244                     err: "Bad response from slice upload"
 245                 });
 246             }
 247             that.locators.push(locator);
 248             _readPos += dataSize;
 249             _currentUploader = null;
 250             goSlice();
 251         }
 252         function onUploaderReject(reason) {
 253             that.state = 'Paused';
 254             setProgress(_readPos);
 255             _currentUploader = null;
 256             _deferred.reject(reason);
 257         }
 258         function onUploaderProgress(sliceDone, sliceSize) {
 259             setProgress(_readPos + sliceDone);
 260         }
 261         function nextSlice() {
 262             var size = Math.min(
 263                 _maxBlobSize,
 264                 that.file.size - _readPos);
 265             setProgress(_readPos);
 266             if (size == 0) {
 267                 return false;
 268             }
 269             var blob = that.file.slice(
 270                 _readPos, _readPos+size,
 271                 'application/octet-stream; charset=x-user-defined');
 272             return {blob: blob, size: size};
 273         }
 274         function setProgress(bytesDone) {
 275             var kBps;
 276             that.progress = Math.min(100, 100 * bytesDone / that.file.size)
 277             if (bytesDone > _startByte) {
 278                 kBps = (bytesDone - _startByte) /
 279                     (Date.now() - _startTime);
 280                 that.statistics = (
 281                     '' + $filter('number')(bytesDone/1024, '0') + 'K ' +
 282                         'at ~' + $filter('number')(kBps, '0') + 'K/s')
 283                 if (that.state == 'Paused') {
 284                     that.statistics += ', paused';
 285                 } else if (that.state == 'Uploading') {
 286                     that.statistics += ', ETA ' +
 287                         $filter('date')(
 288                             new Date(
 289                                 Date.now() + (that.file.size - bytesDone) / kBps),
 290                             'shortTime')
 291                 } else {
 292                     that.statistics += ', finished ' +
 293                         $filter('date')(Date.now(), 'shortTime');
 294                     _finishTime = Date.now();
 295                 }
 296             } else {
 297                 that.statistics = that.state;
 298             }
 299             _deferred.notify();
 300         }
 301     }
 302
 303     function QueueUploader() {
 304         $.extend(this, {
 305             state: 'Idle',
 306             stateReason: null,
 307             statusSuccess: null,
 308             go: go,
 309             stop: stop
 310         });
 311         ////////////////////////////////
 312         var that = this;
 313         var _deferred;
 314         function go() {
 315             if (that.state == 'Running') return _deferred.promise;
 316             _deferred = $.Deferred();
 317             that.state = 'Running';
 318             ArvadosClient.apiPromise(
 319                 'keep_services', 'list',
 320                 {filters: [['service_type','=','proxy']]}).
 321                 then(doQueueWithProxy);
 322             onQueueProgress();
 323             return _deferred.promise();
 324         }
 325         function stop() {
 326             for (var i=0; i<$scope.uploadQueue.length; i++)
 327                 $scope.uploadQueue[i].stop();
 328         }
 329         function doQueueWithProxy(data) {
 330             keepProxy = data.items[0];
 331             if (!keepProxy) {
 332                 that.state = 'Failed';
 333                 that.stateReason =
 334                     'There seems to be no Keep proxy service available.';
 335                 _deferred.reject(null, 'error', that.stateReason);
 336                 return;
 337             }
 338             return doQueueWork();
 339         }
 340         function doQueueWork() {
 341             var i;
 342             that.state = 'Running';
 343             that.stateReason = null;
 344             // Push the done things to the bottom of the queue.
 345             for (i=0; (i<$scope.uploadQueue.length &&
 346                        $scope.uploadQueue[i].state == 'Done'); i++);
 347             if (i>0)
 348                 $scope.uploadQueue.push.apply($scope.uploadQueue, $scope.uploadQueue.splice(0, i));
 349             // If anything is not-done, do it.
 350             if ($scope.uploadQueue.length > 0 &&
 351                 $scope.uploadQueue[0].state != 'Done') {
 352                 return $scope.uploadQueue[0].go().
 353                     then(appendToCollection, null, onQueueProgress).
 354                     then(doQueueWork, onQueueReject);
 355             }
 356             // If everything is done, resolve the promise and clean up.
 357             return onQueueResolve();
 358         }
 359         function onQueueReject(reason) {
 360             that.state = 'Failed';
 361             that.stateReason = (
 362                 (reason.textStatus || 'Error') +
 363                     (reason.xhr && reason.xhr.options
 364                      ? (' (from ' + reason.xhr.options.url + ')')
 365                      : '') +
 366                     ': ' +
 367                     (reason.err || ''));
 368             if (reason.xhr && reason.xhr.responseText)
 369                 that.stateReason += ' -- ' + reason.xhr.responseText;
 370             _deferred.reject(reason);
 371             onQueueProgress();
 372         }
 373         function onQueueResolve() {
 374             that.state = 'Idle';
 375             that.stateReason = 'Done!';
 376             _deferred.resolve();
 377             onQueueProgress();
 378         }
 379         function onQueueProgress() {
 380             // Ensure updates happen after FileUpload promise callbacks.
 381             $timeout(function(){$scope.$apply();});
 382         }
 383         function appendToCollection(uploads) {
 384             var deferred = $q.defer();
 385             return ArvadosClient.apiPromise(
 386                 'collections', 'get',
 387                 { uuid: $scope.uuid }).
 388                 then(function(collection) {
 389                     var manifestText = '';
 390                     var upload, i;
 391                     for (i=0; i<uploads.length; i++) {
 392                         upload = uploads[i];
 393                         filename = ArvadosClient.uniqueNameForManifest(
 394                             collection.manifest_text,
 395                             '.', upload.file.name);
 396                         collection.manifest_text += '. ' +
 397                             upload.locators.join(' ') +
 398                             ' 0:' + upload.file.size.toString() + ':' +
 399                             filename +
 400                             '\n';
 401                     }
 402                     return ArvadosClient.apiPromise(
 403                         'collections', 'update',
 404                         { uuid: $scope.uuid,
 405                           collection:
 406                           { manifest_text:
 407                             collection.manifest_text }
 408                         }).
 409                         then(deferred.resolve);
 410                 }, onQueueReject).then(function() {
 411                     var i;
 412                     for(i=0; i<uploads.length; i++) {
 413                         uploads[i].committed = true;
 414                     }
 415                 });
 416             return deferred.promise.then(doQueueWork);
 417         }
 418     }
 419 }