apps/workbench/app/assets/javascripts/upload_to_collection.js

   1 var app = angular.module('Workbench', ['Arvados']);
   2 app.controller('UploadToCollection', UploadToCollection);
   3 app.directive('arvUuid', arvUuid);
   4
   5 function arvUuid() {
   6     // Copy the given uuid into the current $scope.
   7     return {
   8         restrict: 'A',
   9         link: function(scope, element, attributes) {
  10             scope.uuid = attributes.arvUuid;
  11         }
  12     };
  13 }
  14
  15 UploadToCollection.$inject = ['$scope', '$filter', '$q', '$timeout',
  16                               'ArvadosClient', 'arvadosApiToken'];
  17 function UploadToCollection($scope, $filter, $q, $timeout,
  18                             ArvadosClient, arvadosApiToken) {
  19     $.extend($scope, {
  20         uploadQueue: [],
  21         uploader: new QueueUploader(),
  22         addFilesToQueue: function(files) {
  23             // Angular binding doesn't work its usual magic for file
  24             // inputs, so we need to $scope.$apply() this update.
  25             $scope.$apply(function(){
  26                 var i;
  27                 var insertAt;
  28                 for (insertAt=0; (insertAt<$scope.uploadQueue.length &&
  29                                   $scope.uploadQueue[insertAt].state != 'Done');
  30                      insertAt++);
  31                 for (i=0; i<files.length; i++) {
  32                     $scope.uploadQueue.splice(insertAt+i, 0,
  33                         new FileUploader(files[i]));
  34                 }
  35             });
  36         },
  37         go: function() {
  38             $scope.uploader.go();
  39         },
  40         stop: function() {
  41             $scope.uploader.stop();
  42         },
  43         removeFileFromQueue: function(index) {
  44             var wasRunning = $scope.uploader.running;
  45             $scope.uploadQueue[index].stop();
  46             $scope.uploadQueue.splice(index, 1);
  47             if (wasRunning)
  48                 $scope.go();
  49         },
  50         countInStates: function(want_states) {
  51             var found = 0;
  52             $.each($scope.uploadQueue, function() {
  53                 if (want_states.indexOf(this.state) >= 0) {
  54                     ++found;
  55                 }
  56             });
  57             return found;
  58         }
  59     });
  60     ////////////////////////////////
  61
  62     var keepProxy;
  63
  64     function SliceReader(_slice) {
  65         var that = this;
  66         $.extend(this, {
  67             go: go
  68         });
  69         ////////////////////////////////
  70         var _deferred;
  71         var _reader;
  72         function go() {
  73             // Return a promise, which will be resolved with the
  74             // requested slice data.
  75             _deferred = $.Deferred();
  76             _reader = new FileReader();
  77             _reader.onload = resolve;
  78             _reader.onerror = _deferred.reject;
  79             _reader.onprogress = _deferred.notify;
  80             _reader.readAsArrayBuffer(_slice.blob);
  81             return _deferred.promise();
  82         }
  83         function resolve() {
  84             if (that._reader.result.length != that._slice.size) {
  85                 // Sometimes we get an onload event even if the read
  86                 // did not return the desired number of bytes. We
  87                 // treat that as a fail.
  88                 _deferred.reject(
  89                     null, "Read error",
  90                     "Short read: wanted " + _slice.size +
  91                         ", received " + _reader.result.length);
  92                 return;
  93             }
  94             return _deferred.resolve(_reader.result);
  95         }
  96     }
  97
  98     function SliceUploader(_label, _data, _dataSize) {
  99         $.extend(this, {
 100             go: go,
 101             stop: stop
 102         });
 103         ////////////////////////////////
 104         var that = this;
 105         var _deferred;
 106         var _failCount = 0;
 107         var _failMax = 3;
 108         var _jqxhr;
 109         function go() {
 110             // Send data to the Keep proxy. Retry a few times on
 111             // fail. Return a promise that will get resolved with
 112             // resolve(locator) when the block is accepted by the
 113             // proxy.
 114             _deferred = $.Deferred();
 115             goSend();
 116             return _deferred.promise();
 117         }
 118         function stop() {
 119             _failMax = 0;
 120             _jqxhr.abort();
 121             _deferred.reject({
 122                 textStatus: 'stopped',
 123                 err: 'interrupted at slice '+_label
 124             });
 125         }
 126         function goSend() {
 127             _jqxhr = $.ajax({
 128                 url: proxyUriBase(),
 129                 type: 'POST',
 130                 crossDomain: true,
 131                 headers: {
 132                     'Authorization': 'OAuth2 '+arvadosApiToken,
 133                     'Content-Type': 'application/octet-stream',
 134                     'X-Keep-Desired-Replicas': '2'
 135                 },
 136                 xhr: function() {
 137                     // Make an xhr that reports upload progress
 138                     var xhr = $.ajaxSettings.xhr();
 139                     if (xhr.upload) {
 140                         xhr.upload.onprogress = onSendProgress;
 141                     }
 142                     return xhr;
 143                 },
 144                 processData: false,
 145                 data: _data
 146             });
 147             _jqxhr.then(onSendResolve, onSendReject);
 148         }
 149         function onSendProgress(xhrProgressEvent) {
 150             _deferred.notify(xhrProgressEvent.loaded, _dataSize);
 151         }
 152         function onSendResolve(data, textStatus, jqxhr) {
 153             _deferred.resolve(data, _dataSize);
 154         }
 155         function onSendReject(xhr, textStatus, err) {
 156             if (++_failCount < _failMax) {
 157                 // TODO: nice to tell the user that retry is happening.
 158                 console.log('slice ' + _label + ': ' +
 159                             textStatus + ', retry ' + _failCount);
 160                 goSend();
 161             } else {
 162                 _deferred.reject(
 163                     {xhr: xhr, textStatus: textStatus, err: err});
 164             }
 165         }
 166         function proxyUriBase() {
 167             return ((keepProxy.service_ssl_flag ? 'https' : 'http') +
 168                     '://' + keepProxy.service_host + ':' +
 169                     keepProxy.service_port + '/');
 170         }
 171     }
 172
 173     function FileUploader(file) {
 174         $.extend(this, {
 175             committed: false,
 176             file: file,
 177             locators: [],
 178             progress: 0.0,
 179             state: 'Queued',    // Queued, Uploading, Paused, Done
 180             statistics: null,
 181             go: go,
 182             stop: stop          // User wants to stop.
 183         });
 184         ////////////////////////////////
 185         var that = this;
 186         var _currentUploader;
 187         var _currentSlice;
 188         var _deferred;
 189         var _maxBlobSize = Math.pow(2,26);
 190         var _bytesDone = 0;
 191         var _queueTime = Date.now();
 192         var _startTime;
 193         var _startByte;
 194         var _finishTime;
 195         var _readPos = 0;       // number of bytes confirmed uploaded
 196         function go() {
 197             if (_deferred)
 198                 _deferred.reject({textStatus: 'restarted'});
 199             _deferred = $q.defer();
 200             that.state = 'Uploading';
 201             _startTime = Date.now();
 202             _startByte = _readPos;
 203             setProgress();
 204             goSlice();
 205             return _deferred.promise;
 206         }
 207         function stop() {
 208             if (_deferred) {
 209                 that.state = 'Paused';
 210                 _deferred.reject({textStatus: 'stopped', err: 'interrupted'});
 211             }
 212             if (_currentUploader) {
 213                 _currentUploader.stop();
 214                 _currentUploader = null;
 215             }
 216         }
 217         function goSlice() {
 218             // Ensure this._deferred gets resolved or rejected --
 219             // either right here, or when a new promise arranged right
 220             // here is fulfilled.
 221             _currentSlice = nextSlice();
 222             if (!_currentSlice) {
 223                 that.state = 'Done';
 224                 setProgress(_readPos);
 225                 _currentUploader = null;
 226                 _deferred.resolve([that]);
 227                 return;
 228             }
 229             _currentUploader = new SliceUploader(
 230                 _readPos.toString(),
 231                 _currentSlice.blob,
 232                 _currentSlice.size);
 233             _currentUploader.go().then(
 234                 onUploaderResolve,
 235                 onUploaderReject,
 236                 onUploaderProgress);
 237         }
 238         function onUploaderResolve(locator, dataSize) {
 239             var sizeHint = (''+locator).split('+')[1];
 240             if (!locator || parseInt(sizeHint) != dataSize) {
 241                 console.log("onUploaderResolve, but locator '" + locator +
 242                             "' with size hint '" + sizeHint +
 243                             "' does not look right for dataSize=" + dataSize);
 244                 return onUploaderReject({
 245                     textStatus: "error",
 246                     err: "Bad response from slice upload"
 247                 });
 248             }
 249             that.locators.push(locator);
 250             _readPos += dataSize;
 251             _currentUploader = null;
 252             goSlice();
 253         }
 254         function onUploaderReject(reason) {
 255             that.state = 'Paused';
 256             setProgress(_readPos);
 257             _currentUploader = null;
 258             _deferred.reject(reason);
 259         }
 260         function onUploaderProgress(sliceDone, sliceSize) {
 261             setProgress(_readPos + sliceDone);
 262         }
 263         function nextSlice() {
 264             var size = Math.min(
 265                 _maxBlobSize,
 266                 that.file.size - _readPos);
 267             setProgress(_readPos);
 268             if (size == 0) {
 269                 return false;
 270             }
 271             var blob = that.file.slice(
 272                 _readPos, _readPos+size,
 273                 'application/octet-stream; charset=x-user-defined');
 274             return {blob: blob, size: size};
 275         }
 276         function setProgress(bytesDone) {
 277             var kBps;
 278             that.progress = Math.min(100, 100 * bytesDone / that.file.size)
 279             if (bytesDone > _startByte) {
 280                 kBps = (bytesDone - _startByte) /
 281                     (Date.now() - _startTime);
 282                 that.statistics = (
 283                     '' + $filter('number')(bytesDone/1024, '0') + ' KiB ' +
 284                         'at ~' + $filter('number')(kBps, '0') + ' KiB/s')
 285                 if (that.state == 'Paused') {
 286                     that.statistics += ', paused';
 287                 } else if (that.state == 'Uploading') {
 288                     that.statistics += ', ETA ' +
 289                         $filter('date')(
 290                             new Date(
 291                                 Date.now() + (that.file.size - bytesDone) / kBps),
 292                             'shortTime')
 293                 } else {
 294                     that.statistics += ', finished ' +
 295                         $filter('date')(Date.now(), 'shortTime');
 296                     _finishTime = Date.now();
 297                 }
 298             } else {
 299                 that.statistics = that.state;
 300             }
 301             _deferred.notify();
 302         }
 303     }
 304
 305     function QueueUploader() {
 306         $.extend(this, {
 307             state: 'Idle',
 308             stateReason: null,
 309             statusSuccess: null,
 310             go: go,
 311             stop: stop
 312         });
 313         ////////////////////////////////
 314         var that = this;
 315         var _deferred;
 316         function go() {
 317             if (that.state == 'Running') return _deferred.promise;
 318             _deferred = $.Deferred();
 319             that.state = 'Running';
 320             ArvadosClient.apiPromise(
 321                 'keep_services', 'list',
 322                 {filters: [['service_type','=','proxy']]}).
 323                 then(doQueueWithProxy);
 324             onQueueProgress();
 325             return _deferred.promise();
 326         }
 327         function stop() {
 328             for (var i=0; i<$scope.uploadQueue.length; i++)
 329                 $scope.uploadQueue[i].stop();
 330         }
 331         function doQueueWithProxy(data) {
 332             keepProxy = data.items[0];
 333             if (!keepProxy) {
 334                 that.state = 'Failed';
 335                 that.stateReason =
 336                     'There seems to be no Keep proxy service available.';
 337                 _deferred.reject(null, 'error', that.stateReason);
 338                 return;
 339             }
 340             return doQueueWork();
 341         }
 342         function doQueueWork() {
 343             var i;
 344             that.state = 'Running';
 345             that.stateReason = null;
 346             // Push the done things to the bottom of the queue.
 347             for (i=0; (i<$scope.uploadQueue.length &&
 348                        $scope.uploadQueue[i].state == 'Done'); i++);
 349             if (i>0)
 350                 $scope.uploadQueue.push.apply($scope.uploadQueue, $scope.uploadQueue.splice(0, i));
 351             // If anything is not-done, do it.
 352             if ($scope.uploadQueue.length > 0 &&
 353                 $scope.uploadQueue[0].state != 'Done') {
 354                 return $scope.uploadQueue[0].go().
 355                     then(appendToCollection, null, onQueueProgress).
 356                     then(doQueueWork, onQueueReject);
 357             }
 358             // If everything is done, resolve the promise and clean up.
 359             return onQueueResolve();
 360         }
 361         function onQueueReject(reason) {
 362             that.state = 'Failed';
 363             that.stateReason = (
 364                 (reason.textStatus || 'Error') +
 365                     (reason.xhr && reason.xhr.options
 366                      ? (' (from ' + reason.xhr.options.url + ')')
 367                      : '') +
 368                     ': ' +
 369                     (reason.err || ''));
 370             if (reason.xhr && reason.xhr.responseText)
 371                 that.stateReason += ' -- ' + reason.xhr.responseText;
 372             _deferred.reject(reason);
 373             onQueueProgress();
 374         }
 375         function onQueueResolve() {
 376             that.state = 'Idle';
 377             that.stateReason = 'Done!';
 378             _deferred.resolve();
 379             onQueueProgress();
 380         }
 381         function onQueueProgress() {
 382             // Ensure updates happen after FileUpload promise callbacks.
 383             $timeout(function(){$scope.$apply();});
 384         }
 385         function appendToCollection(uploads) {
 386             var deferred = $q.defer();
 387             return ArvadosClient.apiPromise(
 388                 'collections', 'get',
 389                 { uuid: $scope.uuid }).
 390                 then(function(collection) {
 391                     var manifestText = '';
 392                     var upload, i;
 393                     for (i=0; i<uploads.length; i++) {
 394                         upload = uploads[i];
 395                         filename = ArvadosClient.uniqueNameForManifest(
 396                             collection.manifest_text,
 397                             '.', upload.file.name);
 398                         collection.manifest_text += '. ' +
 399                             upload.locators.join(' ') +
 400                             ' 0:' + upload.file.size.toString() + ':' +
 401                             filename +
 402                             '\n';
 403                     }
 404                     return ArvadosClient.apiPromise(
 405                         'collections', 'update',
 406                         { uuid: $scope.uuid,
 407                           collection:
 408                           { manifest_text:
 409                             collection.manifest_text }
 410                         }).
 411                         then(deferred.resolve);
 412                 }, onQueueReject).then(function() {
 413                     var i;
 414                     for(i=0; i<uploads.length; i++) {
 415                         uploads[i].committed = true;
 416                     }
 417                 });
 418             return deferred.promise.then(doQueueWork);
 419         }
 420     }
 421 }