1 var app = angular.module('Workbench', ['Arvados']);
2 app.controller('UploadToCollection', UploadToCollection);
3 app.directive('arvUuid', arvUuid);
6 // Copy the given uuid into the current $scope.
9 link: function(scope, element, attributes) {
10 scope.uuid = attributes.arvUuid;
15 UploadToCollection.$inject = ['$scope', '$filter', '$q', '$timeout',
16 'ArvadosClient', 'arvadosApiToken'];
17 function UploadToCollection($scope, $filter, $q, $timeout,
18 ArvadosClient, arvadosApiToken) {
21 uploader: new QueueUploader(),
22 addFilesToQueue: function(files) {
23 // Angular binding doesn't work its usual magic for file
24 // inputs, so we need to $scope.$apply() this update.
25 $scope.$apply(function(){
27 // Add these new files after the items already waiting
28 // in the queue -- but before the items that are
29 // 'Done' and have therefore been pushed to the
32 (nItemsTodo < $scope.uploadQueue.length &&
33 $scope.uploadQueue[nItemsTodo].state !== 'Done'); ) {
36 for (i=0; i<files.length; i++) {
37 $scope.uploadQueue.splice(nItemsTodo+i, 0,
38 new FileUploader(files[i]));
46 $scope.uploader.stop();
48 removeFileFromQueue: function(index) {
49 var wasRunning = $scope.uploader.running;
50 $scope.uploadQueue[index].stop();
51 $scope.uploadQueue.splice(index, 1);
55 countInStates: function(want_states) {
57 $.each($scope.uploadQueue, function() {
58 if (want_states.indexOf(this.state) >= 0) {
65 ////////////////////////////////
69 function SliceReader(_slice) {
74 ////////////////////////////////
78 // Return a promise, which will be resolved with the
79 // requested slice data.
80 _deferred = $.Deferred();
81 _reader = new FileReader();
82 _reader.onload = resolve;
83 _reader.onerror = _deferred.reject;
84 _reader.onprogress = _deferred.notify;
85 _reader.readAsArrayBuffer(_slice.blob);
86 return _deferred.promise();
89 if (that._reader.result.length !== that._slice.size) {
90 // Sometimes we get an onload event even if the read
91 // did not return the desired number of bytes. We
92 // treat that as a fail.
95 "Short read: wanted " + _slice.size +
96 ", received " + _reader.result.length);
99 return _deferred.resolve(_reader.result);
103 function SliceUploader(_label, _data, _dataSize) {
108 ////////////////////////////////
115 // Send data to the Keep proxy. Retry a few times on
116 // fail. Return a promise that will get resolved with
117 // resolve(locator) when the block is accepted by the
119 _deferred = $.Deferred();
121 return _deferred.promise();
127 textStatus: 'stopped',
128 err: 'interrupted at slice '+_label
137 'Authorization': 'OAuth2 '+arvadosApiToken,
138 'Content-Type': 'application/octet-stream',
139 'X-Keep-Desired-Replicas': '2'
142 // Make an xhr that reports upload progress
143 var xhr = $.ajaxSettings.xhr();
145 xhr.upload.onprogress = onSendProgress;
152 _jqxhr.then(onSendResolve, onSendReject);
154 function onSendProgress(xhrProgressEvent) {
155 _deferred.notify(xhrProgressEvent.loaded, _dataSize);
157 function onSendResolve(data, textStatus, jqxhr) {
158 _deferred.resolve(data, _dataSize);
160 function onSendReject(xhr, textStatus, err) {
161 if (++_failCount < _failMax) {
162 // TODO: nice to tell the user that retry is happening.
163 console.log('slice ' + _label + ': ' +
164 textStatus + ', retry ' + _failCount);
168 {xhr: xhr, textStatus: textStatus, err: err});
171 function proxyUriBase() {
172 return ((keepProxy.service_ssl_flag ? 'https' : 'http') +
173 '://' + keepProxy.service_host + ':' +
174 keepProxy.service_port + '/');
178 function FileUploader(file) {
184 state: 'Queued', // Queued, Uploading, Paused, Done
187 stop: stop // User wants to stop.
189 ////////////////////////////////
191 var _currentUploader;
194 var _maxBlobSize = Math.pow(2,26);
196 var _queueTime = Date.now();
200 var _readPos = 0; // number of bytes confirmed uploaded
203 _deferred.reject({textStatus: 'restarted'});
204 _deferred = $q.defer();
205 that.state = 'Uploading';
206 _startTime = Date.now();
207 _startByte = _readPos;
210 return _deferred.promise;
214 that.state = 'Paused';
215 _deferred.reject({textStatus: 'stopped', err: 'interrupted'});
217 if (_currentUploader) {
218 _currentUploader.stop();
219 _currentUploader = null;
223 // Ensure this._deferred gets resolved or rejected --
224 // either right here, or when a new promise arranged right
225 // here is fulfilled.
226 _currentSlice = nextSlice();
227 if (!_currentSlice) {
229 setProgress(_readPos);
230 _currentUploader = null;
231 _deferred.resolve([that]);
234 _currentUploader = new SliceUploader(
238 _currentUploader.go().then(
243 function onUploaderResolve(locator, dataSize) {
244 var sizeHint = (''+locator).split('+')[1];
245 if (!locator || parseInt(sizeHint) !== dataSize) {
246 console.log("onUploaderResolve, but locator '" + locator +
247 "' with size hint '" + sizeHint +
248 "' does not look right for dataSize=" + dataSize);
249 return onUploaderReject({
251 err: "Bad response from slice upload"
254 that.locators.push(locator);
255 _readPos += dataSize;
256 _currentUploader = null;
259 function onUploaderReject(reason) {
260 that.state = 'Paused';
261 setProgress(_readPos);
262 _currentUploader = null;
263 _deferred.reject(reason);
265 function onUploaderProgress(sliceDone, sliceSize) {
266 setProgress(_readPos + sliceDone);
268 function nextSlice() {
271 that.file.size - _readPos);
272 setProgress(_readPos);
276 var blob = that.file.slice(
277 _readPos, _readPos+size,
278 'application/octet-stream; charset=x-user-defined');
279 return {blob: blob, size: size};
281 function setProgress(bytesDone) {
283 that.progress = Math.min(100, 100 * bytesDone / that.file.size)
284 if (bytesDone > _startByte) {
285 kBps = (bytesDone - _startByte) /
286 (Date.now() - _startTime);
288 '' + $filter('number')(bytesDone/1024, '0') + ' KiB ' +
289 'at ~' + $filter('number')(kBps, '0') + ' KiB/s')
290 if (that.state === 'Paused') {
291 that.statistics += ', paused';
292 } else if (that.state === 'Uploading') {
293 that.statistics += ', ETA ' +
296 Date.now() + (that.file.size - bytesDone) / kBps),
299 that.statistics += ', finished ' +
300 $filter('date')(Date.now(), 'shortTime');
301 _finishTime = Date.now();
304 that.statistics = that.state;
310 function QueueUploader() {
318 ////////////////////////////////
322 if (that.state === 'Running') return _deferred.promise;
323 _deferred = $.Deferred();
324 that.state = 'Running';
325 ArvadosClient.apiPromise(
326 'keep_services', 'list',
327 {filters: [['service_type','=','proxy']]}).
328 then(doQueueWithProxy);
330 return _deferred.promise();
333 that.state = 'Stopped';
334 for (var i=0; i<$scope.uploadQueue.length; i++)
335 $scope.uploadQueue[i].stop();
337 function doQueueWithProxy(data) {
338 keepProxy = data.items[0];
340 that.state = 'Failed';
342 'There seems to be no Keep proxy service available.';
343 _deferred.reject(null, 'error', that.stateReason);
346 return doQueueWork();
348 function doQueueWork() {
349 that.state = 'Running';
350 that.stateReason = null;
351 // If anything is not Done, do it.
352 if ($scope.uploadQueue.length > 0 &&
353 $scope.uploadQueue[0].state !== 'Done') {
354 return $scope.uploadQueue[0].go().
355 then(appendToCollection, null, onQueueProgress).
356 then(doQueueWork, onQueueReject);
358 // If everything is Done, resolve the promise and clean up.
359 return onQueueResolve();
361 function onQueueReject(reason) {
362 if (that.state !== 'Stopped') {
363 that.state = 'Error';
365 // (else it's not really an error, just a consequence of stop())
368 (reason.textStatus || 'Error') +
369 (reason.xhr && reason.xhr.options
370 ? (' (from ' + reason.xhr.options.url + ')')
374 if (reason.xhr && reason.xhr.responseText)
375 that.stateReason += ' -- ' + reason.xhr.responseText;
376 _deferred.reject(reason);
379 function onQueueResolve() {
381 that.stateReason = 'Done!';
385 function onQueueProgress() {
386 // Ensure updates happen after FileUpload promise callbacks.
387 $timeout(function(){$scope.$apply();});
389 function appendToCollection(uploads) {
390 var deferred = $q.defer();
391 return ArvadosClient.apiPromise(
392 'collections', 'get',
393 { uuid: $scope.uuid }).
394 then(function(collection) {
395 var manifestText = '';
396 $.each(uploads, function(_, upload) {
397 filename = ArvadosClient.uniqueNameForManifest(
398 collection.manifest_text,
399 '.', upload.file.name);
400 collection.manifest_text += '. ' +
401 upload.locators.join(' ') +
402 ' 0:' + upload.file.size.toString() + ':' +
406 return ArvadosClient.apiPromise(
407 'collections', 'update',
411 collection.manifest_text }
413 then(deferred.resolve);
414 }, onQueueReject).then(function() {
415 // Push the completed upload(s) to the bottom of the queue.
416 var i, qLen = $scope.uploadQueue.length;
417 for (i=0; i<qLen; i++) {
418 if (uploads.indexOf($scope.uploadQueue[i]) >= 0) {
419 $scope.uploadQueue[i].committed = true;
420 $scope.uploadQueue.push.apply(
422 $scope.uploadQueue.splice(i, 1));
428 return deferred.promise.then(doQueueWork);