From: Stephen Smith Date: Tue, 15 Aug 2023 16:47:09 +0000 (-0400) Subject: 20219: Switch log panel requests to use live log endpoint, adjust tests to match X-Git-Tag: 2.7.0~15^2~8 X-Git-Url: https://git.arvados.org/arvados-workbench2.git/commitdiff_plain/038d4d4f8ecf66949b321c4ece4aa794719f9f3e 20219: Switch log panel requests to use live log endpoint, adjust tests to match Arvados-DCO-1.1-Signed-off-by: Stephen Smith --- diff --git a/cypress/integration/process.spec.js b/cypress/integration/process.spec.js index b41a443e..62d5d9ea 100644 --- a/cypress/integration/process.spec.js +++ b/cypress/integration/process.spec.js @@ -438,6 +438,7 @@ describe('Process tests', function() { .should('contain', 'No logs yet') .and('not.contain', 'hello world'); + // Append a log line cy.appendLog(adminUser.token, containerRequest.uuid, "stdout.txt", [ "2023-07-18T20:14:48.128642814Z hello world" ]).then(() => { @@ -446,6 +447,7 @@ describe('Process tests', function() { .and('contain', 'hello world'); }); + // Append new log line to different file cy.appendLog(adminUser.token, containerRequest.uuid, "stderr.txt", [ "2023-07-18T20:14:49.128642814Z hello new line" ]).then(() => { diff --git a/cypress/support/commands.js b/cypress/support/commands.js index e4f6fd60..24629d99 100644 --- a/cypress/support/commands.js +++ b/cypress/support/commands.js @@ -34,6 +34,8 @@ const controllerURL = Cypress.env('controller_url'); const systemToken = Cypress.env('system_token'); let createdResources = []; +const containerLogFolderPrefix = 'log for container '; + // Clean up on a 'before' hook to allow post-mortem analysis on individual tests. beforeEach(function () { if (createdResources.length === 0) { @@ -192,6 +194,12 @@ Cypress.Commands.add( } ) +Cypress.Commands.add( + "collectionReplaceFiles", (token, uuid, data) => { + return cy.updateResource(token, 'collections', uuid, JSON.stringify(data)) + } +) + Cypress.Commands.add( "getContainer", (token, uuid) => { return cy.getResource(token, 'containers', uuid) @@ -237,18 +245,19 @@ Cypress.Commands.add( cy.getContainerRequest(token, crUuid).then((containerRequest) => { if (containerRequest.log_uuid) { cy.listContainerRequestLogs(token, crUuid).then((logFiles) => { + const filePath = `${containerRequest.log_uuid}/${containerLogFolderPrefix}${containerRequest.container_uuid}/${fileName}`; if (logFiles.find((file) => (file.name === fileName))) { // File exists, fetch and append return cy.doKeepRequest( "GET", - `c=${containerRequest.log_uuid}/${fileName}`, + `c=${filePath}`, null, null, token ) .then(({ body: contents }) => cy.doKeepRequest( "PUT", - `c=${containerRequest.log_uuid}/${fileName}`, + `c=${filePath}`, contents.split("\n").concat(lines).join("\n"), null, token @@ -257,37 +266,44 @@ Cypress.Commands.add( // File not exists, put new file cy.doKeepRequest( "PUT", - `c=${containerRequest.log_uuid}/${fileName}`, + `c=${filePath}`, lines.join("\n"), null, token ) } }); - // Fetch current log contents and append new line - // let newLines = [...lines]; - // return cy.doKeepRequest('GET', `c=${containerRequest.log_uuid}/${fileName}`, null, null, token) - // .then(({body: contents}) => { - // newLines = [contents.split('\n'), ...newLines]; - // }) - // .then(() => ( - // cy.doKeepRequest('PUT', `c=${containerRequest.log_uuid}/${fileName}`, newLines.join('\n'), null, token) - // )); } else { // Create log collection return cy.createCollection(token, { name: `Test log collection ${Math.floor(Math.random() * 999999)}`, owner_uuid: containerRequest.owner_uuid, manifest_text: "" - }).then((collection) => ( + }).then((collection) => { // Update CR log_uuid to fake log collection cy.updateContainerRequest(token, containerRequest.uuid, { log_uuid: collection.uuid, }).then(() => ( - // Put new log file with contents into fake log collection - cy.doKeepRequest('PUT', `c=${collection.uuid}/${fileName}`, lines.join('\n'), null, token) - )) - )); + // Create empty directory for container uuid + cy.collectionReplaceFiles(token, collection.uuid, { + collection: { + preserve_version: true, + }, + replace_files: { + [`/${containerLogFolderPrefix}${containerRequest.container_uuid}`]: "d41d8cd98f00b204e9800998ecf8427e+0" + } + }).then(() => ( + // Put new log file with contents into fake log collection + cy.doKeepRequest( + 'PUT', + `c=${collection.uuid}/${containerLogFolderPrefix}${containerRequest.container_uuid}/${fileName}`, + lines.join('\n'), + null, + token + ) + )) + )); + }); } }) ) @@ -296,7 +312,7 @@ Cypress.Commands.add( Cypress.Commands.add( "listContainerRequestLogs", (token, crUuid) => ( cy.getContainerRequest(token, crUuid).then((containerRequest) => ( - cy.doKeepRequest('PROPFIND', `c=${containerRequest.log_uuid}`, null, null, token) + cy.doKeepRequest('PROPFIND', `c=${containerRequest.log_uuid}/${containerLogFolderPrefix}${containerRequest.container_uuid}`, null, null, token) .then(({body: data}) => { return extractFilesData(new DOMParser().parseFromString(data, "text/xml")); }) diff --git a/src/services/log-service/log-service.ts b/src/services/log-service/log-service.ts index 03d3c01e..f93a3f81 100644 --- a/src/services/log-service/log-service.ts +++ b/src/services/log-service/log-service.ts @@ -9,6 +9,7 @@ import { ApiActions } from "services/api/api-actions"; import { WebDAV } from "common/webdav"; import { extractFilesData } from "services/collection-service/collection-service-files-response"; import { CollectionFile } from "models/collection-file"; +import { ContainerRequestResource } from "models/container-request"; export type LogFragment = { logType: LogEventType; @@ -20,20 +21,20 @@ export class LogService extends CommonResourceService { super(serverApi, "logs", actions); } - async listLogFiles(containerRequestUuid: string) { - const request = await this.apiWebdavClient.propfind(`container_requests/${containerRequestUuid}/log`); + async listLogFiles(containerRequest: ContainerRequestResource) { + const request = await this.apiWebdavClient.propfind(`container_requests/${containerRequest.uuid}/log/${containerRequest.containerUuid}`); if (request.responseXML != null) { return extractFilesData(request.responseXML) .filter((file) => ( - file.path === `/arvados/v1/container_requests/${containerRequestUuid}/log` + file.path === `/arvados/v1/container_requests/${containerRequest.uuid}/log/${containerRequest.containerUuid}` )); } return Promise.reject(); } - async getLogFileContents(containerRequestUuid: string, fileRecord: CollectionFile, startByte: number, endByte: number): Promise { + async getLogFileContents(containerRequest: ContainerRequestResource, fileRecord: CollectionFile, startByte: number, endByte: number): Promise { const request = await this.apiWebdavClient.get( - `container_requests/${containerRequestUuid}/log/${fileRecord.name}`, + `container_requests/${containerRequest.uuid}/log/${containerRequest.containerUuid}/${fileRecord.name}`, {headers: {Range: `bytes=${startByte}-${endByte}`}} ); const logFileType = logFileToLogType(fileRecord); diff --git a/src/store/process-logs-panel/process-logs-panel-actions.ts b/src/store/process-logs-panel/process-logs-panel-actions.ts index 781c8903..844f0154 100644 --- a/src/store/process-logs-panel/process-logs-panel-actions.ts +++ b/src/store/process-logs-panel/process-logs-panel-actions.ts @@ -13,6 +13,7 @@ import { Process, getProcess } from 'store/processes/process'; import { navigateTo } from 'store/navigation/navigation-action'; import { snackbarActions, SnackbarKind } from 'store/snackbar/snackbar-actions'; import { CollectionFile, CollectionFileType } from "models/collection-file"; +import { ContainerRequestResource } from "models/container-request"; const SNIPLINE = `================ ✀ ================ ✀ ========= Some log(s) were skipped ========= ✀ ================ ✀ ================`; const LOG_TIMESTAMP_PATTERN = /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9}Z/; @@ -44,7 +45,7 @@ export const initProcessLogsPanel = (processUuid: string) => const process = getProcess(processUuid)(getState().resources); if (process?.containerRequest?.uuid) { // Get log file size info - const logFiles = await loadContainerLogFileList(process.containerRequest.uuid, logService); + const logFiles = await loadContainerLogFileList(process.containerRequest, logService); // Populate lastbyte 0 for each file const filesWithProgress = logFiles.map((file) => ({file, lastByte: 0})); @@ -73,7 +74,7 @@ export const pollProcessLogs = (processUuid: string) => // Check if container request is present and initial logs state loaded if (process?.containerRequest?.uuid && Object.keys(currentState.logs).length > 0) { - const logFiles = await loadContainerLogFileList(process.containerRequest.uuid, logService); + const logFiles = await loadContainerLogFileList(process.containerRequest, logService); // Determine byte to fetch from while filtering unchanged files const filesToUpdateWithProgress = logFiles.reduce((acc, updatedFile) => { @@ -107,8 +108,8 @@ export const pollProcessLogs = (processUuid: string) => } }; -const loadContainerLogFileList = async (containerUuid: string, logService: LogService) => { - const logCollectionContents = await logService.listLogFiles(containerUuid); +const loadContainerLogFileList = async (containerRequest: ContainerRequestResource, logService: LogService) => { + const logCollectionContents = await logService.listLogFiles(containerRequest); // Filter only root directory files matching log event types which have bytes return logCollectionContents.filter((file): file is CollectionFile => ( @@ -134,11 +135,11 @@ const loadContainerLogFileContents = async (logFilesWithProgress: FileWithProgre const chunkSize = Math.floor(maxLogFetchSize / 2); const firstChunkEnd = lastByte+chunkSize-1; return Promise.all([ - logService.getLogFileContents(process.containerRequest.uuid, file, lastByte, firstChunkEnd), - logService.getLogFileContents(process.containerRequest.uuid, file, file.size-chunkSize, file.size-1) + logService.getLogFileContents(process.containerRequest, file, lastByte, firstChunkEnd), + logService.getLogFileContents(process.containerRequest, file, file.size-chunkSize, file.size-1) ] as Promise<(LogFragment)>[]); } else { - return Promise.all([logService.getLogFileContents(process.containerRequest.uuid, file, lastByte, file.size-1)]); + return Promise.all([logService.getLogFileContents(process.containerRequest, file, lastByte, file.size-1)]); } })).then((res) => { if (res.length && res.every(promiseResult => (promiseResult.status === 'rejected'))) {