Merge branch 'master' into 14100-process-logs-service

refs #14100

Arvados-DCO-1.1-Signed-off-by: Michal Klobukowski <michal.klobukowski@contractors.roche.com>

diff --git a/src/models/log.ts b/src/models/log.ts
new file mode 100644 (file)
index 0000000..2988656
--- /dev/null
+++ b/src/models/log.ts
@@ -0,0 +1,28 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+import { Resource } from "./resource";
+import { ResourceKind } from '~/models/resource';
+
+export enum LogEventType {
+    CREATE = 'create',
+    UPDATE = 'update',
+    DISPATCH = 'dispatch',
+    CRUNCH_RUN = 'crunch-run',
+    CRUNCHSTAT = 'crunchstat',
+    HOSTSTAT = 'hoststat',
+    NODE_INFO = 'node-info',
+    ARV_MOUNT = 'arv-mount',
+    STDOUT = 'stdout',
+    STDERR = 'stderr',
+}
+
+export interface LogResource extends Resource {
+    kind: ResourceKind.LOG;
+    objectUuid: string;
+    eventAt: string;
+    eventType: string;
+    summary: string;
+    properties: any;
+}

diff --git a/src/models/resource.ts b/src/models/resource.ts
index 3290bdfe06f07ed60fa27accd067cff47f0d0b6b..3c6c11bc57f81b62669a693362d5c3a75b9e2a24 100644 (file)
--- a/src/models/resource.ts
+++ b/src/models/resource.ts
@@ -19,6 +19,7 @@ export enum ResourceKind {
     CONTAINER = "arvados#container",
     CONTAINER_REQUEST = "arvados#containerRequest",
     GROUP = "arvados#group",
+    LOG = "arvados#log",
     PROCESS = "arvados#containerRequest",
     PROJECT = "arvados#group",
     USER = "arvados#user",
@@ -30,6 +31,7 @@ export enum ResourceObjectType {
     CONTAINER = 'dz642',
     CONTAINER_REQUEST = 'xvhdp',
     GROUP = 'j7d0g',
+    LOG = '57u5n',
     USER = 'tpzed',
 }
 
@@ -59,6 +61,8 @@ export const extractUuidKind = (uuid: string = '') => {
             return ResourceKind.CONTAINER_REQUEST;
         case ResourceObjectType.CONTAINER:
             return ResourceKind.CONTAINER;
+        case ResourceObjectType.LOG:
+            return ResourceKind.LOG;
         default:
             return undefined;
     }

diff --git a/src/services/log-service/log-service.ts b/src/services/log-service/log-service.ts
new file mode 100644 (file)
index 0000000..c92475d
--- /dev/null
+++ b/src/services/log-service/log-service.ts
@@ -0,0 +1,13 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+import { CommonResourceService } from "~/common/api/common-resource-service";
+import { AxiosInstance } from "axios";
+import { LogResource } from '~/models/log';
+
+export class LogService extends CommonResourceService<LogResource> {
+    constructor(serverApi: AxiosInstance) {
+        super(serverApi, "logs");
+    }
+}

diff --git a/src/services/services.ts b/src/services/services.ts
index 32e7bd18b2ff9bd54ee1305af28e132e5da4cfe9..53721dd301b64399d77775c3ec09b65240a6c564 100644 (file)
--- a/src/services/services.ts
+++ b/src/services/services.ts
@@ -19,6 +19,7 @@ import { AncestorService } from "~/services/ancestors-service/ancestors-service"
 import { ResourceKind } from "~/models/resource";
 import { ContainerRequestService } from './container-request-service/container-request-service';
 import { ContainerService } from './container-service/container-service';
+import { LogService } from './log-service/log-service';
 
 export type ServiceRepository = ReturnType<typeof createServices>;
 
@@ -29,13 +30,14 @@ export const createServices = (config: Config) => {
     const webdavClient = new WebDAV();
     webdavClient.defaults.baseURL = config.keepWebServiceUrl;
 
+    const containerRequestService = new ContainerRequestService(apiClient);
+    const containerService = new ContainerService(apiClient);
     const groupsService = new GroupsService(apiClient);
     const keepService = new KeepService(apiClient);
     const linkService = new LinkService(apiClient);
+    const logService = new LogService(apiClient);
     const projectService = new ProjectService(apiClient);
     const userService = new UserService(apiClient);
-    const containerRequestService = new ContainerRequestService(apiClient);
-    const containerService = new ContainerService(apiClient);
     
     const ancestorsService = new AncestorService(groupsService, userService);
     const authService = new AuthService(apiClient, config.rootUrl);
@@ -56,6 +58,7 @@ export const createServices = (config: Config) => {
         groupsService,
         keepService,
         linkService,
+        logService,
         projectService,
         tagService,
         userService,

diff --git a/src/store/process-logs-panel/process-logs-panel-actions.ts b/src/store/process-logs-panel/process-logs-panel-actions.ts
new file mode 100644 (file)
index 0000000..532ae11
--- /dev/null
+++ b/src/store/process-logs-panel/process-logs-panel-actions.ts
@@ -0,0 +1,83 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+import { unionize, ofType, UnionOf } from "~/common/unionize";
+import { ProcessLogs } from './process-logs-panel';
+import { LogEventType } from '~/models/log';
+import { RootState } from '~/store/store';
+import { ServiceRepository } from '~/services/services';
+import { Dispatch } from 'redux';
+import { FilterBuilder } from '~/common/api/filter-builder';
+import { groupBy } from 'lodash';
+import { loadProcess } from '~/store/processes/processes-actions';
+import { OrderBuilder } from '~/common/api/order-builder';
+import { LogResource } from '~/models/log';
+import { LogService } from '~/services/log-service/log-service';
+
+export const processLogsPanelActions = unionize({
+    INIT_PROCESS_LOGS_PANEL: ofType<{ filters: string[], logs: ProcessLogs }>(),
+    SET_PROCESS_LOGS_PANEL_FILTER: ofType<string>(),
+    ADD_PROCESS_LOGS_PANEL_ITEM: ofType<{ logType: string, log: string }>(),
+});
+
+export type ProcessLogsPanelAction = UnionOf<typeof processLogsPanelActions>;
+
+export const setProcessLogsPanelFilter = (filter: string) =>
+     processLogsPanelActions.SET_PROCESS_LOGS_PANEL_FILTER(filter);
+
+export const initProcessLogsPanel = (processUuid: string) =>
+    async (dispatch: Dispatch, getState: () => RootState, { logService }: ServiceRepository) => {
+        const process = await dispatch<any>(loadProcess(processUuid));
+        if (process.container) {
+            const logResources = await loadContainerLogs(process.container.uuid, logService);
+            const initialState = createInitialLogPanelState(logResources);
+            dispatch(processLogsPanelActions.INIT_PROCESS_LOGS_PANEL(initialState));
+        }
+    };
+
+const loadContainerLogs = async (containerUuid: string, logService: LogService) => {
+    const requestFilters = new FilterBuilder()
+        .addEqual('objectUuid', containerUuid)
+        .addIn('eventType', PROCESS_PANEL_LOG_EVENT_TYPES)
+        .getFilters();
+    const requestOrder = new OrderBuilder<LogResource>()
+        .addAsc('eventAt')
+        .getOrder();
+    const requestParams = {
+        limit: MAX_AMOUNT_OF_LOGS,
+        filters: requestFilters,
+        order: requestOrder,
+    };
+    const { items } = await logService.list(requestParams);
+    return items;
+};
+
+const createInitialLogPanelState = (logResources: LogResource[]) => {
+    const allLogs = logResources.map(({ properties }) => properties.text);
+    const groupedLogResources = groupBy(logResources, log => log.eventType);
+    const groupedLogs = Object
+        .keys(groupedLogResources)
+        .reduce((grouped, key) => ({
+            ...grouped,
+            [key]: groupedLogResources[key].map(({ properties }) => properties.text)
+        }), {});
+    const filters = [SUMMARIZED_FILTER_TYPE, ...Object.keys(groupedLogs)];
+    const logs = { [SUMMARIZED_FILTER_TYPE]: allLogs, ...groupedLogs };
+    return { filters, logs };
+};
+
+const MAX_AMOUNT_OF_LOGS = 10000;
+
+const SUMMARIZED_FILTER_TYPE = 'Summarized';
+
+const PROCESS_PANEL_LOG_EVENT_TYPES = [
+    LogEventType.ARV_MOUNT,
+    LogEventType.CRUNCH_RUN,
+    LogEventType.CRUNCHSTAT,
+    LogEventType.DISPATCH,
+    LogEventType.HOSTSTAT,
+    LogEventType.NODE_INFO,
+    LogEventType.STDERR,
+    LogEventType.STDOUT,
+];

diff --git a/src/store/process-logs-panel/process-logs-panel-reducer.ts b/src/store/process-logs-panel/process-logs-panel-reducer.ts
new file mode 100644 (file)
index 0000000..39a448b
--- /dev/null
+++ b/src/store/process-logs-panel/process-logs-panel-reducer.ts
@@ -0,0 +1,32 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+import { ProcessLogsPanel } from './process-logs-panel';
+import { RootState } from '~/store/store';
+import { ProcessLogsPanelAction, processLogsPanelActions } from './process-logs-panel-actions';
+
+const initialState: ProcessLogsPanel = {
+    filters: [],
+    selectedFilter: '',
+    logs: { '': [] },
+};
+
+export const processLogsPanelReducer = (state = initialState, action: ProcessLogsPanelAction): ProcessLogsPanel =>
+    processLogsPanelActions.match(action, {
+        INIT_PROCESS_LOGS_PANEL: ({ filters, logs }) => ({
+            filters,
+            logs,
+            selectedFilter: filters[0] || '',
+        }),
+        SET_PROCESS_LOGS_PANEL_FILTER: selectedFilter => ({
+            ...state,
+            selectedFilter
+        }),
+        ADD_PROCESS_LOGS_PANEL_ITEM: ({ logType, log }) => {
+            const logsOfType = [...state.logs[logType], log];
+            const logs = { ...state.logs, [logType]: logsOfType };
+            return { ...state, logs };
+        },
+        default: () => state,
+    });

diff --git a/src/store/process-logs-panel/process-logs-panel.ts b/src/store/process-logs-panel/process-logs-panel.ts
new file mode 100644 (file)
index 0000000..65ed78d
--- /dev/null
+++ b/src/store/process-logs-panel/process-logs-panel.ts
@@ -0,0 +1,17 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+export interface ProcessLogsPanel {
+    filters: string[];
+    selectedFilter: string;
+    logs: ProcessLogs;
+}
+
+export interface ProcessLogs {
+    [logType: string]: string[];
+}
+
+export const getProcessPanelLogs = ({ selectedFilter, logs }: ProcessLogsPanel) => {
+    return logs[selectedFilter];
+};

diff --git a/src/store/processes/processes-actions.ts b/src/store/processes/processes-actions.ts
index be9266b64ec2cb9058ec5c23f8bea6fae2aa3af6..d94cc01e4cad412fab9056885eba44efaf2ab018 100644 (file)
--- a/src/store/processes/processes-actions.ts
+++ b/src/store/processes/processes-actions.ts
@@ -8,16 +8,19 @@ import { ServiceRepository } from '~/services/services';
 import { updateResources } from '~/store/resources/resources-actions';
 import { FilterBuilder } from '~/common/api/filter-builder';
 import { ContainerRequestResource } from '../../models/container-request';
+import { Process } from './process';
 
 export const loadProcess = (containerRequestUuid: string) =>
-    async (dispatch: Dispatch, getState: () => RootState, services: ServiceRepository) => {
+    async (dispatch: Dispatch, getState: () => RootState, services: ServiceRepository): Promise<Process> => {
         const containerRequest = await services.containerRequestService.get(containerRequestUuid);
         dispatch<any>(updateResources([containerRequest]));
         if (containerRequest.containerUuid) {
             const container = await services.containerService.get(containerRequest.containerUuid);
             dispatch<any>(updateResources([container]));
             await dispatch<any>(loadSubprocesses(containerRequest.containerUuid));
+            return { containerRequest, container };
         }
+        return { containerRequest };
     };
 
 export const loadSubprocesses = (containerUuid: string) =>

diff --git a/src/store/store.ts b/src/store/store.ts
index 4fe0f97a697718ee2e5c37b0375291d658b33e25..d0c0dd67b10453d3a11191ecd7418e10dc7b00bc 100644 (file)
--- a/src/store/store.ts
+++ b/src/store/store.ts
@@ -28,6 +28,7 @@ import { resourcesReducer } from '~/store/resources/resources-reducer';
 import { propertiesReducer } from './properties/properties-reducer';
 import { RootState } from './store';
 import { fileUploaderReducer } from './file-uploader/file-uploader-reducer';
+import { processLogsPanelReducer } from './process-logs-panel/process-logs-panel-reducer';
 
 const composeEnhancers =
     (process.env.NODE_ENV === 'development' &&
@@ -68,6 +69,7 @@ const createRootReducer = (services: ServiceRepository) => combineReducers({
     dialog: dialogReducer,
     favorites: favoritesReducer,
     form: formReducer,
+    processLogsPanel: processLogsPanelReducer,
     properties: propertiesReducer,
     resources: resourcesReducer,
     router: routerReducer,

diff --git a/src/websocket/resource-event-message.ts b/src/websocket/resource-event-message.ts
index 274b2e1204a5362e4efa4be93eb8ee86dd498e58..05553bd9c36ab723c27d60ce7b2dea7dc84c03ef 100644 (file)
--- a/src/websocket/resource-event-message.ts
+++ b/src/websocket/resource-event-message.ts
@@ -1,18 +1,11 @@
+import { LogEventType } from '../models/log';
 // Copyright (C) The Arvados Authors. All rights reserved.
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-export enum ResourceEventMessageType {
-    CREATE = 'create',
-    UPDATE = 'update',
-    HOTSTAT = 'hotstat',
-    CRUNCH_RUN = 'crunch-run',
-    NODE_INFO = 'node-info',
-}
-
 export interface ResourceEventMessage {
     eventAt: string;
-    eventType: ResourceEventMessageType;
+    eventType: LogEventType;
     id: string;
     msgID: string;
     objectKind: string;

diff --git a/src/websocket/websocket.ts b/src/websocket/websocket.ts
index d7072d7a29d9c3783347e5c20066ea1e1f46ae14..9ee2d17e225073e2f152404e55a15d961d40cbdb 100644 (file)
--- a/src/websocket/websocket.ts
+++ b/src/websocket/websocket.ts
@@ -6,11 +6,12 @@ import { RootStore } from '~/store/store';
 import { AuthService } from '~/services/auth-service/auth-service';
 import { Config } from '~/common/config';
 import { WebSocketService } from './websocket-service';
-import { ResourceEventMessage, ResourceEventMessageType } from './resource-event-message';
+import { ResourceEventMessage } from './resource-event-message';
 import { ResourceKind } from '~/models/resource';
 import { loadProcess } from '~/store/processes/processes-actions';
 import { loadContainers } from '../store/processes/processes-actions';
 import { FilterBuilder } from '~/common/api/filter-builder';
+import { LogEventType } from '../models/log';
 
 export const initWebSocket = (config: Config, authService: AuthService, store: RootStore) => {
     const webSocketService = new WebSocketService(config.websocketUrl, authService);
@@ -19,7 +20,7 @@ export const initWebSocket = (config: Config, authService: AuthService, store: R
 };
 
 const messageListener = (store: RootStore) => (message: ResourceEventMessage) => {
-    if (message.eventType === ResourceEventMessageType.CREATE || message.eventType === ResourceEventMessageType.UPDATE) {
+    if (message.eventType === LogEventType.CREATE || message.eventType === LogEventType.UPDATE) {
         switch (message.objectKind) {
             case ResourceKind.CONTAINER_REQUEST:
                 return store.dispatch(loadProcess(message.objectUuid));