From: Lucas Di Pentima <lucas@di-pentima.com.ar>
Date: Tue, 23 Jun 2020 14:37:54 +0000 (-0300)
Subject: 15610: Uses a virtualized list to show the collection's file tree. (WIP)
X-Git-Tag: 2.1.0~22^2~9
X-Git-Url: https://git.arvados.org/arvados-workbench2.git/commitdiff_plain/e92207c912aed73a07340b5fb2a9e2cb23e1da5f

15610: Uses a virtualized list to show the collection's file tree. (WIP)

This greatly improves rendering times when showing collections with many
files on a directory.
This is a POC: the whole tree is rendered expanded to show that it doesn't
affect render times, it still needs lots of tweaking to offer the same
behavior as before.

Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas@di-pentima.com.ar>
---

diff --git a/package.json b/package.json
index 0efdbd7d..57c6e311 100644
--- a/package.json
+++ b/package.json
@@ -17,6 +17,8 @@
     "@types/react-copy-to-clipboard": "4.2.6",
     "@types/react-dropzone": "4.2.2",
     "@types/react-highlight-words": "0.12.0",
+    "@types/react-virtualized-auto-sizer": "1.0.0",
+    "@types/react-window": "1.8.2",
     "@types/redux-form": "7.4.12",
     "@types/shell-quote": "1.6.0",
     "axios": "0.18.1",
@@ -53,6 +55,8 @@
     "react-scripts-ts": "3.1.0",
     "react-splitter-layout": "3.0.1",
     "react-transition-group": "2.5.0",
+    "react-virtualized-auto-sizer": "1.0.2",
+    "react-window": "1.8.5",
     "redux": "4.0.3",
     "redux-form": "7.4.2",
     "redux-thunk": "2.3.0",
diff --git a/src/components/file-tree/file-tree.tsx b/src/components/file-tree/file-tree.tsx
index 34a11cd6..b5d98c08 100644
--- a/src/components/file-tree/file-tree.tsx
+++ b/src/components/file-tree/file-tree.tsx
@@ -3,7 +3,8 @@
 // SPDX-License-Identifier: AGPL-3.0
 
 import * as React from "react";
-import { Tree, TreeItem, TreeItemStatus } from "../tree/tree";
+import { TreeItem, TreeItemStatus } from "../tree/tree";
+import { VirtualTree as Tree } from "../tree/virtual-tree";
 import { FileTreeData } from "./file-tree-data";
 import { FileTreeItem } from "./file-tree-item";
 
diff --git a/src/components/tree/tree.tsx b/src/components/tree/tree.tsx
index 76fbf011..b5ce5ec5 100644
--- a/src/components/tree/tree.tsx
+++ b/src/components/tree/tree.tsx
@@ -79,6 +79,7 @@ export interface TreeItem<T> {
     selected?: boolean;
     status: TreeItemStatus;
     items?: Array<TreeItem<T>>;
+    level?: number;
 }
 
 export interface TreeProps<T> {
diff --git a/src/components/tree/virtual-tree.tsx b/src/components/tree/virtual-tree.tsx
new file mode 100644
index 00000000..4615db4f
--- /dev/null
+++ b/src/components/tree/virtual-tree.tsx
@@ -0,0 +1,251 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+import * as React from 'react';
+import { StyleRulesCallback, withStyles, WithStyles } from '@material-ui/core/styles';
+import { ReactElement } from "react";
+import { FixedSizeList, ListChildComponentProps } from "react-window";
+import AutoSizer from "react-virtualized-auto-sizer";
+// import {FixedSizeTree as Tree} from 'react-vtree';
+
+import { ArvadosTheme } from '~/common/custom-theme';
+import { TreeItem } from './tree';
+// import { FileTreeData } from '../file-tree/file-tree-data';
+
+type CssRules = 'list'
+    | 'listItem'
+    | 'active'
+    | 'loader'
+    | 'toggableIconContainer'
+    | 'iconClose'
+    | 'renderContainer'
+    | 'iconOpen'
+    | 'toggableIcon'
+    | 'checkbox'
+    | 'virtualizedList';
+
+const styles: StyleRulesCallback<CssRules> = (theme: ArvadosTheme) => ({
+    list: {
+        padding: '3px 0px',
+    },
+    virtualizedList: {
+        height: '200px',
+    },
+    listItem: {
+        padding: '3px 0px',
+    },
+    loader: {
+        position: 'absolute',
+        transform: 'translate(0px)',
+        top: '3px'
+    },
+    toggableIconContainer: {
+        color: theme.palette.grey["700"],
+        height: '14px',
+        width: '14px',
+    },
+    toggableIcon: {
+        fontSize: '14px'
+    },
+    renderContainer: {
+        flex: 1
+    },
+    active: {
+        color: theme.palette.primary.main,
+    },
+    iconClose: {
+        transition: 'all 0.1s ease',
+    },
+    iconOpen: {
+        transition: 'all 0.1s ease',
+        transform: 'rotate(90deg)',
+    },
+    checkbox: {
+        width: theme.spacing.unit * 3,
+        height: theme.spacing.unit * 3,
+        margin: `0 ${theme.spacing.unit}px`,
+        padding: 0,
+        color: theme.palette.grey["500"],
+    }
+});
+
+export interface TreeProps<T> {
+    disableRipple?: boolean;
+    currentItemUuid?: string;
+    items?: Array<TreeItem<T>>;
+    level?: number;
+    onContextMenu: (event: React.MouseEvent<HTMLElement>, item: TreeItem<T>) => void;
+    render: (item: TreeItem<T>, level?: number) => ReactElement<{}>;
+    showSelection?: boolean | ((item: TreeItem<T>) => boolean);
+    levelIndentation?: number;
+    itemRightPadding?: number;
+    toggleItemActive: (event: React.MouseEvent<HTMLElement>, item: TreeItem<T>) => void;
+    toggleItemOpen: (event: React.MouseEvent<HTMLElement>, item: TreeItem<T>) => void;
+    toggleItemSelection?: (event: React.MouseEvent<HTMLElement>, item: TreeItem<T>) => void;
+
+    /**
+     * When set to true use radio buttons instead of checkboxes for item selection.
+     * This does not guarantee radio group behavior (i.e item mutual exclusivity).
+     * Any item selection logic must be done in the toggleItemActive callback prop.
+     */
+    useRadioButtons?: boolean;
+}
+
+// export const RowA = <T, _>(items: TreeItem<T>[], render:any) => (index: number) => {
+//     return <div>
+//         {render(items[index])}
+//     </div>;
+// };
+
+// For some reason, on TSX files it isn't accepted just one generic param, so
+// I'm using <T, _> as a workaround.
+export const Row = <T, _>(items: TreeItem<T>[], render: any) => (props: React.PropsWithChildren<ListChildComponentProps>) => {
+    const { index, style } = props;
+    const level = items[index].level || 0;
+    const levelIndentation = 20;
+    return <div style={style}>
+        <div style={{ paddingLeft: (level + 1) * levelIndentation,}}>
+            {typeof render === 'function'
+                ? items[index] && render(items[index]) || ''
+                : 'whoops'}
+        </div>
+    </div>;
+    // <div style={style} key={`item/${level}/${idx}`}>
+    //     <ListItem button className={listItem}
+    //         style={{
+    //             paddingLeft: (level + 1) * levelIndentation,
+    //             paddingRight: itemRightPadding,
+    //         }}
+    //         disableRipple={disableRipple}
+    //         onClick={event => toggleItemActive(event, it)}
+    //         selected={showSelection(it) && it.id === currentItemUuid}
+    //         onContextMenu={this.handleRowContextMenu(it)}>
+    //         {it.status === TreeItemStatus.PENDING ?
+    //             <CircularProgress size={10} className={loader} /> : null}
+    //         <i onClick={this.handleToggleItemOpen(it)}
+    //             className={toggableIconContainer}>
+    //             <ListItemIcon className={this.getToggableIconClassNames(it.open, it.active)}>
+    //                 {this.getProperArrowAnimation(it.status, it.items!)}
+    //             </ListItemIcon>
+    //         </i>
+    //         {showSelection(it) && !useRadioButtons &&
+    //             <Checkbox
+    //                 checked={it.selected}
+    //                 className={classes.checkbox}
+    //                 color="primary"
+    //                 onClick={this.handleCheckboxChange(it)} />}
+    //         {showSelection(it) && useRadioButtons &&
+    //             <Radio
+    //                 checked={it.selected}
+    //                 className={classes.checkbox}
+    //                 color="primary" />}
+    //         <div className={renderContainer}>
+    //             {render(it, level)}
+    //         </div>
+    //     </ListItem>
+    //     {it.items && it.items.length > 0 &&
+    //         <Collapse in={it.open} timeout="auto" unmountOnExit>
+    //             <Tree
+    //                 showSelection={this.props.showSelection}
+    //                 items={it.items}
+    //                 render={render}
+    //                 disableRipple={disableRipple}
+    //                 toggleItemOpen={toggleItemOpen}
+    //                 toggleItemActive={toggleItemActive}
+    //                 level={level + 1}
+    //                 onContextMenu={onContextMenu}
+    //                 toggleItemSelection={this.props.toggleItemSelection} />
+    //         </Collapse>}
+    // </div>
+};
+
+export const VirtualList = <T, _>(height: number, width: number, items: TreeItem<T>[], render: any) =>
+    <FixedSizeList
+        height={height}
+        itemCount={items.length}
+        itemSize={30}
+        width={width}
+    >
+        {Row(items, render)}
+    </FixedSizeList>;
+
+export const VirtualTree = withStyles(styles)(
+    class Component<T> extends React.Component<TreeProps<T> & WithStyles<CssRules>, {}> {
+        render(): ReactElement<any> {
+            const { items, render } = this.props;
+
+            return <div className={this.props.classes.virtualizedList}><AutoSizer>
+                {({ height, width }) => {
+                    return VirtualList(height, width, items || [], render);
+                }}
+            </AutoSizer></div>;
+        }
+    }
+);
+
+// const treeWalkerWithTree = (tree: Array<TreeItem<FileTreeData>>) => function* treeWalker(refresh: any) {
+//     const stack = [];
+
+//     // Remember all the necessary data of the first node in the stack.
+//     stack.push({
+//       nestingLevel: 0,
+//       node: tree,
+//     });
+
+//     // Walk through the tree until we have no nodes available.
+//     while (stack.length !== 0) {
+//         const {
+//             node: {items = [], id, name},
+//             nestingLevel,
+//         } = stack.pop()!;
+
+//         // Here we are sending the information about the node to the Tree component
+//         // and receive an information about the openness state from it. The
+//         // `refresh` parameter tells us if the full update of the tree is requested;
+//         // basing on it we decide to return the full node data or only the node
+//         // id to update the nodes order.
+//         const isOpened = yield refresh
+//             ? {
+//                 id,
+//                 isLeaf: items.length === 0,
+//                 isOpenByDefault: true,
+//                 name,
+//                 nestingLevel,
+//             }
+//             : id;
+
+//         // Basing on the node openness state we are deciding if we need to render
+//         // the child nodes (if they exist).
+//         if (children.length !== 0 && isOpened) {
+//             // Since it is a stack structure, we need to put nodes we want to render
+//             // first to the end of the stack.
+//             for (let i = children.length - 1; i >= 0; i--) {
+//                 stack.push({
+//                     nestingLevel: nestingLevel + 1,
+//                     node: children[i],
+//                 });
+//             }
+//         }
+//     }
+// };
+
+// // Node component receives all the data we created in the `treeWalker` +
+// // internal openness state (`isOpen`), function to change internal openness
+// // state (`toggle`) and `style` parameter that should be added to the root div.
+// const Node = ({data: {isLeaf, name}, isOpen, style, toggle}) => (
+//     <div style={style}>
+//         {!isLeaf && (
+//         <button type="button" onClick={toggle}>
+//             {isOpen ? '-' : '+'}
+//         </button>
+//         )}
+//         <div>{name}</div>
+//     </div>
+// );
+
+// export const Example = () => (
+//     <Tree treeWalker={treeWalker} itemSize={30} height={150} width={300}>
+//         {Node}
+//     </Tree>
+// );
\ No newline at end of file
diff --git a/src/models/tree.ts b/src/models/tree.ts
index c7713cbc..69224059 100644
--- a/src/models/tree.ts
+++ b/src/models/tree.ts
@@ -16,6 +16,7 @@ export interface TreeNode<T = any> {
     selected: boolean;
     expanded: boolean;
     status: TreeNodeStatus;
+    level?: number;
 }
 
 export enum TreeNodeStatus {
@@ -193,6 +194,7 @@ export const initTreeNode = <T>(data: Pick<TreeNode<T>, 'id' | 'value'> & { pare
     expanded: false,
     status: TreeNodeStatus.INITIAL,
     parent: '',
+    level: 0,
     ...data,
 });
 
diff --git a/src/store/collection-panel/collection-panel-files/collection-panel-files-actions.ts b/src/store/collection-panel/collection-panel-files/collection-panel-files-actions.ts
index 204d4c0e..175a8cef 100644
--- a/src/store/collection-panel/collection-panel-files/collection-panel-files-actions.ts
+++ b/src/store/collection-panel/collection-panel-files/collection-panel-files-actions.ts
@@ -31,15 +31,25 @@ export const COLLECTION_PANEL_LOAD_FILES_THRESHOLD = 40000;
 
 export const loadCollectionFiles = (uuid: string) =>
     async (dispatch: Dispatch, getState: () => RootState, services: ServiceRepository) => {
+        let step = Date.now();
         dispatch(progressIndicatorActions.START_WORKING(COLLECTION_PANEL_LOAD_FILES));
         const files = await services.collectionService.files(uuid);
+        console.log('Get files: ', (Date.now()-step)/1000);
 
         // Given the array of directories and files, create the appropriate tree nodes,
         // sort them, and add the complete url to each.
+        step = Date.now();
         const tree = createCollectionFilesTree(files);
+        console.log('Create tree: ', (Date.now()-step)/1000);
+        step = Date.now();
         const sorted = sortFilesTree(tree);
+        console.log('Sort tree: ', (Date.now()-step)/1000);
+        step = Date.now();
         const mapped = mapTreeValues(services.collectionService.extendFileURL)(sorted);
+        console.log('Add URL: ', (Date.now()-step)/1000);
+        step = Date.now();
         dispatch(collectionPanelFilesAction.SET_COLLECTION_FILES(mapped));
+        console.log('Dispatch: ', (Date.now()-step)/1000);
         dispatch(progressIndicatorActions.STOP_WORKING(COLLECTION_PANEL_LOAD_FILES));
     };
 
diff --git a/src/views-components/collection-panel-files/collection-panel-files.ts b/src/views-components/collection-panel-files/collection-panel-files.ts
index eb16eb6c..e0798086 100644
--- a/src/views-components/collection-panel-files/collection-panel-files.ts
+++ b/src/views-components/collection-panel-files/collection-panel-files.ts
@@ -32,8 +32,12 @@ const memoizedMapStateToProps = () => {
     return (state: RootState): Pick<CollectionPanelFilesProps, "items" | "currentItemUuid"> => {
         if (prevState !== state.collectionPanelFiles) {
             prevState = state.collectionPanelFiles;
-            prevTree = getNodeChildrenIds('')(state.collectionPanelFiles)
-                .map(collectionItemToTreeItem(state.collectionPanelFiles));
+            prevTree = [].concat.apply(
+                [], getNodeChildrenIds('')(state.collectionPanelFiles)
+                    .map(collectionItemToList(0)(state.collectionPanelFiles)))
+                .map(nodeToTreeItem);
+            // prevTree = getNodeChildrenIds('')(state.collectionPanelFiles)
+            //     .map(collectionItemToTreeItem(state.collectionPanelFiles));
         }
         return {
             items: prevTree,
@@ -77,6 +81,24 @@ const mapDispatchToProps = (dispatch: Dispatch): Pick<CollectionPanelFilesProps,
 
 export const CollectionPanelFiles = connect(memoizedMapStateToProps(), mapDispatchToProps)(Component);
 
+export const collectionItemToList = (level: number) => (tree: Tree<CollectionPanelDirectory | CollectionPanelFile>) =>
+    (id: string): TreeItem<FileTreeData>[] => {
+        const node: TreeNode<CollectionPanelDirectory | CollectionPanelFile> = getNode(id)(tree) || initTreeNode({
+            id: '',
+            parent: '',
+            value: {
+                ...createCollectionDirectory({ name: 'Invalid file' }),
+                selected: false,
+                collapsed: true
+            }
+        });
+        const childs = [].concat.apply([], node.children.map(collectionItemToList(level+1)(tree)));
+        return [
+            {...node, level},
+            ...childs,
+        ];
+    };
+
 const collectionItemToTreeItem = (tree: Tree<CollectionPanelDirectory | CollectionPanelFile>) =>
     (id: string): TreeItem<FileTreeData> => {
         const node: TreeNode<CollectionPanelDirectory | CollectionPanelFile> = getNode(id)(tree) || initTreeNode({
@@ -104,3 +126,21 @@ const collectionItemToTreeItem = (tree: Tree<CollectionPanelDirectory | Collecti
             status: TreeItemStatus.LOADED
         };
     };
+
+const nodeToTreeItem = (node: TreeNode<CollectionPanelDirectory | CollectionPanelFile>): TreeItem<FileTreeData> => {
+    return ({
+        active: false,
+        data: {
+            name: node.value.name,
+            size: node.value.type === CollectionFileType.FILE ? node.value.size : undefined,
+            type: node.value.type,
+            url: node.value.url,
+        },
+        id: node.id,
+        items: [],
+        open: node.value.type === CollectionFileType.DIRECTORY ? !node.value.collapsed : false,
+        selected: node.value.selected,
+        status: TreeItemStatus.LOADED,
+        level: node.level,
+    });
+};
\ No newline at end of file
diff --git a/yarn.lock b/yarn.lock
index d6677a74..742db465 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -427,6 +427,20 @@
   dependencies:
     "@types/react" "*"
 
+"@types/react-virtualized-auto-sizer@1.0.0":
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/@types/react-virtualized-auto-sizer/-/react-virtualized-auto-sizer-1.0.0.tgz#fc32f30a8dab527b5816f3a757e1e1d040c8f272"
+  integrity sha512-NMErdIdSnm2j/7IqMteRiRvRulpjoELnXWUwdbucYCz84xG9PHcoOrr7QfXwB/ku7wd6egiKFrzt/+QK4Imeeg==
+  dependencies:
+    "@types/react" "*"
+
+"@types/react-window@1.8.2":
+  version "1.8.2"
+  resolved "https://registry.yarnpkg.com/@types/react-window/-/react-window-1.8.2.tgz#a5a6b2762ce73ffaab7911ee1397cf645f2459fe"
+  integrity sha512-gP1xam68Wc4ZTAee++zx6pTdDAH08rAkQrWm4B4F/y6hhmlT9Mgx2q8lTCXnrPHXsr15XjRN9+K2DLKcz44qEQ==
+  dependencies:
+    "@types/react" "*"
+
 "@types/react@*":
   version "16.9.11"
   resolved "https://registry.yarnpkg.com/@types/react/-/react-16.9.11.tgz#70e0b7ad79058a7842f25ccf2999807076ada120"
@@ -7063,6 +7077,11 @@ mem@^4.0.0:
     mimic-fn "^2.0.0"
     p-is-promise "^2.0.0"
 
+"memoize-one@>=3.1.1 <6":
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/memoize-one/-/memoize-one-5.1.1.tgz#047b6e3199b508eaec03504de71229b8eb1d75c0"
+  integrity sha512-HKeeBpWvqiVJD57ZUAsJNm71eHTykffzcLZVYWiVfQeI1rJtuEaS7hQiEpWfVVk18donPwJEcFKIkCmPJNOhHA==
+
 memoize-one@^4.0.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/memoize-one/-/memoize-one-4.1.0.tgz#a2387c58c03fff27ca390c31b764a79addf3f906"
@@ -9081,6 +9100,19 @@ react-transition-group@^2.2.1:
     prop-types "^15.6.2"
     react-lifecycles-compat "^3.0.4"
 
+react-virtualized-auto-sizer@1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/react-virtualized-auto-sizer/-/react-virtualized-auto-sizer-1.0.2.tgz#a61dd4f756458bbf63bd895a92379f9b70f803bd"
+  integrity sha512-MYXhTY1BZpdJFjUovvYHVBmkq79szK/k7V3MO+36gJkWGkrXKtyr4vCPtpphaTLRAdDNoYEYFZWE8LjN+PIHNg==
+
+react-window@1.8.5:
+  version "1.8.5"
+  resolved "https://registry.yarnpkg.com/react-window/-/react-window-1.8.5.tgz#a56b39307e79979721021f5d06a67742ecca52d1"
+  integrity sha512-HeTwlNa37AFa8MDZFZOKcNEkuF2YflA0hpGPiTT9vR7OawEt+GZbfM6wqkBahD3D3pUjIabQYzsnY/BSJbgq6Q==
+  dependencies:
+    "@babel/runtime" "^7.0.0"
+    memoize-one ">=3.1.1 <6"
+
 react@16.8.6:
   version "16.8.6"
   resolved "https://registry.yarnpkg.com/react/-/react-16.8.6.tgz#ad6c3a9614fd3a4e9ef51117f54d888da01f2bbe"