From c80ba692e79e271232c93305347c8c5d2a7a8842 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Fri, 12 Jul 2024 15:36:14 +1000 Subject: [PATCH 1/3] chore: added `minimatch` as a dependency [ci skip] --- package-lock.json | 176 ++++++++++++++++++++++++++++++++++++++++------ package.json | 4 +- 2 files changed, 156 insertions(+), 24 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2a6a9a059..0c7465b61 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,6 +43,7 @@ "isomorphic-git": "^1.8.1", "ix": "^5.0.0", "lexicographic-integer": "^1.1.0", + "minimatch": "^10.0.1", "multiformats": "^9.4.8", "pako": "^1.0.11", "prompts": "^2.4.1", @@ -56,6 +57,7 @@ "@swc/jest": "^0.2.29", "@types/cross-spawn": "^6.0.2", "@types/jest": "^29.5.2", + "@types/minimatch": "^5.1.2", "@types/node": "^20.5.7", "@types/pako": "^1.0.2", "@types/prompts": "^2.0.13", @@ -881,12 +883,34 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/@eslint/eslintrc/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", "dev": true }, + "node_modules/@eslint/eslintrc/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/@eslint/js": { "version": "8.51.0", "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.51.0.tgz", @@ -942,6 +966,28 @@ "node": ">=10.10.0" } }, + "node_modules/@humanwhocodes/config-array/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/@humanwhocodes/config-array/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/@humanwhocodes/module-importer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", @@ -2333,6 +2379,12 @@ "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "dev": true }, + "node_modules/@types/minimatch": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz", + "integrity": "sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==", + "dev": true + }, "node_modules/@types/node": { "version": "20.8.5", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.8.5.tgz", @@ -3009,8 +3061,7 @@ "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "node_modules/benchmark": { "version": "2.1.4", @@ -3086,13 +3137,11 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" + "balanced-match": "^1.0.0" } }, "node_modules/braces": { @@ -4207,6 +4256,16 @@ "eslint": "^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8" } }, + "node_modules/eslint-plugin-import/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, "node_modules/eslint-plugin-import/node_modules/debug": { "version": "3.2.7", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", @@ -4228,6 +4287,18 @@ "node": ">=0.10.0" } }, + "node_modules/eslint-plugin-import/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/eslint-plugin-import/node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", @@ -4307,6 +4378,16 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/eslint/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, "node_modules/eslint/node_modules/eslint-scope": { "version": "7.2.2", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", @@ -4338,6 +4419,18 @@ "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", "dev": true }, + "node_modules/eslint/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/esm": { "version": "3.2.25", "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", @@ -4829,6 +4922,28 @@ "node": ">=10.13.0" } }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/globals": { "version": "13.23.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.23.0.tgz", @@ -6489,15 +6604,17 @@ } }, "node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.1.tgz", + "integrity": "sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==", "dependencies": { - "brace-expansion": "^1.1.7" + "brace-expansion": "^2.0.1" }, "engines": { - "node": "*" + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/minimist": { @@ -7919,6 +8036,28 @@ "node": ">=8" } }, + "node_modules/test-exclude/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/test-exclude/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -8284,15 +8423,6 @@ "typescript": "4.6.x || 4.7.x || 4.8.x || 4.9.x || 5.0.x || 5.1.x" } }, - "node_modules/typedoc/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, - "dependencies": { - "balanced-match": "^1.0.0" - } - }, "node_modules/typedoc/node_modules/minimatch": { "version": "9.0.3", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", diff --git a/package.json b/package.json index 0469c74ba..d397bf3cb 100644 --- a/package.json +++ b/package.json @@ -104,7 +104,8 @@ "prompts": "^2.4.1", "resource-counter": "^1.2.4", "sodium-native": "^3.4.1", - "threads": "^1.6.5" + "threads": "^1.6.5", + "minimatch": "^10.0.1" }, "devDependencies": { "@fast-check/jest": "^1.1.0", @@ -116,6 +117,7 @@ "@types/pako": "^1.0.2", "@types/prompts": "^2.0.13", "@types/readable-stream": "^2.3.11", + "@types/minimatch": "^5.1.2", "@typescript-eslint/eslint-plugin": "^5.61.0", "@typescript-eslint/parser": "^5.61.0", "benny": "^3.7.1", From 48b78af477d3d453d86c14c36eeda256cf45adb3 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Thu, 11 Jul 2024 17:15:47 +1000 Subject: [PATCH 2/3] feat: created a globbing file tree walker utility --- src/vaults/types.ts | 61 +++++ src/vaults/utils.ts | 180 +++++++++++++- tests/vaults/VaultOps.test.ts | 52 ++++ tests/vaults/utils.test.ts | 433 +++++++++++++++++++++++++++++++++- 4 files changed, 724 insertions(+), 2 deletions(-) diff --git a/src/vaults/types.ts b/src/vaults/types.ts index 3c1643c1a..6e54e3b10 100644 --- a/src/vaults/types.ts +++ b/src/vaults/types.ts @@ -134,6 +134,58 @@ type VaultName = string; type VaultActions = Partial>; +type FileTree = Array; +type TreeNode = DirectoryNode | FileNode | ContentNode; +type FilePath = string; +type INode = number; +type CNode = number; + +type StatEncoded = { + isSymbolicLink: boolean; + type: 'FILE' | 'DIRECTORY' | 'OTHER'; + dev: number; + ino: number; + mode: number; + nlink: number; + uid: number; + gid: number; + rdev: number; + size: number; + blksize: number; + blocks: number; + atime: number; + mtime: number; + ctime: number; + birthtime: number; +}; + +type DirectoryNode = { + type: 'directory'; + path: FilePath; + iNode: INode; + parent: INode; + children: Array; + stat?: StatEncoded; +}; + +type FileNode = { + type: 'file'; + path: FilePath; + iNode: INode; + parent: INode; + cNode: CNode; + stat?: StatEncoded; +}; + +// Keeping this separate from `FileNode` so we can optionally not include it. +type ContentNode = { + type: 'content'; + path: undefined; + fileName: string; + cNode: CNode; + contents: string; +}; + export { vaultActions }; export type { @@ -148,6 +200,15 @@ export type { FileSystemWritable, VaultName, VaultActions, + FileTree, + TreeNode, + FilePath, + INode, + CNode, + StatEncoded, + DirectoryNode, + FileNode, + ContentNode, }; export { tagLast, refs }; diff --git a/src/vaults/utils.ts b/src/vaults/utils.ts index 449cd4adb..357a04d9e 100644 --- a/src/vaults/utils.ts +++ b/src/vaults/utils.ts @@ -1,14 +1,20 @@ -import type { EncryptedFS } from 'encryptedfs'; +import type { EncryptedFS, Stat } from 'encryptedfs'; +import type { FileSystem } from '../types'; import type { VaultRef, VaultAction, CommitId, FileSystemReadable, FileSystemWritable, + TreeNode, + DirectoryNode, + INode, + StatEncoded, } from './types'; import type { NodeId } from '../ids/types'; import type { Path } from 'encryptedfs/dist/types'; import path from 'path'; +import { minimatch } from 'minimatch'; import { pathJoin } from 'encryptedfs/dist/utils'; import * as vaultsErrors from './errors'; import { tagLast, refs, vaultActions } from './types'; @@ -123,6 +129,177 @@ async function mkdirExists(efs: FileSystemWritable, directory: string) { } } +function genStat(stat: Stat): StatEncoded { + return { + isSymbolicLink: stat.isSymbolicLink(), + type: stat.isFile() ? 'FILE' : stat.isDirectory() ? 'DIRECTORY' : 'OTHER', + dev: stat.dev, + ino: stat.ino, + mode: stat.mode, + nlink: stat.nlink, + uid: stat.uid, + gid: stat.gid, + rdev: stat.rdev, + size: stat.size, + blksize: stat.blksize, + blocks: stat.blocks, + atime: stat.atime.getTime(), + mtime: stat.mtime.getTime(), + ctime: stat.ctime.getTime(), + birthtime: stat.birthtime.getTime(), + }; +} + +/** + * This is a utility for walking a file tree while matching a file path globstar pattern. + * @param fs - file system to work against, supports nodes `fs` and our `FileSystemReadable` provided by vaults. + * @param basePath - The path to start walking from. + * @param pattern - The pattern to match against, defaults to everything + * @param yieldRoot - toggles yielding details of the basePath. Defaults to true. + * @param yieldParents - Toggles yielding details about parents of pattern matched paths. Defaults to false. + * @param yieldDirectories - Toggles yielding directories that match the pattern. Defaults to true. + * @param yieldFiles - Toggles yielding files that match the pattern. Defaults to true. + * @param yieldContents - Toggles yielding file contents after all other details are yielded. Defaults to false. + * @param yieldStats - Toggles including stats in file and directory details. Defaults to false. + */ +async function* globWalk({ + fs, + basePath = '.', + pattern = '**/*', + yieldRoot = true, + yieldParents = false, + yieldDirectories = true, + yieldFiles = true, + yieldContents = false, + yieldStats = false, +}: { + fs: FileSystem | FileSystemReadable; + basePath?: string; + pattern?: string; + yieldRoot?: boolean; + yieldParents?: boolean; + yieldDirectories?: boolean; + yieldFiles?: boolean; + yieldContents?: boolean; + yieldStats?: boolean; +}): AsyncGenerator { + const files: Array = []; + const directoryMap: Map = new Map(); + // Path, node, parent + const queue: Array<[string, INode, INode]> = []; + let iNode = 1; + const basePathNormalised = path.normalize(basePath); + let current: [string, INode, INode] | undefined = [basePathNormalised, 1, 0]; + + const getParents = (parentINode: INode) => { + const parents: Array = []; + let currentParent = parentINode; + while (true) { + const directory = directoryMap.get(currentParent); + directoryMap.delete(currentParent); + if (directory == null) break; + parents.unshift(directory); + currentParent = directory.parent; + } + return parents; + }; + + // Iterate over tree + const patternPath = path.join(basePathNormalised, pattern); + while (current != null) { + const [currentPath, node, parentINode] = current; + + const stat = await fs.promises.stat(currentPath); + if (stat.isDirectory()) { + // `.` and `./` will not partially match the pattern, so we exclude the initial path + if ( + !minimatch(currentPath, patternPath, { partial: true }) && + currentPath !== basePathNormalised + ) { + current = queue.shift(); + continue; + } + // @ts-ignore: While the types don't fully match, it matches enough for our usage. + const childrenPaths = await fs.promises.readdir(currentPath); + const children = childrenPaths.map( + (v) => + [path.join(currentPath!, v.toString()), ++iNode, node] as [ + string, + INode, + INode, + ], + ); + queue.push(...children); + // Only yield root if we specify it + if (yieldRoot || node !== 1) { + directoryMap.set(node, { + type: 'directory', + path: currentPath, + iNode: node, + parent: parentINode, + children: children.map((v) => v[1]), + stat: yieldStats ? genStat(stat) : undefined, + }); + } + // Wildcards can find directories so we need yield them too + if (minimatch(currentPath, patternPath)) { + // Remove current from parent list + directoryMap.delete(node); + // Yield parents + if (yieldParents) { + for (const parent of getParents(parentINode)) yield parent; + } + // Yield directory + if (yieldDirectories) { + yield { + type: 'directory', + path: currentPath, + iNode: node, + parent: parentINode, + children: children.map((v) => v[1]), + stat: yieldStats ? genStat(stat) : undefined, + }; + } + } + } else if (stat.isFile()) { + if (!minimatch(currentPath, patternPath)) { + current = queue.shift(); + continue; + } + // Get the directories in order + if (yieldParents) { + for (const parent of getParents(parentINode)) yield parent; + } + // Yield file. + if (yieldFiles) { + yield { + type: 'file', + path: currentPath, + iNode: node, + parent: parentINode, + cNode: files.length, + stat: yieldStats ? genStat(stat) : undefined, + }; + } + files.push(currentPath); + } + current = queue.shift(); + } + if (!yieldContents) return; + // Iterate over file contents + for (let i = 0; i < files.length; i++) { + const filePath = files[i]; + yield { + type: 'content', + path: undefined, + fileName: path.basename(filePath), + cNode: i, + // @ts-ignore: While the types don't fully match, it matches enough for our usage. + contents: (await fs.promises.readFile(filePath)).toString(), + }; + } +} + export { tagLast, refs, @@ -138,6 +315,7 @@ export { walkFs, deleteObject, mkdirExists, + globWalk, }; export { createVaultIdGenerator, encodeVaultId, decodeVaultId } from '../ids'; diff --git a/tests/vaults/VaultOps.test.ts b/tests/vaults/VaultOps.test.ts index ea34f63eb..6b57097e9 100644 --- a/tests/vaults/VaultOps.test.ts +++ b/tests/vaults/VaultOps.test.ts @@ -2,6 +2,7 @@ import type { VaultId } from '@/vaults/types'; import type { Vault } from '@/vaults/Vault'; import type KeyRing from '@/keys/KeyRing'; import type { LevelPath } from '@matrixai/db'; +import type { FileTree } from '@/vaults/types'; import fs from 'fs'; import path from 'path'; import os from 'os'; @@ -525,4 +526,55 @@ describe('VaultOps', () => { }, globalThis.defaultTimeout * 4, ); + + describe('globWalk', () => { + const relativeBase = '.'; + const dir1: string = 'dir1'; + const dir11: string = path.join(dir1, 'dir11'); + const file0b: string = 'file0.b'; + const file1a: string = path.join(dir1, 'file1.a'); + const file2b: string = path.join(dir1, 'file2.b'); + const file3a: string = path.join(dir11, 'file3.a'); + const file4b: string = path.join(dir11, 'file4.b'); + + beforeEach(async () => { + await vault.writeF(async (fs) => { + await fs.promises.mkdir(dir1); + await fs.promises.mkdir(dir11); + await fs.promises.writeFile(file0b, 'content-file0'); + await fs.promises.writeFile(file1a, 'content-file1'); + await fs.promises.writeFile(file2b, 'content-file2'); + await fs.promises.writeFile(file3a, 'content-file3'); + await fs.promises.writeFile(file4b, 'content-file4'); + }); + }); + + test('Works with efs', async () => { + const files = await vault.readF(async (fs) => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: '.', + yieldDirectories: true, + yieldFiles: true, + yieldParents: true, + yieldRoot: true, + yieldContents: false, + })) { + tree.push(treeNode); + } + return tree.map((v) => v.path ?? ''); + }); + expect(files).toContainAllValues([ + relativeBase, + dir1, + dir11, + file0b, + file1a, + file2b, + file3a, + file4b, + ]); + }); + }); }); diff --git a/tests/vaults/utils.test.ts b/tests/vaults/utils.test.ts index e0725455d..123f9f081 100644 --- a/tests/vaults/utils.test.ts +++ b/tests/vaults/utils.test.ts @@ -1,4 +1,4 @@ -import type { VaultId } from '@/vaults/types'; +import type { FileTree, VaultId } from '@/vaults/types'; import fs from 'fs'; import os from 'os'; import path from 'path'; @@ -83,4 +83,435 @@ describe('Vaults utils', () => { ).toBeUndefined(); expect(vaultsUtils.decodeVaultId('zF4VfxTOOSHORTxTV9')).toBeUndefined(); }); + + describe('globWalk', () => { + let cwd: string; + + const relativeBase = '.'; + const dir1: string = 'dir1'; + const dir2: string = 'dir2'; + const dir11: string = path.join(dir1, 'dir11'); + const dir12: string = path.join(dir1, 'dir12'); + const dir21: string = path.join(dir2, 'dir21'); + const dir22: string = path.join(dir2, 'dir22'); + const file0b: string = 'file0.b'; + const file1a: string = path.join(dir11, 'file1.a'); + const file2b: string = path.join(dir11, 'file2.b'); + const file3a: string = path.join(dir12, 'file3.a'); + const file4b: string = path.join(dir12, 'file4.b'); + const file5a: string = path.join(dir21, 'file5.a'); + const file6b: string = path.join(dir21, 'file6.b'); + const file7a: string = path.join(dir22, 'file7.a'); + const file8b: string = path.join(dir22, 'file8.b'); + const file9a: string = path.join(dir22, 'file9.a'); + + beforeEach(async () => { + await fs.promises.mkdir(path.join(dataDir, dir1)); + await fs.promises.mkdir(path.join(dataDir, dir11)); + await fs.promises.mkdir(path.join(dataDir, dir12)); + await fs.promises.mkdir(path.join(dataDir, dir2)); + await fs.promises.mkdir(path.join(dataDir, dir21)); + await fs.promises.mkdir(path.join(dataDir, dir22)); + await fs.promises.writeFile(path.join(dataDir, file0b), 'content-file0'); + await fs.promises.writeFile(path.join(dataDir, file1a), 'content-file1'); + await fs.promises.writeFile(path.join(dataDir, file2b), 'content-file2'); + await fs.promises.writeFile(path.join(dataDir, file3a), 'content-file3'); + await fs.promises.writeFile(path.join(dataDir, file4b), 'content-file4'); + await fs.promises.writeFile(path.join(dataDir, file5a), 'content-file5'); + await fs.promises.writeFile(path.join(dataDir, file6b), 'content-file6'); + await fs.promises.writeFile(path.join(dataDir, file7a), 'content-file7'); + await fs.promises.writeFile(path.join(dataDir, file8b), 'content-file8'); + await fs.promises.writeFile(path.join(dataDir, file9a), 'content-file9'); + cwd = process.cwd(); + process.chdir(dataDir); + }); + afterEach(async () => { + process.chdir(cwd); + }); + + test('Works with relative base path `.`', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + yieldDirectories: true, + yieldFiles: true, + yieldParents: true, + yieldRoot: true, + yieldContents: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path ?? ''); + expect(files).toContainAllValues([ + relativeBase, + dir1, + dir2, + dir11, + dir12, + dir21, + dir22, + file0b, + file1a, + file2b, + file3a, + file4b, + file5a, + file6b, + file7a, + file8b, + file9a, + ]); + }); + test('Works with relative base path `./`', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: './', + yieldDirectories: true, + yieldFiles: true, + yieldParents: true, + yieldRoot: true, + yieldContents: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path ?? ''); + expect(files).toContainAllValues([ + './', + dir1, + dir2, + dir11, + dir12, + dir21, + dir22, + file0b, + file1a, + file2b, + file3a, + file4b, + file5a, + file6b, + file7a, + file8b, + file9a, + ]); + }); + test('Works with relative base path `./dir1`', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: './dir1', + yieldDirectories: true, + yieldFiles: true, + yieldParents: true, + yieldRoot: true, + yieldContents: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path ?? ''); + expect(files).toContainAllValues([ + dir1, + dir11, + dir12, + file1a, + file2b, + file3a, + file4b, + ]); + }); + test('Works with absolute base path', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: dataDir, + yieldDirectories: true, + yieldFiles: true, + yieldParents: true, + yieldRoot: true, + yieldContents: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path ?? ''); + expect(files).toContainAllValues( + [ + relativeBase, + dir1, + dir2, + dir11, + dir12, + dir21, + dir22, + file0b, + file1a, + file2b, + file3a, + file4b, + file5a, + file6b, + file7a, + file8b, + file9a, + ].map((v) => path.join(dataDir, v)), + ); + }); + test('Yields parent directories with `yieldParents`', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + yieldParents: true, + yieldFiles: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path); + expect(files).toContainAllValues([ + relativeBase, + dir2, + dir1, + dir11, + dir12, + dir21, + dir22, + ]); + }); + test('Does not yield the base path with `yieldParents` and `yieldRoot`', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + yieldRoot: false, + yieldParents: true, + yieldFiles: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path); + expect(files).not.toInclude(relativeBase); + expect(files).toContainAllValues([ + dir2, + dir1, + dir11, + dir12, + dir21, + dir22, + ]); + }); + test('Does not yield the base path with `yieldParents` and `yieldRoot` and absolute paths', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: dataDir, + yieldRoot: false, + yieldParents: true, + yieldFiles: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path); + expect(files).not.toInclude(dataDir); + expect(files).toContainAllValues( + [dir2, dir1, dir11, dir12, dir21, dir22].map((v) => + path.join(dataDir, v), + ), + ); + }); + test('Yields file contents directories with `yieldContents`', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + yieldFiles: false, + yieldDirectories: false, + yieldContents: true, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => (v.type === 'content' ? v.contents : '')); + expect(files).toContainAllValues([ + 'content-file0', + 'content-file9', + 'content-file1', + 'content-file2', + 'content-file3', + 'content-file4', + 'content-file5', + 'content-file6', + 'content-file7', + 'content-file8', + ]); + }); + test('Yields stats with `yieldStats`', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + yieldStats: true, + yieldFiles: true, + yieldDirectories: true, + yieldContents: false, + })) { + tree.push(treeNode); + } + tree.forEach((v) => + v.type === 'directory' || v.type === 'file' + ? expect(v.stat).toBeDefined() + : '', + ); + }); + // Globbing examples + test('glob with wildcard', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + pattern: '*', + yieldFiles: true, + yieldDirectories: true, + yieldParents: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path); + expect(files).toContainAllValues([dir1, dir2, file0b]); + }); + test('glob with wildcard ignores directories with `yieldDirectories: false`', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + pattern: '*', + yieldFiles: true, + yieldDirectories: false, + yieldParents: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path); + expect(files).not.toContainAllValues([relativeBase, dir1, dir2]); + expect(files).toContainAllValues([file0b]); + }); + test('glob with wildcard ignores files with `yieldFiles: false`', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + pattern: '*', + yieldFiles: false, + yieldDirectories: true, + yieldParents: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path); + expect(files).not.toContainAllValues([file0b]); + expect(files).toContainAllValues([dir1, dir2]); + }); + test('glob with globstar', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + pattern: '**', + yieldFiles: true, + yieldDirectories: true, + yieldParents: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path); + expect(files).not.toInclude(relativeBase); + expect(files).toContainAllValues([ + dir1, + dir2, + file0b, + dir11, + dir12, + dir21, + dir22, + file1a, + file2b, + file3a, + file4b, + file5a, + file6b, + file7a, + file8b, + file9a, + ]); + }); + test('glob with globstar and directory pattern', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + pattern: '**/dir2/**', + yieldFiles: true, + yieldDirectories: true, + yieldParents: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path); + expect(files).not.toContainAllValues([ + relativeBase, + dir1, + dir2, + file0b, + dir11, + dir12, + file1a, + file2b, + file3a, + file4b, + ]); + expect(files).toContainAllValues([ + dir21, + dir22, + file5a, + file6b, + file7a, + file8b, + file9a, + ]); + }); + test('glob with globstar and wildcard', async () => { + const tree: FileTree = []; + for await (const treeNode of vaultsUtils.globWalk({ + fs: fs, + basePath: relativeBase, + pattern: '**/*.a', + yieldFiles: true, + yieldDirectories: true, + yieldParents: false, + })) { + tree.push(treeNode); + } + const files = tree.map((v) => v.path); + expect(files).not.toContainAllValues([ + relativeBase, + dir1, + dir2, + file0b, + dir11, + dir12, + dir21, + dir22, + file2b, + file4b, + file6b, + file8b, + ]); + expect(files).toContainAllValues([ + file1a, + file3a, + file5a, + file7a, + file9a, + ]); + }); + }); }); From 1496d2c490cd96fdf4f5426398c80aaa4225f0d8 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Wed, 17 Jul 2024 14:01:00 +1000 Subject: [PATCH 3/3] fix: removed `yieldContents` feature It's not optimal to hold the whole file in memory like that. The file contents will need to be streamed when needed. [ci skip] --- src/vaults/types.ts | 12 +----------- src/vaults/utils.ts | 16 ---------------- tests/vaults/VaultOps.test.ts | 1 - tests/vaults/utils.test.ts | 30 ------------------------------ 4 files changed, 1 insertion(+), 58 deletions(-) diff --git a/src/vaults/types.ts b/src/vaults/types.ts index 6e54e3b10..6511e5ff0 100644 --- a/src/vaults/types.ts +++ b/src/vaults/types.ts @@ -135,7 +135,7 @@ type VaultName = string; type VaultActions = Partial>; type FileTree = Array; -type TreeNode = DirectoryNode | FileNode | ContentNode; +type TreeNode = DirectoryNode | FileNode; type FilePath = string; type INode = number; type CNode = number; @@ -177,15 +177,6 @@ type FileNode = { stat?: StatEncoded; }; -// Keeping this separate from `FileNode` so we can optionally not include it. -type ContentNode = { - type: 'content'; - path: undefined; - fileName: string; - cNode: CNode; - contents: string; -}; - export { vaultActions }; export type { @@ -208,7 +199,6 @@ export type { StatEncoded, DirectoryNode, FileNode, - ContentNode, }; export { tagLast, refs }; diff --git a/src/vaults/utils.ts b/src/vaults/utils.ts index 357a04d9e..340879fbb 100644 --- a/src/vaults/utils.ts +++ b/src/vaults/utils.ts @@ -159,7 +159,6 @@ function genStat(stat: Stat): StatEncoded { * @param yieldParents - Toggles yielding details about parents of pattern matched paths. Defaults to false. * @param yieldDirectories - Toggles yielding directories that match the pattern. Defaults to true. * @param yieldFiles - Toggles yielding files that match the pattern. Defaults to true. - * @param yieldContents - Toggles yielding file contents after all other details are yielded. Defaults to false. * @param yieldStats - Toggles including stats in file and directory details. Defaults to false. */ async function* globWalk({ @@ -170,7 +169,6 @@ async function* globWalk({ yieldParents = false, yieldDirectories = true, yieldFiles = true, - yieldContents = false, yieldStats = false, }: { fs: FileSystem | FileSystemReadable; @@ -180,7 +178,6 @@ async function* globWalk({ yieldParents?: boolean; yieldDirectories?: boolean; yieldFiles?: boolean; - yieldContents?: boolean; yieldStats?: boolean; }): AsyncGenerator { const files: Array = []; @@ -285,19 +282,6 @@ async function* globWalk({ } current = queue.shift(); } - if (!yieldContents) return; - // Iterate over file contents - for (let i = 0; i < files.length; i++) { - const filePath = files[i]; - yield { - type: 'content', - path: undefined, - fileName: path.basename(filePath), - cNode: i, - // @ts-ignore: While the types don't fully match, it matches enough for our usage. - contents: (await fs.promises.readFile(filePath)).toString(), - }; - } } export { diff --git a/tests/vaults/VaultOps.test.ts b/tests/vaults/VaultOps.test.ts index 6b57097e9..6b7a9bb7a 100644 --- a/tests/vaults/VaultOps.test.ts +++ b/tests/vaults/VaultOps.test.ts @@ -559,7 +559,6 @@ describe('VaultOps', () => { yieldFiles: true, yieldParents: true, yieldRoot: true, - yieldContents: false, })) { tree.push(treeNode); } diff --git a/tests/vaults/utils.test.ts b/tests/vaults/utils.test.ts index 123f9f081..e9d989848 100644 --- a/tests/vaults/utils.test.ts +++ b/tests/vaults/utils.test.ts @@ -138,7 +138,6 @@ describe('Vaults utils', () => { yieldFiles: true, yieldParents: true, yieldRoot: true, - yieldContents: false, })) { tree.push(treeNode); } @@ -172,7 +171,6 @@ describe('Vaults utils', () => { yieldFiles: true, yieldParents: true, yieldRoot: true, - yieldContents: false, })) { tree.push(treeNode); } @@ -206,7 +204,6 @@ describe('Vaults utils', () => { yieldFiles: true, yieldParents: true, yieldRoot: true, - yieldContents: false, })) { tree.push(treeNode); } @@ -230,7 +227,6 @@ describe('Vaults utils', () => { yieldFiles: true, yieldParents: true, yieldRoot: true, - yieldContents: false, })) { tree.push(treeNode); } @@ -319,31 +315,6 @@ describe('Vaults utils', () => { ), ); }); - test('Yields file contents directories with `yieldContents`', async () => { - const tree: FileTree = []; - for await (const treeNode of vaultsUtils.globWalk({ - fs: fs, - basePath: relativeBase, - yieldFiles: false, - yieldDirectories: false, - yieldContents: true, - })) { - tree.push(treeNode); - } - const files = tree.map((v) => (v.type === 'content' ? v.contents : '')); - expect(files).toContainAllValues([ - 'content-file0', - 'content-file9', - 'content-file1', - 'content-file2', - 'content-file3', - 'content-file4', - 'content-file5', - 'content-file6', - 'content-file7', - 'content-file8', - ]); - }); test('Yields stats with `yieldStats`', async () => { const tree: FileTree = []; for await (const treeNode of vaultsUtils.globWalk({ @@ -352,7 +323,6 @@ describe('Vaults utils', () => { yieldStats: true, yieldFiles: true, yieldDirectories: true, - yieldContents: false, })) { tree.push(treeNode); }