-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
perf: refactor the walk, use ignore::Walk
1. fix: <🐞> cli init 2. fix: <🐞> tests and refactor the walker 3. fix: <🐞> private pkgs
- Loading branch information
Showing
12 changed files
with
199 additions
and
54 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -32,4 +32,4 @@ jobs: | |
- name: Setup shell env | ||
run: nix develop | ||
- name: test | ||
run: cargo test | ||
run: cargo test --workspace |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,4 +4,5 @@ pub mod ops; | |
pub mod package; | ||
pub mod source; | ||
pub mod target; | ||
pub mod walker; | ||
pub mod workspace; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
pub mod walk; | ||
|
||
use std::path::PathBuf; | ||
pub use walk::{walk, WalkOption}; | ||
|
||
#[derive(Debug)] | ||
pub enum ReceiveMode { | ||
Buffer, | ||
Stream, | ||
} | ||
|
||
pub type ResultItem = PathBuf; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,115 @@ | ||
use anyhow::Result; | ||
use ignore::{WalkBuilder, WalkParallel, WalkState}; | ||
use regex::bytes::Regex; | ||
use std::borrow::Cow; | ||
use std::path::{Path, PathBuf}; | ||
use std::sync::mpsc::channel; | ||
|
||
#[derive(Debug, Clone)] | ||
pub struct WalkOption { | ||
/// How many threads to run. | ||
pub threads: usize, | ||
/// Math depth to iterate the directories. | ||
pub max_depth: usize, | ||
pub patterns: Vec<Regex>, | ||
} | ||
|
||
impl WalkOption { | ||
pub fn new(patterns: Vec<Regex>) -> WalkOption { | ||
WalkOption { | ||
patterns, | ||
..Default::default() | ||
} | ||
} | ||
pub fn max_depth(mut self, size: usize) -> Self { | ||
self.max_depth = size; | ||
self | ||
} | ||
pub fn threads(mut self, size: usize) -> Self { | ||
self.threads = size; | ||
self | ||
} | ||
} | ||
|
||
impl Default for WalkOption { | ||
fn default() -> WalkOption { | ||
WalkOption { | ||
threads: 5, | ||
max_depth: 10, | ||
patterns: Vec::new(), | ||
} | ||
} | ||
} | ||
|
||
/// Create a file tree walker | ||
pub fn create_concurrent_walker( | ||
paths: &[impl AsRef<Path>], | ||
option: Option<WalkOption>, | ||
) -> (WalkParallel, WalkOption) { | ||
let option = option.unwrap_or_default(); | ||
|
||
let first_path = &paths[0]; | ||
let mut walker = WalkBuilder::new(first_path); | ||
walker | ||
.follow_links(false) | ||
// .max_filesize(Some(1024 * 1024 * 30)) | ||
.max_depth(Some(option.max_depth)) | ||
.threads(option.threads); | ||
|
||
// add paths to search | ||
paths.iter().skip(1).for_each(|p| { | ||
walker.add(p); | ||
}); | ||
|
||
(walker.build_parallel(), option) | ||
} | ||
|
||
pub fn walk(paths: &[impl AsRef<Path>], option: Option<WalkOption>) -> Result<Vec<PathBuf>> { | ||
let (walker, option) = create_concurrent_walker(paths, option); | ||
|
||
let patterns = &option.patterns; | ||
let mut buffers = Vec::<PathBuf>::new(); | ||
|
||
let (tx, rx) = channel::<PathBuf>(); | ||
|
||
walker.run(|| { | ||
Box::new(|result| { | ||
if result.is_err() { | ||
return WalkState::Skip; | ||
} | ||
let entry = result.unwrap(); | ||
// skip root. | ||
if entry.depth() == 0 { | ||
return WalkState::Continue; | ||
} | ||
|
||
let entry_path = entry.path(); | ||
if !is_match_pattern(entry_path, patterns.as_slice()) { | ||
return WalkState::Continue; | ||
} | ||
|
||
tx.send(entry_path.to_path_buf()).unwrap(); | ||
|
||
WalkState::Continue | ||
}) | ||
}); | ||
|
||
drop(tx); | ||
|
||
while let Ok(re) = rx.recv() { | ||
buffers.push(re); | ||
} | ||
|
||
Ok(buffers) | ||
} | ||
|
||
fn is_match_pattern(path: &Path, pattern: &[Regex]) -> bool { | ||
let to_match: Cow<[u8]> = { | ||
match path.to_string_lossy() { | ||
Cow::Owned(string) => Cow::Owned(string.into_bytes()), | ||
Cow::Borrowed(string) => Cow::Borrowed(string.as_bytes()), | ||
} | ||
}; | ||
|
||
pattern.iter().all(|pat| pat.is_match(&to_match)) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,41 +1,22 @@ | ||
use crate::walker; | ||
use anyhow::Result; | ||
use regex::bytes::Regex; | ||
use std::path::{Path, PathBuf}; | ||
use wax::{any, Glob}; | ||
|
||
pub(super) fn walk_package_jsons_under_path( | ||
path: impl AsRef<Path>, | ||
) -> Result<impl 'static + Iterator<Item = PathBuf>> { | ||
let glob = Glob::new("**/package.json").unwrap().into_owned(); | ||
let walker = glob | ||
.walk(path) | ||
.not(any([ | ||
"**/node_modules/**", | ||
"**/dist/**", | ||
"**/src/**", | ||
"**/public/**", | ||
"**/.git/**", | ||
"**/.direnv/**", | ||
])) | ||
.unwrap(); | ||
pub(super) fn walk_package_jsons_under_path(path: impl AsRef<Path>) -> Result<Vec<PathBuf>> { | ||
let wo = walker::WalkOption::new(vec![Regex::new(r"package\.json$").unwrap()]); | ||
let paths = walker::walk(&[path], Some(wo))?; | ||
|
||
Ok(walker | ||
.filter(|entry| entry.is_ok()) | ||
.map(|entry| entry.unwrap().path().to_path_buf()) | ||
// clone the path from the iterator results | ||
.collect::<Vec<PathBuf>>() | ||
.into_iter()) | ||
Ok(paths) | ||
} | ||
|
||
#[ignore] | ||
#[test] | ||
fn test_walk_package_jsons_under_path() { | ||
let workspace_root = concat!( | ||
env!("CARGO_WORKSPACE_DIR"), | ||
"assets_/fixtures_npm_workspaces" | ||
); | ||
|
||
let paths = walk_package_jsons_under_path(workspace_root) | ||
.unwrap() | ||
.collect::<Vec<PathBuf>>(); | ||
let paths = walk_package_jsons_under_path(workspace_root).unwrap(); | ||
assert!(paths.len() == 5); | ||
} |
Oops, something went wrong.