gitea-mirror/dust
1
0
Fork 0
mirror of https://github.com/bootandy/dust.git synced 2025-12-05 20:40:11 -08:00
Code Issues Packages Projects Releases Wiki Activity

feat: Custom stack size

Browse Source 
If a user reports a crash from stacksize - we can try different stack
memory sizes to try and figure out the cause.

Do not try and initialise rayon on 32 bit builds

Fix rayon initialisation, may not have been called before due to badly
formed closure
This commit is contained in:
andy.boot
2023-12-11 23:15:31 +00:00
parent c8b61d2f46
commit e8c7990a17
10 changed files with 69 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@@ -83,8 +83,9 @@ Usage: dust -v regex (Exclude files matching this regex (eg dust -v "\.png$" wou
Usage: dust -L (dereference-links - Treat sym links as directories and go into them)
Usage: dust -P (Disable the progress indicator)
Usage: dust -R (For screen readers. Removes bars/symbols. Adds new column: depth level. (May want to use -p for full path too))
Usage: dust -S (Custom Stack size - Use if you see: 'fatal runtime error: stack overflow' (default allocation: low memory=1048576, high memory=1073741824)"),
Usage: dust --skip-total (No total row will be displayed)
Usage: dust -z 4000000 (Exclude files below size 4MB)
Usage: dust -z 4000000 (Exclude output files/directories below size 4MB)
```
## Config file

2
completions/_dust
View File

@@ -29,6 +29,8 @@ _dust() {
'(-t --file_types)*--filter=[Only include filepaths matching this regex. For png files type\: -e "\\.png\$" ]: : ' \
'-w+[Specify width of output overriding the auto detection of terminal width]: : ' \
'--terminal_width=[Specify width of output overriding the auto detection of terminal width]: : ' \
'-S+[Specify memory to use as stack size - use if you see\: '\''fatal runtime error\: stack overflow'\'' (default low memory=1048576, high memory=1073741824)]: : ' \
'--stack-size=[Specify memory to use as stack size - use if you see\: '\''fatal runtime error\: stack overflow'\'' (default low memory=1048576, high memory=1073741824)]: : ' \
'-p[Subdirectories will not have their path shortened]' \
'--full-paths[Subdirectories will not have their path shortened]' \
'-L[dereference sym links - Treat sym links as directories and go into them]' \

2
completions/_dust.ps1
View File

@@ -35,6 +35,8 @@ Register-ArgumentCompleter -Native -CommandName 'dust' -ScriptBlock {
[CompletionResult]::new('--filter', 'filter', [CompletionResultType]::ParameterName, 'Only include filepaths matching this regex. For png files type: -e "\.png$" ')
[CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Specify width of output overriding the auto detection of terminal width')
[CompletionResult]::new('--terminal_width', 'terminal_width', [CompletionResultType]::ParameterName, 'Specify width of output overriding the auto detection of terminal width')
[CompletionResult]::new('-S', 'S ', [CompletionResultType]::ParameterName, 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)')
[CompletionResult]::new('--stack-size', 'stack-size', [CompletionResultType]::ParameterName, 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)')
[CompletionResult]::new('-p', 'p', [CompletionResultType]::ParameterName, 'Subdirectories will not have their path shortened')
[CompletionResult]::new('--full-paths', 'full-paths', [CompletionResultType]::ParameterName, 'Subdirectories will not have their path shortened')
[CompletionResult]::new('-L', 'L ', [CompletionResultType]::ParameterName, 'dereference sym links - Treat sym links as directories and go into them')

10
completions/dust.bash
View File

@@ -19,7 +19,7 @@ _dust() {
case "${cmd}" in
dust)
opts="-d -n -p -X -L -x -s -r -c -b -B -z -R -f -i -v -e -t -w -H -P -D -F -h -V --depth --number-of-lines --full-paths --ignore-directory --dereference-links --limit-filesystem --apparent-size --reverse --no-colors --no-percent-bars --bars-on-right --min-size --screen-reader --skip-total --filecount --ignore_hidden --invert-filter --filter --file_types --terminal_width --si --no-progress --only-dir --only-file --help --version [params]..."
opts="-d -n -p -X -L -x -s -r -c -b -B -z -R -f -i -v -e -t -w -H -P -D -F -S -h -V --depth --number-of-lines --full-paths --ignore-directory --dereference-links --limit-filesystem --apparent-size --reverse --no-colors --no-percent-bars --bars-on-right --min-size --screen-reader --skip-total --filecount --ignore_hidden --invert-filter --filter --file_types --terminal_width --si --no-progress --only-dir --only-file --stack-size --help --version [params]..."
if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
return 0
@@ -81,6 +81,14 @@ _dust() {
COMPREPLY=($(compgen -f "${cur}"))
return 0
;;
--stack-size)
COMPREPLY=($(compgen -f "${cur}"))
return 0
;;
-S)
COMPREPLY=($(compgen -f "${cur}"))
return 0
;;
*)
COMPREPLY=()
;;

2
completions/dust.elv
View File

@@ -32,6 +32,8 @@ set edit:completion:arg-completer[dust] = {|@words|
cand --filter 'Only include filepaths matching this regex. For png files type: -e "\.png$" '
cand -w 'Specify width of output overriding the auto detection of terminal width'
cand --terminal_width 'Specify width of output overriding the auto detection of terminal width'
cand -S 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)'
cand --stack-size 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)'
cand -p 'Subdirectories will not have their path shortened'
cand --full-paths 'Subdirectories will not have their path shortened'
cand -L 'dereference sym links - Treat sym links as directories and go into them'

1
completions/dust.fish
View File

@@ -5,6 +5,7 @@ complete -c dust -s z -l min-size -d 'Minimum size file to include in output' -r
complete -c dust -s v -l invert-filter -d 'Exclude filepaths matching this regex. To ignore png files type: -v "\\.png$" ' -r
complete -c dust -s e -l filter -d 'Only include filepaths matching this regex. For png files type: -e "\\.png$" ' -r
complete -c dust -s w -l terminal_width -d 'Specify width of output overriding the auto detection of terminal width' -r
complete -c dust -s S -l stack-size -d 'Specify memory to use as stack size - use if you see: \'fatal runtime error: stack overflow\' (default low memory=1048576, high memory=1073741824)' -r
complete -c dust -s p -l full-paths -d 'Subdirectories will not have their path shortened'
complete -c dust -s L -l dereference-links -d 'dereference sym links - Treat sym links as directories and go into them'
complete -c dust -s x -l limit-filesystem -d 'Only count the files and directories on the same filesystem as the supplied directory'

5
man-page/dust.1
View File

@@ -4,7 +4,7 @@
.SH NAME
Dust \- Like du but more intuitive
.SH SYNOPSIS
\fBDust\fR [\fB\-d\fR|\fB\-\-depth\fR] [\fB\-n\fR|\fB\-\-number\-of\-lines\fR] [\fB\-p\fR|\fB\-\-full\-paths\fR] [\fB\-X\fR|\fB\-\-ignore\-directory\fR] [\fB\-L\fR|\fB\-\-dereference\-links\fR] [\fB\-x\fR|\fB\-\-limit\-filesystem\fR] [\fB\-s\fR|\fB\-\-apparent\-size\fR] [\fB\-r\fR|\fB\-\-reverse\fR] [\fB\-c\fR|\fB\-\-no\-colors\fR] [\fB\-b\fR|\fB\-\-no\-percent\-bars\fR] [\fB\-B\fR|\fB\-\-bars\-on\-right\fR] [\fB\-z\fR|\fB\-\-min\-size\fR] [\fB\-R\fR|\fB\-\-screen\-reader\fR] [\fB\-\-skip\-total\fR] [\fB\-f\fR|\fB\-\-filecount\fR] [\fB\-i\fR|\fB\-\-ignore_hidden\fR] [\fB\-v\fR|\fB\-\-invert\-filter\fR] [\fB\-e\fR|\fB\-\-filter\fR] [\fB\-t\fR|\fB\-\-file_types\fR] [\fB\-w\fR|\fB\-\-terminal_width\fR] [\fB\-H\fR|\fB\-\-si\fR] [\fB\-P\fR|\fB\-\-no\-progress\fR] [\fB\-D\fR|\fB\-\-only\-dir\fR] [\fB\-F\fR|\fB\-\-only\-file\fR] [\fB\-h\fR|\fB\-\-help\fR] [\fB\-V\fR|\fB\-\-version\fR] [\fIparams\fR]
\fBDust\fR [\fB\-d\fR|\fB\-\-depth\fR] [\fB\-n\fR|\fB\-\-number\-of\-lines\fR] [\fB\-p\fR|\fB\-\-full\-paths\fR] [\fB\-X\fR|\fB\-\-ignore\-directory\fR] [\fB\-L\fR|\fB\-\-dereference\-links\fR] [\fB\-x\fR|\fB\-\-limit\-filesystem\fR] [\fB\-s\fR|\fB\-\-apparent\-size\fR] [\fB\-r\fR|\fB\-\-reverse\fR] [\fB\-c\fR|\fB\-\-no\-colors\fR] [\fB\-b\fR|\fB\-\-no\-percent\-bars\fR] [\fB\-B\fR|\fB\-\-bars\-on\-right\fR] [\fB\-z\fR|\fB\-\-min\-size\fR] [\fB\-R\fR|\fB\-\-screen\-reader\fR] [\fB\-\-skip\-total\fR] [\fB\-f\fR|\fB\-\-filecount\fR] [\fB\-i\fR|\fB\-\-ignore_hidden\fR] [\fB\-v\fR|\fB\-\-invert\-filter\fR] [\fB\-e\fR|\fB\-\-filter\fR] [\fB\-t\fR|\fB\-\-file_types\fR] [\fB\-w\fR|\fB\-\-terminal_width\fR] [\fB\-H\fR|\fB\-\-si\fR] [\fB\-P\fR|\fB\-\-no\-progress\fR] [\fB\-D\fR|\fB\-\-only\-dir\fR] [\fB\-F\fR|\fB\-\-only\-file\fR] [\fB\-S\fR|\fB\-\-stack\-size\fR] [\fB\-h\fR|\fB\-\-help\fR] [\fB\-V\fR|\fB\-\-version\fR] [\fIparams\fR]
.SH DESCRIPTION
Like du but more intuitive
.SH OPTIONS
@@ -81,6 +81,9 @@ Only directories will be displayed.
\fB\-F\fR, \fB\-\-only\-file\fR
Only files will be displayed. (Finds your largest files)
.TP
\fB\-S\fR, \fB\-\-stack\-size\fR
Specify memory to use as stack size \- use if you see: \*(Aqfatal runtime error: stack overflow\*(Aq (default low memory=1048576, high memory=1073741824)
.TP
\fB\-h\fR, \fB\-\-help\fR
Print help
.TP

8
src/cli.rs
View File

@@ -186,6 +186,14 @@ pub fn build_cli() -> Command {
.action(clap::ArgAction::SetTrue)
.help("Only files will be displayed. (Finds your largest files)"),
)
.arg(
Arg::new("stack_size")
.short('S')
.long("stack-size")
.num_args(1)
.value_parser(value_parser!(usize))
.help("Specify memory to use as stack size - use if you see: 'fatal runtime error: stack overflow' (default low memory=1048576, high memory=1073741824)"),
)
.arg(Arg::new("params").num_args(1..)
.value_parser(value_parser!(String)))
}

9
src/config.rs
View File

@@ -26,6 +26,7 @@ pub struct Config {
pub disable_progress: Option<bool>,
pub depth: Option<usize>,
pub bars_on_right: Option<bool>,
pub stack_size: Option<usize>,
}
impl Config {
@@ -95,6 +96,14 @@ impl Config {
pub fn get_bars_on_right(&self, options: &ArgMatches) -> bool {
Some(true) == self.bars_on_right || options.get_flag("bars_on_right")
}
pub fn get_custom_stack_size(&self, options: &ArgMatches) -> Option<usize> {
let from_cmd_line = options.get_one::<usize>("stack_size");
if from_cmd_line.is_none() {
self.stack_size
} else {
from_cmd_line.copied()
}
}
}
fn convert_min_size(input: &str, iso: bool) -> Option<usize> {

48
src/main.rs
View File

@@ -27,7 +27,6 @@ use config::get_config;
use dir_walker::walk_it;
use filter::get_biggest;
use filter_type::get_all_file_types;
use rayon::ThreadPoolBuildError;
use regex::Regex;
use std::cmp::max;
use std::path::PathBuf;
@@ -177,11 +176,8 @@ fn main() {
follow_links,
progress_data: indicator.data.clone(),
};
let result = panic::catch_unwind(|| init_rayon);
if result.is_err() {
eprintln!("Problem initializing rayon, try: export RAYON_NUM_THREADS=1")
}
let stack_size = config.get_custom_stack_size(&options);
init_rayon(&stack_size);
let top_level_nodes = walk_it(simplified_dirs, walk_data);
@@ -227,18 +223,34 @@ fn main() {
}
}
fn init_rayon() -> Result<(), ThreadPoolBuildError> {
let large_stack = usize::pow(1024, 3);
let mut s = System::new();
s.refresh_memory();
let available = s.available_memory();
fn init_rayon(stack_size: &Option<usize>) {
// Rayon seems to raise this error on 32-bit builds
// The global thread pool has not been initialized.: ThreadPoolBuildError { kind: GlobalPoolAlreadyInitialized }
if cfg!(target_pointer_width = "64") {
let result = panic::catch_unwind(|| {
match stack_size {
Some(n) => rayon::ThreadPoolBuilder::new()
.stack_size(*n)
.build_global(),
None => {
let large_stack = usize::pow(1024, 3);
let mut s = System::new();
s.refresh_memory();
let available = s.available_memory();
if available > large_stack.try_into().unwrap() {
// Larger stack size to handle cases with lots of nested directories
rayon::ThreadPoolBuilder::new()
.stack_size(large_stack)
.build_global()
} else {
rayon::ThreadPoolBuilder::new().build_global()
if available > large_stack.try_into().unwrap() {
// Larger stack size to handle cases with lots of nested directories
rayon::ThreadPoolBuilder::new()
.stack_size(large_stack)
.build_global()
} else {
rayon::ThreadPoolBuilder::new().build_global()
}
}
}
});
if result.is_err() {
eprintln!("Problem initializing rayon, try: export RAYON_NUM_THREADS=1")
}
}
}