Spaces:
Sleeping
Sleeping
File size: 834 Bytes
15389e6 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 |
echo "Please input parenet directory, will count all .bin files..."
echo "Example: bash ./count_tokens.sh /workspace/dataset/music"
PARENT_DIR=${1:-/workspace/dataset/music}
LOG_DIR=./count_token_logs/
mkdir -p $LOG_DIR
# find all .bin files
BINS=$(find $PARENT_DIR -name "*.bin" -type f)
for bin in $BINS; do
echo Checking mmap file: $bin
mmap_path=$bin
# mmap size in human readable format (e.g. 1.2G)
mmap_size=$(du -h $mmap_path | awk '{print $1}')
echo "Counting largest mmap file: $mmap_path, size: $mmap_size"
# remove PARENT_DIR, replace / with _
subdir=$(echo $mmap_path | sed "s|$PARENT_DIR/||g" | sed 's/\//_/g')
cmd="nohup python tools/count_mmap_token.py --mmap_path $mmap_path > $LOG_DIR/count.$subdir.log 2>&1 &"
echo $cmd
eval $cmd
echo "Finished!"
done
|