优化以往的图片去重脚本

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

#!/bin/bash

IMG_DIR=/path/to/img
NAME_FILE=/path/to/name.txt
NUM_FILE=/path/to/num.txt

# Deduplicate images
count=0
cd $IMG_DIR
find . -maxdepth 1 -type f -print0 | xargs -0 md5sum | sort >all.txt
cat all.txt | uniq -w 32 >uniq.txt
while read line; do
    if [ $line ]; then
        rm $line
        ((count = count + 1))
    fi
done <<<$(comm all.txt uniq.txt -2 -3 | cut -c 35-)
rm all.txt uniq.txt

# Rename images
count=0
OIFS=$IFS
IFS=$'\n'
mkdir $IMG_DIR/temp && cd $IMG_DIR
while read line; do
    if [ -f $line ]; then
        ((count = count + 1))
        img_type=${line##*.}
        mv $IMG_DIR/$line $IMG_DIR/temp/"$(head -n 4 /dev/urandom | tr -dc "A-Za-z0-9" | cut -c 1-8).$img_type"
    fi
done <<<$(ls $IMG_DIR)
IFS=$OIFS

mv $IMG_DIR/temp/* $IMG_DIR
rm -rf $IMG_DIR/temp
ls -v $IMG_DIR >$NAME_FILE
echo -n $count >$NUM_FILE