Made bash array for invalid/size inner file issues.

dedup.py

@@ -82,7 +82,7 @@ def main():
         delibs.exit_timer(9)
     if size2 != None:
         print(f"ERROR: {size2}")
-        delibs.exit_timer(4) # Mark as Skipped
+        delibs.exit_timer(3) # Mark as Skipped
     
     with delibs.Timer("Hashing"):
         # Quick hashes

do_dups.inc

@@ -0,0 +1,105 @@
+# Check if we found any images
+if [ ${#images[@]} -eq 0 ]; then
+    echo "No images found."
+    exit 1
+fi
+
+MAX_SIZE_for_array=1000
+a_skips=()
+
+truncate_array() {
+    local -n arr_ref=$1  # Nameref (Bash 4.3+)
+    local max_size=$2
+    local array_length=${#arr_ref[@]}
+    
+    if [ $array_length -gt $max_size ]; then
+        # echo "Array too large ($array_length)"
+        #arr_ref=("${arr_ref[@]:0:$max_size}") # Keep oldest data
+        arr_ref=("${arr_ref[@]:1}") # POP First (Shift)
+        # unset arr_ref[${#arr_ref[@]}-1] # POP LAST
+    fi
+}
+
+# Outer loop
+for ((i = 0; i < ${#images[@]}; i++)); do
+    outer_image="${images[$i]}"
+    skip_outer=false  # Reset flag for each outer loop iteration
+    
+    for skip in "${a_skips[@]}"; do
+        if [ "$path/$skip" = "$path/$outer_image" ]; then
+            skip_outer=true
+            break
+        fi    
+    done
+
+    # Skip to next outer iteration if flagged
+    if [ "$skip_outer" = true ]; then
+        continue
+    fi
+
+    # Inner loop (only later images to avoid double-checks)
+    for ((j = i + 1; j < ${#images[@]}; j++)); do
+        inner_image="${images[$j]}"
+        skip_inner=false  # Reset flag for each inner loop iteration
+
+        for skip in "${a_skips[@]}"; do
+            if [ "$path/$skip" = "$path/$inner_image" ]; then
+                skip_inner=true
+                break
+            fi    
+        done
+
+        # Skip to next inner iteration if flagged
+        if [ "$skip_inner" = true ]; then
+            continue
+        fi
+        
+        echo -e "\nComparing files: $outer_image TO $inner_image"
+        python3 dedup.py "$path/$outer_image" "$path/$inner_image" "$2"
+        exit_code=$?
+
+        case $exit_code in
+            1)  # Duplicate found
+                echo "$path/$outer_image # $inner_image" >> dups.txt
+                if [ "$2" = "forreal" ]; then
+                    mv "$path/$outer_image" "$path/dups"
+                fi
+                break
+                ;;
+            2)  # Close match
+                echo "$path/$outer_image # $inner_image" >> alike.txt
+                break
+                ;;
+            3)  # Skip with size issues inner image
+                echo "$path/$inner_image" >> size.txt
+                a_skips+=("$inner_image")
+                truncate_array a_skips $MAX_SIZE_for_array
+                ;;
+            4)  # Skip invalid inner image
+                echo "$path/$inner_image" >> invalid.txt
+                a_skips+=("$inner_image")
+                truncate_array a_skips $MAX_SIZE_for_array
+                ;;
+            5)  # Same GPS
+                echo "$path/$outer_image # $inner_image" >> sameGPS.txt
+                break
+                ;;
+            6)  # Same GPS within a mile
+                echo "$path/$outer_image # $inner_image" >> sameGPSmile.txt
+                break
+                ;;
+            8)  # Invalid outer image
+                echo "$path/$outer_image" >> invalid.txt
+                if [ "$2" = "forreal" ]; then
+                    echo "To remove bad image run: rm $path/$outer_image"
+                fi
+                break
+                ;;
+            9)  # Image size issue
+                echo "$path/$outer_image" >> size.txt
+                break
+                ;;
+        esac
+    done
+done
+# echo "${a_skips[@]}"

get_dups.bat

@@ -46,7 +46,12 @@ for /l %%i in (1,1,%count%) do (
             echo %~1\!outer_image!>> alike.txt
             goto :break_inner
         )
+        if !errorlevel! equ 3 (
+            echo %~1\!inner_image!>> size.txt
+            goto :break_inner
+        )
         if !errorlevel! equ 4 (
+            echo %~1\!inner_image!>> invalid.txt
             goto :break_inner
         )
         if !errorlevel! equ 5 (

get_dups.sh

@@ -18,65 +18,17 @@ if [ "$2" = "forreal" ]; then
     mkdir -p "$path/dups"
 fi    
 
-# Get list of images
+# Get list of images for only png
 shopt -s nullglob
 pushd "$path" || exit 1
-images=(*.jpg *.png)
+images=(*.png)
 popd || exit 1
-
-# Check if we found any images
-if [ ${#images[@]} -eq 0 ]; then
-    echo "No images found."
-    exit 1
-fi
-
-# Outer loop
-for ((i = 0; i < ${#images[@]}; i++)); do
-    outer_image="${images[$i]}"
-
-    # Inner loop (only later images to avoid double-checks)
-    for ((j = i + 1; j < ${#images[@]}; j++)); do
-        inner_image="${images[$j]}"
-		
-		echo -e "\nCompairing files: $outer_image TO $inner_image"
-        python3 dedup.py "$path/$outer_image" "$path/$inner_image" "$2"
-        exit_code=$?
-        if [ $exit_code -eq 1 ]; then
-            echo "$path/$outer_image # $inner_image" >> dups.txt
-            if [ "$2" = "forreal" ]; then
-                mv "$path/$outer_image" "$path/dups"
-            fi    
-            break   # No need to check more once found duplicate
-        fi
-        if [ $exit_code -eq 2 ]; then
-            echo "$path/$outer_image # $inner_image" >> alike.txt
-            break   # No need to check more once found close match to duplicate
-        fi
-        if [ $exit_code -eq 4 ]; then
-            break   # Skip Invaild inner Image
-        fi
-        if [ $exit_code -eq 5 ]; then
-            echo "$path/$outer_image # $inner_image" >> sameGPS.txt
-            break   # No need to check more once found matching GPS image
-        fi
-        if [ $exit_code -eq 6 ]; then
-            echo "$path/$outer_image # $inner_image" >> sameGPSmile.txt
-            break   # No need to check more once found matching GPS image
-        fi
-        if [ $exit_code -eq 8 ]; then
-            echo "$path/$outer_image" >> invalid.txt
-            if [ "$2" = "forreal" ]; then
-                echo "To remove bad image run: rm $path/$outer_image"
-            fi    
-            break   # No need to check more once found bad image
-        fi
-        if [ $exit_code -eq 9 ]; then
-            echo "$path/$outer_image" >> size.txt
-            break   # No need to check more once found image too Small or Large
-        fi
-        
-    done
-done
+source do_dups.inc
+# Get list of images for only jpg
+pushd "$path" || exit 1
+images=(*.jpg)
+popd || exit 1
+source do_dups.inc
 
 end_time=$(date +%s)
 elapsed=$((end_time - start_time))
@@ -89,6 +41,7 @@ seconds=$((elapsed % 60))
 printf "Total time to dedup everything: %02d:%02d:%02d\n" $hours $minutes $seconds
 
 if [ "$2" != "forreal" ] && [ -f "dups.txt" ]; then
+    echo "Here are your duplicate files:"
     more dups.txt
 else
     echo "No Dups found"