Differences

This shows you the differences between two versions of the page.

--- marc:linux:bash_snippets [2023/02/10 12:31] – marcv
+++ marc:linux:bash_snippets [2023/02/16 14:23] (current) – [Download files based on URLs in a file] marcv
@@ Line 227: / Line 227: @@
   done
 done
+</code>
+===== Remove duplicate and empty lines from file =====
+<code>
+#!/bin/bash
+# Remove duplicate lines from url.txt and save to temp file
+sort -u url.txt > temp.txt
+# Remove empty lines from temp file and save back to url.txt
+grep -v "^$" temp.txt > url.txt
+# Remove temp file
+rm temp.txt
+</code>
+===== Download files based on URLs in a file =====
+<code>
+#!/bin/bash
+# read list of URLs from file
+while read url; do
+  # download file using wget
+  wget "$url"
+done < url.txt
+</code>
+Do the same but check if file exists. If it does compare sizes and ask what to do.
+<code>#!/bin/bash
+# Read URLs from the url.txt file and download the files
+while read url; do
+    filename=$(basename "$url")
+    if [ -e "$filename" ]; then
+        local_size=$(wc -c <"$filename")
+        remote_size=$(curl -sI "$url" | awk '/Content-Length/ {sub("\r",""); print $2}')
+        if [ "$local_size" = "$remote_size" ]; then
+            echo "$filename already exists and has the same file size. Skipping download."
+            continue
+        else
+            read -p "A file with the name $filename already exists. Do you want to save the new file with a different name? (y/n) " response
+            if [ "$response" = "y" ]; then
+                read -p "Enter the new file name: " new_filename
+                curl -L -o "$new_filename" "$url"
+                echo "File saved as $new_filename."
+            else
+                echo "Skipping download of $filename."
+            fi
+        fi
+    else
+        curl -L -o "$filename" "$url"
+        echo "Downloaded $filename."
+    fi
+done < url.txt
 </code>