#!/bin/bash
if [ $# -eq 0 ]; then
echo "No arguments supplied. Example: ./script.sh https://vimm.net/vault/GB"
exit 1
fi
arg="$1"
console=$(echo "$arg" | awk -F '/' '{print $(NF)}')
echo "$console"
for letter in \# A B C D E F G H I J K L M N O P Q R S T U V W X Y Z; do
# Construct letterpage
if [ "$letter" = "#" ]; then
letterpage="https://vimm.net/vault/?p=list&system=$console§ion=number"
else
letterpage="https://vimm.net/vault/$console/$letter"
fi
echo
echo "$letter: $letterpage"
echo "======================="
# Base URL (should be without trailing slash)
baseURL="https://vimm.net"
# find all gamepages in a letterpage
for gamepage in $(curl -s "$letterpage" | grep 'a href="\/vault' | awk -F 'href="' '{print $2}' | cut -d '"' -f1 | grep -v '?p=' | grep -v "NES" | grep -v "$console" | grep -o '[[:digit:]]*'); do
# Construct the gamepage URL
gamepage="$baseURL/vault/$gamepage"
skipit=false
retries=1
# Find the mediaID for this specific game page
while true; do
echo "Looking for mediaID into: $gamepage"
id=$(curl -s "$gamepage" | grep 'var media = {"ID":' | awk -F ':' '{ print $2}' | awk -F ',' '{print $1}')
if [ -z "$id" ]; then
echo "\$id is empty. Retrying ...$retries"
((retries = retries + 1))
# But ... if you re-tried 10 times already, then give up on this one
if [[ "$retries" -gt 10 ]]; then
skipit=true
break
fi
else
echo "FOUND! ID is: $id"
break
fi
done
# If you bailed out because the mediaID was not found, then proceed to the next gamepage
if [ "$skipit" = true ]; then
echo 'Giving up on this one...'
echo "$id@$console@$gamepage" >>"$console.err"
continue
fi
# Find Download URL
skipit=false
retries=1
while true; do
echo "Looking for download URL"
URL=$(curl -s "$gamepage" | grep '<form action="//' | awk -F '<form action="//' '{ print $2 }' | awk -F '"' '{ print $1 }')
if [ -z "$URL" ]; then
echo "\$URL is empty. Retrying ...$retries"
((retries = retries + 1))
# But ... if you re-tried 10 times already, then give up on this one
if [[ "$retries" -gt 10 ]]; then
skipit=true
break
fi
else
echo "FOUND! URL is: $URL"
break
fi
done
# If you bailed out because the download URL was not found, then proceed to the next gamepage
if [ "$skipit" = true ]; then
echo 'Giving up on this one...'
echo
echo
echo "$id@$console@$gamepage" >>"$console.err"
continue
fi
# Constrcut the download URL
# e.g. URL="https://download.vimm.net/download/?mediaId=$id"
URL="https://$URL?mediaId=$id"
echo "Looking for filename into: $URL"
# Find the filename (Check there is 404, so skip it)
skipit=false
retries=1
while true; do
filename=$(curl -s --max-time 5 -D - \
-H "Referer: https://vimm.net/" \
-H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36" \
"$URL" | grep "Content-Disposition" | awk -F 'filename="' '{ print $2 }' | awk -F '"' '{ print $1 }' 2>/dev/null)
if [ -z "$filename" ]; then
echo "\$filename is empty. Retrying ... $retries"
((retries = retries + 1))
# But ... if you re-tried 10 times already, then give up on this one
if [[ "$retries" -gt 10 ]]; then
skipit=true
break
fi
else
echo "FOUND! Filename is: $filename"
break
fi
done
# If you bailed out because the filename was not found, then proceed to the next gamepage
if [ "$skipit" = true ]; then
echo 'Giving up on this one...'
echo
echo
echo "$id@$console@$gamepage@$URL@$filename" >>"$console.txt"
continue
fi
# Download the rom
skipit=false
retries=1
while true; do
if curl \
-H "Referer: https://vimm.net/" \
-H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36" \
"$URL" --output "$filename" --progress-bar; then
echo "Start downloading $filename..."
fi
# Check if the file integrity is correct
ZIP=false
SEVEN=false
if test -f "$filename"; then
echo "Check integrity..."
if file "$filename" | grep 'Zip archive data\|7-zip archive data' &>/dev/null; then
echo "VALID!"
if file "$filename" | grep 'Zip archive data' &>/dev/null; then
echo "Compression Type: Zip archive"
ZIP=true
fi
if file "$filename" | grep '7-zip archive data' &>/dev/null; then
echo "Compression Type: 7z archive"
SEVEN=true
fi
if [ "$ZIP" = true ] || [ "$SEVEN" = true ]; then
echo "It's a known compressed format"
else
echo "I don't know this compression format"
((retries = 11))
break
fi
# All fine, break the loop to proceed to the next step that is extraction
break
else
# PROBLEM: In case compressed archive is damaged, retry 10 times or give up
echo "INVALID! $filename is not a valid Zip/7z archive. Retrying ..."
((retries = retries + 1))
# But ... if you re-tried 10 times already, then give up on this one
if [[ "$retries" -gt 10 ]]; then
skipit=true
break
else
continue # retry (this will re-download the file)
fi
fi
else
# PROBLEM: In case download failed, retry 10 times or give up
echo "$filename has not been downloaded. Retrying ... $retries"
((retries = retries + 1))
# But ... if you re-tried 10 times already, then give up on this one
if [[ "$retries" -gt 10 ]]; then
skipit=true
break
fi
continue # Retry (this will re-download the file)
fi
done
# If you bailed out because the zip/7z compressed archive is invalid or failed to download, then proceed to the next GAMELINK
if [ "$skipit" = true ]; then
echo 'Giving up on this one...'
echo "Deleting filename: $filename"
rm "$filename" 2>/dev/null
echo "Writing logs at $console.err"
echo "$id@$console@$gamepage@$URL@$filename" >>"$console.err"
echo "Proceeding to the next game ..."
echo
echo
echo "================================================="
continue # Proceed to the next Game from the outer loop
fi
# ------------ #
# Extract the rom
skipit=false
retries=1
while true; do
folder=$(date)
echo "Trying to extract..."
echo "Creating a ./tmp directory and moving $filename into it"
mkdir ./tmp
mv "$filename" ./tmp
echo "tmp created and file moved into it"
if [ "$SEVEN" = true ]; then
if 7z x "./tmp/$filename" -o./tmp; then
echo "Extracted!"
folder=$(echo "$filename" | awk -F '.7z' '{print $1}')
echo "Deleting the compressed 7z archive: $filename"
rm "./tmp/$filename"
echo "Deleted!"
echo "Renaming the extracted folder ./tmp to $folder"
mv ./tmp "$folder"
echo "Renamed!"
else
echo "FAILED! Cannot un7zip the $filename"
echo "Deleting the ./tmp directory"
rm -rf tmp
echo "Deleted!"
echo "Retrying ... $retries"
((retries = retries + 1))
# But ... if you re-tried 10 times already, then give up on this one
if [[ "$retries" -gt 10 ]]; then
skipit=true
break
else
continue # retry (this will re-extract the file)
fi
fi
fi
if [ "$ZIP" = true ]; then
if unzip -o "tmp/$filename" -d ./tmp; then
echo "Extracted!"
folder=$(echo "$filename" | awk -F '.zip' '{print $1}')
echo "Deleting the compressed zip archive: $filename"
rm "tmp/$filename"
echo "Deleted!"
echo "Renaming the extracted folder ./tmp to $folder"
mv ./tmp "$folder"
echo "Renamed!"
else
echo "FAILED! Cannot unzip the $filename"
echo "Deleting the ./tmp directory"
rm -rf tmp
echo "Deleted!"
echo "Retrying ... $retries"
((retries = retries + 1))
# But ... if you re-tried 10 times already, then give up on this one
if [[ "$retries" -gt 10 ]]; then
skipit=true
break
else
continue # retry (this will re-extract the file)
fi
fi
fi
# All fine, break the loop to proceed to the next game
break
done
# If you bailed out because the zip/7z compressed archive is invalid or failed to download, then proceed to the next GAMELINK
if [ "$skipit" = true ]; then
echo 'Giving up on this one...'
echo "Deleting filename: $filename"
rm "$filename" 2>/dev/null
echo "Writing logs at $console.err"
echo "$id@$console@$gamepage@$URL@$filename" >>"$console.err"
echo "Proceeding to the next game ..."
echo
echo
echo "================================================="
continue # Proceed to the next Game from the outer loop
fi
# ------------ #
echo "DONE! Game $filename with $id has been downloaded and extracted into $folder!"
echo
echo
done
echo "---"
done
Comments