source: ValBot/Bash/drive_slash_adding.sh@ 1190

#!/bin/bash

IFS="
"

CORE="/path/to/Pywikibot/core"
SUMMARY="added ending slash to URL and/or upgrading http to https to satisfy redirect"
RATE=6
FIX_START=0
FIX_END=0

cd "$CORE"
if [ ! -f "pwb.py" ]; then
   echo "drive_slash_adding.sh: Can't launch Pywikibot!"
   exit
fi

echo "drive_slash_adding.sh: Starting at fix $FIX_START..."

FIX_CUR=0
LAST_RUN=0
for THE_LINE in `cat "/path/to/ValExtLinks report.txt"`; do
   #echo "drive_slash_adding.sh: Considering '$THE_LINE'..."
   if [[ "$THE_LINE" =~ .*trailing.* ]] && [[ ! "$THE_LINE" =~ .*w/index.php.* ]]; then
      #echo "drive_slash_adding.sh: This URL needs to be fixed."
      let FIX_CUR+=1

      if [ $FIX_CUR -lt $FIX_START ]; then
         continue
      fi

      if [ $FIX_END -gt 0 ] && [ $FIX_CUR -gt $FIX_END ]; then
         echo "drive_slash_adding.sh: Stopped after fix $FIX_END."
         exit
      fi      

      # Wait for rate limit to expire if we have run the Python script before in this session
      if [ $LAST_RUN -gt 0 ]; then
         CUR_TIME=$(date +%s)
         WAIT_REMAINDER=$(($RATE - $CUR_TIME + $LAST_RUN))
         if [ $WAIT_REMAINDER -gt 0 ]; then
            echo "drive_slash_adding.sh: Waiting $WAIT_REMAINDER second(s)."
            sleep $WAIT_REMAINDER
         fi
      fi
      ON_PAGE=${THE_LINE#*page \'}
      ON_PAGE=${ON_PAGE%%\'*}
      FROM_LINK=${THE_LINE#*URL \'}
      FROM_LINK=${FROM_LINK%%\'*}
      TO_LINK=${THE_LINE%\'*}
      TO_LINK=${TO_LINK##*\'}

      #if [[ "$THE_LINE" =~ ${FROM_LINK}[^a-zA-Z/] ]]; then
      #   echo "URL is not isolated, skipping."
      #   continue
      #fi

      LAST_RUN=$(date +%s)
      echo "pwb.by replace '-page:\"$ON_PAGE\" \"$FROM_LINK\" \"$TO_LINK\""
      python pwb.py replace -page:"$ON_PAGE" "$FROM_LINK" "$TO_LINK" -summary:"$SUMMARY"
   fi
done