Changeset 1148 for Validate External Links/validate_external_links.sh

Timestamp:

Feb 5, 2021, 12:15:20 AM (4 years ago)

Author:

iritscen

Message:

ValExtLinks: Val can now recognize bad YouTube links (no thanks to YouTube). Fixed some math errors. Added error 429 to known codes.

File:

• Validate External Links/validate_external_links.sh (modified) (11 diffs)

Validate External Links/validate_external_links.sh

@@ -48 +48 @@
 
 # Fixed strings -- see the occurrences of these variables to learn their purpose
-AGENT="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36 OPR/70.0.3728.154"
+AGENT="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/88.0.4324.146 Safari/537.36"
 ARCHIVE_API="http://archive.org/wayback/available"
 ARCHIVE_GENERIC="https://web.archive.org/web/*"
@@ -77 +77 @@
 declare -a OK_CODES=(200 401 405 406 418 501)
 declare -a RD_CODES=(301 302 303 307 308)
-declare -a NG_CODES=(000 400 403 404 410 500 502 503 530)
+declare -a NG_CODES=(000 400 403 404 410 429 500 502 503 530)
 
 # Characters not allowed in a URL. Curly braces are sometimes used on the wiki to build a link using
@@ -721 +721 @@
       if [ $LINE == "namespace,title,target" ]; then
          SKIPPED_HEADER_ROW=1
-         LINK_NUM=0 # this line is it's not a link, so reset the link counter
+         LINK_NUM=0 # this line is not a link, so reset the link counter
          valPrint hn "<table>"
          continue
@@ -769 +769 @@
       fi
       let SKIP_UNK_NS+=1
+      let PAGE_LINKS+=1
       continue
    fi
@@ -783 +784 @@
       valPrint trs "Skipping URL '${LINE#$NS_ID,$PAGE_NAME,}' on line $LINK_NUM because it was found on JavaScript page '$PAGE_NAME'."
       let SKIP_JS_PAGE+=1
+      let PAGE_LINKS+=1
       continue
    fi
@@ -804 +806 @@
       valPrint trs "Skipping URL '$URL' (found on page '$PAGE_NAME') because it contains characters illegal in a URL."
       let SKIP_BAD_URL+=1
+      let PAGE_LINKS+=1
       continue
    fi
@@ -811 +814 @@
       valPrint trs "Skipping URL '$URL' (found on page '$PAGE_NAME') because I have not been asked to check Wayback Machine links."
       let SKIP_ARCHIVE_ORG+=1
+      let PAGE_LINKS+=1
       continue
    fi
@@ -828 +832 @@
       valPrint trs "Skipping URL '$URL' (found on page '$PAGE_NAME') because I cannot handle non-ASCII characters."
       let SKIP_NON_ASCII+=1
+      let PAGE_LINKS+=1
       continue
    fi
@@ -905 +910 @@
       STR_TYPE="file"
       let FILE_LINKS+=1
-   elif [ $IS_FILE -eq 0 ]; then
+   else
       STR_TYPE="page"
       let PAGE_LINKS+=1
@@ -953 +958 @@
             STATUS="OK"
             let OK_LINKS+=1
+
+            # If this is a YouTube link, we have to look at the actual page source to know if the video
+            # is good or not
+            if [[ $URL == *www.youtube.com* ]]; then
+               PAGE_TEXT=$(curl --silent --insecure --user-agent '$AGENT' --max-time $TIMEOUT $URL | grep "\"simpleText\":\"Video unavailable\"")
+               if [ ! -z "$PAGE_TEXT" ]; then
+                  STATUS="NG"
+                  let OK_LINKS-=1
+                  let NG_LINKS+=1
+               fi
+            fi
             break
          fi
@@ -1001 +1017 @@
                let OK_LINKS+=1
                let SKIP_SLASH_ADD+=1
-            elif [ $SHOW_YT_RD -eq 0 ] && [ $YOUTU_BE -eq 1 ]; then
-               valPrint trs "Skipping URL '$URL' (found on page '$PAGE_NAME') because I have not been asked to show redirects for youtu.be links, and I was redirected to '$NEW_URL'."
-               STATUS="OK"
-               let OK_LINKS+=1
-               let SKIP_YOUTU_BE+=1
+            elif [ $YOUTU_BE -eq 1 ]; then
+               # We have to look at the actual page source to know if a YouTube video is good or not
+               PAGE_TEXT=$(curl --silent --insecure --user-agent '$AGENT' --max-time $TIMEOUT $NEW_URL | grep "\"simpleText\":\"Video unavailable\"")
+               if [ ! -z "$PAGE_TEXT" ]; then
+                  STATUS="NG"
+                  let NG_LINKS+=1
+               else
+                  if [ $SHOW_YT_RD -eq 0 ]; then
+                     valPrint trs "Skipping URL '$URL' (found on page '$PAGE_NAME') because I have not been asked to show redirects for youtu.be links, and I was redirected to '$NEW_URL'."
+                     STATUS="OK"
+                     let OK_LINKS+=1
+                     let SKIP_YOUTU_BE+=1
+                  else
+                     STATUS="RD"
+                     let RD_LINKS+=1
+                  fi
+               fi
             else
                STATUS="RD"