Changeset 1177 for Validate External Links

Timestamp:

Jan 13, 2023, 11:26:56 PM (2 years ago)

Author:

iritscen

Message:

ValExtLinks now skips URLs that aren't HTTP(S) protocol. Added some error-checking on line parsing. Added my email address.

File:

• Validate External Links/validate_external_links.sh (modified) (9 diffs)

Validate External Links/validate_external_links.sh

@@ -1 +1 @@
 #!/bin/bash
 
-# Validate External Links by Iritscen
+# Validate External Links by Iritscen (iritscen@yahoo.com)
 #
 # Validates a list of external links in CSV format. The resulting logs are produced in three formats:
@@ -32 +32 @@
 LINKS_URL=""           # download external link CSV from this location (can use "file://" protocol)
 EXCEPT_URL=""          # location of wiki page with a list of exceptions for NG results
-OUTPUT_DIR=""          # place reports and all other output in a folder inside this existing folder
+OUTPUT_DIR=""           # place reports and all other output in a folder inside this existing folder
 RECORD_OK_LINKS=0      # record response code to the log even when it's a value in OK_CODES
 SHOW_SLASH=0           # record issue when a slash is added to the end of a URL
@@ -98 +98 @@
 RD_LINKS=0
 NG_LINKS=0
+SKIP_PARSE_FAIL=0
+SKIP_UNK_PROT=0
 SKIP_UNK_NS=0
 SKIP_JS_PAGE=0
@@ -506 +508 @@
    LINKS_PROCESSED=$((LINK_NUM-URL_START+1))
    TRIVIAL_RDS=$((SKIP_SLASH_ADD+SKIP_HTTPS_UP+SKIP_YOUTU_BE))
-   LINK_ERRORS=$((SKIP_UNK_NS+SKIP_JS_PAGE+SKIP_BAD_URL+SKIP_NON_ASCII+SKIP_UNK_SUFFIX+SKIP_UNK_CODE))
+   LINK_ERRORS=$((SKIP_PARSE_FAIL+SKIP_UNK_PROT+SKIP_UNK_NS+SKIP_JS_PAGE+SKIP_BAD_URL+SKIP_NON_ASCII+SKIP_UNK_SUFFIX+SKIP_UNK_CODE))
    LINKS_EXCEPTED=$((SKIP_EXPECT_NG+SKIP_EXPECT_RD+SKIP_EXPECT_EI+SKIP_EXPECT_IW))
    LINK_PROBLEMS_TOTAL=$((NG_LINKS+RD_LINKS+EI_LINKS+IW_LINKS))
@@ -545 +547 @@
       valPrint rt "$LINK_ERRORS link $(pluralCheckNoun error $LINK_ERRORS):"
    fi
+   if [ $SKIP_PARSE_FAIL -gt 0 ]; then valPrint ctrh "- $SKIP_PARSE_FAIL line-parsing $(pluralCheckNoun failure $SKIP_PARSE_FAIL)"; fi
+   if [ $SKIP_UNK_PROT -gt 0 ]; then valPrint ctrh "- $SKIP_UNK_PROT unknown $(pluralCheckNoun protocol $SKIP_UNK_PROT)"; fi
    if [ $SKIP_UNK_NS -gt 0 ]; then valPrint ctrh "- $SKIP_UNK_NS missing/unknown $(pluralCheckNoun namespace $SKIP_UNK_NS)"; fi
    if [ $SKIP_JS_PAGE -gt 0 ]; then valPrint ctrh "- $SKIP_JS_PAGE $(pluralCheckNoun link $SKIP_JS_PAGE) on $(pluralCheckA $SKIP_JS_PAGE)JavaScript $(pluralCheckNoun page $SKIP_JS_PAGE)"; fi
@@ -782 +786 @@
       wrapupAndExit
    fi
+   
+   # Parse line into namespace ID number, containing wiki page, and external link URL
+   NS_ID=${LINE%%,*}
+   PAGE_NAME=${LINE#$NS_ID,}
+   PAGE_NAME=${PAGE_NAME%%,*} # a comma in the page name will break this
+   URL=${LINE#$NS_ID,$PAGE_NAME,} # commas can be in this
+   if [ -z "$NS_ID" ] || [ -z "$PAGE_NAME" ] || [ -z "$URL" ]; then
+      valPrint trs "Skipping line $LINK_NUM ('$LINE') because the namespace, wiki page or link URL could not be read."
+      let SKIP_PARSE_FAIL+=1
+      continue
+   fi
+   
+   # Skip any link that isn't "http://" or "https://"
+   if [[ ! $URL =~ ^http* ]]; then
+      valPrint trs "Skipping line $LINK_NUM ('$LINE') because the protocol isn't 'http://' or 'https://'."
+      let SKIP_UNK_PROT+=1
+      continue
+   fi
 
    # Print progress to screen
@@ -788 +810 @@
    fi
    valPrint cn "Evaluating URL $LINK_NUM/$LINK_COUNT..."
-
-   # The number of the namespace is the element before the first comma on the line
-   NS_ID=${LINE%%,*}
 
    # Find namespace number in NS_IDS and use it to look up namespace's name in NS_NAMES
@@ -815 +834 @@
    fi
 
-   # The name of the page is everything between the namespace ID and the next comma on the line (commas
-   # in page names will break this)
-   PAGE_NAME=${LINE#$NS_ID,}
-   PAGE_NAME=${PAGE_NAME%%,*}
-
    # Build longer wiki page URLs from namespace and page names
    FULL_PAGE_PATH=https://$WIKI_PATH/$NS_NAME:$PAGE_NAME
@@ -839 +853 @@
       continue
    fi
-
-   # The URL being linked to is everything after the previous two fields (this allows commas to be in
-   # the URLs, but a comma in the previous field, the page name, will break this)
-   URL=${LINE#$NS_ID,$PAGE_NAME,}
 
    # Scan for illegal characters