Changeset 1157 for Validate External Links

Timestamp:

May 9, 2021, 11:53:48 PM (4 years ago)

Author:

iritscen

Message:

ValExtLinks: Make sure that bad YT links count as NG. Various tweaks to project organization.

Location:

Validate External Links

Files:

• Development/ValExtLinks to-do.rtf (modified) ( previous)
• Development/print_user_agent.php (moved) (moved from Validate External Links/Development/print_agent.php )
• Documentation/Read Me.rtf (modified) ( previous)
• validate_external_links.command (modified) (1 diff)
• validate_external_links.sh (modified) (3 diffs)

Validate External Links/validate_external_links.command

@@ -22 +22 @@
 #bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots-ng --end-url 70
 
-# Run with start/end URLs, record OK codes, and don't upload
-#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots-ng --start-url 2588 --end-url 2594
+# Run with start/end URLs, record OK codes, suggest OK snapshots, and don't upload
+#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots-ng --suggest-snapshots-ok --start-url 2836 --end-url 3300
 
 # Run with local extlinks and exceptions, start/end URLs, record OK codes, and don't upload

Validate External Links/validate_external_links.sh

@@ -87 +87 @@
 
 # These are parallel arrays giving the prefixes that can be used in place of normal external links to
-# some wikis and other sites
+# some wikis and other sites; based on https://wiki.oni2.net/Special:Interwiki
 declare -a INTERWIKI_PREFIXES=(commons metawikimedia mw wikibooks wikidata wikimedia wikinews wikiquote wikisource wikispecies wikiversity wikivoyage wikt wp)
 declare -a INTERWIKI_DOMAINS=(commons.wikimedia.org meta.wikimedia.org mediawiki.org wikibooks.org wikidata.org wikimediafoundation.org wikinews.org wikiquote.org wikisource.org species.wikimedia.org wikiversity.org wikivoyage.org wiktionary.org wikipedia.org)
@@ -234 +234 @@
       --end-url )              URL_LIMIT=$2;                       shift 2;;
       --upload )               UPLOAD_INFO=$2;                     shift 2;;
-      * )                      echo "Invalid argument $1 detected. Aborting."; exit 1;;
+      * )                      echo "Invalid argument '$1' detected. Aborting."; exit 1;;
   esac
 done
@@ -960 +960 @@
 
             # If this is a YouTube link, we have to look at the actual page source to know if the video
-            # is good or not
+            # is good or not; override the link's info if it's actually NG
             if [[ $URL == *www.youtube.com* ]]; then
                PAGE_TEXT=$(curl --silent --insecure --user-agent '$AGENT' --max-time $TIMEOUT $URL | grep "\"simpleText\":\"Video unavailable\"")
                if [ ! -z "$PAGE_TEXT" ]; then
                   STATUS="NG"
+                  CURL_RESULT=404
                   let OK_LINKS-=1
                   let NG_LINKS+=1