Changeset 1147 for Validate External Links/validate_external_links.command

Timestamp:

Feb 2, 2021, 9:10:39 PM (4 years ago)

Author:

iritscen

Message:

ValExtLinks: Changed --suggest-snapshots to --suggest-snapshots-ng and added --suggest-snapshots-ok for getting snapshot URLs for all good links. This can be used to confirm that sites are backed up in case they die in the future, but note that this argument will take hours to run due to the API rate limit. Added awareness of API rate limit so Archive.org will not start blocking script.

File:

• Validate External Links/validate_external_links.command (modified) (3 diffs)

Validate External Links/validate_external_links.command

@@ -3 +3 @@
 # Runs the Validate External Links script
 # Arguments: --help, --links URL, --exceptions URL, --output DIR, --record-ok-links, --show-added-slashes,
-# --show-https-upgrades, --show-yt-redirects, --suggest-snapshots, --check-archive-links, --take-screenshots FILE,
-# --timeout NUM, --start-url NUM, --end-url NUM, --upload FILE
+# --show-https-upgrades, --show-yt-redirects, --suggest-snapshots-ng, --suggest-snapshots-ok,
+# --check-archive-links, --take-screenshots FILE, --timeout NUM, --start-url NUM, --end-url NUM, --upload FILE
 # Recommended rule:
 # |----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ---|
@@ -20 +20 @@
 
 # Run with end URL, record OK codes, and don't upload
-#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots --end-url 70
+#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots-ng --end-url 70
 
 # Run with start/end URLs, record OK codes, and don't upload
-#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots --start-url 2560 --end-url 2570
+#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots-ng --suggest-snapshots-ok --start-url 1 --end-url 200
 
 # Run with local extlinks and exceptions, start/end URLs, record OK codes, and don't upload
-#bash "$VALEXTLINKS" --links "$LINKS_LOCAL" --exceptions "$EXCEPT_LOCAL" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots --start-url 1 --end-url 25
+#bash "$VALEXTLINKS" --links "$LINKS_LOCAL" --exceptions "$EXCEPT_LOCAL" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots-ng --start-url 1810 --end-url 1815
 
 # Run with unfinished screenshot feature, no upload
@@ -32 +32 @@
 
 # Normal run with no upload
-#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --suggest-snapshots --timeout 10
+bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --suggest-snapshots-ng --timeout 10
 
 # Normal run
-bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --suggest-snapshots --timeout 10 --upload "$UPLOAD_INFO"
+#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --suggest-snapshots-ng --timeout 10 --upload "$UPLOAD_INFO"