source: Validate External Links/validate_external_links.command@ 1147

Last change on this file since 1147 was 1147, checked in by iritscen, 4 years ago

ValExtLinks: Changed --suggest-snapshots to --suggest-snapshots-ng and added --suggest-snapshots-ok for getting snapshot URLs for all good links. This can be used to confirm that sites are backed up in case they die in the future, but note that this argument will take hours to run due to the API rate limit. Added awareness of API rate limit so Archive.org will not start blocking script.
  • Property svn:executable set to *
File size: 2.3 KB
RevLine 
[1139]1#!/bin/bash
2
3# Runs the Validate External Links script
4# Arguments: --help, --links URL, --exceptions URL, --output DIR, --record-ok-links, --show-added-slashes,
[1147]5# --show-https-upgrades, --show-yt-redirects, --suggest-snapshots-ng, --suggest-snapshots-ok,
6# --check-archive-links, --take-screenshots FILE, --timeout NUM, --start-url NUM, --end-url NUM, --upload FILE
[1139]7# Recommended rule:
8# |----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ---|
9
10# Set variables for any arguments we might use below
11VALEXTLINKS="/path/to/Validate External Links/validate_external_links.sh"
12LINKS_LOCAL="file:///path/to/Validate External Links/Sample files/extlinks.csv"
13LINKS_ONLINE="https://wiki.oni2.net/w/extlinks.csv"
14EXCEPT_LOCAL="file:///path/to/Validate External Links/exceptions.txt"
15EXCEPT_ONLINE="https://wiki.oni2.net/Validate_External_Links/Exceptions"
16REPORT_DIR="/path/to/where/you/want/Val Reports"
17UPLOAD_INFO="/path/to/Validate External Links/sftp_login.txt"
18CHROME="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
19
20
21# Run with end URL, record OK codes, and don't upload
[1147]22#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots-ng --end-url 70
[1139]23
24# Run with start/end URLs, record OK codes, and don't upload
[1147]25#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots-ng --suggest-snapshots-ok --start-url 1 --end-url 200
[1139]26
27# Run with local extlinks and exceptions, start/end URLs, record OK codes, and don't upload
[1147]28#bash "$VALEXTLINKS" --links "$LINKS_LOCAL" --exceptions "$EXCEPT_LOCAL" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots-ng --start-url 1810 --end-url 1815
[1139]29
30# Run with unfinished screenshot feature, no upload
31#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --take-screenshots "$CHROME"
32
33# Normal run with no upload
[1147]34bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --suggest-snapshots-ng --timeout 10
[1139]35
36# Normal run
[1147]37#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --suggest-snapshots-ng --timeout 10 --upload "$UPLOAD_INFO"
Note: See TracBrowser for help on using the repository browser.