source: Validate External Links/validate_external_links.command@ 1147

Last change on this file since 1147 was 1147, checked in by iritscen, 4 years ago

ValExtLinks: Changed --suggest-snapshots to --suggest-snapshots-ng and added --suggest-snapshots-ok for getting snapshot URLs for all good links. This can be used to confirm that sites are backed up in case they die in the future, but note that this argument will take hours to run due to the API rate limit. Added awareness of API rate limit so Archive.org will not start blocking script.
  • Property svn:executable set to *
File size: 2.3 KB
Line 
1#!/bin/bash
2
3# Runs the Validate External Links script
4# Arguments: --help, --links URL, --exceptions URL, --output DIR, --record-ok-links, --show-added-slashes,
5# --show-https-upgrades, --show-yt-redirects, --suggest-snapshots-ng, --suggest-snapshots-ok,
6# --check-archive-links, --take-screenshots FILE, --timeout NUM, --start-url NUM, --end-url NUM, --upload FILE
7# Recommended rule:
8# |----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ---|
9
10# Set variables for any arguments we might use below
11VALEXTLINKS="/path/to/Validate External Links/validate_external_links.sh"
12LINKS_LOCAL="file:///path/to/Validate External Links/Sample files/extlinks.csv"
13LINKS_ONLINE="https://wiki.oni2.net/w/extlinks.csv"
14EXCEPT_LOCAL="file:///path/to/Validate External Links/exceptions.txt"
15EXCEPT_ONLINE="https://wiki.oni2.net/Validate_External_Links/Exceptions"
16REPORT_DIR="/path/to/where/you/want/Val Reports"
17UPLOAD_INFO="/path/to/Validate External Links/sftp_login.txt"
18CHROME="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
19
20
21# Run with end URL, record OK codes, and don't upload
22#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots-ng --end-url 70
23
24# Run with start/end URLs, record OK codes, and don't upload
25#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots-ng --suggest-snapshots-ok --start-url 1 --end-url 200
26
27# Run with local extlinks and exceptions, start/end URLs, record OK codes, and don't upload
28#bash "$VALEXTLINKS" --links "$LINKS_LOCAL" --exceptions "$EXCEPT_LOCAL" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots-ng --start-url 1810 --end-url 1815
29
30# Run with unfinished screenshot feature, no upload
31#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --take-screenshots "$CHROME"
32
33# Normal run with no upload
34bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --suggest-snapshots-ng --timeout 10
35
36# Normal run
37#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --suggest-snapshots-ng --timeout 10 --upload "$UPLOAD_INFO"
Note: See TracBrowser for help on using the repository browser.