Changeset 1179
- Timestamp:
- Apr 28, 2023, 2:53:24 AM (19 months ago)
- Location:
- ValBot
- Files:
-
- 2 edited
Legend:
- Unmodified
- Added
- Removed
-
ValBot/Python/check_intrawiki_section_links.py
r1176 r1179 2 2 # by iritscen@yahoo.com 3 3 # Looks at each wikilink on a page (or in all the pages in a category) for a section link ('#'), 4 # and loads the linked page and verifies that the named section actually exists. The output will5 # u se the keywords ADVICE, WARNING or ERROR depending on the nature of issue that it encounters.4 # and loads the linked page and verifies that the named section actually exists. It also 5 # understands section links generated through a call to Template:SectionLink. 6 6 # Recommended viewing width: 7 7 # |---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- --| … … 21 21 from bs4 import BeautifulSoup 22 22 23 # Arrayof OniGalore's namespaces23 # Tuple of OniGalore's namespaces 24 24 intrawiki_prefixes = ('Image', 'Special', 'Talk', 'User', 'User_talk', 'OniGalore', 'OniGalore_talk', 'File', 'File_talk', 'MediaWiki', 'MediaWiki_talk', 'Template', 'Template_talk', 'Help', 'Help_talk', 'Category', 'Category_talk', 'BSL', 'BSL_talk', 'OBD', 'OBD_talk', 'AE', 'AE_talk', 'Oni2', 'Oni2_talk', 'XML', 'XML_talk') 25 25 … … 27 27 onigalore_url = 'https://wiki.oni2.net/' 28 28 29 # Interwiki prefixes, for ruling out theselinks29 # Tuple of interwiki prefixes, for passing over such links 30 30 interwiki_prefixes = ('acronym', 'cache', 'commons', 'dictionary', 'google', 'metawikimedia', 'mw', 'wikibooks', 'wikidata', 'wikimedia', 'wikinews', 'wikipedia', 'wikiquote', 'wikisource', 'wikispecies', 'wikiversity', 'wikivoyage', 'wikt', 'wiktionary', 'wp') 31 31 32 # List of chapter names, for substitution into links that use "{{Cn}}" transclusion 33 chapter_names = ['CHAPTER_00_._COMBAT_TRAINING', 'CHAPTER_01_._TRIAL_RUN', 'CHAPTER_02_._ENGINES_OF_EVIL', 'CHAPTER_03_._PUZZLE_PIECES', 'CHAPTER_04_._TIGER_BY_THE_TAIL', 'CHAPTER_05_._HOT_PURSUIT', 'CHAPTER_06_._COUNTERATTACK', 'CHAPTER_07_._A_FRIEND_IN_NEED', 'CHAPTER_08_._AN_INNOCENT_LIFE', 'CHAPTER_09_._TRUTH_AND_CONSEQUENCES', 'CHAPTER_10_._CAT_AND_MOUSE', 'CHAPTER_11_._DREAM_DIVER', 'CHAPTER_12_._SINS_OF_THE_FATHER', 'CHAPTER_13_._PHOENIX_RISING', 'CHAPTER_14_._DAWN_OF_THE_CHRYSALIS'] 34 35 # Tuple of patterns for recognizing wikilinks 36 # Pattern 1: Detect "[[anything]]", "[[any:thing]]", "[[any|thing]]", "[[any:thi|ng]]" 37 # Pattern 2: Detect "{{SectionLink|Page|Section name}}", "{{SectionLink||Section name}}" 38 link_patterns = ("\[\[[^|\]]*(\||\])", "\{\{SectionLink\|[^|\}]*\|[^|\}]*\}\}") 39 40 # Initialize globals 41 debug = 0 32 42 pages_checked = 0 33 43 iw_found = 0 34 44 advice_issued = 0 35 warnings_issued = 036 45 errors_issued = 0 37 page_name = ''38 46 39 47 # Searches the given page text for intrawiki links with section links in them 40 def scan_for_iw_links(page_text): 41 global pages_checked 42 global iw_found 43 global advice_issued 44 global warnings_issued 45 global errors_issued 46 global page_name 47 pages_checked = pages_checked + 1 48 49 # Isolate strings of pattern "[[anything]]", "[[any:thing]]", "[[any|thing]]" or 50 # "[[any:thi|ng]]" 51 iw_link = "\[\[[^|\]]*(\||\])" 52 for match in re.finditer(iw_link, page_text): 53 found_iw_match = False 54 iw_url = "" 55 page_name2 = page_name 56 57 # Cut out the matched text from the page, and in the process remove the "[[" from the 58 # front and the "|" or "]" from the end 59 s = match.start() + 2 60 e = match.end() - 1 61 link_text = page_text[s:e] 62 63 # Sometimes we used a space char. instead of a '_', so fix that before querying 64 link_text = link_text.replace(' ', '_') 65 #pywikibot.stdout('Found link {0}.'.format(link_text)) 66 67 # If this link doesn't have a section link in it, then we don't care about it, as 68 # MediaWiki takes care of checking basic intrawiki links 69 if not '#' in link_text: 70 #pywikibot.stdout('Link doesn\'t have a section anchor in it. Skipping.') 48 def scan_for_intrawiki_links(page_text, page_name): 49 global debug 50 global pages_checked 51 global iw_found 52 global advice_issued 53 global errors_issued 54 pages_checked += 1 55 name_printed = 0 56 57 for i, the_pattern in enumerate(link_patterns): 58 if debug: 59 if i == 0: 60 pywikibot.stdout(' Checking page for wikilinks with section names.') 61 elif i == 1: 62 pywikibot.stdout(' Checking page for {{SectionLink}} calls.') 63 64 for match in re.finditer(the_pattern, page_text): 65 found_iw_match = False 66 iw_url = "" 67 page_name2 = page_name 68 69 # Cut out the matched text from the page, isolating just the page+section name 70 target_start = 2 # "[[" 71 target_end = 1 # "|" or "]" (we only match the first ending bracket) 72 if i == 1: 73 target_start = 14 # "{{SectionLink|" 74 target_end = 2 # "}}" 75 s = match.start() + target_start # remove the link-opening markup 76 e = match.end() - target_end # remove the link-ending markup 77 link_text = page_text[s:e] 78 79 # The second link type will look like "Page|Section" or "|Section", so fix that pipe 80 if i == 1: 81 link_text = link_text.replace('|', '#') 82 83 # Sometimes we use a space char. instead of a '_', so fix that before querying 84 link_text = link_text.replace(' ', '_') 85 if debug: pywikibot.stdout(' Found link {0}.'.format(link_text)) 86 87 # If this link doesn't have a section link in it, then we don't care about it, as 88 # MediaWiki takes care of checking basic intrawiki links 89 if not '#' in link_text: 90 if debug: pywikibot.stdout(' Link doesn\'t have a section anchor in it. Skipping.') 71 91 continue 72 92 73 # If this link has an interwiki prefix, it can be ignored 74 is_interwiki = False 75 if found_iw_match == False: 93 # If this link has an interwiki prefix, it can be ignored; see check_interwiki_links.py 94 # for the task of checking interwiki page+section links 95 is_interwiki = False 96 if found_iw_match == False: 76 97 for prefix in interwiki_prefixes: 77 78 #pywikibot.stdout('Skipping link {} because it is an interwiki link.'.format(link_text))79 80 81 if is_interwiki:98 if prefix + ":" in link_text: 99 if debug: pywikibot.stdout(' Skipping link {} because it is an interwiki link.'.format(link_text)) 100 is_interwiki = True 101 break 102 if is_interwiki: 82 103 continue 83 84 # If there is a '{' in the link, then probably it's a link built on transcluded text 85 # like "Quotes/Diary#{{C3}}", which we cannot expand and work with, so skip it 86 if '{' in link_text: 87 pywikibot.stdout('ADVICE: Link {} seems to use transclusion, so it can\'t be verified automatically. You should check it manually.'.format(link_text)) 88 advice_issued = advice_issued + 1 89 continue 90 91 # If this is a relative "/" link, use the current page as the basis for the URL. Note 92 # that only a leading slash is looked for, so if there's multiple steps down ("/x/y"), 93 # we're out of luck. 94 if link_text.startswith('/'): 104 105 # If there is a '{' in the link, then probably it's a link built on transcluded text 106 # like "Quotes/Diary#{{C3}}", which we cannot expand and work with, so skip it 107 if '{' in link_text: 108 ch_link_pattern = re.compile(r"{{C[0-9]*}}") 109 ch_link = ch_link_pattern.search(link_text) 110 if debug: pywikibot.stdout(' Found transclusion in link: "{}".'.format(ch_link.group(0))) 111 if ch_link: 112 ch_link_match = ch_link.group(0) 113 ch_num_pattern = re.compile("[0-9]+") 114 ch_num = ch_num_pattern.search(ch_link_match) 115 if ch_num: 116 ch_num_match = int(ch_num.group(0)) 117 if ch_num_match >= 0 and ch_num_match <= 14: 118 ch_name = chapter_names[ch_num_match] 119 replace_pattern = re.compile(r"{{C" + ch_num.group(0) + r"}}") 120 link_text = replace_pattern.sub(ch_name, link_text) 121 if debug: pywikibot.stdout(' After performing transclusion, link is now "{}".'.format(link_text)) 122 else: 123 if not name_printed and not debug: 124 pywikibot.stdout('From page "{}":'.format(page_name)) 125 name_printed = 1 126 pywikibot.stdout(' ADVICE: Link {0} transcludes a chapter name using an out-of-range number, {1}.'.format(link_text, ch_num_match)) 127 advice_issued += 1 128 continue 129 else: 130 if not name_printed and not debug: 131 pywikibot.stdout('From page "{}":'.format(page_name)) 132 name_printed = 1 133 pywikibot.stdout(' ADVICE: Link {} seems to be transcluding a chapter name, but this script couldn\'t read it.'.format(link_text)) 134 advice_issued += 1 135 continue 136 else: 137 if not name_printed and not debug: 138 pywikibot.stdout('From page "{}":'.format(page_name)) 139 name_printed = 1 140 pywikibot.stdout(' ADVICE: Link {0} seems to use transclusion. This script can understand chapter name transclusions such as "{1}" but it doesn\'t recognize this one so it can\'t be verified. You should check the link manually.'.format(link_text, "{{C7}}")) 141 advice_issued += 1 142 continue 143 144 # If this is a relative "/" link, use the current page as the basis for the URL. Note 145 # that only a leading slash is looked for, so if there's multiple steps down ("/x/y"), 146 # we're out of luck. 147 if link_text.startswith('/'): 95 148 link_text = page_name + link_text 96 #pywikibot.stdout('Changed link_text to {} on account of "/".'.format(link_text))97 98 # If this is a relative "../" link, find the parent page andset ourselves to that page,99 # then remove the relative portion of the link. Note that this is only performed once,100 # so if there's multiple steps back ("../../"), we're out of luck.101 if link_text.startswith('../'):149 if debug: pywikibot.stdout(' Changed link_text to {} on account of "/".'.format(link_text)) 150 151 # If this is a relative "../" link, find the parent page, set ourselves to that page, 152 # then remove the relative portion of the link. Note that this is only performed once, 153 # so if there's multiple steps back ("../../"), we're out of luck. 154 if link_text.startswith('../'): 102 155 last_slash = page_name.rfind('/') 103 156 page_name2 = page_name[0:last_slash] 104 #pywikibot.stdout('Changed page_name to {} on account of "../".'.format(page_name2))157 if debug: pywikibot.stdout(' Changed page_name to {} on account of "../".'.format(page_name2)) 105 158 link_text = link_text[3:len(link_text)] 106 #pywikibot.stdout('Changed link_text to {} on account of "../".'.format(link_text))159 if debug: pywikibot.stdout(' Changed link_text to {} on account of "../".'.format(link_text)) 107 160 # If this is now going to be a bare section link for the parent page, don't add a 108 161 # slash, otherwise do because we are drilling down to another subpage 109 162 if link_text.startswith('#'): 110 163 link_text = page_name2 + link_text 111 164 else: 112 113 114 # If this is a bare section link, build URL based on this page115 if link_text.startswith('#'):165 link_text = page_name2 + '/' + link_text 166 167 # If this is a bare section link, build URL based on this page 168 if link_text.startswith('#'): 116 169 iw_url = onigalore_url + page_name2 117 iw_found = iw_found +1118 #pywikibot.stdout('Found link to this very page, {}.'.format(link_text))170 iw_found += 1 171 if debug: pywikibot.stdout(' Found link to this very page, {}.'.format(link_text)) 119 172 found_iw_match = True 120 173 link_text = page_name2 + link_text 121 122 # If there's no ":" in the link (before the section link, where a colon would just be123 # part of the text) then it's a Main namespace article, so constructURL124 if found_iw_match == False:174 175 # If there's no ":" in the link (before the section link, where a colon would just be 176 # part of the text) then it's a Main namespace article; proceed with building URL 177 if found_iw_match == False: 125 178 if not re.search(":.*#", link_text): 126 iw_url = onigalore_url + link_text 127 iw_found = iw_found + 1 128 #pywikibot.stdout('Found link to OniGalore Main namespace page {}.'.format(link_text)) 129 found_iw_match = True 130 131 # If there is a ":", match the prefix against the intrawiki prefixes on OniGalore 132 if found_iw_match == False: 179 iw_url = onigalore_url + link_text 180 iw_found += 1 181 if debug: pywikibot.stdout(' Link is to a Main namespace page.') 182 found_iw_match = True 183 184 # If there is a ":", match the prefix against the intrawiki prefixes on OniGalore 185 # before building URL 186 if found_iw_match == False: 133 187 for prefix in intrawiki_prefixes: 134 #pywikibot.stdout('Comparing link against prefix {}.'.format(prefix)) 135 if prefix + ":" in link_text: 136 iw_url = onigalore_url + link_text 137 _, post_ns = link_text.split(':', 1) 138 #pywikibot.stdout('Found link to OniGalore {0} namespace page {1}.'.format(prefix, post_ns)) 139 iw_found = iw_found + 1 140 found_iw_match = True 141 break 142 143 # If we still haven't turned this match into a URL, something's gone wrong 144 if (found_iw_match == False) or (iw_url == ""): 145 pywikibot.stdout('ERROR: Couldn\'t figure out link {}.'.format(link_text)) 188 if prefix + ":" in link_text: 189 iw_url = onigalore_url + link_text 190 if debug: pywikibot.stdout(' Identified namespace {}.'.format(prefix)) 191 iw_found += 1 192 found_iw_match = True 193 break 194 195 # If we still haven't turned this match into a URL, something's gone wrong 196 if (found_iw_match == False) or (iw_url == ""): 197 if not name_printed and not debug: 198 pywikibot.stdout('From page "{}":'.format(page_name)) 199 name_printed = 1 200 pywikibot.stdout(' ERROR: Couldn\'t figure out link {}.'.format(link_text)) 146 201 continue 147 202 148 # Test the URL 149 iw_url = iw_url.replace(' ', '_') 150 #pywikibot.stdout('Reading page at {}...'.format(iw_url)) 151 response = fetch(iw_url) 152 153 # Redirects are followed automatically by fetch() and treated as "200"s; the way we can 154 # tell that a redirect occurred is by checking fetch's history 155 if response.history != []: 156 pywikibot.stdout('WARNING: Got redirection code ({0}) on URL "{1}".'.format(response.history[0], iw_url)) 157 warnings_issued = warnings_issued + 1 158 elif response.status_code != 200: 159 pywikibot.stdout('WARNING: Got response code {0} on URL {1}.'.format(response.status_code, iw_url)) 160 warnings_issued = warnings_issued + 1 161 else: 203 # Test the URL 204 iw_url = iw_url.replace(' ', '_') 205 if debug: pywikibot.stdout(' Reading page at {}...'.format(iw_url)) 206 response = fetch(iw_url) 207 208 # Redirects are followed automatically by fetch() and treated as "200"s; the way we can 209 # tell that a redirect occurred is by checking fetch's history 210 if response.history != []: 211 if not name_printed and not debug: 212 pywikibot.stdout('From page "{}":'.format(page_name)) 213 name_printed = 1 214 pywikibot.stdout(' ADVICE: Got redirection code ({0}) on URL "{1}". You should check the link manually.'.format(response.history[0], iw_url)) 215 advice_issued += 1 216 elif response.status_code != 200: 217 if not name_printed and not debug: 218 pywikibot.stdout('From page "{}":'.format(page_name)) 219 name_printed = 1 220 pywikibot.stdout(' ERROR: Got response code {0} on URL {1}. The target page may not exist.'.format(response.status_code, iw_url)) 221 errors_issued += 1 222 else: 162 223 # Isolate section link 163 224 pre_section, section_name = link_text.split('#', 1) 164 #pywikibot.stdout('Searching for section link {} on page.'.format(section_name))165 225 if debug: pywikibot.stdout(' Searching for section link {} on page.'.format(section_name)) 226 166 227 # Convert slash character to the dot-notation hex encoding that MediaWiki uses 167 228 section_name = section_name.replace('/', '.2F') 168 229 169 230 # Read linked page to see if it really has this anchor link 170 231 soup = BeautifulSoup(response.text, 'html.parser') 171 232 found_section = False 172 233 for span_tag in soup.findAll('span'): 173 174 175 #pywikibot.stdout('Found section!')176 177 234 span_name = span_tag.get('id', None) 235 if span_name == section_name: 236 if debug: pywikibot.stdout(' Found section!') 237 found_section = True 238 break 178 239 if found_section == False: 179 pywikibot.stdout('ERROR: Could not find section {0} on page {1}!'.format(section_name, pre_section)) 180 errors_issued = errors_issued + 1 240 if not name_printed and not debug: 241 pywikibot.stdout('From page "{}":'.format(page_name)) 242 name_printed = 1 243 pywikibot.stdout(' ERROR: Could not find section {0} on page {1}!'.format(section_name, pre_section)) 244 errors_issued += 1 181 245 182 246 def main(*args): 183 cat_name = '' 184 global page_name 185 186 local_args = pywikibot.handle_args(args) 187 genFactory = pagegenerators.GeneratorFactory() 188 189 for arg in local_args: 190 if arg.startswith('-cat:'): 191 cat_name = arg[5:] 192 elif arg.startswith('-page:'): 193 page_name = arg[6:] 194 195 site = pywikibot.Site() 196 197 # This line of code enumerates the methods in the 'page' class 198 #pywikibot.stdout(format(dir(page))) 199 200 if cat_name != '': 201 cat_obj = pywikibot.Category(site, cat_name) 202 generator = pagegenerators.CategorizedPageGenerator(cat_obj, recurse=True) 203 for page in pagegenerators.PreloadingGenerator(generator, 100): 204 pywikibot.stdout('Checking page {0}'.format(page.title())) 205 page_name = page.title() 206 scan_for_iw_links(page.text) 207 elif page_name != '': 208 page = pywikibot.Page(site, page_name) 209 pywikibot.stdout('Checking page {0}'.format(page.title())) 210 scan_for_iw_links(page.text) 211 212 global pages_checked 213 global iw_found 214 global advice_issued 215 global warnings_issued 216 global errors_issued 217 218 page_str = "pages" 219 if pages_checked == 1: 220 page_str = "page" 221 222 link_str = "links" 223 if iw_found == 1: 224 link_str = "link" 225 226 pywikibot.stdout('Checked {0} {1} and found {2} intrawiki {3}.'.format(pages_checked, page_str, iw_found, link_str)) 227 pywikibot.stdout('While attempting to follow section links...') 228 229 if advice_issued == 0: 230 pywikibot.stdout(' No advice on potential problems was issued.') 231 elif advice_issued == 1: 232 pywikibot.stdout(' 1 piece of advice on a potential problem was issued.') 233 else: 234 pywikibot.stdout(' {} pieces of advice on potential problems were issued.'.format(advice_issued)) 235 236 warning_str = "warnings were" 237 if warnings_issued == 1: 238 warning_str = "warning was" 239 pywikibot.stdout(' {0} {1} issued.'.format(warnings_issued, warning_str)) 240 241 error_str = "errors were" 242 if errors_issued == 1: 243 error_str = "error was" 244 pywikibot.stdout(' {0} {1} encountered.'.format(errors_issued, error_str)) 247 global debug 248 global pages_checked 249 global iw_found 250 global advice_issued 251 global errors_issued 252 search_cat = '' 253 search_page = '' 254 255 local_args = pywikibot.handle_args(args) 256 genFactory = pagegenerators.GeneratorFactory() 257 258 for arg in local_args: 259 if arg.startswith('-cat:'): 260 search_cat = arg[5:] 261 elif arg.startswith('-page:'): 262 search_page = arg[6:] 263 elif arg == '-dbg': 264 debug = 1 265 else: 266 pywikibot.stdout('Unknown argument "{}".'.format(arg)) 267 return 268 269 site = pywikibot.Site() 270 271 # This line of code enumerates the methods in the 'page' class 272 #pywikibot.stdout(format(dir(page))) 273 274 if search_cat != '': 275 cat_obj = pywikibot.Category(site, search_cat) 276 generator = pagegenerators.CategorizedPageGenerator(cat_obj, recurse=True) 277 for page in pagegenerators.PreloadingGenerator(generator, 100): 278 if debug: pywikibot.stdout('Checking page {0}'.format(page.title())) 279 scan_for_intrawiki_links(page.text, page.title()) 280 elif search_page != '': 281 page = pywikibot.Page(site, search_page) 282 if debug: pywikibot.stdout('Checking page {0}'.format(page.title())) 283 scan_for_intrawiki_links(page.text, page.title()) 284 285 page_str = "pages" 286 if pages_checked == 1: 287 page_str = "page" 288 289 link_str = "links" 290 if iw_found == 1: 291 link_str = "link" 292 293 pywikibot.stdout('Checked {0} {1} and found {2} intrawiki {3}.'.format(pages_checked, page_str, iw_found, link_str)) 294 pywikibot.stdout('While attempting to follow section links...') 295 296 if advice_issued == 0: 297 pywikibot.stdout(' No advice on potential problems was issued.') 298 elif advice_issued == 1: 299 pywikibot.stdout(' 1 piece of advice on a potential problem was issued.') 300 else: 301 pywikibot.stdout(' {} pieces of advice on potential problems were issued.'.format(advice_issued)) 302 303 error_str = "errors were" 304 if errors_issued == 1: 305 error_str = "error was" 306 pywikibot.stdout(' {0} {1} encountered.'.format(errors_issued, error_str)) 245 307 246 308 if __name__ == '__main__': 247 309 main()
Note:
See TracChangeset
for help on using the changeset viewer.