| There is no RationalWiki without you. We are a small non-profit with no staff – we are hundreds of volunteers who document pseudoscience and crankery around the world every day. We will never allow ads because we must remain independent. We cannot rely on big donors with corresponding big agendas. We are not the largest website around, but we believe we play an important role in defending truth and objectivity. If everyone who saw this today donated $5, we would meet our goal for 2019. |
Fighting pseudoscience isn't free. We are 100% user-supported! Help and donate $5, $20 or whatever you can today with |
User:CarpetBot/Source
From RationalWiki
Requires mwapi; only tested with Python 3.
#!/usr/bin/env python3
# encoding: utf8
#
# Copyright © 2015 Martin Tournoij <martin@arp242.net>
# See below for full copyright
#
from __future__ import print_function
import json, sys, time, re
import mwapi
session = mwapi.Session('http://rationalwiki.org', 'Carpetbot/1.0 <martin@arp242.net>')
l = session.login('Carpetbot', 'XXX')
_noop = False
#_noop = True
# Like pprint but more readable
def pp(d):
json.dump(d, sys.stdout, sort_keys=True, indent=4)
print()
# Split list +l+ into +s+ sized chunks with the remained appended
def chunk(l, s):
if len(l) <= s: return l
nl = []
for i in range(0, len(l) // s):
nl += (l[i*s:i*s+s],)
if len(l) % s != 0:
nl += (l[i*s+s:],)
return nl
# Check if a page has the template +tpl+ in +templates+ (being the return value
# from action=query&prop=templates). +content+ can also be given for a double
# check, as sometimes not the complete list of templates is returned with this
# (?!?! example: Talk:Michele Bachmann doesn't have {{person}}).
#
# Example usage:
# page = list(session.get(action='query',
# titles='Main page',
# prop='revisions|templates',
# rvprop='content')['query']['pages'].values())[0]
#
# has = has_template('Person', page.get('templates', []), page['revisions'][0]['*']
def has_template(tpl, templates, content=None):
if templates is not None and 'Template:{}'.format(tpl) in [ t['title'] for t in templates ]:
return True
# Do case-insensitive search; this could clash since SomeTpl and Sometpl are
# not the same page, but the chance of one being a redirect to the other is
# large, and typically we prefer a false positive over a false negative
# (can't be arsed to check redirects here)
if content is not None and re.search(re.escape(tpl), content, re.IGNORECASE) is not None:
return True
return False
# Like session.post(), and solve the turing test if we encounter one
def action_with_captcha(**params):
if _noop: return
ret = session.post(**params)
ret = ret[params['action']]
if ret.get('result') and ret['result'] == 'Failure' and ret.get('captcha'):
q = ret['captcha']['question'].replace('−', '-')
# We're trusting RW here...
answer = eval(q)
ret = session.post(
captchaid=ret['captcha']['id'],
captchaword=answer,
**params
)
if ret.get('result') and ret['result'] == 'Failure' and ret.get('captcha'):
print('Unable to solve CAPTCHA?')
print("Our answer to `{}' was `{}'".format(q, answer))
print()
pp(ret)
sys.exit(0)
return ret
# Enhanced version of list=querypage&qpage=Unusedimages which gets the image
# pages with some info
def get_unused(limit=50, offset=0):
images = session.get(action='query',
list='querypage',
qppage='Unusedimages',
qplimit=limit,
qpoffset=offset
)
if len(images) == 0:
return []
# For some reason getting a whole bunch of pages in one go doesn't work
# correctly, after the third page "templates" is always empty...
titles = chunk([ i['title'] for i in images['query']['querypage']['results'] ], 3)
pages = []
for t in titles:
pages += session.get(action='query',
titles='|'.join(t),
prop='revisions|info|templates',
rvprop='content')['query']['pages'].values()
return pages
# Get token. You can re-use a token as many times as you want
def get_token(which='edit'):
if which == 'deletedrevs': which = 'edit'
#if which == 'deletedrevs':
# data = session.get(action='query',
# list='deletedrevs',
# drprop='token')
# pp(data)
# sys.exit(1)
data = session.get(action='query',
titles='Main Page',
prop='info',
intoken=which)
try:
token = list(data['query']['pages'].values())[0]['{}token'.format(which)]
except:
pp(data)
print(sys.exc_info()[1])
sys.exit(1)
return token
# Find all unused images without copyright and delete them
def unused_images_wo_copyright(start=1):
print("THIS IS BROKEN! It doesn't check if a file is linked to; only if it's embeded.")
sys.exit(1)
token = get_token('delete')
limit = 50
offset = start * limit
for i in range(0, 999):
print('=>', i)
unused = get_unused(limit, offset)
if len(unused) == 0:
print('Our work here is done')
sys.exit(0)
pages = [ ]
for p in unused:
# Redirect page, file with no revisions, or other "special"
# situation
if p.get('templates') is None:
print('Warning: {} has no templates'.format(p['title']))
continue
if 'Template:Nolicense' in [ t['title'] for t in p['templates'] ]:
pages.append(p)
if len(pages) == 0:
offset += limit
continue
offset += limit - len(pages)
for page in pages:
print(' ', page['title'])
#print(page)
action_with_captcha(
action='delete',
pageid=page['pageid'],
token=token,
reason='Removing unused image with no copyright')
time.sleep(1)
def delete_pages_from_stdin():
token = get_token('delete')
while True:
line = sys.stdin.readline().strip()
if line == '': break
print(line)
action_with_captcha(
action='delete',
title=line,
token=token,
reason='Removing unused image with no copyright')
# Get all images
def all(limit=100, offset=0):
images = session.get(action='query',
list='allimages',
qplimit=limit,
qpoffset=offset
)
if len(images) == 0:
return []
# For some reason getting a whole bunch of pages in one go doesn't work
# correctly, after the third page "templates" is always empty...
titles = chunk([ i['title'] for i in images['query']['querypage']['results'] ], 3)
pages = []
for t in titles:
pages += session.get(action='query',
titles='|'.join(t),
prop='revisions|info|templates',
rvprop='content')['query']['pages'].values()
return pages
def fix_cz_screencaps():
token = get_token('edit')
while True:
line = sys.stdin.readline().strip()
if line == '': break
print(line)
page = list(session.get(action='query',
titles=line,
prop='revisions|info',
rvprop='content')['query']['pages'].values())[0]
url = page['revisions'][0]['*'].split('\n')[0]
action_with_captcha(
action='edit',
bot=True,
title=line,
token=token,
reason='Add license for Citizendium screencaps',
text=url + '\n{{CZ screenshot}}\n[[Category:Citizendium screencaps]]')
# Make talk page title from page title:
# David Cameron -> Talk:David Cameron
# Fun:David Cameron -> Fun talk:David Cameron
# User:David Cameron -> User talk:David Cameron
def make_talk_title(title):
if ':' in title:
title = title.split(':')
return '{} talk:{}'.format(title[0], ':'.join(title[1:]))
else:
return 'Talk:{}'.format(title)
def living_people():
token = get_token('edit')
cont = 0
while True:
if cont is None: break
pages = session.get(action='query',
list='categorymembers',
cmtitle='Category:Living people',
cmlimit=500,
cmcontinue=cont
)
if pages.get('query-continue') is None:
cont = None
else:
cont = pages['query-continue']['categorymembers']['cmcontinue']
pages = pages['query']['categorymembers']
if len(pages) == 0: break
for p in pages:
talktitle = make_talk_title(p['title'])
print('=>', talktitle.jlust(40), end=' ')
talk = list(session.get(action='query',
titles=talktitle,
prop='revisions|templates',
rvprop='content')['query']['pages'].values())[0]
#pp(talk)
# No talk page yet, so create one
if talk.get('missing') is not None:
print('Creating talk page')
action_with_captcha(
action='edit',
bot=True,
title=talktitle,
token=token,
reason='Add {{person}} template for living person',
summary='Add {{person}} template for living person',
text='{{person}}')
continue
if has_template('Person', talk.get('templates', []), talk['revisions'][0]['*']):
print('Already has the template')
continue
print('Adding template')
action_with_captcha(
action='edit',
bot=True,
title=talktitle,
token=token,
reason='Add {{person}} template for living person',
summary='Add {{person}} template for living person',
text='{{person}}\n' + talk['revisions'][0]['*'])
# TODO: There's got to be some bot for this already?
def double_redirect():
pass
# List all images embeded or linked to in mainspace with either no copyright
# information, or listed as fair use.
def list_mainspace_images():
apfrom = ''
nolicense = []
fairuse = []
while True:
if apfrom is None: break
images = session.get(action='query',
list='allpages',
apnamespace=6,
aplimit=500,
apfrom=apfrom)
if images.get('query-continue') is None:
apfrom = None
else:
apfrom = images['query-continue']['allpages']['apfrom']
images = images['query']['allpages']
# Conservapedia, Template, Category, Main, Fun, Help, RationalWiki
mainns = '100|10|14|0|106|12|4'
for img in images:
try:
print(img['title'].ljust(60), end='')
links = session.get(action='query',
prop='templates',
list='backlinks|imageusage|embeddedin',
titles=img['title'], bltitle=img['title'], iutitle=img['title'], eititle=img['title'],
blnamespace=mainns, iunamespace=mainns, einamespace=mainns,
bllimit=1, iulimit=1, tllimit=500, eilimit=1
)['query']
if len(links['backlinks']) == 0 and len(links['embeddedin']) == 0 and len(links['imageusage']) == 0:
print('not in mainspace')
continue
tpls = list(links['pages'].values())[0]
if tpls.get('templates') is None:
tpls = ['Template:Nolicense']
else:
tpls = [ t['title'] for t in tpls['templates'] ]
if 'Template:Nolicense' in tpls:
print('No license')
nolicense.append(img['title'])
elif 'Template:Fair use' in tpls:
print('Fair use')
fairuse.append(img['title'])
else:
print()
except:
print('ERROR')
print(sys.exc_info()[1])
print('\n')
print('==> No license ({})'.format(len(nolicense)))
for img in nolicense: print('http://rationalwiki.org/wiki/{}'.format(img))
print('==> Fair use ({})'.format(len(fairuse)))
for img in fairuse: print('http://rationalwiki.org/wiki/{}'.format(img))
def list_namespaces():
lst = session.get(action='query',
meta='siteinfo',
siprop='namespaces|namespacealiases')
for k, ns in lst['query']['namespaces'].items():
print(' {} -> {}'.format(ns['id'], ns['*']))
print('\n==> Namespace aliases <==')
for ns in lst['query']['namespacealiases']:
print(' {} -> {}'.format(ns['id'], ns['*']))
def fix_the_fucking_mess():
import urllib
token = get_token('deletedrevs')
while True:
line = sys.stdin.readline().strip()
if line == '': break
line = urllib.parse.unquote(line)
print(line.ljust(80), end='')
# backlinks
data = session.get(action='query',
list='backlinks',
bltitle=line)['query']['backlinks']
data = [ d for d in data if d['title'] != 'User:Carpetbot/Removed' ]
if len(data) == 0:
print()
continue
page = session.get(action='query',
titles=line,
prop='info')
if page['query']['pages'].get('-1') is None or page['query']['pages']['-1'].get('missing') is None:
print('Already restored')
else:
print('Restoring')
session.post(action='undelete',
title=line,
token=token,
reason='Page is not embeded, but is linked to')
if __name__ == '__main__':
ops = {
'unused_without_copyright': (
lambda: unused_images_wo_copyright(0),
'Delete all images that are not used (embeded) and have no copyright'
),
'delete_pages': (
delete_pages_from_stdin,
'delete all the pages from stdin, seperated by newline'
),
'cz_copyright': (
fix_cz_screencaps,
'Fix CZ captures; images to fix are from stdin'
),
'people': (
living_people,
'Add {{person}} to all talkpages for pages with [[Category: Living people]]'
),
'double_redirect': (
double_redirect,
'Fix all double redirects'
),
'list_mainspace_images': (
list_mainspace_images,
'List all mainspace images no copyright or fair use'
),
'list_namespaces': (
list_namespaces,
'List all namespaces'
),
'fix_the_fucking_mess': (
fix_the_fucking_mess,
'Fix the fucking UnusedImages mess'
),
}
def show_help(err=None):
if err: print('Error: {}\n'.format(err), file=sys.stderr)
print('Available operations\n')
longest = max([ len(k) for k in ops.keys() ])
for k, v in ops.items():
print(' {}{} {}'.format(k.ljust(longest), v[1]))
if err: sys.exit(1)
if len(sys.argv) < 2: show_help('No operation given')
op = ops.get(sys.argv[1])
if op is None: show_help("Unknown operation `{}'".format(sys.argv[1]))
op[0]()
# The MIT License (MIT)
#
# Copyright © 2015 Martin Tournoij
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# The software is provided "as is", without warranty of any kind, express or
# implied, including but not limited to the warranties of merchantability,
# fitness for a particular purpose and noninfringement. In no event shall the
# authors or copyright holders be liable for any claim, damages or other
# liability, whether in an action of contract, tort or otherwise, arising
# from, out of or in connection with the software or the use or other dealings
# in the software.