User:CarpetBot/Source
Jump to navigation
Jump to search
Requires mwapi; only tested with Python 3.
#!/usr/bin/env python3
# encoding: utf8
#
# Copyright © 2015 Martin Tournoij <martin@arp242.net>
# See below for full copyright
#
from __future__ import print_function
import json, sys, time, re
import mwapi
session = mwapi.Session('http://rationalwiki.org', 'Carpetbot/1.0 <martin@arp242.net>')
l = session.login('Carpetbot', 'XXX')
_noop = False
#_noop = True
# Like pprint but more readable
def pp(d):
json.dump(d, sys.stdout, sort_keys=True, indent=4)
print()
# Split list +l+ into +s+ sized chunks with the remained appended
def chunk(l, s):
if len(l) <= s: return l
nl = []
for i in range(0, len(l) // s):
nl += (l[i*s:i*s+s],)
if len(l) % s != 0:
nl += (l[i*s+s:],)
return nl
# Check if a page has the template +tpl+ in +templates+ (being the return value
# from action=query&prop=templates). +content+ can also be given for a double
# check, as sometimes not the complete list of templates is returned with this
# (?!?! example: Talk:Michele Bachmann doesn't have {{person}}).
#
# Example usage:
# page = list(session.get(action='query',
# titles='Main page',
# prop='revisions|templates',
# rvprop='content')['query']['pages'].values())[0]
#
# has = has_template('Person', page.get('templates', []), page['revisions'][0]['*']
def has_template(tpl, templates, content=None):
if templates is not None and 'Template:{}'.format(tpl) in [ t['title'] for t in templates ]:
return True
# Do case-insensitive search; this could clash since SomeTpl and Sometpl are
# not the same page, but the chance of one being a redirect to the other is
# large, and typically we prefer a false positive over a false negative
# (can't be arsed to check redirects here)
if content is not None and re.search(re.escape(tpl), content, re.IGNORECASE) is not None:
return True
return False
# Like session.post(), and solve the turing test if we encounter one
def action_with_captcha(**params):
if _noop: return
ret = session.post(**params)
ret = ret[params['action']]
if ret.get('result') and ret['result'] == 'Failure' and ret.get('captcha'):
q = ret['captcha']['question'].replace('−', '-')
# We're trusting RW here...
answer = eval(q)
ret = session.post(
captchaid=ret['captcha']['id'],
captchaword=answer,
**params
)
if ret.get('result') and ret['result'] == 'Failure' and ret.get('captcha'):
print('Unable to solve CAPTCHA?')
print("Our answer to `{}' was `{}'".format(q, answer))
print()
pp(ret)
sys.exit(0)
return ret
# Enhanced version of list=querypage&qpage=Unusedimages which gets the image
# pages with some info
def get_unused(limit=50, offset=0):
images = session.get(action='query',
list='querypage',
qppage='Unusedimages',
qplimit=limit,
qpoffset=offset
)
if len(images) == 0:
return []
# For some reason getting a whole bunch of pages in one go doesn't work
# correctly, after the third page "templates" is always empty...
titles = chunk([ i['title'] for i in images['query']['querypage']['results'] ], 3)
pages = []
for t in titles:
pages += session.get(action='query',
titles='|'.join(t),
prop='revisions|info|templates',
rvprop='content')['query']['pages'].values()
return pages
# Get token. You can re-use a token as many times as you want
def get_token(which='edit'):
if which == 'deletedrevs': which = 'edit'
#if which == 'deletedrevs':
# data = session.get(action='query',
# list='deletedrevs',
# drprop='token')
# pp(data)
# sys.exit(1)
data = session.get(action='query',
titles='Main Page',
prop='info',
intoken=which)
try:
token = list(data['query']['pages'].values())[0]['{}token'.format(which)]
except:
pp(data)
print(sys.exc_info()[1])
sys.exit(1)
return token
# Find all unused images without copyright and delete them
def unused_images_wo_copyright(start=1):
print("THIS IS BROKEN! It doesn't check if a file is linked to; only if it's embeded.")
sys.exit(1)
token = get_token('delete')
limit = 50
offset = start * limit
for i in range(0, 999):
print('=>', i)
unused = get_unused(limit, offset)
if len(unused) == 0:
print('Our work here is done')
sys.exit(0)
pages = [ ]
for p in unused:
# Redirect page, file with no revisions, or other "special"
# situation
if p.get('templates') is None:
print('Warning: {} has no templates'.format(p['title']))
continue
if 'Template:Nolicense' in [ t['title'] for t in p['templates'] ]:
pages.append(p)
if len(pages) == 0:
offset += limit
continue
offset += limit - len(pages)
for page in pages:
print(' ', page['title'])
#print(page)
action_with_captcha(
action='delete',
pageid=page['pageid'],
token=token,
reason='Removing unused image with no copyright')
time.sleep(1)
def delete_pages_from_stdin():
token = get_token('delete')
while True:
line = sys.stdin.readline().strip()
if line == '': break
print(line)
action_with_captcha(
action='delete',
title=line,
token=token,
reason='Removing unused image with no copyright')
# Get all images
def all(limit=100, offset=0):
images = session.get(action='query',
list='allimages',
qplimit=limit,
qpoffset=offset
)
if len(images) == 0:
return []
# For some reason getting a whole bunch of pages in one go doesn't work
# correctly, after the third page "templates" is always empty...
titles = chunk([ i['title'] for i in images['query']['querypage']['results'] ], 3)
pages = []
for t in titles:
pages += session.get(action='query',
titles='|'.join(t),
prop='revisions|info|templates',
rvprop='content')['query']['pages'].values()
return pages
def fix_cz_screencaps():
token = get_token('edit')
while True:
line = sys.stdin.readline().strip()
if line == '': break
print(line)
page = list(session.get(action='query',
titles=line,
prop='revisions|info',
rvprop='content')['query']['pages'].values())[0]
url = page['revisions'][0]['*'].split('\n')[0]
action_with_captcha(
action='edit',
bot=True,
title=line,
token=token,
reason='Add license for Citizendium screencaps',
text=url + '\n{{CZ screenshot}}\n[[Category:Citizendium screencaps]]')
# Make talk page title from page title:
# David Cameron -> Talk:David Cameron
# Fun:David Cameron -> Fun talk:David Cameron
# User:David Cameron -> User talk:David Cameron
def make_talk_title(title):
if ':' in title:
title = title.split(':')
return '{} talk:{}'.format(title[0], ':'.join(title[1:]))
else:
return 'Talk:{}'.format(title)
def living_people():
token = get_token('edit')
cont = 0
while True:
if cont is None: break
pages = session.get(action='query',
list='categorymembers',
cmtitle='Category:Living people',
cmlimit=500,
cmcontinue=cont
)
if pages.get('query-continue') is None:
cont = None
else:
cont = pages['query-continue']['categorymembers']['cmcontinue']
pages = pages['query']['categorymembers']
if len(pages) == 0: break
for p in pages:
talktitle = make_talk_title(p['title'])
print('=>', talktitle.jlust(40), end=' ')
talk = list(session.get(action='query',
titles=talktitle,
prop='revisions|templates',
rvprop='content')['query']['pages'].values())[0]
#pp(talk)
# No talk page yet, so create one
if talk.get('missing') is not None:
print('Creating talk page')
action_with_captcha(
action='edit',
bot=True,
title=talktitle,
token=token,
reason='Add {{person}} template for living person',
summary='Add {{person}} template for living person',
text='{{person}}')
continue
if has_template('Person', talk.get('templates', []), talk['revisions'][0]['*']):
print('Already has the template')
continue
print('Adding template')
action_with_captcha(
action='edit',
bot=True,
title=talktitle,
token=token,
reason='Add {{person}} template for living person',
summary='Add {{person}} template for living person',
text='{{person}}\n' + talk['revisions'][0]['*'])
# TODO: There's got to be some bot for this already?
def double_redirect():
pass
# List all images embeded or linked to in mainspace with either no copyright
# information, or listed as fair use.
def list_mainspace_images():
apfrom = ''
nolicense = []
fairuse = []
while True:
if apfrom is None: break
images = session.get(action='query',
list='allpages',
apnamespace=6,
aplimit=500,
apfrom=apfrom)
if images.get('query-continue') is None:
apfrom = None
else:
apfrom = images['query-continue']['allpages']['apfrom']
images = images['query']['allpages']
# Conservapedia, Template, Category, Main, Fun, Help, RationalWiki
mainns = '100|10|14|0|106|12|4'
for img in images:
try:
print(img['title'].ljust(60), end='')
links = session.get(action='query',
prop='templates',
list='backlinks|imageusage|embeddedin',
titles=img['title'], bltitle=img['title'], iutitle=img['title'], eititle=img['title'],
blnamespace=mainns, iunamespace=mainns, einamespace=mainns,
bllimit=1, iulimit=1, tllimit=500, eilimit=1
)['query']
if len(links['backlinks']) == 0 and len(links['embeddedin']) == 0 and len(links['imageusage']) == 0:
print('not in mainspace')
continue
tpls = list(links['pages'].values())[0]
if tpls.get('templates') is None:
tpls = ['Template:Nolicense']
else:
tpls = [ t['title'] for t in tpls['templates'] ]
if 'Template:Nolicense' in tpls:
print('No license')
nolicense.append(img['title'])
elif 'Template:Fair use' in tpls:
print('Fair use')
fairuse.append(img['title'])
else:
print()
except:
print('ERROR')
print(sys.exc_info()[1])
print('\n')
print('==> No license ({})'.format(len(nolicense)))
for img in nolicense: print('http://rationalwiki.org/wiki/{}'.format(img))
print('==> Fair use ({})'.format(len(fairuse)))
for img in fairuse: print('http://rationalwiki.org/wiki/{}'.format(img))
def list_namespaces():
lst = session.get(action='query',
meta='siteinfo',
siprop='namespaces|namespacealiases')
for k, ns in lst['query']['namespaces'].items():
print(' {} -> {}'.format(ns['id'], ns['*']))
print('\n==> Namespace aliases <==')
for ns in lst['query']['namespacealiases']:
print(' {} -> {}'.format(ns['id'], ns['*']))
def fix_the_fucking_mess():
import urllib
token = get_token('deletedrevs')
while True:
line = sys.stdin.readline().strip()
if line == '': break
line = urllib.parse.unquote(line)
print(line.ljust(80), end='')
# backlinks
data = session.get(action='query',
list='backlinks',
bltitle=line)['query']['backlinks']
data = [ d for d in data if d['title'] != 'User:Carpetbot/Removed' ]
if len(data) == 0:
print()
continue
page = session.get(action='query',
titles=line,
prop='info')
if page['query']['pages'].get('-1') is None or page['query']['pages']['-1'].get('missing') is None:
print('Already restored')
else:
print('Restoring')
session.post(action='undelete',
title=line,
token=token,
reason='Page is not embeded, but is linked to')
if __name__ == '__main__':
ops = {
'unused_without_copyright': (
lambda: unused_images_wo_copyright(0),
'Delete all images that are not used (embeded) and have no copyright'
),
'delete_pages': (
delete_pages_from_stdin,
'delete all the pages from stdin, seperated by newline'
),
'cz_copyright': (
fix_cz_screencaps,
'Fix CZ captures; images to fix are from stdin'
),
'people': (
living_people,
'Add {{person}} to all talkpages for pages with [[Category: Living people]]'
),
'double_redirect': (
double_redirect,
'Fix all double redirects'
),
'list_mainspace_images': (
list_mainspace_images,
'List all mainspace images no copyright or fair use'
),
'list_namespaces': (
list_namespaces,
'List all namespaces'
),
'fix_the_fucking_mess': (
fix_the_fucking_mess,
'Fix the fucking UnusedImages mess'
),
}
def show_help(err=None):
if err: print('Error: {}\n'.format(err), file=sys.stderr)
print('Available operations\n')
longest = max([ len(k) for k in ops.keys() ])
for k, v in ops.items():
print(' {}{} {}'.format(k.ljust(longest), v[1]))
if err: sys.exit(1)
if len(sys.argv) < 2: show_help('No operation given')
op = ops.get(sys.argv[1])
if op is None: show_help("Unknown operation `{}'".format(sys.argv[1]))
op[0]()
# The MIT License (MIT)
#
# Copyright © 2015 Martin Tournoij
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# The software is provided "as is", without warranty of any kind, express or
# implied, including but not limited to the warranties of merchantability,
# fitness for a particular purpose and noninfringement. In no event shall the
# authors or copyright holders be liable for any claim, damages or other
# liability, whether in an action of contract, tort or otherwise, arising
# from, out of or in connection with the software or the use or other dealings
# in the software.