User:CarpetBot/Source

Requires mwapi; only tested with Python 3.

#!/usr/bin/env python3
# encoding: utf8
#
# Copyright © 2015 Martin Tournoij <martin@arp242.net>
# See below for full copyright
#

from __future__ import print_function
import json, sys, time, re
import mwapi

session = mwapi.Session('http://rationalwiki.org', 'Carpetbot/1.0 <martin@arp242.net>')
l = session.login('Carpetbot', 'XXX')

_noop = False
#_noop = True

# Like pprint but more readable
def pp(d):
	json.dump(d, sys.stdout, sort_keys=True, indent=4)
	print()


# Split list +l+ into +s+ sized chunks with the remained appended
def chunk(l, s):
	if len(l) <= s: return l

	nl = []
	for i in range(0, len(l) // s):
		nl += (l[i*s:i*s+s],)
	if len(l) % s != 0:
		nl += (l[i*s+s:],)
	return nl


# Check if a page has the template +tpl+ in +templates+ (being the return value
# from action=query&prop=templates). +content+ can also be given for a double
# check, as sometimes not the complete list of templates is returned with this
# (?!?! example: Talk:Michele Bachmann doesn't have {{person}}).
#
# Example usage:
# page = list(session.get(action='query',
#	titles='Main page',
#	prop='revisions|templates',
#	rvprop='content')['query']['pages'].values())[0]
#
# has = has_template('Person', page.get('templates', []), page['revisions'][0]['*']
def has_template(tpl, templates, content=None):
	if templates is not None and 'Template:{}'.format(tpl) in [ t['title'] for t in templates ]:
		return True

	# Do case-insensitive search; this could clash since SomeTpl and Sometpl are
	# not the same page, but the chance of one being a redirect to the other is
	# large, and typically we prefer a false positive over a false negative
	# (can't be arsed to check redirects here)
	if content is not None and re.search(re.escape(tpl), content, re.IGNORECASE) is not None:
		return True

	return False


# Like session.post(), and solve the turing test if we encounter one
def action_with_captcha(**params):
	if _noop: return

	ret = session.post(**params)
	ret = ret[params['action']]

	if ret.get('result') and ret['result'] == 'Failure' and ret.get('captcha'):
		q = ret['captcha']['question'].replace('−', '-')
		# We're trusting RW here...
		answer = eval(q)
		ret = session.post(
			captchaid=ret['captcha']['id'],
			captchaword=answer,
			**params
		)

		if ret.get('result') and ret['result'] == 'Failure' and ret.get('captcha'):
			print('Unable to solve CAPTCHA?')
			print("Our answer to `{}' was `{}'".format(q, answer))
			print()
			pp(ret)
			sys.exit(0)

	return ret


# Enhanced version of list=querypage&qpage=Unusedimages which gets the image
# pages with some info
def get_unused(limit=50, offset=0):
	images = session.get(action='query',
		list='querypage',
		qppage='Unusedimages',
		qplimit=limit,
		qpoffset=offset
	)

	if len(images) == 0:
		return []

	# For some reason getting a whole bunch of pages in one go doesn't work
	# correctly, after the third page "templates" is always empty...
	titles = chunk([ i['title'] for i in images['query']['querypage']['results'] ], 3)

	pages = []
	for t in titles:
		pages += session.get(action='query',
			titles='|'.join(t),
			prop='revisions|info|templates',
			rvprop='content')['query']['pages'].values()
	return pages


# Get token. You can re-use a token as many times as you want
def get_token(which='edit'):
	if which == 'deletedrevs': which = 'edit'

	#if which == 'deletedrevs':
	#	data = session.get(action='query',
	#		list='deletedrevs',
	#		drprop='token')
	#	pp(data)
	#	sys.exit(1)
		
	data = session.get(action='query',
		titles='Main Page',
		prop='info',
		intoken=which)
	try:
		token = list(data['query']['pages'].values())[0]['{}token'.format(which)]
	except:
		pp(data)
		print(sys.exc_info()[1])
		sys.exit(1)
	return token


# Find all unused images without copyright and delete them
def unused_images_wo_copyright(start=1):
	print("THIS IS BROKEN! It doesn't check if a file is linked to; only if it's embeded.")
	sys.exit(1)

	token = get_token('delete')
	limit = 50
	offset = start * limit
	for i in range(0, 999):
		print('=>', i)
		unused = get_unused(limit, offset)
		if len(unused) == 0:
			print('Our work here is done')
			sys.exit(0)

		pages = [ ]
		for p in unused:
			# Redirect page, file with no revisions, or other "special"
			# situation
			if p.get('templates') is None:
				print('Warning: {} has no templates'.format(p['title']))
				continue

			if 'Template:Nolicense' in [ t['title'] for t in p['templates'] ]:
				pages.append(p)

		if len(pages) == 0:
			offset += limit
			continue

		offset += limit - len(pages)

		for page in pages:
			print('   ', page['title'])
			#print(page)
			action_with_captcha(
				action='delete',
				pageid=page['pageid'],
				token=token,
				reason='Removing unused image with no copyright')
		time.sleep(1)


def delete_pages_from_stdin():
	token = get_token('delete')

	while True:
		line = sys.stdin.readline().strip()
		if line == '': break
		print(line)
		action_with_captcha(
			action='delete',
			title=line,
			token=token,
			reason='Removing unused image with no copyright')


# Get all images
def all(limit=100, offset=0):
	images = session.get(action='query',
		list='allimages',
		qplimit=limit,
		qpoffset=offset
	)

	if len(images) == 0:
		return []

	# For some reason getting a whole bunch of pages in one go doesn't work
	# correctly, after the third page "templates" is always empty...
	titles = chunk([ i['title'] for i in images['query']['querypage']['results'] ], 3)

	pages = []
	for t in titles:
		pages += session.get(action='query',
			titles='|'.join(t),
			prop='revisions|info|templates',
			rvprop='content')['query']['pages'].values()
	return pages


def fix_cz_screencaps():
	token = get_token('edit')

	while True:
		line = sys.stdin.readline().strip()
		if line == '': break
		print(line)

		page = list(session.get(action='query',
			titles=line,
			prop='revisions|info',
			rvprop='content')['query']['pages'].values())[0]

		url = page['revisions'][0]['*'].split('\n')[0]
		action_with_captcha(
			action='edit',
			bot=True,
			title=line,
			token=token,
			reason='Add license for Citizendium screencaps',
			text=url + '\n{{CZ screenshot}}\n[[Category:Citizendium screencaps]]')


# Make talk page title from page title:
#	  David Cameron -> Talk:David Cameron
#	  Fun:David Cameron -> Fun talk:David Cameron
#	  User:David Cameron -> User talk:David Cameron
def make_talk_title(title):
	if ':' in title:
		title = title.split(':')
		return '{} talk:{}'.format(title[0], ':'.join(title[1:]))
	else:
		return 'Talk:{}'.format(title)


def living_people():
	token = get_token('edit')

	cont = 0
	while True:
		if cont is None: break
		pages = session.get(action='query',
			list='categorymembers',
			cmtitle='Category:Living people',
			cmlimit=500,
			cmcontinue=cont
		)
		if pages.get('query-continue') is None:
			cont = None
		else:
			cont = pages['query-continue']['categorymembers']['cmcontinue']
		pages = pages['query']['categorymembers']

		if len(pages) == 0: break

		for p in pages:
			talktitle = make_talk_title(p['title'])
			print('=>', talktitle.jlust(40), end=' ')
			talk = list(session.get(action='query',
				titles=talktitle,
				prop='revisions|templates',
				rvprop='content')['query']['pages'].values())[0]
			#pp(talk)

			# No talk page yet, so create one
			if talk.get('missing') is not None:
				print('Creating talk page')
				action_with_captcha(
					action='edit',
					bot=True,
					title=talktitle,
					token=token,
					reason='Add {{person}} template for living person',
					summary='Add {{person}} template for living person',
					text='{{person}}')
				continue

			if has_template('Person', talk.get('templates', []), talk['revisions'][0]['*']):
				print('Already has the template')
				continue

			print('Adding template')
			action_with_captcha(
				action='edit',
				bot=True,
				title=talktitle,
				token=token,
				reason='Add {{person}} template for living person',
				summary='Add {{person}} template for living person',
				text='{{person}}\n' + talk['revisions'][0]['*'])


# TODO: There's got to be some bot for this already?
def double_redirect():
	pass


# List all images embeded or linked to in mainspace with either no copyright
# information, or listed as fair use.
def list_mainspace_images():
	apfrom = ''
	nolicense = []
	fairuse = []

	while True:
		if apfrom is None: break

		images = session.get(action='query',
			list='allpages',
			apnamespace=6,
			aplimit=500,
			apfrom=apfrom)

		if images.get('query-continue') is None:
			apfrom = None
		else:
			apfrom = images['query-continue']['allpages']['apfrom']
		images = images['query']['allpages']

		# Conservapedia, Template, Category, Main, Fun, Help, RationalWiki
		mainns = '100|10|14|0|106|12|4'
		for img in images:
			try:
				print(img['title'].ljust(60), end='')
				links = session.get(action='query',
					prop='templates',
					list='backlinks|imageusage|embeddedin',
					titles=img['title'], bltitle=img['title'], iutitle=img['title'], eititle=img['title'],
					blnamespace=mainns, iunamespace=mainns, einamespace=mainns,
					bllimit=1, iulimit=1, tllimit=500, eilimit=1
				)['query']

				if len(links['backlinks']) == 0 and len(links['embeddedin']) == 0 and len(links['imageusage']) == 0:
					print('not in mainspace')
					continue

				tpls = list(links['pages'].values())[0]
				if tpls.get('templates') is None:
					tpls = ['Template:Nolicense']
				else:
					tpls = [ t['title'] for t in tpls['templates'] ]

				if 'Template:Nolicense' in tpls:
					print('No license')
					nolicense.append(img['title'])
				elif 'Template:Fair use' in tpls:
					print('Fair use')
					fairuse.append(img['title'])
				else:
					print()
			except:
				print('ERROR')
				print(sys.exc_info()[1])

	print('\n')
	print('==> No license ({})'.format(len(nolicense)))
	for img in nolicense: print('http://rationalwiki.org/wiki/{}'.format(img))

	print('==> Fair use ({})'.format(len(fairuse)))
	for img in fairuse: print('http://rationalwiki.org/wiki/{}'.format(img))


def list_namespaces():
	lst = session.get(action='query',
		meta='siteinfo',
		siprop='namespaces|namespacealiases')

	for k, ns in lst['query']['namespaces'].items():
		print('  {} -> {}'.format(ns['id'], ns['*']))

	print('\n==> Namespace aliases <==')
	for ns in lst['query']['namespacealiases']:
		print('  {} -> {}'.format(ns['id'], ns['*']))


def fix_the_fucking_mess():
	import urllib
	token = get_token('deletedrevs')
	while True:
		line = sys.stdin.readline().strip()
		if line == '': break
		line = urllib.parse.unquote(line)
		print(line.ljust(80), end='')

		# backlinks
		data = session.get(action='query',
			list='backlinks',
			bltitle=line)['query']['backlinks']

		data = [ d for d in data if d['title'] != 'User:Carpetbot/Removed' ]

		if len(data) == 0:
			print()
			continue

		page = session.get(action='query',
			titles=line,
			prop='info')
		if page['query']['pages'].get('-1') is None or page['query']['pages']['-1'].get('missing') is None:
			print('Already restored')
		else:
			print('Restoring')
			session.post(action='undelete',
				title=line,
				token=token,
				reason='Page is not embeded, but is linked to')


if __name__ == '__main__':
	ops = {
		'unused_without_copyright': (
			lambda: unused_images_wo_copyright(0),
			'Delete all images that are not used (embeded) and have no copyright'
		),
		'delete_pages': (
			delete_pages_from_stdin,
			'delete all the pages from stdin, seperated by newline'
		),
		'cz_copyright': (
			fix_cz_screencaps,
			'Fix CZ captures; images to fix are from stdin'
		),
		'people': (
			living_people,
			'Add {{person}} to all talkpages for pages with [[Category: Living people]]'
		),
		'double_redirect': (
			double_redirect,
			'Fix all double redirects'
		),
		'list_mainspace_images': (
			list_mainspace_images,
			'List all mainspace images no copyright or fair use'
		),
		'list_namespaces': (
			list_namespaces,
			'List all namespaces'
		),
		'fix_the_fucking_mess': (
			fix_the_fucking_mess,
			'Fix the fucking UnusedImages mess'
		),
	}

	def show_help(err=None):
		if err: print('Error: {}\n'.format(err), file=sys.stderr)

		print('Available operations\n')

		longest = max([ len(k) for k in ops.keys() ])

		for k, v in ops.items():
			print('  {}{}  {}'.format(k.ljust(longest), v[1]))

		if err: sys.exit(1)

	if len(sys.argv) < 2: show_help('No operation given')
	op = ops.get(sys.argv[1])
	if op is None: show_help("Unknown operation `{}'".format(sys.argv[1]))
	op[0]()


# The MIT License (MIT)
#
# Copyright © 2015 Martin Tournoij
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# The software is provided "as is", without warranty of any kind, express or
# implied, including but not limited to the warranties of merchantability,
# fitness for a particular purpose and noninfringement. In no event shall the
# authors or copyright holders be liable for any claim, damages or other
# liability, whether in an action of contract, tort or otherwise, arising
# from, out of or in connection with the software or the use or other dealings
# in the software.