User:Capturebot2/capturebot2.py
From RationalWiki
#!/usr/bin/env python # -*- coding: utf-8 -*- import wikipedia, config, pagegenerators, query, upload from datetime import datetime from datetime import timedelta from datetime import date import time, re, os import locale sleeptime = 60 # seconds sleeping = False; checking = True; lastrevs = { 'console' : 0, 'options' : 0, } #script = "" # since javascript is a security risk, this script is hardcoded now script = "if (el = document.getElementById('content')) { el.style.marginLeft = '0px';el.style.zIndex = '100'; } if (el = document.getElementById('footer')) { el.style.marginLeft = '0px';el.style.zIndex = '100'; } if (el = document.getElementById('column-one')) { el.style.position = 'relative';el.style.left = '-200px';el.style.top = '-1000px';el.style.width = '0';el.style.height = '0'; } if (el = document.getElementById('p-logo')) { el.style.display = 'none'; } if (el = document.getElementById('p-cactions')) { el.style.left = '200px';el.style.top = '1016px';el.style.zIndex = '102'; } if (el = document.getElementById('p-personal')) { el.style.display = 'none'; }" #script2 = "toc = document.getElementById(\"toc\"); toc.style.display = \"none\";\n" + "spans = getElementsByClassName(document.getElementById(\"bodyContent\"),\"span\",\"mw-headline\"); \n" + "section = location.href.match(/#([^&?]*)/);\n" + "if (section.length > 1) section = section[1];\n" + "i=0; while (i < spans.length && spans[i].parentNode.previousSibling.id != section) ++i;\n" + "if (i < spans.length) {\n" + " start0 = document.getElementById(\"bodyContent\").firstChild;\n" + " cur = start0;\n" + " hiding = true;\n" + " key1 = spans[i].parentNode.previousSibling;\n" + " if (i+1 < spans.length) { key2 = spans[i+1].parentNode.previousSibling; } else { key2 = null; }\n" + " do { if (key1 == cur) { hiding = false; } else if (key2 != null && key2 == cur) {hiding = true; } if (cur.style != undefined && hiding) cur.style.display = \"none\"; } while ((cur = cur.nextSibling) != null);\n" + "}" script2 = '''toc = document.getElementById("toc"); toc.style.display = ""; spans = getElementsByClassName(document.getElementById("bodyContent"),"span","mw-headline"); section = location.href.match(/#([^&?]*)/); if (section.length > 1) section = section[1]; i=0; while (i < spans.length && spans[i].parentNode.previousSibling.id != section) ++i; headinglevel = parseInt(spans[i].parentNode.tagName.substring(1)); //alert(spans[i].innerHTML); if (i < spans.length) { start0 = document.getElementById("bodyContent").firstChild; cur = start0; hiding = true; key1 = spans[i].parentNode.previousSibling; j = i+1; key2 = null; while (j < spans.length) { headinglevel2 = parseInt(spans[j].parentNode.tagName.substring(1)); if (headinglevel2 <= headinglevel) { key2 = spans[j].parentNode.previousSibling; break; } j++; } toremove = new Array(); do { if (key1 == cur) { hiding = false; } else if (key2 != null && key2 == cur) { hiding = true; } if (hiding) { toremove.push(cur); } } while ((cur = cur.nextSibling) != null); for (var i=0; i<toremove.length; ++i) { toremove[i].parentNode.removeChild(toremove[i]); } }''' def recentChanges(site = None, delay=0, block=70): ''' Return a pagegenerator containing all the pages edited in a certain timespan. The delay is the amount of minutes to wait and the block is the timespan to return pages in. Should probably copied to somewhere else ''' result = [] dateformat ="%Y-%m-%dT%H:%M:%SZ" rcstart = datetime.utcnow() + timedelta(minutes=-delay-block) rcend = datetime.utcnow() + timedelta(minutes=-delay) params = { 'action' :'query', 'list' :'recentchanges', 'rcstart' :rcstart.strftime(dateformat), 'rcend' :rcend.strftime(dateformat), 'rcdir' :'newer', 'rcprop' :'title', 'rcshow' :'!bot', 'rclimit' :'5000', 'rctype' :'edit|new', } data = query.GetData(params, site, useAPI = True, encodeTitle = False) try: for item in data['query']['recentchanges']: result.append(item['title']) except IndexError: wikipedia.output(u'API Error, nothing found in the APIs') except KeyError: wikipedia.output(u'API Error, nothing found in the APIs') return pagegenerators.PagesFromTitlesGenerator(result, site) def execConsole(pagetext, site): modified = False output = '' summary = 'By your command' lines = pagetext.splitlines() global sleeping for line in lines: if line != '': if line[0] != '*': modified = True output += '*' + line + '\n' if re.match('\s*status\s*',line,re.I) != None: if not sleeping: output += '**Online' + '\n' else: output += '**Sleeping' + '\n' elif re.match('\s*sleep\s*',line,re.I) != None: if not sleeping: output += '**Deactivating' + '\n' sleeping = True else: output += '**Already inactive' + '\n' elif re.match('\s*wake\s*',line,re.I) != None: if sleeping: output += '**Activating' + '\n' sleeping = False else: output += '**Already active' + '\n' elif re.match('\s*capture\s*(.*)',line,re.I) != None: m = re.match('\s*capture\s*(.*)',line,re.I) output += '**Analyzing ' + m.group(1) + '\n' cpage = wikipedia.Page(site,m.group(1)) if cpage.exists(): check(site.getUrl(site.get_address(m.group(1))),site) output += '**Done' + '\n' else: output += '**Error: Page does not exist ' + '\n' else: output += '**Unknown command' + '\n' summary = 'Error' else: output += line + '\n' return modified, output, summary def execOptions(pagetext): summary = 'Updating configuration' output = '' modified = False lines = pagetext.splitlines() newlr = {} for line in lines: if line != '': if line[0] != '*': valid = False m = re.match('\s*watch\s*=\s*(.*)',line,re.I) if m != None: valid = True output += line + '\n' items = m.group(1).split('&') for item in items: item = item.strip() newlr[item] = 0 m = re.match('\s*script\s*=\s*(.*)',line,re.I) if m != None: valid = False output += '*This option is a security risk, and has been disabled. You will have to modify the script variable in capturebot2.py instead.\n' # security risk # global script # script = m.group(1) m = re.match('\s*match\s*=\s*(.*)',line,re.I) if m != None: valid = True global checking if m.group(1) == 'true': output += line + '\n' checking = True elif m.group(1) == 'false': output += line + '\n' checking = False else: valid = False if not valid: modified = True output += '*' + line + '\n' output += '**Unknown command' + '\n' summary = 'Error' else: output += line + '\n' global lastrevs # copy over those that stayed for key, value in newlr.iteritems(): if key in lastrevs: newlr[key] = lastrevs[key] newlr['console'] = lastrevs['console'] newlr['options'] = lastrevs['options'] lastrevs = newlr return modified, output, summary def unescape(s): s = s.replace("<", "<") s = s.replace(">", ">") s = s.replace(""", '"') s = s.replace("'", "'") s = s.replace("&", "&") return s def check(pagetext, site): regexp = re.compile('<a[^>]*\s+href="([^"]*)"[^>]*>(?:[^<]|<[^/]|</[^a]|</a[^>])*</a><span\s*class="wigocapture"><sup><a\s*href="[^"]*"\s*class="new"\s*title="(?:Image:|File:)([^&"]*).png', re.UNICODE) imglist = {} for m in regexp.finditer(pagetext): # wikipedia.output(m.group(2) + ' : ' + m.group(1)) imglist[unescape(m.group(2))] = unescape(m.group(1)) if len(imglist) == 0: return f = open('capturebotuploads','w') usesectionscript = False for img, link in imglist.iteritems(): if link.find('&diff=') >= 0: link += '&diffonly=1' if link.find('#') >= 0: usesectionscript = True f.write('\'' + img.encode('utf8').encode('string-escape') + '\'' + ' : ' + '\'' + link.encode('utf8').encode('string-escape') + '\'' + '\n') f.close() f = open('capturebotscript','w') global script f.write(script) if usesectionscript: f.write("\n") f.write(script2) f.close() wikipedia.output('capturing') # remove -x to use existing X server or when running on Windows os.system("./webkit2png.py -l capturebotuploads --geometry 1024 1 -x -t 90 --script capturebotscript") os.remove('capturebotuploads') os.remove('capturebotscript') wikipedia.output('uploading') for img, link in imglist.iteritems(): imgname = img + '.png' if os.path.exists(imgname): wikipedia.output('uploading' + imgname + ' for ' + link) if link.find('conservapedia.com') != -1: desc = link + '\n{{CP screenshot}}\n[[Category:' + date.today().strftime('%B %Y') + ' Conservapedia screencaps]]' # Check for the existence of monthly category and create it catpage = wikipedia.Page(site,'Category:' + date.today().strftime('%B %Y') + ' Conservapedia screencaps') if not catpage.exists(): catpage.put('[[Category:Conservapedia screencaps]]', comment='Creating monthly category', minorEdit=False ) else: desc = link + '\n{{subst:Nolicense/subst}}\n[[Category:Screencaps]]' bot = upload.UploadRobot(url = imgname, description = desc, targetSite = site, verifyDescription = False, keepFilename = True, ignoreWarning = True) try: targetFilename = bot.run() except IOError: wikipedia.output('upload failed!') page = wikipedia.Page(site,'File:' + imgname) try: page.put(link+'\n[[Category:Failed screencaps]]','upload failed') except PageNotSaved: wikipedia.output("Saving page failed") os.remove(imgname) wikipedia.output(imgname) else: wikipedia.output(imgname + ' failed') return def main(): # locale gets set to user's locale (by pywikipedia?), which affects month names in categories and could cause exceptions with foreign characters # reset it to C (aka english) locale.setlocale(locale.LC_ALL, 'C') site = wikipedia.getSite('en','rationalwiki') # site = wikipedia.getSite('en','testwiki') site.forceLogin() username = site.loggedInAs() consolepage = 'User:'+username+'/console' optionspage = 'User:'+username+'/configuration' global lastrevs # process the options and console pages page = wikipedia.Page(site,optionspage) try: lastrevs['options'] = page.latestRevision() r, t, s = execOptions(page.get(False,True)) if r: try: page.put(t,s,False,False) except PageNotSaved: wikipedia.output("Saving page failed") except wikipedia.NoPage: # TODO: logging pass page = wikipedia.Page(site,consolepage) try: lastrevs['console'] = page.latestRevision() r, t, s = execConsole(page.get(False,True),site) if r: try: page.put(t,s,False,False) except PageNotSaved: wikipedia.output("Saving page failed") except wikipedia.NoPage: # TODO: logging pass # now process all pages on watch for pagename, lr in lastrevs.iteritems(): page = wikipedia.Page(site,pagename) try: if page.latestRevision() > lastrevs[page.title()]: lastrevs[page.title()] = page.latestRevision() wikipedia.output('analyzing ' + page.title()) check(site.getUrl(site.get_address(page.title())),site) except wikipedia.NoPage: # TODO: logging pass except KeyError: pass while True: generator = recentChanges(site) global sleeping global checking pregenerator = pagegenerators.PreloadingGenerator(generator) for page in pregenerator: if page.title() == consolepage: try: if page.latestRevision() > lastrevs['console']: lastrevs['console'] = page.latestRevision() r, t, s = execConsole(page.get(False,True),site) if r: try: page.put(t,s,False,False) except PageNotSaved: wikipedia.output("Saving page failed") except wikipedia.NoPage: # TODO: logging pass elif page.title() == optionspage: try: if page.latestRevision() > lastrevs['options']: lastrevs['options'] = page.latestRevision() r, t, s = execOptions(page.get(False,True)) if r: try: page.put(t,s,False,False) except PageNotSaved: wikipedia.output("Saving page failed") except wikipedia.NoPage: # TODO: logging pass else: if not sleeping: if checking: if page.title() in lastrevs: try: if page.latestRevision() > lastrevs[page.title()]: lastrevs[page.title()] = page.latestRevision() if page.exists(): wikipedia.output('analyzing ' + page.title()) check(site.getUrl(site.get_address(page.title())),site) except wikipedia.NoPage: # TODO: logging pass except KeyError: pass else: if page.exists(): wikipedia.output('analyzing ' + page.title()) check(site.getUrl(site.get_address(page.title())),site) wikipedia.output('sleeping') global sleeptime time.sleep(sleeptime) if __name__ == '__main__': try: main() finally: wikipedia.stopme()