SCP基金会中文分部

import random
import string

from fake_useragent import UserAgent
import requests
import time
import re

NUM_THREADS = 30

def getHTML(link):

    rHeader = {
            'User-Agent': f'{UserAgent().random}',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Referer': 'https://scp-wiki-cn.wikidot.com/',
        }
    requestRes = requests.get(link,headers=rHeader)

    return requestRes

def postHTML(link, data):
    header = {
        "User-Agent": f'{UserAgent().random}',
        "Content-Type": "application/x-www-form-urlencoded",
        "Referer": "https://scp-wiki-cn.wikidot.com/",
    }

    session = requests.Session()

    url = 'https://scp-wiki-cn.wikidot.com/ajax-module-connector.php'
    session.cookies.set('wikidot_token7', token, domain='scp-wiki-cn.wikidot.com', path='/')
    response = session.post(url, headers= header, data=data).json()["body"]

    return response

def getAuthurAndLink(mainRes):
    div_pattern = re.compile(r'<div class="list-pages-box">\s*<p>(.*?)\s*</div>', re.DOTALL)
    div_table = re.findall(div_pattern,mainRes)[0]

    info_pattern = re.compile(r'<a href="(/[^"]+)">([^<]+)</a>.*?<a href="(http://www.wikidot.com/user:info/[^"]+)"[^>]+>([^<]+)</a>.*?(\d+)')
    info_match = re.findall(info_pattern,div_table)

    searchDictList = []
    verifyDictList = []

    for link, title, userlink, username, score in info_match:
        searchDict = {"link": "https://scp-wiki-cn.wikidot.com"+link, "userlink": userlink}
        verifyDict = {"title": title, "username": username.replace(" ",""), "score": score}

        searchDictList.append(searchDict)
        verifyDictList.append(verifyDict)

    return searchDictList,verifyDictList

mainRes = getHTML("https://scp-wiki-cn.wikidot.com/eddy-sandbox-1")
searchDictList,verifyDictList = getAuthurAndLink(mainRes.text)

karmaList = ["none", "low", "medium", "high", "veryhigh", "guru"]

token = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(10))

globalKarma = {}

import threading
from concurrent.futures import ThreadPoolExecutor

def getKarma(userLink,expectUserName):
    if expectUserName in globalKarma.keys():
        print(f"Output User {expectUserName} of Karma {globalKarma[expectUserName]}")
        return globalKarma[expectUserName]
    else:
        requestRes = getHTML(userLink)

        pattern = re.compile(r'<h1 class="profile-title">\s*<img [^>]+>\s*([^\n]+)\s*</h1>.*?<dt>Karma level:</dt>\s*<dd>\s*([^<]+)', re.DOTALL)
        match = re.search(pattern,requestRes.text)

        foundUserName, karma = match.groups()
        foundUserName = foundUserName.replace(" ","")
        karma = karma.replace(" ","")

        if(foundUserName != expectUserName):
            print(f"{foundUserName} != {expectUserName}")

        if not(karma in karmaList):
            print(f"Given level {karma} not valid")

        time.sleep(1)
        globalKarma[expectUserName] = karma
        print(f"Record User {expectUserName} of Karma {karma}")
        return karma

def getVoterList(pageLink,expectVoterNum):
    requestsRes = getHTML(pageLink)
    pattern = re.compile(r'WIKIREQUEST.info.pageId = (.*?);', re.DOTALL)

    pageId = re.search(pattern, requestsRes.text).group(1)

    data = {
        'wikidot_token7': token,
        'pageId': pageId,
        'moduleName': 'pagerate/WhoRatedPageModule'
    }

    votersHtml = postHTML(pageLink,data)

    votersPattern = re.compile(r'<a href="(http://www.wikidot.com/user:info/[^"]+)"[^>]*>([^<]+)</a></span>\s*<span style="color:#777">\s*([+-])', re.DOTALL)

    deleteNum = votersHtml.count('deleted')

    user_ratings = votersPattern.findall(votersHtml)

    time.sleep(1)

    return user_ratings

def process_author(inputTupel):

    searchDict,verifyDict = inputTupel

    print(f'Processing {verifyDict["username"]}')

    authorDict = {
        "authorKarma": getKarma(searchDict["userlink"], verifyDict["username"]),
        "none_upvote": 0, "none_downvote": 0,
        "low_upvote": 0, "low_downvote": 0,
        "medium_upvote": 0, "medium_downvote": 0,
        "high_upvote": 0, "high_downvote": 0,
        "veryhigh_upvote": 0, "veryhigh_downvote": 0,
        "guru_upvote": 0, "guru_downvote": 0,
        "link": searchDict["link"]
    }

    votersList = getVoterList(searchDict["link"], verifyDict["score"])

    for voterLink, voterName, vote in votersList:
        voterKarma = getKarma(voterLink, voterName.replace(" ", ""))
        vote_type = "upvote" if vote == '+' else "downvote"
        karma_key = f"{voterKarma}_{vote_type}"
        if karma_key in authorDict:
            authorDict[karma_key] += 1
        else:
            print(f"Unrecognized karma level or vote type: {karma_key}")

    return authorDict

infoDictList = []
with ThreadPoolExecutor(max_workers=NUM_THREADS) as executor:
    infoDictList += list(executor.map(process_author, zip(searchDictList, verifyDictList)))