import requests, threading, time

from bs4 import BeautifulSoup

from collections import Counter

from queue import Queue

import sys, os



def get_session():

    return requests.session()



# 獲取resp

def fetch(session, url):

    headers = {

        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36',

        'refer': 'https://www.amazon.com/'

    }

    resp = session.get(url, headers=headers)

    return resp



# 獲取當(dāng)前的pages數(shù)

def get_pages(session, url):

    url = url.format(asin, 2)

    resp = fetch(session, url)

    soup = BeautifulSoup(resp.text, 'lxml')

    pages = soup.select('.a-pagination > li:nth-last-child(2) > a')[0].text

    return pages



# 獲取QA List

def get_qa(session, urlList):

    while urlList._qsize():

        url = urlList.get()

        resp = fetch(session, url)

        soup = BeautifulSoup(resp.text, 'lxml')

        group_qa = soup.select('.askTeaserQuestions > .a-fixed-left-grid.a-spacing-base > .a-fixed-left-grid-inner > .a-fixed-left-grid-col.a-col-right')

        for i in range(len(group_qa)):

            question = group_qa[i].select('.a-fixed-left-grid-col.a-col-right > a> span')[0].text.strip('\n').lstrip().rstrip()

            answer = group_qa[i].select('.a-fixed-left-grid.a-spacing-base .a-fixed-left-grid-col.a-col-right > span')[0].text

            # f.write(question + '\t' + answer + '\n')

            fq.write(question + '\n')

            fa.write(answer + '\n')



def main(url, asin):

    session = get_session()

    # 獲取當(dāng)前的頁數(shù)總數(shù)

    pages = int(get_pages(session, url))

    urlList = Queue() # 將要抓取的url存到queue中

    for i in range(pages):

        i += 1

        furl = url.format(asin, i)

        urlList.put(furl)

    # 抓取QA

    thread_list = 

    # 設(shè)置線程數(shù)

    thread_count = 15

    for _ in range(thread_count):

        t = threading.Thread(target=get_qa, args=(session, urlList))

        t.start()

        thread_list.append(t)

    for i in range(thread_count):

        thread_list[i].join()



# 已廢棄，原本想去除掉常用冠詞和人稱代詞后再統(tǒng)計(jì)詞頻的

def get_most_count(TEXT):

    for char in '\n\t.?-':

        TEXT = TEXT.replace(char, ' ')

    for char in ['the', 'I', 'to', 'you ', 'and ', 'a', 'these', 'it', 'they', 'with', 'have', 'can', 'be', 'at', 'of', 'are', 'them', 'Are']:

        TEXT = TEXT.replace(char, '')

    word_list = TEXT.split()

    print(Counter(word_list).most_common())



if __name__ == "__main__":

    os.chdir(sys.path[0])

    ### configuration ###

    # 改asin

    asin = 'B07DPJVN6P'

    us = 'https://www.amazon.com/'

    uk = 'https://www.amazon.co.uk/'

    # 改站點(diǎn)

    Marketplace = uk

    baseurl = Marketplace + 'ask/questions/asin/{}/{}/ref=ask_dp_iaw_ql_hza?isAnswered=true'

    fq = open('./questions.txt', 'w', encoding='utf-8') #存放question的文件夾

    fa = open('./answers.txt', 'w', encoding='utf-8') #存放question的文件夾

    start = time.time()

    # 開始運(yùn)行

    main(baseurl, asin)

    fq.close()

    fa.close()

    print('duration: %.2f' % (time.time() - start))

    # 計(jì)算詞頻

    # f = open('a.txt', 'r', encoding='utf-8')

    # get_most_count(f.read())

    # f.close()[/i][/i][/i]