

# from get_cookie import Get_cookie
import requests
import logging
logging.captureWarnings(True)
import os
from lxml import etree
import re
session = requests.Session()
os.environ['NO_PROXY'] = 'stackoverflow.com'
from datetime import datetime, timedelta
from sqlalchemy import create_engine, delete
import pandas as pd
import json
from multiprocessing import Pool
from concurrent.futures import ThreadPoolExecutor
import time
from requests.exceptions import ConnectionError
import threading
from queue import Queue
import queue
from all_connect import ConnectSpider
from switch_ip import pppoe_ip
import random
from cookie_list import cookie_lists

Con = ConnectSpider()
class Amazon_details:
    def __init__(self):
        self.headers = {
    "Host": "www.amazon.com",
    "sec-ch-ua": "\"Google Chrome\";v=\"123\", \"Not:A-Brand\";v=\"8\", \"Chromium\";v=\"123\"",
    "sec-ch-ua-mobile": "?0",
    "sec-ch-ua-platform": "\"Windows\"",
    "upgrade-insecure-requests": "1",
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    "sec-fetch-site": "none",
    "sec-fetch-mode": "navigate",
    "sec-fetch-user": "?1",
    "sec-fetch-dest": "document",
    "accept-language": "zh-CN,zh;q=0.9"
}
        self.params = {"th": "1"}
        self.item_queue = Queue()
        self.MAX_RETRIES = 3

    def response_data(self, asin, cookie_json, start_id, limit, retry_count=1):
        url = f'https://www.amazon.com/dp/{asin}'
        retries_left = self.MAX_RETRIES - retry_count
        cookie = json.loads(cookie_json)
        try:
            response = session.get(url, params=self.params, cookies=cookie, headers=self.headers, timeout=30)
            html = etree.HTML(response.text)
            # 判断是否出现验证码
            verify = html.xpath(
                '//h4[contains(text(),"Type the characters you see in this image:")]/text()') if html.xpath(
                '//h4[contains(text(),"Type the characters you see in this image:")]/text()') else None
            print(f'verify:{verify}{asin}')
            if verify is None:
                # 判断正确邮编
                new_york = html.xpath(
                    '//span[@id="glow-ingress-line2" and contains(text(),"New York 10010‌")]/text()') if html.xpath(
                    '//span[@id="glow-ingress-line2" and contains(text(),"New York 10010‌")]/text()') else None
                print(f'new_york:{new_york}{asin}')
                if new_york is not None:
                    print(f'verify:{verify}{asin}')
                    print(f'new_york:{new_york}{asin}')
                    self.parse_data(response, asin, start_id, limit)
                else:
                    print(f'邮编不是10010：{new_york} {asin}')
                    del cookie_lists[0]
                    print('删除第一个cookie',cookie)
                    new_cookie = cookie_lists[0]
                    print('cookie剩余：',len(cookie_lists))
                    if new_cookie:
                        self.response_data(asin, cookie_json, start_id, limit)

            else:
                print(f'验证码页面 {asin}')
                Con.upstate_to_3(start_id, limit)
                pass

        except ConnectionError as exc:
            if retries_left > 0:
                wait_time = 2 ** retry_count
                print(f"遇到错误，将在{wait_time}秒后重试...")
                time.sleep(wait_time)
                return self.response_data(asin, cookie_json, start_id, limit)
            else:
                print(f"重试次数耗尽，放弃处理ASIN: {asin}, 错误: {exc}")
                pass

    def parse_data(self, response, asin,start_id, limit):
        try:

            html = etree.HTML(response.text)
            compressed_html_code = re.sub(r'>\s+<', '><', response.text)
            compressed_html_code = re.sub(r'\s{2,}', ' ', compressed_html_code)

            img_url = re.findall("'dp60MainImage': '(.*?)',", response.text)[0] if re.findall(
                "'dp60MainImage': '(.*?)',",
                response.text) else None
            img_num = len(html.xpath('//ul/li[@class="a-spacing-small item"]'))
            if not img_url:
                img_url = re.findall(',"main":\{"(.*?jpg)"', compressed_html_code)
                img_num = len(re.findall(',"main":\{"(.*?jpg)"', compressed_html_code))
            title = html.xpath('//div[@id="imgTagWrapperId"]/img/@alt')[0]
            title_len = len(title)
            price = html.xpath('//div[@id="apex_offerDisplay_desktop" ]//span[@class="a-offscreen"]/text()')[
                0] if html.xpath('//div[@id="apex_offerDisplay_desktop" ]//span[@class="a-offscreen"]/text()') else None

            if price and '$' in price:
                price = price.replace('$', '')

            category = html.xpath('//ul[@class="a-unordered-list a-horizontal a-size-small"]//li//text()')
            category = ''.join([item.strip() for item in category if item.strip()])
            rating = html.xpath('//a/span[@class="a-size-base a-color-base"]/text()')[0] if html.xpath(
                '//a/span[@class="a-size-base a-color-base"]/text()') else None
            total_comments = html.xpath('//a/span[@id="acrCustomerReviewText"]/text()')[0].split(' ratings')[0].replace(
                ',',
                '') if html.xpath(
                '//a/span[@id="acrCustomerReviewText"]/text()') else None
            if total_comments == '1 rating':
                total_comments = 1

            seller_type = html.xpath(
                '//div[@class="a-section a-spacing-none _p13n-desktop-sims-fbt_fbt-desktop_shipping-info-show-box__17yWM"]/div/span[@class="a-size-base a-color-secondary"]//text()')[
                0] if html.xpath(
                '//div[@class="a-section a-spacing-none _p13n-desktop-sims-fbt_fbt-desktop_shipping-info-show-box__17yWM"]/div/span[@class="a-size-base a-color-secondary"]//text()') else None
            if seller_type:
                if seller_type == 'Ships from and sold by Amazon.com.':
                    buy_box_seller_type = 1
                else:
                    sold = html.xpath('//div[@id="merchantInfoFeature_feature_div"]//a/text()')[
                        0].strip() if html.xpath(
                        '//div[@id="merchantInfoFeature_feature_div"]//a/text()') else None
                    if not sold:
                        sold = html.xpath(
                            '//div[@id="shipFromSoldByAbbreviated_feature_div"]//span[@class="a-size-small"]/text()')[
                            3].strip() if html.xpath(
                            '//div[@id="shipFromSoldByAbbreviated_feature_div"]//span[@class="a-size-small"]/text()') else None

                    ships_from = html.xpath(
                        '//div[@id="fulfillerInfoFeature_feature_div"]//span[@class="a-size-small offer-display-feature-text-message"]/text()')[
                        0].strip()
                    if not ships_from:
                        ships_from = html.xpath(
                            '//div[@id="shipFromSoldByAbbreviated_feature_div"]//span[@class="a-size-small"]/text()')[
                            1].strip() if html.xpath(
                            '//div[@id="shipFromSoldByAbbreviated_feature_div"]//span[@class="a-size-small"]/text()') else None
                    if (ships_from == 'Amazon' or ships_from == 'Amazon.com') and (
                            sold == 'Amazon' or sold == 'Amazon.com'):
                        buy_box_seller_type = 1
                    elif (ships_from == 'Amazon' or ships_from == 'Amazon.com') and (
                            sold != 'Amazon' or sold != 'Amazon.com'):
                        buy_box_seller_type = 2
                    elif (ships_from != 'Amazon' or ships_from != 'Amazon.com') and (
                            sold != 'Amazon' or sold != 'Amazon.com'):
                        buy_box_seller_type = 3
                    else:
                        buy_box_seller_type = 4
            else:
                buy_box_seller_type = 4

            if not seller_type:
                seller_type = html.xpath('//div[contains(text(),"Sold by")]/a/text()') if html.xpath(
                    '//div[contains(text(),"Sold by")]/a/text()') else None
                if seller_type:
                    buy_box_seller_type = 3
                if not seller_type:
                    seller_type = html.xpath(
                        '//div[@class="a-column a-span12 a-text-left truncate"]//text()') if html.xpath(
                        '//div[@class="a-column a-span12 a-text-left truncate"]//text()') else None
                    if seller_type:
                        seller_type = [x.strip() for x in seller_type if x.strip()]
                        ships_from = seller_type[1]
                        sold = seller_type[3]
                        if (ships_from == 'Amazon' or ships_from == 'Amazon.com') and (
                                sold == 'Amazon' or sold == 'Amazon.com'):
                            buy_box_seller_type = 1
                        elif (ships_from == 'Amazon' or ships_from == 'Amazon.com') and (
                                sold != 'Amazon' or sold != 'Amazon.com'):
                            buy_box_seller_type = 2
                        elif (ships_from != 'Amazon' or ships_from != 'Amazon.com') and (
                                sold != 'Amazon' or sold != 'Amazon.com'):
                            buy_box_seller_type = 3
                        else:
                            buy_box_seller_type = 4

            if not seller_type:
                buy_box_seller_type = 4
            # print(seller_type)
            # print(buy_box_seller_type)

            page_inventory = len(html.xpath('//div[@id="selectQuantity"]//select[@id="quantity" ]/option/text()'))
            if page_inventory > 20:
                page_inventory = 1
            elif 1 <= page_inventory <= 20:
                page_inventory = 2
            elif page_inventory < 1:
                page_inventory = 3

            volume = html.xpath(
                '//th[@class="a-color-secondary a-size-base prodDetSectionEntry" and normalize-space(.)="Product Dimensions"]/../td/text()')[
                0].strip().replace('\u200e', '') if html.xpath(
                '//th[@class="a-color-secondary a-size-base prodDetSectionEntry" and normalize-space(.)="Product Dimensions"]/../td/text()') else None
            weight = html.xpath(
                '//th[@class="a-color-secondary a-size-base prodDetSectionEntry" and normalize-space(.)="Item Weight"]/../td/text()')[
                0].strip().replace('\u200e', '') if html.xpath(
                '//th[@class="a-color-secondary a-size-base prodDetSectionEntry" and normalize-space(.)="Item Weight"]/../td/text()') else None
            if not volume and not weight:
                if html.xpath(
                        '//span[contains(text(), "Package Dimensions")]/following-sibling::span/text() | //span[contains(text(), "Product Dimensions")]/following-sibling::span/text()'):
                    volume_weight = html.xpath(
                        '//span[contains(text(), "Package Dimensions")]/following-sibling::span/text() | //span[contains(text(), "Product Dimensions")]/following-sibling::span/text()')[
                        0]
                    volume = volume_weight.split(';')[0].replace('\u200e', '')
                    weight = volume_weight.split(';')[1].replace('\u200e', '')
                else:
                    volume = None
                    weight = None

            if weight and 'ounces' in weight:
                weight = weight.split('ounces')[0].strip().replace('\u200e', '')
                weight = float(weight) / 16

            if weight and 'Grams' in weight:
                weight = weight.split('Grams')[0].strip().replace('\u200e', '')
                weight = float(weight) / 453.592

            if weight and 'Kilograms' in weight:
                weight = weight.split('Kilograms')[0].replace('\u200e', '')
                weight = float(weight) * 2.20462

            elif weight and 'pounds' in weight:
                weight = weight.split('pounds')[0].replace('\u200e', '')

            elif weight and 'Pounds' in weight:
                weight = weight.split('Pounds')[0].replace('\u200e', '')

            rank = ' '.join(html.xpath(
                '//th[@class="a-color-secondary a-size-base prodDetSectionEntry" and normalize-space(.)="Best Sellers Rank"]/../td//text()')).split(
                '#')[1].split(' ')[0] if html.xpath(
                '//th[@class="a-color-secondary a-size-base prodDetSectionEntry" and normalize-space(.)="Best Sellers Rank"]/../td//text()') else None
            if not rank:
                rank = \
                    ' '.join(html.xpath('//span[contains(text(), "Best Sellers Rank")]/../text()')).split('#')[1].split(
                        ' ')[
                        0] if html.xpath('//span[contains(text(), "Best Sellers Rank")]/../text()') else None
            if rank:
                rank = int(rank.replace(',', ''))
            launch_time = html.xpath(
                '//th[@class="a-color-secondary a-size-base prodDetSectionEntry" and normalize-space(.)="Date First Available"]/../td//text()')[
                0] if html.xpath(
                '//th[@class="a-color-secondary a-size-base prodDetSectionEntry" and normalize-space(.)="Date First Available"]/../td//text()') else None
            if not launch_time:
                launch_time = \
                html.xpath('//span[contains(text(), "Date First Available")]/following-sibling::span/text()')[
                    0] if html.xpath(
                    '//span[contains(text(), "Date First Available")]/following-sibling::span/text()') else None
            if launch_time:
                trimmed_date_str = launch_time.strip()
                date_obj = datetime.strptime(trimmed_date_str, "%B %d, %Y")
                launch_time = date_obj.strftime("%Y-%m-%d")

            img_type = []
            if img_num != 0:
                type = 1
                img_type.append(type)
            if html.xpath(
                    '//span[@class="a-size-mini a-color-secondary video-count a-text-bold a-nowrap"and contains(text(), "VIDEOS")]'):
                type = 2
                img_type.append(type)
            if html.xpath('//div[@class="aplus-v2 desktop celwidget"]//img'):
                type = 3
                img_type.append(type)

            img_type = ', '.join(map(str, img_type))
            brand = html.xpath('//a[@id="bylineInfo"]/text()')[0] if html.xpath(
                '//a[@id="bylineInfo"]/text()') else None
            node_id = html.xpath('//div[@id="wayfinding-breadcrumbs_feature_div"]//a/@href')[-1].split('node=')[
                -1] if html.xpath('//div[@id="wayfinding-breadcrumbs_feature_div"]//a/@href') else None

            buy_sales = ' '.join(
                html.xpath('//span[@id="social-proofing-faceout-title-tk_bought"]//text()')).strip() if html.xpath(
                '//span[@id="social-proofing-faceout-title-tk_bought"]//text()') else None
            current_date = datetime.now()
            date_info = f"{current_date.year}-{current_date.month:02d}"
            created_time = datetime.now()

            item = {}
            item['asin'] = asin
            item['img_url'] = img_url
            item['img_num'] = img_num
            item['title'] = title
            item['title_len'] = title_len
            item['price'] = price
            item['category'] = category
            item['rating'] = rating
            item['total_comments'] = total_comments
            item['buy_box_seller_type'] = buy_box_seller_type
            item['page_inventory'] = page_inventory
            item['volume'] = volume
            item['weight'] = weight
            item['rank'] = rank
            item['launch_time'] = launch_time
            item['img_type'] = img_type
            item['brand'] = brand
            item['node_id'] = node_id
            item['buy_sales'] = buy_sales
            item['date_info'] = date_info
            item['created_time'] = created_time
            # print('try完成')
            # print(item)
            self.item_queue.put(item)
            print(f'解析成功：{asin}')
        except Exception as e:
            print(f'解析出错: {asin}',e)


    def workers(self, task_queue,start_id, limit):
        while not task_queue.empty():
            asin, cookie_str = task_queue.get(timeout=3)
            self.response_data(asin, cookie_str,start_id, limit)
            task_queue.task_done()

    def run(self, asin_list, cookie_str,start_id, limit):
        task_queue = queue.Queue()

        for asin in asin_list:
            # print(asin)
            task_queue.put((asin, cookie_str))

        threads = []
        for th in range(1):
            t = threading.Thread(target=self.workers, args=(task_queue,start_id, limit,))
            threads.append(t)
            t.start()

        # 等待所有线程完成
        for t in threads:
            t.join()

        # 保存所有收集到的数据
        while not self.item_queue.empty():
            item = self.item_queue.get()
            try:
                Con.save_asin_detail(item)
                Con.upstate_to_21(start_id, limit)
            except Exception as e:
                print(e)
            self.item_queue.task_done()


def worker(start_id, limit):
    # asin_list = Con.us_all_syn_st_month_2024_05(start_id, limit)
    asin_list = ['B0CBSC5ZNR','B0CBSCNSQ7','B0CBSB12FP','B0CC5LCHDC','B0CBVKVD56']
    print(len(asin_list),11111111111111111111111)
    lists = list(cookie_lists)
    cookie_str = lists[0]
    if asin_list and cookie_str:
        Amazon_details().run(asin_list,cookie_str,start_id, limit)

def main():
    # pppoe_ip()
    start_time = datetime.now()
    for i in range(1):
        print(f'第{i + 1}批次')
        minid_maxid = Con.us_asin_split_test_wj(i)
        minid = minid_maxid.split('-')[0]
        maxid = minid_maxid.split('-')[1]
        print(minid, maxid)
        num_processes = 1
        batch_size = 200
        p = Pool(num_processes)
        data_range = []
        start_id = int(minid)
        while start_id <= int(maxid):
            # end_id 每次加200
            end_id = min(start_id + batch_size - 1, int(maxid))  # 防止超出最大ID,
            data_range.append((start_id, end_id - start_id + 1))  # 计算每个范围的行数
            start_id = end_id + 1  # 更新起始ID为下一个范围的起始ID
        print(data_range,22222222222222222)
        # 创建多个进程并行处理数据
        # for start_id, limit in data_range:
            # print(start_id)
        p.apply_async(worker, args=(start_id, 2))
        p.close()
        p.join()

    #     current_time = datetime.now()
    #     elapsed_time = current_time - start_time
    #     if elapsed_time >= timedelta(minutes=5):
    #         print('时间超过5分钟，切换IP')
    #         pppoe_ip()
    #         start_time = current_time  # 重置开始时间
    #
    # end_time = datetime.now()
    # duration = end_time - start_time
    # hours, remainder = divmod(duration.seconds, 3600)
    # minutes, seconds = divmod(remainder, 60)
    # print(f'项目总共耗时: {hours}小时 {minutes}分钟 {seconds}秒')


if __name__ == '__main__':
    main()



