from all_connect import ConnectSpider
import ast
import gzip
import requests
from lxml import etree
from  amazon_params import py_ja3
import re
from datetime import datetime
from collections import defaultdict
import json
import io
from multiprocessing import Pool
from multiprocessing import Queue
from multiprocessing import Process, Manager, Lock
Con = ConnectSpider()
import queue


class GetHtmlDetails(object):
    def __init__(self):
        self.cookies = {
            'session-id': '138-7972032-1402868',
            'i18n-prefs': 'USD',
            'ubid-main': '134-9153986-6135400',
            'lc-main': 'en_US',
            'session-id-time': '2082787201l',
            'x-amz-captcha-1': '1720064461331751',
            'x-amz-captcha-2': '9FJ9aWmeMgYXgPtd1k0BZA==',
            'session-token': 'f64VywAY2c6ZHVCOG21LVT/c3KP1qlgpC2AxNQlDmuh/CFYUBWjL+g1yk9Pp8o9QFJTzsGoK8rxsN+Bh8QhPZ8/YWziaAb7UMDmgohc08qZOINXortgh++N0i9v9y3Uswrh0+TRgbMvGvxswQB+x1bVR2zDVyxFK3jWGtZzmNOuHYIMJks9koNEe7KFmFAsIg3tyo49hpu/ayzx/zPDRHy4sbortQxTeAgUZMusWI6Cuh817sVzra97/rk4zpuqNz9IS5EetHRxSnr6yIpJe/NQlPMU9htTun+NsnI8ZovDVWRDLHpZ7NGAyfrEqQTM7yt99hIklwbCXUDaA9kc/hkGw4XcMPGsO',
            'csm-hit': 'tb:6G2FNM91K9DN8GNR7WJ6+s-GMD3X5FQRFPQF1K92NT9|1720057322140&t:1720057322140&adb:adblk_no',
        }
        self.headers = {
            'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
            'accept-language': 'zh-CN,zh;q=0.9',
            'cache-control': 'max-age=0',
            # 'cookie': 'session-id=138-7972032-1402868; i18n-prefs=USD; ubid-main=134-9153986-6135400; lc-main=en_US; session-id-time=2082787201l; x-amz-captcha-1=1720064461331751; x-amz-captcha-2=9FJ9aWmeMgYXgPtd1k0BZA==; session-token=f64VywAY2c6ZHVCOG21LVT/c3KP1qlgpC2AxNQlDmuh/CFYUBWjL+g1yk9Pp8o9QFJTzsGoK8rxsN+Bh8QhPZ8/YWziaAb7UMDmgohc08qZOINXortgh++N0i9v9y3Uswrh0+TRgbMvGvxswQB+x1bVR2zDVyxFK3jWGtZzmNOuHYIMJks9koNEe7KFmFAsIg3tyo49hpu/ayzx/zPDRHy4sbortQxTeAgUZMusWI6Cuh817sVzra97/rk4zpuqNz9IS5EetHRxSnr6yIpJe/NQlPMU9htTun+NsnI8ZovDVWRDLHpZ7NGAyfrEqQTM7yt99hIklwbCXUDaA9kc/hkGw4XcMPGsO; csm-hit=tb:6G2FNM91K9DN8GNR7WJ6+s-GMD3X5FQRFPQF1K92NT9|1720057322140&t:1720057322140&adb:adblk_no',
            'device-memory': '8',
            'downlink': '10',
            'dpr': '1',
            'ect': '4g',
            'rtt': '50',
            'sec-ch-device-memory': '8',
            'sec-ch-dpr': '1',
            'sec-ch-ua': '"Google Chrome";v="123", "Not:A-Brand";v="8", "Chromium";v="123"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': '"Windows"',
            'sec-ch-ua-platform-version': '"10.0.0"',
            'sec-ch-viewport-width': '1919',
            'sec-fetch-dest': 'document',
            'sec-fetch-mode': 'navigate',
            'sec-fetch-site': 'none',
            'sec-fetch-user': '?1',
            'upgrade-insecure-requests': '1',
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
            'viewport-width': '1919',
        }


    def decompress_bytes(self, input_bytes):
        if isinstance(input_bytes, str):
            input_bytes = ast.literal_eval(input_bytes)
        return gzip.decompress(input_bytes).decode('utf-8')

    def request_data(self,asin):
        sess = requests.Session()
        sess.mount('https://www.amazon.com/', py_ja3.DESAdapter())
        # 'B00F7VP5NY'
        response = sess.get(f'https://www.amazon.com/dp/{asin}', cookies=self.cookies, headers=self.headers)
        self.parse_data(response,asin)

    def parse_data(self,asin,term_html):

        # print(response)
        response_text = term_html
        # with open('get_pic.py', 'w',encoding='utf-8') as f:
        #     f.write(response_text)
        filename = f"{asin}.html"
        # with open(filename, 'w', encoding='utf-8') as f:
        #     f.write(term_html)

        html = etree.HTML(response_text)
        # with open('get_pic.py', 'r',encoding='utf-8') as f:
        #     response_text = f.read()
        # print(response)
        # html = etree.HTML(response_text)
        img_url = html.xpath('//div[@id="imgTagWrapperId"]/img/@src')[0] if html.xpath(
            '//div[@id="imgTagWrapperId"]/img/@src') else None
        title = html.xpath('//span[@id="productTitle"]/text()')[0].replace('  ', '').rstrip()
        title_len = len(title)
        price = [item.strip() for item in html.xpath(
            '//div[@class="a-section a-spacing-none aok-align-center"]/span/span[@class="a-offscreen"]/text() | //input[@id="twister-plus-price-data-price"]/@value ')
                 if item.strip()][
            0] if [item.strip() for item in html.xpath(
            '//div[@class="a-section a-spacing-none aok-align-center"]/span/span[@class="a-offscreen"]/text() | //input[@id="twister-plus-price-data-price"]/@value ')
                   if item.strip()] else None
        if not price:
            price = [item.strip() for item in html.xpath(
                '//span[@class="a-price a-text-price a-size-medium apexPriceToPay"]/span[@class="a-offscreen"]/text() | //div[@id="corePriceDisplay_desktop_feature_div"]/div/span[1]/span/span/text()')
                     if item.strip()][0] if [item.strip() for item in html.xpath(
                '//span[@class="a-price a-text-price a-size-medium apexPriceToPay"]/span[@class="a-offscreen"]/text() | //div[@id="corePriceDisplay_desktop_feature_div"]/div/span[1]/span/span/text()')
                                             if item.strip()] else None

        if price and '$' in price:
            price = price.split('$')[1]

        rat_title = html.xpath('//span[@id="acrPopover" ]/@title')[0] if html.xpath(
            '//span[@id="acrPopover" ]/@title') else None
        if rat_title:
            rating = rat_title.split(' ')[0]
        else:
            rating = None
        comment = html.xpath('//span[@id="acrCustomerReviewText" ]/text()')[0] if html.xpath(
            '//span[@id="acrCustomerReviewText" ]/text()') else None
        if comment:
            total_comments = comment.split(' ')[0].replace(',', '')
        else:
            total_comments = None

        ships = \
            [aa.strip() for aa in html.xpath('//div[@id="fulfillerInfoFeature_feature_div"]//text()') if aa.strip()][
                1] if html.xpath('//div[@id="fulfillerInfoFeature_feature_div"]//text()') else None
        if not ships:
            ships = html.xpath('//a[@id="SSOFpopoverLink_ubb"]/text()')[0].split('Fulfilled by')[-1] if html.xpath(
                '//a[@id="SSOFpopoverLink_ubb"]/text()') else None
        # sold = html.xpath('//div[@id="merchantInfoFeature_feature_div"]//a/text()')[0] if html.xpath(
        #     '//div[@id="merchantInfoFeature_feature_div"]//a/text()') else None
        # if not sold:
        sold = html.xpath('//div[@id="merchantInfoFeature_feature_div"]//span/text()')[1] if html.xpath(
            '//div[@id="merchantInfoFeature_feature_div"]//span/text()') else None
        if not sold:
            sold = html.xpath('//a[@id="sellerProfileTriggerId"]/text()')[0] if html.xpath(
                '//a[@id="sellerProfileTriggerId"]/text()') else None

        if ships and sold:
            if 'amazon' in ships.lower() and 'amazon' in sold.lower():
                buy_box_seller_type = 1
            elif 'amazon' in ships.lower() and 'amazon' not in sold.lower():
                buy_box_seller_type = 2
            elif 'amazon' not in ships.lower() and 'amazon' not in sold.lower():
                buy_box_seller_type = 3

        elif sold and not ships:
            buy_box_seller_type = 3
        else:
            buy_box_seller_type = 4

        in_stock = len(html.xpath(
            '//div[@id="quantityRelocate_feature_div"]//select[@id="quantity" ]/option/text()')) if html.xpath(
            '//div[@id="quantityRelocate_feature_div"]//select[@id="quantity" ]/option/text()') else None
        if not in_stock:
            in_stock = int(
                html.xpath('//span[@class="a-size-base a-color-price a-text-bold"]/text()')[0].split('Only')[1].split(
                    'left')[0]) if html.xpath('//span[@class="a-size-base a-color-price a-text-bold"]/text()') else None
        if in_stock and in_stock > 20:
            page_inventory = 1
        elif (in_stock and 1 <= in_stock <= 20) or html.xpath(
                '//span[@class="a-size-medium a-color-success" and contains(text(),"In stock")]/text()'):
            page_inventory = 2
        else:
            page_inventory = 3

        category = ''.join(
            [item.strip() for item in html.xpath('//ul[@class="a-unordered-list a-horizontal a-size-small"]//text()') if
             item.strip()])

        weight_strs = [item.strip() for item in html.xpath(
            '//span[contains(text(),"Item Weight")]/../span[2]/text() | //th[contains(text(),"Item Weight")]/../td//text()')
                       if item.strip()][0] if [item.strip() for item in html.xpath(
            '//span[contains(text(),"Item Weight")]/../span[2]/text() | //th[contains(text(),"Item Weight")]/../td//text()')
                                               if item.strip()] else None
        if not weight_strs:
            weight_strs = \
                [item.strip() for item in html.xpath(' //span[contains(text(),"Item Weight")]/../../td[2]//text() ') if
                 item.strip()][0] if [item.strip() for item in
                                      html.xpath(' //span[contains(text(),"Item Weight")]/../../td[2]//text() ') if
                                      item.strip()] else None

        volume_str = [item.strip().replace('\u200e', '') for item in html.xpath(
            '//span[contains(text(),"Dimensions")]/../../div[3]/span/text() | //span[contains(text(),"Product Dimensions")]/../span[2]/text() | //span[contains(text(),"Package Dimensions")]/../span[2]/text() | //th[contains(text(),"Item Package Dimensions L x W x H")]/../td//text()')][
            0] if [item.strip().replace('\u200e', '') for item in html.xpath(
            '//span[contains(text(),"Dimensions")]/../../div[3]/span/text() | //span[contains(text(),"Product Dimensions")]/../span[2]/text() | //span[contains(text(),"Package Dimensions")]/../span[2]/text() | //th[contains(text(),"Item Package Dimensions L x W x H")]/../td//text()')] else None
        if volume_str and ';' in volume_str:
            volume = volume_str.split(';')[0]
            weight_strs = volume_str.split(';')[1]
        else:
            volume = volume_str

        if weight_strs:
            weight_strs = weight_strs.lower()
            if weight_strs and ' pounds' in weight_strs:
                weight = weight_strs.split(' pounds')[0].replace('\u200e', '')
            elif weight_strs and ' ounces' in weight_strs:
                weight = round(float(weight_strs.split(' ounces')[0].replace('\u200e', '')) / 16, 3)
            elif weight_strs and ' kilograms' in weight_strs:
                weight = round(float(weight_strs.split(' kilograms')[0].replace('\u200e', '')) * 2.20462, 3)
            elif weight_strs and ' grams' in weight_strs:
                weight = round(float(weight_strs.split(' grams')[0].replace('\u200e', '')) / 453.592, 3)
            elif weight_strs and ' g' in weight_strs:
                weight = round(float(weight_strs.split(' g')[0].replace('\u200e', '')) * 0.00220462, 3)
            elif weight_strs and ' milligrams' in weight_strs:
                weight = round(float(weight_strs.split(' milligrams')[0].replace('\u200e', '')) / 453.59237, 3)
        else:
            weight = None

        rank = [item.strip().replace(',', '') for item in html.xpath(
            '//span[contains(text(),"Best Sellers Rank")]/../text() | //th[contains(text(),"Best Sellers Rank")]/../td//text()')
                if item.strip()][0] if [item.strip().replace(',', '') for item in html.xpath(
            '//span[contains(text(),"Best Sellers Rank")]/../text() | //th[contains(text(),"Best Sellers Rank")]/../td//text()')
                                        if item.strip()] else None

        if rank and ' ' in rank:
            rank = ' '.join(re.findall(r'\d+', rank))

        launch_time = [item.strip() for item in html.xpath(
            '//span[contains(text(),"Date First Available")]/../span[2]/text() | //th[contains(text(),"Date First Available")]/../td/text()  | //span[text()="Publication date"]/../../div[3]//text() | //span[contains(text(),"Release date")]/../span[2]/text() ')
                       if item.strip()][0] if [item.strip() for item in html.xpath(
            '//span[contains(text(),"Date First Available")]/../span[2]/text() | //th[contains(text(),"Date First Available")]/../td/text()  | //span[text()="Publication date"]/../../div[3]//text() | //span[contains(text(),"Release date")]/../span[2]/text() ')
                                               if item.strip()] else None
        if launch_time:
            date_obj = datetime.strptime(launch_time.replace('\u200e', ''), "%B %d, %Y")
            launch_time = date_obj.strftime("%Y-%m-%d")

        img_type = []
        if img_url:
            img_type.append(1)
        video = [item.strip() for item in html.xpath(
            '//ul//li[@class="a-spacing-small item videoThumbnail videoBlockIngress videoBlockDarkIngress a-declarative multiple-videos"] | //ul//li[@class="a-spacing-small videoCountTemplate aok-hidden"]/span/text() | //ul//li[@class="a-align-top a-spacing-small item videoBlockIngress videoBlockDarkIngress"]//img/@src')
                 if item.strip()] if [item.strip() for item in html.xpath(
            '//ul//li[@class="a-spacing-small item videoThumbnail videoBlockIngress videoBlockDarkIngress a-declarative multiple-videos"] | //ul//li[@class="a-spacing-small videoCountTemplate aok-hidden"]/span/text() | //ul//li[@class="a-align-top a-spacing-small item videoBlockIngress videoBlockDarkIngress"]//img/@src')
                                      if item.strip()] else None
        if video:
            img_type.append(2)
        pics = html.xpath(
            '//div[@class="celwidget aplus-module 3p-module-b aplus-standard"]//img/@src | //h2[contains(text(),"From the manufacturer")]/..//div[@class="apm-hero-image"]//@src | //div[@id="aplus"]//img/@src') if html.xpath(
            '//div[@class="celwidget aplus-module 3p-module-b aplus-standard"]//img/@src | //h2[contains(text(),"From the manufacturer")]/..//div[@class="apm-hero-image"]//@src | //div[@id="aplus"]//img/@src') else None
        if pics:
            img_type.append(3)
        img_type = ','.join(map(str, img_type))

        img_num = len(html.xpath(
            '//ul//li[@class="a-spacing-small item imageThumbnail a-declarative"] | //ul[@class="a-unordered-list a-nostyle a-button-list a-vertical a-spacing-top-micro gridAltImageViewLayoutIn1x7"]//li[@class="a-spacing-small item"] | //ul//li[@class="a-spacing-small item"]')) if html.xpath(
            '//ul//li[@class="a-spacing-small item imageThumbnail a-declarative"] | //ul[@class="a-unordered-list a-nostyle a-button-list a-vertical a-spacing-top-micro gridAltImageViewLayoutIn1x7"]//li[@class="a-spacing-small item"] | //ul//li[@class="a-spacing-small item"]') else None

        if img_num:
            img_num = img_num
            if video:
                img_num = img_num + 1
        elif not img_num and img_url:
            img_num = 0
        else:
            img_num = 0

        # activity_type
        activity_type = []
        coupon_type = html.xpath(
            '//i[contains(text(),"Coupon:")]/..//span[@class="a-color-success"]/label/text()')[
            0] if html.xpath(
            '//i[contains(text(),"Coupon:")]/..//span[@class="a-color-success"]/label/text()') else None
        if coupon_type and '%' in coupon_type:
            activity_type.append('1')
        elif coupon_type and '$' in coupon_type:
            activity_type.append('2')

        if '1' in activity_type and 'Apply' in coupon_type:
            one_two_val = coupon_type.split('Apply ')[1].split('% ')[0]
        elif '1' in activity_type and 'Save' in coupon_type:
            one_two_val = coupon_type.split('Save ')[1].split('%')[0]
        else:
            one_two_val = None
        if one_two_val and ' ' in one_two_val:
            one_two_val = ' '.join(re.findall(r'\d+', one_two_val))

        join_prime = html.xpath('//span[contains(text(),"Join Prime")]/../span[2]/text()')[0] if html.xpath(
            '//span[contains(text(),"Join Prime")]/../span[2]/text()') else None
        if join_prime and '%' in join_prime:
            activity_type.append('3')
            three_four_val = join_prime.split('%')[1]
        elif join_prime and '$' in join_prime:
            activity_type.append('4')
            three_four_val = join_prime.split('$')[1]
        else:
            three_four_val = None

        eight_val = \
            [item.strip() for item in html.xpath('//td[contains(text(),"You Save")]/../td[2]/span/text()') if
             item.strip()][0] if [item.strip() for item in
                                  html.xpath('//td[contains(text(),"You Save")]/../td[2]/span/text()') if
                                  item.strip()] else None
        if eight_val:
            activity_type.append('8')
            eight_val = ' '.join(re.findall(r'\d+', eight_val))

        five_six_val = None
        activity_type = ','.join(activity_type)

        one_star = html.xpath(
            '//span[contains(text(),"1 star")]/../../td[3]//text() | //a[contains(text(),"1 star")]/../../td[3]//text()')[
            0].replace('%', '') if html.xpath(
            '//span[contains(text(),"1 star")]/../../td[3]//text() | //a[contains(text(),"1 star")]/../../td[3]//text()') else None
        two_star = html.xpath(
            '//a[contains(text(),"2 star")]/../../td[3]//text() | //span[contains(text(),"2 star")]/../../td[3]//text()')[
            0].replace('%', '') if html.xpath(
            '//a[contains(text(),"2 star")]/../../td[3]//text() | //span[contains(text(),"2 star")]/../../td[3]//text()') else None
        three_star = html.xpath(
            '//span[contains(text(),"3 star")]/../../td[3]//text() | //a[contains(text(),"3 star")]/../../td[3]//text()')[
            0].replace('%', '') if html.xpath(
            '//span[contains(text(),"3 star")]/../../td[3]//text() | //a[contains(text(),"3 star")]/../../td[3]//text()') else None
        four_star = html.xpath('//a[contains(text(),"4 star")]/../../td[3]//text()')[0].replace('%', '') if html.xpath(
            '//a[contains(text(),"4 star")]/../../td[3]//text()') else None
        five_star = html.xpath('//a[contains(text(),"5 star")]/../../td[3]//text()')[0].replace('%', '') if html.xpath(
            '//a[contains(text(),"5 star")]/../../td[3]//text()') else None
        low_star = int(one_star) + int(two_star) + int(three_star)

        together_asin_list = [re.findall('pd_rd_i=(.*?)&psc', item) for item in html.xpath(
            '//div[@class="a-cardui _p13n-desktop-sims-fbt_fbt-desktop_new-thumbnail-box__36bD3"]//div//a[@class="a-link-normal _p13n-desktop-sims-fbt_fbt-desktop_image-link__17L3C"]/@href')] if html.xpath(
            '//div[@class="a-cardui _p13n-desktop-sims-fbt_fbt-desktop_new-thumbnail-box__36bD3"]//div//a[@class="a-link-normal _p13n-desktop-sims-fbt_fbt-desktop_image-link__17L3C"]/@href') else None

        if together_asin_list:
            flat_list = [item for sublist in together_asin_list for item in sublist]
            together_asin = ','.join(flat_list)

        else:
            together_asin = None

        if not together_asin_list:
            together_asin_list = [re.findall('dp/(.*?)/ref', item) for item in html.xpath(
                '//div[@class="a-section a-spacing-none _p13n-desktop-sims-fbt_fbt-desktop_link-area__1VLAZ"]/a/@href')] if html.xpath(
                '//div[@class="a-section a-spacing-none _p13n-desktop-sims-fbt_fbt-desktop_link-area__1VLAZ"]/a/@href') else None
            if together_asin_list:
                flat_list = [item for sublist in together_asin_list for item in sublist]
                together_asin = ','.join(flat_list)

            else:
                together_asin = [re.findall('product-reviews/(.*?)/ref', item) for item in html.xpath(
                    '//div[@class="a-cardui _sp-desktop-thematic-bundle_thematicBundle-desktop_new-thumbnail-box__1W9Ku _sp-desktop-thematic-bundle_thematicBundle-desktop_two-item-thumbnail-box__7kF95"]/div[position() > 1]//a[@class="a-link-normal"]/@href')] if html.xpath(
                    '//div[@class="a-cardui _sp-desktop-thematic-bundle_thematicBundle-desktop_new-thumbnail-box__1W9Ku _sp-desktop-thematic-bundle_thematicBundle-desktop_two-item-thumbnail-box__7kF95"]/div[position() > 1]//a[@class="a-link-normal"]/@href') else None
                if together_asin_list:
                    flat_list = [item for sublist in together_asin_list for item in sublist]
                    together_asin = ','.join(flat_list)

                else:
                    together_asin = None

        # print(together_asin)

        brand_str = html.xpath('//a[@id="bylineInfo"]/text()')[0] if html.xpath(
            '//a[@id="bylineInfo"]/text()') else None
        if brand_str and 'Brand: ' in brand_str:
            brand = brand_str.split('Brand: ')[1]
        elif brand_str and 'Visit the ' in brand_str and ' Store' in brand_str:
            brand = brand_str.split('Visit the ')[1].split(' Store')[0]
        else:
            brand = None

        try:
            ac_name = \
            [item.strip() for item in html.xpath('//span[@class="ac-for-text"]//text()') if item.strip()][0].split(
                'in ')[1]
        except IndexError:
            ac_name = None

        material = \
            html.xpath('//table[@class="a-normal a-spacing-micro"]//span[text() = "Material"]/../../td[2]/span/text()')[
                0] if html.xpath(
                '//table[@class="a-normal a-spacing-micro"]//span[text() = "Material"]/../../td[2]/span/text()') else None

        node_id_str = html.xpath('//ul[@class="a-unordered-list a-horizontal a-size-small"]//li[last()]//a/@href')[
            0] if html.xpath('//ul[@class="a-unordered-list a-horizontal a-size-small"]//li[last()]//a/@href') else None
        if node_id_str and 'node=' in node_id_str:
            node_id = node_id_str.split('node=')[1]
        elif node_id_str and 'fashion' in node_id_str and 'ref' in node_id_str:
            node_id = node_id_str.split('fashion')[1].split('ref')[0]
        else:
            node_id = node_id_str

        describe = '|-|'.join([item.lstrip().rstrip() for item in html.xpath(
            '//ul[@class="a-unordered-list a-vertical a-spacing-mini"]/li//text() | //ul[@class="a-unordered-list a-vertical a-spacing-small"]//li//text() | //div[@id="bookDescription_feature_div"]//li//text()')]) if html.xpath(
            '//ul[@class="a-unordered-list a-vertical a-spacing-mini"]/li//text() | //ul[@class="a-unordered-list a-vertical a-spacing-small"]//li//text() | //div[@id="bookDescription_feature_div"]//li//text()') else None
        now = datetime.now()
        # created_time = now.strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
        # updated_time = now.strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
        date_info = now.strftime('%Y-%m')
        weight_str = \
            [item.strip().replace('\u200e', '') for item in html.xpath(
                '//th[contains(text(),"Item Weight")]/../td//text() | //span[contains(text(),"Item Weight")]/../span[2]/text() | //span[contains(text(),"Product Dimensions")]/../span[2]/text()')
             if item.strip()][0] if [item.strip().replace('\u200e', '') for item in html.xpath(
                '//th[contains(text(),"Item Weight")]/../td//text() | //span[contains(text(),"Item Weight")]/../span[2]/text() | //span[contains(text(),"Product Dimensions")]/../span[2]/text()')
                                     if item.strip()] else None

        pattern_name = \
            html.xpath('//div[@id="customer_review-RYQ2ES848T5PV"]//span[@data-hook="format-strip-linkless"]/text()')[
                0].split('Pattern Name: ')[1] if html.xpath(
                '//div[@id="customer_review-RYQ2ES848T5PV"]//span[@data-hook="format-strip-linkless"]/text()') else None
        if not pattern_name:
            pattern_name = html.xpath('//div[@id="variation_pattern_name"]/div/span/text()')[0].strip() if html.xpath(
                '//div[@id="variation_pattern_name"]/div/span/text()') else None
        follow_sellers = \
            html.xpath('//span[@class="a-declarative"]/span[@class="a-color-base"]/text()')[0].split('(')[1].split(')')[
                0] if html.xpath('//span[@class="a-declarative"]/span[@class="a-color-base"]/text()') else None
        sp_related_to_term = html.xpath('//div[@id="sp_detail2"]/@data-a-carousel-options')[0] if html.xpath(
            '//div[@id="sp_detail2"]/@data-a-carousel-options') else None
        if sp_related_to_term:
            sp_1 = re.findall('"set_size":(.*?),"filteredItems"', sp_related_to_term)[0]
        else:
            sp_1 = 0
        sp_4_stars_and_above = \
            html.xpath('//h2[contains(text(),"4 stars and above")]/../../../@data-a-carousel-options')[0] if html.xpath(
                '//h2[contains(text(),"4 stars and above")]/../../../@data-a-carousel-options') else None
        if sp_4_stars_and_above:
            sp_2 = re.findall('"set_size":(.*?),"filteredItems"', sp_4_stars_and_above)[0]
        else:
            sp_2 = 0
        free_delivery = html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../@data-a-carousel-options')[
            0] if html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../@data-a-carousel-options') else None
        if free_delivery:
            sp_3 = re.findall('"set_size":(.*?),"filteredItems"', free_delivery)[0]
        else:
            sp_3 = 0
        sp_num = str(sp_1) + ',' + str(sp_2) + ',' + str(sp_3)

        product_description = '|-|'.join([item.strip() for item in html.xpath(
            '//div[@id="productDescription"]//span/text() | //h2[contains(text(),"Product Description")]/..//div[@class="celwidget aplus-module premium-module-3-four-column-images aplus-premium"]//text()')
                                          if item.strip()]) if html.xpath(
            '//div[@id="productDescription"]//span/text() | //h2[contains(text(),"Product Description")]/..//div[@class="celwidget aplus-module premium-module-3-four-column-images aplus-premium"]//text()') else None
        buy_sales = html.xpath('//span[@id="social-proofing-faceout-title-tk_bought"]/span/text()')[0].replace(' ',
                                                                                                               '') if html.xpath(
            '//span[@id="social-proofing-faceout-title-tk_bought"]/span/text()') else None
        package_quantity = html.xpath('//label[contains(text(),"Item Package Quantity:")]/../span/text()')[
            0].strip() if html.xpath('//label[contains(text(),"Item Package Quantity:")]/../span/text()') else None

        product_info_dict = html.xpath('//table[@class="a-normal a-spacing-micro"]//span/text()') if html.xpath(
            '//table[@class="a-normal a-spacing-micro"]//span/text()') else None
        if product_info_dict:
            product_json = json.dumps(
                {product_info_dict[i]: product_info_dict[i + 1] for i in range(0, len(product_info_dict), 2)})
        else:
            product_json = product_info_dict

        detail_json = [re.sub(r'\s+', ' ', item.strip().replace('\u200e', '').replace('\u200f', '')) for item in
                       html.xpath(
                           '//h1[contains(text(),"Technical Details")]/../../..//tr//text() | //h1[contains(text(),"Additional Information")]/..//td[@class="a-size-base prodDetAttrValue"]/..//text() | //h2[contains(text(),"Product details")]/../div[@id="detailBullets_feature_div"]/ul//text()')
                       if item.strip()] if html.xpath(
            '//h1[contains(text(),"Technical Details")]/../../..//tr//text() | //h1[contains(text(),"Additional Information")]/..//td[@class="a-size-base prodDetAttrValue"]/..//text() | //h2[contains(text(),"Product details")]/../div[@id="detailBullets_feature_div"]/ul//text()') else None

        if detail_json:
            product_detail_json = json.dumps(
                {detail_json[i]: detail_json[i + 1] for i in range(0, len(detail_json), 2)})
        else:
            product_detail_json = detail_json

        review_ai_text = html.xpath('//h3[contains(text(),"Customers say")]/../../p/span/text()')[0] if html.xpath(
            '//h3[contains(text(),"Customers say")]/../../p/span/text()') else None

        # sp_initial_seen_asins_json
        seen_asins = [item.split('_')[-1] for item in html.xpath(
            '//h2[contains(text(),"Products related to this item")]/../../../div[@class="a-row"]//ol/li/div/@id')] if html.xpath(
            '//h2[contains(text(),"Products related to this item")]/../../../div[@class="a-row"]//ol/li/div/@id') else None
        seen_asins_title = html.xpath(
            '//h2[contains(text(),"Products related to this item")]/../../../div[@class="a-row"]//ol/li/div/a/@title') if html.xpath(
            '//h2[contains(text(),"Products related to this item")]/../../../div[@class="a-row"]//ol/li/div/a/@title') else None
        seen_asins_src = html.xpath(
            '//h2[contains(text(),"Products related to this item")]/../../../div[@class="a-row"]//ol/li/div//img/@src') if html.xpath(
            '//h2[contains(text(),"Products related to this item")]/../../../div[@class="a-row"]//ol/li/div//img/@src') else None
        seen_asins_total_comments = [item.xpath('./div//span[@class="a-color-link"]/text()')[0] if item.xpath(
            './div//span[@class="a-color-link"]/text()') else None for item in html.xpath(
            '//h2[contains(text(),"Products related to this item")]/../../../div[@class="a-row"]//ol/li')]
        seen_asins_price = [item.xpath('./div//span[@class="a-offscreen"]/text()')[0] if item.xpath(
            './div//span[@class="a-offscreen"]/text()') else None for item in html.xpath(
            '//h2[contains(text(),"Products related to this item")]/../../../div[@class="a-row"]//ol/li')]
        if seen_asins and seen_asins_title and seen_asins_src and seen_asins_total_comments and seen_asins_price:
            initial_combined = zip(seen_asins, seen_asins_title, seen_asins_src, seen_asins_total_comments,
                                   seen_asins_price)
            sp_initial_seen_asins_json = json.dumps([{"seen_asins": seen_asins, "seen_asins_title": seen_asins_title,
                                                      "seen_asins_src": seen_asins_src,
                                                      "seen_asins_total_comments": seen_asins_total_comments,
                                                      "seen_asins_price": seen_asins_price} for
                                                     seen_asins, seen_asins_title, seen_asins_src, seen_asins_total_comments, seen_asins_price
                                                     in initial_combined])
        else:
            sp_initial_seen_asins_json = None

        # sp_4stars_initial_seen_asins_json
        seen_asins = [item.split('_')[-1] for item in html.xpath(
            '//h2[contains(text(),"4 stars and above")]/../../../div[@class="a-row"]//ol/li/div/@id')] if html.xpath(
            '//h2[contains(text(),"4 stars and above")]/../../../div[@class="a-row"]//ol/li/div/@id') else None
        seen_asins_title = html.xpath(
            '//h2[contains(text(),"4 stars and above")]/../../../div[@class="a-row"]//ol/li/div/a/@title') if html.xpath(
            '//h2[contains(text(),"4 stars and above")]/../../../div[@class="a-row"]//ol/li/div/a/@title') else None
        seen_asins_src = html.xpath(
            '//h2[contains(text(),"4 stars and above")]/../../../div[@class="a-row"]//ol/li/div//img/@src') if html.xpath(
            '//h2[contains(text(),"4 stars and above")]/../../../div[@class="a-row"]//ol/li/div//img/@src') else None
        seen_asins_total_comments = [item.xpath('./div//span[@class="a-color-link"]/text()')[0] if item.xpath(
            './div//span[@class="a-color-link"]/text()') else None for item in html.xpath(
            '//h2[contains(text(),"4 stars and above")]/../../../div[@class="a-row"]//ol/li')]
        seen_asins_price = [item.xpath('./div//span[@class="a-offscreen"]/text()')[0] if item.xpath(
            './div//span[@class="a-offscreen"]/text()') else None for item in html.xpath(
            '//h2[contains(text(),"4 stars and above")]/../../../div[@class="a-row"]//ol/li')]

        if seen_asins and seen_asins_title and seen_asins_src and seen_asins_total_comments and seen_asins_price:
            delivery_initial_combined = zip(seen_asins, seen_asins_title, seen_asins_src, seen_asins_total_comments,
                                            seen_asins_price)
            sp_4stars_initial_seen_asins_json = json.dumps([{"seen_asins": seen_asins,
                                                             "seen_asins_title": seen_asins_title,
                                                             "seen_asins_src": seen_asins_src,
                                                             "seen_asins_total_comments": seen_asins_total_comments,
                                                             "seen_asins_price": seen_asins_price} for
                                                            seen_asins, seen_asins_title, seen_asins_src, seen_asins_total_comments, seen_asins_price
                                                            in delivery_initial_combined])
        else:
            sp_4stars_initial_seen_asins_json = None

        # sp_delivery_initial_seen_asins_json
        seen_asins = [item.split('_')[-1] for item in html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../div[@class="a-row"]//ol/li/div/@id')] if html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../div[@class="a-row"]//ol/li/div/@id') else None
        seen_asins_title = html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../div[@class="a-row"]//ol/li/div/a/@title') if html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../div[@class="a-row"]//ol/li/div/a/@title') else None
        seen_asins_src = html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../div[@class="a-row"]//ol/li/div/a/img/@src') if html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../div[@class="a-row"]//ol/li/div/a/img/@src') else None
        seen_asins_total_comments = [item.xpath('.//span[@class="a-color-link"]/text()')[0] if item.xpath(
            './/span[@class="a-color-link"]/text()') else None for item in html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../div[@class="a-row"]//ol/li')]
        seen_asins_price = [item.xpath('.//span[@class="a-offscreen"]/text()')[0] if item.xpath(
            './/span[@class="a-offscreen"]/text()') else None for item in html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../div[@class="a-row"]//ol/li')]

        if seen_asins and seen_asins_title and seen_asins_src and seen_asins_total_comments and seen_asins_price:
            delivery_initial_combined = zip(seen_asins, seen_asins_title, seen_asins_src, seen_asins_total_comments,
                                            seen_asins_price)
            sp_delivery_initial_seen_asins_json = json.dumps([{"seen_asins": seen_asins,
                                                               "seen_asins_title": seen_asins_title,
                                                               "seen_asins_src": seen_asins_src,
                                                               "seen_asins_total_comments": seen_asins_total_comments,
                                                               "seen_asins_price": seen_asins_price} for
                                                              seen_asins, seen_asins_title, seen_asins_src, seen_asins_total_comments, seen_asins_price
                                                              in delivery_initial_combined])
        else:
            sp_delivery_initial_seen_asins_json = None

        # compare_similar_asin_json
        compare_asin = [item.split('-')[-1] for item in html.xpath(
            '//h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@tabindex="0"]/div/@id | //h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@class="a-section a-spacing-mini"]/a/div/@id')] if html.xpath(
            '//h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@tabindex="0"]/div/@id | //h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@class="a-section a-spacing-mini"]/a/div/@id') else None
        compare_asin_src = html.xpath(
            '//h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@tabindex="0"]//img/@src | //h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@class="a-section a-spacing-mini"]//img/@src') if html.xpath(
            '//h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@tabindex="0"]//img/@src | //h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@class="a-section a-spacing-mini"]//img/@src') else None
        compare_asin_title = html.xpath(
            '//h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@tabindex="0"]//img/@alt | //h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@class="a-section a-spacing-mini"]//img/@alt') if html.xpath(
            '//h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@tabindex="0"]//img/@alt | //h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@class="a-section a-spacing-mini"]//img/@alt') else None
        # print(compare_asin)
        # print(compare_asin_src)
        # print(compare_asin_title)
        # print(11111111111111111111111111111)
        if compare_asin and compare_asin_src and compare_asin_title:
            compare_combined = zip(compare_asin, compare_asin_src, compare_asin_title)
            compare_similar_asin_json = json.dumps([{"compare_asin": compare_asin, "compare_asin_src": compare_asin_src,
                                                     "compare_asin_title": compare_asin_title} for
                                                    compare_asin, compare_asin_src, compare_asin_title in
                                                    compare_combined])
        else:
            compare_similar_asin_json = None

        toge_asin = [item for sublist in [re.findall('/dp/(.*?)/ref', item) for item in html.xpath(
            '//h2[contains(text(),"Frequently bought together")]/../../..//a[@class="a-link-normal _p13n-desktop-sims-fbt_fbt-desktop_image-link__17L3C"]/@href | //h2[contains(text(),"Buy it with")]/../../..//a[@class="a-link-normal _p13n-desktop-sims-fbt_fbt-desktop_image-link__17L3C"]/@href')]
                     for item in sublist] if html.xpath(
            '//h2[contains(text(),"Frequently bought together")]/../../..//a[@class="a-link-normal _p13n-desktop-sims-fbt_fbt-desktop_image-link__17L3C"]/@href | //h2[contains(text(),"Buy it with")]/../../..//a[@class="a-link-normal _p13n-desktop-sims-fbt_fbt-desktop_image-link__17L3C"]/@href') else None
        if not toge_asin:
            toge_asin = [re.findall('product-reviews/(.*?)/ref=', item) for item in html.xpath(
                '//div[@class="a-cardui _sp-desktop-thematic-bundle_thematicBundle-desktop_new-thumbnail-box__1W9Ku _sp-desktop-thematic-bundle_thematicBundle-desktop_two-item-thumbnail-box__7kF95"]/div[position() > 1]//a[@class="a-link-normal"]/@href')] if html.xpath(
                '//div[@class="a-cardui _sp-desktop-thematic-bundle_thematicBundle-desktop_new-thumbnail-box__1W9Ku _sp-desktop-thematic-bundle_thematicBundle-desktop_two-item-thumbnail-box__7kF95"]/div[position() > 1]//a[@class="a-link-normal"]/@href') else None

        toge_title = html.xpath(
            '//div[@class="a-cardui _p13n-desktop-sims-fbt_fbt-desktop_new-detail-faceout-box___WyNy"][position() > 1]//span[@class="a-size-base"]/text() | //div[@class="a-cardui _sp-desktop-thematic-bundle_thematicBundle-desktop_new-thumbnail-box__1W9Ku _sp-desktop-thematic-bundle_thematicBundle-desktop_two-item-thumbnail-box__7kF95"]/div[position() > 1]//span[@class="a-size-base"]/text()')
        toge_price = html.xpath(
            '//div[@class="a-cardui _p13n-desktop-sims-fbt_fbt-desktop_new-detail-faceout-box___WyNy"][position() > 1]//span[@class="a-offscreen"]/text() | //div[@class="a-cardui _sp-desktop-thematic-bundle_thematicBundle-desktop_new-thumbnail-box__1W9Ku _sp-desktop-thematic-bundle_thematicBundle-desktop_two-item-thumbnail-box__7kF95"]/div[position() > 1]//span[@class="a-offscreen"]/text()')
        if toge_asin and toge_title and toge_price:
            together_asin_combined = zip(toge_asin, toge_title, toge_price)
            together_asin_json = json.dumps(
                [{"together_asin": asin, "together_asin_title": title, "together_asin_price": price} for
                 asin, title, price in together_asin_combined])
        else:
            together_asin_json = None
        reviews_json = html.xpath(
            '//div[@id="customerReviewsAttribute_feature_div"]//div[@class="a-section a-spacing-none"]//span[@class="a-size-base a-color-base"]/text() |//div[@id="customerReviewsAttribute_feature_div"]//div[@class="a-section a-spacing-none"]//span[@class="a-icon-alt"]/text()') if html.xpath(
            '//div[@id="customerReviewsAttribute_feature_div"]//div[@class="a-section a-spacing-none"]//span[@class="a-size-base a-color-base"]/text() |//div[@id="customerReviewsAttribute_feature_div"]//div[@class="a-section a-spacing-none"]//span[@class="a-icon-alt"]/text()') else None
        if reviews_json:
            customer_reviews_json = json.dumps(
                [{reviews_json[i].rstrip().lstrip(): reviews_json[i + 1]} for i in range(0, len(reviews_json), 2)])
        else:
            customer_reviews_json = None

        # lob_asin_json
        lob_asin = html.xpath('//div[@id="pba-lob-carousel-row"]//ol/li//a/@href') if html.xpath(
            '//div[@id="pba-lob-carousel-row"]//ol/li//a/@href') else None
        lob_asin_img = html.xpath('//div[@id="pba-lob-carousel-row"]//ol/li//img/@src') if html.xpath(
            '//div[@id="pba-lob-carousel-row"]//ol/li//img/@src') else None
        lob_asin_title = html.xpath(
            '//div[@id="pba-lob-carousel-row"]//ol/li//div[@class="a-section a-spacing-none"]/span/span/text()')
        lob_asin_price = [
            item.xpath('.//span[@class="a-price pba-lob-bundle-buy-price"]/span[@class="a-offscreen"]/text()')[
                0] if item.xpath(
                './/span[@class="a-price pba-lob-bundle-buy-price"]/span[@class="a-offscreen"]/text()') else None for
            item in html.xpath('//div[@id="pba-lob-carousel-row"]//ol/li')]

        if lob_asin and lob_asin_img and lob_asin_title and lob_asin_price:
            lob_combined = zip(lob_asin, lob_asin_img, lob_asin_title, lob_asin_price)
            lob_asin_json = json.dumps([{"lob_asin": lob_asin, "lob_asin_img": lob_asin_img,
                                         "lob_asin_title": lob_asin_title, "lob_asin_price": lob_asin_price,
                                         "lob_asin_total_comment": None} for
                                        lob_asin, lob_asin_img, lob_asin_title, lob_asin_price in lob_combined])
        else:
            lob_asin_json = None

        # review_label_json
        review_lable_title = html.xpath(
            '//div[@class="a-section a-spacing-small a-spacing-top-small _cr-product-insights_style_aspect-symbol-list__24amT"]/a/text() | //div[@data-hook="cr-insights-widget-aspects"]//button//span[@class="a-size-base"]/text()') if html.xpath(
            '//div[@class="a-section a-spacing-small a-spacing-top-small _cr-product-insights_style_aspect-symbol-list__24amT"]/a/text() | //div[@data-hook="cr-insights-widget-aspects"]//button//span[@class="a-size-base"]/text()') else None
        if review_lable_title:
            review_label_json = {}
            for lable_title in review_lable_title:
                reviews = '|-|'.join([item.replace('Read more', '') for item in
                                      html.xpath(f'//div[@data-aspect= "{lable_title}"]//text()')])
                review_label_json[lable_title] = reviews
            json.dumps(review_label_json)
        else:
            review_label_json = None

        # views = html.xpath('//div[@id="ivImagesTab"]') if html.xpath('//div[@id="ivImagesTab"]') else None
        views = html.xpath('//div[@id="main-image-container"]/script[@type="text/javascript"]') if html.xpath(
            '//div[@id="main-image-container"]/script[@type="text/javascript"]') else None

        if views:
            image_view = 1
        else:
            image_view = 0

        fulfilled_by = html.xpath('//a[@id="SSOFpopoverLink_ubb"]/text()')[0].split('Fulfilled by')[-1] if html.xpath(
            '//a[@id="SSOFpopoverLink_ubb"]/text()') else None
        seller_id = html.xpath('//input[@id="merchantID"]/@value')[0] if html.xpath(
            '//input[@id="merchantID"]/@value') else None
        seller_json = json.dumps(
            [{"seller_id": seller_id, "ship_from": ships, "sold_by": sold, "fulfilled_by": fulfilled_by}])

        # variat_num = len(html.xpath(
        #     '//ul[@class="a-unordered-list a-nostyle a-button-list a-declarative a-button-toggle-group a-horizontal a-spacing-top-micro swatches swatchesSquare imageSwatches"]//li')) if html.xpath(
        #     '//ul[@class="a-unordered-list a-nostyle a-button-list a-declarative a-button-toggle-group a-horizontal a-spacing-top-micro swatches swatchesSquare imageSwatches"]//li') else 0

        curr_asin = html.xpath('//link[@rel="canonical"]/@href')[0].split('dp/')[1]
        if curr_asin == asin:
            current_asin = None
        else:
            current_asin = curr_asin

        img_src = html.xpath(
            '//ul//li[@class="a-spacing-small item imageThumbnail a-declarative"]//img/@src | //ul//li[@class="a-spacing-small item"]//img/@src') if html.xpath(
            '//ul//li[@class="a-spacing-small item imageThumbnail a-declarative"]//img/@src | //ul//li[@class="a-spacing-small item"]//img/@src') else None
        video_src = html.xpath(
            '//ul/li[@class="a-align-top a-spacing-small item videoBlockIngress videoBlockDarkIngress"]//img/@src') if html.xpath(
            '//ul/li[@class="a-align-top a-spacing-small item videoBlockIngress videoBlockDarkIngress"]//img/@src') else None
        pics_src = html.xpath(
            '//h2[contains(text(),"Product Description")]/..//div[@class="aplus-module-wrapper apm-spacing apm-floatnone apm-fixed-width"]//img/@data-src') if html.xpath(
            '//h2[contains(text(),"Product Description")]/..//div[@class="aplus-module-wrapper apm-spacing apm-floatnone apm-fixed-width"]//img/@data-src') else None
        if img_src and video_src and pics_src:
            img_list = json.dumps([
                [asin, src, idx, type]
                for type, srcs in enumerate([img_src, video_src, pics_src], start=1)
                if srcs is not None
                for idx, src in enumerate(srcs, start=1)])
        else:
            img_list = None

        parent_asin = re.findall('"parent_asin":"(.*?)",', response_text)[0] if re.findall('"parent_asin":"(.*?)",',
                                                                                           response_text) else None
        if not parent_asin:
            parent_asin = html.xpath('//link[@rel="canonical"]/@href')[0].split('/')[-1] if html.xpath(
                '//link[@rel="canonical"]/@href') else None

        min_match_asin_json = None

        asin_detail = re.findall('"dimensionValuesDisplayData" : \{(.*?)},', response_text) if re.findall(
            '"dimensionValuesDisplayData" : \{(.*?)},', response_text) else None
        if asin_detail:
            # 使用正则表达式提取 asins 和 details
            asins = re.findall(r'"(\w+)":\[', str(asin_detail))
            variat_num = len(asins)
            try:
                details = [json.loads(item) for item in re.findall('":(\[".*?"])', str(asin_detail))]
                variat_list = [[asin] + detail for asin, detail in zip(asins, details)]
            except:
                color_size = re.findall('":(\[".*?"])', str(asin_detail))
                details = [item.replace('\\\'', '').replace('\\\\"', '') for item in color_size]
                variat_list = [[asin] + ast.literal_eval(detail) for asin, detail in zip(asins, details)]

            # 向每个子列表添加额外的值
            for detail in variat_list:
                detail.extend([parent_asin, 1, None, None])
            variat_list = json.dumps(variat_list)
        else:
            variat_list = None
            variat_num = 0

        item = {}
        item['asin'] = asin
        item['img_url'] = img_url
        item['title'] = title
        item['title_len'] = title_len
        item['price'] = price
        item['rating'] = rating
        item['total_comments'] = total_comments
        item['buy_box_seller_type'] = buy_box_seller_type
        item['page_inventory'] = page_inventory
        item['category'] = category
        item['volume'] = volume
        item['weight'] = weight
        item['rank'] = rank
        item['launch_time'] = launch_time
        item['category_state'] = None
        item['img_num'] = img_num
        item['img_type'] = img_type
        item['activity_type'] = activity_type
        item['one_two_val'] = one_two_val
        item['three_four_val'] = three_four_val
        item['five_six_val'] = five_six_val
        item['eight_val'] = eight_val
        item['qa_num'] = None
        item['one_star'] = one_star
        item['two_star'] = two_star
        item['three_star'] = three_star
        item['four_star'] = four_star
        item['five_star'] = five_star
        item['low_star'] = low_star
        item['together_asin'] = together_asin
        item['brand'] = brand
        item['ac_name'] = ac_name
        item['material'] = material
        item['node_id'] = node_id
        item['data_type'] = None
        item['sp_num'] = sp_num
        item['describe'] = describe
        item['date_info'] = date_info
        item['weight_str'] = weight_str
        item['package_quantity'] = package_quantity
        item['pattern_name'] = pattern_name
        item['spider_int'] = None
        item['follow_sellers'] = follow_sellers
        item['product_description'] = product_description
        item['buy_sales'] = buy_sales
        item['image_view'] = image_view
        item['product_json'] = product_json
        item['product_detail_json'] = product_detail_json
        item['review_ai_text'] = review_ai_text
        item['review_label_json'] = review_label_json
        item['lob_asin_json'] = lob_asin_json
        item['sp_initial_seen_asins_json'] = sp_initial_seen_asins_json
        item['sp_4stars_initial_seen_asins_json'] = sp_4stars_initial_seen_asins_json
        item['sp_delivery_initial_seen_asins_json'] = sp_delivery_initial_seen_asins_json
        item['compare_similar_asin_json'] = compare_similar_asin_json
        item['customer_reviews_json'] = customer_reviews_json
        item['together_asin_json'] = together_asin_json
        item['min_match_asin_json'] = min_match_asin_json
        item['seller_json'] = seller_json
        item['variat_num'] = variat_num
        item['current_asin'] = current_asin
        item['img_list'] = img_list
        item['variat_list'] = variat_list
        item['parent_asin'] = parent_asin

        print(item)
        Con.save_us_asin_detail_2024_wj(item)

    def execution_parse(self, asin_html_queue):
        while True:
            if asin_html_queue.empty() == False:
                asin_html = asin_html_queue.get()
                keywords_scraper_url_list = asin_html.split('||-||')
                asin_org = keywords_scraper_url_list[0]
                html_org = keywords_scraper_url_list[1]
                print(f'当前解析：{asin_org}')
                html_str = json.loads(html_org)
                html = self.decompress_bytes(html_str)
                # with open('get_pic.py', 'w', encoding='utf-8') as f:
                #     f.write(html)
                asin_html_queue.task_done()
                self.parse_data(asin_org,html)

            else:
                break

    def run(self):
        asin_html_pairs = Con.get_asin_html_06()
        asin_html_queue = queue.Queue()
        if asin_html_pairs:
            for asin_html in asin_html_pairs:
                asin_html_queue.put(asin_html)
            self.execution_parse(asin_html_queue)

        else:
            print("没有符合条件的数据需要处理")
            return



if __name__ == '__main__':
    GetHtmlDetails().run()



