from all_connect import ConnectSpider
import ast
import gzip
import requests
from lxml import etree
from  amazon_params import py_ja3
from datetime import datetime
import re
from datetime import datetime
from collections import defaultdict
import json
import io
from multiprocessing import Pool
from multiprocessing import Queue
from multiprocessing import Process, Manager, Lock
Con = ConnectSpider()
import queue
from parse_html_parse import HtmlDetailsParser
htmldetailsparser = HtmlDetailsParser()

class GetHtmlDetails(object):
    def __init__(self):
        self.cookies = {
            'session-id': '138-7972032-1402868',
            'i18n-prefs': 'USD',
            'ubid-main': '134-9153986-6135400',
            'lc-main': 'en_US',
            'session-id-time': '2082787201l',
            'x-amz-captcha-1': '1720064461331751',
            'x-amz-captcha-2': '9FJ9aWmeMgYXgPtd1k0BZA==',
            'session-token': 'f64VywAY2c6ZHVCOG21LVT/c3KP1qlgpC2AxNQlDmuh/CFYUBWjL+g1yk9Pp8o9QFJTzsGoK8rxsN+Bh8QhPZ8/YWziaAb7UMDmgohc08qZOINXortgh++N0i9v9y3Uswrh0+TRgbMvGvxswQB+x1bVR2zDVyxFK3jWGtZzmNOuHYIMJks9koNEe7KFmFAsIg3tyo49hpu/ayzx/zPDRHy4sbortQxTeAgUZMusWI6Cuh817sVzra97/rk4zpuqNz9IS5EetHRxSnr6yIpJe/NQlPMU9htTun+NsnI8ZovDVWRDLHpZ7NGAyfrEqQTM7yt99hIklwbCXUDaA9kc/hkGw4XcMPGsO',
            'csm-hit': 'tb:6G2FNM91K9DN8GNR7WJ6+s-GMD3X5FQRFPQF1K92NT9|1720057322140&t:1720057322140&adb:adblk_no',
        }
        self.headers = {
            'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
            'accept-language': 'zh-CN,zh;q=0.9',
            'cache-control': 'max-age=0',
            # 'cookie': 'session-id=138-7972032-1402868; i18n-prefs=USD; ubid-main=134-9153986-6135400; lc-main=en_US; session-id-time=2082787201l; x-amz-captcha-1=1720064461331751; x-amz-captcha-2=9FJ9aWmeMgYXgPtd1k0BZA==; session-token=f64VywAY2c6ZHVCOG21LVT/c3KP1qlgpC2AxNQlDmuh/CFYUBWjL+g1yk9Pp8o9QFJTzsGoK8rxsN+Bh8QhPZ8/YWziaAb7UMDmgohc08qZOINXortgh++N0i9v9y3Uswrh0+TRgbMvGvxswQB+x1bVR2zDVyxFK3jWGtZzmNOuHYIMJks9koNEe7KFmFAsIg3tyo49hpu/ayzx/zPDRHy4sbortQxTeAgUZMusWI6Cuh817sVzra97/rk4zpuqNz9IS5EetHRxSnr6yIpJe/NQlPMU9htTun+NsnI8ZovDVWRDLHpZ7NGAyfrEqQTM7yt99hIklwbCXUDaA9kc/hkGw4XcMPGsO; csm-hit=tb:6G2FNM91K9DN8GNR7WJ6+s-GMD3X5FQRFPQF1K92NT9|1720057322140&t:1720057322140&adb:adblk_no',
            'device-memory': '8',
            'downlink': '10',
            'dpr': '1',
            'ect': '4g',
            'rtt': '50',
            'sec-ch-device-memory': '8',
            'sec-ch-dpr': '1',
            'sec-ch-ua': '"Google Chrome";v="123", "Not:A-Brand";v="8", "Chromium";v="123"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': '"Windows"',
            'sec-ch-ua-platform-version': '"10.0.0"',
            'sec-ch-viewport-width': '1919',
            'sec-fetch-dest': 'document',
            'sec-fetch-mode': 'navigate',
            'sec-fetch-site': 'none',
            'sec-fetch-user': '?1',
            'upgrade-insecure-requests': '1',
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
            'viewport-width': '1919',
        }


    def decompress_bytes(self, input_bytes):
        if isinstance(input_bytes, str):
            input_bytes = ast.literal_eval(input_bytes)
        return gzip.decompress(input_bytes).decode('utf-8')

    def request_data(self,asin):
        sess = requests.Session()
        sess.mount('https://www.amazon.com/', py_ja3.DESAdapter())
        # 'B00F7VP5NY'
        response = sess.get(f'https://www.amazon.com/dp/{asin}', cookies=self.cookies, headers=self.headers)
        # self.parse_data(asin,response)

    def parse_data(self,asin,response_text):

        # print(response)
        # with open('get_pic.py', 'w',encoding='utf-8') as f:
        #     f.write(response_text)
        html = etree.HTML(response_text)
        # response_text = response.text

        # print(response)
        # html = etree.HTML(response_text)

        img_url = html.xpath('//div[@id="imgTagWrapperId"]/img/@src')[0] if html.xpath(
            '//div[@id="imgTagWrapperId"]/img/@src') else None
        title = html.xpath('//span[@id="productTitle"]/text()')[0].replace('  ', '').rstrip()
        title_len = len(title)
        price = [item.strip().replace(',','') for item in html.xpath(
            '//div[@class="a-section a-spacing-none aok-align-center"]/span/span[@class="a-offscreen"]/text() | //input[@id="twister-plus-price-data-price"]/@value ') if item.strip()][
            0] if [item.strip() for item in html.xpath(
            '//div[@class="a-section a-spacing-none aok-align-center"]/span/span[@class="a-offscreen"]/text() | //input[@id="twister-plus-price-data-price"]/@value ') if item.strip()] else None
        if not price:
            price =[item.strip().replace(',','') for item in html.xpath( '//span[@class="a-price a-text-price a-size-medium apexPriceToPay"]/span[@class="a-offscreen"]/text() | //span[@class="a-price aok-align-center reinventPricePriceToPayMargin priceToPay"]/span/text()') if item.strip()][0] if [item.strip() for item in html.xpath( '//span[@class="a-price a-text-price a-size-medium apexPriceToPay"]/span[@class="a-offscreen"]/text() | //span[@class="a-price aok-align-center reinventPricePriceToPayMargin priceToPay"]/span/text()') if item.strip()]else None


        if price and '$' in price:
            price = price.split('$')[1]

        rat_title = html.xpath('//span[@id="acrPopover" ]/@title')[0] if html.xpath(
            '//span[@id="acrPopover" ]/@title') else None
        if rat_title:
            rating = rat_title.split(' ')[0]
        else:
            rating = None
        comment = html.xpath('//span[@id="acrCustomerReviewText" ]/text()')[0] if html.xpath(
            '//span[@id="acrCustomerReviewText" ]/text()') else None
        if comment:
            total_comments = comment.split(' ')[0].replace(',', '')
        else:
            total_comments = None

        ships = [aa.strip() for aa in html.xpath('//div[@id="fulfillerInfoFeature_feature_div"]//text()') if aa.strip()][
                1] if [aa.strip() for aa in html.xpath('//div[@id="fulfillerInfoFeature_feature_div"]//text()') if aa.strip()] else None
        if not ships:
            ships = html.xpath('//a[@id="SSOFpopoverLink_ubb"]/text()')[0].split('Fulfilled by')[-1] if html.xpath(
                '//a[@id="SSOFpopoverLink_ubb"]/text()') else None
        # sold = html.xpath('//div[@id="merchantInfoFeature_feature_div"]//a/text()')[0] if html.xpath(
        #     '//div[@id="merchantInfoFeature_feature_div"]//a/text()') else None
        # if not sold:
        sold = html.xpath('//div[@id="merchantInfoFeature_feature_div"]//span/text()')[1] if html.xpath(
            '//div[@id="merchantInfoFeature_feature_div"]//span/text()') else None
        if not sold:
            sold = html.xpath('//a[@id="sellerProfileTriggerId"]/text()')[0] if html.xpath(
                '//a[@id="sellerProfileTriggerId"]/text()') else None

        if ships and sold:
            if 'amazon' in ships.lower() and 'amazon' in sold.lower():
                buy_box_seller_type = 1
            elif 'amazon' in ships.lower() and 'amazon' not in sold.lower():
                buy_box_seller_type = 2
            elif 'amazon' not in ships.lower() and 'amazon' not in sold.lower():
                buy_box_seller_type = 3

        else:
            buy_box_seller_type = 4

        in_stock = len(html.xpath(
            '//div[@id="quantityRelocate_feature_div"]//select[@id="quantity" ]/option/text()')) if html.xpath(
            '//div[@id="quantityRelocate_feature_div"]//select[@id="quantity" ]/option/text()') else None
        if not in_stock:
            in_stock = html.xpath('//span[@class="a-size-base a-color-price a-text-bold"]/text()')[0] if html.xpath('//span[@class="a-size-base a-color-price a-text-bold"]/text()') else None
            if in_stock and 'usually ships' in in_stock.lower():
                in_stock = None
            if in_stock:
                in_stock = int(in_stock.split('Only')[1].split('left')[0])
        if in_stock and in_stock > 20:
            page_inventory = 1
        elif (in_stock and 1 <= in_stock <= 20) or html.xpath(
                '//span[@class="a-size-medium a-color-success" and contains(text(),"In stock")]/text()'):
            page_inventory = 2
        else:
            page_inventory = 3

        category = ''.join(
            [item.strip() for item in html.xpath('//ul[@class="a-unordered-list a-horizontal a-size-small"]//text()') if
             item.strip()])

        weight_strs = [item.strip() for item in html.xpath(
            '//div[@class="a-row a-expander-container a-expander-inline-container"]//span[contains(text(),"Item Weight")]/../span[2]/text() | //div[@class="a-row a-expander-container a-expander-inline-container"]//th[contains(text(),"Item Weight")]/../td//text()')
                       if item.strip()][0] if [item.strip() for item in html.xpath(
            '//div[@class="a-row a-expander-container a-expander-inline-container"]//span[contains(text(),"Item Weight")]/../span[2]/text() | //div[@class="a-row a-expander-container a-expander-inline-container"]//th[contains(text(),"Item Weight")]/../td//text()')if item.strip()] else None


        volume_str = [item.strip().replace('\u200e', '') for item in html.xpath(
            '//span[contains(text(),"Dimensions")]/../../div[3]/span/text() | //span[contains(text(),"Product Dimensions")]/../span[2]/text() | //span[contains(text(),"Package Dimensions")]/../span[2]/text() | //th[contains(text(),"Item Package Dimensions L x W x H")]/../td//text()')][0] if [item.strip().replace('\u200e', '') for item in html.xpath(
            '//span[contains(text(),"Dimensions")]/../../div[3]/span/text() | //span[contains(text(),"Product Dimensions")]/../span[2]/text() | //span[contains(text(),"Package Dimensions")]/../span[2]/text() | //th[contains(text(),"Item Package Dimensions L x W x H")]/../td//text()')] else None
        if volume_str and ';' in volume_str:
            volume = volume_str.split(';')[0]
            weight_strs = volume_str.split(';')[1]
        else:
            volume = volume_str
        if weight_strs:
            weight_strs = weight_strs.lower()
            if weight_strs and ' hundredths pounds' in weight_strs:
                weight = round(float(weight_strs.split(' hundredths pounds')[0].replace('\u200e', '')) / 100, 3)
            elif weight_strs and ' pounds' in weight_strs:
                weight = weight_strs.split(' pounds')[0].replace('\u200e', '')
            elif weight_strs and ' ounces' in weight_strs:
                weight = round(float(weight_strs.split(' ounces')[0].replace('\u200e', '')) / 16, 3)
            elif weight_strs and ' kilograms' in weight_strs:
                weight = round(float(weight_strs.split(' kilograms')[0].replace('\u200e', '')) * 2.20462, 3)
            elif weight_strs and ' grams' in weight_strs:
                weight = round(float(weight_strs.split(' grams')[0].replace('\u200e', '')) / 453.592, 3)
            elif weight_strs and ' g' in weight_strs:
                weight = round(float(weight_strs.split(' g')[0].replace('\u200e', '')) * 0.00220462, 3)
            elif weight_strs and ' milligrams' in weight_strs:
                weight = round(float(weight_strs.split(' milligrams')[0].replace('\u200e', '')) / 453.59237, 3)

        else:
            weight = None


        rank = [item.strip().replace(',', '') for item in html.xpath(
            '//span[contains(text(),"Best Sellers Rank")]/../text() | //th[contains(text(),"Best Sellers Rank")]/../td//text()')if item.strip()][0] if [item.strip().replace(',', '') for item in html.xpath(
            '//span[contains(text(),"Best Sellers Rank")]/../text() | //th[contains(text(),"Best Sellers Rank")]/../td//text()')if item.strip()] else None

        if rank and ' ' in rank:
            rank = ' '.join(re.findall(r'\d+', rank))

        launch_time = [item.strip() for item in html.xpath('//span[contains(text(),"Date First Available")]/../span[2]/text() | //th[contains(text(),"Date First Available")]/../td/text()  | //span[text()="Publication date"]/../../div[3]//text() | //span[contains(text(),"Release date")]/../span[2]/text() ') if item.strip()][0] if [item.strip() for item in html.xpath('//span[contains(text(),"Date First Available")]/../span[2]/text() | //th[contains(text(),"Date First Available")]/../td/text()  | //span[text()="Publication date"]/../../div[3]//text() | //span[contains(text(),"Release date")]/../span[2]/text() ') if item.strip()] else None
        if launch_time:
            date_obj = datetime.strptime(launch_time.replace('\u200e', ''), "%B %d, %Y")
            launch_time = date_obj.strftime("%Y-%m-%d")

        img_type = []
        if img_url:
            img_type.append(1)
        video = [item.strip() for item in html.xpath(
            '//ul//li[@class="a-spacing-small item videoThumbnail videoBlockIngress videoBlockDarkIngress a-declarative multiple-videos"] | //ul//li[@class="a-spacing-small videoCountTemplate aok-hidden"]/span/text() | //ul//li[@class="a-align-top a-spacing-small item videoBlockIngress videoBlockDarkIngress"]//img/@src')
                 if item.strip()] if [item.strip() for item in html.xpath(
            '//ul//li[@class="a-spacing-small item videoThumbnail videoBlockIngress videoBlockDarkIngress a-declarative multiple-videos"] | //ul//li[@class="a-spacing-small videoCountTemplate aok-hidden"]/span/text() | //ul//li[@class="a-align-top a-spacing-small item videoBlockIngress videoBlockDarkIngress"]//img/@src')
                 if item.strip()] else None
        if video:
            img_type.append(2)
        pics = html.xpath(
            '//div[@class="celwidget aplus-module 3p-module-b aplus-standard"]//img/@src | //h2[contains(text(),"From the manufacturer")]/..//div[@class="apm-hero-image"]//@src | //div[@id="aplus"]//img/@src') if html.xpath(
            '//div[@class="celwidget aplus-module 3p-module-b aplus-standard"]//img/@src | //h2[contains(text(),"From the manufacturer")]/..//div[@class="apm-hero-image"]//@src | //div[@id="aplus"]//img/@src') else None
        if pics:
            img_type.append(3)
        img_type = ','.join(map(str, img_type))

        img_num = len(html.xpath(
            '//ul//li[@class="a-spacing-small item imageThumbnail a-declarative"] | //ul[@class="a-unordered-list a-nostyle a-button-list a-vertical a-spacing-top-micro gridAltImageViewLayoutIn1x7"]//li[@class="a-spacing-small item"] | //ul//li[@class="a-spacing-small item"]')) if html.xpath(
            '//ul//li[@class="a-spacing-small item imageThumbnail a-declarative"] | //ul[@class="a-unordered-list a-nostyle a-button-list a-vertical a-spacing-top-micro gridAltImageViewLayoutIn1x7"]//li[@class="a-spacing-small item"] | //ul//li[@class="a-spacing-small item"]') else None

        if img_num:
            img_num = img_num
            if video:
                img_num = img_num + 1
        elif not img_num and img_url:
            img_num = 0
        else:
            img_num = 0

        # activity_type
        activity_type = []
        coupon_type = html.xpath(
            '//i[contains(text(),"Coupon:")]/..//span[@class="a-color-success"]/label/text()')[
            0] if html.xpath(
            '//i[contains(text(),"Coupon:")]/..//span[@class="a-color-success"]/label/text()') else None
        if coupon_type and '%' in coupon_type:
            activity_type.append('1')
        elif coupon_type and '$' in coupon_type:
            activity_type.append('2')

        if '1' in activity_type and 'Apply' in coupon_type:
            one_two_val = coupon_type.split('Apply ')[1].split('% ')[0]
        elif '1' in activity_type and 'Save' in coupon_type:
            one_two_val = coupon_type.split('Save ')[1].split('%')[0]
        else:
            one_two_val = None
        if one_two_val and ' ' in one_two_val:
            one_two_val = ' '.join(re.findall(r'\d+', one_two_val))

        join_prime = html.xpath('//span[contains(text(),"Join Prime")]/../span[2]/text()')[0] if html.xpath(
            '//span[contains(text(),"Join Prime")]/../span[2]/text()') else None
        if join_prime and '%' in join_prime:
            activity_type.append('3')
            three_four_val = join_prime.split('%')[1]
        elif join_prime and '$' in join_prime:
            activity_type.append('4')
            three_four_val = join_prime.split('$')[1]
        else:
            three_four_val = None

        eight_val = \
            [item.strip() for item in html.xpath('//td[contains(text(),"You Save")]/../td[2]/span/text()') if
             item.strip()][0] if [item.strip() for item in html.xpath('//td[contains(text(),"You Save")]/../td[2]/span/text()') if item.strip()] else None
        if eight_val:
            activity_type.append('8')
            eight_val = ' '.join(re.findall(r'\d+', eight_val))

        five_six_val = None
        activity_type = ','.join(activity_type)

        one_star = html.xpath(
            '//a[contains(@aria-label,"percent of reviews have 1 stars")]//text() | //span[contains(@aria-label,"percent of reviews have 1 stars")]//text()')[
            0].replace('%', '') if html.xpath(
            '//a[contains(@aria-label,"percent of reviews have 1 stars")]//text() | //span[contains(@aria-label,"percent of reviews have 1 stars")]//text()') else None
        two_star = html.xpath(
            '//a[contains(@aria-label,"percent of reviews have 2 stars")]//text() | //span[contains(@aria-label,"percent of reviews have 2 stars")]//text()')[
            0].replace('%', '') if html.xpath(
            '//a[contains(@aria-label,"percent of reviews have 2 stars")]//text() | //span[contains(@aria-label,"percent of reviews have 2 stars")]//text()') else None

        three_star = html.xpath(
            '//a[contains(@aria-label,"percent of reviews have 3 stars")]//text() | //span[contains(@aria-label,"percent of reviews have 3 stars")]//text()')[
            0].replace('%', '') if html.xpath(
            '//a[contains(@aria-label,"percent of reviews have 3 stars")]//text() | //span[contains(@aria-label,"percent of reviews have 3 stars")]//text()') else None

        four_star = html.xpath('//a[contains(@aria-label,"percent of reviews have 4 stars")]//text() | //span[contains(@aria-label,"percent of reviews have 4 stars")]//text()')[0].replace('%', '') if html.xpath(
            '//a[contains(@aria-label,"percent of reviews have 4 stars")]//text() | //span[contains(@aria-label,"percent of reviews have 4 stars")]//text()') else None
        five_star = html.xpath('//a[contains(@aria-label,"percent of reviews have 5 stars")]//text() | //span[contains(@aria-label,"percent of reviews have 5 stars")]//text()')[0].replace('%', '') if html.xpath(
            '//a[contains(@aria-label,"percent of reviews have 5 stars")]//text() | //span[contains(@aria-label,"percent of reviews have 5 stars")]//text()') else None
        low_star = int(one_star) + int(two_star) + int(three_star)

        together_asin_list = [re.findall('pd_rd_i=(.*?)&psc', item) for item in html.xpath(
            '//div[@class="a-cardui _p13n-desktop-sims-fbt_fbt-desktop_new-thumbnail-box__36bD3"]//div//a[@class="a-link-normal _p13n-desktop-sims-fbt_fbt-desktop_image-link__17L3C"]/@href')] if html.xpath(
            '//div[@class="a-cardui _p13n-desktop-sims-fbt_fbt-desktop_new-thumbnail-box__36bD3"]//div//a[@class="a-link-normal _p13n-desktop-sims-fbt_fbt-desktop_image-link__17L3C"]/@href') else None

        if together_asin_list:
            flat_list = [item for sublist in together_asin_list for item in sublist]
            together_asin = ','.join(flat_list)

        else:
            together_asin = None

        if not together_asin_list:
            together_asin_list = [re.findall('dp/(.*?)/ref', item) for item in html.xpath(
                '//div[@class="a-section a-spacing-none _p13n-desktop-sims-fbt_fbt-desktop_link-area__1VLAZ"]/a/@href')] if html.xpath(
                '//div[@class="a-section a-spacing-none _p13n-desktop-sims-fbt_fbt-desktop_link-area__1VLAZ"]/a/@href') else None
            if together_asin_list:
                flat_list = [item for sublist in together_asin_list for item in sublist]
                together_asin = ','.join(flat_list)

            else:
                together_asin = [re.findall('product-reviews/(.*?)/ref', item) for item in html.xpath(
                    '//div[@class="a-cardui _sp-desktop-thematic-bundle_thematicBundle-desktop_new-thumbnail-box__1W9Ku _sp-desktop-thematic-bundle_thematicBundle-desktop_two-item-thumbnail-box__7kF95"]/div[position() > 1]//a[@class="a-link-normal"]/@href')] if html.xpath(
                    '//div[@class="a-cardui _sp-desktop-thematic-bundle_thematicBundle-desktop_new-thumbnail-box__1W9Ku _sp-desktop-thematic-bundle_thematicBundle-desktop_two-item-thumbnail-box__7kF95"]/div[position() > 1]//a[@class="a-link-normal"]/@href') else None
                if together_asin_list:
                    flat_list = [item for sublist in together_asin_list for item in sublist]
                    together_asin = ','.join(flat_list)

                else:
                    together_asin = None

        # print(together_asin)

        brand_str = html.xpath('//a[@id="bylineInfo"]/text()')[0] if html.xpath(
            '//a[@id="bylineInfo"]/text()') else None
        if brand_str and 'Brand: ' in brand_str:
            brand = brand_str.split('Brand: ')[1]
        elif brand_str and 'Visit the ' in brand_str and ' Store' in brand_str:
            brand = brand_str.split('Visit the ')[1].split(' Store')[0]
        else:
            brand = None

        try:
            ac_name = [item.strip() for item in html.xpath('//span[@class="ac-for-text"]//text()') if item.strip()][0].split(
                    'in ')[1]
        except IndexError:
            ac_name = None

        material = \
            html.xpath('//table[@class="a-normal a-spacing-micro"]//span[text() = "Material"]/../../td[2]/span/text()')[
                0] if html.xpath(
                '//table[@class="a-normal a-spacing-micro"]//span[text() = "Material"]/../../td[2]/span/text()') else None

        node_id_str = html.xpath('//ul[@class="a-unordered-list a-horizontal a-size-small"]//li[last()]//a/@href')[
            0] if html.xpath('//ul[@class="a-unordered-list a-horizontal a-size-small"]//li[last()]//a/@href') else None
        if node_id_str and 'node=' in node_id_str:
            node_id = node_id_str.split('node=')[1]
        elif node_id_str and 'fashion' in node_id_str and 'ref' in node_id_str:
            node_id = node_id_str.split('fashion')[1].split('ref')[0]
        else:
            node_id = node_id_str

        describe = '|-|'.join([item.lstrip().rstrip() for item in html.xpath(
            '//ul[@class="a-unordered-list a-vertical a-spacing-mini"]/li//text() | //ul[@class="a-unordered-list a-vertical a-spacing-small"]//li//text() | //div[@id="bookDescription_feature_div"]//li//text()')]) if html.xpath(
            '//ul[@class="a-unordered-list a-vertical a-spacing-mini"]/li//text() | //ul[@class="a-unordered-list a-vertical a-spacing-small"]//li//text() | //div[@id="bookDescription_feature_div"]//li//text()') else None
        now = datetime.now()
        # created_time = now.strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
        # updated_time = now.strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
        date_info = now.strftime('%Y-%m')
        weight_str = \
            [item.strip().replace('\u200e', '') for item in html.xpath(
                '//th[contains(text(),"Item Weight")]/../td//text() | //span[contains(text(),"Item Weight")]/../span[2]/text() | //span[contains(text(),"Product Dimensions")]/../span[2]/text()')
             if item.strip()][0] if [item.strip().replace('\u200e', '') for item in html.xpath(
                '//th[contains(text(),"Item Weight")]/../td//text() | //span[contains(text(),"Item Weight")]/../span[2]/text() | //span[contains(text(),"Product Dimensions")]/../span[2]/text()')
             if item.strip()] else None

        pattern_name = \
            html.xpath('//div[@id="customer_review-RYQ2ES848T5PV"]//span[@data-hook="format-strip-linkless"]/text()')[
                0].split('Pattern Name: ')[1] if html.xpath(
                '//div[@id="customer_review-RYQ2ES848T5PV"]//span[@data-hook="format-strip-linkless"]/text()') else None
        if not pattern_name:
            pattern_name = html.xpath('//div[@id="variation_pattern_name"]/div/span/text()')[0].strip() if html.xpath(
                '//div[@id="variation_pattern_name"]/div/span/text()') else None

        try:
            follow_sellers = html.xpath('//span[@class="a-declarative"]/span[@class="a-color-base"]/text()')[0].split('(')[1].split(')')[0]
        except:
            follow_sellers = None
        sp_related_to_term = html.xpath('//div[@id="sp_detail2"]/@data-a-carousel-options')[0] if html.xpath(
            '//div[@id="sp_detail2"]/@data-a-carousel-options') else None
        if sp_related_to_term:
            sp_1 = re.findall('"set_size":(.*?),"filteredItems"', sp_related_to_term)[0]
        else:
            sp_1 = 0
        sp_4_stars_and_above = \
            html.xpath('//h2[contains(text(),"4 stars and above")]/../../../@data-a-carousel-options')[0] if html.xpath(
                '//h2[contains(text(),"4 stars and above")]/../../../@data-a-carousel-options') else None
        if sp_4_stars_and_above:
            sp_2 = re.findall('"set_size":(.*?),"filteredItems"', sp_4_stars_and_above)[0]
        else:
            sp_2 = 0
        free_delivery = html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../@data-a-carousel-options')[
            0] if html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../@data-a-carousel-options') else None
        if free_delivery:
            sp_3 = re.findall('"set_size":(.*?),"filteredItems"', free_delivery)[0]
        else:
            sp_3 = 0
        sp_num = str(sp_1) + ',' + str(sp_2) + ',' + str(sp_3)

        product_description = '|-|'.join([item.strip() for item in html.xpath(
            '//div[@id="productDescription"]//span/text() | //h2[contains(text(),"Product Description")]/..//div[@class="celwidget aplus-module premium-module-3-four-column-images aplus-premium"]//text()')
                                          if item.strip()]) if html.xpath(
            '//div[@id="productDescription"]//span/text() | //h2[contains(text(),"Product Description")]/..//div[@class="celwidget aplus-module premium-module-3-four-column-images aplus-premium"]//text()') else None
        buy_sales = html.xpath('//span[@id="social-proofing-faceout-title-tk_bought"]/span/text()')[0].replace(' ',
                                                                                                               '') if html.xpath(
            '//span[@id="social-proofing-faceout-title-tk_bought"]/span/text()') else None
        package_quantity = html.xpath('//label[contains(text(),"Item Package Quantity:")]/../span/text()')[
            0].strip() if html.xpath('//label[contains(text(),"Item Package Quantity:")]/../span/text()') else None

        product_info_dict = html.xpath('//table[@class="a-normal a-spacing-micro"]//span/text()') if html.xpath(
            '//table[@class="a-normal a-spacing-micro"]//span/text()') else None
        if product_info_dict:
            product_json = json.dumps(
                {product_info_dict[i]: product_info_dict[i + 1] for i in range(0, len(product_info_dict), 2)})
        else:
            product_json = product_info_dict


        detail_json = [re.sub(r'\s+', ' ', item.strip().replace('\u200e', '').replace('\u200f', '')) for item in
                       html.xpath(
                           '//h1[contains(text(),"Technical Details")]/../../..//tr//text() | //h1[contains(text(),"Additional Information")]/..//td[@class="a-size-base prodDetAttrValue"]/..//text() | //h2[contains(text(),"Product details")]/../div[@id="detailBullets_feature_div"]/ul//text()')
                       if item.strip()] if html.xpath(
            '//h1[contains(text(),"Technical Details")]/../../..//tr//text() | //h1[contains(text(),"Additional Information")]/..//td[@class="a-size-base prodDetAttrValue"]/..//text() | //h2[contains(text(),"Product details")]/../div[@id="detailBullets_feature_div"]/ul//text()') else None

        if detail_json:
            detail_json = [text for text in detail_json if 'Learn More' not in text]
            product_detail_json = json.dumps(
                {detail_json[i]: detail_json[i + 1] for i in range(0, len(detail_json), 2)})
        else:
            product_detail_json = detail_json

        review_ai_text = html.xpath('//h3[contains(text(),"Customers say")]/../../p/span/text()')[0] if html.xpath(
            '//h3[contains(text(),"Customers say")]/../../p/span/text()') else None

        # sp_initial_seen_asins_json
        seen_asins = [item.split('_')[-1] for item in html.xpath(
            '//h2[contains(text(),"Products related to this item")]/../../../div[@class="a-row"]//ol/li/div/@id')] if html.xpath(
            '//h2[contains(text(),"Products related to this item")]/../../../div[@class="a-row"]//ol/li/div/@id') else None
        seen_asins_title = html.xpath(
            '//h2[contains(text(),"Products related to this item")]/../../../div[@class="a-row"]//ol/li/div/a/@title') if html.xpath(
            '//h2[contains(text(),"Products related to this item")]/../../../div[@class="a-row"]//ol/li/div/a/@title') else None
        seen_asins_src = html.xpath(
            '//h2[contains(text(),"Products related to this item")]/../../../div[@class="a-row"]//ol/li/div//img/@src') if html.xpath(
            '//h2[contains(text(),"Products related to this item")]/../../../div[@class="a-row"]//ol/li/div//img/@src') else None
        seen_asins_total_comments = [item.xpath('./div//span[@class="a-color-link"]/text()')[0] if item.xpath(
            './div//span[@class="a-color-link"]/text()') else None for item in html.xpath(
            '//h2[contains(text(),"Products related to this item")]/../../../div[@class="a-row"]//ol/li')]
        seen_asins_price = [item.xpath('./div//span[@class="a-offscreen"]/text()')[0] if item.xpath(
            './div//span[@class="a-offscreen"]/text()') else None for item in html.xpath(
            '//h2[contains(text(),"Products related to this item")]/../../../div[@class="a-row"]//ol/li')]
        if seen_asins and seen_asins_title and seen_asins_src and seen_asins_total_comments and seen_asins_price:
            initial_combined = zip(seen_asins, seen_asins_title, seen_asins_src, seen_asins_total_comments,
                                   seen_asins_price)
            sp_initial_seen_asins_json = json.dumps([{"seen_asins": seen_asins, "seen_asins_title": seen_asins_title,
                                                      "seen_asins_src": seen_asins_src,
                                                      "seen_asins_total_comments": seen_asins_total_comments,
                                                      "seen_asins_price": seen_asins_price} for
                                                     seen_asins, seen_asins_title, seen_asins_src, seen_asins_total_comments, seen_asins_price
                                                     in initial_combined])
        else:
            sp_initial_seen_asins_json = None

        # sp_4stars_initial_seen_asins_json
        seen_asins = [item.split('_')[-1] for item in html.xpath(
            '//h2[contains(text(),"4 stars and above")]/../../../div[@class="a-row"]//ol/li/div/@id')] if html.xpath(
            '//h2[contains(text(),"4 stars and above")]/../../../div[@class="a-row"]//ol/li/div/@id') else None
        seen_asins_title = html.xpath(
            '//h2[contains(text(),"4 stars and above")]/../../../div[@class="a-row"]//ol/li/div/a/@title') if html.xpath(
            '//h2[contains(text(),"4 stars and above")]/../../../div[@class="a-row"]//ol/li/div/a/@title') else None
        seen_asins_src = html.xpath(
            '//h2[contains(text(),"4 stars and above")]/../../../div[@class="a-row"]//ol/li/div//img/@src') if html.xpath(
            '//h2[contains(text(),"4 stars and above")]/../../../div[@class="a-row"]//ol/li/div//img/@src') else None
        seen_asins_total_comments = [item.xpath('./div//span[@class="a-color-link"]/text()')[0] if item.xpath(
            './div//span[@class="a-color-link"]/text()') else None for item in html.xpath(
            '//h2[contains(text(),"4 stars and above")]/../../../div[@class="a-row"]//ol/li')]
        seen_asins_price = [item.xpath('./div//span[@class="a-offscreen"]/text()')[0] if item.xpath(
            './div//span[@class="a-offscreen"]/text()') else None for item in html.xpath(
            '//h2[contains(text(),"4 stars and above")]/../../../div[@class="a-row"]//ol/li')]

        if seen_asins and seen_asins_title and seen_asins_src and seen_asins_total_comments and seen_asins_price:
            delivery_initial_combined = zip(seen_asins, seen_asins_title, seen_asins_src, seen_asins_total_comments,
                                            seen_asins_price)
            sp_4stars_initial_seen_asins_json = json.dumps([{"seen_asins": seen_asins,
                                                             "seen_asins_title": seen_asins_title,
                                                             "seen_asins_src": seen_asins_src,
                                                             "seen_asins_total_comments": seen_asins_total_comments,
                                                             "seen_asins_price": seen_asins_price} for
                                                            seen_asins, seen_asins_title, seen_asins_src, seen_asins_total_comments, seen_asins_price
                                                            in delivery_initial_combined])
        else:
            sp_4stars_initial_seen_asins_json = None

        # sp_delivery_initial_seen_asins_json
        seen_asins = [item.split('_')[-1] for item in html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../div[@class="a-row"]//ol/li/div/@id')] if html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../div[@class="a-row"]//ol/li/div/@id') else None
        seen_asins_title = html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../div[@class="a-row"]//ol/li/div/a/@title') if html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../div[@class="a-row"]//ol/li/div/a/@title') else None
        seen_asins_src = html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../div[@class="a-row"]//ol/li/div/a/img/@src') if html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../div[@class="a-row"]//ol/li/div/a/img/@src') else None
        seen_asins_total_comments = [item.xpath('.//span[@class="a-color-link"]/text()')[0] if item.xpath(
            './/span[@class="a-color-link"]/text()') else None for item in html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../div[@class="a-row"]//ol/li')]
        seen_asins_price = [item.xpath('.//span[@class="a-offscreen"]/text()')[0] if item.xpath(
            './/span[@class="a-offscreen"]/text()') else None for item in html.xpath(
            '//h2[contains(text(),"Related products with free delivery on eligible orders")]/../../../div[@class="a-row"]//ol/li')]

        if seen_asins and seen_asins_title and seen_asins_src and seen_asins_total_comments and seen_asins_price:
            delivery_initial_combined = zip(seen_asins, seen_asins_title, seen_asins_src, seen_asins_total_comments,
                                            seen_asins_price)
            sp_delivery_initial_seen_asins_json = json.dumps([{"seen_asins": seen_asins,
                                                               "seen_asins_title": seen_asins_title,
                                                               "seen_asins_src": seen_asins_src,
                                                               "seen_asins_total_comments": seen_asins_total_comments,
                                                               "seen_asins_price": seen_asins_price} for
                                                              seen_asins, seen_asins_title, seen_asins_src, seen_asins_total_comments, seen_asins_price
                                                              in delivery_initial_combined])
        else:
            sp_delivery_initial_seen_asins_json = None

        # compare_similar_asin_json
        compare_asin = [item.split('-')[-1] for item in html.xpath(
            '//h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@tabindex="0"]/div/@id | //h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@class="a-section a-spacing-mini"]/a/div/@id')] if html.xpath(
            '//h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@tabindex="0"]/div/@id | //h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@class="a-section a-spacing-mini"]/a/div/@id') else None
        compare_asin_src = html.xpath(
            '//h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@tabindex="0"]//img/@src | //h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@class="a-section a-spacing-mini"]//img/@src') if html.xpath(
            '//h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@tabindex="0"]//img/@src | //h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@class="a-section a-spacing-mini"]//img/@src') else None
        compare_asin_title = html.xpath(
            '//h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@tabindex="0"]//img/@alt | //h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@class="a-section a-spacing-mini"]//img/@alt') if html.xpath(
            '//h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@tabindex="0"]//img/@alt | //h2[contains(text(),"Compare with similar items")]/../div//tr[1]//div[@class="a-section a-spacing-mini"]//img/@alt') else None

        if compare_asin and compare_asin_src and compare_asin_title:
            compare_combined = zip(compare_asin, compare_asin_src, compare_asin_title)
            compare_similar_asin_json = json.dumps([{"compare_asin": compare_asin, "compare_asin_src": compare_asin_src,
                                                     "compare_asin_title": compare_asin_title} for
                                                    compare_asin, compare_asin_src, compare_asin_title in
                                                    compare_combined])
        else:
            compare_similar_asin_json = None

        toge_asin = [item for sublist in [re.findall('/dp/(.*?)/ref', item) for item in html.xpath(
            '//h2[contains(text(),"Frequently bought together")]/../../..//a[@class="a-link-normal _p13n-desktop-sims-fbt_fbt-desktop_image-link__17L3C"]/@href | //h2[contains(text(),"Buy it with")]/../../..//a[@class="a-link-normal _p13n-desktop-sims-fbt_fbt-desktop_image-link__17L3C"]/@href')]
                     for item in sublist] if html.xpath(
            '//h2[contains(text(),"Frequently bought together")]/../../..//a[@class="a-link-normal _p13n-desktop-sims-fbt_fbt-desktop_image-link__17L3C"]/@href | //h2[contains(text(),"Buy it with")]/../../..//a[@class="a-link-normal _p13n-desktop-sims-fbt_fbt-desktop_image-link__17L3C"]/@href') else None
        if not toge_asin:
            toge_asin = [re.findall('product-reviews/(.*?)/ref=', item) for item in html.xpath(
                '//div[@class="a-cardui _sp-desktop-thematic-bundle_thematicBundle-desktop_new-thumbnail-box__1W9Ku _sp-desktop-thematic-bundle_thematicBundle-desktop_two-item-thumbnail-box__7kF95"]/div[position() > 1]//a[@class="a-link-normal"]/@href')] if html.xpath(
                '//div[@class="a-cardui _sp-desktop-thematic-bundle_thematicBundle-desktop_new-thumbnail-box__1W9Ku _sp-desktop-thematic-bundle_thematicBundle-desktop_two-item-thumbnail-box__7kF95"]/div[position() > 1]//a[@class="a-link-normal"]/@href') else None

        toge_title = html.xpath(
            '//div[@class="a-cardui _p13n-desktop-sims-fbt_fbt-desktop_new-detail-faceout-box___WyNy"][position() > 1]//span[@class="a-size-base"]/text() | //div[@class="a-cardui _sp-desktop-thematic-bundle_thematicBundle-desktop_new-thumbnail-box__1W9Ku _sp-desktop-thematic-bundle_thematicBundle-desktop_two-item-thumbnail-box__7kF95"]/div[position() > 1]//span[@class="a-size-base"]/text()')
        toge_price = html.xpath(
            '//div[@class="a-cardui _p13n-desktop-sims-fbt_fbt-desktop_new-detail-faceout-box___WyNy"][position() > 1]//span[@class="a-offscreen"]/text() | //div[@class="a-cardui _sp-desktop-thematic-bundle_thematicBundle-desktop_new-thumbnail-box__1W9Ku _sp-desktop-thematic-bundle_thematicBundle-desktop_two-item-thumbnail-box__7kF95"]/div[position() > 1]//span[@class="a-offscreen"]/text()')
        if toge_asin and toge_title and toge_price:
            together_asin_combined = zip(toge_asin, toge_title, toge_price)
            together_asin_json = json.dumps(
                [{"together_asin": asin, "together_asin_title": title, "together_asin_price": price} for
                 asin, title, price in together_asin_combined])
        else:
            together_asin_json = None
        reviews_json = html.xpath(
            '//div[@id="customerReviewsAttribute_feature_div"]//div[@class="a-section a-spacing-none"]//span[@class="a-size-base a-color-base"]/text() |//div[@id="customerReviewsAttribute_feature_div"]//div[@class="a-section a-spacing-none"]//span[@class="a-icon-alt"]/text()') if html.xpath(
            '//div[@id="customerReviewsAttribute_feature_div"]//div[@class="a-section a-spacing-none"]//span[@class="a-size-base a-color-base"]/text() |//div[@id="customerReviewsAttribute_feature_div"]//div[@class="a-section a-spacing-none"]//span[@class="a-icon-alt"]/text()') else None
        if reviews_json:
            customer_reviews_json = json.dumps(
                [{reviews_json[i].rstrip().lstrip(): reviews_json[i + 1]} for i in range(0, len(reviews_json), 2)])
        else:
            customer_reviews_json = None

        # lob_asin_json
        lob_asin = html.xpath('//div[@id="pba-lob-carousel-row"]//ol/li//a/@href') if html.xpath(
            '//div[@id="pba-lob-carousel-row"]//ol/li//a/@href') else None
        lob_asin_img = html.xpath('//div[@id="pba-lob-carousel-row"]//ol/li//img/@src') if html.xpath(
            '//div[@id="pba-lob-carousel-row"]//ol/li//img/@src') else None
        lob_asin_title = html.xpath(
            '//div[@id="pba-lob-carousel-row"]//ol/li//div[@class="a-section a-spacing-none"]/span/span/text()')
        lob_asin_price = [
            item.xpath('.//span[@class="a-price pba-lob-bundle-buy-price"]/span[@class="a-offscreen"]/text()')[
                0] if item.xpath(
                './/span[@class="a-price pba-lob-bundle-buy-price"]/span[@class="a-offscreen"]/text()') else None for
            item in html.xpath('//div[@id="pba-lob-carousel-row"]//ol/li')]

        if lob_asin and lob_asin_img and lob_asin_title and lob_asin_price:
            lob_combined = zip(lob_asin, lob_asin_img, lob_asin_title, lob_asin_price)
            lob_asin_json = json.dumps([{"lob_asin": lob_asin, "lob_asin_img": lob_asin_img,
                                         "lob_asin_title": lob_asin_title, "lob_asin_price": lob_asin_price,
                                         "lob_asin_total_comment": None} for
                                        lob_asin, lob_asin_img, lob_asin_title, lob_asin_price in lob_combined])
        else:
            lob_asin_json = None

        # review_label_json
        review_lable_title = html.xpath(
            '//div[@class="a-section a-spacing-small a-spacing-top-small _cr-product-insights_style_aspect-symbol-list__24amT"]/a/text() | //div[@data-hook="cr-insights-widget-aspects"]//button//span[@class="a-size-base"]/text()') if html.xpath(
            '//div[@class="a-section a-spacing-small a-spacing-top-small _cr-product-insights_style_aspect-symbol-list__24amT"]/a/text() | //div[@data-hook="cr-insights-widget-aspects"]//button//span[@class="a-size-base"]/text()') else None
        if review_lable_title:
            review_label_json = {}
            for lable_title in review_lable_title:
                reviews = '|-|'.join([item.replace('Read more', '') for item in
                                      html.xpath(f'//div[@data-aspect= "{lable_title}"]//text()')])
                review_label_json[lable_title] = reviews
            json.dumps(review_label_json)
        else:
            review_label_json = None

        # views = html.xpath('//div[@id="ivImagesTab"]') if html.xpath('//div[@id="ivImagesTab"]') else None
        views = html.xpath('//div[@id="main-image-container"]/script[@type="text/javascript"]') if html.xpath(
            '//div[@id="main-image-container"]/script[@type="text/javascript"]') else None

        if views:
            image_view = 1
        else:
            image_view = 0

        fulfilled_by = html.xpath('//a[@id="SSOFpopoverLink_ubb"]/text()')[0].split('Fulfilled by')[-1] if html.xpath(
            '//a[@id="SSOFpopoverLink_ubb"]/text()') else None
        seller_id = html.xpath('//input[@id="merchantID"]/@value')[0] if html.xpath(
            '//input[@id="merchantID"]/@value') else None
        seller_json = json.dumps(
            [{"seller_id": seller_id, "ship_from": ships, "sold_by": sold, "fulfilled_by": fulfilled_by}])

        # variat_num = len(html.xpath(
        #     '//ul[@class="a-unordered-list a-nostyle a-button-list a-declarative a-button-toggle-group a-horizontal a-spacing-top-micro swatches swatchesSquare imageSwatches"]//li')) if html.xpath(
        #     '//ul[@class="a-unordered-list a-nostyle a-button-list a-declarative a-button-toggle-group a-horizontal a-spacing-top-micro swatches swatchesSquare imageSwatches"]//li') else 0

        curr_asin = html.xpath('//link[@rel="canonical"]/@href')[0].split('dp/')[1]
        curr_asin = html.xpath('//div[@id="imageBlock_feature_div"]/span/@data-ssf-share-icon')


        # with open('get_pic.py','w',encoding='utf-8') as f:
        #     f.write(compressed_html_code)

        # print(compressed_html_code)
        print(curr_asin,11111111111111)
        if curr_asin == asin:
            current_asin = None
        else:
            current_asin = curr_asin

        img_src = html.xpath(
            '//ul//li[@class="a-spacing-small item imageThumbnail a-declarative"]//img/@src | //ul//li[@class="a-spacing-small item"]//img/@src') if html.xpath(
            '//ul//li[@class="a-spacing-small item imageThumbnail a-declarative"]//img/@src | //ul//li[@class="a-spacing-small item"]//img/@src') else None
        video_src = html.xpath(
            '//ul/li[@class="a-align-top a-spacing-small item videoBlockIngress videoBlockDarkIngress"]//img/@src') if html.xpath(
            '//ul/li[@class="a-align-top a-spacing-small item videoBlockIngress videoBlockDarkIngress"]//img/@src') else None
        pics_src = html.xpath(
            '//h2[contains(text(),"Product Description")]/..//div[@class="aplus-module-wrapper apm-spacing apm-floatnone apm-fixed-width"]//img/@data-src') if html.xpath(
            '//h2[contains(text(),"Product Description")]/..//div[@class="aplus-module-wrapper apm-spacing apm-floatnone apm-fixed-width"]//img/@data-src') else None
        if img_src and video_src and pics_src:
            img_list = json.dumps([
                [asin, src, idx, type]
                for type, srcs in enumerate([img_src, video_src, pics_src], start=1)
                if srcs is not None
                for idx, src in enumerate(srcs, start=1)])
        else:
            img_list = None

        parent_asin = re.findall('"parent_asin":"(.*?)",', response_text)[0] if re.findall('"parent_asin":"(.*?)",',
                                                                                           response_text) else None
        if not parent_asin:
            parent_asin = html.xpath('//link[@rel="canonical"]/@href')[0].split('/')[-1] if html.xpath(
                '//link[@rel="canonical"]/@href') else None

        min_match_asin_json = None

        asin_detail = re.findall('"dimensionValuesDisplayData" : \{(.*?)},', response_text) if re.findall(
            '"dimensionValuesDisplayData" : \{(.*?)},', response_text) else None
        # print(asin_detail)
        if asin_detail:
            # 使用正则表达式提取 asins 和 details
            asins = re.findall(r'"(\w+)":\[', str(asin_detail))
            variat_num = len(asins)
            try:

                details = [json.loads(item) for item in re.findall('":(\[".*?"])', str(asin_detail))]
                # print(222222,details)
                variat_list = [[asin] + detail for asin, detail in zip(asins, details)]
                # print(333333,variat_list)
            except:
                try:
                    color_size = re.findall('":(\[".*?"])', str(asin_detail))
                    details = [item.replace('\\\'', '').replace('\\\\"', '') for item in color_size]
                    variat_list = [[asin] + ast.literal_eval(detail) for asin, detail in zip(asins, details)]
                except:
                    color_size = re.findall('":(\[".*?"])', str(asin_detail))
                    details = [item.replace('\\\'', '').replace('\\\\"', '').replace('"', '').replace('[', '').replace(']', '') for item in color_size]
                    variat_list = [[asins[i], details[i]] for i in range(len(asins))]

            # 向每个子列表添加额外的值
            for detail in variat_list:
                detail.extend([parent_asin, 1, None, None])
            variat_list = json.dumps(variat_list)
        else:
            variat_list = None
            variat_num = 0


        item = {}
        item['id'] = 1
        item['asin'] = asin
        item['img_url'] = img_url
        item['title'] = title
        item['title_len'] = title_len
        item['price'] = price
        item['rating'] = rating
        item['total_comments'] = total_comments
        item['buy_box_seller_type'] = buy_box_seller_type
        item['page_inventory'] = page_inventory
        item['category'] = category
        item['volume'] = volume
        item['weight'] = weight
        item['rank'] = rank
        item['launch_time'] = launch_time
        item['category_state'] = None
        item['img_num'] = img_num
        item['img_type'] = img_type
        item['activity_type'] = activity_type
        item['one_two_val'] = one_two_val
        item['three_four_val'] = three_four_val
        item['five_six_val'] = five_six_val
        item['eight_val'] = eight_val
        item['qa_num'] = None
        item['one_star'] = one_star
        item['two_star'] = two_star
        item['three_star'] = three_star
        item['four_star'] = four_star
        item['five_star'] = five_star
        item['low_star'] = low_star
        item['together_asin'] = together_asin
        item['brand'] = brand
        item['ac_name'] = ac_name
        item['material'] = material
        item['node_id'] = node_id
        item['data_type'] = None
        item['sp_num'] = sp_num
        item['describe'] = describe
        item['date_info'] = date_info
        item['weight_str'] = weight_str
        item['package_quantity'] = package_quantity
        item['pattern_name'] = pattern_name
        item['spider_int'] = None
        item['follow_sellers'] = follow_sellers
        item['product_description'] = product_description
        item['buy_sales'] = buy_sales
        item['image_view'] = image_view
        item['product_json'] = product_json
        item['product_detail_json'] = product_detail_json
        item['review_ai_text'] = review_ai_text
        item['review_label_json'] = review_label_json
        item['lob_asin_json'] = lob_asin_json
        item['sp_initial_seen_asins_json'] = sp_initial_seen_asins_json
        item['sp_4stars_initial_seen_asins_json'] = sp_4stars_initial_seen_asins_json
        item['sp_delivery_initial_seen_asins_json'] = sp_delivery_initial_seen_asins_json
        item['compare_similar_asin_json'] = compare_similar_asin_json
        item['customer_reviews_json'] = customer_reviews_json
        item['together_asin_json'] = together_asin_json
        item['min_match_asin_json'] = min_match_asin_json
        item['seller_json'] = seller_json
        item['variat_num'] = variat_num
        item['current_asin'] = current_asin
        item['img_list'] = img_list
        item['variat_list'] = variat_list
        item['parent_asin'] = parent_asin

        # print(item)



    def execution_parse(self, asin_html_queue):
        while True:
            if asin_html_queue.empty() == False:
                asin_html = asin_html_queue.get()
                keywords_scraper_url_list = asin_html.split('||-||')
                asin_org = keywords_scraper_url_list[0]
                html_org = keywords_scraper_url_list[1]
                print(f'当前解析：{asin_org}')
                html_str = json.loads(html_org)
                html = self.decompress_bytes(html_str)
                # with open('get_pic.py', 'w', encoding='utf-8') as f:
                #     f.write(html)
                asin_html_queue.task_done()

                # self.parse_data(asin_org,html)


            else:
                break

    def run(self):
        # B0CY4MYP37
        asin = 'B0BPF1QNQ4'
        # self.request_data(asin)
        with open('B0BPF1QNQ41.html', 'r',encoding='utf-8') as f:
            response_text = f.read()
        item = htmldetailsparser.parse_data_new(asin, response_text, id_org=123)
        print(item)
        Con.save_us_asin_detail_2024_wj(item)
        # self.parse_data(asin,response_text)
if __name__ == '__main__':
    GetHtmlDetails().run()


