import time
from playwright.sync_api import sync_playwright
from sqlalchemy import create_engine
from sqlalchemy.engine import URL
from lxml import etree
import random
import traceback
import pandas as pd


class One688LoginSpider(object):
    def __init__(self, site):
        self.site_name = site
        self.data = None
    def mysql_connect(self, site='us'):
        if site == 'us':
            db = 'selection'
        else:
            db = f'selection_{site}'
        DB_CONN_DICT = {
            "mysql_port": 3306,
            "mysql_user": "XP_Yswg2025_PY",
            "mysql_pwd": "Gd1pGJog1ysLMLBdML8w81",
            "mysql_host": "rm-wz9yg9bsb2zf01ea4yo.mysql.rds.aliyuncs.com",
        }
        url = URL.create(
            drivername="mysql+pymysql",
            username=DB_CONN_DICT["mysql_user"],
            password=DB_CONN_DICT["mysql_pwd"],  # 原始密码，含 @ 也没问题
            host=DB_CONN_DICT["mysql_host"],
            port=int(DB_CONN_DICT["mysql_port"]),
            database=db,
            query={"charset": "utf8mb4"}
        )
        self.engine_mysql = create_engine(url)
        url_us = URL.create(
            drivername="mysql+pymysql",
            username=DB_CONN_DICT["mysql_user"],
            password=DB_CONN_DICT["mysql_pwd"],  # 原始密码，含 @ 也没问题
            host=DB_CONN_DICT["mysql_host"],
            port=int(DB_CONN_DICT["mysql_port"]),
            database='selection',
            query={"charset": "utf8mb4"}
        )
        self.engine_us_mysql = create_engine(url_us)

        self.engine_pg = create_engine(
            f"postgresql+psycopg2://postgres:F9kL2sXe81rZq@113.100.143.162:5432/{db}",
            encoding='utf-8')
        self.num = 0
        week = time.strftime("%W")
        yaer = time.strftime('%Y', time.localtime(time.time()))
        self.y_w = f"{yaer}-{week}"

    def crawl(self):
        self.page.wait_for_timeout(10000)
        page_content = self.page.content()
        html = etree.HTML(page_content)
        Category_list = html.xpath('//h2[contains(text(),"Category")]/following-sibling::div/div')
        for Category in Category_list:
            Category_name = Category.xpath('.//@label')
            print("Category_name 名称 11111", Category_name)
            self.page.evaluate_handle(
                f"""document.querySelector("kat-radiobutton[label='{Category_name[0]}']").click()""")
            self.page.wait_for_timeout(5000)
            page_content = self.page.content()
            html = etree.HTML(page_content)
            Product_Type_list = html.xpath(
                '//h2[contains(text(),"Product Type")]/following-sibling::div/div')
            del Product_Type_list[0]
            for Product_Type in Product_Type_list:
                Product_name = Product_Type.xpath('./@id')
                self.page.evaluate_handle(f"document.querySelector('#{Product_name[0]} > kat-radiobutton').click()")
                self.page.wait_for_timeout(3000)
                page_content = self.page.content()
                html = etree.HTML(page_content)
                Item_Type_Keyword_id_list = html.xpath(
                    '//h2[contains(text(),"Item Type Keyword")]/following-sibling::div/div')
                kw_list = []
                for Item_Type_Keyword_id in Item_Type_Keyword_id_list:
                    Keyword_id = Item_Type_Keyword_id.xpath('./@id')
                    print("Keyword_id:", Keyword_id)
                    Keyword = html.xpath(f"//div[@id='{Keyword_id[0]}']/kat-radiobutton/@label")
                    print('Keyword', Keyword)
                    self.page.query_selector(f"xpath=//kat-radiobutton[@value='{Keyword_id[0]}']").click()
                    self.page.wait_for_timeout(6000)
                    self.page.on("requestfinished", self.print_request_finished)
                    self.page.wait_for_timeout(6000)
                    time.sleep(15)
                    if self.data:
                        kw_list.append(self.data)
                        self.data = None
                    print('\n')
                    self.page.wait_for_timeout(3500)
                    print('\n')
                    self.data = None
                    # break
                self.save_data(kw_list)

    def print_request_finished(self, request):
        # 拦截请求获取数据
        post_url = 'https://sellercentral.amazon.com/next/v2/getPerformanceDashboard'
        if post_url in request.url:
            if request.response():
                try:
                    self.data = request.response().json()
                    print('self.data:: ',self.data)
                except Exception as e:
                    print('拦截url报错：', e, f"\n{traceback.format_exc()}")
                    self.data = None


    def get_category_data(self, Category_list):
        print('Category_list:::', Category_list)
        num = 0
        for Category in Category_list:
            print(Category, '   22222222222222222222222222222222222222')
            num += 1
            print("Category_name 名称 11111", Category)
            self.page.evaluate(f"""document.querySelector("kat-radiobutton[label='{Category}']").click()""")
            time.sleep(5)
            html = etree.HTML(self.page.content())
            Product_Type_list = html.xpath(
                '//h2[contains(text(),"Product Type")]/following-sibling::div/div')
            for Product_Type in Product_Type_list:
                time.sleep(5)
                Product_name = Product_Type.xpath('./@id')
                print("Product_name3222222222::", Product_name[0].upper())
                self.page.evaluate(f"document.querySelector('#{Product_name[0]} > kat-radiobutton').click()")
                time.sleep(2)
                html = etree.HTML(self.page.content())
                Item_Type_Keyword_id_list = html.xpath(
                    '//h2[contains(text(),"Item Type Keyword")]/following-sibling::div/div')
                kw_list = []
                for Item_Type_Keyword_id in Item_Type_Keyword_id_list:
                    Keyword_id = Item_Type_Keyword_id.xpath('./@id')
                    print("Keyword_id:", Keyword_id)
                    Keyword = html.xpath(f"//div[@id='{Keyword_id[0]}']/kat-radiobutton/@label")
                    print('Keyword', Keyword)
                    self.page.locator(f'//kat-radiobutton[@value="{Keyword_id[0]}"]').click()
                    self.page.wait_for_timeout(6000)
                    self.page.on("requestfinished", self.print_request_finished)
                    if self.data:
                        kw_list.append(self.data)
                        self.data = None
                    print('\n')
                    self.page.wait_for_timeout(3500)
                    print('\n')
                    self.data = None
                    # break
                self.save_data(kw_list)

    def save_data(self, kw_list):
        with open(r'data.json', 'a', encoding='utf-8') as f:
            for data_dict in kw_list:
                # f.write(json.dumps(data_dict)+'\n')
                print(data_dict['demand']['unitSold'], '售出單位')
                print(data_dict['demand']['glanceViews'], '瀏覽量')
                print(data_dict['demand']['yearOnYearUnitSold'], '同比銷量')
                print('\n')
                print(data_dict['demand']['yearOnYearGlanceViews'], '年同比瀏覽次數')
                print('\n')
                print(data_dict['demand']['unitSoldYOY'], '銷售量EO')
                print('\n')
                print(data_dict['demand']['glanceViewsYOY'], '瀏覽量同比')
                print('\n')
                print(data_dict['demand']['mostPopularKeywords'], '最熱門關鍵字')
                print('\n')
                print(data_dict['demand']['searchToPurchaseRatio'], '搜尋購買比率')
                print('\n')
                print(data_dict['demand']['returnRatio'], '回報率')
                print('\n')
                print(data_dict['demand']['returnReasons'], '退貨原因')
                print('\n')
                print(data_dict['demand']['priceConversionRates'], '價格轉換率')
                print('\n')
                print(data_dict['demand']['netShippedGMS'], '網運GMS')
                print('\n')
                print(data_dict['demand']['searchVolume'], '搜尋量')
                print('\n')
                print(data_dict['demand']['clickCount'], '點擊次數')
                print('\n')
                print(data_dict['demand']['buyBoxPrice'], '買盒價格')
                print('\n')

                print(data_dict['competition']['starRatings'])
                print(data_dict['competition']['sellerCount'])
                print(data_dict['competition']['asinCount'])
                print(data_dict['competition']['offersPerAsin'])
                print(data_dict['competition']['medianAdSpendPerClick'])
                print(data_dict['competition']['avgAdSpendPerClick'])
                print(data_dict['competition']['majorityAdSpendPerClick'])
                print(data_dict['competition']['newAsinCount'])
                print(data_dict['competition']['newBrandCount'])


    def read_category(self):
        # 接着上次中断的继续
        print('接着上次中断的继续')
        self.mysql_connect(site=self.site_name)
        select_sql = 'select category  from seller_category_insights_syn where state =1'
        df = pd.read_sql(select_sql, con=self.engine_pg)
        category_list = list(df.category)
        if category_list:
            return category_list
        else:
            workflow_everyday_list = [
                [self.site_name, self.y_w, '类目分析抓取完成', 3, f'{self.site_name}_aba_profit_category_insights', 'week',
                 '类目分析', '是']]
            df_seller_asin_account = pd.DataFrame(data=workflow_everyday_list,
                                                  columns=['site_name', 'date_info', 'status', 'status_val',
                                                           'table_name', 'date_type', 'page', 'is_end'])
            df_seller_asin_account.to_sql('workflow_progress', con=self.engine_us_mysql, if_exists='append',
                                          index=False)

    def run(self):
        for i in range(2):
            try:
                self.page.goto('https://sellercentral.amazon.com/selection/category-insights')
                self.page.wait_for_timeout(2000)
                time.sleep(random.uniform(10, 25.25))
                break
            except:
                time.sleep(5)
        if self.site_name == 'us':
            self.page.evaluate(
                'document.querySelector("#ATVPDKIKX0DER > kat-radiobutton").shadowRoot.querySelector("div > div.text > slot > kat-label:nth-child(1)").click()')
        elif self.site_name == 'uk':
            self.page.evaluate(
                'document.querySelector("#A1F83G8C2ARO7P > kat-radiobutton").shadowRoot.querySelector("div > div.text > slot > kat-label:nth-child(1)").click()')

    def main(self):
        # 初始化
        with sync_playwright() as _playwright:
            # _playwright.chromium.launch_persistent_context
            browser = _playwright.chromium.launch_persistent_context(
                # 指定本机用户缓存地址
                user_data_dir=r"C:\Users\Administrator\AppData\Local\Google\Chrome\User Data",
                # 指定本机google客户端exe的路径
                executable_path=r"C:\Program Files\Google\Chrome\Application\chrome.exe",
                # 要想通过这个下载文件这个必然要开  默认是False
                accept_downloads=True,
                # 设置不是无头模式
                headless=False,
                bypass_csp=True,
                locale='en-GB',
                ignore_https_errors=True,
                no_viewport=True,
                slow_mo=10,
                # 跳过检测
                args=['--disable-blink-features=AutomationControlled', '--remote-debugging-port=9222']
            )

            self.page = browser.new_page()
            js = """
                    Object.defineProperties(navigator, {webdriver:{get:()=>undefined}});
                    """
            self.page.add_init_script(js)
            self.page.evaluate_handle('''() =>{ window.chrome = { runtime: {}, }; }''')
            self.page.evaluate_handle(
                '''() =>{ Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5,6], }); }''')
            # 模拟浏览器参数
            self.page.locator("body").click()
            js = """
                    Object.defineProperties(navigator, {webdriver:{get:()=>undefined}});
                    """
            self.page.add_init_script(js)
            time.sleep(3)
            Category_list = self.read_category()
            if Category_list:
                self.run()
                self.get_category_data(Category_list)


if __name__ == '__main__':
    One688 = One688LoginSpider('us')
    One688.main()
