from playwright.sync_api import sync_playwright
from sqlalchemy import create_engine
import pandas as pd
import urllib.parse
import json
import traceback
import time
from sqlalchemy.engine import URL
'商机探测器。下载bsr分类数据'


class One688LoginSpider(object):
    def __init__(self, site='us'):
        self.site = site
        self.data = None
        month = time.strftime("%m")
        yaer = time.strftime('%Y', time.localtime(time.time()))
        self.y_w = f"{yaer}-{month}"
        self.mysql_connect()
    def mysql_connect(self):
        if self.site == 'us':
            db = 'selection'
        else:
            db = f'selection_{self.site}'
        DB_CONN_DICT = {
            "mysql_port": 3306,
            "mysql_db": db,
            "mysql_user": "XP_Yswg2025_PY",
            "mysql_pwd": "Gd1pGJog1ysLMLBdML8w81",
            "mysql_host": "rm-wz9yg9bsb2zf01ea4yo.mysql.rds.aliyuncs.com",
        }
        url = URL.create(
            drivername="mysql+pymysql",
            username=DB_CONN_DICT["mysql_user"],
            password=DB_CONN_DICT["mysql_pwd"],  # 原始密码，含 @ 也没问题
            host=DB_CONN_DICT["mysql_host"],
            port=int(DB_CONN_DICT["mysql_port"]),
            database=db,
            query={"charset": "utf8mb4"}
        )
        self.engine_us_mysql = create_engine( url)

        self.engine_pg = create_engine(
            f"postgresql+psycopg2://postgres:F9kL2sXe81rZq@113.100.143.162:5432/{db}",
            encoding='utf-8')
        return self.engine_us_mysql

    def print_request_finished(self, request):
        # 拦截请求获取数据
        if self.post_url in request.url:
            if request.response():
                try:
                    self.data = request.response().json()
                    print(self.data)
                except Exception as e:
                    print('拦截url报错：', e, f"\n{traceback.format_exc()}")
                    self.data = None

    def select_category_json(self):
        sql = 'SELECT category_json,id FROM seller_product_opportunity_syn where state=1'
        engine_mysql = self.mysql_connect()
        df_category_json = pd.read_sql(sql, con=engine_mysql)
        category_data_list = list(df_category_json['category_json'] + '|=|=|' + df_category_json['id'].astype("U"))
        data_list = []
        for i in category_data_list:
            data = i.split('|=|=|')
            data_list.append(data)
        return data_list

    def get_category(self):
        data_list = self.select_category_json()
        for data in data_list:
            item = json.loads(data[0])
            print('请求分类：', item)
            category_name = item['browseNodeName']
            item_list = []
            item_list.append(item)
            for i in range(3):
                try:
                    base_url = "https://sellercentral.amazon.com/opportunity-explorer/search?mons_redirect=stck_reroute&browsenodes="
                    json_data_item = json.dumps(item_list)
                    encoded_data = urllib.parse.quote(json_data_item)
                    full_url = f"{base_url}{encoded_data}"
                    print('分类url：', full_url)
                    self.post_url = '/ox-api/graphql'
                    self.crawl(full_url)
                    if self.data:
                        self.mysql_connect()
                        category_data_list = []
                        for i in self.data['data']['niches']:
                            i_json_data = json.dumps(i)
                            print('json_data:::', i_json_data)
                            niche_title = i['nicheTitle']
                            product_count = i['nicheSummary']['productCount']
                            search_volume_t360 = i['nicheSummary']['searchVolumeT360']
                            search_volume_t90 = i['nicheSummary']['searchVolumeT90']
                            search_volume_growth_t180 = i['nicheSummary']['searchVolumeGrowthT180']
                            search_volume_growth_t90 = i['nicheSummary']['searchVolumeGrowthT90']
                            minimum_units_sold_t360 = i['nicheSummary']['minimumUnitsSoldT360']
                            maximum_units_sold_t360 = i['nicheSummary']['maximumUnitsSoldT360']
                            minimum_average_units_sold_t360 = i['nicheSummary']['minimumAverageUnitsSoldT360']
                            maximum_average_units_sold_t360 = i['nicheSummary']['maximumAverageUnitsSoldT360']
                            minimum_price = i['nicheSummary']['minimumPrice']
                            maximum_price = i['nicheSummary']['maximumPrice']
                            avg_price = i['nicheSummary']['avgPrice']
                            return_rate_t360 = i['nicheSummary']['returnRateT360']
                            search_volume_growth_t360 = i['nicheSummary']['searchVolumeGrowthT360']
                            searchTerm_list = []
                            for i in i['topSearchTermMetrics']:
                                searchTerm_list.append(i['searchTerm'])
                            if searchTerm_list:
                                popular_search = ' && '.join(searchTerm_list)
                            else:
                                popular_search = None
                            category_data_list.append(
                                [category_name, i_json_data, niche_title, product_count, search_volume_t360,
                                 search_volume_t90, search_volume_growth_t180, search_volume_growth_t90,
                                 minimum_units_sold_t360, maximum_units_sold_t360, minimum_average_units_sold_t360,
                                 maximum_average_units_sold_t360, minimum_price, maximum_price,
                                 avg_price, return_rate_t360, search_volume_growth_t360, self.site, self.y_w,
                                 popular_search])

                        df_category_data = pd.DataFrame(data=category_data_list,
                                                        columns=['category_name', 'category_data_json', 'niche_title',
                                                                 'product_count',
                                                                 'search_volume_t360', 'search_volume_t90',
                                                                 'search_volume_growth_t180',
                                                                 'search_volume_growth_t90', 'minimum_units_sold_t360',
                                                                 'maximum_units_sold_t360',
                                                                 'minimum_average_units_sold_t360',
                                                                 'maximum_average_units_sold_t360',
                                                                 'minimum_price', 'maximum_price', 'avg_price',
                                                                 'return_rate_t360', 'search_volume_growth_t360',
                                                                 'site', 'date_info', 'search_term'])
                        df_category_data.to_sql('seller_product_opportunity', con=self.engine_pg, if_exists='append',
                                                index=False)
                        print('存储成功：', len(category_data_list))
                        with self.engine_us_mysql.begin() as conn:
                            sql_update = f"update seller_product_opportunity_syn set state=3 where id={int(data[1])};"
                            print('修改状态：', sql_update)
                            conn.execute(sql_update)
                        self.data = None
                    break
                except Exception as e:
                    print("并等待5s继续", e, f"\n{traceback.format_exc()}")
                    time.sleep(10)
                    print('报错重试')
            time.sleep(10)

        self.save_workflow_everyday()

    def save_workflow_everyday(self):
        self.mysql_connect()
        workflow_everyday_list = [
            [self.site, self.y_w, '商机探测器抓取完成', 3, 'seller_product_opportunity', 'month',
             '商机探测器', '是']]
        df_seller_asin_account = pd.DataFrame(data=workflow_everyday_list,
                                              columns=['site_name', 'date_info', 'status', 'status_val',
                                                       'table_name', 'report_date', 'page', 'is_end'])
        df_seller_asin_account.to_sql('workflow_everyday', con=self.engine_us_mysql, if_exists='append',
                                      index=False)

    def crawl(self, url):
        self.page.on("requestfinished", self.print_request_finished)
        # 请求 指定的 URL
        self.page.goto(url)
        self.page.wait_for_timeout(25000)
        # 等待页面加载一段时间

    def run(self):
        self.page.goto('https://sellercentral.amazon.com/gp/homepage.html/ref=xx_home_logo_xx?')
        self.page.wait_for_timeout(20000)
        # # 开始持续拦截请求
        self.get_category()

    def main(self):
        # 初始化
        with sync_playwright() as _playwright:
            # _playwright.chromium.launch_persistent_context
            browser = _playwright.chromium.launch_persistent_context(
                # 指定本机用户缓存地址
                user_data_dir=r"C:\Users\Administrator\AppData\Local\Google\Chrome\User Data",
                # 指定本机google客户端exe的路径
                executable_path=r"C:\Program Files\Google\Chrome\Application\chrome.exe",
                # 要想通过这个下载文件这个必然要开  默认是False
                accept_downloads=True,
                # 设置不是无头模式
                headless=False,
                bypass_csp=True,
                locale='en-GB',
                ignore_https_errors=True,
                no_viewport=True,
                slow_mo=10,
                # 跳过检测
                args=['--disable-blink-features=AutomationControlled', '--remote-debugging-port=9222']
            )

            self.page = browser.new_page()
            js = """
                    Object.defineProperties(navigator, {webdriver:{get:()=>undefined}});
                    """
            self.page.add_init_script(js)
            self.page.evaluate_handle('''() =>{ window.chrome = { runtime: {}, }; }''')
            self.page.evaluate_handle(
                '''() =>{ Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5,6], }); }''')
            # 模拟浏览器参数
            self.page.locator("body").click()
            js = """
                    Object.defineProperties(navigator, {webdriver:{get:()=>undefined}});
                    """
            self.page.add_init_script(js)

            self.run()
            self.page.close()


if __name__ == '__main__':
    One688 = One688LoginSpider()
    One688.main()
