# import sys
# import os
#
# sys.path.append(os.path.dirname(sys.path[0]))  # 上级目录
# from threading_spider.db_connectivity import connect_db
# import traceback
# import random
# from lxml import etree
# import json
# # import requests
# from curl_cffi import requests
# import time
# import pymysql
# from urllib.parse import quote
#
# """获取 junglescout bsr分类排名 销量"""
#
# rank_list = [1, 10, 30, 50, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500, 1600, 1700,
#              1800,
#              1900,
#              2000, 2100, 2200, 2300, 2400, 2500, 2600, 2700, 2800, 2900, 3000, 3100, 3200, 3300, 3400, 3500, 3600, 3700,
#              3800, 3900, 4000, 4100, 4200, 4300, 4400, 4500, 4600, 4700, 4800, 4900, 5000, 5100, 5200, 5300, 5400, 5500,
#              5600, 5700, 5800, 5900, 6000, 6100, 6200, 6300, 6400, 6500, 6600, 6700, 6800, 6900, 7000, 7100, 7200, 7300,
#              7400, 7500, 7600, 7700, 7800, 7900, 8000, 8100, 8200, 8300, 8400, 8500, 8600, 8700, 8800, 8900, 9000, 9100,
#              9200, 9300, 9400, 9500, 9600, 9700, 9800, 9900, 10000, 11000, 12000, 13000, 14000, 15000, 16000, 17000,
#              18000, 19000, 20000, 25000, 30000, 35000, 40000, 45000, 50000, 55000, 60000, 65000, 70000, 75000, 80000,
#              85000, 90000, 95000, 100000, 150000, 200000, 250000, 300000, 350000, 400000, 450000, 500000, 550000,
#              600000, 650000, 700000, 750000, 800000, 850000, 900000, 950000, 1000000]
#
#
# def send_mes(site_name):
#
#     month = time.strftime("%m")
#     year = time.strftime("%Y")
#     _year_month = f'{year}_{int(month)}'
#     db_class = connect_db(site_name)
#     cursor_mysql_db, db = db_class.mysql_db()  # mysql
#     cursor = db.cursor(cursor=pymysql.cursors.DictCursor)  # 以字典的格式输出
#     update_sql_state = f"""
#        UPDATE {site_name}_one_category
#            SET STATE=4
#            WHERE `name` IN (
#                SELECT `name` FROM (
#                    SELECT `name` FROM {site_name}_one_category WHERE rank=1 AND orders=0 AND `year_month`="{_year_month}"
#                ) AS temp_table
#            );
#        """
#     cursor.execute(update_sql_state)
#     db.commit()
#
# def get_jl_rank(db_base):
#     month = time.strftime("%m")
#     year = time.strftime("%Y")
#     year_month = f'{year}_{int(month)}'
#     while True:
#         try:
#             db_class = connect_db(db_base)
#             print(db_base)
#             cursor_mysql_db, db = db_class.mysql_db()  # mysql
#             cursor = db.cursor(cursor=pymysql.cursors.DictCursor)  # 以字典的格式输出
#             category_name_sql_select = f"select distinct `name` , orders,category_first_id from {db_base}_one_category where rank=1 and `year_month` ='{year_month}'"
#             print(category_name_sql_select)
#             cursor.execute(category_name_sql_select)
#             category_name_list = cursor.fetchall()
#             print(category_name_list,'3444444444')
#             update_sql_state1 = f"""UPDATE {db_base}_one_category SET STATE= 1"""
#             cursor.execute(update_sql_state1)
#             db.commit()
#             break
#         except:
#             time.sleep(20)
#     url = "https://www.junglescout.cn/sales-estimator/?gspk=OTQ0&gsxid=joW2dg6ZmAJA&utm_medium=944&utm_source=affiliate"
#     h = {
#         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
#         'Accept-Encoding': 'gzip, deflate, br',
#         'Accept-Language': 'zh-CN,zh;q=0.9',
#         'Cache-Control': 'no-cache',
#         'Connection': 'keep-alive',
#         'Host': 'www.junglescout.cn',
#         'Pragma': 'no-cache',
#         'sec-ch-ua-mobile': '?0',
#         'sec-ch-ua-platform': '"Windows"',
#         'Sec-Fetch-Dest': 'document',
#         'Sec-Fetch-Mode': 'navigate',
#         'Sec-Fetch-Site': 'none',
#         'Sec-Fetch-User': '?1',
#         'Upgrade-Insecure-Requests': '1',
#         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36'
#     }
#     while True:
#         try:
#
#             resp_html = requests.get(url, impersonate="chrome", headers=h,
#                                                 timeout=25, verify=False)
#             if resp_html.status_code == 200 or resp_html.status_code == 201:
#                 break
#             else:
#                 print(resp_html.text)
#                 print("请求首页报错----重新请求 ")
#                 time.sleep(2)
#                 continue
#         except:
#             time.sleep(5)
#             continue
#     html = etree.HTML(resp_html.text)
#     name_lsit = html.xpath(
#         f'//table/tbody/tr[4]/td[2]/div/ul/li[@class="category {db_base}-available"]/span[1]/text()')
#     print('获取 js 分类名称：', name_lsit)
#     for name in name_lsit:
#         if db_base == 'fr':
#             name = name.replace(' & ', ' et ')
#         if name == 'Camera & Photo':
#             name = 'Camera & Photo Products'
#         while True:
#             try:
#                 select_name_sql = f"select id from {db_base}_one_category where name='{name}'"
#                 cursor.execute(select_name_sql)
#                 name_list = cursor.fetchall()
#                 if name_list:
#                     pass
#                 else:
#                     print('junglescout 新增分类：', name)
#                     insert_sql = f'insert into {db_base}_one_category (name, rank, orders)values (%s,%s,%s)'
#                     cursor.execute(insert_sql, (name, 1, 1))
#                     db.commit()
#                 break
#             except:
#                 time.sleep(15)
#
#     print(category_name_list)
#     for category_name in category_name_list:
#         print("历史销量：", category_name['name'], category_name['orders'], category_name['category_first_id'])
#         name_rnak_list = []
#         Handmade_Products_list = []
#         sales = 31
#         for i in rank_list:
#             if sales == 0:
#                 break
#             token_num = 0
#             while True:
#                 try:
#                     print(i)
#                     if db_base == 'fr':
#                         c_name = category_name['name'].replace(' et ', ' & ')
#                     else:
#                         c_name = category_name['name']
#                         if c_name == 'Camera & Photo Products':
#                             c_name = 'Camera & Photo'
#                     num = random.randint(115, 126)
#                     print(num)
#                     f'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{num}.0.0.0 Safari/537.36'
#                     headers = {
#                         "Upgrade-Insecure-Requests": "1",
#                         "User-Agent": f'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{num}.0.0.0 Safari/537.36',
#                         "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
#                         "Sec-Fetch-Site": "none",
#                         "Sec-Fetch-Mode": "navigate",
#                         "Sec-Fetch-User": "?1",
#                         "Sec-Fetch-Dest": "document",
#                         "Accept-Encoding": "gzip, deflate, br",
#                         "Accept-Language": "zh-CN,zh;q=0.9",
#                         "Pragma": "no-cache",
#                         "Cache-Control": "no-cache"
#                     }
#                     params = (
#                         ('rank', f'{i}'),
#                         ('category', f'{c_name}'),
#                         ('store', f'{db_base}'),
#                     )
#                     for num in range(2):
#                         try:
#                             session = requests.Session()
#                             session.headers = headers
#                             session.get('https://d2ogs1k0ty8acr.cloudfront.net/sales', headers=headers,
#                                         params=params, verify=False, timeout=10,impersonate="chrome")
#                             response = session.get('https://d2ogs1k0ty8acr.cloudfront.net/sales', headers=headers,
#                                                    params=params, verify=False, timeout=10,impersonate="chrome")
#                             print(response.url)
#                             print(response.text)
#                             break
#                         except Exception as e:
#                             print(e)
#                             time.sleep(6)
#                             continue
#                     response = json.loads(response.text)
#                     data = response['data']
#                     if data['sales']:
#                         sales = int(data['sales'])
#                         name_rnak_list.append(
#                             (category_name['name'], i, int(data['sales']), category_name['category_first_id'],
#                              year_month))
#                         if category_name['name'] == "Musical Instruments":
#                             Handmade_Products_list.append(
#                                 ('Handmade Products', i, int(data['sales']), category_name['category_first_id'],
#                                  year_month))
#                         token_num = 0
#                         if category_name['orders'] == int(data['sales']):
#                             print("销量不变")
#                             sales = 0
#                             break
#                     else:
#                         name_rnak_list.append(
#                             (category_name['name'], i, 0, category_name['category_first_id'], year_month))
#                         if category_name['name'] == "Musical Instruments":
#                             Handmade_Products_list.append(
#                                 ('Handmade Products', i, 0, category_name['category_first_id'], year_month))
#                         sales = 0
#                 except Exception as e:
#                     print('错误', e, f"\n{traceback.format_exc()}")
#                     time.sleep(random.uniform(20, 60.5))
#                     token_num += 1
#                     continue
#                 time.sleep(random.uniform(15, 40.75))
#                 break
#         print(name_rnak_list)
#         while True:
#             try:
#                 db.ping(reconnect=True)
#                 inset_sql = f"INSERT INTO {db_base}_one_category (name, rank,orders,category_first_id,`year_month`) values (%s, %s, %s, %s, %s)"
#                 print(inset_sql)
#                 cursor.executemany(inset_sql, name_rnak_list)
#                 db.commit()
#                 break
#             except:
#                 db_class = connect_db(db_base)
#                 print(db_base)
#                 cursor_mysql_db, db = db_class.mysql_db()  # mysql
#                 cursor = db.cursor(cursor=pymysql.cursors.DictCursor)  # 以字典的格式输出
#                 time.sleep(20)
#         if category_name['name'] == "Musical Instruments" and db_base == 'us':
#             while True:
#                 try:
#                     inset_sql = f"INSERT INTO {db_base}_one_category (name, rank,orders,category_first_id,`year_month`) values (%s, %s, %s, %s, %s)"
#                     print(inset_sql)
#                     cursor.executemany(inset_sql, Handmade_Products_list)
#                     db.commit()
#                     break
#                 except:
#                     db_class = connect_db(db_base)
#                     print(db_base)
#                     cursor_mysql_db, db = db_class.mysql_db()  # mysql
#                     cursor = db.cursor(cursor=pymysql.cursors.DictCursor)  # 以字典的格式输出
#                     time.sleep(20)
#     update_sql_state = f"""
#     UPDATE {db_base}_one_category
#         SET STATE=4
#         WHERE `name` IN (
#             SELECT `name` FROM (
#                 SELECT `name` FROM {db_base}_one_category WHERE rank=1 AND orders=0 AND `year_month`="{year_month}"
#             ) AS temp_table
#         );
#     """
#     cursor.execute(update_sql_state)
#     db.commit()
#     cursor.close()
#     db.close()
#
#
# if __name__ == '__main__':
#     site = 'us'
#     get_jl_rank(site)
#     send_mes(site)


import sys
import os

sys.path.append(os.path.dirname(sys.path[0]))  # 上级目录
import curl_cffi
from lxml import etree
from threading_spider.db_connectivity import connect_db
import json
import time
import pymysql
import random


def get_cid():
    # 获取所有站点的bsr 大类名称 和 分类id，存储到us站点
    url = 'https://www.sellersprite.com/v2/tools/sales-estimator'
    headers = {

        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
        "Accept-Encoding": "gzip, deflate, br, zstd",
        "Accept-Language": "zh-CN,zh-TW;q=0.9,zh;q=0.8",
        "Cache-Control": "no-cache",
        "User-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
    }
    resp = curl_cffi.get(url, headers=headers, impersonate="chrome")
    html = etree.HTML(resp.text)
    data_category = html.xpath("//script[@id='data-category']/text()")[0]
    print(data_category)
    save_site_category(json.loads(data_category))


def junglescout_spider(db_base):
    month = time.strftime("%m")
    year = time.strftime("%Y")
    year_month = f'{year}_{int(month)}'

    db_class = connect_db(db_base)
    cursor_mysql_db, connect_mysql_db = db_class.mysql_db()  # mysql
    cursor_us, connect_us = db_class.us_mysql_db()  # us站点
    cursor_us = connect_us.cursor(cursor=pymysql.cursors.DictCursor)  # 以字典的格式输出
    category_name_sql_select = f"select `name`,c_id  from all_site_category where site='{db_base}' and state =1"
    print(category_name_sql_select)
    cursor_us.execute(category_name_sql_select)
    category_name_list = cursor_us.fetchall()
    print('category_name_list:',category_name_list)
    for category_name in category_name_list:
        db_class = connect_db(db_base)
        print(db_base)
        cursor_mysql_db, db = db_class.mysql_db()  # mysql
        db_class_us = connect_db('us')
        cursor_us_mysql_db, db_us = db_class_us.us_mysql_db()  # us 站点 mysql
        # cursor = db.cursor(cursor=pymysql.cursors.DictCursor)  # 以字典的格式输出
        print(category_name['name'], category_name['c_id'])
        name_rnak_list = []
        rank_list = [1, 10, 30, 50, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500,
                     1600,
                     1700,
                     1800,
                     1900,
                     2000, 2100, 2200, 2300, 2400, 2500, 2600, 2700, 2800, 2900, 3000, 3100, 3200, 3300, 3400, 3500,
                     3600,
                     3700,
                     3800, 3900, 4000, 4100, 4200, 4300, 4400, 4500, 4600, 4700, 4800, 4900, 5000, 5100, 5200, 5300,
                     5400,
                     5500,
                     5600, 5700, 5800, 5900, 6000, 6100, 6200, 6300, 6400, 6500, 6600, 6700, 6800, 6900, 7000, 7100,
                     7200,
                     7300,
                     7400, 7500, 7600, 7700, 7800, 7900, 8000, 8100, 8200, 8300, 8400, 8500, 8600, 8700, 8800, 8900,
                     9000,
                     9100,
                     9200, 9300, 9400, 9500, 9600, 9700, 9800, 9900, 10000, 11000, 12000, 13000, 14000, 15000, 16000,
                     17000,
                     18000, 19000, 20000, 25000, 30000, 35000, 40000, 45000, 50000
                     # 55000, 60000, 65000, 70000, 75000,
                     # 80000,
                     # 85000, 90000, 95000, 100000, 150000, 200000, 250000, 300000, 350000, 400000, 450000, 500000,
                     # 550000,
                     # 600000, 650000, 700000, 750000, 800000, 850000, 900000, 950000, 1000000
                     ]
        for i in rank_list:
            headers = {
                "Referer": "https://www.sellersprite.com/v2/tools/sales-estimator",
                "Origin":"https://www.sellersprite.com",
                "Accept": "application/json, text/javascript, */*; q=0.01",
                "Accept-Encoding": "gzip, deflate, br, zstd",
                "Accept-Language": "zh-CN,zh-TW;q=0.9,zh;q=0.8",
                "Cache-Control": "no-cache",
                'Cookie': '_ga=GA1.1.522737765.1749119222; _fp=65dbbe41a37f8f9fbe702eba96328267; MEIQIA_TRACK_ID=2y5KvHOzkFTlJAhOLENKAKWsOeb; MEIQIA_VISIT_ID=2y5KvGrMsL4O61rUcCdsLjChlRa; current_guest=r0hgXGqjbSw0_250605-186810; ecookie=xOHgcnYmcZIZKG0z_CN; x-hng=lang=zh-CN&domain=www.sellersprite.com; 8f00639f9c446a2d0213=54fb71d3f2c9e8acb7878e0f73abbf33; _gcl_au=1.1.420472597.1749119222.719336435.1751886424.1751886424; Hm_lvt_e0dfc78949a2d7c553713cb5c573a486=1751516385,1751886410,1751973053,1752031904; HMACCOUNT=800EBCCFB4C6BBFB; 65722c3d8208b58d42f9=7dc2ebaa5e4a51182da4ade1aacd8dc4; rank-guest-user=6159802571t3e3obe8rwmCywrH0Xq28vOMfd8Q+siSpAi1WiGPGuuMcYrYhXyf/QpgeBCBdgCT; rank-login-user=6159802571t3e3obe8rwmCywrH0Xq28mIqu6gO0eXYPrSqY9RlSIznMsavLuIJkOkjELzcr/d1; rank-login-user-info="eyJuaWNrbmFtZSI6Iuilv+mXqOWQuembqiIsImlzQWRtaW4iOmZhbHNlLCJhY2NvdW50IjoiMTMzKioqKjU0MDciLCJ0b2tlbiI6IjYxNTk4MDI1NzF0M2Uzb2JlOHJ3bUN5d3JIMFhxMjhtSXF1NmdPMGVYWVByU3FZOVJsU0l6bk1zYXZMdUlKa09rakVMemNyL2QxIn0="; Sprite-X-Token=eyJhbGciOiJSUzI1NiIsImtpZCI6IjE2Nzk5NjI2YmZlMDQzZTBiYzI5NTEwMTE4ODA3YWExIn0.eyJqdGkiOiJJUGFLc3VqMkZsUmpPR1NRQnIxYkJRIiwiaWF0IjoxNzUyMDMxOTE2LCJleHAiOjE3NTIxMTgzMTYsIm5iZiI6MTc1MjAzMTg1Niwic3ViIjoieXVueWEiLCJpc3MiOiJyYW5rIiwiYXVkIjoic2VsbGVyU3BhY2UiLCJpZCI6MTMzNDkzLCJwaSI6bnVsbCwibm4iOiLopb_pl6jlkLnpm6oiLCJzeXMiOiJTU19DTiIsImVkIjoiTiIsInBobiI6IjEzMzkyNDE1NDA3IiwiZW0iOiJxcTE2NTMxMjE4NjUzQDE2My5jb20iLCJtbCI6IkcifQ.mLIjN_qO4K8w18IDVa0GCRY3MODTmJhZlQaPbgBjeYJRPDwteHfkfqFS_GFyLu4svoahzyFRxkdnKhxs1x90QxQ-7QCwjwypbk8On6gMarKl8jopo9sJbZITvk8mrqtoT6N34LZ1ash35iAkIuPZONPMH8_cp5NxiSC70J12fvIT9ZXp-9zvEk6WV8qQ3pRr0yRuGnSsuWjVvDE9WRNpE3ZmYS_EUBroA51yBEPdS8aBThRuuVGt4HuqrPXp9ZwHoiOcRYu1VcQu-wpIAhLfXcnY1vJA3FXm7w_H00DOGZuM9HRcxdg6Fj-2WP5FvCxbE8z5n1-zbQMs_J8JVaVXgQ; ao_lo_to_n="6159802571t3e3obe8rwmCywrH0Xq28osFyhyxlRsfXXDx9AUjMD2qAFgWUPkLF84KewBkZoL5OL21x5jznuxdPNdiJfglPNE7YH03Vk5CofaP+MGH3y8="; _gaf_fp=01fef3c14bfcaf5a01438f74a677e95a; _ga_38NCVF2XST=GS2.1.s1752031904$o47$g1$t1752031923$j41$l0$h1543227925; _ga_CN0F80S6GL=GS2.1.s1752031906$o46$g1$t1752031924$j42$l0$h0; Hm_lpvt_e0dfc78949a2d7c553713cb5c573a486=1752035308; JSESSIONID=165F9BAA752FE5B22CCD7C5BB7B62F2F',
                "User-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
            }
            url = "https://www.sellersprite.com/v2/tools/sales-estimator/bsr.json"
            data = {
                "station": "DE",
                "cid": category_name['c_id'],  # 分类id
                "bsr": f"{i}"  # 排名
            }
            print(category_name['name'], '请求参数 data::', data)
            for i1 in range(3):
                try:
                    response = curl_cffi.post(url, headers=headers, data=data, impersonate="chrome",timeout=300)
                    print(response.url)
                    # print(response.text)
                    response = json.loads(response.text)
                    break
                except:
                    time.sleep(random.uniform(15, 35.75))

            response_data = response['data']
            print('code::', response['code'])
            print('message::', response['message'])
            print('estMonSales::', response_data['estMonSales'])
            est = response_data.get('estMonSales')
            if est is None:
                # 没拿到数据，跳出循环
                break
            if est == 0.0:
                print(f"{category_name['name']} 排名{i}：销量 0，跳出循环。")
                break

            # 2) 既然不会有 0.3、0.99 这种情况，剩下的 est 都是 ≥ 1
            print(type(est))
            print('获取数据：', category_name['name'], i, est, year_month)
            sales = int(est)
            name_rnak_list.append((category_name['name'], i, sales, year_month))
            time.sleep(random.uniform(20, 75.75))
            # break
        for i in range(4):
            try:
                inset_sql = f"INSERT INTO {db_base}_one_category (name, rank,orders,`year_month`) values (%s, %s, %s, %s)"
                print(inset_sql)
                cursor_mysql_db.executemany(inset_sql, name_rnak_list)
                db.commit()
                up_sql = f"UPDATE all_site_category set state=3 WHERE site='{db_base}' and state=1 and c_id='{category_name['c_id']}'"
                print('更新状态：', up_sql)
                cursor_us_mysql_db.execute(up_sql)
                db_us.commit()
                break
            except Exception as e:
                print('存储失败：', e)
                db_class = connect_db(db_base)
                print(db_base)
                cursor_mysql_db, db = db_class.mysql_db()  # mysql
                cursor_us_mysql_db, db_us = db_class_us.us_mysql_db()  # us 站点 mysql
                time.sleep(20)
        print('当前完成。获取下一个分类销量')
        time.sleep(random.uniform(120, 240.5))


def save_site_category(site_bsr_dict=None):
    db_class = connect_db('us')
    cursor_us_mysql_db, db = db_class.us_mysql_db()  # mysql
    for i in site_bsr_dict.keys():
        print(i)
        delete_sql = f'DELETE from all_site_category where `name` ="{i}"'  # 删除旧分类
        print(delete_sql)
        cursor_us_mysql_db.execute(delete_sql)
        db.commit()
        site_category_list = site_bsr_dict[i]
        for site_category in site_category_list:
            insert_sql = f'insert into all_site_category (site,`name`, c_id)values (%s,%s,%s)'
            cursor_us_mysql_db.execute(insert_sql, (i, site_category['categoryLabel'], site_category['cid']))
            db.commit()
        db_class = connect_db(i)
        cursor_site_mysql_db, db = db_class.mysql_db()  # mysql


def run():
    # get_cid()
    junglescout_spider('de')


if __name__ == '__main__':
    run()
