# import sys
# import os
#
# sys.path.append(os.path.dirname(sys.path[0]))  # 上级目录
# import curl_cffi
# from lxml import etree
# import json
# import time
# import random
# from utils.secure_db_client import get_remote_engine
#
#
# def db_engine(site_name, db_type):
#
#     """
#     "mysql": "mysql",  # 阿里云mysql
#     "postgresql_14": "postgresql_14",  # pg14爬虫库-内网
#     "postgresql_14_outer": "postgresql_14_outer",  # pg14爬虫库-外网
#     "postgresql_15": "postgresql_15",  # pg15正式库-内网
#     "postgresql_15_outer": "postgresql_15_outer",  # pg15正式库-外网
#     "postgresql_cluster": "postgresql_cluster",  # pg集群-内网
#     "postgresql_cluster_outer": "postgresql_cluster_outer",  # pg集群-外网
#     "doris": "doris",  # doris集群-内网
#     """
#     engine = get_remote_engine(
#         site_name=site_name,  # -> database "selection"
#         db_type=db_type,  # -> 服务端 alias "mysql"
#     )
#     return engine
#
# def db_cursor_connect_update(sql):
#     for i in range(3):
#         try:
#             engine_us_mysql = get_remote_engine('us','mysql')
#             print('执行sql：', sql)
#             with engine_us_mysql.begin() as conn:
#                 conn.execute(sql)
#             break
#         except:
#             print('db_cursor_connect 报错：', sql)
# def get_cid():
#     # 获取所有站点的bsr 大类名称 和 分类id，存储到us站点
#     url = 'https://www.sellersprite.com/v2/tools/sales-estimator'
#     headers = {
#
#         "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
#         "Accept-Encoding": "gzip, deflate, br, zstd",
#         "Accept-Language": "zh-CN,zh-TW;q=0.9,zh;q=0.8",
#         "Cache-Control": "no-cache",
#         "User-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
#     }
#     resp = curl_cffi.get(url, headers=headers, impersonate="chrome")
#     html = etree.HTML(resp.text)
#     data_category = html.xpath("//script[@id='data-category']/text()")[0]
#     print(data_category)
#     save_site_category(json.loads(data_category))
#
#
# def junglescout_spider(db_base):
#     month = time.strftime("%m")
#     year = time.strftime("%Y")
#     year_month = f'{year}_{int(month)}'
#     category_name_sql_select = f"select `name`,c_id  from all_site_category where site='{db_base}' and state =1"
#     print(category_name_sql_select)
#     _engine = db_engine('us','mysql')
#     df_ = _engine.read_sql(category_name_sql_select)
#     category_name_list = list(df_.name + '|==|--|'+df_.c_id)
#     print('category_name_list:',category_name_list)
#     for category_name_cid in category_name_list:
#         category_name = category_name_cid.split('|==|--|')[0]
#         c_id = category_name_cid.split('|==|--|')[1]
#         print(category_name, c_id)
#         name_rnak_list = []
#         up_sql = f"UPDATE all_site_category set state=2 WHERE site='{db_base}' and state=1 and c_id='{c_id}'"
#         print('更新状态：', up_sql)
#         db_cursor_connect_update(up_sql)
#         rank_list = [1, 10, 30, 50, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500,
#                      1600,
#                      1700,
#                      1800,
#                      1900,
#                      2000, 2100, 2200, 2300, 2400, 2500, 2600, 2700, 2800, 2900, 3000, 3100, 3200, 3300, 3400, 3500,
#                      3600,
#                      3700,
#                      3800, 3900, 4000, 4100, 4200, 4300, 4400, 4500, 4600, 4700, 4800, 4900, 5000, 5100, 5200, 5300,
#                      5400,
#                      5500,
#                      5600, 5700, 5800, 5900, 6000, 6100, 6200, 6300, 6400, 6500, 6600, 6700, 6800, 6900, 7000, 7100,
#                      7200,
#                      7300,
#                      7400, 7500, 7600, 7700, 7800, 7900, 8000, 8100, 8200, 8300, 8400, 8500, 8600, 8700, 8800, 8900,
#                      9000,
#                      9100,
#                      9200, 9300, 9400, 9500, 9600, 9700, 9800, 9900, 10000, 11000, 12000, 13000, 14000, 15000, 16000,
#                      17000,
#                      18000, 19000, 20000, 25000, 30000, 35000, 40000, 45000, 50000
#                      ]
#         for i in rank_list:
#             headers = {
#                 "Referer": "https://www.sellersprite.com/v2/tools/sales-estimator",
#                 "Origin":"https://www.sellersprite.com",
#                 "Accept": "application/json, text/javascript, */*; q=0.01",
#                 "Accept-Encoding": "gzip, deflate, br, zstd",
#                 "Accept-Language": "zh-CN,zh-TW;q=0.9,zh;q=0.8",
#                 "Cache-Control": "no-cache",
#                 'Cookie': 'Hm_lvt_e0dfc78949a2d7c553713cb5c573a486=1754980411; HMACCOUNT=470336FBA166E984; _ga=GA1.1.369458666.1754980412; _gcl_au=1.1.264141837.1754980412; MEIQIA_TRACK_ID=31AwurWIdtN2Ob4FQ2QW7kcpB0x; MEIQIA_VISIT_ID=31AwuwdPS1Vv4z7uSPqpc0Gj1ce; ecookie=0oXdazBemDJwQj8M_CN; 5d6b3550f67a0d98a3f2=300e7c0221464bf96a29eee60c456f00; _fp=65dbbe41a37f8f9fbe702eba96328267; current_guest=WEMjtTq1tsth_250812-144558; _gaf_fp=59df4b0c2b58ce924ed353a3d4aff048; rank-guest-user=6308305571KK6FnhfedvWg9tSSyk3xj0WOO7cLm/YtvwwmR8H9ligCErogWOBlL8kpVxO2TpkM; rank-login-user=6308305571KK6FnhfedvWg9tSSyk3xj2GRIc/8HSm4vuPYVHI5vKIJMH4fZg1mmDOwxfDtgQRc; rank-login-user-info=eyJuaWNrbmFtZSI6IuWViuWTiOWTiOWTiCIsImlzQWRtaW4iOmZhbHNlLCJhY2NvdW50IjoiMTUzKioqKjEyNzAiLCJ0b2tlbiI6IjYzMDgzMDU1NzFLSzZGbmhmZWR2V2c5dFNTeWszeGoyR1JJYy84SFNtNHZ1UFlWSEk1dktJSk1INGZaZzFtbURPd3hmRHRnUVJjIn0=; Sprite-X-Token=eyJhbGciOiJSUzI1NiIsImtpZCI6IjE2Nzk5NjI2YmZlMDQzZTBiYzI5NTEwMTE4ODA3YWExIn0.eyJqdGkiOiJ4RTZsWDJPbDdhWEphOVRjZVdfbER3IiwiaWF0IjoxNzU0OTgwNDM2LCJleHAiOjE3NTUwNjY4MzYsIm5iZiI6MTc1NDk4MDM3Niwic3ViIjoieXVueWEiLCJpc3MiOiJyYW5rIiwiYXVkIjoic2VsbGVyU3BhY2UiLCJpZCI6MTQ2NjIwMSwicGkiOm51bGwsIm5uIjoi5ZWK5ZOI5ZOI5ZOIIiwic3lzIjoiU1NfQ04iLCJlZCI6Ik4iLCJwaG4iOiIxNTM2ODA1MTI3MCIsImVtIjoibWVpeW91bGFAbWVpeW91bGEuY29tIiwibWwiOiJHIn0.ijVJw3SbLMEwdgwRIK-WviTFwmhYpp5_Kme31vu9V1HdiBLuyNjCwO97G6c5Pjlpio4jJjVdmaWXjB0rgAfpZA0XG3eui7bENUjqeOnAlBseIVfPBUjDWp9vhgV8uGGtpkD5I8wBAu26KSyE6tLuE4ML3mGMaWuNCPc9NpiLszyIaGmp0FkCU761-PmV0K9mNDKPQmUSlCOqHsNn5mscdfWfIaUaYdlAKEh0ojJoIOrDnFK3hLy9qWQ4W2ChgI8HQUU4Y48QwZCXsTn8fvZ6cNstRNuTjeT2Iq16hzdpWyuVzBgFe2YkwRYBdvi4plyPZkauZtb6RRkoi6oNA1qCXA; ao_lo_to_n=6308305571KK6FnhfedvWg9tSSyk3xjw9sOVkTMvBIw9HsBCFNE4fph+5i+ZmhNt9gtZL7px6VG01jHqLt1SVP6xii5hleGt0VSRpt/FdjNQpaxROiyf0=; JSESSIONID=57A66ED8AB79C6F6E5B12EC4414C0E00; _ga_38NCVF2XST=GS2.1.s1754980411$o1$g1$t1754980455$j16$l0$h797116043; Hm_lpvt_e0dfc78949a2d7c553713cb5c573a486=1754980455; _ga_CN0F80S6GL=GS2.1.s1754980412$o1$g1$t1754980455$j17$l0$h0',
#                 "User-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
#             }
#             url = "https://www.sellersprite.com/v2/tools/sales-estimator/bsr.json"
#             data = {
#                 "station": "UK",
#                 "cid": c_id,  # 分类id
#                 "bsr": f"{i}"  # 排名
#             }
#             print(category_name, '请求参数 data::', data)
#             for i1 in range(3):
#                 try:
#                     response = curl_cffi.post(url, headers=headers, data=data, impersonate="chrome",timeout=300)
#                     print(response.url)
#                     # print(response.text)
#                     response = json.loads(response.text)
#                     break
#                 except:
#                     time.sleep(random.uniform(15, 35.75))
#
#             response_data = response['data']
#             print('code::', response['code'])
#             print('message::', response['message'])
#             print('estMonSales::', response_data['estMonSales'])
#             est = response_data.get('estMonSales')
#             if est is None:
#                 # 没拿到数据，跳出循环
#                 break
#             if est == 0.0:
#                 print(f"{category_name} 排名{i}：销量 0，跳出循环。")
#                 break
#
#             # 2) 既然不会有 0.3、0.99 这种情况，剩下的 est 都是 ≥ 1
#             print(type(est))
#             print('获取数据：', category_name, i, est, year_month)
#             sales = int(est)
#             name_rnak_list.append((category_name, i, sales, year_month))
#             time.sleep(random.uniform(30, 95.75))
#             # break
#         for i in range(4):
#             try:
#                 inset_sql = f"INSERT INTO {db_base}_one_category (name, rank,orders,`year_month`) values (%s, %s, %s, %s)"
#                 print(inset_sql)
#                 engine_mysql = db_engine(db_base, 'mysql')
#                 with engine_mysql.begin() as conn:
#                     conn.execute(
#                         f"INSERT INTO {db_base}_one_category (name, rank,orders,`year_month`) values (%s, %s, %s, %s)",
#                         name_rnak_list)
#                 up_sql = f"UPDATE all_site_category set state=3 WHERE site='{db_base}' and state=2 and c_id='{c_id}'"
#                 print('更新状态：', up_sql)
#                 db_cursor_connect_update(up_sql)
#                 break
#             except Exception as e:
#                 print('存储失败：',e)
#                 _engine = db_engine('us','mysql')
#
#         print('当前完成。获取下一个分类销量')
#         time.sleep(random.uniform(120, 240.5))
#
#
# def save_site_category(site_bsr_dict=None):
#     engine_mysql = db_engine('us', 'mysql')
#     for i in site_bsr_dict.keys():
#         print(i)
#         delete_sql = f'DELETE from all_site_category where `name` ="{i}"'  # 删除旧分类
#         db_cursor_connect_update(delete_sql)
#         site_category_list = site_bsr_dict[i]
#         with engine_mysql.begin() as conn:
#             for site_category in site_category_list:
#                 insert_sql = f'insert into all_site_category (site,`name`, c_id)values (%s,%s,%s)'
#                 conn.execute(insert_sql, (i, site_category['categoryLabel'], site_category['cid']))
# def run():
#     # get_cid()
#     junglescout_spider('uk')
#
#
# if __name__ == '__main__':
#     run()






import sys
import os

sys.path.append(os.path.dirname(sys.path[0]))  # 上级目录
import curl_cffi
from lxml import etree
from threading_spider.db_connectivity import connect_db
import json
import time
from utils.secure_db_client import get_remote_engine
import random


def get_cid():
    # 获取所有站点的bsr 大类名称 和 分类id，存储到us站点
    url = 'https://www.sellersprite.com/v2/tools/sales-estimator'
    headers = {

        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
        "Accept-Encoding": "gzip, deflate, br, zstd",
        "Accept-Language": "zh-CN,zh-TW;q=0.9,zh;q=0.8",
        "Cache-Control": "no-cache",
        "User-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
    }
    resp = curl_cffi.get(url, headers=headers, impersonate="chrome")
    html = etree.HTML(resp.text)
    data_category = html.xpath("//script[@id='data-category']/text()")[0]
    print(data_category)

def mysql_connect(site='us'):
    engine_mysql = get_remote_engine(
        site_name=site,  # -> database "selection"
        db_type='mysql',  # -> 服务端 alias "mysql"
    )
    return engine_mysql

def db_cursor_connect_update(sql, site):
    for i in range(3):
        try:
            engine_us_mysql = mysql_connect(site=site)
            print('更新sql：', sql)
            with engine_us_mysql.begin() as conn:
                conn.execute(sql)
            break
        except:
            print(site, 'db_cursor_connect 报错：', sql)
def db_cursor_connect_msyql_read(site,select_state1_sql):
    for i in range(3):
        try:
            engine_mysql = mysql_connect(site=site)
            df = engine_mysql.read_sql(select_state1_sql)
            return df
        except Exception as e:
            import traceback
            traceback.print_exc()  # ★ 打印完整栈到终端
            print(e, 'db_cursor_connect_msyql_read 报错：', select_state1_sql)

def junglescout_spider(db_base):
    month = time.strftime("%m")
    year = time.strftime("%Y")
    year_month = f'{year}_{int(month)}'


    category_name_sql_select = f"select `name`,c_id  from all_site_category where site='{db_base}' and state =1"
    print(category_name_sql_select)
    category_name_list_df = db_cursor_connect_msyql_read('us',category_name_sql_select)
    print(category_name_list_df)

    category_name_list = list(category_name_list_df['name']+'|-|==|'+category_name_list_df['c_id'])
    for category_name_id in category_name_list:
        print(category_name_id, '2323232323')
        c_name = category_name_id.split('|-|==|')[0]
        c_id = category_name_id.split('|-|==|')[1]
        print(c_name, c_id)
        name_rnak_list = []
        up_sql = f"UPDATE all_site_category set state=2 WHERE site='{db_base}' and state=1 and c_id='{c_id}'"
        db_cursor_connect_update(up_sql,'us')
        rank_list = [1, 10, 30, 50, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500,
                     1600,
                     1700,
                     1800,
                     1900,
                     2000, 2100, 2200, 2300, 2400, 2500, 2600, 2700, 2800, 2900, 3000, 3100, 3200, 3300, 3400, 3500,
                     3600,
                     3700,
                     3800, 3900, 4000, 4100, 4200, 4300, 4400, 4500, 4600, 4700, 4800, 4900, 5000, 5100, 5200, 5300,
                     5400,
                     5500,
                     5600, 5700, 5800, 5900, 6000, 6100, 6200, 6300, 6400, 6500, 6600, 6700, 6800, 6900, 7000, 7100,
                     7200,
                     7300,
                     7400, 7500, 7600, 7700, 7800, 7900, 8000, 8100, 8200, 8300, 8400, 8500, 8600, 8700, 8800, 8900,
                     9000,
                     9100,
                     9200, 9300, 9400, 9500, 9600, 9700, 9800, 9900, 10000, 11000, 12000, 13000, 14000, 15000, 16000,
                     17000,
                     18000, 19000, 20000, 25000, 30000, 35000, 40000, 45000, 50000
                     # 55000, 60000, 65000, 70000, 75000,
                     # 80000,
                     # 85000, 90000, 95000, 100000, 150000, 200000, 250000, 300000, 350000, 400000, 450000, 500000,
                     # 550000,
                     # 600000, 650000, 700000, 750000, 800000, 850000, 900000, 950000, 1000000
                     ]
        for i in rank_list:
            headers = {
                "Referer": "https://www.sellersprite.com/v2/tools/sales-estimator",
                "Origin":"https://www.sellersprite.com",
                "Accept": "application/json, text/javascript, */*; q=0.01",
                "Accept-Encoding": "gzip, deflate, br, zstd",
                "Accept-Language": "zh-CN,zh-TW;q=0.9,zh;q=0.8",
                "Cache-Control": "no-cache",
                'Cookie': '_gcl_au=1.1.1447326361.1758264089; Hm_lvt_e0dfc78949a2d7c553713cb5c573a486=1758264089; HMACCOUNT=7F9DD13A227E1D2B; _ga=GA1.1.991212207.1758264089; MEIQIA_TRACK_ID=32uIa4AhuxRLTcHXxm0PIk6Neqn; MEIQIA_VISIT_ID=32uIa1sK6QRdxkjk2DRJqfS7UaG; ecookie=qmC7o7xcw0S1xZD9_CN; d0c5b9780b50a33ad822=dc2936bc8106c9a8ee1714818e7e7a72; _fp=65dbbe41a37f8f9fbe702eba96328267; _gaf_fp=8dc236c6b0cc21a83974c129be61cfc2; current_guest=BWQNYACe6Zku_250919-145796; rank-login-user=6961238571i+0e7rddchJDh/F/NHVdW5HCX9QRmjEXsHOplEmuEXlp0Qwv/G3CXc4Z8WBR8qa8; rank-login-user-info="eyJuaWNrbmFtZSI6IuWViuWTiOWTiOWTiCIsImlzQWRtaW4iOmZhbHNlLCJhY2NvdW50IjoiMTUzKioqKjEyNzAiLCJ0b2tlbiI6IjY5NjEyMzg1NzFpKzBlN3JkZGNoSkRoL0YvTkhWZFc1SENYOVFSbWpFWHNIT3BsRW11RVhscDBRd3YvRzNDWGM0WjhXQlI4cWE4In0="; Sprite-X-Token=eyJhbGciOiJSUzI1NiIsImtpZCI6IjE2Nzk5NjI2YmZlMDQzZTBiYzI5NTEwMTE4ODA3YWExIn0.eyJqdGkiOiJhYnA1OTJtVWFjeElMQm96TkRVTEFBIiwiaWF0IjoxNzU4MjY0MDk2LCJleHAiOjE3NTgzNTA0OTYsIm5iZiI6MTc1ODI2NDAzNiwic3ViIjoieXVueWEiLCJpc3MiOiJyYW5rIiwiYXVkIjoic2VsbGVyU3BhY2UiLCJpZCI6MTQ2NjIwMSwicGkiOm51bGwsIm5uIjoi5ZWK5ZOI5ZOI5ZOIIiwic3lzIjoiU1NfQ04iLCJlZCI6Ik4iLCJwaG4iOiIxNTM2ODA1MTI3MCIsImVtIjoibWVpeW91bGFAbWVpeW91bGEuY29tIiwibWwiOiJHIn0.Q4Sne6pW_Lhytp1g9sR99PbRNN7BcE5azPHptecj7JIKFyRhFNJ7ZuwDnXW755Cs64JwS16Ra2R67pb1WhuxFoFFP6MBNZIql3jMnpBUO33LKBWbHkN3x5r4OXHCjCM6BvhhAyDWftUHnG-QeF-zTuQDfgVPiWSxEPnJj0pBvpDSOAYWgSs7G3J7pWE1Mz5nL4m7VhsKyIpQ0oT3zv5zgPOfXq6CaB0Ud5LjBJ9ADpNlQOi2-7hT05lQoZRgzce8Irx8jrla4icnWsBSYEUyTZSlNEf1pZVly4aK_txRfgYj5PjOyph4axuvZGq1am3wcEiD8MRBNz-ksEspXCeW0Q; ao_lo_to_n="6961238571i+0e7rddchJDh/F/NHVdW+To037jtP43UpoX84C25sG3pqka4v1jufgWxEucssJpM3EjpAFpLcWIRm7MR3R6a8lu71Loi14HqnMls77u9v8="; rank-guest-user=6961238571i+0e7rddchJDh/F/NHVdW6XmQntBFYjoP/KdgG5QRJAelN+6DbB+BnVatmY+ztdl; JSESSIONID=0FEB643072E7552D57ED5DEC085D20D8; _ga_38NCVF2XST=GS2.1.s1758264089$o1$g1$t1758264103$j46$l0$h1039295416; Hm_lpvt_e0dfc78949a2d7c553713cb5c573a486=1758264103; _ga_CN0F80S6GL=GS2.1.s1758264090$o1$g1$t1758264103$j47$l0$h0',
                "User-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
            }
            url = "https://www.sellersprite.com/v2/tools/sales-estimator/bsr.json"
            data = {
                "station": "DE",
                "cid": c_id,  # 分类id
                "bsr": f"{i}"  # 排名
            }
            print(c_name, '请求参数 data::', data)
            for i1 in range(3):
                try:
                    response = curl_cffi.post(url, headers=headers, data=data, impersonate="chrome",timeout=300)
                    print(response.url)
                    # print(response.text)
                    response = json.loads(response.text)
                    break
                except:
                    time.sleep(random.uniform(15, 30.75))

            response_data = response['data']
            print('code::', response['code'])
            print('message::', response['message'])
            print('estMonSales::', response_data['estMonSales'])
            est = response_data.get('estMonSales')
            if est is None:
                # 没拿到数据，跳出循环
                break
            if est == 0.0:
                print(f"{c_name} 排名{i}：销量 0，跳出循环。")
                break

            # 2) 既然不会有 0.3、0.99 这种情况，剩下的 est 都是 ≥ 1
            print(type(est))
            print('获取数据：', c_name, i, est, year_month)
            sales = int(est)
            name_rnak_list.append((c_name, i, sales, year_month))
            time.sleep(random.uniform(20, 65.75))
            # break
        for i in range(4):
            try:
                inset_sql = f"INSERT INTO {db_base}_one_category (name, rank,orders,`year_month`) values (%s, %s, %s, %s)"
                print(inset_sql)
                engine_db_msyql = mysql_connect(site=db_base)
                with engine_db_msyql.begin() as conn:
                    conn.execute(
                        f"INSERT INTO {db_base}_one_category (name, rank,orders,`year_month`) values (%s, %s, %s, %s)",
                        name_rnak_list)

                up_sql = f"UPDATE all_site_category set state=3 WHERE site='{db_base}' and state=2 and c_id='{c_id}'"
                print('更新状态：', up_sql)
                db_cursor_connect_update(up_sql,'us')
                break
            except Exception as e:
                print('存储失败：', e)

                time.sleep(20)
        print('当前完成。获取下一个分类销量')
        time.sleep(random.uniform(90, 200.5))

def run():
    junglescout_spider('de')

if __name__ == '__main__':
    run()