
import os
import sys
from datetime import datetime, timedelta

sys.path.append(os.path.dirname(sys.path[0]))  # 上级目录
import multiprocessing
import time
import random
from amazon_spider.day_asin_spider import async_asin_pg
from utils.db_connect import BaseUtils
from threading_spider.post_to_dolphin import DolphinschedulerHelper
from amazon_spider.VPS_IP import pppoe_ip


def db_cursor_connect_update(sql, site):
    for i in range(3):
        try:
            engine_us_mysql = BaseUtils(site_name='us').mysql_connect()
            print('更新sql：', sql)
            with engine_us_mysql.begin() as conn:
                conn.execute(sql)
            break
        except:
            print(site, 'db_cursor_connect 报错：', sql)


def read_data_from_pg(site, next_date, read_size=1500):
    """主进程：走代理PG read_then_update（含FOR UPDATE）读取数据"""
    try:
        date_info = next_date.replace('-', '_')
        engine_pg = BaseUtils(site_name=site).pg_connect()
        table = f'{site}_all_syn_st_day_{date_info}'
        sql_read = f"SELECT asin, id, date_info, asin_is_variation, data_type, volume, weight_str FROM {table} WHERE state=1 FOR UPDATE SKIP LOCKED LIMIT {read_size}"
        print(sql_read)
        df_read = engine_pg.read_then_update(
            select_sql=sql_read,
            update_table=table,
            set_values={"state": 2},
            where_keys=["id"],
        )
        df_read.drop_duplicates(['asin'], inplace=True)
        if df_read.shape[0] > 0:
            df_read['volume'] = df_read['volume'].fillna('null')
            df_read['weight_str'] = df_read['weight_str'].fillna('null')
            asin_list = list(
                df_read.id.astype("U") + '|' + df_read.asin + '|' + df_read.date_info + '|' + df_read.asin_is_variation.astype(
                    "U") + '|' + df_read.data_type.astype("U") + '|' + df_read.volume.astype(
                    "U") + '|' + df_read.weight_str.astype("U"))
            print(f'读取 {len(asin_list)} 条ASIN')
            return asin_list
        else:
            print('没有 state=1 的数据了')
            return []
    except Exception as e:
        print(f'读取数据失败: {e}')
        return []


def long_time_task(site, next_date, data_chunk):
    print(f"当前进程 {os.getpid()} 分配到 {len(data_chunk)} 条数据")
    async_asin_us = async_asin_pg(site_name=site, next_date=next_date)
    async_asin_us.run_with_data(data_chunk)


def update_mysql_day_asin_state(site_name, next_date, num_state):
    redis_client = BaseUtils().redis_db()
    engine_mysql = BaseUtils(site_name='us').mysql_connect()
    lock_key_state = f"{next_date}_day_asin_lock_state"
    lock_state = redis_client.lock(lock_key_state, timeout=55)  # 10秒超时
    if num_state == 1:
        select_day_status_val = f"select status_val from workflow_progress where date_type='day' and page='ASIN详情' and site_name='us' and date_info='{next_date}'  and kafka_flow_state=1 and spider_state=1"
        print('开始抓取 查询状态1 ::', select_day_status_val)
        df_status_val = engine_mysql.read_sql(select_day_status_val)
        status_val = df_status_val.loc[0, "status_val"] if not df_status_val.empty else None
        if status_val == 1:
            if lock_state.acquire(blocking=True):
                sql_select_ = f"select status_val from workflow_progress where date_type='day' and page='ASIN详情' and site_name='{site_name}' and date_info='{next_date}'  and kafka_flow_state=1 and spider_state=1"
                print('sql_select 2222222:',sql_select_)
                df_status_dict = engine_mysql.read_sql(sql_select_)
                if not df_status_dict.empty:
                    print('查询kafka是否开启', df_status_dict.status_val[0])
                    if df_status_dict.status_val[0] in (1, 2):
                        update_month_spider_state = f"update workflow_progress set spider_state=2,status_val=2 WHERE site_name='{site_name}' and date_type='day' and date_info='{next_date}' and page='ASIN详情'"
                        print('spider_state=2,status_val=2 修改状态：：',update_month_spider_state)
                        db_cursor_connect_update(update_month_spider_state, site_name)
                        DolphinschedulerHelper.start_process_instance_common(
                            project_name="big_data_selection",
                            process_df_name='ALL-实时day-插件+流量选品',
                            startParams={
                                "site_name": f"{site_name}",
                                "date_type": "day",
                                "date_info": f'{next_date}'
                            }
                        )

        n = 0
        while True:
            try:
                sql_read = f"select id from workflow_progress where date_type='day' and page='ASIN详情' and site_name='{site_name}' and date_info='{next_date}'  and kafka_flow_state=3 and spider_state=2"
                print('等待es启动：：',sql_read)
                df_report_date = engine_mysql.read_sql(sql_read)
                if not df_report_date.empty:
                    print('抓取 day asin')
                    break
                else:
                    n += 1
                    time.sleep(120)
                    if n > 25:
                        break
            except:
                time.sleep(10)

    if num_state == 3:
        print('抓取完成 抓取完成')
        if lock_state.acquire(blocking=True):
            sql_read = f"select id from workflow_progress where date_type='day' and page='ASIN详情' and site_name='{site_name}' and date_info='{next_date}'  and kafka_flow_state=3 and spider_state=2"
            print(sql_read, '抓取完成 修改状态 333333444444')
            df_report_id = engine_mysql.read_sql(sql_read)
            if not df_report_id.empty:
                update_month_spider_state = f"update workflow_progress set spider_state=3,status_val=3 WHERE site_name='{site_name}' and date_type='day' and date_info='{next_date}' and page='ASIN详情'"
                print(update_month_spider_state)
                db_cursor_connect_update(update_month_spider_state, site_name)

    if lock_state.locked():
        lock_state.release()



def select_day_asin_id(site, next_date):
    date_info = next_date.replace('-', '_')
    engine_pg_14 = BaseUtils(site_name=site).pg_connect()
    sql_read = f'SELECT id FROM {site}_all_syn_st_day_{date_info} where state in (1,2) LIMIT 1'
    print('查询 {} 是否有asin需要抓取 ::'.format(date_info), sql_read)
    df = engine_pg_14.read_sql(sql_read)
    if not df.empty:
        id_tuple = [1]
    else:
        id_tuple = None
    if id_tuple is None:
        update_mysql_day_asin_state(site, next_date, 3)
        return False
    else:
        print("sql_read:", sql_read)
        update_mysql_day_asin_state(site, next_date, 1)
        return True


def check_workflow_status(site_name, date_info):
    """
    检查 workflow_progress 表中 status_val 状态
    :param site_name: 站点名称，如 'us'
    :param date_info: 日期信息，如 '2026-02-26'
    :return: True-有值可继续，False-无值跳过
    """
    while True:
        try:
            engine_mysql = BaseUtils(site_name='us').mysql_connect()
            sql_read = f"select status_val from workflow_progress where date_type='day' and page='ASIN详情' and site_name='{site_name}' and date_info='{date_info}'"
            print(f'check_workflow_status SQL: {sql_read}')
            df = engine_mysql.read_sql(sql_read)
            if df.empty:
                return False              # 没记录，跳过

            else:
                return True              # 有记录 可以往下抓取数据
        except:
            time.sleep(10)


def start_day_spider():
    pppoe_ip()
    site_list = ['us']
    # 动态生成：前3天到今天，共4个日期
    today = datetime.now()
    date_list = [(today - timedelta(days=i)).strftime('%Y-%m-%d') for i in range(4, -1, -1)]
    print(f'本次日ASIN任务日期范围: {date_list}')
#本次日ASIN任务日期范围: ['2026-03-23', '2026-03-24', '2026-03-25', '2026-03-26']
    for site in site_list:
        for next_date in date_list:
            if not check_workflow_status(site, next_date):
                print(f'跳过 {next_date}，继续下一个日期')
                continue

            num_processes = 5
            while True:
                if select_day_asin_id(site, next_date):
                    current_time = datetime.now()
                    date_info = next_date.replace('-', '_')
                    five_minutes_later = current_time + timedelta(minutes=2)
                    print('后五分钟时间', five_minutes_later)

                    # 主进程统一读取数据
                    all_data = read_data_from_pg(site, next_date, read_size=1500)
                    if not all_data:
                        print('读取数据为空，等待重试')
                        time.sleep(30)
                        continue

                    # 平均分配给子进程
                    chunks = [all_data[i::num_processes] for i in range(num_processes)]

                    processes = []
                    for idx in range(num_processes):
                        if chunks[idx]:
                            process = multiprocessing.Process(target=long_time_task, args=(site, date_info, chunks[idx]))
                            processes.append(process)
                            process.start()
                    for process in processes:
                        process.join()

                    print('所有进程运行完毕！开始切换ip')
                    current_time = datetime.now()
                    if current_time > five_minutes_later:
                        pppoe_ip()
                    else:
                        time.sleep(random.uniform(150, 220))
                        pppoe_ip()
                else:
                    break
    return None
if __name__ == '__main__':
    print('抓取 日 asin')
    start_day_spider()