import os
import sys
from datetime import datetime, timedelta

sys.path.append(os.path.dirname(sys.path[0]))  # 上级目录
import multiprocessing
from amazon_spider.search_term_pg import search_temp_pg
from threading_spider.db_connectivity import connect_db
import time
import random
from amazon_spider.VPS_IP import pppoe_ip
from threading_spider.post_to_dolphin import DolphinschedulerHelper
from utils.secure_db_client import get_remote_engine
from utils.db_connect import BaseUtils
from amazon_params.params import DB_SEARCH_TERM_PARAMS_SPIDER

def db_engine(site_name, db_type):
    """
    "mysql": "mysql",  # 阿里云mysql
    "postgresql_14": "postgresql_14",  # pg14爬虫库-内网
    "postgresql_14_outer": "postgresql_14_outer",  # pg14爬虫库-外网
    "postgresql_15": "postgresql_15",  # pg15正式库-内网
    "postgresql_15_outer": "postgresql_15_outer",  # pg15正式库-外网
    "postgresql_cluster": "postgresql_cluster",  # pg集群-内网
    "postgresql_cluster_outer": "postgresql_cluster_outer",  # pg集群-外网
    "doris": "doris",  # doris集群-内网
    """
    engine_mysql = get_remote_engine(
        site_name=site_name,  # -> database "selection"
        db_type=db_type,  # -> 服务端 alias "mysql"
    )
    return engine_mysql


def select_search_term_state(month, site):
    # Bug1修复：return 1 移到 for 循环外，确保5次重试都能执行
    for i in range(5):
        try:
            sql_read = f"SELECT id FROM {site}_search_term_month_syn where state in (1,2) and date_info='2026-{month}' LIMIT 1"
            engine_pg = db_engine(site, 'postgresql_14_outer')
            df = engine_pg.read_sql(sql_read)
            if not df.empty:
                id_tuple = [1]
            else:
                id_tuple = None
            return id_tuple
        except Exception as e:
            print(e, '报错11。', sql_read)
            time.sleep(3)
    return 1  # 5次全部失败才返回1


def db_cursor_connect_update(sql, site):
    for i in range(3):
        try:
            engine_us_mysql = db_engine('us', 'mysql')
            print('更新sql：', sql)
            with engine_us_mysql.begin() as conn:
                conn.execute(sql)
            break
        except:
            print(site, 'db_cursor_connect 报错：', sql)


def select_sate_mysql(site, num=None, month=None, week=None):
    db_class = connect_db(site)
    print('month::', month)
    if num == 1:
        # Bug2修复：先抢锁，再在锁内查DB状态，避免多台机器同时通过判断
        redis_client = BaseUtils().redis_db()
        lock_key = "ALL站点-asin同步-pg-api_lock"
        lock = redis_client.lock(lock_key, timeout=60)
        if not lock.acquire(blocking=False):
            print(f'{site} 站点：其他机器已抢到锁，跳过调度通知')
            return False

        try:
            sql_select_ = f"select status_val from workflow_progress where date_info='2026-{week}' and date_type='week' and page='反查搜索词' and site_name='{site}'"
            print(sql_select_)
            engine_us_mysql = db_engine('us', 'mysql')
            df = engine_us_mysql.read_sql(sql_select_)
            # Bug3修复：df为空时不再抛 IndexError
            if df.empty or int(df.status_val[0]) not in (1, 2):
                print('5555555555555555555555555555555555')
                lock.release()
                return False
        except Exception as e:
            print(f'查询 workflow_progress 失败: {e}')
            lock.release()
            return False

        update_workflow_progress = f"update workflow_progress set status_val=3,status='抓取结束' where page='反查搜索词' and date_info='2026-{week}' and site_name='{site}' and date_type='week'"
        print('update_workflow_progress: 修改状态3 ', update_workflow_progress)
        db_cursor_connect_update(update_workflow_progress, site)
        lock.release()

        account = 'pengyanbing'
        title = site + '站点 搜索词'
        content = f'{month} 月 搜索词 已结束,请确认下一步流程!时间:'
        db_class.send_mg(account, title, content)

        ii = 0
        id_tuple = None
        for i in range(11):
            id_tuple = select_search_term_state(month, site)
            time.sleep(200)
            if id_tuple is None:
                ii += 1
                if ii > 8:
                    break

        if id_tuple is None:
            DolphinschedulerHelper.start_process_instance_common(
                project_name="big_data_selection",
                process_df_name='ALL站点-asin同步-pg-api',
                startParams={
                    "site_name": f"{site}",
                    "date_type": "month",
                    "date_info": f'2026-{month}'
                }
            )
            account = 'pengyanbing,chenyuanjie,hezhe,wangrui4,fangxingjun,chenjianyun,zhouyuchen'
            title = site + '站点 搜索词'
            content = f'{month} 月 搜索词 已结束,成功调度 ALL站点-asin同步-pg-api'
            db_class.send_mg(account, title, content)
            return True

    if num == 3:
        # 搜索词多进程已经抓完。最后执行单进程抓取。
        id_tuple = select_search_term_state(month, site)
        if id_tuple is None:
            select_sate_mysql(site, num=1, month=month, week=week)
            return False
        else:
            return True


def read_data_from_pg(site, month, read_size=1500):
    """主进程：走代理PG read_then_update（含FOR UPDATE）读取数据"""
    try:
        engine_pg = db_engine(site, 'postgresql_14_outer')
        date_info = f'2026-{month}'
        db_search_term = site + DB_SEARCH_TERM_PARAMS_SPIDER["db_search_term"][2:] + '_month_syn'
        sql_read = f"SELECT id, search_term, url FROM {db_search_term} where state=1 and date_info='{date_info}' LIMIT {read_size} for update"
        print(sql_read)
        df_read = engine_pg.read_then_update(
            select_sql=sql_read,
            update_table=db_search_term,
            set_values={"state": 2},
            where_keys=["id"],
        )
        if df_read.shape[0] > 0:
            search_term_list = list(
                df_read.id.astype("U") + '|-|' + df_read.search_term + '|-|' + df_read.url)
            print(f'读取 {len(search_term_list)} 条，已更新state=2')
            return search_term_list
        else:
            print('没有 state=1 的数据了')
            return []
    except Exception as e:
        print(f'读取数据失败: {e}')
        return []


def long_time_task(site, proxy_name, month, data_chunk):
    """子进程：用分配到的数据抓取"""
    print(f"当前进程 {os.getpid()} 分配到 {len(data_chunk)} 条数据")
    spider_us = search_temp_pg(site_name=site, read_size=100, proxy_name=proxy_name, month=month)
    spider_us.run_pol_with_data(data_chunk)


if __name__ == '__main__':
    pppoe_ip()
    # site_list = ['us', 'de', 'uk']
    site_list = ['de']
    num_processes = 5
    month = int(sys.argv[1])
    week = int(sys.argv[2])
    proxy_name = None
    if month < 10:
        month = '0' + str(month)
    if week < 10:
        week = '0' + str(week)
    print('抓取月：',month)
    for site in site_list:
        while True:
            current_time = datetime.now()
            one_minute_later = current_time + timedelta(minutes=1)  # Bug4修复：变量名与实际值对齐
            print('后一分钟时间', one_minute_later)

            # 主进程统一读取数据
            all_data = read_data_from_pg(site, month, read_size=1800)
            if not all_data:
                print('没有数据，检查是否全部完成')
                if select_sate_mysql(site, num=3, month=month, week=week) == False:
                    break
                else:
                    print('还有数据但读取失败，等待重试')
                    time.sleep(30)
                    continue

            # 平均分配给子进程
            chunks = [all_data[i::num_processes] for i in range(num_processes)]

            processes = []
            for idx in range(num_processes):
                if chunks[idx]:
                    process = multiprocessing.Process(target=long_time_task, args=(site, proxy_name, month, chunks[idx]))
                    processes.append(process)
                    process.start()
            for process in processes:
                process.join()

            print('所有进程运行完毕！')
            current_time = datetime.now()
            if current_time > one_minute_later:
                pppoe_ip()
            else:
                time.sleep(random.uniform(150, 220))
                pppoe_ip()
