import os
import sys
from datetime import datetime, timedelta
sys.path.append(os.path.dirname(sys.path[0]))  # 上级目录
import multiprocessing
from amazon_spider.sp_serch_term_spider import search_temp_pg
import time
import random
from amazon_spider.VPS_IP import pppoe_ip
from utils.db_connect import BaseUtils
import redis
from amazon_params.params import REDIS_CONN, DORIS_ADV_DIRECT_CONN
import pymysql




def read_data_with_lock(site, read_size=500):
    """主进程：Redis分布式锁 + 直连Doris读取数据 + 更新state=2"""
    try:
        r = redis.Redis(
            host=REDIS_CONN['redis_host'],
            port=REDIS_CONN['redis_port'],
            password=REDIS_CONN['redis_pwd'],
            db=REDIS_CONN['redis_db']
        )
    except Exception as e:
        print(f'Redis连接失败: {e}')
        return []

    lock_key = f'lock:sp_search_term_read:{site}'
    lock_value = f'{os.getpid()}_{time.time()}'
    lock_acquired = False

    # 加锁，120秒过期
    for _ in range(60):
        try:
            if r.set(lock_key, lock_value, nx=True, ex=120):
                lock_acquired = True
                break
        except Exception as e:
            print(f'Redis加锁异常: {e}')
            return []
        print('等待其他VPS释放锁...')
        time.sleep(2)
    else:
        print('等待锁超时，跳过本轮')
        return []

    conn = None
    try:
        conn = pymysql.connect(**DORIS_ADV_DIRECT_CONN)
        cursor = conn.cursor()
        table = f'{site}_sp_search_term_syn'

        # SELECT
        sql_select = f'SELECT id, search_term, url, time_batch FROM {table} WHERE state=1 ORDER BY id ASC LIMIT {read_size}'
        print(sql_select)
        cursor.execute(sql_select)
        rows = cursor.fetchall()
        if not rows:
            print('没有 state=1 的数据了')
            return []

        # UPDATE state=2
        ids = [str(row[0]) for row in rows]
        print(len(ids), '#' * 100)
        id_str = ','.join(ids)
        sql_update = f'UPDATE {table} SET state=2 WHERE id IN ({id_str})'
        cursor.execute(sql_update)
        conn.commit()
        print(f'直连Doris读取 {len(rows)} 条，已更新state=2')

        # 组装成 search_term_list 格式
        search_term_list = [f'{row[0]}|-|{row[1]}|-|{row[2]}|-|{row[3]}' for row in rows]
        return search_term_list
    except Exception as e:
        print(f'读取数据失败: {e}')
        return []
    finally:
        # 关闭数据库连接
        if conn:
            try:
                conn.close()
            except:
                pass
        # 释放锁（只释放自己的锁）
        if lock_acquired:
            try:
                if r.get(lock_key) == lock_value.encode():
                    r.delete(lock_key)
            except Exception as e:
                print(f'释放Redis锁失败（不影响主流程）: {e}')


def long_time_task(site, data_chunk):
    """子进程：用分配到的数据抓取"""
    print(f"当前进程 {os.getpid()} 分配到 {len(data_chunk)} 条数据")
    spider_us = search_temp_pg(site_name=site, read_size=100)
    spider_us.run_pol_with_data(data_chunk)


if __name__ == '__main__':
    pppoe_ip()
    site_list = ['us']
    num_processes = 5
    start_time = datetime.now()
    for site in site_list:
        while True:
            try:
                current_time = datetime.now()
                five_minutes_later = current_time + timedelta(minutes=1)
                print('后五分钟时间', five_minutes_later)

                # 主进程统一读取数据
                all_data = read_data_with_lock(site, read_size=1500)
                if not all_data:
                    print('没有数据，等待60秒后重新检查...')
                    time.sleep(60)
                    continue

                # 平均分配给子进程
                chunks = [all_data[i::num_processes] for i in range(num_processes)]

                processes = []
                for idx in range(num_processes):
                    if chunks[idx]:
                        process = multiprocessing.Process(target=long_time_task, args=(site, chunks[idx]))
                        processes.append(process)
                        process.start()
                for process in processes:
                    process.join()

                print('所有进程运行完毕！')
                current_time = datetime.now()
                if current_time > five_minutes_later:
                    pppoe_ip()
                else:
                    time.sleep(random.uniform(150, 220))
                    pppoe_ip()

            except Exception as e:
                print(f'[主循环异常] 等待30秒后重试: {e}')
                import traceback
                traceback.print_exc()
                time.sleep(30)
                continue
