print('抓取 月 asin')
import os
import sys
from datetime import datetime, timedelta

sys.path.append(os.path.dirname(sys.path[0]))  # 上级目录
import time, random
import multiprocessing
from amazon_spider.asin_detail_pg import async_asin_pg
from amazon_spider.VPS_IP import pppoe_ip
from threading_spider.update_spdier_state import select_sate_mysql
from amazon_save_db.save_asin_detail_pg import Save_asin_detail  # 主进程读数据用
from threading_spider.Poll_site_spider import start_day_spider


def read_data_from_pg(save_detail_reader):
    """主进程：复用 Save_asin_detail.read_db_data()，保留 minid_maxid 分段读取"""
    asin_6field = save_detail_reader.read_db_data()
    if not asin_6field:
        return []
    # read_db_data() 执行后 save_detail_reader.df_read 里有 id 列
    # 组装成7字段格式：id|asin|date_info|is_variation|data_type|volume|weight_str
    df = save_detail_reader.df_read
    all_data = list(
        df.id.astype("U") + '|' + df.asin + '|' + df.date_info + '|' +
        df.asin_is_variation.astype("U") + '|' + df.data_type.astype("U") + '|' +
        df.volume.astype("U") + '|' + df.weight_str.astype("U"))
    print(f'主进程读取 {len(all_data)} 条ASIN')
    return all_data


def long_time_task(site, month, spider_int, data_chunk):
    """子进程：接收分发数据，不自己读数据库"""
    print(f"当前进程 {os.getpid()} 分配到 {len(data_chunk)} 条数据")
    print(f"开始=============== {site} asin 抓取 =======================")
    async_asin_us = async_asin_pg(site_name=site, month=month, spider_int=spider_int, skip_read_init=True)
    async_asin_us.run_with_data(data_chunk)


if __name__ == '__main__':
    pppoe_ip()
    site_list = ['uk','de']
    month = int(sys.argv[1])
    spider_int = 0
    week = int(sys.argv[2])
    if int(month) < 10:
        month = '0' + str(month)
    if week < 10:
        week = '0' + str(week)
    num_processes = 5
    day_spider_ran_date = None  # 记录当天是否已触发日任务，防止重复执行
    for site in site_list:
        if select_sate_mysql(site=site, week=week, spider_int=spider_int, month=month):
            # 主进程创建 reader（保留 minid_maxid 分段机制）
            reader = Save_asin_detail(site_name=site, month=month, spider_int=spider_int)
            reader.read_size = 1500  # 原来180，主进程一次读1500条分给子进程
            empty_count = 0
            while True:
                # ====== 凌晨2点切换日ASIN任务 ======
                _today = datetime.now().strftime('%Y-%m-%d')
                if 2 <= datetime.now().hour <= 5 and day_spider_ran_date != _today:
                    day_spider_ran_date = _today
                    print(f'==== {datetime.now()} 凌晨2点，执行日ASIN抓取 ====')
                    try:
                        start_day_spider()
                    except Exception as e:
                        print(f'日ASIN任务异常: {e}')
                    print(f'==== {datetime.now()} 日ASIN任务完成，继续月度任务 ====')
                # ====================================

                current_time = datetime.now()
                five_minutes_later = current_time + timedelta(minutes=2)
                print('后五分钟时间', five_minutes_later)

                # ====== 主进程统一读取数据（带 minid_maxid 分段 + FOR UPDATE SKIP LOCKED）======
                all_data = read_data_from_pg(reader)

                if not all_data:
                    empty_count += 1
                    # 分段区间内读空 → 清掉分段限制，下次走 read_db_data2() 全表扫
                    if reader.minid_maxid_list:
                        print(f'分段区间无数据，清除分段限制，尝试无分段读取')
                        reader.minid_maxid_list = []
                        continue  # 立即重试，不sleep
                    # 无分段也读空 → 可能数据被其他机器抢完了
                    if empty_count >= 3:
                        print(f'连续{empty_count}次读空，检查是否全部完成')
                        if select_sate_mysql(site=site, num=3, month=month, week=week,
                                             spider_int=spider_int) == False:
                            break  # 全部完成，退出
                        # 还没完 → 其他机器还在处理，sleep退让减少空转
                        time.sleep(60)
                        empty_count = 0
                        continue
                    time.sleep(10)
                    continue

                # ====== 读到数据，重置空计数 ======
                empty_count = 0

                # 动态调整进程数：数据少时减少进程避免浪费
                if len(all_data) < 300:
                    actual_processes = max(1, len(all_data) // 60)
                else:
                    actual_processes = num_processes

                # 平均分配给子进程
                chunks = [all_data[i::actual_processes] for i in range(actual_processes)]

                processes = []
                for idx in range(actual_processes):
                    if chunks[idx]:
                        process = multiprocessing.Process(
                            target=long_time_task,
                            args=(site, month, spider_int, chunks[idx]))
                        processes.append(process)
                        process.start()
                for process in processes:
                    process.join()

                print('所有进程运行完毕！开始切换ip')
                current_time = datetime.now()
                if current_time > five_minutes_later:
                    pppoe_ip()
                else:
                    time.sleep(random.uniform(150, 220))
                    pppoe_ip()

                if select_sate_mysql(site=site, num=3, month=month, week=week,
                                     spider_int=spider_int) == False:
                    break
