"""
@Description : ABA搜索词全历史出现中间表 - 每月增量更新脚本

@业务背景    :
    本中间表 dim_st_detail_history 服务于 dwt_aba_st_analytics.is_first_ever_text 字段判断
    （全历史首次出现）。每月 dim_st_detail 跑完后，通过本脚本把当月数据合并进中间表，
    保持中间表始终是"截止当月"的最新状态。

@数据流向    :
    dim_st_detail_history (M-1 状态) + dim_st_detail (当月分区)
        ↓ UNION + groupby search_term + MIN(first) / MAX(last)
    dim_st_detail_history (M 状态)

@调度位置    : 每月 dim_st_detail 跑完之后、dwt_aba_st_analytics 跑之前

@幂等性     :
    - 当月 dim 词集合不变 → 重跑结果完全一致
    - 当月 dim 词集合缩小（某词被剔除）→ 该词在中间表里被自动清除
    - 当月 dim 词集合扩大（多了新词）→ 新词被正确追加

@SourceTable : dim_st_detail (date_type='month', date_info=当月)
                + dim_st_detail_history (本表自身，读取历史状态)
@SinkTable   : dim_st_detail_history
@CreateTime  : 2026-05-07
@Param       : site_name (us / uk / de) + date_info (YYYY-MM, 当月)
"""
import os
import sys

sys.path.append(os.path.dirname(sys.path[0]))

from utils.hdfs_utils import HdfsUtils
from utils.common_util import CommonUtil
from utils.spark_util import SparkUtil
from pyspark.sql import functions as F


class DimStDetailHistory(object):

    def __init__(self, site_name, date_info):
        self.site_name = site_name
        self.date_info = date_info
        self.hive_tb = "dim_st_detail_history"
        self.hdfs_path = f"/home/{SparkUtil.DEF_USE_DB}/dim/{self.hive_tb}/site_name={self.site_name}"
        self.partitions_by = ['site_name']
        app_name = f"{self.hive_tb}: {site_name}, {date_info}"
        self.spark = SparkUtil.get_spark_session(app_name)

        # 占位 DataFrame
        self.df_history = self.spark.sql("select 1+1;")
        self.df_current_month = self.spark.sql("select 1+1;")
        self.df_save = self.spark.sql("select 1+1;")

    def run(self):
        # 启动校验：防止误传历史月份破坏中间表
        self.check_rollback()
        # 读取中间表历史数据 + dim 当月数据
        self.read_data()
        # UNION + GROUP BY 合并
        self.handle_data()
        # 写入中间表（覆盖 site_name 分区）
        self.save_data()

    def check_rollback(self):
        # 中间表 max(last) 应该 ≤ 当前传入的 date_info
        # 如果 > 当前 date_info，说明中间表已包含未来月份数据 → 判断为回溯历史月
        # 直接跑增量会让 first=当月 的少量词的 last 字段被错误重置 → 走 init 重建更安全
        check_sql = f"""
            SELECT MAX(date_info_last) AS max_last
            FROM {self.hive_tb}
            WHERE site_name = '{self.site_name}'
        """
        max_last = self.spark.sql(check_sql).collect()[0]['max_last']
        if max_last is not None and max_last > self.date_info:
            raise Exception(
                f"中间表 max(date_info_last)={max_last} > 当前 {self.date_info}，"
                f"判断为回溯历史月场景。\n"
                f"请改跑 dim_st_detail_history_init.py 重建中间表。"
            )
        print(f"[校验通过] 中间表 max(date_info_last)={max_last} ≤ 当前 {self.date_info}")

    def read_data(self):
        # 分支1的源：中间表里 first ≠ 当月 的所有词
        # 用 != 而不是 <：误操作传历史月时数据损失最小
        # （比如当前 2026-05 误传 2026-04，!= 只排除 first=2026-04 的少数词，
        #   而 < 会排除 first=2026-04 + first=2026-05 一大批词）
        history_sql = f"""
            SELECT
                search_term,
                date_info_first,
                date_info_last
            FROM {self.hive_tb}
            WHERE site_name = '{self.site_name}'
              AND date_info_first != '{self.date_info}'
        """
        print(f"\n[读中间表历史 SQL]\n{history_sql}")
        self.df_history = self.spark.sql(history_sql)

        # 分支2的源：dim 当月所有词（每词去重一次，因为 dim_st_detail 当月分区每词应该只一行，
        # 但 distinct 防御一下避免脏数据）
        current_sql = f"""
            SELECT DISTINCT search_term
            FROM dim_st_detail
            WHERE site_name = '{self.site_name}'
              AND date_type = 'month'
              AND date_info = '{self.date_info}'
        """
        print(f"\n[读 dim 当月 SQL]\n{current_sql}")
        self.df_current_month = self.spark.sql(current_sql)

    def handle_data(self):
        # 分支2：当月词加上占位 first=last=当月
        # 这样 UNION 后 GROUP BY MIN/MAX 时：
        #   - 老词：分支1 (first=老月份) UNION 分支2 (first=当月) → MIN=老月份不变 ✓
        #          分支1 (last=之前月份) UNION 分支2 (last=当月) → MAX=当月（更新成功）✓
        #   - 新词：只在分支2 → first=last=当月 ✓
        #   - 当月被 dim 剔除的词：既不在分支1（被 != 过滤）也不在分支2（dim 没了）→ 自动消失 ✓
        df_branch2 = self.df_current_month.select(
            'search_term',
            F.lit(self.date_info).alias('date_info_first'),
            F.lit(self.date_info).alias('date_info_last')
        )

        # UNION + GROUP BY 聚合
        df_union = self.df_history.unionByName(df_branch2)
        self.df_save = df_union.groupBy('search_term').agg(
            F.min('date_info_first').alias('date_info_first'),
            F.max('date_info_last').alias('date_info_last')
        )

    def save_data(self):
        # 加分区列（saveAsTable + partitionBy 需要 DataFrame 里有该列）
        self.df_save = self.df_save.withColumn('site_name', F.lit(self.site_name))

        # repartition 控制文件数（与 init 一致策略：us 5 个、uk/de 3 个）
        target_partitions = 15 if self.site_name == 'us' else 10
        self.df_save = self.df_save.repartition(target_partitions)
        print(f"\n[repartition] 目标文件数: {target_partitions}")

        # 关键：先 cache + count 强制把数据物化到内存
        # 因为本脚本"自读自写"（读 dim_st_detail_history → 写 dim_st_detail_history），
        # 必须在 delete_file_in_folder 之前完成对源数据的读取，否则 Spark 的 lazy execution
        # 会让 saveAsTable 时才去读累加表，但此时累加表已被 delete 删空，导致历史数据丢失
        self.df_save = self.df_save.cache()
        row_count = self.df_save.count()
        print(f"\n[已物化] 待写入行数: {row_count}, 目标文件数: {target_partitions}")
        
        # delete_file_in_folder + saveAsTable(format='hive', mode='append')：项目主流写法
        # delete_file_in_folder 删文件保目录，避免 Hive 找不到目录的 WARN
        # format='hive' 让 Spark 按表 SerDe（LZO 文本）自动写入
        # mode='append' 配合前面的 delete_file_in_folder 实现"覆盖整个 site_name 分区"语义
        print(f"\n清除分区文件: {self.hdfs_path}")
        HdfsUtils.delete_file_in_folder(self.hdfs_path)
        print(f"写入表: {self.hive_tb}, 分区: {self.partitions_by}")
        self.df_save.write.saveAsTable(
            name=self.hive_tb,
            format='hive',
            mode='append',
            partitionBy=self.partitions_by
        )
        self.df_save.unpersist()
        print("[写入完成]")

        # 验证
        verify_sql = f"""
            SELECT
                COUNT(*) AS row_count,
                COUNT(DISTINCT search_term) AS distinct_terms,
                MIN(date_info_first) AS earliest_first,
                MAX(date_info_last) AS latest_last
            FROM {self.hive_tb}
            WHERE site_name = '{self.site_name}'
        """
        print(f"\n[验证 SQL]\n{verify_sql}")
        self.spark.sql(verify_sql).show(truncate=False)
        print(f"\n[完成] 增量更新 dim_st_detail_history site_name={self.site_name}, date_info={self.date_info}")


if __name__ == '__main__':
    site_name = CommonUtil.get_sys_arg(1, None)
    date_info = CommonUtil.get_sys_arg(2, None)
    if not site_name or not date_info:
        print("用法: spark-submit dim_st_detail_history.py <site_name> <date_info>")
        print("示例: spark-submit dim_st_detail_history.py us 2026-05")
        sys.exit(1)

    print(f"{'=' * 60}")
    print(f"开始增量更新中间表 dim_st_detail_history")
    print(f"site_name = {site_name}, date_info = {date_info}")
    print(f"{'=' * 60}")

    obj = DimStDetailHistory(site_name=site_name, date_info=date_info)
    obj.run()
