import os
import sys

sys.path.append(os.path.dirname(sys.path[0]))

from utils.hdfs_utils import HdfsUtils
from utils.common_util import CommonUtil
from utils.spark_util import SparkUtil
from pyspark.sql import functions as F


class DimStDetailHistoryInit(object):

    def __init__(self, site_name):
      self.site_name = site_name
      self.hive_tb = "dim_st_detail_history"
      # 累加表 HDFS 路径，用于 delete_file_in_folder 清空目标分区文件
      self.hdfs_path = f"/home/{SparkUtil.DEF_USE_DB}/dim/{self.hive_tb}/site_name={self.site_name}"
      self.partitions_by = ['site_name']
      app_name = f"{self.hive_tb}_init: {site_name}"
      self.spark = SparkUtil.get_spark_session(app_name)

    def run(self):
      # 第一步：聚合（用默认 200 partitions 做 GROUP BY，保证聚合性能）
      aggregate_sql = f"""
          SELECT
              search_term,
              MIN(date_info) AS date_info_first,
              MAX(date_info) AS date_info_last
          FROM dim_st_detail
          WHERE site_name = '{self.site_name}'
            AND date_type = 'month'
          GROUP BY search_term
      """
      print(f"\n[聚合 SQL]\n{aggregate_sql}")
      df = self.spark.sql(aggregate_sql)

      # 第二步：加分区列（saveAsTable + partitionBy 需要 DataFrame 里有该列）
      df = df.withColumn('site_name', F.lit(self.site_name))

      # 第三步：repartition 控制最终输出文件数，避免小文件
      # us 数据量大（~1183万行）→ 5 个文件；uk/de 数据量小（~350万行）→ 3 个文件
      target_partitions = 5 if self.site_name == 'us' else 3
      df = df.repartition(target_partitions)
      print(f"\n[repartition] 目标文件数: {target_partitions}")

      # 第四步：清空目标分区文件 + saveAsTable 写入
      # delete_file_in_folder：删文件保目录，避免 Hive 后续找不到目录报 WARN
      # saveAsTable(format='hive', mode='append'): 项目主流写法
      #   format='hive' 让 Spark 按表的 SerDe 定义写入（这里是 LZO 文本格式），不会强制写 ORC
      #   mode='append' 配合 delete_file_in_folder 实现"覆盖"语义（项目通用约定）
      print(f"\n清除分区文件: {self.hdfs_path}")
      HdfsUtils.delete_file_in_folder(self.hdfs_path)
      print(f"写入表: {self.hive_tb}, 分区: {self.partitions_by}")
      df.write.saveAsTable(
          name=self.hive_tb,
          format='hive',
          mode='append',
          partitionBy=self.partitions_by
      )
      print(f"[写入完成]")

      # 第五步：验证
      verify_sql = f"""
          SELECT
              COUNT(*) AS row_count,
              COUNT(DISTINCT search_term) AS distinct_terms,
              MIN(date_info_first) AS earliest_first,
              MAX(date_info_last) AS latest_last
          FROM {self.hive_tb}
          WHERE site_name = '{self.site_name}'
      """
      print(f"\n[验证 SQL]\n{verify_sql}")
      self.spark.sql(verify_sql).show(truncate=False)
      print(f"\n[完成] init dim_st_detail_history site_name={self.site_name}")


if __name__ == '__main__':
    site_name = CommonUtil.get_sys_arg(1, None)
    if not site_name:
      print("用法: spark-submit dim_st_detail_history_init.py <site_name>")
      print("示例: spark-submit dim_st_detail_history_init.py us")
      sys.exit(1)

    print(f"{'=' * 60}")
    print(f"开始初始化累加表 dim_st_detail_history")
    print(f"site_name = {site_name}")
    print(f"{'=' * 60}")

    obj = DimStDetailHistoryInit(site_name=site_name)
    obj.run()
