"""
author: CT
description: Keepa 数据聚合 — 一站式 Hive → Hive + Doris
             步骤：
               1) 读 Hive ods_keepa_asin_detail 当日分区，解析 last_detail JSON 各字段
                  派生 keepa_launch_time = min(listed_since, tracking_since) 转 yyyy-MM-dd HH:mm:ss
                  weight 字段已废弃置 NULL（Doris 端不再保留）
               2) 与 Hive 历史 dim_keepa_asin_info union 按 asin 去重保留 updated_time 最新
               3) 写入 Hive dim_keepa_asin_info（当日分区）+ 删除所有 date_info < 今日的历史分区
               4) 当日新数据（不含历史）写入 Doris dwd.dwd_keepa_asin_detail
                  Doris UNIQUE KEY(site_name, asin) + sequence_col=updated_time 自动取最新
执行示例: spark-submit dim_keepa_asin_info.py us 2026-05-15
"""
import os
import sys

sys.path.append(os.path.dirname(sys.path[0]))

from utils.spark_util import SparkUtil
from pyspark.sql import functions as F, Window
from utils.common_util import CommonUtil
from utils.DorisHelper import DorisHelper
from utils.hdfs_utils import HdfsUtils
from datetime import datetime, timedelta


class DimKeepaAsinInfo(object):

    def __init__(self, site_name, date_info):
        self.site_name = site_name
        self.date_info = date_info
        self.last_date_info = (datetime.strptime(date_info, "%Y-%m-%d").date() - timedelta(days=1)).strftime("%Y-%m-%d")
        self.spark = SparkUtil.get_spark_session(f"{self.__class__.__name__}: {self.site_name} {self.date_info}")

        self.df_keepa_asin = self.spark.sql(f"select 1+1;")
        self.df_keepa_asin_history = self.spark.sql(f"select 1+1;")
        self.df_save = self.spark.sql(f"select 1+1;")

        # doris相关配置
        self.doris_db = "dwd"
        self.doris_table = "dwd_keepa_asin_detail"
        self.df_to_doris = self.spark.sql(f"select 1+1;")

    def run(self):
        self.read_data()
        self.handle_data()
        self.save_data()

    def read_data(self):
        sql = f"""
            select asin, last_detail, update_at as updated_time from ods_keepa_asin_detail 
            where site_name = '{self.site_name}' and date_info = '{self.date_info}' and last_detail is not null;
        """
        self.df_keepa_asin = self.spark.sql(sqlQuery=sql).repartition(40, 'asin').select(
            F.col('asin'),
            F.get_json_object("last_detail", "$.packageLength").cast("int").alias("package_length"),
            F.get_json_object("last_detail", "$.packageWidth").cast("int").alias("package_width"),
            F.get_json_object("last_detail", "$.packageHeight").cast("int").alias("package_height"),
            F.get_json_object("last_detail", "$.packageWeight").cast("int").alias("package_weight"),
            F.get_json_object("last_detail", "$.itemWeight").cast("int").alias("item_weight"),
            F.get_json_object("last_detail", "$.listedSince").cast("int").alias("listed_since"),
            F.get_json_object("last_detail", "$.releaseDate").cast("int").alias("release_date"),
            F.get_json_object("last_detail", "$.trackingSince").cast("int").alias("tracking_since"),
            F.get_json_object("last_detail", "$.productType").cast("int").alias("product_type"),
            F.get_json_object("last_detail", "$.title").alias("title"),
            F.col('updated_time')
        ).withColumn(
            # weight 字段已废弃（Doris 端 dwd_keepa_asin_detail 不再保留），Hive 兼容 schema 保留列但置 NULL
            'weight', F.lit(None).cast('int')
        ).withColumn(
            "min_time", F.least(
                F.when(F.col("listed_since") != 0, F.col("listed_since")),
                F.when(F.col("tracking_since") > 0, F.col("tracking_since"))
            )
        ).withColumn(
            "keepa_launch_time",
            F.when(
                F.col("min_time").isNull(), F.lit(None)
            ).otherwise(
                F.date_format(F.from_unixtime((F.col("min_time") + F.lit(21564000)) * 60), "yyyy-MM-dd HH:mm:ss")
            )
        ).select(
            F.col('asin'),
            F.col('package_length'),
            F.col('package_width'),
            F.col('package_height'),
            F.col('package_weight'),
            F.col('item_weight'),
            F.col('weight'),
            F.col('keepa_launch_time'),
            F.col('updated_time'),
            F.col('listed_since'),
            F.col('release_date'),
            F.col('tracking_since'),
            F.col('product_type'),
            F.col('title')
        ).filter(
            # 过滤脏数据：productType in (3,4,5) 且 title 为空的异常数据不做保留
            ~(F.col("product_type").isin(3, 4, 5) & F.col("title").isNull())
        ).cache()
        # 写入 Doris 需带 site_name 分区字段，并把 keepa_launch_time / updated_time 转为 DATETIME（DDL 类型已改）
        # weight 字段在 Doris dwd_keepa_asin_detail 已废弃，select 不带
        self.df_to_doris = self.df_keepa_asin.select(
            F.lit(self.site_name).alias('site_name'),
            'asin', 'package_length', 'package_width', 'package_height', 'package_weight', 'item_weight',
            'listed_since', 'release_date', 'tracking_since',
            F.to_timestamp(F.col('keepa_launch_time')).alias('keepa_launch_time'),
            F.to_timestamp(F.col('updated_time')).alias('updated_time')
        )

        # 读取历史数据
        sql = f"""
            select asin, package_length, package_width, package_height, package_weight, item_weight, weight, keepa_launch_time, updated_time, listed_since, release_date, tracking_since, product_type, title
            from dim_keepa_asin_info where site_name = '{self.site_name}';
        """
        self.df_keepa_asin_history = self.spark.sql(sqlQuery=sql).repartition(40, 'asin').cache()

    def handle_data(self):
        # 去重
        self.df_save = self.df_keepa_asin.unionByName(self.df_keepa_asin_history)
        window = Window.partitionBy(['asin']).orderBy(
            self.df_save.updated_time.desc_nulls_last()
        )
        self.df_save = self.df_save.withColumn(
            'rank', F.row_number().over(window=window)
        ).filter('rank = 1').drop('rank')

    def save_data(self):
        self.df_save = self.df_save.withColumn(
            "site_name", F.lit(self.site_name)
        ).withColumn(
            "date_info", F.lit(self.date_info)
        ).repartition(40).cache()
        new_count = self.df_save.count()
        old_count = self.df_keepa_asin_history.count()
        hive_tb = "dim_keepa_asin_info"
        partition_dict = {
            "site_name": self.site_name,
            "date_info": self.date_info
        }
        partition_by = list(partition_dict.keys())
        hdfs_path = CommonUtil.build_hdfs_path(hive_tb, partition_dict)
        print(f"正在进行数据存储，当前存储的表名为：{hive_tb}，存储路径：{hdfs_path}")
        self.df_save.write.saveAsTable(name=hive_tb, format='hive', mode='append', partitionBy=partition_by)
        print("success!")

        if new_count >= old_count:
            # 删除所有 date_info < 当前 date_info 的历史分区（兼容间隔几天才执行的场景）
            print(f"正在删除所有 date_info < {self.date_info} 的历史分区数据")
            partitions = self.spark.sql(f"SHOW PARTITIONS {hive_tb}").collect()
            for row in partitions:
                # row[0] 形如 "site_name=us/date_info=2026-05-10"
                parts = dict(p.split('=') for p in row[0].split('/'))
                p_site = parts.get('site_name')
                p_date = parts.get('date_info')
                if p_site == self.site_name and p_date < self.date_info:
                    print(f"  删除分区: site_name={p_site}, date_info={p_date}")
                    self.spark.sql(
                        f"ALTER TABLE {hive_tb} DROP IF EXISTS PARTITION (site_name='{p_site}', date_info='{p_date}')"
                    )
                    HdfsUtils.delete_hdfs_file(
                        CommonUtil.build_hdfs_path(hive_tb, partition_dict={"site_name": p_site, "date_info": p_date})
                    )
            print("历史分区清理完毕!")

        # 写入Doris表
        print("往doris存储最新keepa详情信息：")
        doris_table_columns = """
        site_name, asin, package_length, package_width, package_height, package_weight, item_weight,
        listed_since, release_date, tracking_since, keepa_launch_time, updated_time
        """
        DorisHelper.spark_export_with_columns(
            df_save=self.df_to_doris,
            db_name=self.doris_db,
            table_name=self.doris_table,
            table_columns=doris_table_columns
        )
        print("success!")


if __name__ == "__main__":
    site_name = sys.argv[1]
    date_info = sys.argv[2]
    handle_obj = DimKeepaAsinInfo(site_name, date_info)
    handle_obj.run()
