"""
author: CT
description: 同步 Hive tmp_jm_info 竞卖/SKU 数据到 Doris dwd.dwd_asin_auction
             按 asin UPSERT，sequence_col=updated_at 控制旧数据不覆盖新数据
执行示例: spark-submit doris_update_auction.py
"""
import os
import sys

sys.path.append(os.path.dirname(sys.path[0]))

from pyspark.sql import functions as F

from utils.spark_util import SparkUtil
from utils.DorisHelper import DorisHelper

DORIS_DB    = 'dwd'
DORIS_TABLE = 'dwd_asin_auction'

if __name__ == '__main__':
    spark = SparkUtil.get_spark_session('doris_update_auction')

    sql = """
        SELECT asin,
               CAST(auctions_num       AS INT) AS auctions_num,
               CAST(auctions_num_all   AS INT) AS auctions_num_all,
               CAST(skus_num_creat     AS INT) AS skus_num_creat,
               CAST(skus_num_creat_all AS INT) AS skus_num_creat_all
        FROM big_data_selection.tmp_jm_info
    """
    print(f"读取 Hive: {sql}")
    df = spark.sql(sqlQuery=sql).repartition(40)
    df = df.na.fill({
        "auctions_num":       0,
        "auctions_num_all":   0,
        "skus_num_creat":     0,
        "skus_num_creat_all": 0,
    })
    df = df.withColumn("updated_at", F.current_timestamp()).cache()

    count = df.count()
    print(f"写入数据量：{count:,}")
    df.show(10, truncate=False)

    table_columns = "asin, auctions_num, auctions_num_all, skus_num_creat, skus_num_creat_all, updated_at"
    DorisHelper.spark_export_with_columns(
        df_save=df,
        db_name=DORIS_DB,
        table_name=DORIS_TABLE,
        table_columns=table_columns,
    )

    df.unpersist()
    print('success！')
