import os
import sys
import time
import json

sys.path.append(os.path.dirname(sys.path[0]))  # 上级目录
from utils.templates import Templates
from pyspark.sql.window import Window
from pyspark.sql import functions as F
from pyspark.sql.types import IntegerType, StringType
from utils.db_util import DBUtil
from utils.spark_util import SparkUtil
from utils.common_util import CommonUtil
from yswg_utils.common_udf import udf_detect_phrase_reg


# ============================================================
# filter_refinements 维度黑名单（全部小写）
# 【这是什么】filter_refinements 是爬虫抓的"该搜索词市场的筛选维度"（JSON：维度名 -> 值列表），
#   里面混了品牌/平台/物流/交易类维度（如 Local Stores、Subscribe & Save），前端只想展示
#   材质/风格/功能/场景等商品属性维度，所以要按黑名单把非属性维度整条丢掉。
# 【匹配规则】维度名转小写后，只要"包含"下列任一关键词 → 整个维度连同其值一起丢弃（业务确认按包含匹配）。
# 【'brand' 这条】单独一条覆盖所有带品牌的写法（Brands / Sub-Brand / Premium Brands /
#   From Our Brands / All Top Brands / Top Brands / Top Brands in Health & Household 等），
#   等价于业务原始 22 项里那 6 个 brand 相关项，故不再逐条重复列。
# 【注意】'color'/'price'/'seller'/'condition' 也按包含丢（业务确认），会连带丢掉
#   Frame Color / Skin Condition 这类含该词的维度——这是预期行为。
# 【怎么加词】后续要新增过滤维度，只在本列表加一行小写关键词即可。
# ============================================================
FILTER_REFINEMENTS_BLACKLIST = [
    'brand',                              # 含 brand 全丢（覆盖原 22 项里 6 个 brand 相关项）
    'eligible for free shipping',
    'delivery day',
    'color',
    'price',
    'deals & discounts',
    'customer reviews',
    'amazon fashion',
    'care instructions',
    'seller',
    'customizable products',
    'subscribe & save',
    'handmade products',
    'condition',
    'amazon certified',
    'amazon beauty',
    'local stores',
]


class DwtAbaStAnalytics(Templates):

    def __init__(self, site_name="us", date_type="week", date_info="2022-40"):
        super().__init__()
        self.site_name = site_name
        self.date_type = date_type
        self.date_info = date_info
        self.db_save = f"dwt_aba_st_analytics"
        self.spark = self.create_spark_object(
            app_name=f"{self.db_save}: {self.site_name},{self.date_type}, {self.date_info}")

        # 写入、分区初始化
        self.df_save = self.spark.sql(f"select 1+1;")
        self.partitions_by = ['site_name', 'date_type', 'date_info']
        self.reset_partitions(partitions_num=40)

        # 初始化列表
        self.sp_symbols = []

        # 初始化全局df
        self.df_st_measure = self.spark.sql(f"select 1+1;")
        self.df_asin_measure = self.spark.sql(f"select 1+1;")
        self.df_st_asin_measure = self.spark.sql(f"select 1+1;")
        self.df_asin_detail = self.spark.sql(f"select 1+1;")
        self.df_seller_asin_info = self.spark.sql(f"select 1+1;")
        self.df_st_asin_cal = self.spark.sql(f"select 1+1;")
        self.df_st_asin_join = self.spark.sql(f"select 1+1;")
        self.df_seller_asin_country = self.spark.sql(f"select 1+1;")
        self.df_st_brand_cal = self.spark.sql(f"select 1+1;")
        self.df_top3_st_brand_cal = self.spark.sql(f"select 1+1;")
        self.df_st_seller_cal = self.spark.sql(f"select 1+1;")
        self.df_top3_st_seller_cal = self.spark.sql(f"select 1+1;")
        self.df_st_num_stats = self.spark.sql(f"select 1+1;")
        self.df_st_detail = self.spark.sql(f"select 1+1;")
        self.df_st_key = self.spark.sql(f"select 1+1;")
        self.df_st_market = self.spark.sql(f"select 1+1;")
        # self.df_st_volume_fba = self.spark.sql(f"select 1+1;")
        self.df_st_brand = self.spark.sql(f"select 1+1;")
        self.df_asin_label = self.spark.sql(f"select 1+1;")
        self.df_is_hidden_cate = self.spark.sql(f"select 1+1;")
        self.df_asin_profit_rate = self.spark.sql(f"select 1+1;")
        self.df_history_st = self.spark.sql(f"select 1+1;")  # 累加表 dim_st_detail_history 历史搜索词，用于 is_first_ever_text 判断
        # 搜索词属性标签 df，来源 dws_st_theme 聚合 theme_ch（材质/颜色/细分人群等）
        # 仅 month 流程在 read_data 阶段真正读取数据；非 month 流程下游统一用 lit("-1") 填充
        self.df_st_attribute = self.spark.sql(f"select 1+1;")
        # 自身历史 11 个月分区的 rank（峰值月 peak_month / 常年可卖 all_year_text_flag 计算用）
        # 仅 month 流程在 read_data 阶段真正读取数据；非 month 流程下游统一用占位值填充
        self.df_history_rank = self.spark.sql(f"select 1+1;")
        # 搜索词市场筛选维度 df，来源 ods_st_quantity_being_sold.filter_refinements（清洗后 JSON）
        # 仅 month 流程在 read_data 阶段真正读取数据；非 month 流程下游统一用 lit("{}") 填充
        self.df_st_filter = self.spark.sql(f"select 1+1;")

        # 自定义udf函数注册
        self.u_contains = self.spark.udf.register('u_contains', self.udf_contains, IntegerType())
        # filter_refinements 清洗 UDF：解析 JSON→按黑名单删维度→重序列化，返回干净 JSON 串
        self.u_clean_filter = self.spark.udf.register(
            'u_clean_filter', self.udf_clean_filter_refinements, StringType()
        )
        self.u_judge_color = self.spark.udf.register('u_judge_color', self.udf_judge_color, IntegerType())
        self.u_judge_title_color = self.spark.udf.register(
            'u_judge_title_color', self.udf_judge_title_color, IntegerType()
        )
        self.u_judge_multi_size = self.spark.udf.register(
            'u_judge_multi_size', self.udf_judge_multi_size, IntegerType()
        )

    # 解析aba搜索词的拆词个数
    def st_word_count(self, sp_symbols):
        def udf_st_word_count(name):
            # 特殊字符基准列表---迁移到数据库维护 -已处理
            # sp_symbols = ['?', '!', '-', '%', '&', '|']
            split_list = name.split(" ")
            # 取切割list中包含的特殊字符
            sp_list = list(set(split_list).intersection(set(sp_symbols)))
            # 排除掉特殊list中的特殊字符
            word_list = list(filter(lambda x: x not in sp_list, split_list))
            word_count = len(word_list)
            # 存在多个特殊字符都设定为 1
            if len(sp_list) > 0:
                symbol_count = 1
            else:
                symbol_count = 0
            return word_count + symbol_count

        return F.udf(udf_st_word_count, IntegerType())

    @staticmethod
    def udf_contains(sub, text):
        if text is None:
            return None
        if str(sub).lower() in str(text).lower():
            return 1
        else:
            return 0

    # @staticmethod
    # def udf_get_volume(volume):
    #     # print("get_volume", volume)
    #     volume = str(volume)
    #     if volume == "null":
    #         return None
    #     else:
    #         pattern = r"\d+\.?\d*"
    #         volumeList = re.findall(pattern, volume)
    #         if len(volumeList):
    #             volumeList = list(map(float, volumeList))
    #             result = reduce((lambda x, y: x * y), volumeList)
    #             return result
    #         else:
    #             return None

    @staticmethod
    def udf_judge_color(color):
        if color is None:
            return None
        color = str(color).lower()
        color_len = len(color)
        if color not in ['null', 'none'] and color_len > 1:
            return 1
        else:
            return 0

    @staticmethod
    def udf_judge_title_color(asin_title):
        if asin_title is None:
            return None
        title = str(asin_title).lower()
        modeTypes = ['colorful', 'assorted color', 'multi color']
        for color in modeTypes:
            if color in title:
                return 1
        return 0

    @staticmethod
    def udf_judge_multi_size(size, style):
        size = str(size).lower()
        style = str(style).lower()
        # 变体表中即有size又有style时，取size进行计数。如果无size，则判断是否有style进行计数
        if size not in ['none', 'null']:
            return 1
        else:
            if style not in ['none', 'null']:
                return 1
        return 0

    @staticmethod
    def udf_clean_filter_refinements(s):
        """
        清洗 filter_refinements：解析原始 JSON → 按黑名单丢维度 → 重序列化成干净 JSON 串。
        空 / 异常 / 过滤后无维度剩余 → 返回 None（下游 na.fill 补 "{}"）。
        """
        # ============================================================
        # Step1 解析：null / 空串 / 非法 JSON / 不是 dict 一律判空
        # 为什么全判空：这些都取不出"维度名->值列表"结构，直接交给下游补 "{}"
        # 例：s=None → None；s='' → None；s='abc'(非JSON) → None；s='[1,2]'(非dict) → None
        # ============================================================
        if s is None:
            return None
        s = s.strip()
        if s == '':
            return None
        try:
            obj = json.loads(s)
        except Exception:
            return None
        if not isinstance(obj, dict):
            return None

        # ============================================================
        # Step2 过滤：维度名转小写后"包含"任一黑名单词 → 整条维度(连同值)丢弃
        # 为什么转小写再判断：黑名单匹配忽略大小写，'Top Brands'/'top brands' 都要命中
        # 例：{'Flavor':[...],'Top Brands':[...],'Frame Color':[...]}
        #     → 'top brands' 含 'brand' 丢、'frame color' 含 'color' 丢 → 只留 {'Flavor':[...]}
        # ============================================================
        kept = {}
        for k, v in obj.items():
            kl = str(k).lower()
            if any(bad in kl for bad in FILTER_REFINEMENTS_BLACKLIST):  # 含任一黑名单词=非商品属性维度
                continue
            kept[k] = v

        # Step3 过滤后一个维度都不剩 → 判空（如整条 JSON 全是品牌/平台维度）
        if not kept:
            return None

        # ============================================================
        # Step4 重序列化成 JSON 串
        # ensure_ascii=False：保留原字符（维度名/值可能含非 ASCII），否则会变成 \uXXXX 转义
        # 不加 sort_keys：保持爬虫原始维度顺序（PG 端为 jsonb，键顺序本就会被 jsonb 重排，排序无意义）
        # ============================================================
        return json.dumps(kept, ensure_ascii=False)

    def read_data(self):
        # 一些不涵盖month_old的分区,重定义成month,其他正常
        spe_date_type = 'month' if 'month_old' == self.date_type else self.date_type

        # 获取ods_st_key, st唯一主键
        sql = f"""
        select 
            search_term,
            cast(st_key as int) as id 
        from ods_st_key 
        where site_name = '{self.site_name}' 
        """
        self.df_st_key = self.spark.sql(sqlQuery=sql)
        self.df_st_key = self.df_st_key.repartition(80, 'search_term').cache()
        print("self.df_st_key:")
        # self.df_st_key.show(10, truncate=True)

        # 获取dwd_st_measure 事实表
        sql = f"""
        select 
            search_term,
            st_zr_orders,
            st_bsr_orders,
            st_ao_val                     as st_ao_avg,
            st_ao_val_rate,
            st_zr_page1_title_appear_rate as page1_title_proportion,
            null                          as st_4_20_ao_avg,
            null                          as st_4_20_ao_rate,
            st_volume_avg                 as volume_avg,
            st_weight_avg                 as weight_avg,
            st_price_avg                  as price_avg,
            st_zr_page123_title_appear_rate,
            st_sp_page123_title_appear_rate,
            st_zr_flow_proportion,
            st_ao_val_matrix,
            st_flow_proportion_matrix,
            st_zr_counts,
            st_sp_counts,
            st_self_asin_counts,
            st_self_asin_proportion,
            st_dd50_proportion,
            st_dd100_proportion,
            st_dd200_proportion
        from dwd_st_measure
        where site_name = '{self.site_name}' 
        and date_type = '{self.date_type}' 
        and date_info = '{self.date_info}' 
        """
        self.df_st_measure = self.spark.sql(sqlQuery=sql)
        self.df_st_measure = self.df_st_measure.repartition(80, 'search_term').cache()
        print("self.df_st_measure:")
        self.df_st_measure.show(10, truncate=True)

        # 获取dwd_st_asin_measure 事实表
        sql = f"""
        select 
            search_term,
            asin 
        from dwd_st_asin_measure 
        where site_name = '{self.site_name}' 
        and date_type = '{self.date_type}' 
        and date_info = '{self.date_info}'
        """
        self.df_st_asin_measure = self.spark.sql(sqlQuery=sql)
        self.df_st_asin_measure = self.df_st_asin_measure.repartition(80, 'asin').cache()
        print("self.df_st_asin_measure:")
        # self.df_st_asin_measure.show(10, truncate=True)

        # 获取dwd_asin_measure 事实表
        sql = f"""
        select 
            asin,
            asin_bsr_orders,
            asin_zr_orders,
            asin_amazon_orders 
        from dwd_asin_measure 
        where site_name = '{self.site_name}' 
        and date_type = '{self.date_type}' 
        and date_info = '{self.date_info}' 
        """
        self.df_asin_measure = self.spark.sql(sqlQuery=sql)
        self.df_asin_measure = self.df_asin_measure.repartition(80, 'asin').cache()
        print("self.df_asin_measure:")
        # self.df_asin_measure.show(10, truncate=True)

        # 获取dim_asin_detail表
        sql = f"""
        select 
            asin, 
            asin_title, 
            asin_title_len, 
            asin_category_desc,
            asin_rank, 
            asin_color, 
            asin_size, 
            asin_style, 
            asin_price, 
            asin_rating,
            asin_total_comments, 
            asin_material, 
            asin_brand_name,
            bsr_cate_1_id, 
            asin_buy_box_seller_type, 
            asin_is_amazon,
            asin_is_fba, 
            asin_is_fbm, 
            asin_is_other, 
            asin_is_sale,
            asin_launch_time, 
            asin_is_new, 
            asin_img_num, 
            asin_img_type,
            asin_is_picture, 
            asin_is_video, 
            asin_is_aadd 
        from dim_asin_detail  
        where site_name = '{self.site_name}' 
        and date_type = '{spe_date_type}' 
        and date_info = '{self.date_info}' 
        """
        self.df_asin_detail = self.spark.sql(sqlQuery=sql)
        self.df_asin_detail = self.df_asin_detail.repartition(80, 'asin').cache()
        print("self.df_asin_detail:")
        # self.df_asin_detail.show(10, truncate=True)

        # 仅获取 asin和country_name,对country_name进行了聚合处理
        sql = f"""
        select 
            asin,
            concat_ws(\",\",collect_list(cast(fd_country_name as string))) as country_name 
        from dim_fd_asin_info  
        where site_name = '{self.site_name}'  
        group by asin;
        """
        self.df_seller_asin_country = self.spark.sql(sqlQuery=sql)
        self.df_seller_asin_country = self.df_seller_asin_country.repartition(80, 'asin').cache()
        print("self.df_seller_asin_country:")
        # self.df_seller_asin_country.show(10, truncate=True)

        # 获取 dim_fd_asin_info 表
        sql = f"""
        select 
            asin,
            fd_unique as account_id,
            fd_country_name as country_name 
        from dim_fd_asin_info 
        where site_name = '{self.site_name}' 
        """
        self.df_seller_asin_info = self.spark.sql(sqlQuery=sql)
        self.df_seller_asin_info = self.df_seller_asin_info.drop_duplicates(['asin']).repartition(80, 'asin').cache()
        print("self.df_seller_asin_info:")
        # self.df_seller_asin_info.show(10, truncate=True)

        # 获取 dim_st_detail asin1-3共享点击信息表
        sql = f"""
        select 
            search_term,
            st_rank                                                          as rank,
            st_asin1                                                         as asin1,
            st_asin2                                                         as asin2,
            st_asin3                                                         as asin3,
            st_click_share1                                                  as click_share1,
            st_click_share2                                                  as click_share2,
            st_click_share3                                                  as click_share3,
            st_click_share_sum                                               as total_click_share,
            st_is_new_market_segment                                         as is_new_market_segment,
            st_conversion_share1                                             as conversion_share1,
            st_conversion_share2                                             as conversion_share2,
            st_conversion_share3                                             as conversion_share3,
            st_conversion_share_sum                                          as total_conversion_share,
            st_quantity_being_sold                                           as quantity_being_sold,
            cast(st_bsr_cate_1_id as int)                                    as category_id,
            st_search_num                                                    as search_volume,
            st_is_first_text                                                 as is_first_text,
            st_is_ascending_text                                             as is_ascending_text,
            st_is_search_text                                                as is_search_text,
            cast(st_bsr_cate_current_id as int)                              as category_current_id,
            st_appear_history_counts                                         as st_num,
            cast((st_quantity_being_sold / st_search_num) as decimal(10, 3)) as supply_demand,
            st_brand1,
            st_category1,
            st_brand2,
            st_category2,
            st_brand3,
            st_category3,
            st_bsr_cate_1_id_new,
            st_bsr_cate_current_id_new,
            if(st_appear_history_counts>=4 and (st_click_share_sum > st_conversion_share_sum),1,0) as is_high_return_text,
            date_format(st_updated_time, 'yyyy-MM-dd HH:mm:ss')              as st_crawl_date,
            st_competition_level
        from dim_st_detail
        where site_name = '{self.site_name}' 
        and date_type = '{self.date_type}' 
        and date_info = '{self.date_info}'
        """
        self.df_st_detail = self.spark.sql(sqlQuery=sql)
        # 清洗品牌/类目字段里的 \x00（NUL 字节）  脏数据流到本表后，sqoop 导出 PG 会报错
        for col in ['st_brand1', 'st_brand2', 'st_brand3', 'st_category1', 'st_category2', 'st_category3']:
            self.df_st_detail = self.df_st_detail.withColumn(col, F.regexp_replace(F.col(col), '\x00', ''))
        self.df_st_detail = self.df_st_detail.repartition(80, 'search_term').cache()
        print("self.df_st_detail:")
        # self.df_st_detail.show(10, truncate=True)

        # 获取dws_st_num_stats表 取max_num、most_proportion
        sql = f""" 
        select 
            search_term,
            cast(max_num as double) as max_num,
            most_proportion,
            max_num_asin,
            is_self_max_num_asin 
        from dws_st_num_stats 
        where site_name = '{self.site_name}' 
        and date_type = '{self.date_type}' 
        and date_info = '{self.date_info}'
        and max_num_asin is not null 
        """
        self.df_st_num_stats = self.spark.sql(sqlQuery=sql)
        self.df_st_num_stats = self.df_st_num_stats.repartition(80, 'search_term').cache()
        print("self.df_st_num_stats:")
        # self.df_st_num_stats.show(10, truncate=True)

        # 获取dwt_st_market表 取market_cycle_type
        sql = f"""
        select 
            search_term,
            cast(market_cycle_type as int ) as market_cycle_type 
        from dwt_st_market 
        where site_name = '{self.site_name}' 
        and date_type = '{self.date_type}' 
        and date_info = '{self.date_info}'
        """
        self.df_st_market = self.spark.sql(sqlQuery=sql)
        self.df_st_market = self.df_st_market.repartition(80, 'search_term').cache()
        print("self.df_st_market:")
        # self.df_st_market.show(10, truncate=True)

        # 获取dwd_st_volume_fba 取gross_profit_fee_air 和 gross_profit_fee_sea
        # sql = f"""
        # select
        #     search_term,
        #     gross_profit_fee_air,
        #     gross_profit_fee_sea
        # from dwd_st_volume_fba
        # where site_name = '{self.site_name}'
        # and date_type = '{self.date_type}'
        # and date_info = '{self.date_info}'
        # """
        # self.df_st_volume_fba = self.spark.sql(sqlQuery=sql)
        # self.df_st_volume_fba = self.df_st_volume_fba.repartition(80, 'search_term').cache()
        # print("self.df_st_volume_fba:")
        # self.df_st_volume_fba.show(10, truncate=True)

        # 获取影视标签dim_asin_label 取 asin_label_type
        sql = f"""
        select 
            asin, 
            asin_label_type 
        from dim_asin_label 
        where site_name = '{self.site_name}' 
        and date_type = '{self.date_type}' 
        and date_info = '{self.date_info}' 
        """
        self.df_asin_label = self.spark.sql(sqlQuery=sql)
        self.df_asin_label = self.df_asin_label.repartition(80, 'asin').cache()
        print("self.df_asin_label:")
        # self.df_asin_label.show(10, truncate=True)

        # 获取品牌词库
        sql = f"""
        select 
            search_term,
            st_brand_label 
        from dws_st_brand_info
        where site_name = '{self.site_name}'
        and date_type = '{self.date_type}'
        and date_info = '{self.date_info}'
        and st_brand_label = 1
        """
        self.df_st_brand = self.spark.sql(sqlQuery=sql)
        self.df_st_brand = self.df_st_brand.repartition(80, 'search_term').cache()
        print("self.df_st_brand:")
        # self.df_st_brand.show(10, truncate=True)

        # 从pgsql获取特殊字符匹配字典表：match_character_dict
        pg_sql = f"""
        select 
            character_name 
        from match_character_dict 
        where match_type = '特殊字符'
        """
        conn_info = DBUtil.get_connection_info("mysql", "us")
        chart_dict_df = SparkUtil.read_jdbc_query(
            session=self.spark,
            url=conn_info["url"],
            pwd=conn_info["pwd"],
            username=conn_info["username"],
            query=pg_sql
        )
        # 将数据转换成pandas df
        dict_df = chart_dict_df.toPandas()
        # 提取特殊字符list
        self.sp_symbols = dict_df["character_name"].values.tolist()

        # 隐藏分类
        # Apps & Games、Audible Books & Originals、Books、CDs & Vinyl、Digital Music、Kindle Store、Movies & TV、Software
        sql = f"""
        select 
            category_id as st_bsr_cate_1_id_new, 
            1 as is_hidden_cate 
        from dim_bsr_category_tree
        where site_name = '{self.site_name}'
          and en_name in ("Apps & Games", "Audible Books & Originals", "Books", "CDs & Vinyl", "Digital Music", "Kindle Store", "Movies & TV", "Software")
          and category_parent_id = 0;
        """
        self.df_is_hidden_cate = self.spark.sql(sqlQuery=sql)
        self.df_is_hidden_cate = self.df_is_hidden_cate.repartition(80).cache()
        print("self.df_is_hidden_cate:")
        # self.df_is_hidden_cate.show(10, truncate=True)

        # asin利润率
        sql = f"""
        select asin, price as asin_price, ocean_profit, air_profit
        from dim_asin_profit_rate_info where site_name = '{self.site_name}'
        """
        self.df_asin_profit_rate = self.spark.sql(sqlQuery=sql).repartition(80, 'asin').dropDuplicates(['asin', 'asin_price']).cache()
        print("self.df_asin_profit_rate:")
        # self.df_asin_profit_rate.show(10, truncate=True)

        if self.date_type == 'month':
            # 读累加表 dim_st_detail_history，用于判断 is_first_ever_text（全历史首次出现）
            # 用 date_info_first < self.date_info 过滤：first 早于当月 → 该词历史出现过
            # date_info_first 字段永不被覆盖（MIN 聚合天然幂等），任何重跑场景都安全
            has_data = self.spark.sql(f"""
                SELECT 1 FROM dim_st_detail_history WHERE site_name = '{self.site_name}' LIMIT 1
            """).take(1)
            assert len(has_data) > 0, f"累加表 dim_st_detail_history site_name={self.site_name} 分区为空，请先跑 dim_st_detail_history_init.py"

            sql = f"""
                select
                    search_term
                from dim_st_detail_history
                where site_name = '{self.site_name}'
                  and date_info_first < '{self.date_info}'
            """
            # date_info_first < 当月 → 该词在当月之前已出现过（历史词）
            # 后续 join 上 → is_first_ever_text=0；join 不上 → is_first_ever_text=1（全历史首次）
            self.df_history_st = self.spark.sql(sqlQuery=sql).repartition(80, 'search_term').cache()
            print("self.df_history_st:")
            # self.df_history_st.show(10, truncate=True)

            #  读 dws_st_theme 计算搜索词属性标签 st_attribute_label
            # 【为什么加站点判断】st_attribute_label 只有 us 计算 因为属性是基于us统计 业务要求 uk de不展示 这里填充默认值
            #   非 us 直接跳过读取，st_attribute_label 在 handle_data 里统一填占位 '-1'（Java 转 null 返前端）。
            if self.site_name == 'us':
                # ===== 临时注释（补数用）：补数时 dws_st_theme 可能还没跑好，先关掉存在性校验避免阻塞任务 =====
                # 影响范围仅 st_attribute_label 一个字段：无数据时它填占位 '-1'（Java 转 null），不影响任何其他字段。
                # dws_st_theme 调度恢复正常后，请取消下面注释、恢复 assert。
                # has_theme_data = self.spark.sql(f"""
                #     SELECT 1 FROM dws_st_theme
                #     WHERE site_name='{self.site_name}'
                #       AND date_type='{self.date_type}'
                #       AND date_info='{self.date_info}'
                #     LIMIT 1
                # """).take(1)
                # assert len(has_theme_data) > 0, (
                #     f"上游 dws_st_theme 分区无数据 "
                #     f"site_name={self.site_name}/date_type={self.date_type}/date_info={self.date_info}，"
                #     f"请确认dws_st_theme 调度是否完成；若强行跑 dwt_aba_st_analytics 会导致所有词的 st_attribute_label 错填 '-1'"
                # )

                sql = f"""
                select
                    search_term,
                    concat_ws(',', sort_array(collect_set(theme_ch))) as st_attribute_label
                from dws_st_theme
                where site_name = '{self.site_name}'
                  and date_type = '{self.date_type}'
                  and date_info = '{self.date_info}'
                  and theme_ch is not null
                group by search_term
                """
                self.df_st_attribute = self.spark.sql(sqlQuery=sql).repartition(80, 'search_term').cache()
                print("self.df_st_attribute:")
                # self.df_st_attribute.show(10, truncate=True)

            # ============================================================
            # 读自身历史 11 个月分区的 rank，用于计算峰值月 peak_month / 常年可卖 all_year_text_flag
            # ============================================================
            last_11_month = [CommonUtil.get_month_offset(self.date_info, -i) for i in range(1, 12)]
            # ===== 原逻辑(保留备查):读 dwt_aba_st_analytics 近11月 rank =====
            # 问题:dwt_aba_st_analytics = 爬虫 ∩ ABA(inner join),某月爬虫漏抓 → 该词该月无行 →
            #      peak_month/常年可卖(all_year_text_flag)被漏抓月带偏(峰值月算错、常年可卖误判为否)。
            # sql = f"""
            # select
            #     search_term,
            #     rank,
            #     date_info
            # from dwt_aba_st_analytics
            # where site_name = '{self.site_name}'
            #   and date_type = 'month'
            #   and date_info in ({CommonUtil.list_to_insql(last_11_month)})
            # """
            # 【改】换源 dim_st_detail(ABA 清洗表,dwt 的 rank 本就取自它的 st_rank):
            #   dim_st_detail 不经爬虫过滤,漏抓月 ABA 侧仍有该词行 →
            #   峰值月/常年可卖按 ABA 的 presence + 真实 rank 计算,不再受爬虫漏抓影响。
            sql = f"""
            select
                search_term,
                st_rank as rank,
                date_info
            from dim_st_detail
            where site_name = '{self.site_name}'
              and date_type = 'month'
              and date_info in ({CommonUtil.list_to_insql(last_11_month)})
            """
            self.df_history_rank = self.spark.sql(sqlQuery=sql).repartition(80, 'search_term').cache()
            print("self.df_history_rank:")
            # self.df_history_rank.show(10, truncate=True)

            # ============================================================
            # 读 ods_st_quantity_being_sold.filter_refinements，清洗出 st_filter_refinements
            # 【业务背景】前端要在 ABA 搜索词页展示该词所在市场的"筛选维度"（材质/风格/功能/场景等商品属性），
            #   数据源是爬虫抓的 filter_refinements（JSON：维度名->值列表），需过滤掉品牌/平台/物流/交易类维度
            # 【为什么读 ODS】该字段在 DIM 层(dim_st_detail)已被丢弃，只能回 ODS 原始表取
            # 【为什么要去重】ods_st_quantity_being_sold 未按 search_term 去重，同一词多次抓取会有多行、
            #   各行 filter_refinements 值和抓取时间都不同 → 按 created_time 降序只取最新一行（对齐 dwd_st_brand_badge）
            # 【数据边界】该字段 2026-06 才开始抓，之前的月份 filter_refinements 全为 null → 清洗后统一为 "{}"，无害
            # ============================================================
            sql = f"""
                select search_term, filter_refinements, created_time
                from ods_st_quantity_being_sold
                where site_name = '{self.site_name}'
                  and date_type = '{self.date_type}'
                  and date_info = '{self.date_info}'
                  and search_term is not null
            """
            df_fr = self.spark.sql(sqlQuery=sql)
            # 按 search_term 分区、created_time 降序，row_number=1 只保留最新一行
            # 【只按时间取最新、不看内容】即便最新行 filter_refinements 为空也取它（业务确认口径）；
            #   放在清洗前：去重只看时间与内容无关，先去重能让每个词只跑一次清洗 UDF，避免无用计算
            w_fr = Window.partitionBy('search_term').orderBy(F.col('created_time').desc_nulls_last())
            self.df_st_filter = df_fr.withColumn('_rn', F.row_number().over(w_fr)) \
                .filter(F.col('_rn') == 1) \
                .withColumn('st_filter_refinements', self.u_clean_filter(F.col('filter_refinements'))) \
                .select('search_term', 'st_filter_refinements') \
                .repartition(80, 'search_term').cache()
            print("self.df_st_filter:")
            # self.df_st_filter.show(10, truncate=True)

    def handle_data(self):
        # 对基础计算表进行关联
        self.handle_base_join()

        # 对st_asin按st进行指标聚合
        self.handle_st_agg()

        # 对品牌和卖家按st进行指标聚合
        self.handle_brand_seller_agg()

        # 计算最终指标
        self.handle_st_cal()

        # 语种处理
        self.handle_calc_lang()

        # ============================================================
        # month-only 字段统一处理：month 流程计算/关联，非 month 流程统一 lit 占位
        # 涉及 5 个字段（都只在 month 有业务意义）：
        #   - is_first_ever_text                : 全历史首次出现标记（handle_first_ever_flag，基于累加表 dim_st_detail_history）
        #   - peak_month / all_year_text_flag   : 峰值月 + 常年可卖（handle_peak_month，基于近 12 个月 rank）
        #   - st_attribute_label                : 属性标签（left join read_data 读好的 dws_st_theme 聚合，占位 "-1"）
        #   - st_filter_refinements             : 市场筛选维度（left join read_data 清洗好的 ods filter_refinements，占位 "{}"）
        # join 不上的词在 handle_column 阶段各自 na.fill 占位；非 month 直接 lit 占位（占位值语义见各行注释）
        # ============================================================
        if self.date_type == 'month':
            self.handle_first_ever_flag()
            # ============================================================
            # 【日期判断】派生字段各自"起算月"（早于此填默认值，前端隐藏）：
            #   peak_month / all_year_text_flag：>= '2025-07' 才算
            #     （2025-07 前历史 dwt 曾被删 / 老逻辑读空自身历史补过，峰值月/常年可卖口径不可靠）
            #   st_attribute_label：>= '2025-01' 才算（属性数据单独口径，2025-01 起可靠）
            # 默认值与非 month 流程一致：peak_month='' / all_year_text_flag=-1 / st_attribute_label='-1'。
            # ============================================================
            calc_peak = self.date_info >= '2025-07'   # 峰值月/常年可卖 起算月
            calc_attr = self.date_info >= '2025-01'   # 属性标签 起算月（单独口径）
            if calc_peak:
                self.handle_peak_month()
            else:
                self.df_save = self.df_save \
                    .withColumn('peak_month', F.lit('')) \
                    .withColumn('all_year_text_flag', F.lit(-1))
            # st_attribute_label 只有 us 且 date_info>=2025-01 才 join 计算
            # 非 us（uk/de）或早于 2025-01：直接填占位 '-1'（Java 转 null 返前端）
            if calc_attr and self.site_name == 'us':
                self.df_save = self.df_save.join(self.df_st_attribute, on='search_term', how='left')
            else:
                self.df_save = self.df_save.withColumn('st_attribute_label', F.lit('-1'))
            self.df_st_attribute.unpersist()   # 无论走哪支都释放（非 us/未算时是空 df，unpersist 无害）
            self.df_save = self.df_save.join(self.df_st_filter, on='search_term', how='left')
            self.df_st_filter.unpersist()
        else:
            self.df_save = self.df_save \
                .withColumn('is_first_ever_text', F.lit(-1)) \
                .withColumn('peak_month', F.lit('')) \
                .withColumn('all_year_text_flag', F.lit(-1)) \
                .withColumn('st_attribute_label', F.lit('-1')) \
                .withColumn('st_filter_refinements', F.lit('{}'))

        # 处理输出字段
        self.handle_column()

    def handle_base_join(self):
        self.df_st_asin_join = self.df_st_asin_measure.join(
            self.df_asin_measure, on=['asin'], how='left'
        ).join(
            self.df_asin_detail, on=['asin'], how='left'
        ).join(
            self.df_asin_label, on=['asin'], how='left'
        ).join(
            self.df_asin_profit_rate, on=['asin', 'asin_price'], how='left'
        ).cache()
        self.df_st_asin_measure.unpersist()
        self.df_asin_measure.unpersist()
        self.df_asin_detail.unpersist()
        self.df_asin_label.unpersist()
        self.df_asin_profit_rate.unpersist()

        self.df_st_asin_cal = self.df_st_asin_join.join(
            self.df_seller_asin_country, on=['asin'], how='left'
        )

        # 计算品牌相关指标df
        self.df_st_brand_cal = self.df_st_asin_join

        # 计算卖家相关指标df
        self.df_st_seller_cal = self.df_st_asin_join.join(
            self.df_seller_asin_info, on=['asin'], how='left'
        )

    def handle_st_agg(self):
        self.df_st_asin_cal = self.df_st_asin_cal.withColumn(
            # 打上是否中国卖家标签
            "asin_is_cn",
            self.u_contains(F.lit('CN'), F.col("country_name"))
        ).withColumn(
            # 新品bsr销量
            "asin_is_new_bsr_orders",
            F.when(F.col("asin_is_new") == 1, F.col("asin_amazon_orders"))
        ).withColumn(
            # 新品zr销量
            "asin_is_new_zr_orders",
            F.when(F.col("asin_is_new") == 1, F.col("asin_zr_orders"))
        ).withColumn(
            # 标记是否有颜色标签
            "asin_is_color_flag",
            self.u_judge_color(F.col("asin_color"))
        ).withColumn(
            # 标记标题中是否出现多色关键词
            "asin_is_multi_color",
            self.u_judge_title_color(F.col("asin_title"))
        ).na.fill({
            # 判断是否多尺寸时，先对多尺寸判断字段进行空值处理
            "asin_size": "None",
            "asin_style": "None"
        }).withColumn(
            # 判断是否多尺寸
            "asin_is_multi_size",
            self.u_judge_multi_size(F.col("asin_size"), F.col("asin_style"))
        ).withColumn(
            # 增加判断是否是影视产品标签
            "asin_is_movie_flag",
            F.when(F.col("asin_label_type") == 1, F.lit(1))
        )

        # group by 按search_term 聚合
        self.df_st_asin_cal = self.df_st_asin_cal.groupby(['search_term']).agg(
            F.count("asin").alias("asin_count"),
            F.sum("asin_is_new").alias("asin_is_new_total"),
            F.sum("asin_is_aadd").alias("asin_aadd_count"),
            F.sum("asin_is_video").alias("asin_video_count"),
            F.sum("asin_is_fbm").alias("asin_fbm_count"),
            F.sum("asin_is_amazon").alias("asin_amazon_count"),
            F.sum("asin_is_cn").alias("asin_cn_count"),
            F.sum("asin_is_new_bsr_orders").alias("new_asin_bsr_orders"),
            F.sum("asin_is_new_zr_orders").alias("new_asin_orders"),
            F.sum("asin_is_color_flag").alias("asin_color_count"),
            F.sum("asin_is_multi_color").alias("asin_multi_color_count"),
            F.sum("asin_is_multi_size").alias("asin_multi_size_count"),
            F.sum("asin_is_movie_flag").alias("asin_movie_type_count"),
            F.sum("asin_amazon_orders").alias("amazon_monthly_sales"),
            F.avg("asin_title_len").alias("title_length_avg"),
            F.avg("asin_rating").alias("rating_avg"),
            F.avg("asin_total_comments").alias("total_comments_avg"),
            F.avg("ocean_profit").alias("ocean_profit_avg"),
            F.avg("air_profit").alias("air_profit_avg")
        ).repartition(80, 'search_term').cache()

    def handle_brand_seller_agg(self):
        # 计算品牌top3销量和总销量
        self.df_st_brand_cal = self.df_st_brand_cal.filter("asin_brand_name is not null")
        self.df_st_brand_cal = self.df_st_brand_cal.filter("asin_brand_name not in('null','None')")
        self.df_st_brand_cal = self.df_st_brand_cal.groupby(['search_term', 'asin_brand_name']).agg(
            F.sum("asin_amazon_orders").alias("asin_brand_bsr_orders_total"),
            F.sum("asin_zr_orders").alias("asin_brand_zr_orders_total"),
            F.count("asin").alias("brand_asin_count")  # 需求3：每个品牌在该搜索词下的 ASIN 数（用于后续算占比）
        )

        self.df_top3_st_brand_cal = self.df_st_brand_cal
        # top3品牌bsr销量
        brand_bsr_window = Window.partitionBy(["search_term"]).orderBy(
            self.df_top3_st_brand_cal.asin_brand_bsr_orders_total.desc_nulls_last()
        )
        df_st_brand_top3_bsr_orders = self.df_top3_st_brand_cal.withColumn(
            "brand_rank",
            F.row_number().over(window=brand_bsr_window)
        )
        df_st_brand_top3_bsr_orders = df_st_brand_top3_bsr_orders.filter("brand_rank<=3")
        df_st_brand_top3_bsr_orders = df_st_brand_top3_bsr_orders.groupby(["search_term"]).agg(
            F.sum("asin_brand_bsr_orders_total").alias("top3_brand_bsr_orders")
        )

        # top3品牌zr销量
        brand_zr_window = Window.partitionBy(["search_term"]).orderBy(
            self.df_top3_st_brand_cal.asin_brand_zr_orders_total.desc_nulls_last()
        )
        df_st_brand_top3_zr_orders = self.df_top3_st_brand_cal.withColumn(
            "brand_rank",
            F.row_number().over(window=brand_zr_window)
        )
        df_st_brand_top3_zr_orders = df_st_brand_top3_zr_orders.filter("brand_rank<=3")
        df_st_brand_top3_zr_orders = df_st_brand_top3_zr_orders.groupby(["search_term"]).agg(
            F.sum("asin_brand_zr_orders_total").alias("top3_brand_orders")
        )

        # 品牌总销量
        # 合并 max(brand_asin_count) 到这次 groupby，避免多一次 shuffle
        self.df_st_brand_cal = self.df_st_brand_cal.groupby(['search_term']).agg(
            F.count_distinct("asin_brand_name").alias("page3_brand_num"),
            F.max("brand_asin_count").alias("max_brand_asin_count")  # 需求3：取该搜索词下 ASIN 数最多品牌的 ASIN 数
        ).repartition(80, 'search_term')

        # 聚合得到st_brand
        self.df_st_brand_cal = self.df_st_brand_cal.join(
            df_st_brand_top3_bsr_orders, on=['search_term'], how='left'
        ).join(
            df_st_brand_top3_zr_orders, on=['search_term'], how='left'
        )

        self.df_st_brand_cal = self.df_st_brand_cal.select(
            "search_term", "page3_brand_num", "top3_brand_bsr_orders", "top3_brand_orders", "max_brand_asin_count"
        ).cache()

        # 计算卖家top3销量和总销量
        self.df_st_seller_cal = self.df_st_seller_cal.filter("account_id is not null")
        self.df_st_seller_cal = self.df_st_seller_cal.groupby(['search_term', 'account_id']).agg(
            F.sum("asin_amazon_orders").alias("asin_seller_bsr_orders_total"),
            F.sum("asin_zr_orders").alias("asin_seller_zr_orders_total"),
            F.count("asin").alias("seller_asin_count")  # 需求3：每个卖家在该搜索词下的 ASIN 数
        )

        self.df_top3_st_seller_cal = self.df_st_seller_cal

        # 计算top3卖家bsr销量
        seller_bsr_window = Window.partitionBy(["search_term"]).orderBy(
            self.df_top3_st_seller_cal.asin_seller_bsr_orders_total.desc_nulls_last()
        )
        df_st_seller_top3_bsr_orders = self.df_top3_st_seller_cal.withColumn(
            "seller_rank",
            F.row_number().over(window=seller_bsr_window)
        )
        df_st_seller_top3_bsr_orders = df_st_seller_top3_bsr_orders.filter("seller_rank<=3")
        df_st_seller_top3_bsr_orders = df_st_seller_top3_bsr_orders.groupby(["search_term"]).agg(
            F.sum("asin_seller_bsr_orders_total").alias("top3_seller_bsr_orders")
        )

        # 计算top3卖家的zr销量
        seller_zr_window = Window.partitionBy(["search_term"]).orderBy(
            self.df_st_seller_cal.asin_seller_zr_orders_total.desc_nulls_last()
        )
        df_st_seller_top3_zr_orders = self.df_st_seller_cal.withColumn(
            "seller_rank",
            F.row_number().over(window=seller_zr_window)
        )
        df_st_seller_top3_zr_orders = df_st_seller_top3_zr_orders.filter("seller_rank<=3")
        df_st_seller_top3_zr_orders = df_st_seller_top3_zr_orders.groupby(["search_term"]).agg(
            F.sum("asin_seller_zr_orders_total").alias("top3_seller_orders")
        )

        # 卖家总数量
        # 合并 max(seller_asin_count) 到这次 groupby，避免多一次 shuffle
        self.df_st_seller_cal = self.df_st_seller_cal.groupby(['search_term']).agg(
            F.countDistinct("account_id").alias("page3_seller_num"),
            F.max("seller_asin_count").alias("max_seller_asin_count")  # 需求3：取该搜索词下 ASIN 数最多卖家的 ASIN 数
        ).repartition(80, 'search_term')

        # 聚合得到st_seller
        self.df_st_seller_cal = self.df_st_seller_cal.join(
            df_st_seller_top3_bsr_orders, on=['search_term'], how='left'
        ).join(
            df_st_seller_top3_zr_orders, on=['search_term'], how='left'
        )

        self.df_st_seller_cal = self.df_st_seller_cal.select(
            "search_term", "page3_seller_num", "top3_seller_bsr_orders", "top3_seller_orders", "max_seller_asin_count"
        ).cache()

    # 计算最终指标
    def handle_st_cal(self):
        # 将st,st_asin,st_seller,st_brand按search_term聚合
        df_st_agg = self.df_st_measure.join(
            self.df_st_asin_cal, on=['search_term'], how='left'
        ).join(
            self.df_st_brand_cal, on=['search_term'], how='left'
        ).join(
            self.df_st_seller_cal, on=['search_term'], how='left'
        )
        self.df_st_measure.unpersist()
        self.df_st_asin_cal.unpersist()
        self.df_st_brand_cal.unpersist()
        self.df_st_seller_cal.unpersist()

        # 求值
        df_st_agg = df_st_agg.withColumn(
            # 新品产品数量/前三页产品总数
            "new_asin_proportion",
            F.round(F.col("asin_is_new_total") / F.col("asin_count"), 4)
        ).withColumn(
            # 当日A+商品占比
            "aadd_proportion",
            F.round(F.col("asin_aadd_count") / F.col("asin_count"), 4)
        ).withColumn(
            # 当日视频商品占比
            "sp_proportion",
            F.round(F.col("asin_video_count") / F.col("asin_count"), 4)
        ).withColumn(
            # 当日FBM商品占比
            "fbm_proportion",
            F.round(F.col("asin_fbm_count") / F.col("asin_count"), 4)
        ).withColumn(
            # 中国卖家占比
            "cn_proportion",
            F.round(F.col("asin_cn_count") / F.col("asin_count"), 4)
        ).withColumn(
            # Amazon自营占比
            "amzon_proportion",
            F.round(F.col("asin_amazon_count") / F.col("asin_count"), 4)
        ).withColumn(
            # 多颜色占比 = 关键字有颜色的asin数/关键字的asin数
            "color_proportion",
            F.round(F.col("asin_color_count") / F.col("asin_count"), 4)
        ).withColumn(
            # 多色比例 = 关键词前三页产品标题中出现colorful/assorted color/multi color的词产品个数/前三页产品数量
            "multi_color_proportion",
            F.round(F.col("asin_multi_color_count") / F.col("asin_count"), 4)
        ).withColumn(
            # 多尺寸占比
            "multi_size_proportion",
            F.round(F.col("asin_multi_size_count") / F.col("asin_count"), 4)
        ).withColumnRenamed(
            # 新品总数
            "asin_is_new_total",
            "new_asin_num"
        ).withColumnRenamed(
            # 产品总数
            "asin_count",
            "total_asin_num"
        ).withColumnRenamed(
            # 总预估销量
            "st_zr_orders",
            "orders"
        ).withColumn(
            # 销量占比 新品销量占比
            "new_bsr_orders_proportion",
            F.round(F.col("new_asin_bsr_orders") / F.col("amazon_monthly_sales"), 4)
        ).withColumn(
            # 品牌垄断系数（向上取整保留4位，避免低估垄断程度）
            "brand_monopoly",
            F.round(F.col("top3_brand_bsr_orders") / F.col("amazon_monthly_sales"), 4)
        ).withColumn(
            # 卖家垄断系数（向上取整保留4位，避免低估垄断程度）
            "seller_monopoly",
            F.round(F.col("top3_seller_bsr_orders") / F.col("amazon_monthly_sales"), 4)
        ).withColumn(
            # 品牌ASIN占比 = 该搜索词下 ASIN 数最多的品牌的 ASIN 数 / 前三页所有产品数
            # 分子来自 handle_brand_seller_agg 的 max_brand_asin_count（已过滤 null/None 品牌）
            # 分母 total_asin_num 含全量 ASIN（含无品牌 ASIN）
            "brand_asin_proportion",
            F.round(F.col("max_brand_asin_count") / F.col("total_asin_num"), 4)
        ).withColumn(
            # 卖家ASIN占比 = 该搜索词下 ASIN 数最多的卖家的 ASIN 数 / 前三页所有产品数
            "seller_asin_proportion",
            F.round(F.col("max_seller_asin_count") / F.col("total_asin_num"), 4)
        ).withColumn(
            # ABA搜索词拆分的单词个数
            "st_word_num",
            self.st_word_count(self.sp_symbols)(F.col("search_term"))
        ).withColumn(
            # aba搜索词影视比例
            "movie_prop",
            F.round((F.col("asin_movie_type_count") / F.col("total_asin_num"))*100, 2)
        ).withColumn(
            # 影视标记类型 0：非影视; 1:0< 比例 <= 20%; 2: 20% < 比例 <= 50%; 3:50% < 比例
            "st_movie_label",
            F.when(
                (F.col("movie_prop") > 0) & (F.col("movie_prop") <= 20), F.lit(1)
            ).when(
                (F.col("movie_prop") > 20) & (F.col("movie_prop") <= 50), F.lit(2)
            ).when(
                (F.col("movie_prop") > 50), F.lit(3)
            ).otherwise(F.lit(0))
        )

        self.df_save = df_st_agg.join(
            self.df_st_detail, on=['search_term'], how='inner'
        ).join(
            self.df_st_key, on=['search_term'], how='inner'
        ).join(
            self.df_st_num_stats, on=['search_term'], how='left'
        ).join(
            self.df_st_market, on=['search_term'], how='left'
        ).join(
            self.df_st_brand, on=['search_term'], how='left'
        ).join(
            self.df_is_hidden_cate, on=['st_bsr_cate_1_id_new'], how='left'
        ).cache()
        self.df_st_detail.unpersist()
        self.df_st_key.unpersist()
        self.df_st_num_stats.unpersist()
        self.df_st_market.unpersist()
        self.df_st_brand.unpersist()
        self.df_is_hidden_cate.unpersist()

    # 语种处理
    def handle_calc_lang(self):
        sql = """
            select 
                word, 
                langs 
            from big_data_selection.tmp_lang_word_frequency;
        """
        lang_word_list = self.spark.sql(sql).collect()
        # 转为map
        lang_word_map = {row['word']: row['langs'] for row in lang_word_list}

        self.df_save = self.df_save.withColumn(
            "lang",
            F.coalesce(udf_detect_phrase_reg(lang_word_map)(F.col("search_term")).getField("lang"), F.lit("other"))
        )

    # 全历史首次出现标记（基于累加表 dim_st_detail_history 的历史搜索词集合）
    def handle_first_ever_flag(self):
        # self.df_history_st 在 read_data 阶段已经过滤了"first < 当月"——即在当月之前出现过的词
        # 给这部分词标记 is_first_ever_text=0（历史出现过）
        # 当月词 left join：
        #   - join 上 → 累加表里有该词的"早于当月的首次记录" → 历史出现过 → is_first_ever_text=0
        #   - join 不上 → 累加表里没有该词早于当月的记录 → 全历史首次出现 → fillna(1)
        df_flag = self.df_history_st.withColumn('is_first_ever_text', F.lit(0))
        self.df_save = self.df_save.join(df_flag, on='search_term', how='left') \
                                   .fillna({'is_first_ever_text': 1})
        self.df_history_st.unpersist()

    # 峰值月 + 常年可卖：基于近 12 个月（自身历史 11 个月分区 + 当月）的 rank 计算
    # ============================================================
    # 【重要提醒·前端展示口径】peak_month / all_year_text_flag 本表（月表 dwt_aba_st_analytics）
    #   仍照常计算并落库，但前端页面【不读本表这两列】。
    # 【为什么不读本表】本表每个月分区各自算"该月往前 12 个月"的窗口，跑完就固定——
    #   后续数据补抓/修正不会回刷，历史月（如 25-12）会一直停在当时算出的旧结论
    #   （典型：某月漏抓 → 当时判"非常年可卖"，之后数据补齐了本表也不会自动改过来）。
    #   而业务要求"所有月页面都显示最近一年的最新结果"，本表这套按月固定的口径满足不了。
    # 【实际展示从哪取】前端按 search_term 从年表 us_aba_last_365_day 取这两列。
    #   年表 = dwt_aba_last365（date_type='month'，每月滚动算近 12 个月）导出到 PG 集群，
    #   导出是"整表替换"（见 sqoop_export/dwt_aba_last365.py），所以该表永远只存最新一份近 12 月快照。
    #   年表每月一替换 → 所有月页面 join 到的自动是最新值，天然满足"历史月也显示最新"。
    # 【结论】本表这两列保留计算（兼容/备查），前端取数源改到年表即可，切勿因为"看起来重复"而删。
    # ============================================================
    def handle_peak_month(self):
        """
        peak_month：近 12 个月窗口内 rank 最小（排名最好）的月份，并列全保留，
                    按时间升序逗号拼接，格式 'YYYY-MM'（如 '2025-11,2026-04'）。
        all_year_text_flag：近 12 个月每个月都出现在 ABA 月度搜索词中 → 1，否则 0（常年可卖）。
        """
        df_current_rank = self.df_save.select(
            'search_term', 'rank', F.lit(self.date_info).alias('date_info')
        )
        df_rank_all = self.df_history_rank.unionByName(df_current_rank)

        df_agg = df_rank_all.groupBy('search_term').agg(
            F.min('rank').alias('min_rank'),
            F.collect_list(F.struct('rank', 'date_info')).alias('rank_arr'),
            F.count_distinct('date_info').alias('month_cnt')
        )
        # map 端表达式算出 peak_month / all_year_text_flag（不触发 shuffle）
        # peak_month  从 rank_arr 里 filter 出 rank=min_rank 的项 → transform 取 date_info → array_sort 保证时间升序、重跑结果稳定 → concat_ws 拼成逗号串
        #  all_year_text_flag  去重月份数 = 12 → 常年可卖；只出现 6 个月 → 0
        # ============================================================
        df_peak = df_agg.select(
            'search_term',
            F.expr("""
                nullif(
                    concat_ws(',',
                        array_sort(transform(
                            filter(rank_arr, x -> x.rank = min_rank),
                            x -> x.date_info
                        ))
                    ),
                    ''
                )
            """).alias('peak_month'),
            F.when(F.col('month_cnt') >= 12, F.lit(1)).otherwise(F.lit(0)).alias('all_year_text_flag')
        )

        self.df_save = self.df_save.join(df_peak, on='search_term', how='left')
        self.df_history_rank.unpersist()

    def handle_column(self):
        # 入库前字段处理，精度以 PG 集群列约束为基准
        self.df_save = self.df_save.select(
            "id",
            "search_term",
            "rank",
            "category_id",
            F.round("orders", 0).alias("orders"),
            F.col("amazon_monthly_sales").alias("bsr_orders"),
            "search_volume",
            "quantity_being_sold",
            F.round("st_ao_avg", 4).alias("st_ao_avg"),
            F.round("st_ao_val_rate", 4).alias("st_ao_val_rate"),
            "new_bsr_orders_proportion",
            "new_asin_proportion",
            F.round("page1_title_proportion", 4).alias("page1_title_proportion"),
            F.round("price_avg", 2).alias("price_avg"),
            F.round("total_comments_avg", 0).alias("total_comments_avg"),
            F.round("rating_avg", 1).alias("rating_avg"),
            F.round("weight_avg", 4).alias("weight_avg"),
            F.round("volume_avg", 4).alias("volume_avg"),
            F.round("title_length_avg", 0).alias("title_length_avg"),
            "st_num",
            "aadd_proportion",
            "sp_proportion",
            "fbm_proportion",
            "cn_proportion",
            "amzon_proportion",
            F.round("most_proportion", 4).alias("most_proportion"),
            F.round("max_num", 4).alias("max_num"),
            "asin1",
            "asin2",
            "asin3",
            F.round("click_share1", 4).alias("click_share1"),
            F.round("click_share2", 4).alias("click_share2"),
            F.round("click_share3", 4).alias("click_share3"),
            F.round("total_click_share", 4).alias("total_click_share"),
            F.round("conversion_share1", 4).alias("conversion_share1"),
            F.round("conversion_share2", 4).alias("conversion_share2"),
            F.round("conversion_share3", 4).alias("conversion_share3"),
            F.round("total_conversion_share", 4).alias("total_conversion_share"),
            "new_asin_num",
            "total_asin_num",
            "new_asin_orders",
            "new_asin_bsr_orders",
            "is_first_text",
            "is_ascending_text",
            "is_search_text",
            "top3_seller_orders",
            "top3_seller_bsr_orders",
            "top3_brand_orders",
            "top3_brand_bsr_orders",
            "page3_brand_num",
            "page3_seller_num",
            "brand_monopoly",
            "seller_monopoly",
            "max_num_asin",
            "is_self_max_num_asin",
            "is_new_market_segment",
            F.when(F.col('category_current_id').isNull(), F.col('category_id'))
                .otherwise(F.col('category_current_id')).alias('category_current_id'),
            F.round("supply_demand", 4).alias("supply_demand"),
            "market_cycle_type",
            "color_proportion",
            "multi_color_proportion",
            "multi_size_proportion",
            "st_4_20_ao_avg",
            "st_4_20_ao_rate",
            "asin_aadd_count",
            "asin_video_count",
            "asin_fbm_count",
            "asin_cn_count",
            "asin_amazon_count",
            "asin_color_count",
            "asin_multi_color_count",
            "asin_multi_size_count",
            "st_word_num",
            "st_movie_label",
            "st_brand_label",
            "st_brand1",
            "st_category1",
            "st_brand2",
            "st_category2",
            "st_brand3",
            "st_category3",
            "st_bsr_cate_1_id_new",
            "st_bsr_cate_current_id_new",
            "st_crawl_date",
            "is_high_return_text",
            F.round("st_zr_page123_title_appear_rate", 4).alias("st_zr_page123_title_appear_rate"),
            F.round("st_sp_page123_title_appear_rate", 4).alias("st_sp_page123_title_appear_rate"),
            "st_competition_level",
            "amazon_monthly_sales",
            F.round("st_zr_flow_proportion", 4).alias("st_zr_flow_proportion"),
            F.round("st_ao_val_matrix", 4).alias("st_ao_val_matrix"),
            F.round("st_flow_proportion_matrix", 4).alias("st_flow_proportion_matrix"),
            "st_zr_counts",
            "st_sp_counts",
            "st_self_asin_counts",
            F.round("st_self_asin_proportion", 4).alias("st_self_asin_proportion"),
            "lang",
            "asin_movie_type_count",
            "is_hidden_cate",
            F.round("st_dd50_proportion", 4).alias("st_dd50_proportion"),
            F.round("st_dd100_proportion", 4).alias("st_dd100_proportion"),
            F.round("st_dd200_proportion", 4).alias("st_dd200_proportion"),
            F.round("ocean_profit_avg", 4).alias("gross_profit_fee_sea"),
            F.round("air_profit_avg", 4).alias("gross_profit_fee_air"),
            "is_first_ever_text",       # 全历史首次出现标记（1=当月首次, 0=历史出现过）
            "brand_asin_proportion",    # 前三页ASIN数最多品牌的ASIN数占比
            "seller_asin_proportion",   # 前三页ASIN数最多卖家的ASIN数占比
            "st_attribute_label",       # 搜索词属性标签（逗号分隔字符串，sqoop 导出 PG 后转 VARCHAR[]）
            # 【前端展示提醒】下面两列本表仍落库，但页面【从年表 us_aba_last_365_day 按 search_term 取，不读本表这两列】——
            #   本表按月分区固定不回刷、历史月会停在旧值；年表整表替换永远最新（详见 handle_peak_month 顶部提醒）
            "peak_month",               # 峰值月：近12个月 rank 最小的月份，并列保留，'YYYY-MM' 逗号分隔（PG 转 VARCHAR[]）
            "all_year_text_flag",         # 常年可卖：近12个月每月都出现在 ABA 月度搜索词中=1，否则=0
            "st_filter_refinements"       # 搜索词市场筛选维度（清洗后 JSON：维度名->值列表，已过滤黑名单维度；PG 端 jsonb，空为 {}）
        )

        # 空值处理
        self.df_save = self.df_save.na.fill({
            "is_first_text": 0,
            "is_ascending_text": 0,
            "is_search_text": 0,
            "st_movie_label": 0,
            "st_brand_label": 0,
            "is_self_max_num_asin": 0,
            "market_cycle_type": 0,
            "is_new_market_segment": 0,
            "is_high_return_text": 0,
            "amazon_monthly_sales": 0,
            "bsr_orders": 0,
            "is_hidden_cate": 0,
            "brand_asin_proportion": -1,   # 分子为 null（搜索词全无品牌） → 占位 -1
            "seller_asin_proportion": -1,  # 分子为 null（搜索词全无账号） → 占位 -1
            "st_attribute_label": "-1",    # 词典无匹配 → 占位 "-1"（Java 侧转 null 返前端）
            "peak_month": "",              # 极端兜底（rank 全 null）→ 空串，PG string_to_array 转空数组 {}
            "all_year_text_flag": 0,         # 理论必有值，兜底 0（非 month 流程已提前填 -1，不受影响）
            "st_filter_refinements": "{}"  # 空/异常/无维度剩余/join不上 → 空对象 "{}"（PG jsonb）
        })

        # 日期字段补全
        self.df_save = self.df_save.withColumn(
            "created_time",
            F.date_format(F.current_timestamp(), 'yyyy-MM-dd HH:mm:SS')
        ).withColumn(
            "updated_time",
            F.date_format(F.current_timestamp(), 'yyyy-MM-dd HH:mm:SS')
        )

        # 预留字段补全
        self.df_save = self.df_save.withColumn(
            "re_string_field1", F.lit(None)
        )

        # 用户标记字段(已废弃,济苍直接填充)
        self.df_save = self.df_save.withColumn(
            "usr_mask_type", F.lit(None)
        ).withColumn(
            "usr_mask_progress", F.lit(None)
        )

        # 分区字段补全
        self.df_save = self.df_save.withColumn(
            "site_name", F.lit(self.site_name)
        ).withColumn(
            "date_type", F.lit(self.date_type)
        ).withColumn(
            "date_info", F.lit(self.date_info)
        )


if __name__ == '__main__':
    start_time = time.time()
    site_name = sys.argv[1]  # 参数1：站点
    date_type = sys.argv[2]  # 参数2：类型：week/4_week/month/quarter
    date_info = sys.argv[3]  # 参数3：年-周/年-月/年-季, 比如: 2022-1
    handle_obj = DwtAbaStAnalytics(site_name=site_name, date_type=date_type, date_info=date_info)
    handle_obj.run()
    end_time = time.time()
    consumer_time = end_time - start_time
    print("aba数据存储完毕, 执行时长为：" + str(consumer_time))
