import os
import sys
from collections import Counter

import inflect
import re
from textblob import TextBlob
from textblob import Word

sys.path.append(os.path.dirname(sys.path[0]))
from functools import reduce
from pyspark.sql import DataFrame
from pyspark.sql.window import Window
from pyspark.sql.types import StringType, MapType, IntegerType, StructType, StructField, DoubleType
from utils.common_util import CommonUtil, DateTypes
from utils.word_normalize import to_base_form, SEMANTIC_DISTINCT_WORDS  # 归一/白名单统一公共模块
from utils.hdfs_utils import HdfsUtils
from utils.spark_util import SparkUtil
from utils.db_util import DBUtil
from pyspark.sql import functions as F
from yswg_utils.common_udf import udf_ele_mattch
import numpy as np


# 模块级 inflect 引擎单例：driver 端 st_word_filter_condition / _build_word_to_base_mapping 调 to_base_form 时传入
# （SEMANTIC_DISTINCT_WORDS / IRREGULAR_PLURAL / to_base_form 已统一抽到 utils.word_normalize）
_INFLECT_ENGINE = inflect.engine()


class DwtStThemeAgg(object):
    def __init__(self, site_name, date_type, date_info):
        self.site_name = site_name
        self.date_type = date_type
        self.date_info = date_info
        self.hive_tb = "dwt_st_theme_agg"
        self.partition_dict = {
            "site_name": site_name,
            "date_type": date_type,
            "date_info": date_info
        }
        # 创建spark_session对象相关
        app_name = f"{self.__class__.__name__}:{site_name}:{date_info}"
        self.spark = SparkUtil.get_spark_session(app_name)

        # 把 word_normalize.py 分发到各 executor：udf_word_restoration 在 executor 端要 import 它，
        # 但 executor 的 sys.path 没有 driver 顶部 sys.path.append 的 py_demo 目录，直接 import utils.word_normalize 会 ModuleNotFoundError。
        # addPyFile 运行时把该文件临时发到每个 executor 并加进其 sys.path（用完即焚），UDF 即可 import 顶层 word_normalize。
        import utils.word_normalize as _wn_mod
        self.spark.sparkContext.addPyFile(_wn_mod.__file__)

        # 注册自定义函数 (UDF)
        self.u_theme_pattern = F.udf(udf_ele_mattch, StringType())
        self.u_theme_contain_judge = F.udf(self.udf_theme_contain_judge, IntegerType())
        self.u_judge_twin_words = F.udf(self.udf_judge_twin_words, IntegerType())
        self.u_filter_pattern_words = F.udf(self.udf_filter_pattern_words, IntegerType())

        # 全局df初始化
        self.df_st_base = self.spark.sql(f"select 1+1;")
        self.df_usr_mask = self.spark.sql(f"select 1+1;")
        self.df_base_filter_date = self.spark.sql(f"select 1+1;")
        self.df_pattern_words_base = self.spark.sql(f"select 1+1;")
        self.df_sec_words = self.spark.sql(f"select 1+1;")
        self.df_third_words = self.spark.sql(f"select 1+1;")
        self.df_theme = self.spark.sql(f"select 1+1;")
        self.df_st_theme = self.spark.sql(f"select 1+1;")
        self.df_st_theme_base = self.spark.sql(f"select 1+1;")
        self.df_st_filter = self.spark.sql(f"select 1+1;")
        self.df_pattern_st_agg = self.spark.sql(f"select 1+1;")
        self.df_pattern_st_words = self.spark.sql(
            f"select null as pattern_st,id as st_key,search_term,bsr_orders from dwt_aba_st_analytics limit 0;"
        )
        self.df_st_theme_agg = self.spark.sql(f"select 1+1;")
        self.df_st_match_topic_detail = self.spark.sql(f"select 1+1;")
        self.df_st_match_topic_agg = self.spark.sql(f"select 1+1;")
        self.df_match_brand = self.spark.sql(f"select 1+1;")
        self.df_match_blacklist = self.spark.sql(f"select 1+1;")

        # 其他变量
        self.brand_pattern = str()  # 正则匹配
        self.st_word_list = []

    @staticmethod
    def udf_unionAll(*dfs):
        return reduce(DataFrame.unionAll, dfs)

    @staticmethod
    def udf_theme_contain_judge(pattern_word, pattern_list):
        count = sum(1 for word in pattern_list if pattern_word in word)
        # 如果匹配到的pattern_word大于1则说明有已经匹配过的单词
        return 0 if count > 1 else 1

    @staticmethod
    def udf_inflect_word():
        p = inflect.engine()

        def udf_split_words(st_word):
            words = st_word.split(" ")
            word_list = []
            for word in words:
                # 跳过空串：search_term 带双空格/首尾空格时 split(" ") 会拆出 ''，
                # 新版 inflect 用 typeguard 校验入参（Word 要求长度≥1），p.plural('') 直接抛
                # TypeCheckError: argument "text" (str) is not an instance of inflect.Word
                # 上游 rlike 过滤是对"去空格后"的串校验的（'gift  box'→'giftbox' 合法通过），挡不住双空格
                if not word:
                    continue
                word_list.append(word)
                # 白名单词（short/shorts、person/people 等单复数语义不同）不加复数对偶。
                # inflect.plural() 是"双向 toggle"——plural('short')='shorts'，
                #   而 plural('shorts') 又变回 'short'，一来一回闭环。
                #   于是 'short' 和 'shorts' 凑出的集合都是 {short, shorts} → 签名相同
                #   → 在下面 Window.partitionBy('similar_word_list') 去重时被判为重复、删掉一个，
                #   把"长度(short)"和"短裤(shorts)"两个不同语义的模板词合并掉。
                # 必须显式判白名单，不能赌 inflect 对某些词(person→people→peoples)碰巧不闭环。
                # 与 udf_word_restoration / st_word_filter_condition 三处保持同一套白名单逻辑。
                if word.lower() not in SEMANTIC_DISTINCT_WORDS:
                    word_list.append(p.plural(word))
            word_list.sort()
            return ','.join(set(word_list))

        return F.udf(udf_split_words, StringType())

    @staticmethod
    def udf_word_restoration():
        # ============================================================
        # dedup 第二轮 UDF：对 search_term 拆词后逐词归一，拼成签名用于"词形相同"去重。
        # 归一统一调 utils.word_normalize.to_base_form（与匹配侧 _build_word_to_base_mapping 同一套口径）。
        # 为什么"函数内 import + function attribute 单例"：
        #   UDF 在 executor 端逐行跑，函数内 import 让每个 executor 进程构造一次依赖（单例缓存），
        #   且 to_base_form 在独立模块里、cloudpickle 按"模块名+函数名"引用而非按值 pickle，
        #   避开了原先内联想绕的那个 `args[0] from __newobj__` PickleError。
        #   注：to_base_form 内部已带白名单/不规则表，不再需要本地内联那几张表。
        # ============================================================
        def udf_restoration_words(st_word):
            # executor 端首次调用时初始化（function attribute 当 per-process 单例缓存）
            if not getattr(udf_restoration_words, '_inited', False):
                import inflect as _inflect
                from textblob import TextBlob as _TB
                # 顶层 word_normalize（由 __init__ 的 addPyFile 发到 executor），不是 utils.word_normalize：
                # executor sys.path 无 py_demo 目录，addPyFile 把单文件按顶层模块名分发
                from word_normalize import to_base_form as _to_base_form
                udf_restoration_words._engine = _inflect.engine()
                udf_restoration_words._TextBlob = _TB
                udf_restoration_words._to_base_form = _to_base_form
                udf_restoration_words._inited = True

            p = udf_restoration_words._engine
            TextBlob = udf_restoration_words._TextBlob
            to_base = udf_restoration_words._to_base_form

            word_list = []
            blob = TextBlob(st_word)
            for word in blob.words:
                word_list.append(to_base(str(word), p))
            word_list.sort()
            return ','.join(set(word_list))

        return F.udf(udf_restoration_words, StringType())

    @staticmethod
    def udf_judge_twin_words(st_word):
        # 剔除空串后再拆词：search_term 带多余空格时 split(" ") 会拆出 ''，混进 set 让叠词判断失效
        # 修复前：'gun gun '.split(" ") → ['gun','gun',''] → set={'gun',''} 2个元素 ≠ 1 → 叠词漏网 ✗
        # 修复后：剔除空串 → ['gun','gun'] → set={'gun'} 1个元素 → 正常识别为叠词过滤掉 ✓
        # 不带多余空格的正常词（'gun gun'/'red dress'）拆词结果与修复前完全一致，行为不变
        words = [w for w in st_word.split(" ") if w]
        judge_flag = 0
        # 先判断是否完全一致的同属性叠词，如gun gun；这种用户不需要
        # len(words) > 1：叠词定义要求"至少2个词且全相同"。
        #   不加的话 "gun "（单词+尾空格）剔空串后只剩 ['gun']，len(set)==1 会被误判成叠词丢弃；
        #   外层 Spark 侧的单词豁免（F.size(F.split)==1）对 "gun " 不生效（Spark split 出 ["gun",""] size=2 进了本UDF），
        #   所以必须在这里自己兜住单词情况
        if len(words) > 1 and len(set(words)) == 1:
            judge_flag = 1
        return judge_flag

    @staticmethod
    def udf_theme_regex(pattern):

        def udf_theme_pattern(match_text):
            ele_list = re.findall(pattern, match_text)
            if ele_list:
                return ','.join(set(ele_list))
            else:
                return None
        return F.udf(udf_theme_pattern, StringType())

    @staticmethod
    def st_word_filter_condition(search_term):
        base_keywords = search_term.split(" ")
        # 生成排序后的单数和复数列表
        singular_plural_list = []
        for word in base_keywords:
            # 白名单词不做单复数对偶，只用原词生成 rlike：
            # 否则 short 模板会拼出 \b(short|shorts)\b 把 shorts 的搜索词也匹配进来（反之亦然），
            # 两个不同语义的模板词互相"借"对方的流量，指标被污染
            if word.lower() in SEMANTIC_DISTINCT_WORDS:
                sort_list = [word]
            else:
                # 用 to_base_form + round-trip 校验拿对偶形式，替代老的 CommonUtil.convert_singular_plural
                # 修复两个老 bug：①错拼词借真词流量（sunglasse→sunglasses）②-ss/-sis 真词被砍 s（dress→dres）
                w = word.lower()
                base = to_base_form(w, _INFLECT_ENGINE)
                if base != w:
                    # word 是复数 → 对偶是单数 base（如 shorts→short、dresses→dress）
                    counterpart = base
                else:
                    # word 是单数或错拼词 → 算复数再 round-trip 校验：
                    # 复数转回单数 == base 才接受（真单数 short→shorts ✓）；
                    # 错拼词转回对不上则不配对（sunglasse 的复数 sunglasses 转回是 sunglass≠sunglasse ✗）
                    try:
                        plural = _INFLECT_ENGINE.plural(base)
                    except Exception:
                        plural = base
                    counterpart = plural if to_base_form(plural, _INFLECT_ENGINE) == base else word
                sort_list = [word, counterpart]
            sort_list.sort()
            singular_plural_word = "|".join(sort_list)
            singular_plural_list.append(singular_plural_word)

        # 统计排序后的单数和复数的词频
        condition_counts = Counter(singular_plural_list)

        # 构建重复词和唯一词的条件
        result_condition = ""
        condition_list = []
        for word, count in condition_counts.items():
            word_condition = f"search_term rlike '\\\\b({word})\\\\b"
            word_condition += "".join([f".*\\\\b({word})\\\\b" for _ in range(count - 1)]) + "'"

            if count > 1:
                condition_list.append(word_condition)
            else:
                condition_list.append(word_condition)
        if condition_list:
            result_condition = (" and ").join(condition_list)
        return result_condition

    @staticmethod
    def filter_blacklist_words(blacklist):
        # 获取二三级词包含词黑名单
        pd_contain_list = blacklist.loc[blacklist.specical_match_type == '1']
        contain_word_list = list(pd_contain_list.st_blacklist_word_lower)
        contain_word_pattern = re.compile(
            r'(?<!\+|\*|\-|\%|\.)\b({})\b'.format('|'.join([re.escape(x) for x in contain_word_list])),
            flags=re.IGNORECASE)
        # 获取二三级词搜索词黑名单
        pd_st_blacklist = blacklist.loc[blacklist.specical_match_type == '2']
        st_blacklist = list(pd_st_blacklist.st_blacklist_word_lower)

        def udf_filter_blacklist(st_word):
            # 包含词匹配
            black_flag = 0
            pattern_flag = re.search(contain_word_pattern, st_word)
            if pattern_flag:
                black_flag = 1
            # 黑名单词匹配--此处不能使用二元表达，重新置为 0
            if black_flag == 0:
                if st_word in st_blacklist:
                    black_flag = 1
            return black_flag

        return F.udf(udf_filter_blacklist, IntegerType())

    @staticmethod
    def udf_filter_pattern_words(st_word, pattern_list):
        filter_flag = 0
        theme_list = ['combination', 'size']
        if pattern_list:
            if any(theme in str(pattern_list) for theme in theme_list):
                # 说明匹配到了组合和size两种匹配词，则需要给标记
                return 1
        # 进行单项 数字+month/months的所有二级词 和 数字连接t+ boys/girls的二级词特殊匹配
        date_pattern = re.compile(r"(\d+(?:\.\d+)?) +(month|months)\b", flags=re.IGNORECASE)
        numt_pattern = re.compile(r"((?:\d+)t)(?: +)(boys|girls|boy|girl)\b", flags=re.IGNORECASE)
        other_pattern = re.compile(r"\b(womens?|mens?|mans?|womans?|fors?|cutes?|fashions?|kids?|adults?|girls?|boys?)\b", flags=re.IGNORECASE)
        if re.search(date_pattern, st_word):
            return 1
        if re.search(numt_pattern, st_word):
            return 1
        if re.search(other_pattern, st_word):
            return 1
        return filter_flag

    def read_data(self):
        # 月ABA词基础数据
        sql = f"""
            select 
                id as st_key,
                search_term,
                bsr_orders
            from dwt_aba_st_analytics
            where site_name = '{self.site_name}'
              and date_type = '{self.date_type}'
              and date_info = '{self.date_info}'
              and st_bsr_cate_1_id_new is not null
              and st_bsr_cate_1_id_new not in ("audible", "books", "digital-text", "dmusic", "mobile-apps", "movies-tv",
               "music", "software", "videogames")
        """
        print(sql)
        self.df_st_base = self.spark.sql(sql)

        # 根据ABA词计算匹配模板词，要求：
        # 1. ABA词汇数是1和2
        # 2. 与ABA月数据默认类型保持一致（济苍处理）
        # 3. 只能由数字和字母组成，且不能为纯数字
        # 4. 必须有分类
        # 5. 单复数关系、词序不同的问题，做归一化处理，保留排名最好的词
        sql = f"""
            select search_term, st_word_num, rank, st_brand_label, st_bsr_cate_1_id_new, st_bsr_cate_current_id_new from (
                select search_term,
                       regexp_replace(search_term,' ','') as search_term_without_space,
                       st_word_num,
                       rank,
                       st_brand_label,
                       st_bsr_cate_1_id_new,
                       st_bsr_cate_current_id_new
                from dwt_aba_st_analytics
                where site_name = '{site_name}'
                  and date_type = '{date_type}'
                  and date_info = '{date_info}'
                  and st_bsr_cate_1_id_new is not null
                  and st_bsr_cate_1_id_new not in ("audible", "books", "digital-text", "dmusic", "mobile-apps", 
                  "movies-tv", "music", "software", "videogames")
                  and st_word_num <= 2
                  and st_brand_label <= 1
                ) t1
            where search_term_without_space rlike '^(?![0-9]+$)[0-9a-zA-Z]+$'
        """
        self.df_pattern_words_base = self.spark.sql(sql)

        # 读取ABA词排名变化数据
        sql = f"""
            select 
                search_term,
                rank_change_rate,
                rank_rate_of_change 
            from dwt_aba_last_change_rate
            where site_name = '{self.site_name}'
              and date_type = '{self.date_type}'
              and date_info = '{self.date_info}'
        """
        df_st_rank = self.spark.sql(sql).repartition(40, 'search_term')

        # 过滤掉叠词脏数据，例如 gun gun
        self.df_pattern_words_base = self.df_pattern_words_base.withColumn(
            "twin_words_flag",
            F.when(
                F.size(F.split(F.col("search_term"), " ")) == 1, F.lit(0)
            ).otherwise(
                self.u_judge_twin_words(F.col("search_term"))
            )
        ).filter("twin_words_flag = 0").repartition(40, "search_term")
        # 关联排名变化数据
        self.df_pattern_words_base = self.df_pattern_words_base.join(
            df_st_rank, on=['search_term'], how='left'
        ).cache()
        print('一二级词基础数据如下：')
        self.df_pattern_words_base.show(10, True)

        # 读取ABA词主题标签数据
        sql = f"""
            select 
                st_key, 
                search_term, 
                theme_ch, 
                theme_en, 
                theme_label_ch, 
                theme_label_en, 
                pattern_type, 
                theme_label_num_info, 
                theme_label_unit_info 
            from big_data_selection.dws_st_theme
            where site_name = '{self.site_name}'
              and date_type = '{self.date_type}'
              and date_info = '{self.date_info}'
        """
        self.df_st_theme_base = self.spark.sql(sql).repartition(40, 'search_term').cache()
        print('ABA词主题标签数据如下：')
        self.df_st_theme_base.show(10, True)

        # 聚合搜索词+主题数据，后续过滤二级词用
        sql = f"""
            select 
                search_term, 
                concat_ws(",", collect_list(theme_en)) as pattern_list 
            from big_data_selection.dws_st_theme
            where site_name = '{self.site_name}'
              and date_type = '{self.date_type}'
              and date_info = '{self.date_info}'
            group by st_key, search_term
        """
        self.df_theme = self.spark.sql(sql).repartition(40, 'search_term').cache()

        # sql获取最终品牌词匹配需保留得品牌词库
        pg_sql = f"""
            select 
                lower(trim(character_name)) as st_brand_name_lower 
            from match_character_dict where match_type = '二三级词专用品牌词库'
        """
        conn_info = DBUtil.get_connection_info("mysql", "us")
        self.df_match_brand = SparkUtil.read_jdbc_query(
            session=self.spark,
            url=conn_info["url"],
            pwd=conn_info["pwd"],
            username=conn_info["username"],
            query=pg_sql
        )
        pdf_match_brand = self.df_match_brand.toPandas()
        match_brand = list(set(pdf_match_brand.st_brand_name_lower))
        self.brand_pattern = re.compile(
            r'(?<!\+|\*|\-|\%|\.)\b({})\b'.format('|'.join([re.escape(x) for x in match_brand])), flags=re.IGNORECASE
        )

        # sql获取二三级词黑名单库
        pg_sql = f"""
            select 
                lower(trim(character_name)) as st_blacklist_word_lower, 
                specical_match_type 
            from match_character_dict where match_type = '二三级词匹配黑名单'
        """
        conn_info = DBUtil.get_connection_info("mysql", "us")
        self.df_match_blacklist = SparkUtil.read_jdbc_query(
            session=self.spark,
            url=conn_info["url"],
            pwd=conn_info["pwd"],
            username=conn_info["username"],
            query=pg_sql
        ).cache()

    def handle_data(self):
        self.read_data()
        # 模板词归一化处理
        self.handle_base_pattern_data()
        # 将一级二级模板词和搜索词进行匹配，做中间存储
        self.handle_st_filter_table()
        # 统计各模板词的指标 pattern_type=0
        self.handle_st_pattern_common_agg()
        # 统计各模板词的指标 pattern_type=1
        self.handle_st_pattern_special_agg()
        self.save_data()

    def handle_base_pattern_data(self):
        self.df_base_filter_date = self.df_pattern_words_base
        # 数据归一化处理，单复数、词序标准化，保留rank最小
        self.df_base_filter_date = self.df_base_filter_date.withColumn(
            'similar_word_list', self.udf_inflect_word()(F.col('search_term'))
        )
        similar_words_window = Window.partitionBy(["similar_word_list"]).orderBy(
            self.df_base_filter_date.rank.asc_nulls_last()
        )
        self.df_base_filter_date = self.df_base_filter_date.withColumn(
            'row_num', F.row_number().over(window=similar_words_window)
        ).filter('row_num = 1').drop(*['similar_word_list', 'row_num'])

        # 二次过滤相似词 采用textblob词库词性还原方式过滤
        self.df_base_filter_date = self.df_base_filter_date.withColumn(
            'similar_word_list', self.udf_word_restoration()(F.col('search_term'))
        )
        similar_words_window = Window.partitionBy(["similar_word_list"]).orderBy(
            self.df_base_filter_date.rank.asc_nulls_last()
        )
        self.df_base_filter_date = self.df_base_filter_date.withColumn(
            'row_num', F.row_number().over(window=similar_words_window)
        ).filter('row_num = 1').drop(*['similar_word_list', 'row_num']).cache()

        # 处理品牌词逻辑【已注释：不再做品牌过滤】
        # 注释原因：原逻辑把 st_brand_label=1 的词只保留命中"二三级词专用品牌词库"白名单的，其余砍掉。
        #   但上游 dim_st_brand_info 会把常用词误判为品牌（如某卖家品牌名就叫 "Dress" → 搜索词 dress 被标 st_brand_label=1），
        #   而专用白名单又没收录 dress，导致 dress（rank 108 的真实热词）被错误砍掉，
        #   反而留下错拼词 dresse。直接去掉这层过滤，让所有词正常通过，由后续 dedup/黑名单/主题过滤把关。
        # 注意：去掉后纯品牌搜索词（如 "anrabess dress"）也会进模板词集合，业务若不接受需重新设计品牌识别
        #   （根因在上游 dim_st_brand_info 品牌库被常用词污染，要先清洗上游再谈下游怎么用）。
        # df_without_brand_words = self.df_base_filter_date.filter('st_brand_label = 0')
        # df_brand_words = self.df_base_filter_date.filter('st_brand_label = 1')
        # df_brand_words = df_brand_words.withColumn(
        #     'brand_match_detail', self.udf_theme_regex(self.brand_pattern)(F.col("search_term"))
        # ).filter('brand_match_detail is not null').drop('brand_match_detail')
        # # 将处理后的品牌词与非品牌词合并
        # self.df_base_filter_date = df_without_brand_words.unionByName(df_brand_words)

        # 处理黑名单词逻辑
        pd_match_blacklist = self.df_match_blacklist.toPandas()
        self.df_base_filter_date = self.df_base_filter_date.withColumn(
            'st_blacklist_flag', self.filter_blacklist_words(pd_match_blacklist)("search_term")
        ).filter('st_blacklist_flag != 1').cache()

    def _build_word_to_base_mapping(self, df_st_base, df_pat_base):
        """
        Driver 端构建权威 word → base_form 映射并 broadcast。
        子词侧 + 模板词侧 union 后归一，保证两侧用同一份规则；复用 to_base_form（带白名单）。
        """
        # ============================================================
        # Step 1：子词侧 + 模板词侧 union 后 distinct
        # 为什么 union：保证两侧归一规则一致，避免同一个词在两侧 base_form 不同 → join miss
        # 拆词用 r'\W+'（非 word 字符全当分隔符），与下游 explode 完全一致
        #   例 "tree-skirt" → ["tree","skirt"]（对齐原 rlike `\btree\b` 在 "tree-skirt" 命中行为）
        #   例 "5tree"      → ["5tree"]（数字+字母粘连不拆，与 `\btree\b` 不命中一致）
        # ============================================================
        df_st_words = df_st_base.select(
            F.explode(F.split(F.lower('search_term'), r'\W+')).alias('word')
        )
        df_pat_words = df_pat_base.select(
            F.explode(F.split(F.lower('search_term'), r'\W+')).alias('word')
        )
        df_all = df_st_words.union(df_pat_words) \
            .filter((F.col('word') != '') & F.col('word').isNotNull()) \
            .distinct() \
            .cache()
        cnt = df_all.count()
        print(f"distinct word 总数（子词+模板词 union）: {cnt}")

        # ============================================================
        # Step 2：toPandas 拉到 driver 单线程归一
        # 为什么 driver 端跑：inflect/WordNet 没有 Spark UDF 版本，row-by-row PythonUDF 序列化开销
        #   反而比 driver 单线程还慢；distinct word ~10w 量级，driver 处理可控（1~2 分钟）
        # 与 dws_aba_word_heat._build_word_to_base_mapping 同款做法
        # ============================================================
        pdf = df_all.toPandas()
        df_all.unpersist()

        p = inflect.engine()
        mappings = [(w, to_base_form(w, p)) for w in pdf['word']]

        # ============================================================
        # Step 3：list + 显式 schema 创建 DataFrame，broadcast 出去
        # 为什么 broadcast：下游子词侧/模板词侧 join 都用这份映射，小表（几 MB），无 shuffle
        # 为什么 list+schema 而不是 createDataFrame(pdf)：pandas 2.x 移除 iteritems，PySpark 3.1.x
        #   内部还在调，走 list 方式与 pandas 版本完全解耦
        # ============================================================
        df_map = self.spark.createDataFrame(mappings, schema=['word', 'base_form'])
        return F.broadcast(df_map)

    def handle_st_filter_table(self):
        # 过滤特殊词
        self.df_base_filter_date = self.df_base_filter_date.join(
            self.df_theme, on=['search_term'], how='left'
        ).withColumn(
            "filter_flag", self.u_filter_pattern_words(F.col("search_term"), F.col("pattern_list"))
        ).filter(
            "filter_flag != 1"
        ).select(
            'search_term', 'st_word_num', 'st_bsr_cate_1_id_new', 'st_bsr_cate_current_id_new', 'rank', 'rank_change_rate', 'rank_rate_of_change'
        ).cache()

        df_st_filter_base = self.df_st_base.select(
            F.col('st_key'),
            F.col('search_term'),
            F.col('bsr_orders'),
            F.lit(self.site_name).alias('site_name'),
            F.lit(self.date_type).alias('date_type'),
            F.lit(self.date_info).alias('date_info')
        ).cache()

        # 落表路径校验（新旧实现都需要：写入前先清当前分区，保证幂等）
        del_hdfs_path = CommonUtil.build_hdfs_path('tmp_pattern_st_info', partition_dict=self.partition_dict)
        print(f"清除hdfs目录中:{del_hdfs_path}")
        HdfsUtils.delete_file_in_folder(del_hdfs_path)
        partition_by = ["site_name", "date_type", "date_info"]

        # ===== 原写法（逐模板词 rlike 全表扫描，保留备查，已停用）=====
        # 问题：每个模板词单独 rlike 扫一遍 df_st_filter_base，N 模板词(~3w) × 写批次(~150) 起步
        #   ≈ 3×10¹⁰ 次行级正则判定 → 单月跑批 1~2 天。
        # 雪上加霜：repartition(1) 写入单线程；150 次独立 saveAsTable append 各起一个 Spark job。
        # 详细问题/方案/口径对齐见
        #   Pyspark_job/spark_hjm/aba_month_new/分支B_搜索词主题标签/反查子体join化优化方案.md
        # pattern_words = self.df_base_filter_date.select('search_term')
        # # 将数据转换成pandas_df
        # dict_df = pattern_words.toPandas()
        # # 提取二级词和是否叠词标签转换成list[dict{}]
        # self.st_word_list = dict_df.to_dict(orient='records')
        # row_size = 40000
        # batch_size = 200
        # word_batches = [self.st_word_list[i:i + row_size] for i in range(0, len(self.st_word_list), row_size)]
        # for word_batch in word_batches:
        #     df_list = []  # 用于存储 DataFrame
        #     for row in word_batch:
        #         # 获取处理后的多级词
        #         pattern_st = row["search_term"]
        #         # 通过方法拆分，获取完全匹配的过滤条件
        #         filter_condition = self.st_word_filter_condition(pattern_st)
        #         filter_condition_expr = F.expr(filter_condition)
        #         df_union_filter = df_st_filter_base.filter(
        #             filter_condition_expr
        #         ).withColumn(
        #             "pattern_st", F.lit(pattern_st)
        #         )
        #         df_list.append(df_union_filter)
        #     for i in range(0, len(df_list), batch_size):
        #         print(f"当前是word_batches的轮回：f{word_batches.index(word_batch)}，当前写入表的df索引位置：{i + 1}")
        #         tmp_df = df_list[i:i + batch_size]
        #         result_df = self.udf_unionAll(*tmp_df)
        #         result_df = result_df.repartition(1)
        #         result_df.write.saveAsTable(name='tmp_pattern_st_info', format='hive', mode='append', partitionBy=partition_by)

        # ===== 新写法：单词级 broadcast join + 多重性判定 =====
        # 核心思路：把"子词 rlike 匹配模板词所有单词"改写成
        #   "拆词 → base_form 归一 → 计数 → 按 base_form join → 验证全 key 命中"，
        # 从 O(N模板 × N子词) 的暴力扫描降到 O(N子词)，预计从天级降到分钟级。

        # Step 1：构建权威 word → base_form 映射（driver 端 inflect 归一 + broadcast）
        # 复用 to_base_form（带 SEMANTIC_DISTINCT_WORDS 白名单）：
        #   - 白名单词（short/shorts/person/people 等）原样保留 → 与原 rlike `\b(short)\b` 不加复数对偶一致
        #   - 非白名单（girl/girls/dress/dresses）归到同一 base → 与原 rlike `\b(girl|girls)\b` 双形态等价
        df_word_to_base = self._build_word_to_base_mapping(df_st_filter_base, self.df_base_filter_date)

        # Step 2：子词侧拆词 + 归一 + 按 (st_key, base_form) 计数
        # 例：search_term="cashmere pashmina wrap"
        #   → 拆词 [cashmere, pashmina, wrap] → 归一 [cashmere, pashmina, wrap]
        #   → 3 行 (st_key=X, ..., base_form=cashmere, st_cnt=1) / (...,pashmina,1) / (...,wrap,1)
        # join 不上 word_to_base 时 coalesce 用原词兜底（理论上 Step1 已 union 全量、不会发生）
        df_st_word_cnt = df_st_filter_base.select(
            'st_key', 'search_term', 'bsr_orders',
            F.explode(F.split(F.lower('search_term'), r'\W+')).alias('word')
        ).filter("word != ''") \
         .join(df_word_to_base, on='word', how='left') \
         .withColumn('base_form', F.coalesce(F.col('base_form'), F.col('word'))) \
         .groupBy('st_key', 'search_term', 'bsr_orders', 'base_form') \
         .agg(F.count('*').alias('st_cnt'))

        # Step 3：模板词侧同款拆词 + 归一 + 按 (pattern_st, base_form) 计数
        # 例：pattern_st="red dress" → 拆出 {red:1, dress:1}
        #     pattern_st="cashmere"  → {cashmere:1}
        # 叠词模板（"gun gun"）已在 read_data 阶段被 udf_judge_twin_words 过滤，这里不会出现
        df_pat_word_cnt = self.df_base_filter_date.select(
            F.col('search_term').alias('pattern_st'),
            F.explode(F.split(F.lower('search_term'), r'\W+')).alias('word')
        ).filter("word != ''") \
         .join(df_word_to_base, on='word', how='left') \
         .withColumn('base_form', F.coalesce(F.col('base_form'), F.col('word'))) \
         .groupBy('pattern_st', 'base_form') \
         .agg(F.count('*').alias('pat_cnt'))

        # 模板词需要满足的 distinct base_form 总数（Step5 "全 key 命中"判定用）
        # 例：pattern_st="red dress" → total_keys=2；pattern_st="cashmere" → 1
        df_pat_total_keys = df_pat_word_cnt.groupBy('pattern_st').agg(
            F.count('*').alias('total_keys')
        )

        # Step 4：broadcast 模板词侧（小表 ~4w 行）+ inner join 子词侧 on base_form
        # 为什么 broadcast：模板词 base_form 表 ~4w 行（3w 模板 × 1.5 词），几 MB 可全量广播；
        #   子词侧（百万级）无需 shuffle，避免高频 base_form（"for"/"set"/"with"）倾斜爆炸。
        # filter st_cnt >= pat_cnt：满足多重性要求（"tree tree"需子词至少 2 个 tree）
        df_match = df_st_word_cnt.join(
            F.broadcast(df_pat_word_cnt),
            on='base_form', how='inner'
        ).filter(F.col('st_cnt') >= F.col('pat_cnt'))

        # Step 5：按 (pattern_st, st_key) 汇总命中的 distinct base_form 数 → 与 total_keys 比对
        # matched_keys == total_keys → 该子词包含模板词的所有单词（顺序无关，与原 rlike 一致）
        # 例：pattern_st="red dress" (total_keys=2)
        #   子词 "red dress for women" 命中 {red, dress} → matched_keys=2 → 保留 ✓
        #   子词 "red shoes"           只命中 {red}      → matched_keys=1 → 过滤 ✗
        df_matched_keys = df_match.groupBy('pattern_st', 'st_key', 'search_term', 'bsr_orders').agg(
            F.countDistinct('base_form').alias('matched_keys')
        )
        df_result = df_matched_keys.join(
            F.broadcast(df_pat_total_keys), on='pattern_st', how='inner'
        ).filter(F.col('matched_keys') == F.col('total_keys')) \
         .select('st_key', 'search_term', 'bsr_orders', 'pattern_st')

        # Step 6：补分区字段 + 合理并行度一次性写入
        # 取代原来 150 次 append + repartition(1) → 1 次 saveAsTable + 20 分区并行写
        df_result = df_result.withColumn('site_name', F.lit(self.site_name)) \
                             .withColumn('date_type', F.lit(self.date_type)) \
                             .withColumn('date_info', F.lit(self.date_info)) \
                             .repartition(20)
        df_result.write.saveAsTable(
            name='tmp_pattern_st_info', format='hive', mode='append',
            partitionBy=partition_by
        )

        sql = f"""
            select
               st_key,
               search_term,
               bsr_orders,
               pattern_st
            from big_data_selection.tmp_pattern_st_info
            where site_name = '{self.site_name}'
              and date_type = '{self.date_type}'
              and date_info = '{self.date_info}'
        """
        self.df_pattern_st_words = self.spark.sql(sql).cache()
        self.df_pattern_st_words.show(10, truncate=True)

    def handle_st_pattern_common_agg(self):
        # 计算模板词的总销量和匹配到的ABA词个数
        self.df_pattern_st_agg = self.df_pattern_st_words.groupBy(['pattern_st']).agg(
            F.sum("bsr_orders").alias("pattern_bsr_orders_total"),
            F.count("search_term").alias("pattern_st_count")
        ).join(
            self.df_base_filter_date, on=self.df_pattern_st_words.pattern_st == self.df_base_filter_date.search_term
        ).cache()

        # 将模板词与搜索词主题标签关联
        # pattern_type=0的情况
        df_common_st_theme = self.df_st_theme_base.filter("pattern_type = 0")
        self.df_st_theme_agg = self.df_pattern_st_words.join(
            df_common_st_theme, on=['st_key', 'search_term'], how='left'
        ).filter("theme_en is not null")

        # 进行分组累加（按照匹配词中文进行累加，业务要求中文含义为准计数）
        self.df_st_theme_agg = self.df_st_theme_agg.groupBy(['pattern_st', 'theme_label_ch', 'theme_en', 'theme_ch']).agg(
            F.count("st_key").alias("theme_label_counts"),
            F.sum("bsr_orders").alias("theme_label_bsr_orders"),
            F.collect_set("theme_label_en").alias("theme_label_en_list")
        )
        # 转换成字符串拼接
        self.df_st_theme_agg = self.df_st_theme_agg.withColumn(
            'label_en_str', F.concat_ws("/", F.col('theme_label_en_list'))
        )
        # 关联模板词的聚合数据
        self.df_st_theme_agg = self.df_st_theme_agg.join(
            self.df_pattern_st_agg, on=['pattern_st'], how='left'
        )
        # 计算占比
        self.df_st_theme_agg = self.df_st_theme_agg.withColumn(
            'pattern_bsr_orders_rate',
            F.when(
                F.col('pattern_bsr_orders_total') > 0,
                F.round(F.col('theme_label_bsr_orders') / F.col('pattern_bsr_orders_total'), 4)
            ).otherwise(F.lit(0.0))
        ).withColumn(
            'pattern_num_rate',
            F.when(
                F.col('pattern_st_count') > 0,
                F.round(F.col('theme_label_counts') / F.col('pattern_st_count'), 4)
            ).otherwise(F.lit(0.0))
        )

        self.df_st_theme_agg = self.df_st_theme_agg.select(
            F.col('pattern_st'),
            F.col('pattern_bsr_orders_total'),
            F.col('pattern_st_count'),
            F.col('theme_ch'),
            F.col('theme_en'),
            F.col('theme_label_ch'),
            F.col('label_en_str').alias('theme_label_en'),
            F.col('theme_label_bsr_orders'),
            F.col('theme_label_counts'),
            F.col('pattern_bsr_orders_rate'),
            F.col('pattern_num_rate'),
            F.col('st_word_num'),
            F.col('st_bsr_cate_1_id_new'),
            F.col('st_bsr_cate_current_id_new'),
            F.col('rank'),
            F.col('rank_change_rate'),
            F.col('rank_rate_of_change')
        ).cache()

    def handle_st_pattern_special_agg(self):
        # 将模板词与搜索词主题标签关联
        # pattern_type=1的情况
        df_special_st_theme = self.df_st_theme_base.filter("pattern_type = 1")
        self.df_st_match_topic_detail = self.df_pattern_st_words.join(
            df_special_st_theme, on=['st_key', 'search_term'], how='left'
        ).filter("theme_label_en is not null")

        df_st_match_agg = self.df_st_match_topic_detail.groupby(
            ['pattern_st', 'theme_ch', 'theme_en', 'theme_label_ch', 'theme_label_num_info', 'theme_label_unit_info']
        ).agg(
            F.count('bsr_orders').alias("same_info_count"),
            F.sum('bsr_orders').alias("same_info_bsr_orders")
        ).cache()
        df_st_match_no_num_agg = df_st_match_agg.filter("theme_label_num_info is null")
        df_st_match_no_num_info = df_st_match_no_num_agg.groupby(
            ['pattern_st', 'theme_ch', 'theme_en', 'theme_label_ch', 'theme_label_unit_info']
        ).agg(
            F.sum('same_info_count').alias("st_label_num"),
            F.sum('same_info_bsr_orders').alias("st_label_bsr_orders"),
            F.col('theme_label_unit_info').alias("label_info")
        ).drop("theme_label_unit_info")
        df_st_match_no_unit_agg = df_st_match_agg.filter("theme_label_unit_info in ('x', 'by')")
        df_st_match_no_unit_info = df_st_match_no_unit_agg.groupby(
            ['pattern_st', 'theme_ch', 'theme_en', 'theme_label_ch', 'theme_label_num_info']
        ).agg(
            F.sum('same_info_count').alias("st_label_num"),
            F.sum('same_info_bsr_orders').alias("st_label_bsr_orders"),
            F.col("theme_label_num_info").alias("label_info")
        ).drop("theme_label_num_info")
        df_st_match_complete_agg = df_st_match_agg.filter(
            (F.col("theme_label_num_info").isNotNull()) & (F.col("theme_label_unit_info").isNotNull()) &
            (F.col("theme_label_unit_info") != 'x') & (F.col("theme_label_unit_info") != 'by')
        )
        df_st_match_complete_agg = df_st_match_complete_agg.withColumn(
            "complete_info", F.concat_ws(' ', F.col("theme_label_num_info"), F.col("theme_label_unit_info"))
        )
        df_st_match_complete_info = df_st_match_complete_agg.groupby(
            ['pattern_st', 'theme_ch', 'theme_en', 'theme_label_ch', 'theme_label_num_info']
        ).agg(
            F.sum('same_info_count').alias("st_label_num"),
            F.sum('same_info_bsr_orders').alias("st_label_bsr_orders"),
            F.concat_ws("/", F.collect_set(F.col("complete_info"))).alias("label_info")
        ).drop("theme_label_num_info")
        self.df_st_match_topic_agg = df_st_match_no_num_info.unionByName(
            df_st_match_no_unit_info
        ).unionByName(
            df_st_match_complete_info
        ).join(
            self.df_pattern_st_agg, on=['pattern_st'], how='left'
        )
        self.df_st_match_topic_agg = self.df_st_match_topic_agg.withColumn(
            "pattern_bsr_orders_rate",
            F.when(
                F.col("pattern_bsr_orders_total") > 0,
                F.round((F.col("st_label_bsr_orders") / F.col("pattern_bsr_orders_total")), 4)
            ).otherwise(F.lit(0.0))
        ).withColumn(
            "pattern_num_rate",
            F.when(
                F.col("pattern_st_count") > 0,
                F.round((F.col("st_label_num") / F.col("pattern_st_count")), 4)
            ).otherwise(F.lit(0.0))
        )

        self.df_st_match_topic_agg = self.df_st_match_topic_agg.select(
            F.col('pattern_st'),
            F.col('pattern_bsr_orders_total'),
            F.col('pattern_st_count'),
            F.col('theme_ch'),
            F.col('theme_en'),
            F.col('theme_label_ch'),
            F.col('label_info').alias('theme_label_en'),
            F.col('st_label_bsr_orders').alias('theme_label_bsr_orders'),
            F.col('st_label_num').alias('theme_label_counts'),
            F.col('pattern_bsr_orders_rate'),
            F.col('pattern_num_rate'),
            F.col('st_word_num'),
            F.col('st_bsr_cate_1_id_new'),
            F.col('st_bsr_cate_current_id_new'),
            F.col('rank'),
            F.col('rank_change_rate'),
            F.col('rank_rate_of_change')
        ).cache()

    def save_data(self):
        hdfs_path_asin_info = CommonUtil.build_hdfs_path(self.hive_tb, partition_dict=self.partition_dict)
        print(f"清除hdfs目录中:{hdfs_path_asin_info}")
        HdfsUtils.delete_file_in_folder(hdfs_path_asin_info)
        self.df_st_theme_agg = self.df_st_theme_agg.unionByName(self.df_st_match_topic_agg)
        # 如果模板词本身也有匹配词，则相应的统计需要过滤掉（把"模板词自己名字里就含的属性"从它的反查子体里剔除）

        # ===== 原写法（保留备查，已停用）=====
        # 问题：自匹配按英文 theme_label_en 配对。但聚合侧的 theme_label_en 是 concat_ws("/", ...) 拼接串
        #   （如 "cashmere/pashmina"、"cars/automotive/auto/automobile"——多个同义词/单复数/拼写变体都映射到同一个中文标签，
        #    在 handle_st_pattern_common_agg 里被 collect_set 收进同一组拼起来），而明细侧是单个英文词。
        #   "cashmere/pashmina" == "cashmere" 永远不成立 → 凡是命中多个同义词的属性组都漏删（普遍存在，历史月份均有）。
        # df_agg_filter = self.df_st_theme_base.select(
        #     F.col('search_term'),
        #     F.col('theme_label_en').alias('theme_label_en_join'),
        #     F.lit(1).alias('join_flag')
        # )
        # self.df_st_theme_agg = self.df_st_theme_agg.join(
        #     df_agg_filter, on=(self.df_st_theme_agg.pattern_st == df_agg_filter.search_term) & (self.df_st_theme_agg.theme_label_en == df_agg_filter.theme_label_en_join), how='left'
        # ).filter(F.col('join_flag').isNull())

        # ===== 新写法：按中文标签 theme_label_ch + 主题类别 theme_ch 配对，用 left_anti 做减法 =====
        # 为什么按 theme_label_ch（中文含义）而不是英文拼接串：
        #   聚合本就是按 theme_label_ch 分组的，一个 "/" 组里的英文词必然共享同一个中文标签（这正是它们被收进一组的原因），
        #   所以按中文含义配对、删掉整组是精确的——整组本就都是同一个含义，不会误删别的含义。
        #   且 theme_label_ch 是单值、不受英文 "/" 拼接影响，从根上避开漏删。
        # 为什么再加 theme_ch：万一同一个中文标签挂在两个不同主题类别下，只按 theme_label_ch 会把另一类的行也误删，
        #   加 theme_ch 精确到"同一主题类别下的同一中文标签"（防御性加固，正常数据用不到）。
        # 为什么用 left_anti：语义就是"聚合表 减去 模板词自身命中的 (中文标签, 主题类别)"，
        #   且对右表重复天然不敏感（无需 distinct 防放大），比 left join + filter(flag is null) 更直观。
        df_self_label = self.df_st_theme_base.select(
            F.col('search_term').alias('self_search_term'),
            F.col('theme_label_ch').alias('self_theme_label_ch'),
            F.col('theme_ch').alias('self_theme_ch')
        )
        self.df_st_theme_agg = self.df_st_theme_agg.join(
            df_self_label,
            on=(self.df_st_theme_agg.pattern_st == df_self_label.self_search_term) &
               (self.df_st_theme_agg.theme_label_ch == df_self_label.self_theme_label_ch) &
               (self.df_st_theme_agg.theme_ch == df_self_label.self_theme_ch),
            how='left_anti'
        )

        self.df_st_theme_agg = self.df_st_theme_agg.select(
            F.col('pattern_st'),
            F.col('pattern_bsr_orders_total'),
            F.col('pattern_st_count'),
            F.col('theme_ch'),
            F.col('theme_en'),
            F.col('theme_label_ch'),
            F.col('theme_label_en'),
            F.col('theme_label_bsr_orders'),
            F.col('theme_label_counts'),
            F.col('pattern_bsr_orders_rate'),
            F.col('pattern_num_rate'),
            F.date_format(F.current_timestamp(), 'yyyy-MM-dd HH:mm:SS').alias('created_time'),
            F.date_format(F.current_timestamp(), 'yyyy-MM-dd HH:mm:SS').alias('updated_time'),
            F.col('st_word_num').alias('pattern_word_num'),
            F.col('st_bsr_cate_1_id_new').alias('pattern_category_id'),
            F.col('st_bsr_cate_current_id_new').alias('pattern_category_current_id'),
            F.col('rank').alias('pattern_rank'),
            F.col('rank_change_rate').alias('pattern_rank_change_rate'),
            F.col('rank_rate_of_change').alias('pattern_rank_rate_of_change'),
            F.lit(self.site_name).alias('site_name'),
            F.lit(self.date_type).alias('date_type'),
            F.lit(self.date_info).alias('date_info')
        )

        self.df_st_theme_agg = self.df_st_theme_agg.repartition(20)
        partition_by = ["site_name", "date_type", "date_info"]
        print(f"当前存储的表名为：{self.hive_tb},分区为{partition_by}", )
        self.df_st_theme_agg.write.saveAsTable(name=self.hive_tb, format='hive', mode='append', partitionBy=partition_by)
        print("success")


if __name__ == '__main__':
    site_name = CommonUtil.get_sys_arg(1, None)
    date_type = CommonUtil.get_sys_arg(2, None)
    date_info = CommonUtil.get_sys_arg(3, None)  # 参数3：年-周/年-月/年-季/年-月-日, 比如: 2022-1
    assert site_name is not None, "site_name 不能为空！"
    assert date_type is not None, "date_type 不能为空！"
    assert date_info is not None, "date_info 不能为空！"
    obj = DwtStThemeAgg(site_name=site_name, date_type=date_type, date_info=date_info)
    obj.handle_data()
