import os
import sys
import re

import inflect

sys.path.append(os.path.dirname(sys.path[0]))
from pyspark.sql.types import StringType, MapType, IntegerType, ArrayType
from utils.common_util import CommonUtil
from utils.hdfs_utils import HdfsUtils
from utils.spark_util import SparkUtil
from utils.db_util import DBUtil
from pyspark.sql import functions as F
from yswg_utils.common_udf import udf_handle_string_null_value
from utils.templates import Templates
# 单复数归一 / 白名单 / 不规则复数表统一从公共模块引入（原先各文件各写一份，已抽到 utils.word_normalize）
from utils.word_normalize import to_base_form, IRREGULAR_PLURAL, SEMANTIC_DISTINCT_WORDS


class DwsStTheme(Templates):
    def __init__(self, site_name, date_type, date_info):
        super().__init__()
        self.site_name = site_name
        self.date_type = date_type
        self.date_info = date_info
        app_name = f"{self.__class__.__name__}:{site_name},{date_type},{date_info}"
        self.spark = SparkUtil.get_spark_session(app_name)
        self.db_save = "dws_st_theme"
        self.partition_dict = {
            "site_name": site_name,
            "date_type": date_type,
            "date_info": date_info
        }
        hdfs_path = CommonUtil.build_hdfs_path(self.db_save, partition_dict=self.partition_dict)
        print(f"清除hdfs目录中:{hdfs_path}")
        HdfsUtils.delete_file_in_folder(hdfs_path)
        self.partitions_by = ["site_name", "date_type", "date_info"]
        self.reset_partitions(1)

        # 全局df初始化
        self.df_st_asin_info = self.spark.sql(f"select 1+1;")
        self.df_st_detail = self.spark.sql(f"select 1+1;")
        self.df_st_key = self.spark.sql(f"select 1+1;")
        self.df_theme = self.spark.sql(f"select 1+1;")
        self.df_st_base = self.spark.sql(f"select 1+1;")
        self.df_st_theme = self.spark.sql(f"select 1+1;")
        self.df_st_theme_vertical = self.spark.sql(f"select 1+1;")
        self.df_st_save = self.spark.sql(f"select 1+1;")
        self.df_st_topic_base = self.spark.sql(f"select 1+1;")
        self.df_st_match_topic_detail = self.spark.sql(f"select 1+1;")
        self.df_save = self.spark.sql(f"select 1+1;")
        self.topic_rules_regexp_dict = self.create_regexp_rules()

        # 注册自定义函数 (UDF)
        self.u_theme_contain_judge = F.udf(self.u_theme_contain_judge, IntegerType())
        self.u_handle_string_null_value = F.udf(udf_handle_string_null_value, StringType())

    @staticmethod
    def parse_ele_match_regexp(pattern):
        def udf_ele_mattch(match_text):
            ele_list = re.findall(pattern, match_text)
            if ele_list:
                return ','.join(set(ele_list))
            else:
                return None

        return F.udf(udf_ele_mattch, StringType())

    # 通过主题正则关系表拼接正则规范
    @staticmethod
    def create_regexp_rules():
        theme_rules_regexp_dict = {}
        engine = DBUtil.get_db_engine('mysql', 'us')
        # GROUP_CONCAT 按长度降序：让长标签在 | 交替组里排在它的"词前缀"短标签前面
        # type 0 正则形如 \b(plus|plus size)\b，交替组后面没有必匹配内容（没有回溯保护），
        #   短标签 plus 先匹配成功后引擎不会回溯，plus size 永远没机会 → 标签错打成 plus
        #   （与 handle_st_theme 词典大正则的排序修复是同一个病；实测规则表有 plus/plus size、
        #    heavy/heavy duty、full/full length 三对词前缀冲突，都在 尺寸 type 0）
        # type 1-6 交替组后面有必匹配的数字等内容，短标签选错会回溯重试长标签，加排序无影响也无害
        rules_sql = f"""
                       select theme_ch,regular_expression_type, GROUP_CONCAT(label_en_lower ORDER BY LENGTH(label_en_lower) DESC) as key_list from aba_match_theme_rules group by theme_ch, regular_expression_type;
                   """
        with engine.connect() as connect:
            rules_result = connect.execute(rules_sql)
            for rules_row in rules_result:
                if rules_row.regular_expression_type == 0:
                    regexp_str = r"(?<!\+|\*|\-|\%|\.|\')\b(" + "|".join(str(rules_row.key_list).split(',')) + r")\b"
                elif rules_row.regular_expression_type == 1:
                    regexp_str = r"(\d+(?:\.\d+)?) +(" + "|".join(str(rules_row.key_list).split(',')) + r")\b"
                elif rules_row.regular_expression_type == 2:
                    regexp_str = r"\b(" + "|".join(str(rules_row.key_list).split(',')) + r") +(\d+(?:\.\d+)?)"
                elif rules_row.regular_expression_type == 3:
                    regexp_str = r"(\d+(?:\.\d+)?)(" + "|".join(str(rules_row.key_list).split(',')) + r")\b"
                elif rules_row.regular_expression_type == 4:
                    regexp_str = r"(\d+(?:\.\d+)?) *(" + "|".join(
                        str(rules_row.key_list).split(',')) + r") *(\d+(?:\.\d+)?)(?: |$)"
                elif rules_row.regular_expression_type == 5:
                    regexp_str = r"(\d+(?:\.\d+)?) *(-) *(\d+(?:\.\d+)?) *(" + "|".join(
                        str(rules_row.key_list).split(',')) + r")\b"
                else:
                    regexp_str = r"\b(" + "|".join(
                        str(rules_row.key_list).split(',')) + r") +(\d+(?:\.\d+)?) *(-) *(\d+(?:\.\d+)?)"
                if rules_row.theme_ch in theme_rules_regexp_dict:
                    rules_list = theme_rules_regexp_dict.get(rules_row.theme_ch)
                    rules_list.append(regexp_str)
                    theme_rules_regexp_dict[rules_row.theme_ch] = rules_list
                else:
                    rules_list = []
                    rules_list.append(regexp_str)
                    theme_rules_regexp_dict[rules_row.theme_ch] = rules_list
            rules_list = theme_rules_regexp_dict['尺寸']
            rules_list.append(r'\b(\d+) *(\'\')')
        connect.close()
        return theme_rules_regexp_dict

    @staticmethod
    def parse_search_term_theme(topic_rules_regexp_dict):
        def parse_theme(search_term):
            if len(topic_rules_regexp_dict) > 0:
                matches_map = {}
                for theme_rule in topic_rules_regexp_dict.keys():
                    matches_list = []
                    rules_regexp_list = topic_rules_regexp_dict[theme_rule]
                    for rules_regexp in rules_regexp_list:
                        matches = re.findall(rules_regexp, search_term, re.IGNORECASE)
                        if len(matches) > 0:
                            matches_list.append(matches[0])
                    if len(matches_list) > 0:
                        matches_map[theme_rule] = matches_list
                if len(matches_map) > 0:
                    theme_array = []
                    for theme_rule in matches_map.keys():
                        matches_list = matches_map[theme_rule]
                        for target_matches in matches_list:
                            num_info = ""
                            unit_info = ""
                            num_pattern = r'^\d+(\.\d+)?$'
                            # target_matches = max(matches_list, key=len)
                            if isinstance(target_matches, tuple):
                                match_low = [match.lower() for match in target_matches]
                            else:
                                match_low = target_matches
                            if isinstance(match_low, str):
                                unit_info = match_low
                            else:
                                if len(match_low) == 2:
                                    if re.match(num_pattern, str(match_low[0])):
                                        num_info = match_low[0]
                                        unit_info = match_low[1]
                                    else:
                                        num_info = match_low[1]
                                        unit_info = match_low[0]
                                elif len(match_low) == 3:
                                    num_info = "".join(match_low)
                                    unit_info = match_low[1]
                                elif len(match_low) == 4:
                                    if re.match(num_pattern, str(match_low[0])):
                                        num_info = "".join(match_low[0:3])
                                        unit_info = match_low[3]
                                    else:
                                        unit_info = match_low[0]
                                        num_info = "".join(match_low[1:4])
                            theme_array.append(
                                {"theme": str(theme_rule), "num_info": str(num_info), "unit_info": str(unit_info)})
                    return theme_array

        return F.udf(parse_theme, ArrayType(MapType(StringType(), StringType())))

    @staticmethod
    def u_theme_contain_judge(pattern_word, pattern_list):
        num_pattern = r'^\d'
        pattern_flag = bool(re.match(num_pattern, pattern_word))
        count = 0
        if not pattern_flag:
            # set() 去重后再数：本意是"被另一个更长的标签整词包含才删"（AB词去重，如 make 被 make up 包含）。
            # 但同一个匹配词可能经"关键词词典(pattern_type=0)"+"数字规则表(pattern_type=1)"两条路各命中一次，
            #   在 pattern_list 里出现两次（如 weighted 既在 keywords_match_theme=功能、又在 aba_match_theme_rules=尺寸）。
            #   老逻辑按出现次数 count，"自己撞自己"也会 count=2 被误判成"已被包含"→整条搜索词标签被删光。
            #   去重后只有"存在另一个不同的、更长的标签包含本词"时才 count>1，真AB词去重不受影响。
            count = sum(1 for word in set(pattern_list) if re.search(r'\b{}\b'.format(re.escape(pattern_word)), word))
        # 如果匹配到的pattern_word大于1则说明有已经匹配过的单词
        return 0 if count > 1 else 1

    def read_data(self):
        sql1 = f"""
            select 
                search_term
            from dim_st_asin_info
            where site_name = '{site_name}'
            and date_type = '{date_type}'
            and date_info = '{date_info}';
        """
        print("sql:", sql1)
        self.df_st_asin_info = self.spark.sql(sql1).repartition(40, 'search_term').cache()

        sql2 = f"""
            select 
                search_term
            from dim_st_detail
            where site_name = '{site_name}'
            and date_type = '{date_type}'
            and date_info = '{date_info}';
        """
        print("sql:", sql2)
        self.df_st_detail = self.spark.sql(sql2).repartition(40, 'search_term').cache()

        sql3 = f"""
            select 
                st_key,
                search_term
            from ods_st_key
            where site_name = '{site_name}';
        """
        print("sql:", sql3)
        self.df_st_key = self.spark.sql(sql3).repartition(40, 'search_term').cache()

        # 获取主题词
        # 词典源从 MySQL selection.aba_match_theme 改为 PG 集群 public.keywords_match_theme
        # 改动原因：业务方词典统一迁移到 PG 集群维护，跟现有 PG 数据架构对齐
        sql4 = f"""
            select
                theme_en,
                theme_ch,
                label_ch,
                label_en_lower
            from public.keywords_match_theme
            where label_ch is not null
        """
        print("sql:", sql4)
        conn_info = DBUtil.get_connection_info("postgresql_cluster", "us")
        self.df_theme = SparkUtil.read_jdbc_query(
            session=self.spark,
            url=conn_info["url"],
            pwd=conn_info["pwd"],
            username=conn_info["username"],
            query=sql4
        )

        # 词典主题双值拆分：theme_ch / theme_en 成对的逗号双值（如 "主题,风格" ↔ "topic,style"）拆成多行
        # 必须放在单复数扩展之前：让下游 collect_set 去重、sort_array 排序都基于单个原子主题
        self.df_theme = self._split_theme_multi_value(self.df_theme)

        # 词典单复数双向扩展（替代业务方手工录复数列）
        # 业务问题：业务方录入只覆盖一种形式（girl 或 girls），搜索词里的另一种形式正则匹配不上
        # 解决方案：driver 端用 inflect 双向扩展原词 + base + plural 三种形式
        # 性能：1w 多行词典在 driver 单机跑只需几秒
        self.df_theme = self._expand_theme_with_plural(self.df_theme)

    def _split_theme_multi_value(self, df_theme):
        """
        词典主题双值拆分：把 theme_ch / theme_en 里逗号拼接的双值（成对一一对应）按位置拆成多行。
        修复早期 bug：下游 dwt_aba_st_analytics 用 concat_ws(',', collect_set(theme_ch)) 算 st_attribute_label，
        若 theme_ch 存的是整块 "主题,风格"，collect_set 按整串去重 → "风格" 这种重复值去不掉、且拆不开。
        """
        # ============================================================
        # Step 1：全角逗号 → 半角逗号，统一分隔符
        # 为什么：业务方录入两种逗号都可能出现（"主题,风格" / "主题，风格"），
        #   先归一成半角 ","，下面 split 只需按一种分隔符处理
        # ============================================================
        df_theme = df_theme.withColumn(
            "theme_en", F.regexp_replace(F.col("theme_en"), "，", ",")
        ).withColumn(
            "theme_ch", F.regexp_replace(F.col("theme_ch"), "，", ",")
        )

        # ============================================================
        # Step 2：theme_en / theme_ch 各自 split 成数组，再 arrays_zip 按位置配对后 explode
        # 为什么用 arrays_zip：两个字段是"成对双值"（theme_en="topic,style" 对 theme_ch="主题,风格"），
        #   必须按下标一一对应配对，不能各自 explode（那样会变成 2×2 笛卡尔积错配）
        # 例：theme_en="topic,style", theme_ch="主题,风格"
        #     → zip 配对 [(topic,主题),(style,风格)] → explode 成 2 行
        # 单值行（无逗号）：split 得长度 1 数组 → explode 仍是 1 行，不受影响
        # ============================================================
        df_theme = df_theme.withColumn(
            "theme_pair",
            F.explode(
                F.arrays_zip(
                    F.split(F.col("theme_en"), ",").alias("theme_en"),
                    F.split(F.col("theme_ch"), ",").alias("theme_ch")
                )
            )
        )

        # ============================================================
        # Step 3：取出配对后的单值，trim 去掉 "主题, 风格" 逗号后的空格，并过滤空值
        # 为什么 trim：业务方录入逗号后常带空格（"主题, 风格"），不去掉会产出 " 风格" 脏标签
        # 为什么过滤空串：末尾多打逗号（"主题,"）会 split 出空元素，过滤掉避免落入空主题
        # ============================================================
        df_theme = df_theme.select(
            F.trim(F.col("theme_pair.theme_en")).alias("theme_en"),
            F.trim(F.col("theme_pair.theme_ch")).alias("theme_ch"),
            F.col("label_ch"),
            F.col("label_en_lower")
        ).filter("theme_ch is not null and theme_ch != '' and theme_en is not null and theme_en != ''")

        return df_theme

    def _expand_theme_with_plural(self, df_theme):
        """
        词典单复数双向扩展：业务方录"girl"自动补出"girls"（反之亦然），让搜索词两种形式都能匹配上。
        单词扩展 [原词, base, p lural] 三种形式；短语保留原词不扩展（避免错拆）。
        """
        # ============================================================
        # Step 1：toPandas 拉到 driver（词典约 1.1w 行，单机可承载）
        # 为什么在 driver 跑：inflect/WordNet 没有 spark UDF 版本，
        #   每行词典要单独跑归一+加复数，spark UDF 序列化开销反而比 driver 单线程慢
        # 性能：1.1w 行约 3-5 秒，可接受
        # ============================================================
        pdf = df_theme.toPandas()
        p = inflect.engine()

        expanded = []          # 最终收集的 (variant, theme_en, theme_ch, label_ch) 行
        seen = set()           # 去重 (variant, theme_en) 组合，避免重复入表
        phrase_count = 0       # 统计短语数量（仅用于跑批日志，方便业务抽查）

        # ============================================================
        # Step 2：遍历词典每行，按"短语 vs 单词"两种情况扩展
        # ============================================================
        for row in pdf.itertuples(index=False):
            word = row.label_en_lower
            if not word:
                continue

            # --------------------------------------------------------
            # Step 2a：短语跳过扩展（关键 bug 防护）
            # 为什么：如果对短语按单词扩展，会错误拆出短语第一个词单独打标
            # 反例：词典 'dusty rose' → 颜色
            #       如果扩展会拆出 base='dusty'（仅第一个词）加进词典
            #       → 任何含 'dusty' 的搜索词（如 "dusty old furniture"）都被错标"颜色"
            # 正确做法：短语保留原词不扩展，让正则匹配整段短语
            # --------------------------------------------------------
            if word in SEMANTIC_DISTINCT_WORDS:
                # 白名单词只保留原词，不扩展单复数：
                # 否则词典录了 short(长度) 会扩展出 shorts，导致"短裤"搜索词被误打"长度"主题（跨语义污染）
                variants = {word}
            elif len(word.split()) > 1:
                phrase_count += 1
                variants = {word}
            else:
                # --------------------------------------------------------
                # Step 2b：单词扩展 [原词, 单数 base, 复数 plural] 三种形式
                # 为什么三种都加：业务方录哪种形式都行，三种 variant 都能匹配上搜索词
                # 例：业务方录 'girl'  → base='girl',  plural='girls'  → 扩展 [girl, girls]
                #     业务方录 'girls' → base='girl'（还原）, plural='girls' → 同样扩展 [girl, girls]
                # 不规则复数：业务方录 'goose' → [goose, geese]（inflect 内置支持）
                # --------------------------------------------------------
                base = to_base_form(word, p)
                try:
                    plural = p.plural(base) or base
                except Exception:
                    plural = base
                variants = {word, base, plural}  # set 自动去重（word==base==plural 时只保留 1 个）

                # 补不规则名词的"省撇号所有格"写法（womens/womans/mens/mans）：
                # ：woman 的复数是 women（不规则），inflect 产不出 womens；但搜索词里 womens/mens很多 dws 搜索词侧是【字面正则匹配、不做归一】，导致匹配不到数据
                #   在这条路用不上，只能反过来把这些写法补进词典变体，让搜索词的 womens 能字面撞上 women 的标签。
                #   例：base=woman → 补 {womens, womans}；base=man → 补 {mens, mans}
                # 只影响 base 落在 woman/man/tooth 的词，其它词不变，blast radius 极小
                variants |= {k for k, v in IRREGULAR_PLURAL.items() if v == base}

            # --------------------------------------------------------
            # Step 2c：把所有 variant 加入 expanded 列表（按 (variant, theme_en) 去重）
            # 为什么按 (variant, theme_en) 去重而不是只按 variant：
            #   同一个 word 在词典里可能对应多个 theme_en
            #   例：'rose' 既是 颜色(color) 又是 元素(element)
            #   按 (variant, theme_en) 去重既能保留两个 theme，
            #   又避免业务方同时录入 girl/girls 两行造成 (girl, crowd) 重复入表
            # --------------------------------------------------------
            for variant in variants:
                if not variant:
                    continue
                key = (variant, row.theme_en)
                if key in seen:
                    continue
                seen.add(key)
                expanded.append((variant, row.theme_en, row.theme_ch, row.label_ch))

        print(f"单复数扩展：词典原始 {len(pdf)} 行 → 扩展后 {len(expanded)} 行（其中短语 {phrase_count} 个未扩展）")

        # ============================================================
        # Step 3：list + 显式 schema 创建 DataFrame，避开 pandas iteritems bug
        # 为什么不走 createDataFrame(pdf)：pandas 2.x 移除了 DataFrame.iteritems()，
        #   PySpark 3.1.x 内部还在调用，会报 "'DataFrame' object has no attribute 'iteritems'"
        #   走 list + schema 方式与 pandas 版本完全解耦
        # cache 原因：下游 handle_st_theme 多次用到这个扩展后的词典 df
        # ============================================================
        return self.spark.createDataFrame(
            expanded, schema=['label_en_lower', 'theme_en', 'theme_ch', 'label_ch']
        ).cache()

    def handle_data(self):
        self.handle_base()
        self.handle_st_theme()
        self.match_st_topic()
        self.match_search_term_ch_unit()
        self.handle_contains_theme()
        self.handle_save()

    # 去重处理
    def handle_base(self):
        self.df_st_base = self.df_st_asin_info.unionByName(
            self.df_st_detail
        ).drop_duplicates(['search_term'])

        # 业务规则：单个单词的搜索词（如 gloves / wool）不打任何属性标签（关键词路 + 数字规则路都不打）。
        # 在这里（两条匹配路之前）过滤掉单词搜索词，让它们根本不参与匹配 → dws_st_theme 不产生其主题行。
        # 词数判定要稳，避免把"一个词"误算成两个：
        #   trim 去首尾空白 + 按连续空白 \s+ 拆 → "gloves"/"gloves "/"gloves  "(带尾/多空格) 都算 1 个词；
        #   只按空白拆、不按符号拆 → "men's"/"tree-skirt"(带符号无空格) 仍算 1 个词。
        #   词数 > 1 才保留（如 "wool gloves"）。
        self.df_st_base = self.df_st_base.filter(
            F.size(F.split(F.trim(F.col('search_term')), r'\s+')) > 1
        )

        self.df_st_base = self.df_st_base.join(
            self.df_st_key, on='search_term', how='inner'
        ).cache()

    # 给每个搜索词打上主题标签
    def handle_st_theme(self):
        pdf_theme = self.df_theme.toPandas()
        # 按长度降序排，让长短语优先于它的子词被匹配
        # 为什么：Python 正则 | 是"先到先得"而非最长匹配，且 set 顺序随机；
        #   如果 'mothers' 排在 'mothers day' 前面，findall 在位置 0 先吃掉 'mothers'（不重叠），
        #   'mothers day'（节日）就永远匹配不上 → 节日标签随机丢失
        # 例：'mothers day gifts' 修复前可能 → [mothers, gifts]（丢节日）
        #     长度降序后固定 → [mothers day, gifts] ✓（子词让位短语，与 AB 词去重设计意图一致）
        theme_list = sorted(set(pdf_theme.label_en_lower), key=len, reverse=True)
        pattern = re.compile(r'(?<!\+|\*|\-|\%|\.|\')\b({})\b'.format('|'.join([re.escape(x) for x in theme_list])),
                             flags=re.IGNORECASE)
        self.df_st_theme = self.df_st_base.withColumn(
            "theme_en_pattern_str",
            self.parse_ele_match_regexp(pattern)(F.col("search_term"))
        )
        # 将匹配到的字符串拆分成list
        self.df_st_theme = self.df_st_theme.withColumn(
            "label_en_lower_list",
            F.split(F.col("theme_en_pattern_str"), ",")
        )
        # 对list进行explode炸裂，转换成多行数据
        self.df_st_theme = self.df_st_theme.withColumn(
            "label_en_lower",
            F.explode(F.col("label_en_lower_list"))
        )
        self.df_st_theme = self.df_st_theme.select(
            'st_key', 'search_term', 'label_en_lower'
        )
        # 进行主题补回，根据匹配词(label_en_lower)关联
        self.df_st_theme = self.df_st_theme.join(
            self.df_theme, on=['label_en_lower'], how='left'
        )

        self.df_st_theme_vertical = self.df_st_theme.drop_duplicates(['st_key', 'search_term', 'theme_en', 'label_ch'])
        self.df_st_theme_vertical = self.df_st_theme_vertical.filter('theme_en is not null')
        self.df_st_theme_vertical = self.df_st_theme_vertical.select(
            F.col('st_key'),
            F.col('search_term'),
            F.col('theme_ch'),
            F.col('theme_en'),
            F.col('label_ch').alias('theme_label_ch'),
            F.col('label_en_lower').alias('theme_label_en'),
            F.lit(0).alias('pattern_type'),
            F.lit(None).alias('theme_label_num_info'),
            F.lit(None).alias('theme_label_unit_info')
        )

    # 通过正则规则匹配搜索词
    def match_st_topic(self):
        self.df_st_topic_base = self.df_st_base.withColumn(
            "all_theme_field",
            F.explode(self.parse_search_term_theme(self.topic_rules_regexp_dict)(F.col("search_term")))
        ).withColumn(
            "theme",
            F.col("all_theme_field").getItem("theme")
        ).withColumn(
            "num_info",
            F.col("all_theme_field").getItem("num_info")
        ).withColumn(
            "unit_info",
            F.col("all_theme_field").getItem("unit_info")
        ).drop("all_theme_field")

    # 通过正则匹配结果拿回中文单位信息
    def match_search_term_ch_unit(self):
        # 过滤掉 num_info 和 unit_info 都为空的行（没有任何匹配信息的脏行）
        # 为什么这么写：每个字段的"空"要同时覆盖 null 和空串 ""（用 isNull() | == ""），
        #   再整体取反 ~(A空 and B空) → 只要有一个字段非空就保留。
        # 老写法的 bug：用 isNull() & (== "")，但一个字段不可能既是 null 又等于 ""，
        #   该条件恒为 False → ~False 恒为 True → 整个 filter 形同虚设、一行都没过滤掉。
        df_st_mate = self.df_st_topic_base.filter(
            ~(
                (F.col("num_info").isNull() | (F.col("num_info") == "")) &
                (F.col("unit_info").isNull() | (F.col("unit_info") == ""))
            )
        )
        df_st_no_num = df_st_mate.filter(F.col("num_info") == '')
        df_st_with_num = df_st_mate.exceptAll(df_st_no_num)
        key_word_in_theme_info_sql = f"""
            select
            theme_ch as theme, 
            theme_en, 
            label_en_lower as unit_info,
            label_ch 
            from aba_match_theme_rules 
            where regular_expression_type = 0
        """
        key_word_with_num_in_theme_info_sql = f"""
            select 
            theme_ch as theme, 
            theme_en, 
            label_en_lower as unit_info,
            label_ch 
            from aba_match_theme_rules 
            where regular_expression_type != 0 
            group by theme_ch, theme_en, label_en_lower, label_ch
        """
        con_info = DBUtil.get_connection_info('mysql', 'us')
        if con_info is not None:
            df_key_word_in_theme = SparkUtil.read_jdbc_query(
                session=self.spark, url=con_info['url'],
                pwd=con_info['pwd'],
                username=con_info['username'],
                query=key_word_in_theme_info_sql
            )
            df_key_word_with_num_in_theme = SparkUtil.read_jdbc_query(
                session=self.spark, url=con_info['url'],
                pwd=con_info['pwd'],
                username=con_info['username'],
                query=key_word_with_num_in_theme_info_sql
            )
            df_st_no_num = df_st_no_num.join(
                df_key_word_in_theme, how='inner', on=['theme', 'unit_info']
            )
            df_st_with_num = df_st_with_num.join(
                df_key_word_with_num_in_theme, how='inner', on=['theme', 'unit_info']
            )
            self.df_st_match_topic_detail = df_st_no_num.unionByName(df_st_with_num)

            self.df_st_match_topic_detail = self.df_st_match_topic_detail.select(
                F.col('st_key'),
                F.col('search_term'),
                F.col('theme').alias('theme_ch'),
                F.col('theme_en'),
                F.col('label_ch').alias('theme_label_ch'),
                F.when(
                    F.col('num_info') == '', F.col('unit_info')
                ).when(
                    (F.col('unit_info') == 'x') | (F.col('unit_info') == 'by'), F.col('num_info')
                ).when(
                    (F.col("num_info") != '') & (F.col("unit_info") != '') &
                    (F.col("unit_info") != 'x') & (F.col("unit_info") != 'by'),
                    F.concat_ws(' ', F.col("num_info"), F.col("unit_info"))
                ).alias('theme_label_en'),
                F.lit(1).alias('pattern_type'),
                F.col('num_info').alias('theme_label_num_info'),
                F.col('unit_info').alias('theme_label_unit_info')
            )

    def handle_contains_theme(self):
        # 处理ab词包含关系的匹配
        df_st_join = self.df_st_theme_vertical.unionByName(self.df_st_match_topic_detail)
        df_st_join = df_st_join.filter(' theme_label_en is not null ')
        # 处理AB匹配词
        df_st_pattern_list = df_st_join.groupBy(['st_key', 'search_term']).agg(
            F.collect_list("theme_label_en").alias("pattern_label_list")
        )
        df_st_join = df_st_join.join(
            df_st_pattern_list, on=['st_key', 'search_term'], how='left'
        )

        # 自定义方法判断是否包含多个匹配词，打上标记标签
        df_st_join = df_st_join.withColumn(
            'pattern_flag',
            self.u_theme_contain_judge(F.col('theme_label_en'), F.col('pattern_label_list'))
        ).drop('pattern_label_list')

        # 过滤掉为0的数据（即已经被ab词匹配过的那些a词和b词）
        self.df_save = df_st_join.filter('pattern_flag = 1')

    def handle_save(self):
        self.df_save = self.df_save.select(
            F.col('st_key'),
            F.col('search_term'),
            F.col('theme_ch'),
            F.col('theme_en'),
            F.col('theme_label_ch'),
            F.col('theme_label_en'),
            F.date_format(F.current_timestamp(), 'yyyy-MM-dd HH:mm:SS').alias('created_time'),
            F.date_format(F.current_timestamp(), 'yyyy-MM-dd HH:mm:SS').alias('updated_time'),
            F.col('pattern_type'),
            self.u_handle_string_null_value(F.col('theme_label_num_info')).alias('theme_label_num_info'),
            F.col('theme_label_unit_info'),
            F.lit(self.site_name).alias('site_name'),
            F.lit(self.date_type).alias('date_type'),
            F.lit(self.date_info).alias('date_info')
        )


if __name__ == '__main__':
    site_name = CommonUtil.get_sys_arg(1, None)
    date_type = CommonUtil.get_sys_arg(2, None)
    date_info = CommonUtil.get_sys_arg(3, None)
    assert site_name is not None, "site_name 不能为空！"
    assert date_type is not None, "date_type 不能为空！"
    assert date_info is not None, "date_info 不能为空！"
    handle_obj = DwsStTheme(site_name=site_name, date_type=date_type, date_info=date_info)
    handle_obj.run()
