用户固定选品模式纳入每日刷新流程

Pyspark_job/doris_handle/dws_user_selection_pattern.py → Pyspark_job/doris_handle/dwt_user_selection_pattern.py

 """
    @Author      : CT
-   @Description : 根据用户筛选模式日志批量计算选品结果，写入 Doris selection.user_selection_pattern
-                  - 串行逐 filter_id 执行，减小数据库压力
+   @Description : 按站点增量追平用户筛选模式选品结果，写入 Doris selection.user_selection_pattern
+                  - 传入 site_name，只处理该站点的 active filter
+                  - 纯增量追平：水位线落后于该站点最新月表时补齐缺失月份，不做刷新判断
+                    （us 站点利润率/Keepa 字段每日刷新型 filter，最新月差异化刷新见 dws_flow_asin_refresh.py 第5步）
                   - 支持断点续算：latest_computed_month 作为水位线，跳过已算月份
                   - Doris AGGREGATE KEY(filter_id, asin) + MIN(date_info) 自动保留首次入选月份
-                  - us 站点利润率/Keepa 字段每日刷新：TRUNCATE 分区(p{filter_id}) + 全量重算
-                  - 水位线规则：成功→维护到最新月份；失败→维护到失败的前一个月
+                  - 每月执行一次即可（只有新月表出现时才会产生缺口）
+执行示例: python dwt_user_selection_pattern.py us
 """
 import os
-import re
 import sys
 from datetime import datetime
 
@@ -23,11 +24,6 @@ MYSQL_FILTER_TABLE = 'flow_increment_filter_sql'  # MySQL 筛选模式日志表
 DORIS_RESULT_DB    = 'selection'
 DORIS_RESULT_TABLE = 'user_selection_pattern'
 SUPPORTED_SITES    = ('us', 'uk', 'de')
-REFRESH_FIELDS   = {
-    'ocean_profit', 'air_profit',
-    'launch_time', 'launch_time_type',
-    'tracking_since', 'tracking_since_type',
-}
 
 
 # ===== 连接工厂 =====
@@ -46,6 +42,7 @@ def _get_doris_conn():
 
 
 def _get_mysql_engine():
+    """flow_increment_filter_sql 是跨站点共享的控制库表(带 site 字段区分)，固定连 us 实例"""
     return DBUtil.get_db_engine(db_type=DbTypes.mysql.name, site_name='us')
 
 
@@ -79,10 +76,6 @@ def _get_available_months(doris_cur, site):
     return months
 
 
-def _needs_refresh(where_sql):
-    return any(re.search(rf'\b{re.escape(field)}\b', where_sql) for field in REFRESH_FIELDS)
-
-
 def _update_mysql_log(mysql_conn, filter_id, latest_month, status, msg):
     now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
     if latest_month is not None:
@@ -95,7 +88,7 @@ def _update_mysql_log(mysql_conn, filter_id, latest_month, status, msg):
                      WHERE filter_id = %s"""
         params = (latest_month, status, now, msg[:500], filter_id)
     else:
-        # 首月即失败：不更新水位线，仅记录失败状态
+        # latest_month 为 None：仅记录失败状态，不更新水位线
         sql    = f"""UPDATE `{MYSQL_FILTER_TABLE}`
                      SET last_run_status = %s,
                          last_run_at     = %s,
@@ -110,57 +103,24 @@ def _update_mysql_log(mysql_conn, filter_id, latest_month, status, msg):
 
 # ===== 核心计算 =====
 
-def _compute_one_filter(row, doris_cur, mysql_conn, months_by_site):
+def _backfill_one_filter(row, site_name, available, doris_cur, mysql_conn):
+    """水位线落后于最新月表时，纯新增 INSERT 追平缺失月份（不做刷新判断）"""
     filter_id   = row['filter_id']
-    site        = row['site']
     base_month  = row['base_month']
     where_sql   = row['where_sql']
     latest_done = row['latest_computed_month']  # None 表示从未计算过
 
-    available = months_by_site.get(site, [])
-    if not available:
-        print(f"    [SKIP] 站点 {site} 无可用月表")
-        return
-
-    is_refresh = _needs_refresh(where_sql) and site == 'us'
-
-    if is_refresh:
-        # 刷新模式：TRUNCATE 分区 + 从 base_month 全量重算
-        doris_cur.execute(f"SELECT auto_partition_name('list', {filter_id})")
-        partition_name = doris_cur.fetchone()[0]
-        truncate_sql = (f"TRUNCATE TABLE `{DORIS_RESULT_DB}`.`{DORIS_RESULT_TABLE}` "
-                        f"PARTITION ({partition_name})")
-        try:
-            doris_cur.execute(truncate_sql)
-            print(f"    [TRUNCATE] 已清空分区 {partition_name}")
-        except Exception as e:
-            err_str = str(e)
-            if 'does not exist' in err_str:
-                # 首次计算时分区尚未创建，跳过 TRUNCATE 直接 INSERT
-                print(f"    [TRUNCATE SKIP] 分区 {partition_name} 不存在（首次计算），跳过清空")
-            else:
-                err = err_str[:200]
-                print(f"    [TRUNCATE FAIL] {err}")
-                _update_mysql_log(mysql_conn, filter_id, latest_done, 'failed', f'TRUNCATE失败: {err}')
-                return
-        all_months  = [m for m in available if m >= base_month]
-        mode_label  = '刷新重算'
-        last_ok_month = None
-    else:
-        # 补算模式：水位线增量
+    latest_available = available[-1]
     start_month = _next_month(latest_done) if latest_done else base_month
-        all_months  = [m for m in available if start_month <= m <= available[-1]]
-        mode_label  = '新增'
-        last_ok_month = None
-
+    all_months  = [m for m in available if start_month <= m <= latest_available]
     if not all_months:
-        print(f"    [SKIP] 无需计算（latest_computed={latest_done}，refresh={is_refresh}）")
+        print(f"    [SKIP] 无需计算（latest_computed={latest_done}，已追平至最新月 {latest_available}）")
         return
 
-    print(f"    [{mode_label}] 计算范围：{all_months[0]} ~ {all_months[-1]}，共 {len(all_months)} 个月")
-
+    print(f"    [新增] 计算范围：{all_months[0]} ~ {all_months[-1]}，共 {len(all_months)} 个月")
+    last_ok_month = None
     for month in all_months:
-        table = f'{site}_flow_asin_month_{month.replace("-", "_")}'
+        table = f'{site_name}_flow_asin_month_{month.replace("-", "_")}'
         sql = f"""
             INSERT INTO `{DORIS_RESULT_DB}`.`{DORIS_RESULT_TABLE}`
                 (filter_id, asin, date_info)
@@ -183,40 +143,43 @@ def _compute_one_filter(row, doris_cur, mysql_conn, months_by_site):
 
     _update_mysql_log(
         mysql_conn, filter_id, last_ok_month, 'success',
-        f'完成：{mode_label} {len(all_months)} 个月（{all_months[0]} ~ {all_months[-1]}）'
+        f'完成：新增 {len(all_months)} 个月（{all_months[0]} ~ {all_months[-1]}）'
     )
 
 
 # ===== 入口 =====
 
 def main():
+    assert len(sys.argv) == 2 and sys.argv[1] in SUPPORTED_SITES, \
+        f"用法: python dwt_user_selection_pattern.py <site_name>，site_name 需为 {SUPPORTED_SITES} 之一"
+    site_name = sys.argv[1]
+
     doris_conn = _get_doris_conn()
     engine     = _get_mysql_engine()
     mysql_conn = engine.raw_connection()
     try:
         doris_cur = doris_conn.cursor()
 
-        # 预加载各站点可用月份
-        months_by_site = {site: _get_available_months(doris_cur, site) for site in SUPPORTED_SITES}
-        for site, months in months_by_site.items():
-            rng = f"{months[0]} ~ {months[-1]}" if months else "无"
-            print(f"[站点 {site}] 可用月份：{rng}（共 {len(months)} 个）")
+        available = _get_available_months(doris_cur, site_name)
+        if not available:
+            print(f"[SKIP] 站点 {site_name} 无可用月表")
+            return
+        print(f"[站点 {site_name}] 可用月份：{available[0]} ~ {available[-1]}（共 {len(available)} 个）")
 
-        # 读取全部 active 筛选模式
         df = pd.read_sql(
-            f"""SELECT filter_id, site, base_month, where_sql, latest_computed_month
+            f"""SELECT filter_id, base_month, where_sql, latest_computed_month
                 FROM `{MYSQL_FILTER_TABLE}`
-                WHERE status = 'active'
+                WHERE status = 'active' AND site = '{site_name}'
                 ORDER BY filter_id""",
             engine
         )
         total = len(df)
-        print(f"\n共 {total} 条 active 筛选模式，开始串行计算...\n")
+        print(f"\n共 {total} 条 active 筛选模式（站点={site_name}），开始串行计算...\n")
 
         for i, row in df.iterrows():
-            print(f"[{i + 1}/{total}] filter_id={row['filter_id']}  site={row['site']}")
+            print(f"[{i + 1}/{total}] filter_id={row['filter_id']}")
             try:
-                _compute_one_filter(row, doris_cur, mysql_conn, months_by_site)
+                _backfill_one_filter(row, site_name, available, doris_cur, mysql_conn)
             except Exception as e:
                 print(f"    [ERROR] 未预期异常：{e}")