店铺计算从ods改为dim读取

Pyspark_job/dwt/dwt_fb_category_report.py

 """
-   @Author      : HuangJian
+   @Author      : hejiangming
    @Description : 店铺分类统计表
+                  数据源从 ODS 切换到 flow:
+                  - ods_seller_asin_account → dwt_flow_asin(seller-asin + asin_is_new + category)
+                  - ods_seller_account_feedback → 不再读取(以 flow 为主表)
+                  - dim_cal_asin_history_detail → 仅用于 bsr_asin_num 市场总量
    @SourceTable :
-                  ①ods_seller_asin_account
+                  ①dwt_flow_asin
                   ②dim_cal_asin_history_detail
-
+                  ③dim_bsr_category_tree
    @SinkTable   :
                   ①dwt_fb_category_report
    @CreateTime  : 2023/07/18 17:33
-   @UpdateTime  : 2023/07/18 17:33
+   @UpdateTime  : 2026/07/29
 """
 
 import os
 import sys
-import re
-from datetime import datetime
-import traceback
-from functools import reduce
 
-sys.path.append(os.path.dirname(sys.path[0]))  # 上级目录
+sys.path.append(os.path.dirname(sys.path[0]))
 
-# 分组排序的udf窗口函数
-from pyspark.sql.window import Window
 from pyspark.sql import functions as F
-from pyspark.sql.types import StringType, IntegerType, DoubleType
-from utils.common_util import CommonUtil, DateTypes
+from utils.common_util import CommonUtil
 from utils.spark_util import SparkUtil
-from yswg_utils.common_udf import udf_new_asin_flag
 from utils.hdfs_utils import HdfsUtils
 
 
@@ -49,9 +45,6 @@ class DwtFbCategoryReport(object):
         app_name = f"{self.__class__.__name__}:{site_name}:{date_info}"
         self.spark = SparkUtil.get_spark_session(app_name)
 
-        # 获取不同维度日期下的计算日期YYYY-MM-DD
-        self.cal_date = CommonUtil.get_calDay_by_dateInfo(self.spark, self.date_type, self.date_info)
-
         # 初始化全局df
         self.df_fb_asin_info = self.spark.sql(f"select 1+1;")
         self.df_asin_history = self.spark.sql(f"select 1+1;")
@@ -61,42 +54,41 @@ class DwtFbCategoryReport(object):
         self.df_bsr_asin_cal = self.spark.sql(f"select 1+1;")
         self.df_result_cal = self.spark.sql(f"select 1+1;")
 
-        # 初始化UDF函数
-        self.udf_new_asin_flag = F.udf(udf_new_asin_flag, IntegerType())
-
     def read_data(self):
-        print("==============================获取 原始数据sql==========================================")
-        # 获取ods_seller_account_feedback
-
-        # 获取ods_seller_asin_account,获取卖家和asin关系
-        sql = f"""select fb.seller_id,
-       oa.asin
-from (
-         select seller_id
-         from ods_seller_account_feedback
-         where site_name = '{site_name}'
-           and date_type = '{date_type}'
-           and date_info = '{date_info}') fb
-         left join (select seller_id ,asin 
-              from ods_seller_asin_account
+        # 从 flow 读 seller-asin 关系 + asin_is_new + category_first_id, 以 flow 为主表
+        # 替代原来的 ods_seller_account_feedback(seller集合) + ods_seller_asin_account(seller-asin)
+        # + dim_cal_asin_history_detail(asin_launch_time+category)
+        # asin_is_new 直接用 flow 已算好的值, 不再 UDF 重算
+        print("获取 dwt_flow_asin (seller-asin + asin_is_new + category)")
+        sql = f"""
+        select account_id as seller_id, asin,
+               asin_is_new as is_asin_new,
+               category_first_id as bsr_cate_1_id
+        from dwt_flow_asin
         where site_name = '{self.site_name}'
-              and date_format(created_at,'yyyy-MM-dd') <= '{self.cal_date}') oa
-                   on fb.seller_id = oa.seller_id  """
+          and date_type = '{self.date_type}'
+          and date_info = '{self.date_info}'
+          and account_id is not null
+        """
+        # 原来从 ods_seller_asin_account 读, 有 created_at 过滤(爬虫先删后增导致活跃店铺被误筛)
+        # 现在从 flow 读, 不存在该问题
         self.df_fb_asin_info = self.spark.sql(sqlQuery=sql)
         self.df_fb_asin_info = self.df_fb_asin_info.drop_duplicates(['seller_id', 'asin']).cache()
         print(sql)
 
-        # 获取dim_cal_asin_history_detail，历史asin取上架时间
+        # dim_cal_asin_history_detail: 仅用于 bsr_asin_num(按一级分类统计全量 asin 数, 作为市场占比分母)
+        # seller 的 asin_is_new 和 category 已从 flow 取, 这里不再取 asin_launch_time
+        print("获取 dim_cal_asin_history_detail (bsr_asin_num)")
         sql = f"""
-            select asin,
-                   asin_launch_time,
-                   category_first_id as bsr_cate_1_id 
+            select asin, category_first_id as bsr_cate_1_id
             from dim_cal_asin_history_detail
-              where site_name = '{self.site_name}'"""
+            where site_name = '{self.site_name}'
+        """
         self.df_asin_history = self.spark.sql(sqlQuery=sql).cache()
         print(sql)
 
-        # 获取一级分类的名称
+        # 一级分类名称
+        print("获取 dim_bsr_category_tree")
         sql = f"""
             select category_id as bsr_cate_1_id, en_name as bsr_cate_1_name
             from big_data_selection.dim_bsr_category_tree
@@ -114,28 +106,23 @@ from (
         # 字段处理
         self.sava_data()
 
-    # 计算卖家下的asin数量和新品数量
     def handle_fb_agg(self):
-        self.df_fb_cate_asin_cal = self.df_fb_asin_info.join(self.df_asin_history, on='asin', how='left')
-
+        # df_fb_asin_info 已含 is_asin_new 和 bsr_cate_1_id(从 flow 读取)
+        # 不需要 join asin_history 取 category/launch_time, 也不需要 UDF 算新品
+        self.df_fb_cate_asin_cal = self.df_fb_asin_info
         self.df_fb_cate_asin_cal = self.df_fb_cate_asin_cal.na.fill({'bsr_cate_1_id': '无'})
 
-        # 通过days_diff走自定义udf，生成is_asin_new字段（是否asin新品标记）
-        self.df_fb_cate_asin_cal = self.df_fb_cate_asin_cal.withColumn("is_asin_new",
-                                                                       self.udf_new_asin_flag(F.col('asin_launch_time'),
-                                                                                              F.lit(self.cal_date)))
-
-        # 按照seller_id和category_id进行分组聚合
-        self.df_fb_cate_asin_cal = self.df_fb_cate_asin_cal.groupby(['seller_id', 'bsr_cate_1_id']). \
-            agg(
+        # 按 seller_id + 一级分类聚合: 分类下 asin 数 + 新品数
+        self.df_fb_cate_asin_cal = self.df_fb_cate_asin_cal.groupby(['seller_id', 'bsr_cate_1_id']).agg(
             F.count("asin").alias("fb_cate_asin_num"),
             F.sum("is_asin_new").alias("fb_cate_new_asin_num"),
         )
 
-        # 计算卖家分类下的asin数量
-        self.df_fb_asin_cal = self.df_fb_asin_info.groupby(['seller_id']).agg(F.count("asin").alias("fb_asin_num"))
+        # 店铺总 asin 数(flow 口径)
+        self.df_fb_asin_cal = self.df_fb_asin_info.groupby(['seller_id']).agg(
+            F.count("asin").alias("fb_asin_num"))
 
-        # 计算bsr分类下的asin数量
+        # BSR 分类市场总量(仍用 dim_cal_asin_history_detail 全量 asin, 作为市场占比分母)
         self.df_bsr_asin_cal = self.df_asin_history.groupby(['bsr_cate_1_id']).agg(
             F.count("asin").alias("bsr_asin_num"))
 

Pyspark_job/dwt/dwt_fb_top20_asin_info.py

 """
-   @Author      : HuangJian
+   @Author      : hejiangming
    @Description : 店铺top20数据详情表
+                  数据源优化: ods_seller_account_syn → dim_fb_detail(account_name)
+                  dim_cal_asin_history_detail → dwt_flow_asin(asin详情)
    @SourceTable :
-                  ①ods_st_key
-                  ②dim_st_detail
-
-   @SinkTable   : dwt_fb_top20_info
+                  ①ods_asin_detail_product    (top20 seller_id+asin+row_num)
+                  ②dwt_flow_asin              (asin详情+is_asin_new)
+                  ③dim_fb_detail              (account_name)
+   @SinkTable   : dwt_fb_top20_asin_info
    @CreateTime  : 2022/07/24 14:55
-   @UpdateTime  : 2022/07/24 14:55
+   @UpdateTime  : 2026/07/29
 """
 
 import os
@@ -16,10 +18,8 @@ import sys
 sys.path.append(os.path.dirname(sys.path[0]))
 from utils.hdfs_utils import HdfsUtils
 from utils.common_util import CommonUtil
-from pyspark.sql.types import IntegerType
 from utils.spark_util import SparkUtil
 from pyspark.sql import functions as F
-from yswg_utils.common_udf import udf_new_asin_flag
 
 
 class DwtFbTop20Info(object):
@@ -41,87 +41,57 @@ class DwtFbTop20Info(object):
         app_name = f"{self.__class__.__name__}:{site_name}:{date_info}"
         self.spark = SparkUtil.get_spark_session(app_name)
 
-        # 获取不同维度日期下的计算日期YYYY-MM-DD
-        self.cal_date = CommonUtil.get_calDay_by_dateInfo(self.spark, self.date_type, self.date_info)
-
         # 初始化全局df
         self.df_fb_top20_asin_info = self.spark.sql(f"select 1+1;")
         self.df_seller_account = self.spark.sql(f"select 1+1;")
 
-        # 初始化UDF函数
-        self.udf_new_asin_flag = self.spark.udf.register("udf_new_asin_flag", udf_new_asin_flag, IntegerType())
-
     def read_data(self):
-        # 获取店铺抓取top20的基础信息数据
+        # top20 排名关系: 只读 seller_id + asin + row_num
+        print("获取 ods_asin_detail_product (top20 排名)")
         sql = f"""
-            with base_table as(
-                select 
-                    seller_id,
-                    asin,
-                    title,
-                    img_url,
-                    price,
-                    rating,
-                    total_comments,
-                    row_num
+            select seller_id, asin, row_num as fb_row_num
             from ods_asin_detail_product
             where site_name = '{self.site_name}'
               and date_type = '{self.date_type}'
               and date_info = '{self.date_info}'
-            ), 
-            history_asin as(
-                select
-                    asin,
-                    asin_volume,
-                    asin_weight,
-                    asin_launch_time
-                from dim_cal_asin_history_detail
+        """
+        self.df_fb_top20_asin_info = self.spark.sql(sqlQuery=sql)
+        self.df_fb_top20_asin_info = self.df_fb_top20_asin_info.drop_duplicates(['seller_id', 'asin'])
+        print(sql)
+
+        # asin 详情 + 新品标记从 flow 取(替代 dim_cal_asin_history_detail + UDF)
+        print("获取 dwt_flow_asin (asin 详情)")
+        sql = f"""
+            select asin, asin_title, asin_img_url,
+                   asin_price, asin_rating, asin_total_comments,
+                   asin_volume, asin_weight, asin_launch_time,
+                   asin_is_new as is_asin_new
+            from dwt_flow_asin
             where site_name = '{self.site_name}'
-            )
-            select
-                base_table.seller_id,
-                base_table.asin,
-                base_table.title as asin_title,
-                base_table.img_url as asin_img_url,
-                base_table.price as asin_price,
-                base_table.rating as asin_rating,
-                base_table.total_comments as asin_total_comments,
-                base_table.row_num as fb_row_num,
-                history_asin.asin_volume,
-                history_asin.asin_weight,
-                history_asin.asin_launch_time,
-                udf_new_asin_flag(history_asin.asin_launch_time,'{self.cal_date}') as is_asin_new
-            from base_table
-            left join history_asin
-            on base_table.asin = history_asin.asin
+              and date_type = '{self.date_type}'
+              and date_info = '{self.date_info}'
         """
-        self.df_fb_top20_asin_info = self.spark.sql(sqlQuery=sql).cache()
+        df_flow = self.spark.sql(sqlQuery=sql).drop_duplicates(['asin'])
+        self.df_fb_top20_asin_info = self.df_fb_top20_asin_info.join(df_flow, on='asin', how='left').cache()
         print(sql)
-        self.df_fb_top20_asin_info = self.df_fb_top20_asin_info.drop_duplicates(['seller_id', 'asin'])
-        # print("self.df_fb_top20_asin_info", self.df_fb_top20_asin_info.show(10, truncate=False))
 
-        # 获取ods_seller_account_syn提取account_name
-        print("获取 ods_seller_account_syn")
+        # account_name 从 dim_fb_detail 取(替代 ods_seller_account_syn)
+        print("获取 dim_fb_detail (account_name)")
         sql = f"""
-            select 
-                seller_id,
-                account_name,
-                id 
-            from ods_seller_account_syn 
-            where site_name='{self.site_name}'
+            select seller_id, account_name
+            from dim_fb_detail
+            where site_name = '{self.site_name}'
+              and date_type = '{self.date_type}'
+              and date_info = '{self.date_info}'
         """
         self.df_seller_account = self.spark.sql(sqlQuery=sql)
-        # 进行去重
-        self.df_seller_account = self.df_seller_account.orderBy(self.df_seller_account.id.desc())
-        self.df_seller_account = self.df_seller_account.drop_duplicates(['seller_id'])
-        self.df_seller_account = self.df_seller_account.drop('id')
         print(sql)
 
     def sava_data(self):
-        # 关联ods_seller_account_syn,带回account_name
         df_save = self.df_fb_top20_asin_info.join(
             self.df_seller_account, on='seller_id', how='inner'
         )
+        df_save = df_save.filter(F.col('account_name').isNotNull())
 
         df_save = df_save.select(
             F.col('seller_id'),