Commit a05de552 by hejiangming

店铺+搜索词增加兜底 历史回刷读取原数据/快照

parent df3cd66b
......@@ -362,12 +362,13 @@ class DwtAbaStAnalytics(Templates):
# self.df_asin_detail.show(10, truncate=True)
# 仅获取 asin和country_name,对country_name进行了聚合处理
# dim_fb_asin_info 从 2026-06 开始有分区,历史月读 2026-06
asin_info_date = self.date_info if self.date_info >= '2026-06' else '2026-06'
sql = f"""
select
asin,
select asin,
concat_ws(\",\",collect_list(cast(fb_country_name as string))) as country_name
from dim_fb_asin_info
where site_name = '{self.site_name}' and date_type = '{self.date_type}' and date_info = '{self.date_info}'
where site_name = '{self.site_name}' and date_type = '{self.date_type}' and date_info = '{asin_info_date}'
group by asin;
"""
self.df_seller_asin_country = self.spark.sql(sqlQuery=sql)
......@@ -375,14 +376,11 @@ class DwtAbaStAnalytics(Templates):
print("self.df_seller_asin_country:")
# self.df_seller_asin_country.show(10, truncate=True)
# 获取 dim_fd_asin_info 表
# 获取卖家-asin 信息
sql = f"""
select
asin,
seller_id as account_id,
fb_country_name as country_name
select asin, seller_id as account_id, fb_country_name as country_name
from dim_fb_asin_info
where site_name = '{self.site_name}' and date_type = '{self.date_type}' and date_info = '{self.date_info}'
where site_name = '{self.site_name}' and date_type = '{self.date_type}' and date_info = '{asin_info_date}'
"""
self.df_seller_asin_info = self.spark.sql(sqlQuery=sql)
self.df_seller_asin_info = self.df_seller_asin_info.drop_duplicates(['asin']).repartition(80, 'asin').cache()
......@@ -1073,7 +1071,7 @@ class DwtAbaStAnalytics(Templates):
select
word,
langs
from big_data_selection.tmp_lang_word_frequency;
from big_data_selection.lang_word_frequency;
"""
lang_word_list = self.spark.sql(sql).collect()
# 转为map
......
......@@ -324,7 +324,9 @@ class DwtAbaStAnalyticsReport(Templates):
self.df_asin_measure = self.spark.sql(sql).repartition(80, 'asin').cache()
self.df_asin_measure = self.df_asin_measure.na.fill({"asin_bsr_orders": 0})
print("1.7 读取dim_seller_asin_history_info系列表")
# dim_fb_asin_info 从 2026-06 开始有分区,历史月读 2026-06
print("1.7 读取dim_fb_asin_info卖家国家信息")
asin_info_date = self.date_info if self.date_info >= '2026-06' else '2026-06'
sql = f"""
select asin,
case when upper(fb_country_name) = 'US' then 'US'
......@@ -333,7 +335,7 @@ class DwtAbaStAnalyticsReport(Templates):
when upper(fb_country_name) = 'FR' then 'FR'
when upper(fb_country_name) = 'DE' then 'DE'
else 'OTHER' end as seller_name
from dim_fb_asin_info where site_name = '{self.site_name}' and date_type = '{self.date_type}' and date_info = '{self.date_info}';
from dim_fb_asin_info where site_name = '{self.site_name}' and date_type = '{self.date_type}' and date_info = '{asin_info_date}';
"""
print("sql:", sql)
self.df_seller_asin = self.spark.sql(sql).repartition(80, 'asin').cache()
......
......@@ -76,13 +76,15 @@ class DwtFbTop20Info(object):
print(sql)
# account_name 从 dim_fb_detail 取(替代 ods_seller_account_syn)
print("获取 dim_fb_detail (account_name)")
# dim_fb_detail 从 2026-06 开始有分区,历史月读 2026-06
detail_date = self.date_info if self.date_info >= '2026-06' else '2026-06'
print(f"获取 dim_fb_detail (account_name, date_info={detail_date})")
sql = f"""
select seller_id, account_name
from dim_fb_detail
where site_name = '{self.site_name}'
and date_type = '{self.date_type}'
and date_info = '{self.date_info}'
and date_info = '{detail_date}'
"""
self.df_seller_account = self.spark.sql(sqlQuery=sql)
print(sql)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment