Commit a05de552 by hejiangming

店铺+搜索词增加兜底 历史回刷读取原数据/快照

parent df3cd66b
...@@ -362,12 +362,13 @@ class DwtAbaStAnalytics(Templates): ...@@ -362,12 +362,13 @@ class DwtAbaStAnalytics(Templates):
# self.df_asin_detail.show(10, truncate=True) # self.df_asin_detail.show(10, truncate=True)
# 仅获取 asin和country_name,对country_name进行了聚合处理 # 仅获取 asin和country_name,对country_name进行了聚合处理
# dim_fb_asin_info 从 2026-06 开始有分区,历史月读 2026-06
asin_info_date = self.date_info if self.date_info >= '2026-06' else '2026-06'
sql = f""" sql = f"""
select select asin,
asin,
concat_ws(\",\",collect_list(cast(fb_country_name as string))) as country_name concat_ws(\",\",collect_list(cast(fb_country_name as string))) as country_name
from dim_fb_asin_info from dim_fb_asin_info
where site_name = '{self.site_name}' and date_type = '{self.date_type}' and date_info = '{self.date_info}' where site_name = '{self.site_name}' and date_type = '{self.date_type}' and date_info = '{asin_info_date}'
group by asin; group by asin;
""" """
self.df_seller_asin_country = self.spark.sql(sqlQuery=sql) self.df_seller_asin_country = self.spark.sql(sqlQuery=sql)
...@@ -375,14 +376,11 @@ class DwtAbaStAnalytics(Templates): ...@@ -375,14 +376,11 @@ class DwtAbaStAnalytics(Templates):
print("self.df_seller_asin_country:") print("self.df_seller_asin_country:")
# self.df_seller_asin_country.show(10, truncate=True) # self.df_seller_asin_country.show(10, truncate=True)
# 获取 dim_fd_asin_info 表 # 获取卖家-asin 信息
sql = f""" sql = f"""
select select asin, seller_id as account_id, fb_country_name as country_name
asin,
seller_id as account_id,
fb_country_name as country_name
from dim_fb_asin_info from dim_fb_asin_info
where site_name = '{self.site_name}' and date_type = '{self.date_type}' and date_info = '{self.date_info}' where site_name = '{self.site_name}' and date_type = '{self.date_type}' and date_info = '{asin_info_date}'
""" """
self.df_seller_asin_info = self.spark.sql(sqlQuery=sql) self.df_seller_asin_info = self.spark.sql(sqlQuery=sql)
self.df_seller_asin_info = self.df_seller_asin_info.drop_duplicates(['asin']).repartition(80, 'asin').cache() self.df_seller_asin_info = self.df_seller_asin_info.drop_duplicates(['asin']).repartition(80, 'asin').cache()
...@@ -1073,7 +1071,7 @@ class DwtAbaStAnalytics(Templates): ...@@ -1073,7 +1071,7 @@ class DwtAbaStAnalytics(Templates):
select select
word, word,
langs langs
from big_data_selection.tmp_lang_word_frequency; from big_data_selection.lang_word_frequency;
""" """
lang_word_list = self.spark.sql(sql).collect() lang_word_list = self.spark.sql(sql).collect()
# 转为map # 转为map
......
...@@ -324,7 +324,9 @@ class DwtAbaStAnalyticsReport(Templates): ...@@ -324,7 +324,9 @@ class DwtAbaStAnalyticsReport(Templates):
self.df_asin_measure = self.spark.sql(sql).repartition(80, 'asin').cache() self.df_asin_measure = self.spark.sql(sql).repartition(80, 'asin').cache()
self.df_asin_measure = self.df_asin_measure.na.fill({"asin_bsr_orders": 0}) self.df_asin_measure = self.df_asin_measure.na.fill({"asin_bsr_orders": 0})
print("1.7 读取dim_seller_asin_history_info系列表") # dim_fb_asin_info 从 2026-06 开始有分区,历史月读 2026-06
print("1.7 读取dim_fb_asin_info卖家国家信息")
asin_info_date = self.date_info if self.date_info >= '2026-06' else '2026-06'
sql = f""" sql = f"""
select asin, select asin,
case when upper(fb_country_name) = 'US' then 'US' case when upper(fb_country_name) = 'US' then 'US'
...@@ -333,7 +335,7 @@ class DwtAbaStAnalyticsReport(Templates): ...@@ -333,7 +335,7 @@ class DwtAbaStAnalyticsReport(Templates):
when upper(fb_country_name) = 'FR' then 'FR' when upper(fb_country_name) = 'FR' then 'FR'
when upper(fb_country_name) = 'DE' then 'DE' when upper(fb_country_name) = 'DE' then 'DE'
else 'OTHER' end as seller_name else 'OTHER' end as seller_name
from dim_fb_asin_info where site_name = '{self.site_name}' and date_type = '{self.date_type}' and date_info = '{self.date_info}'; from dim_fb_asin_info where site_name = '{self.site_name}' and date_type = '{self.date_type}' and date_info = '{asin_info_date}';
""" """
print("sql:", sql) print("sql:", sql)
self.df_seller_asin = self.spark.sql(sql).repartition(80, 'asin').cache() self.df_seller_asin = self.spark.sql(sql).repartition(80, 'asin').cache()
......
...@@ -76,13 +76,15 @@ class DwtFbTop20Info(object): ...@@ -76,13 +76,15 @@ class DwtFbTop20Info(object):
print(sql) print(sql)
# account_name 从 dim_fb_detail 取(替代 ods_seller_account_syn) # account_name 从 dim_fb_detail 取(替代 ods_seller_account_syn)
print("获取 dim_fb_detail (account_name)") # dim_fb_detail 从 2026-06 开始有分区,历史月读 2026-06
detail_date = self.date_info if self.date_info >= '2026-06' else '2026-06'
print(f"获取 dim_fb_detail (account_name, date_info={detail_date})")
sql = f""" sql = f"""
select seller_id, account_name select seller_id, account_name
from dim_fb_detail from dim_fb_detail
where site_name = '{self.site_name}' where site_name = '{self.site_name}'
and date_type = '{self.date_type}' and date_type = '{self.date_type}'
and date_info = '{self.date_info}' and date_info = '{detail_date}'
""" """
self.df_seller_account = self.spark.sql(sqlQuery=sql) self.df_seller_account = self.spark.sql(sqlQuery=sql)
print(sql) print(sql)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment