Commit 9cba6f71 by chenyuanjie

搜索词中间表兼容landing_asin字段做asin跳转

parent 01428841
...@@ -177,7 +177,7 @@ class DwdStMeasure(Templates): ...@@ -177,7 +177,7 @@ class DwdStMeasure(Templates):
self.df_asin_bs = self.spark.sql(sql).cache() self.df_asin_bs = self.spark.sql(sql).cache()
self.df_asin_bs.show(10) self.df_asin_bs.show(10)
sql = f"select asin, asin_title, asin_price, parent_asin, asin_bought_month, current_asin, updated_time " \ sql = f"select asin, asin_title, asin_price, parent_asin, asin_bought_month, updated_time, landing_asin " \
f"from dim_asin_detail where site_name='{self.site_name}' and date_type='{self.date_type.replace('_old', '')}' and date_info='{self.date_info}';" f"from dim_asin_detail where site_name='{self.site_name}' and date_type='{self.date_type.replace('_old', '')}' and date_info='{self.date_info}';"
print("sql:", sql) print("sql:", sql)
self.df_asin_detail = self.spark.sql(sql).cache() self.df_asin_detail = self.spark.sql(sql).cache()
...@@ -361,21 +361,21 @@ class DwdStMeasure(Templates): ...@@ -361,21 +361,21 @@ class DwdStMeasure(Templates):
self.df_st_asin_flow, on=['page_rank'], how='left' self.df_st_asin_flow, on=['page_rank'], how='left'
) )
# === asin跳转处理:将df_st_asin和df_asin_detail的asin统一替换为current_asin === # === asin跳转处理:将df_st_asin和df_asin_detail的asin统一替换为landing_asin ===
# 1. 保留原始缓存引用,从中构建跳转映射:原始asin → current_asin # 1. 保留原始缓存引用,从中构建跳转映射:原始asin → landing_asin
df_asin_detail_cached = self.df_asin_detail df_asin_detail_cached = self.df_asin_detail
redirect_map = df_asin_detail_cached.filter(F.col('current_asin').isNotNull()) \ redirect_map = df_asin_detail_cached.filter(F.col('landing_asin').isNotNull()) \
.select(F.col('asin').alias('redirect_src'), F.col('current_asin')) .select(F.col('asin').alias('redirect_src'), F.col('landing_asin'))
# 2. df_st_asin中的原始asin替换为current_asin(无跳转则保留原值) # 2. df_st_asin中的原始asin替换为landing_asin(无跳转则保留原值)
self.df_st_asin = self.df_st_asin \ self.df_st_asin = self.df_st_asin \
.join(redirect_map, self.df_st_asin.asin == redirect_map.redirect_src, how='left') \ .join(redirect_map, self.df_st_asin.asin == redirect_map.redirect_src, how='left') \
.withColumn('asin', F.coalesce(F.col('current_asin'), F.col('asin'))) \ .withColumn('asin', F.coalesce(F.col('landing_asin'), F.col('asin'))) \
.drop('redirect_src', 'current_asin') .drop('redirect_src', 'landing_asin')
# 3. df_asin_detail的asin键同步替换,asin重复时按updated_time降序保留最新一条;释放原始缓存 # 3. df_asin_detail的asin键同步替换,asin重复时按updated_time降序保留最新一条;释放原始缓存
window_redirect = Window.partitionBy('asin').orderBy(F.col('updated_time').desc()) window_redirect = Window.partitionBy('asin').orderBy(F.col('updated_time').desc())
self.df_asin_detail = df_asin_detail_cached \ self.df_asin_detail = df_asin_detail_cached \
.withColumn('asin', F.coalesce(F.col('current_asin'), F.col('asin'))) \ .withColumn('asin', F.coalesce(F.col('landing_asin'), F.col('asin'))) \
.drop('current_asin') \ .drop('landing_asin') \
.withColumn('_rk', F.row_number().over(window_redirect)) \ .withColumn('_rk', F.row_number().over(window_redirect)) \
.filter(F.col('_rk') == 1) \ .filter(F.col('_rk') == 1) \
.drop('_rk', 'updated_time') \ .drop('_rk', 'updated_time') \
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment