Skip to content

A
Amazon-Selection-Data
Commit 81c19a7f by chenyuanjie
Options

年度流量选品-增加变体销量相关指标

parent 667390e5
Showing with 74 additions and 5 deletions
Pyspark_job/doris_handle/dwt_flow_asin_month.py
......@@ -732,7 +732,13 @@ EXTRA_COLUMNS = (
"bought_month_9, bought_month_10, bought_month_11, bought_month_12, "
"bought_month_q1, bought_month_q2, bought_month_q3, bought_month_q4, "
"total_appear_month, bought_month_peak, peak_month_arr, "
"is_periodic_flag, is_seasonal_flag, bsr_seen_count_total, nsr_seen_count_total"
"is_periodic_flag, is_seasonal_flag, bsr_seen_count_total, nsr_seen_count_total, "
"variation_bought_month_total, bought_month_total_yoy, variation_bought_month_total_yoy, "
"variation_bought_month_1, variation_bought_month_2, variation_bought_month_3, variation_bought_month_4, "
"variation_bought_month_5, variation_bought_month_6, variation_bought_month_7, variation_bought_month_8, "
"variation_bought_month_9, variation_bought_month_10, variation_bought_month_11, variation_bought_month_12, "
"variation_bought_month_q1, variation_bought_month_q2, variation_bought_month_q3, variation_bought_month_q4, "
"variation_bought_month_peak, variation_peak_month_arr"
)
......@@ -765,7 +771,28 @@ CREATE TABLE IF NOT EXISTS `dwt`.`{table_name}`
`is_periodic_flag` INT,
`is_seasonal_flag` INT,
`bsr_seen_count_total` INT,
`nsr_seen_count_total` INT
`nsr_seen_count_total` INT,
`variation_bought_month_total` INT,
`bought_month_total_yoy` DECIMAL(20,4),
`variation_bought_month_total_yoy` DECIMAL(20,4),
`variation_bought_month_1` INT,
`variation_bought_month_2` INT,
`variation_bought_month_3` INT,
`variation_bought_month_4` INT,
`variation_bought_month_5` INT,
`variation_bought_month_6` INT,
`variation_bought_month_7` INT,
`variation_bought_month_8` INT,
`variation_bought_month_9` INT,
`variation_bought_month_10` INT,
`variation_bought_month_11` INT,
`variation_bought_month_12` INT,
`variation_bought_month_q1` INT,
`variation_bought_month_q2` INT,
`variation_bought_month_q3` INT,
`variation_bought_month_q4` INT,
`variation_bought_month_peak` INT,
`variation_peak_month_arr` ARRAY<INT>
) ENGINE=OLAP
UNIQUE KEY(`asin`)
COMMENT '流量选品年度聚合指标'
......@@ -793,13 +820,19 @@ def sync_extra_table(spark, site_name, date_info):
total_appear_month,
bought_month_peak,
peak_month_arr,
is_periodic_flag, is_seasonal_flag, bsr_seen_count_total, nsr_seen_count_total
is_periodic_flag, is_seasonal_flag, bsr_seen_count_total, nsr_seen_count_total,
variation_bought_month_total, bought_month_total_yoy, variation_bought_month_total_yoy,
variation_bought_month_1, variation_bought_month_2, variation_bought_month_3, variation_bought_month_4,
variation_bought_month_5, variation_bought_month_6, variation_bought_month_7, variation_bought_month_8,
variation_bought_month_9, variation_bought_month_10, variation_bought_month_11, variation_bought_month_12,
variation_bought_month_q1, variation_bought_month_q2, variation_bought_month_q3, variation_bought_month_q4,
variation_bought_month_peak, variation_peak_month_arr
FROM dwt_flow_asin_year
WHERE site_name = '{site_name}' AND date_info = '{date_info}'
"""
df_raw = spark.sql(sqlQuery=sql)
# total_appear_month / peak_month_arr 在 Hive 那边存的是逗号拼接的 STRING
# total_appear_month / peak_month_arr / variation_peak_month_arr 在 Hive 那边存的是逗号拼接的 STRING
df_extra = df_raw.withColumn(
'total_appear_month', F.split(F.col('total_appear_month'), ',')
).withColumn(
......@@ -812,6 +845,12 @@ def sync_extra_table(spark, site_name, date_info):
'peak_month_arr', F.expr("transform(peak_month_arr, x -> cast(x as int))")
).withColumn(
'peak_month_arr', F.to_json(F.col('peak_month_arr'))
).withColumn(
'variation_peak_month_arr', F.split(F.col('variation_peak_month_arr'), ',')
).withColumn(
'variation_peak_month_arr', F.expr("transform(variation_peak_month_arr, x -> cast(x as int))")
).withColumn(
'variation_peak_month_arr', F.to_json(F.col('variation_peak_month_arr'))
).repartition(20, 'asin').cache()
row_count = df_extra.count()
......@@ -1159,6 +1198,27 @@ CREATE TABLE IF NOT EXISTS `selection`.`{table_name}`
`peak_month_arr` ARRAY<INT> NULL,
`is_periodic_flag` INT NULL,
`is_seasonal_flag` INT NULL,
`variation_bought_month_total` INT NULL,
`bought_month_total_yoy` DECIMAL(20,4) NULL,
`variation_bought_month_total_yoy` DECIMAL(20,4) NULL,
`variation_bought_month_1` INT NULL,
`variation_bought_month_2` INT NULL,
`variation_bought_month_3` INT NULL,
`variation_bought_month_4` INT NULL,
`variation_bought_month_5` INT NULL,
`variation_bought_month_6` INT NULL,
`variation_bought_month_7` INT NULL,
`variation_bought_month_8` INT NULL,
`variation_bought_month_9` INT NULL,
`variation_bought_month_10` INT NULL,
`variation_bought_month_11` INT NULL,
`variation_bought_month_12` INT NULL,
`variation_bought_month_q1` INT NULL,
`variation_bought_month_q2` INT NULL,
`variation_bought_month_q3` INT NULL,
`variation_bought_month_q4` INT NULL,
`variation_bought_month_peak` INT NULL,
`variation_peak_month_arr` ARRAY<INT> NULL,
INDEX idx_title (`title`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题倒排索引',
INDEX idx_title_stem (`title_stem`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题词干倒排索引',
INDEX idx_title_stem_15 (`title_stem_15`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题词干前15个词倒排索引',
......@@ -1317,7 +1377,16 @@ SELECT
agg.bought_month_peak,
agg.peak_month_arr,
COALESCE(uma.is_periodic_flag, agg.is_periodic_flag, 0) AS is_periodic_flag,
COALESCE(uma.is_seasonal_flag, agg.is_seasonal_flag, 0) AS is_seasonal_flag
COALESCE(uma.is_seasonal_flag, agg.is_seasonal_flag, 0) AS is_seasonal_flag,
COALESCE(agg.variation_bought_month_total, 0) AS variation_bought_month_total,
agg.bought_month_total_yoy,
agg.variation_bought_month_total_yoy,
agg.variation_bought_month_1, agg.variation_bought_month_2, agg.variation_bought_month_3, agg.variation_bought_month_4,
agg.variation_bought_month_5, agg.variation_bought_month_6, agg.variation_bought_month_7, agg.variation_bought_month_8,
agg.variation_bought_month_9, agg.variation_bought_month_10, agg.variation_bought_month_11, agg.variation_bought_month_12,
agg.variation_bought_month_q1, agg.variation_bought_month_q2, agg.variation_bought_month_q3, agg.variation_bought_month_q4,
agg.variation_bought_month_peak,
agg.variation_peak_month_arr
FROM `dwt`.`{site_name}_flow_asin_365day` f
LEFT JOIN `dwt`.`{site_name}_flow_asin_365day_extra` agg ON f.asin = agg.asin
-- ===== 利润率 =====
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment