Skip to content

A
Amazon-Selection-Data
Commit af5627d1 by chenyuanjie
Options

导出Doris新增字段

parent ebe53613
Showing with 99 additions and 10 deletions
Pyspark_job/doris_handle/dwt_flow_asin_month.py
......@@ -234,6 +234,14 @@ CREATE TABLE IF NOT EXISTS `selection`.`{table_name}`
`keepa_item_type` STRING NULL COMMENT 'Keepa itemTypeKeyword（小写、_转-）',
`keepa_type_cn` STRING NULL COMMENT 'keepa_type 中文翻译',
`keepa_item_type_cn` STRING NULL COMMENT 'keepa_item_type 中文翻译',
`age_range` STRING NULL COMMENT '适用年龄范围(voyager Age Range Description)',
`seasons` STRING NULL COMMENT '季节(voyager Seasons)',
`additional_features` STRING NULL COMMENT '附加功能(voyager Additional Features/Other Special Features of the Product)',
`closure_type` STRING NULL COMMENT '安装/闭合方式(voyager Closure Type/Mounting Type/Installation Type)',
`shape` STRING NULL COMMENT '形状(voyager Shape)',
`theme` STRING NULL COMMENT '主题(voyager Theme)',
`pattern` STRING NULL COMMENT '图案(voyager Pattern)',
`node_id` STRING NULL COMMENT '最小分类id',
INDEX idx_title (`title`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题倒排索引',
INDEX idx_title_stem (`title_stem`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题词干倒排索引',
INDEX idx_title_stem_15 (`title_stem_15`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题词干前15个词倒排索引',
......@@ -318,7 +326,9 @@ def read_and_normalize_month_data(spark, site_name, date_info):
title_matching_degree,
multi_color_flag, multi_color_str,
amazon_label,
auctions_num, auctions_num_all, skus_num_creat, skus_num_creat_all
auctions_num, auctions_num_all, skus_num_creat, skus_num_creat_all,
asin_age_range, asin_seasons, asin_additional_features, asin_closure_type,
asin_shape, asin_theme, asin_pattern, node_id
FROM dwt_flow_asin
WHERE site_name = '{site_name}' AND date_type = 'month' AND date_info = '{date_info}'
"""
......@@ -466,6 +476,15 @@ def read_and_normalize_month_data(spark, site_name, date_info):
F.col('variation_bought_month').cast('int').alias('variation_bought_month'),
F.round(F.col('variation_bought_month_mom').cast('double'), 4).cast('decimal(20,4)').alias('variation_bought_month_mom'),
F.round(F.col('variation_bought_month_yoy').cast('double'), 4).cast('decimal(20,4)').alias('variation_bought_month_yoy'),
# ===== voyager_detail_json 解析属性及最小分类id（新增字段放末尾）=====
F.col('asin_age_range').alias('age_range'),
F.col('asin_seasons').alias('seasons'),
F.col('asin_additional_features').alias('additional_features'),
F.col('asin_closure_type').alias('closure_type'),
F.col('asin_shape').alias('shape'),
F.col('asin_theme').alias('theme'),
F.col('asin_pattern').alias('pattern'),
F.col('node_id'),
).cache()
count = df_save.count()
......@@ -505,7 +524,8 @@ def write_dwt_month_table(df_save, doris_table, date_info):
"product_features, img_info, collapse_asin, follow_sellers_count, asin_describe, fbm_price, describe_len, "
"title_matching_degree, multi_color_flag, multi_color_str, amazon_label, "
"auctions_num, auctions_num_all, skus_num_creat, skus_num_creat_all, "
"variation_bought_month, variation_bought_month_mom, variation_bought_month_yoy"
"variation_bought_month, variation_bought_month_mom, variation_bought_month_yoy, "
"age_range, seasons, additional_features, closure_type, shape, theme, pattern, node_id"
)
print(f"[Step 3] 写入 Doris {DORIS_DB}.{doris_table}")
DorisHelper.spark_export_with_columns(
......@@ -646,7 +666,15 @@ CREATE TABLE IF NOT EXISTS `dwt`.`{site_name}_flow_asin_365day`
`skus_num_creat_all` INT,
`variation_bought_month` INT,
`variation_bought_month_mom` DECIMAL(20,4),
`variation_bought_month_yoy` DECIMAL(20,4)
`variation_bought_month_yoy` DECIMAL(20,4),
`age_range` STRING,
`seasons` STRING,
`additional_features` STRING,
`closure_type` STRING,
`shape` STRING,
`theme` STRING,
`pattern` STRING,
`node_id` STRING
)
ENGINE=OLAP
UNIQUE KEY(`asin`)
......@@ -687,7 +715,8 @@ STG_INSERT_COLUMNS = """
asin_describe, fbm_price, describe_len, title_matching_degree,
multi_color_flag, multi_color_str, amazon_label,
auctions_num, auctions_num_all, skus_num_creat, skus_num_creat_all,
variation_bought_month, variation_bought_month_mom, variation_bought_month_yoy
variation_bought_month, variation_bought_month_mom, variation_bought_month_yoy,
age_range, seasons, additional_features, closure_type, shape, theme, pattern, node_id
""".strip()
......@@ -1035,7 +1064,15 @@ SELECT
kp.keepa_type,
kp.keepa_item_type,
kp.keepa_type_cn,
kp.keepa_item_type_cn
kp.keepa_item_type_cn,
f.age_range,
f.seasons,
f.additional_features,
f.closure_type,
f.shape,
f.theme,
f.pattern,
f.node_id
FROM `dwt`.`{site_name}_flow_asin_month` f
LEFT JOIN `dwd`.`dwd_asin_profit_rate_latest` pr
ON f.asin = pr.asin AND f.price = pr.price AND pr.site_name = '{site_name}'
......@@ -1260,6 +1297,14 @@ CREATE TABLE IF NOT EXISTS `selection`.`{table_name}`
`keepa_item_type` STRING NULL COMMENT 'Keepa itemTypeKeyword（小写、_转-）',
`keepa_type_cn` STRING NULL COMMENT 'keepa_type 中文翻译',
`keepa_item_type_cn` STRING NULL COMMENT 'keepa_item_type 中文翻译',
`age_range` STRING NULL COMMENT '适用年龄范围(voyager Age Range Description)',
`seasons` STRING NULL COMMENT '季节(voyager Seasons)',
`additional_features` STRING NULL COMMENT '附加功能(voyager Additional Features/Other Special Features of the Product)',
`closure_type` STRING NULL COMMENT '安装/闭合方式(voyager Closure Type/Mounting Type/Installation Type)',
`shape` STRING NULL COMMENT '形状(voyager Shape)',
`theme` STRING NULL COMMENT '主题(voyager Theme)',
`pattern` STRING NULL COMMENT '图案(voyager Pattern)',
`node_id` STRING NULL COMMENT '最小分类id',
INDEX idx_title (`title`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题倒排索引',
INDEX idx_title_stem (`title_stem`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题词干倒排索引',
INDEX idx_title_stem_15 (`title_stem_15`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题词干前15个词倒排索引',
......@@ -1435,7 +1480,15 @@ SELECT
kp.keepa_type,
kp.keepa_item_type,
kp.keepa_type_cn,
kp.keepa_item_type_cn
kp.keepa_item_type_cn,
f.age_range,
f.seasons,
f.additional_features,
f.closure_type,
f.shape,
f.theme,
f.pattern,
f.node_id
FROM `dwt`.`{site_name}_flow_asin_365day` f
LEFT JOIN `dwt`.`{site_name}_flow_asin_365day_extra` agg ON f.asin = agg.asin
-- ===== 利润率 =====
......
Pyspark_job/doris_handle/dwt_flow_asin_week.py
......@@ -218,7 +218,15 @@ CREATE TABLE IF NOT EXISTS `dwt`.`{site_name}_flow_asin_week`
`bought_month_wow_last_2w` DECIMAL(20,4) COMMENT '前2周月销近7天环比',
`bought_month_wow_last_3w` DECIMAL(20,4) COMMENT '前3周月销近7天环比',
`bought_month_rise` INT COMMENT '月销差值(对比上周)',
`variation_bought_month_rise` INT COMMENT '变体销量和差值(对比上周)'
`variation_bought_month_rise` INT COMMENT '变体销量和差值(对比上周)',
`age_range` STRING COMMENT '适用年龄范围(voyager Age Range Description)',
`seasons` STRING COMMENT '季节(voyager Seasons)',
`additional_features` STRING COMMENT '附加功能(voyager Additional Features/Other Special Features of the Product)',
`closure_type` STRING COMMENT '安装/闭合方式(voyager Closure Type/Mounting Type/Installation Type)',
`shape` STRING COMMENT '形状(voyager Shape)',
`theme` STRING COMMENT '主题(voyager Theme)',
`pattern` STRING COMMENT '图案(voyager Pattern)',
`node_id` STRING COMMENT '最小分类id'
)
UNIQUE KEY(`date_info`, `asin`)
COMMENT '流量选品周维度主表'
......@@ -397,6 +405,14 @@ CREATE TABLE IF NOT EXISTS `selection`.`{table_name}`
`bought_month_wow_last_3w` DECIMAL(20,4) NULL COMMENT '前3周月销近7天环比',
`bought_month_rise` INT NULL COMMENT '月销差值(对比上周)',
`variation_bought_month_rise` INT NULL COMMENT '变体销量和差值(对比上周)',
`age_range` STRING NULL COMMENT '适用年龄范围(voyager Age Range Description)',
`seasons` STRING NULL COMMENT '季节(voyager Seasons)',
`additional_features` STRING NULL COMMENT '附加功能(voyager Additional Features/Other Special Features of the Product)',
`closure_type` STRING NULL COMMENT '安装/闭合方式(voyager Closure Type/Mounting Type/Installation Type)',
`shape` STRING NULL COMMENT '形状(voyager Shape)',
`theme` STRING NULL COMMENT '主题(voyager Theme)',
`pattern` STRING NULL COMMENT '图案(voyager Pattern)',
`node_id` STRING NULL COMMENT '最小分类id',
INDEX idx_title (`title`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题倒排索引',
INDEX idx_title_stem (`title_stem`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题词干倒排索引',
INDEX idx_title_stem_15 (`title_stem_15`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题词干前15个词倒排索引',
......@@ -486,7 +502,9 @@ def read_and_normalize_week_data(spark, site_name, date_info):
asin_rank_wow, asin_ao_wow, asin_price_wow, asin_rating_wow, asin_comments_wow,
asin_bsr_orders_wow, asin_sales_wow, asin_variation_wow, asin_bought_wow, variation_bought_month_wow,
asin_bought_wow_last_1w, asin_bought_wow_last_2w, asin_bought_wow_last_3w,
asin_bought_rise, variation_bought_month_rise
asin_bought_rise, variation_bought_month_rise,
asin_age_range, asin_seasons, asin_additional_features, asin_closure_type,
asin_shape, asin_theme, asin_pattern, node_id
FROM dwt_flow_asin_week
WHERE site_name = '{site_name}' AND date_type = 'week' AND date_info = '{date_info}'
"""
......@@ -659,6 +677,15 @@ def read_and_normalize_week_data(spark, site_name, date_info):
_dec4('asin_bought_wow_last_3w', 'bought_month_wow_last_3w'),
F.col('asin_bought_rise').cast('int').alias('bought_month_rise'),
F.col('variation_bought_month_rise').cast('int').alias('variation_bought_month_rise'),
# ===== voyager_detail_json 解析属性及最小分类id（新增字段放末尾）=====
F.col('asin_age_range').alias('age_range'),
F.col('asin_seasons').alias('seasons'),
F.col('asin_additional_features').alias('additional_features'),
F.col('asin_closure_type').alias('closure_type'),
F.col('asin_shape').alias('shape'),
F.col('asin_theme').alias('theme'),
F.col('asin_pattern').alias('pattern'),
F.col('node_id'),
).cache()
count = df_save.count()
......@@ -697,7 +724,8 @@ DWT_WEEK_COLUMNS = (
"rank_wow, ao_wow, price_wow, rating_wow, comments_wow, bsr_orders_wow, sales_wow, variation_wow, "
"bought_month_wow, variation_bought_month_wow, "
"bought_month_wow_last_1w, bought_month_wow_last_2w, bought_month_wow_last_3w, "
"bought_month_rise, variation_bought_month_rise"
"bought_month_rise, variation_bought_month_rise, "
"age_range, seasons, additional_features, closure_type, shape, theme, pattern, node_id"
)
......@@ -864,7 +892,15 @@ SELECT
f.bought_month_wow_last_2w,
f.bought_month_wow_last_3w,
f.bought_month_rise,
f.variation_bought_month_rise
f.variation_bought_month_rise,
f.age_range,
f.seasons,
f.additional_features,
f.closure_type,
f.shape,
f.theme,
f.pattern,
f.node_id
FROM `dwt`.`{site_name}_flow_asin_week` f
LEFT JOIN `dwd`.`dwd_asin_profit_rate_latest` pr
ON f.asin = pr.asin AND f.price = pr.price AND pr.site_name = '{site_name}'
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment