导出Doris新增字段

Pyspark_job/doris_handle/dwt_flow_asin_month.py

@@ -234,6 +234,14 @@ CREATE TABLE IF NOT EXISTS `selection`.`{table_name}`
     `keepa_item_type`             STRING           NULL     COMMENT 'Keepa itemTypeKeyword（小写、_转-）',
     `keepa_type_cn`               STRING           NULL     COMMENT 'keepa_type 中文翻译',
     `keepa_item_type_cn`          STRING           NULL     COMMENT 'keepa_item_type 中文翻译',
+    `age_range`                   STRING           NULL     COMMENT '适用年龄范围(voyager Age Range Description)',
+    `seasons`                     STRING           NULL     COMMENT '季节(voyager Seasons)',
+    `additional_features`         STRING           NULL     COMMENT '附加功能(voyager Additional Features/Other Special Features of the Product)',
+    `closure_type`                STRING           NULL     COMMENT '安装/闭合方式(voyager Closure Type/Mounting Type/Installation Type)',
+    `shape`                       STRING           NULL     COMMENT '形状(voyager Shape)',
+    `theme`                       STRING           NULL     COMMENT '主题(voyager Theme)',
+    `pattern`                     STRING           NULL     COMMENT '图案(voyager Pattern)',
+    `node_id`                     STRING           NULL     COMMENT '最小分类id',
     INDEX idx_title      (`title`)      USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题倒排索引',
     INDEX idx_title_stem (`title_stem`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题词干倒排索引',
     INDEX idx_title_stem_15 (`title_stem_15`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题词干前15个词倒排索引',
@@ -318,7 +326,9 @@ def read_and_normalize_month_data(spark, site_name, date_info):
             title_matching_degree,
             multi_color_flag, multi_color_str,
             amazon_label,
-            auctions_num, auctions_num_all, skus_num_creat, skus_num_creat_all
+            auctions_num, auctions_num_all, skus_num_creat, skus_num_creat_all,
+            asin_age_range, asin_seasons, asin_additional_features, asin_closure_type,
+            asin_shape, asin_theme, asin_pattern, node_id
         FROM dwt_flow_asin
         WHERE site_name = '{site_name}' AND date_type = 'month' AND date_info = '{date_info}'
     """
@@ -466,6 +476,15 @@ def read_and_normalize_month_data(spark, site_name, date_info):
         F.col('variation_bought_month').cast('int').alias('variation_bought_month'),
         F.round(F.col('variation_bought_month_mom').cast('double'), 4).cast('decimal(20,4)').alias('variation_bought_month_mom'),
         F.round(F.col('variation_bought_month_yoy').cast('double'), 4).cast('decimal(20,4)').alias('variation_bought_month_yoy'),
+        # ===== voyager_detail_json 解析属性及最小分类id（新增字段放末尾）=====
+        F.col('asin_age_range').alias('age_range'),
+        F.col('asin_seasons').alias('seasons'),
+        F.col('asin_additional_features').alias('additional_features'),
+        F.col('asin_closure_type').alias('closure_type'),
+        F.col('asin_shape').alias('shape'),
+        F.col('asin_theme').alias('theme'),
+        F.col('asin_pattern').alias('pattern'),
+        F.col('node_id'),
     ).cache()
 
     count = df_save.count()
@@ -505,7 +524,8 @@ def write_dwt_month_table(df_save, doris_table, date_info):
         "product_features, img_info, collapse_asin, follow_sellers_count, asin_describe, fbm_price, describe_len, "
         "title_matching_degree, multi_color_flag, multi_color_str, amazon_label, "
         "auctions_num, auctions_num_all, skus_num_creat, skus_num_creat_all, "
-        "variation_bought_month, variation_bought_month_mom, variation_bought_month_yoy"
+        "variation_bought_month, variation_bought_month_mom, variation_bought_month_yoy, "
+        "age_range, seasons, additional_features, closure_type, shape, theme, pattern, node_id"
     )
     print(f"[Step 3] 写入 Doris {DORIS_DB}.{doris_table}")
     DorisHelper.spark_export_with_columns(
@@ -646,7 +666,15 @@ CREATE TABLE IF NOT EXISTS `dwt`.`{site_name}_flow_asin_365day`
     `skus_num_creat_all`          INT,
     `variation_bought_month`      INT,
     `variation_bought_month_mom`  DECIMAL(20,4),
-    `variation_bought_month_yoy`  DECIMAL(20,4)
+    `variation_bought_month_yoy`  DECIMAL(20,4),
+    `age_range`                   STRING,
+    `seasons`                     STRING,
+    `additional_features`         STRING,
+    `closure_type`                STRING,
+    `shape`                       STRING,
+    `theme`                       STRING,
+    `pattern`                     STRING,
+    `node_id`                     STRING
 )
 ENGINE=OLAP
 UNIQUE KEY(`asin`)
@@ -687,7 +715,8 @@ STG_INSERT_COLUMNS = """
     asin_describe, fbm_price, describe_len, title_matching_degree,
     multi_color_flag, multi_color_str, amazon_label,
     auctions_num, auctions_num_all, skus_num_creat, skus_num_creat_all,
-    variation_bought_month, variation_bought_month_mom, variation_bought_month_yoy
+    variation_bought_month, variation_bought_month_mom, variation_bought_month_yoy,
+    age_range, seasons, additional_features, closure_type, shape, theme, pattern, node_id
 """.strip()
 
 
@@ -1035,7 +1064,15 @@ SELECT
     kp.keepa_type,
     kp.keepa_item_type,
     kp.keepa_type_cn,
-    kp.keepa_item_type_cn
+    kp.keepa_item_type_cn,
+    f.age_range,
+    f.seasons,
+    f.additional_features,
+    f.closure_type,
+    f.shape,
+    f.theme,
+    f.pattern,
+    f.node_id
 FROM `dwt`.`{site_name}_flow_asin_month` f
 LEFT JOIN `dwd`.`dwd_asin_profit_rate_latest` pr
     ON f.asin = pr.asin AND f.price = pr.price AND pr.site_name = '{site_name}'
@@ -1260,6 +1297,14 @@ CREATE TABLE IF NOT EXISTS `selection`.`{table_name}`
     `keepa_item_type`                   STRING           NULL     COMMENT 'Keepa itemTypeKeyword（小写、_转-）',
     `keepa_type_cn`                     STRING           NULL     COMMENT 'keepa_type 中文翻译',
     `keepa_item_type_cn`                STRING           NULL     COMMENT 'keepa_item_type 中文翻译',
+    `age_range`                         STRING           NULL     COMMENT '适用年龄范围(voyager Age Range Description)',
+    `seasons`                           STRING           NULL     COMMENT '季节(voyager Seasons)',
+    `additional_features`               STRING           NULL     COMMENT '附加功能(voyager Additional Features/Other Special Features of the Product)',
+    `closure_type`                      STRING           NULL     COMMENT '安装/闭合方式(voyager Closure Type/Mounting Type/Installation Type)',
+    `shape`                             STRING           NULL     COMMENT '形状(voyager Shape)',
+    `theme`                             STRING           NULL     COMMENT '主题(voyager Theme)',
+    `pattern`                           STRING           NULL     COMMENT '图案(voyager Pattern)',
+    `node_id`                           STRING           NULL     COMMENT '最小分类id',
     INDEX idx_title      (`title`)      USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题倒排索引',
     INDEX idx_title_stem (`title_stem`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题词干倒排索引',
     INDEX idx_title_stem_15 (`title_stem_15`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题词干前15个词倒排索引',
@@ -1435,7 +1480,15 @@ SELECT
     kp.keepa_type,
     kp.keepa_item_type,
     kp.keepa_type_cn,
-    kp.keepa_item_type_cn
+    kp.keepa_item_type_cn,
+    f.age_range,
+    f.seasons,
+    f.additional_features,
+    f.closure_type,
+    f.shape,
+    f.theme,
+    f.pattern,
+    f.node_id
 FROM `dwt`.`{site_name}_flow_asin_365day` f
 LEFT JOIN `dwt`.`{site_name}_flow_asin_365day_extra` agg ON f.asin = agg.asin
 -- ===== 利润率 =====

Pyspark_job/doris_handle/dwt_flow_asin_week.py

@@ -218,7 +218,15 @@ CREATE TABLE IF NOT EXISTS `dwt`.`{site_name}_flow_asin_week`
     `bought_month_wow_last_2w`    DECIMAL(20,4)    COMMENT '前2周月销近7天环比',
     `bought_month_wow_last_3w`    DECIMAL(20,4)    COMMENT '前3周月销近7天环比',
     `bought_month_rise`           INT              COMMENT '月销差值(对比上周)',
-    `variation_bought_month_rise` INT              COMMENT '变体销量和差值(对比上周)'
+    `variation_bought_month_rise` INT              COMMENT '变体销量和差值(对比上周)',
+    `age_range`                   STRING           COMMENT '适用年龄范围(voyager Age Range Description)',
+    `seasons`                     STRING           COMMENT '季节(voyager Seasons)',
+    `additional_features`         STRING           COMMENT '附加功能(voyager Additional Features/Other Special Features of the Product)',
+    `closure_type`                STRING           COMMENT '安装/闭合方式(voyager Closure Type/Mounting Type/Installation Type)',
+    `shape`                       STRING           COMMENT '形状(voyager Shape)',
+    `theme`                       STRING           COMMENT '主题(voyager Theme)',
+    `pattern`                     STRING           COMMENT '图案(voyager Pattern)',
+    `node_id`                     STRING           COMMENT '最小分类id'
 )
 UNIQUE KEY(`date_info`, `asin`)
 COMMENT '流量选品周维度主表'
@@ -397,6 +405,14 @@ CREATE TABLE IF NOT EXISTS `selection`.`{table_name}`
     `bought_month_wow_last_3w`    DECIMAL(20,4)    NULL     COMMENT '前3周月销近7天环比',
     `bought_month_rise`           INT              NULL     COMMENT '月销差值(对比上周)',
     `variation_bought_month_rise` INT              NULL     COMMENT '变体销量和差值(对比上周)',
+    `age_range`                   STRING           NULL     COMMENT '适用年龄范围(voyager Age Range Description)',
+    `seasons`                     STRING           NULL     COMMENT '季节(voyager Seasons)',
+    `additional_features`         STRING           NULL     COMMENT '附加功能(voyager Additional Features/Other Special Features of the Product)',
+    `closure_type`                STRING           NULL     COMMENT '安装/闭合方式(voyager Closure Type/Mounting Type/Installation Type)',
+    `shape`                       STRING           NULL     COMMENT '形状(voyager Shape)',
+    `theme`                       STRING           NULL     COMMENT '主题(voyager Theme)',
+    `pattern`                     STRING           NULL     COMMENT '图案(voyager Pattern)',
+    `node_id`                     STRING           NULL     COMMENT '最小分类id',
     INDEX idx_title      (`title`)      USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题倒排索引',
     INDEX idx_title_stem (`title_stem`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题词干倒排索引',
     INDEX idx_title_stem_15 (`title_stem_15`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '标题词干前15个词倒排索引',
@@ -486,7 +502,9 @@ def read_and_normalize_week_data(spark, site_name, date_info):
             asin_rank_wow, asin_ao_wow, asin_price_wow, asin_rating_wow, asin_comments_wow,
             asin_bsr_orders_wow, asin_sales_wow, asin_variation_wow, asin_bought_wow, variation_bought_month_wow,
             asin_bought_wow_last_1w, asin_bought_wow_last_2w, asin_bought_wow_last_3w,
-            asin_bought_rise, variation_bought_month_rise
+            asin_bought_rise, variation_bought_month_rise,
+            asin_age_range, asin_seasons, asin_additional_features, asin_closure_type,
+            asin_shape, asin_theme, asin_pattern, node_id
         FROM dwt_flow_asin_week
         WHERE site_name = '{site_name}' AND date_type = 'week' AND date_info = '{date_info}'
     """
@@ -659,6 +677,15 @@ def read_and_normalize_week_data(spark, site_name, date_info):
         _dec4('asin_bought_wow_last_3w', 'bought_month_wow_last_3w'),
         F.col('asin_bought_rise').cast('int').alias('bought_month_rise'),
         F.col('variation_bought_month_rise').cast('int').alias('variation_bought_month_rise'),
+        # ===== voyager_detail_json 解析属性及最小分类id（新增字段放末尾）=====
+        F.col('asin_age_range').alias('age_range'),
+        F.col('asin_seasons').alias('seasons'),
+        F.col('asin_additional_features').alias('additional_features'),
+        F.col('asin_closure_type').alias('closure_type'),
+        F.col('asin_shape').alias('shape'),
+        F.col('asin_theme').alias('theme'),
+        F.col('asin_pattern').alias('pattern'),
+        F.col('node_id'),
     ).cache()
 
     count = df_save.count()
@@ -697,7 +724,8 @@ DWT_WEEK_COLUMNS = (
     "rank_wow, ao_wow, price_wow, rating_wow, comments_wow, bsr_orders_wow, sales_wow, variation_wow, "
     "bought_month_wow, variation_bought_month_wow, "
     "bought_month_wow_last_1w, bought_month_wow_last_2w, bought_month_wow_last_3w, "
-    "bought_month_rise, variation_bought_month_rise"
+    "bought_month_rise, variation_bought_month_rise, "
+    "age_range, seasons, additional_features, closure_type, shape, theme, pattern, node_id"
 )
 
 
@@ -864,7 +892,15 @@ SELECT
     f.bought_month_wow_last_2w,
     f.bought_month_wow_last_3w,
     f.bought_month_rise,
-    f.variation_bought_month_rise
+    f.variation_bought_month_rise,
+    f.age_range,
+    f.seasons,
+    f.additional_features,
+    f.closure_type,
+    f.shape,
+    f.theme,
+    f.pattern,
+    f.node_id
 FROM `dwt`.`{site_name}_flow_asin_week` f
 LEFT JOIN `dwd`.`dwd_asin_profit_rate_latest` pr
     ON f.asin = pr.asin AND f.price = pr.price AND pr.site_name = '{site_name}'