内部asin改为从mysql导入

Pyspark_job/sqoop_import/ods_self_asin_detail.py

@@ -14,8 +14,8 @@ if __name__ == '__main__':
     assert date_type is not None, "date_type 不能为空！"
     assert date_info is not None, "date_info 不能为空！"
     year = date_info.split("-")[0]
-    db_type = 'postgresql_15'
-    import_table = f"{site_name}_self_asin_detail_{year}"
+    db_type = 'mysql'
+    import_table = f"{site_name}_self_asin_detail"
 
     hive_table = "ods_self_asin_detail"
     partition_dict = {
@@ -32,32 +32,32 @@ if __name__ == '__main__':
     query = f"""
     SELECT
   asin,
-  REPLACE(REPLACE(REPLACE(img_url, E'\n',' '), E'\r',' '), E'\t',' ') AS img_url,
-  REPLACE(REPLACE(REPLACE(title, E'\n',' '), E'\r',' '), E'\t',' ') AS title,
+  REPLACE(REPLACE(REPLACE(img_url, '\n',' '), '\r',' '), '\t',' ') AS img_url,
+  REPLACE(REPLACE(REPLACE(title, '\n',' '), '\r',' '), '\t',' ') AS title,
   title_len,
   price,
   rating,
   total_comments,
   buy_box_seller_type,
   page_inventory,
-  REPLACE(REPLACE(REPLACE(category, E'\n',' '), E'\r',' '), E'\t',' ') AS category,
-  REPLACE(REPLACE(REPLACE(volume, E'\n',' '), E'\r',' '), E'\t',' ') AS volume,
+  REPLACE(REPLACE(REPLACE(category, '\n',' '), '\r',' '), '\t',' ') AS category,
+  REPLACE(REPLACE(REPLACE(volume, '\n',' '), '\r',' '), '\t',' ') AS volume,
   weight,
-  rank,
+  `rank`,
   launch_time,
-  REPLACE(REPLACE(REPLACE(video_url, E'\n',' '), E'\r',' '), E'\t',' ') AS video_url,
-  REPLACE(REPLACE(REPLACE(add_url, E'\n',' '), E'\r',' '), E'\t',' ') AS add_url,
-  REPLACE(REPLACE(REPLACE(material, E'\n',' '), E'\r',' '), E'\t',' ') AS material,
+  REPLACE(REPLACE(REPLACE(video_url, '\n',' '), '\r',' '), '\t',' ') AS video_url,
+  REPLACE(REPLACE(REPLACE(add_url, '\n',' '), '\r',' '), '\t',' ') AS add_url,
+  REPLACE(REPLACE(REPLACE(material, '\n',' '), '\r',' '), '\t',' ') AS material,
   img_num,
-  REPLACE(REPLACE(REPLACE(img_type, E'\n',' '), E'\r',' '), E'\t',' ') AS img_type,
+  REPLACE(REPLACE(REPLACE(img_type, '\n',' '), '\r',' '), '\t',' ') AS img_type,
   qa_num,
-  REPLACE(REPLACE(REPLACE(brand, E'\n',' '), E'\r',' '), E'\t',' ') AS brand,
-  REPLACE(REPLACE(REPLACE(ac_name, E'\n',' '), E'\r',' '), E'\t',' ') AS ac_name,
-  REPLACE(REPLACE(REPLACE(node_id, E'\n',' '), E'\r',' '), E'\t',' ') AS node_id,
-  REPLACE(REPLACE(REPLACE(sp_num, E'\n',' '), E'\r',' '), E'\t',' ') AS sp_num,
-  REPLACE(REPLACE(REPLACE(mpn, E'\n',' '), E'\r',' '), E'\t',' ') AS mpn,
+  REPLACE(REPLACE(REPLACE(brand, '\n',' '), '\r',' '), '\t',' ') AS brand,
+  REPLACE(REPLACE(REPLACE(ac_name, '\n',' '), '\r',' '), '\t',' ') AS ac_name,
+  REPLACE(REPLACE(REPLACE(node_id, '\n',' '), '\r',' '), '\t',' ') AS node_id,
+  REPLACE(REPLACE(REPLACE(sp_num, '\n',' '), '\r',' '), '\t',' ') AS sp_num,
+  REPLACE(REPLACE(REPLACE(mpn, '\n',' '), '\r',' '), '\t',' ') AS mpn,
   online_time,
-  REPLACE(REPLACE(REPLACE(describe, E'\n',' '), E'\r',' '), E'\t',' ') AS describe,
+  REPLACE(REPLACE(REPLACE(`describe`, '\n',' '), '\r',' '), '\t',' ') AS `describe`,
   one_star,
   two_star,
   three_star,
@@ -65,45 +65,46 @@ if __name__ == '__main__':
   five_star,
   low_star,
   asin_type,
-  REPLACE(REPLACE(REPLACE(is_coupon, E'\n',' '), E'\r',' '), E'\t',' ') AS is_coupon,
-  REPLACE(REPLACE(REPLACE(search_category, E'\n',' '), E'\r',' '), E'\t',' ') AS search_category,
-  REPLACE(REPLACE(REPLACE(weight_str, E'\n',' '), E'\r',' '), E'\t',' ') AS weight_str,
-  REPLACE(REPLACE(REPLACE(account_name, E'\n',' '), E'\r',' '), E'\t',' ') AS account_name,
-  REPLACE(REPLACE(REPLACE(other_seller_name, E'\n',' '), E'\r',' '), E'\t',' ') AS other_seller_name,
-  REPLACE(REPLACE(REPLACE(bsr_date_info, E'\n',' '), E'\r',' '), E'\t',' ') AS bsr_date_info,
-  REPLACE(REPLACE(REPLACE(account_id, E'\n',' '), E'\r',' '), E'\t',' ') AS account_id,
-  REPLACE(REPLACE(REPLACE(package_quantity, E'\n',' '), E'\r',' '), E'\t',' ') AS package_quantity,
-  REPLACE(REPLACE(REPLACE(pattern_name, E'\n',' '), E'\r',' '), E'\t',' ') AS pattern_name,
-  REPLACE(REPLACE(REPLACE(together_asin, E'\n',' '), E'\r',' '), E'\t',' ') AS together_asin,
-  REPLACE(REPLACE(REPLACE(activity_type, E'\n',' '), E'\r',' '), E'\t',' ') AS activity_type,
+  REPLACE(REPLACE(REPLACE(is_coupon, '\n',' '), '\r',' '), '\t',' ') AS is_coupon,
+  REPLACE(REPLACE(REPLACE(search_category, '\n',' '), '\r',' '), '\t',' ') AS search_category,
+  REPLACE(REPLACE(REPLACE(weight_str, '\n',' '), '\r',' '), '\t',' ') AS weight_str,
+  REPLACE(REPLACE(REPLACE(account_name, '\n',' '), '\r',' '), '\t',' ') AS account_name,
+  REPLACE(REPLACE(REPLACE(other_seller_name, '\n',' '), '\r',' '), '\t',' ') AS other_seller_name,
+  REPLACE(REPLACE(REPLACE(bsr_date_info, '\n',' '), '\r',' '), '\t',' ') AS bsr_date_info,
+  REPLACE(REPLACE(REPLACE(account_id, '\n',' '), '\r',' '), '\t',' ') AS account_id,
+  REPLACE(REPLACE(REPLACE(package_quantity, '\n',' '), '\r',' '), '\t',' ') AS package_quantity,
+  REPLACE(REPLACE(REPLACE(pattern_name, '\n',' '), '\r',' '), '\t',' ') AS pattern_name,
+  REPLACE(REPLACE(REPLACE(together_asin, '\n',' '), '\r',' '), '\t',' ') AS together_asin,
+  REPLACE(REPLACE(REPLACE(activity_type, '\n',' '), '\r',' '), '\t',' ') AS activity_type,
   one_two_val, 
   three_four_val, 
   five_six_val, 
   eight_val, 
-  REPLACE(REPLACE(REPLACE(product_description, E'\n',' '), E'\r',' '), E'\t',' ') AS product_description,
+  REPLACE(REPLACE(REPLACE(product_description, '\n',' '), '\r',' '), '\t',' ') AS product_description,
   follow_sellers,
-  REPLACE(REPLACE(REPLACE(buy_sales, E'\n',' '), E'\r',' '), E'\t',' ') AS buy_sales,
+  REPLACE(REPLACE(REPLACE(buy_sales, '\n',' '), '\r',' '), '\t',' ') AS buy_sales,
   image_view, 
-  REPLACE(REPLACE(REPLACE(product_json, E'\n',' '), E'\r',' '), E'\t',' ') AS product_json,
-  REPLACE(REPLACE(REPLACE(productdetail_json, E'\n',' '), E'\r',' '), E'\t',' ') AS productdetail_json,
-  REPLACE(REPLACE(REPLACE(review_ai_text, E'\n',' '), E'\r',' '), E'\t',' ') AS review_ai_text,
-  REPLACE(REPLACE(REPLACE(review_label_json, E'\n',' '), E'\r',' '), E'\t',' ') AS review_label_json,
-  REPLACE(REPLACE(REPLACE(lob_asin_json, E'\n',' '), E'\r',' '), E'\t',' ') AS lob_asin_json,
-  REPLACE(REPLACE(REPLACE(sp_initial_seen_asins_json, E'\n',' '), E'\r',' '), E'\t',' ') AS sp_initial_seen_asins_json,
-  REPLACE(REPLACE(REPLACE(sp_4stars_initial_seen_asins_json, E'\n',' '), E'\r',' '), E'\t',' ') AS sp_4stars_initial_seen_asins_json,
-  REPLACE(REPLACE(REPLACE(sp_delivery_initial_seen_asins_json, E'\n',' '), E'\r',' '), E'\t',' ') AS sp_delivery_initial_seen_asins_json,
-  REPLACE(REPLACE(REPLACE(compare_similar_asin_json, E'\n',' '), E'\r',' '), E'\t',' ') AS compare_similar_asin_json,
-  REPLACE(REPLACE(REPLACE(customer_reviews_json, E'\n',' '), E'\r',' '), E'\t',' ') AS customer_reviews_json,
-  REPLACE(REPLACE(REPLACE(together_asin_json, E'\n',' '), E'\r',' '), E'\t',' ') AS together_asin_json,
-  REPLACE(REPLACE(REPLACE(min_match_asin_json, E'\n',' '), E'\r',' '), E'\t',' ') AS min_match_asin_json,
-  REPLACE(REPLACE(REPLACE(seller_json, E'\n',' '), E'\r',' '), E'\t',' ') AS seller_json,
-  returns, 
+  REPLACE(REPLACE(REPLACE(product_json, '\n',' '), '\r',' '), '\t',' ') AS product_json,
+  REPLACE(REPLACE(REPLACE(productdetail_json, '\n',' '), '\r',' '), '\t',' ') AS productdetail_json,
+  REPLACE(REPLACE(REPLACE(review_ai_text, '\n',' '), '\r',' '), '\t',' ') AS review_ai_text,
+  REPLACE(REPLACE(REPLACE(review_label_json, '\n',' '), '\r',' '), '\t',' ') AS review_label_json,
+  REPLACE(REPLACE(REPLACE(lob_asin_json, '\n',' '), '\r',' '), '\t',' ') AS lob_asin_json,
+  REPLACE(REPLACE(REPLACE(sp_initial_seen_asins_json, '\n',' '), '\r',' '), '\t',' ') AS sp_initial_seen_asins_json,
+  REPLACE(REPLACE(REPLACE(sp_4stars_initial_seen_asins_json, '\n',' '), '\r',' '), '\t',' ') AS sp_4stars_initial_seen_asins_json,
+  REPLACE(REPLACE(REPLACE(sp_delivery_initial_seen_asins_json, '\n',' '), '\r',' '), '\t',' ') AS sp_delivery_initial_seen_asins_json,
+  REPLACE(REPLACE(REPLACE(compare_similar_asin_json, '\n',' '), '\r',' '), '\t',' ') AS compare_similar_asin_json,
+  REPLACE(REPLACE(REPLACE(customer_reviews_json, '\n',' '), '\r',' '), '\t',' ') AS customer_reviews_json,
+  REPLACE(REPLACE(REPLACE(together_asin_json, '\n',' '), '\r',' '), '\t',' ') AS together_asin_json,
+  REPLACE(REPLACE(REPLACE(min_match_asin_json, '\n',' '), '\r',' '), '\t',' ') AS min_match_asin_json,
+  REPLACE(REPLACE(REPLACE(seller_json, '\n',' '), '\r',' '), '\t',' ') AS seller_json,
+  `returns`,
   created_at, updated_at,
-  REPLACE(REPLACE(REPLACE(result_list_json, E'\n',' '), E'\r',' '), E'\t',' ') AS result_list_json,
-  REPLACE(REPLACE(REPLACE(variat_list, E'\n',' '), E'\r',' '), E'\t',' ') AS variat_list,
-  REPLACE(REPLACE(REPLACE(bundle_asin_component_json, E'\n',' '), E'\r',' '), E'\t',' ') AS bundle_asin_component_json,
-  REPLACE(REPLACE(REPLACE(cart_type, E'\n',' '), E'\r',' '), E'\t',' ') AS cart_type, 
-  REPLACE(REPLACE(REPLACE(site, E'\n',' '), E'\r',' '), E'\t',' ') AS site
+  REPLACE(REPLACE(REPLACE(result_list_json, '\n',' '), '\r',' '), '\t',' ') AS result_list_json,
+  REPLACE(REPLACE(REPLACE(variat_list, '\n',' '), '\r',' '), '\t',' ') AS variat_list,
+  REPLACE(REPLACE(REPLACE(bundle_asin_component_json, '\n',' '), '\r',' '), '\t',' ') AS bundle_asin_component_json,
+  REPLACE(REPLACE(REPLACE(cart_type, '\n',' '), '\r',' '), '\t',' ') AS cart_type, 
+  REPLACE(REPLACE(REPLACE(site, '\n',' '), '\r',' '), '\t',' ') AS site,
+  REPLACE(REPLACE(REPLACE(comment_json, '\n',' '), '\r',' '), '\t',' ') AS comment_json
 
 FROM {import_table}
 WHERE  date_info='{date_info}' and 1=1 AND \$CONDITIONS
@@ -111,10 +112,10 @@ WHERE  date_info='{date_info}' and 1=1 AND \$CONDITIONS
 
 #   spider_int,
 #   variat_num,
-#   REPLACE(REPLACE(REPLACE(img_list, E'\n',' '), E'\r',' '), E'\t',' ') AS img_list,
-#   REPLACE(REPLACE(REPLACE(parent_asin, E'\n',' '), E'\r',' '), E'\t',' ') AS parent_asin,
-#   REPLACE(REPLACE(REPLACE(video_m3u8_url, E'\n',' '), E'\r',' '), E'\t',' ') AS video_m3u8_url,
-#   REPLACE(REPLACE(REPLACE(review_json_list, E'\n',' '), E'\r',' '), E'\t',' ') AS review_json_list,
+#   REPLACE(REPLACE(REPLACE(img_list, '\n',' '), '\r',' '), '\t',' ') AS img_list,
+#   REPLACE(REPLACE(REPLACE(parent_asin, '\n',' '), '\r',' '), '\t',' ') AS parent_asin,
+#   REPLACE(REPLACE(REPLACE(video_m3u8_url, '\n',' '), '\r',' '), '\t',' ') AS video_m3u8_url,
+#   REPLACE(REPLACE(REPLACE(review_json_list, '\n',' '), '\r',' '), '\t',' ') AS review_json_list,
 #   fbm_delivery_price
 
     engine.sqoop_raw_import(