no message

Pyspark_job/sqoop_export/dim_st_detail_week.py

@@ -27,12 +27,15 @@ if __name__ == '__main__':
     week = int(week_str)
 
     export_master_tb = f"{site_name}_aba_report_week_{year}"
+    #   用上一年母表 like 出当年母表(including all 带那4个分区索引)、按 week 分区；已存在则 if not exists 跳过。
+    export_master_tb_before = f"{site_name}_aba_report_week_{year - 1}"
     export_tb = f"{export_master_tb}_{week}"
     next_week = week + 1
 
     engine = DBUtil.get_db_engine(db_type, site_name)
     with engine.connect() as connection:
         sql = f"""
+            create table if not exists {export_master_tb} (like {export_master_tb_before} including all) partition by range (week);
             drop table if exists {export_tb};
             create table if not exists {export_tb}
             (
@@ -132,61 +135,63 @@ if __name__ == '__main__':
         cp_index_flag=True,
     )
 
-    # 第2步：Spark SQL 从 dim_st_detail_week 写入 dwt_st_base_report
-    print("================================写入周趋势数据到Hive================================")
-    spark = SparkUtil.get_spark_session(f"export_dim_st_detail_week_trend:{site_name} {date_type} {date_info}")
-    dwt_hive_tb = "dwt_st_base_report"
-    dwt_hdfs_path = f"/home/big_data_selection/dwt/{dwt_hive_tb}/site_name={site_name}/date_type=week/date_info={date_info}"
-    print(f"清除hdfs目录中数据:{dwt_hdfs_path}")
-    HdfsUtils.delete_hdfs_file(dwt_hdfs_path)
-    spark.sql(f"""
-        INSERT INTO TABLE {dwt_hive_tb} PARTITION(site_name='{site_name}', date_type='week', date_info='{date_info}')
-        SELECT
-            st_key,
-            search_term,
-            search_volume as st_volume,
-            orders as st_orders,
-            {year} as years,
-            current_timestamp() as created_time,
-            current_timestamp() as updated_time,
-            `rank` as st_rank
-        FROM dim_st_detail_week
-        WHERE site_name='{site_name}' AND date_type='week' AND date_info='{date_info}'
-    """)
-    spark.stop()
-    print("================================写入周趋势数据完成================================")
-
-    # 第3步：Sqoop 导出 dwt_st_base_report 到 PG 趋势表
-    print("================================导出周趋势图到PG集群================================")
-    trend_master_tb = f"{site_name}_aba_last_total_week"
-    trend_year_next = str(year + 1)
-    trend_export_tb = f"{trend_master_tb}_{year_str}"
-
-    trend_sql = f"""
-        create table if not exists {trend_export_tb} partition of {trend_master_tb} for values from ('{year_str}') to ('{trend_year_next}');
-        create index if not exists {trend_export_tb}_st_key_idx on {trend_export_tb} using btree (st_key);
-        delete from {trend_export_tb} where date_info = '{date_info}';
-    """
-    DBUtil.engine_exec_sql(engine, trend_sql)
-
-    trend_columns = ["st_key", "search_term", "st_volume", "st_rank", "st_orders", "years",
-                     "created_time", "updated_time", "date_type", "date_info"]
-    trend_sh = CommonUtil.build_export_sh(
-        site_name=site_name,
-        db_type=db_type,
-        hive_tb=dwt_hive_tb,
-        export_tb=trend_export_tb,
-        col=trend_columns,
-        partition_dict={
-            "site_name": site_name,
-            "date_type": "week",
-            "date_info": date_info
-        }
-    )
-    trend_client = SSHUtil.get_ssh_client()
-    SSHUtil.exec_command_async(trend_client, trend_sh, ignore_err=False)
-    trend_client.close()
-    print("================================周趋势图导出完成================================")
+    # 趋势改由独立脚本产：dwt/dwt_st_base_report_week.py(算)+sqoop_export/dwt_st_base_report_week.py(导)，
+    # 三站共用、口径逐字段一致。此处不再产趋势，避免与独立趋势任务并行写 {site}_aba_last_total_week
+    # # 第2步：Spark SQL 从 dim_st_detail_week 写入 dwt_st_base_report
+    # print("================================写入周趋势数据到Hive================================")
+    # spark = SparkUtil.get_spark_session(f"export_dim_st_detail_week_trend:{site_name} {date_type} {date_info}")
+    # dwt_hive_tb = "dwt_st_base_report"
+    # dwt_hdfs_path = f"/home/big_data_selection/dwt/{dwt_hive_tb}/site_name={site_name}/date_type=week/date_info={date_info}"
+    # print(f"清除hdfs目录中数据:{dwt_hdfs_path}")
+    # HdfsUtils.delete_hdfs_file(dwt_hdfs_path)
+    # spark.sql(f"""
+    #     INSERT INTO TABLE {dwt_hive_tb} PARTITION(site_name='{site_name}', date_type='week', date_info='{date_info}')
+    #     SELECT
+    #         st_key,
+    #         search_term,
+    #         search_volume as st_volume,
+    #         orders as st_orders,
+    #         {year} as years,
+    #         current_timestamp() as created_time,
+    #         current_timestamp() as updated_time,
+    #         `rank` as st_rank
+    #     FROM dim_st_detail_week
+    #     WHERE site_name='{site_name}' AND date_type='week' AND date_info='{date_info}'
+    # """)
+    # spark.stop()
+    # print("================================写入周趋势数据完成================================")
+
+    # # 第3步：Sqoop 导出 dwt_st_base_report 到 PG 趋势表
+    # print("================================导出周趋势图到PG集群================================")
+    # trend_master_tb = f"{site_name}_aba_last_total_week"
+    # trend_year_next = str(year + 1)
+    # trend_export_tb = f"{trend_master_tb}_{year_str}"
+    #
+    # trend_sql = f"""
+    #     create table if not exists {trend_export_tb} partition of {trend_master_tb} for values from ('{year_str}') to ('{trend_year_next}');
+    #     create index if not exists {trend_export_tb}_st_key_idx on {trend_export_tb} using btree (st_key);
+    #     delete from {trend_export_tb} where date_info = '{date_info}';
+    # """
+    # DBUtil.engine_exec_sql(engine, trend_sql)
+    #
+    # trend_columns = ["st_key", "search_term", "st_volume", "st_rank", "st_orders", "years",
+    #                  "created_time", "updated_time", "date_type", "date_info"]
+    # trend_sh = CommonUtil.build_export_sh(
+    #     site_name=site_name,
+    #     db_type=db_type,
+    #     hive_tb=dwt_hive_tb,
+    #     export_tb=trend_export_tb,
+    #     col=trend_columns,
+    #     partition_dict={
+    #         "site_name": site_name,
+    #         "date_type": "week",
+    #         "date_info": date_info
+    #     }
+    # )
+    # trend_client = SSHUtil.get_ssh_client()
+    # SSHUtil.exec_command_async(trend_client, trend_sh, ignore_err=False)
+    # trend_client.close()
+    # print("================================周趋势图导出完成================================")
 
     # 插入流程记录表
     sql = f"""

Pyspark_job/sqoop_export/dwt_st_base_report_week.py

@@ -14,6 +14,7 @@ sys.path.append(os.path.dirname(sys.path[0]))
 from utils.ssh_util import SSHUtil
 from utils.common_util import CommonUtil
 from utils.db_util import DBUtil, DbTypes
+from utils.hdfs_utils import HdfsUtils
 
 if __name__ == '__main__':
     site_name = CommonUtil.get_sys_arg(1, None)   # 参数1：站点 us/uk/de
@@ -40,6 +41,16 @@ if __name__ == '__main__':
 
     engine = DBUtil.get_db_engine(db_type, site_name)
 
+    hive_path = CommonUtil.build_hdfs_path(
+        "dwt_st_base_report",
+        partition_dict={"site_name": site_name, "date_type": date_type, "date_info": date_info},
+    )
+    if not HdfsUtils.read_list(hive_path):
+        print(f"[ERROR] Hive 分区无数据文件：{hive_path}，跳过导出(不删PG)，请先检查趋势计算任务！")
+        engine.dispose()
+        sys.exit(1)
+    print(f"Hive 分区有数据：{hive_path}，继续导出")
+
     # ============================================================
     # 建当年分区子表 + st_key 索引 + 清本周
     # 【为什么 create if not exists partition of】趋势表是「顶层母表 + 按年 RANGE 分区子表」，