Skip to content

A
Amazon-Selection-Data
Commit e22e3efe by hejiangming
Options

no message

parent c62a2353
Showing with 2 additions and 2 deletions
Pyspark_job/sqoop_export/dws_st_theme.py
...@@ -28,7 +28,7 @@ if __name__ == '__main__': ...@@ -28,7 +28,7 @@ if __name__ == '__main__':
# 获取数据库连接 # 获取数据库连接
engine = DBUtil.get_db_engine(db_type, site_name) engine = DBUtil.get_db_engine(db_type, site_name)
# 导出前校验 Hive 分区是否有数据，避免空分区触发交换导致 PG 数据被清空 # 导出前校验 Hive 分区是否有数据，避免空分区触发交换导致 PG 数据被清空
hive_partition_path = f"/home/big_data_selection/dwt/dwt_aba_last_change_rate/site_name={site_name}/date_type={date_type}/date_info={date_info}" hive_partition_path = f"/home/big_data_selection/dwt/dws_st_theme/site_name={site_name}/date_type={date_type}/date_info={date_info}"
hive_files = HdfsUtils.read_list(hive_partition_path) hive_files = HdfsUtils.read_list(hive_partition_path)
if not hive_files: if not hive_files:
print(f"[ERROR] Hive 分区无数据文件，路径：{hive_partition_path}，跳过导出，请先检查 DWT 计算任务是否正常写入！") print(f"[ERROR] Hive 分区无数据文件，路径：{hive_partition_path}，跳过导出，请先检查 DWT 计算任务是否正常写入！")
......
Pyspark_job/sqoop_export/dwt_st_theme_agg.py
...@@ -27,7 +27,7 @@ if __name__ == '__main__': ...@@ -27,7 +27,7 @@ if __name__ == '__main__':
# 获取数据库连接 # 获取数据库连接
engine = DBUtil.get_db_engine(db_type, site_name) engine = DBUtil.get_db_engine(db_type, site_name)
# 导出前校验 Hive 分区是否有数据，避免空分区触发交换导致 PG 数据被清空 # 导出前校验 Hive 分区是否有数据，避免空分区触发交换导致 PG 数据被清空
hive_partition_path = f"/home/big_data_selection/dwt/dwt_aba_last_change_rate/site_name={site_name}/date_type={date_type}/date_info={date_info}" hive_partition_path = f"/home/big_data_selection/dwt/dwt_st_theme_agg/site_name={site_name}/date_type={date_type}/date_info={date_info}"
hive_files = HdfsUtils.read_list(hive_partition_path) hive_files = HdfsUtils.read_list(hive_partition_path)
if not hive_files: if not hive_files:
print(f"[ERROR] Hive 分区无数据文件，路径：{hive_partition_path}，跳过导出，请先检查 DWT 计算任务是否正常写入！") print(f"[ERROR] Hive 分区无数据文件，路径：{hive_partition_path}，跳过导出，请先检查 DWT 计算任务是否正常写入！")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment