Skip to content

A
Amazon-Selection-Data
Commit 17c74913 by hejiangming
Options

no message

parent d6c5d1af
Showing with 72 additions and 56 deletions
Pyspark_job/sqoop_export/dim_st_detail_week.py
...@@ -27,14 +27,17 @@ if __name__ == '__main__': ...@@ -27,14 +27,17 @@ if __name__ == '__main__':
week = int(week_str) week = int(week_str)
export_master_tb = f"{site_name}_aba_report_week_{year}" export_master_tb = f"{site_name}_aba_report_week_{year}"
# 用上一年母表 like 出当年母表(including all 带那4个分区索引)、按 week 分区；已存在则 if not exists 跳过。
export_master_tb_before = f"{site_name}_aba_report_week_{year - 1}"
export_tb = f"{export_master_tb}_{week}" export_tb = f"{export_master_tb}_{week}"
next_week = week + 1 next_week = week + 1
engine = DBUtil.get_db_engine(db_type, site_name) engine = DBUtil.get_db_engine(db_type, site_name)
with engine.connect() as connection: with engine.connect() as connection:
sql = f""" sql = f"""
create table if not exists {export_master_tb} (like {export_master_tb_before} including all) partition by range (week);
drop table if exists {export_tb}; drop table if exists {export_tb};
create table if not exists {export_tb} create table if not exists {export_tb}
( (
like {export_master_tb} including defaults including comments like {export_master_tb} including defaults including comments
); );
...@@ -132,61 +135,63 @@ if __name__ == '__main__': ...@@ -132,61 +135,63 @@ if __name__ == '__main__':
cp_index_flag=True, cp_index_flag=True,
) )
# 第2步：Spark SQL 从 dim_st_detail_week 写入 dwt_st_base_report # 趋势改由独立脚本产：dwt/dwt_st_base_report_week.py(算)+sqoop_export/dwt_st_base_report_week.py(导)，
print("================================写入周趋势数据到Hive================================") # 三站共用、口径逐字段一致。此处不再产趋势，避免与独立趋势任务并行写 {site}_aba_last_total_week
spark = SparkUtil.get_spark_session(f"export_dim_st_detail_week_trend:{site_name} {date_type} {date_info}") # # 第2步：Spark SQL 从 dim_st_detail_week 写入 dwt_st_base_report
dwt_hive_tb = "dwt_st_base_report" # print("================================写入周趋势数据到Hive================================")
dwt_hdfs_path = f"/home/big_data_selection/dwt/{dwt_hive_tb}/site_name={site_name}/date_type=week/date_info={date_info}" # spark = SparkUtil.get_spark_session(f"export_dim_st_detail_week_trend:{site_name} {date_type} {date_info}")
print(f"清除hdfs目录中数据:{dwt_hdfs_path}") # dwt_hive_tb = "dwt_st_base_report"
HdfsUtils.delete_hdfs_file(dwt_hdfs_path) # dwt_hdfs_path = f"/home/big_data_selection/dwt/{dwt_hive_tb}/site_name={site_name}/date_type=week/date_info={date_info}"
spark.sql(f""" # print(f"清除hdfs目录中数据:{dwt_hdfs_path}")
INSERT INTO TABLE {dwt_hive_tb} PARTITION(site_name='{site_name}', date_type='week', date_info='{date_info}') # HdfsUtils.delete_hdfs_file(dwt_hdfs_path)
SELECT # spark.sql(f"""
st_key, # INSERT INTO TABLE {dwt_hive_tb} PARTITION(site_name='{site_name}', date_type='week', date_info='{date_info}')
search_term, # SELECT
search_volume as st_volume, # st_key,
orders as st_orders, # search_term,
{year} as years, # search_volume as st_volume,
current_timestamp() as created_time, # orders as st_orders,
current_timestamp() as updated_time, # {year} as years,
`rank` as st_rank # current_timestamp() as created_time,
FROM dim_st_detail_week # current_timestamp() as updated_time,
WHERE site_name='{site_name}' AND date_type='week' AND date_info='{date_info}' # `rank` as st_rank
""") # FROM dim_st_detail_week
spark.stop() # WHERE site_name='{site_name}' AND date_type='week' AND date_info='{date_info}'
print("================================写入周趋势数据完成================================") # """)
# spark.stop()
# 第3步：Sqoop 导出 dwt_st_base_report 到 PG 趋势表 # print("================================写入周趋势数据完成================================")
print("================================导出周趋势图到PG集群================================")
trend_master_tb = f"{site_name}_aba_last_total_week" # # 第3步：Sqoop 导出 dwt_st_base_report 到 PG 趋势表
trend_year_next = str(year + 1) # print("================================导出周趋势图到PG集群================================")
trend_export_tb = f"{trend_master_tb}_{year_str}" # trend_master_tb = f"{site_name}_aba_last_total_week"
# trend_year_next = str(year + 1)
trend_sql = f""" # trend_export_tb = f"{trend_master_tb}_{year_str}"
create table if not exists {trend_export_tb} partition of {trend_master_tb} for values from ('{year_str}') to ('{trend_year_next}'); #
create index if not exists {trend_export_tb}_st_key_idx on {trend_export_tb} using btree (st_key); # trend_sql = f"""
delete from {trend_export_tb} where date_info = '{date_info}'; # create table if not exists {trend_export_tb} partition of {trend_master_tb} for values from ('{year_str}') to ('{trend_year_next}');
""" # create index if not exists {trend_export_tb}_st_key_idx on {trend_export_tb} using btree (st_key);
DBUtil.engine_exec_sql(engine, trend_sql) # delete from {trend_export_tb} where date_info = '{date_info}';
# """
trend_columns = ["st_key", "search_term", "st_volume", "st_rank", "st_orders", "years", # DBUtil.engine_exec_sql(engine, trend_sql)
"created_time", "updated_time", "date_type", "date_info"] #
trend_sh = CommonUtil.build_export_sh( # trend_columns = ["st_key", "search_term", "st_volume", "st_rank", "st_orders", "years",
site_name=site_name, # "created_time", "updated_time", "date_type", "date_info"]
db_type=db_type, # trend_sh = CommonUtil.build_export_sh(
hive_tb=dwt_hive_tb, # site_name=site_name,
export_tb=trend_export_tb, # db_type=db_type,
col=trend_columns, # hive_tb=dwt_hive_tb,
partition_dict={ # export_tb=trend_export_tb,
"site_name": site_name, # col=trend_columns,
"date_type": "week", # partition_dict={
"date_info": date_info # "site_name": site_name,
} # "date_type": "week",
) # "date_info": date_info
trend_client = SSHUtil.get_ssh_client() # }
SSHUtil.exec_command_async(trend_client, trend_sh, ignore_err=False) # )
trend_client.close() # trend_client = SSHUtil.get_ssh_client()
print("================================周趋势图导出完成================================") # SSHUtil.exec_command_async(trend_client, trend_sh, ignore_err=False)
# trend_client.close()
# print("================================周趋势图导出完成================================")
# 插入流程记录表 # 插入流程记录表
sql = f""" sql = f"""
......
Pyspark_job/sqoop_export/dwt_st_base_report_week.py
...@@ -14,6 +14,7 @@ sys.path.append(os.path.dirname(sys.path[0])) ...@@ -14,6 +14,7 @@ sys.path.append(os.path.dirname(sys.path[0]))
from utils.ssh_util import SSHUtil from utils.ssh_util import SSHUtil
from utils.common_util import CommonUtil from utils.common_util import CommonUtil
from utils.db_util import DBUtil, DbTypes from utils.db_util import DBUtil, DbTypes
from utils.hdfs_utils import HdfsUtils
if __name__ == '__main__': if __name__ == '__main__':
site_name = CommonUtil.get_sys_arg(1, None) # 参数1：站点 us/uk/de site_name = CommonUtil.get_sys_arg(1, None) # 参数1：站点 us/uk/de
...@@ -40,6 +41,16 @@ if __name__ == '__main__': ...@@ -40,6 +41,16 @@ if __name__ == '__main__':
engine = DBUtil.get_db_engine(db_type, site_name) engine = DBUtil.get_db_engine(db_type, site_name)
hive_path = CommonUtil.build_hdfs_path(
"dwt_st_base_report",
partition_dict={"site_name": site_name, "date_type": date_type, "date_info": date_info},
)
if not HdfsUtils.read_list(hive_path):
print(f"[ERROR] Hive 分区无数据文件：{hive_path}，跳过导出(不删PG)，请先检查趋势计算任务！")
engine.dispose()
sys.exit(1)
print(f"Hive 分区有数据：{hive_path}，继续导出")
# ============================================================ # ============================================================
# 建当年分区子表 + st_key 索引 + 清本周 # 建当年分区子表 + st_key 索引 + 清本周
# 【为什么 create if not exists partition of】趋势表是「顶层母表 + 按年 RANGE 分区子表」， # 【为什么 create if not exists partition of】趋势表是「顶层母表 + 按年 RANGE 分区子表」，
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment