Skip to content

A
Amazon-Selection-Data
Commit 1978d698 by hejiangming
Options

新增品类调研计算 读取流量选品数据

parent d059a090
Showing with 295 additions and 0 deletions
Pyspark_job/sqoop_export/dwt_bs_top100.py 0 → 100644
"""
@Author : hejiangming
@Description : 品类调研主表导出 PG 集群：Hive dwt_bs_top100 → PG {site}_category_top_analysis。
PG 侧是"单基表 + 按月 RANGE(date_info) 分区"（趋势图跨月/跨年查一张表, 详见 us_category_top_analysis.sql）。
每月导数：建当月空分区 + copy 表 → sqoop 灌 copy 表 → 分区交换换入(exchange_pg_part_tb), 幂等重导、无空窗期。
@SourceTable : dwt_bs_top100 (Hive, 分区 site_name/date_type/date_info)
@SinkTable : {site}_category_top_analysis (PG 集群, 按 date_info 月分区)
@CreateTime : 2026/07/15
@UpdateTime : 2026/07/15
"""
import os
import sys
sys.path.append(os.path.dirname(sys.path[0]))
from utils.ssh_util import SSHUtil
from utils.common_util import CommonUtil, DateTypes
from utils.db_util import DBUtil
from utils.hdfs_utils import HdfsUtils
if __name__ == '__main__':
site_name = CommonUtil.get_sys_arg(1, None)
date_type = CommonUtil.get_sys_arg(2, None)
date_info = CommonUtil.get_sys_arg(3, None)
# 最后一个参数判断是否导测试库
test_flag = CommonUtil.get_sys_arg(len(sys.argv) - 1, None)
print(f"执行参数为{sys.argv}")
# 品类调研只有月粒度
assert date_type == DateTypes.month.name, f"仅支持 month, 传入 date_type={date_type}"
if test_flag == 'test':
db_type = 'postgresql_test'
print("导出到PG测试库中")
else:
CommonUtil.judge_is_work_hours(site_name=site_name, date_type=date_type, date_info=date_info,
principal='hejiangming', priority=2, export_tools_type=1,
belonging_to_process=f'品类调研流程_{date_type}')
db_type = 'postgresql_cluster'
print("导出到PG集群库中")
# 获取数据库连接
engine = DBUtil.get_db_engine(db_type, site_name)
# 导出前校验 Hive 分区有数据：空分区会导致灌进空 copy 表、分区交换后把 PG 当月换成空
hive_path = CommonUtil.build_hdfs_path(
"dwt_bs_top100",
partition_dict={"site_name": site_name, "date_type": date_type, "date_info": date_info},
)
if not HdfsUtils.read_list(hive_path):
print(f"[ERROR] Hive 分区无数据文件：{hive_path}，跳过导出，请先检查 DWT 计算任务！")
engine.dispose()
sys.exit(1)
print(f"Hive 分区有数据：{hive_path}，继续导出")
# 用"copy 表 + 分区交换"方式导数（比先 truncate 再灌更稳：先把数据灌进 copy 表，成功后再原子换入分区，
# sqoop 中途失败也不会清空线上当月数据、无空窗期；机制同 dwt_aba_st_analytics.py 的 month 分支）。
base_tb = f"{site_name}_category_top_analysis"
date_partition = str(date_info).replace("-", "_")
export_table_original = f"{base_tb}_{date_partition}" # 当月分区子表, 如 us_category_top_analysis_2026_05
export_tb_copy = f"{export_table_original}_copy" # 灌数用的临时普通表
next_val = CommonUtil.get_next_val(date_type, date_info) # 下月, 作分区右边界(不含)
# 1) 建当月分区(交换时需 detach 它, 首次跑先建空的)；2) 建 copy 表(形如分区子表, 含索引)并清空
sql = f"""
create table if not exists {export_table_original} partition of {base_tb} for values from ('{date_info}') to ('{next_val}');
create table if not exists {export_tb_copy} (like {export_table_original} including all);
truncate table {export_tb_copy};
"""
DBUtil.engine_exec_sql(engine, sql)
# 导出列 = 86 业务列 + 3 分区列（列名与 Hive dwt_bs_top100 一致, sqoop 按名映射 PG 同名列）
export_cols = [
# 维度 / 键
"asin_bs_cate_current_id", "asin_bs_cate_parent_id", "asin_bs_cate_parent_id_join",
"level", "cur_category_name",
# 销量类
"asin_amazon_orders_sum", "asin_bsr_orders_sum", "asin_bsr_orders_sum_new", "asin_bsr_orders_sum_brand",
"asin_bsr_orders_mean", "asin_bsr_orders_mean_new", "asin_bsr_orders_mean_brand", "asin_bsr_orders_sum_parent",
# 销售额类
"asin_bsr_orders_sales_sum", "asin_bsr_orders_sales_sum_new", "asin_bsr_orders_sales_sum_brand",
"asin_bsr_orders_sales_mean", "asin_bsr_orders_sales_mean_new", "asin_bsr_orders_sales_mean_brand",
# 数量类
"asin_count", "asin_count_new", "asin_count_brand",
"asin_count_rate_new", "asin_count_rate_brand",
"asin_bsr_orders_sum_rate_new", "asin_bsr_orders_sum_rate_brand", "asin_bsr_orders_sum_rate",
# 价格/评分/排名/评论 均值
"asin_price_mean", "asin_rating_mean", "asin_bs_cate_1_rank_mean", "asin_total_comments_mean",
# 上架时间分布
"launch_time_type0_rate", "launch_time_type1_rate", "launch_time_type2_rate", "launch_time_type3_rate",
"launch_time_type4_rate", "launch_time_type5_rate", "launch_time_type6_rate", "launch_time_type7_rate",
# 卖家类型分布
"asin_buy_box_seller_type0_rate", "asin_buy_box_seller_type1_rate", "asin_buy_box_seller_type2_rate",
"asin_buy_box_seller_type3_rate", "asin_buy_box_seller_type4_rate",
# 卖家所属地占比
"seller_country_us_rate", "seller_country_cn_rate", "seller_country_other_rate",
# 月销阈值占比
"asin_orders_ge_50_rate", "asin_orders_ge_100_rate", "asin_orders_ge_200_rate",
# 利润率均值
"asin_ocean_freight_gross_margin_avg", "asin_air_freight_gross_margin_avg",
# 区间利润率（海运12）
"asin_ocean_freight_gross_margin_price_0_10", "asin_ocean_freight_gross_margin_price_10_20",
"asin_ocean_freight_gross_margin_price_20_30", "asin_ocean_freight_gross_margin_price_30_50",
"asin_ocean_freight_gross_margin_price_50_80", "asin_ocean_freight_gross_margin_price_80",
"asin_ocean_freight_gross_margin_orders_0_50", "asin_ocean_freight_gross_margin_orders_50_100",
"asin_ocean_freight_gross_margin_orders_100_200", "asin_ocean_freight_gross_margin_orders_200_500",
"asin_ocean_freight_gross_margin_orders_500_1000", "asin_ocean_freight_gross_margin_orders_1000",
# 区间利润率（空运12）
"asin_air_freight_gross_margin_price_0_10", "asin_air_freight_gross_margin_price_10_20",
"asin_air_freight_gross_margin_price_20_30", "asin_air_freight_gross_margin_price_30_50",
"asin_air_freight_gross_margin_price_50_80", "asin_air_freight_gross_margin_price_80",
"asin_air_freight_gross_margin_orders_0_50", "asin_air_freight_gross_margin_orders_50_100",
"asin_air_freight_gross_margin_orders_100_200", "asin_air_freight_gross_margin_orders_200_500",
"asin_air_freight_gross_margin_orders_500_1000", "asin_air_freight_gross_margin_orders_1000",
# 打包段月销占比
"asin_pkg_1_orders_rate", "asin_pkg_2_4_orders_rate", "asin_pkg_5_10_orders_rate",
"asin_pkg_10_20_orders_rate", "asin_pkg_20_50_orders_rate", "asin_pkg_50_100_orders_rate",
"asin_pkg_100_orders_rate",
# 集中度
"asin_concentration", "brand_concentration", "seller_concentration",
# 分区列（也落成普通列, 取分区值）
"site_name", "date_type", "date_info",
]
sh = CommonUtil.build_export_sh(
site_name=site_name,
db_type=db_type,
hive_tb="dwt_bs_top100",
export_tb=export_tb_copy,
col=export_cols,
partition_dict={"site_name": site_name, "date_type": date_type, "date_info": date_info},
)
client = SSHUtil.get_ssh_client()
SSHUtil.exec_command_async(client, sh, ignore_err=False)
client.close()
# 分区交换：把灌好的 copy 表原子换入当月分区（旧分区在函数内转到 copy 名下），避免 truncate 的空窗期。
# 分区表主键/索引由父表继承, cp_index_flag=False 不额外建。
DBUtil.exchange_pg_part_tb(
engine,
source_tb_name=export_tb_copy,
part_master_tb=base_tb,
part_target_tb=export_table_original,
cp_index_flag=False,
part_val={"from": [date_info], "to": [next_val]},
)
# 交换后 copy 表里是旧数据, 删掉省空间（同 aba）
DBUtil.engine_exec_sql(engine, f"drop table if exists {export_tb_copy};")
# 关闭链接
engine.dispose()
# 导出完成状态更新：主表 + 同比环比表两个导出脚本共用同一 belonging_to_process(品类调研流程_{date_type})。
# modify_export_workflow_status 会把本脚本置完成(status=3)、统计同组未完成数——只有最后一个跑完的
# (未完成数=0)才执行下面的 update_workflow_sql，即"两个都导完，页面才展示"(机制同 dwt_aba_last_change_rate.py)。
# 两脚本传同一条 update SQL，谁最后完成都能正确触发。本地/test 运行时该函数自动跳过。
# 本功能不像 aba 会在流程启动时预写一条 workflow_everyday，故用 replace INTO 直接插入/覆盖该行(不存在则建、存在则替换)。
# 只有主表+同比环比两个导出都完成时，modify_export_workflow_status 才会执行这条(即页面此时才出现"完成"行)。
update_workflow_sql = f"""
replace INTO selection.workflow_everyday
(site_name, report_date, status, status_val, table_name, date_type, page, is_end, remark, export_db_type)
VALUES('{site_name}', '{date_info}', '导出PG数据库完成', 14, '{site_name}_category_top_analysis', '{date_type}', '品类调研', '是', '品类调研月表', '{db_type}');
"""
CommonUtil.modify_export_workflow_status(update_workflow_sql, site_name, date_type, date_info)
print("success")
Pyspark_job/sqoop_export/dwt_bs_top100_change_rate.py 0 → 100644
"""
@Author : hejiangming
@Description : 品类调研同比/环比表导出 PG 集群：Hive dwt_bs_top100_change_rate → PG {site}_category_top_analysis_change_rate。
PG 侧同主表：单基表 + 按月 RANGE(date_info) 分区（详见 us_category_top_analysis.sql）。
每月导数：建当月空分区 + copy 表 → sqoop 灌 copy 表 → 分区交换换入(exchange_pg_part_tb), 幂等重导、无空窗期。
@SourceTable : dwt_bs_top100_change_rate (Hive, 分区 site_name/date_type/date_info)
@SinkTable : {site}_category_top_analysis_change_rate (PG 集群, 按 date_info 月分区)
@CreateTime : 2026/07/15
@UpdateTime : 2026/07/15
"""
import os
import sys
sys.path.append(os.path.dirname(sys.path[0]))
from utils.ssh_util import SSHUtil
from utils.common_util import CommonUtil, DateTypes
from utils.db_util import DBUtil
from utils.hdfs_utils import HdfsUtils
if __name__ == '__main__':
site_name = CommonUtil.get_sys_arg(1, None)
date_type = CommonUtil.get_sys_arg(2, None)
date_info = CommonUtil.get_sys_arg(3, None)
# 最后一个参数判断是否导测试库
test_flag = CommonUtil.get_sys_arg(len(sys.argv) - 1, None)
print(f"执行参数为{sys.argv}")
# 品类调研只有月粒度
assert date_type == DateTypes.month.name, f"仅支持 month, 传入 date_type={date_type}"
if test_flag == 'test':
db_type = 'postgresql_test'
print("导出到PG测试库中")
else:
# 非上班时段导出约束（本地/test 会自动跳过）
CommonUtil.judge_is_work_hours(site_name=site_name, date_type=date_type, date_info=date_info,
principal='hejiangming', priority=2, export_tools_type=1,
belonging_to_process=f'品类调研流程_{date_type}')
db_type = 'postgresql_cluster'
print("导出到PG集群库中")
# 获取数据库连接
engine = DBUtil.get_db_engine(db_type, site_name)
# 导出前校验 Hive 分区有数据：空分区会导致灌进空 copy 表、分区交换后把 PG 当月换成空
hive_path = CommonUtil.build_hdfs_path(
"dwt_bs_top100_change_rate",
partition_dict={"site_name": site_name, "date_type": date_type, "date_info": date_info},
)
if not HdfsUtils.read_list(hive_path):
print(f"[ERROR] Hive 分区无数据文件：{hive_path}，跳过导出，请先检查 DWT 计算任务！")
engine.dispose()
sys.exit(1)
print(f"Hive 分区有数据：{hive_path}，继续导出")
# 用"copy 表 + 分区交换"方式导数（比先 truncate 再灌更稳：先把数据灌进 copy 表，成功后再原子换入分区，
# sqoop 中途失败也不会清空线上当月数据、无空窗期；机制同 dwt_aba_st_analytics.py 的 month 分支）。
base_tb = f"{site_name}_category_top_analysis_change_rate"
date_partition = str(date_info).replace("-", "_")
export_table_original = f"{base_tb}_{date_partition}" # 当月分区子表, 如 us_category_top_analysis_change_rate_2026_05
export_tb_copy = f"{export_table_original}_copy" # 灌数用的临时普通表
next_val = CommonUtil.get_next_val(date_type, date_info) # 下月, 作分区右边界(不含)
# 1) 建当月分区(交换时需 detach 它, 首次跑先建空的)；2) 建 copy 表(形如分区子表, 含索引)并清空
sql = f"""
create table if not exists {export_table_original} partition of {base_tb} for values from ('{date_info}') to ('{next_val}');
create table if not exists {export_tb_copy} (like {export_table_original} including all);
truncate table {export_tb_copy};
"""
DBUtil.engine_exec_sql(engine, sql)
# 导出列 = join键2 + level + 28 变化率列 + 3 分区列（列名与 Hive dwt_bs_top100_change_rate 一致）
export_cols = [
"asin_bs_cate_current_id", "asin_bs_cate_parent_id_join", "level",
"asin_amazon_orders_sum_ym_rate", "asin_amazon_orders_sum_lm_rate",
"asin_bsr_orders_sum_ym_rate", "asin_bsr_orders_sum_lm_rate",
"asin_bsr_orders_mean_ym_rate", "asin_bsr_orders_mean_lm_rate",
"asin_bsr_orders_sum_new_ym_rate", "asin_bsr_orders_sum_new_lm_rate",
"asin_bsr_orders_sum_brand_ym_rate", "asin_bsr_orders_sum_brand_lm_rate",
"asin_count_ym_rate", "asin_count_lm_rate",
"asin_count_new_ym_rate", "asin_count_new_lm_rate",
"asin_count_brand_ym_rate", "asin_count_brand_lm_rate",
"asin_bs_cate_1_rank_mean_ym_rate", "asin_bs_cate_1_rank_mean_lm_rate",
"asin_price_mean_ym_rate", "asin_price_mean_lm_rate",
"asin_rating_mean_ym_rate", "asin_rating_mean_lm_rate",
"asin_total_comments_mean_ym_rate", "asin_total_comments_mean_lm_rate",
"asin_ocean_freight_gross_margin_avg_ym_rate", "asin_ocean_freight_gross_margin_avg_lm_rate",
"asin_air_freight_gross_margin_avg_ym_rate", "asin_air_freight_gross_margin_avg_lm_rate",
# 分区列（也落成普通列, 取分区值）
"site_name", "date_type", "date_info",
]
sh = CommonUtil.build_export_sh(
site_name=site_name,
db_type=db_type,
hive_tb="dwt_bs_top100_change_rate",
export_tb=export_tb_copy,
col=export_cols,
partition_dict={"site_name": site_name, "date_type": date_type, "date_info": date_info},
)
client = SSHUtil.get_ssh_client()
SSHUtil.exec_command_async(client, sh, ignore_err=False)
client.close()
# 分区交换：把灌好的 copy 表原子换入当月分区（旧分区在函数内转到 copy 名下），避免 truncate 的空窗期。
# 分区表主键/索引由父表继承, cp_index_flag=False 不额外建。
DBUtil.exchange_pg_part_tb(
engine,
source_tb_name=export_tb_copy,
part_master_tb=base_tb,
part_target_tb=export_table_original,
cp_index_flag=False,
part_val={"from": [date_info], "to": [next_val]},
)
# 交换后 copy 表里是旧数据, 删掉省空间（同 aba）
DBUtil.engine_exec_sql(engine, f"drop table if exists {export_tb_copy};")
# 关闭链接
engine.dispose()
# 导出完成状态更新：主表 + 同比环比表两个导出脚本共用同一 belonging_to_process(品类调研流程_{date_type})。
# modify_export_workflow_status 会把本脚本置完成(status=3)、统计同组未完成数——只有最后一个跑完的
# (未完成数=0)才执行下面的 update_workflow_sql，即"两个都导完，页面才展示"(机制同 dwt_aba_last_change_rate.py)。
# 两脚本传同一条 update SQL，谁最后完成都能正确触发。本地/test 运行时该函数自动跳过。
# 本功能不像 aba 会在流程启动时预写一条 workflow_everyday，故用 replace INTO 直接插入/覆盖该行(不存在则建、存在则替换)。
# 只有主表+同比环比两个导出都完成时，modify_export_workflow_status 才会执行这条(即页面此时才出现"完成"行)。
update_workflow_sql = f"""
replace INTO selection.workflow_everyday
(site_name, report_date, status, status_val, table_name, date_type, page, is_end, remark, export_db_type)
VALUES('{site_name}', '{date_info}', '导出PG数据库完成', 14, '{site_name}_category_top_analysis', '{date_type}', '品类调研', '是', '品类调研月表', '{db_type}');
"""
CommonUtil.modify_export_workflow_status(update_workflow_sql, site_name, date_type, date_info)
print("success")
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment