""" @Author : HuangJian @Description : 各站点店铺名称与店铺id关系全量表--传参为单站点 @SourceTable : us_seller_account_feedback @SinkTable : ods_seller_account_feedback @CreateTime : 2022/05/19 14:55 @UpdateTime : 2022/05/19 14:55 """ import os import sys sys.path.append(os.path.dirname(sys.path[0])) from utils.ssh_util import SSHUtil from utils.db_util import DBUtil from utils.common_util import CommonUtil from utils.common_util import DateTypes from utils.hdfs_utils import HdfsUtils from utils.spark_util import SparkUtil if __name__ == '__main__': site_name = CommonUtil.get_sys_arg(1, None) assert site_name is not None, "site_name 不能为空!" hive_table = f"ods_seller_account_syn" partition_dict = { "site_name": site_name } # 落表路径校验 hdfs_path = CommonUtil.build_hdfs_path(hive_table, partition_dict=partition_dict) print(f"hdfs_path is {hdfs_path}") import_table = f"{site_name}_seller_account_syn_distinct" db_type = 'mysql' sql_query = f""" select id, account_name, url, state, created_at, updated_at, seller_id from {import_table} where 1=1 and \$CONDITIONS """ # 进行schema和数据校验 CommonUtil.check_schema_before_import(db_type=db_type, site_name=site_name, query=sql_query, hive_tb_name=hive_table, msg_usr=['chenyuanjie']) # 生成导出脚本 import_sh = CommonUtil.build_import_sh(site_name=site_name, db_type=db_type, query=sql_query, hdfs_path=hdfs_path) # 导入前先删除原始hdfs数据 HdfsUtils.delete_hdfs_file(hdfs_path) # 创建ssh Client对象--用于执行cmd命令 client = SSHUtil.get_ssh_client() SSHUtil.exec_command_async(client, import_sh, ignore_err=False) # 创建lzo索引和修复元数据 CommonUtil.after_import(hdfs_path=hdfs_path, hive_tb=hive_table) # 关闭链接 client.close()