import os import sys sys.path.append(os.path.dirname(sys.path[0])) from utils.ssh_util import SSHUtil from utils.common_util import CommonUtil from utils.hdfs_utils import HdfsUtils from utils.db_util import DbTypes if __name__ == '__main__': site_name = CommonUtil.get_sys_arg(1, None) assert site_name is not None, "site_name 不能为空!" import_tb = f"{site_name}_all_syn_st_asin" query = f""" select asin, state, updated_at from {import_tb} where state = 4 and \$CONDITIONS """ hive_tb = "ods_asin_err_state" partition_dict = { "site_name": site_name } hdfs_path = CommonUtil.build_hdfs_path(hive_tb, partition_dict=partition_dict) print(f"hdfs_path is {hdfs_path}") db_type = DbTypes.postgresql.name empty_flag, check_flag = CommonUtil.check_schema_before_import(db_type=db_type, site_name=site_name, query=query, hive_tb_name=hive_tb, msg_usr=['wujicang'] ) assert check_flag, f"导入hive表{hive_tb}表结构检查失败!请检查query是否异常!!" if not empty_flag: sh = CommonUtil.build_import_sh(site_name=site_name, db_type=db_type, query=query, hdfs_path=hdfs_path) # 导入前先删除 HdfsUtils.delete_hdfs_file(hdfs_path) client = SSHUtil.get_ssh_client() SSHUtil.exec_command_async(client, sh, ignore_err=False) CommonUtil.after_import(hdfs_path=hdfs_path, hive_tb=hive_tb) client.close() pass