dwd_all_syn_asin.py

import os
import sys

sys.path.append(os.path.dirname(sys.path[0]))

from utils.db_util import DBUtil
from utils.ssh_util import SSHUtil
from utils.common_util import CommonUtil

if __name__ == '__main__':
    site_name = CommonUtil.get_sys_arg(1, None)
    if not (site_name in ['uk', 'de']):
        print("暂不计算该维度数据！")
        quit()

    CommonUtil.judge_is_work_hours(site_name=site_name, principal='chenyuanjie', priority=3, export_tools_type=1)

    db_type = "postgresql"
    print("导出到PG数据库")
    master_tb = f"{site_name}_all_syn_asin"
    export_tb = f"{site_name}_all_syn_asin_copy"

    # 获取数据库连接
    engine = DBUtil.get_db_engine(db_type, site_name)

    with engine.connect() as connection:
        sql = f"""
            drop table if exists {export_tb};
            CREATE TABLE IF NOT EXISTS {export_tb}
            AS
            SELECT *
            FROM {master_tb}
            WITH NO DATA;
        """
        print("================================执行sql================================")
        print(sql)
        connection.execute(sql)

    # 导出脚本
    sh = CommonUtil.build_export_sh(
        site_name=site_name,
        db_type=db_type,
        hive_tb="dwd_all_syn_asin",
        export_tb=export_tb,
        col=[
            "asin",
            "time_list"
        ],
        partition_dict={
            "site_name": site_name
        }
    )

    client = SSHUtil.get_ssh_client()
    SSHUtil.exec_command_async(client, sh, ignore_err=False)
    client.close()

    print("导出完成，准备创建索引和交换表名")
    DBUtil.exchange_tb(engine, export_tb, master_tb, True)

    engine.dispose()
    print("success")

pass