dwd_day_asin.py

import os
import sys

sys.path.append(os.path.dirname(sys.path[0]))
from utils.ssh_util import SSHUtil
from utils.common_util import CommonUtil
from utils.db_util import DBUtil

if __name__ == '__main__':
    site_name = CommonUtil.get_sys_arg(1, None)
    date_info = CommonUtil.get_sys_arg(2, None)
    #  获取最后一个参数
    test_flag = CommonUtil.get_sys_arg(len(sys.argv) - 1, None)
    print(f"执行参数为{sys.argv}")

    db_type = "mysql"
    print("导出到mysql库中")

    export_tb = f"bsr_day_asin"
    engine = DBUtil.get_db_engine(db_type, site_name)

    with engine.connect() as connection:
        sql = f""" select * from {export_tb} where site_name ='{site_name}' and date_info ='{date_info}' limit 1;"""
        print("================================执行sql================================")
        print(sql)
        size = len(list(connection.execute(sql)))
        if size != 0:
            print(f"当天数据{date_info}已经导出，请勿重复导出！")
            sys.exit(0)

    # # 导出表名
    sh = CommonUtil.build_export_sh(
        site_name=site_name,
        db_type=db_type,
        hive_tb="dwd_day_asin",
        export_tb=export_tb,
        col=[
            "asin",
            "queue_name",
            "site_name",
            "date_info",
        ],
        partition_dict={
            "site_name": site_name,
            "date_info": date_info
        }
    )

    client = SSHUtil.get_ssh_client()
    SSHUtil.exec_command_async(client, sh, ignore_err=False)
    client.close()
    #  导出完毕后删除之前日的数据
    with engine.connect() as connection:
        sql = f""" delete from {export_tb} where site_name ='{site_name}' and date_info < '{date_info}' """
        print("================================删除之前数据中================================")
        connection.execute(sql)
    print("success")