dwd_st_volume_fba.py

import os
import sys
from datetime import datetime

sys.path.append(os.path.dirname(sys.path[0]))
from utils.ssh_util import SSHUtil
from utils.common_util import CommonUtil, DateTypes
from utils.db_util import DBUtil

if __name__ == '__main__':
    site_name = CommonUtil.get_sys_arg(1, None)
    date_type = CommonUtil.get_sys_arg(2, None)
    date_info = CommonUtil.get_sys_arg(3, None)
    #  获取最后一个参数
    test_flag = CommonUtil.get_sys_arg(len(sys.argv) - 1, None)
    print(f"执行参数为{sys.argv}")

    if test_flag == 'test':
        db_type = 'postgresql_test'
        print("导出到测试库中")
    else:
        CommonUtil.judge_is_work_hours(site_name=site_name, date_type=date_type, date_info=date_info,
                                       principal='wujicang', priority=2, export_tools_type=1, belonging_to_process=f'新ABA流程_{date_type}')
        if date_type in (DateTypes.month.name, DateTypes.week.name, DateTypes.month_week.name):
            db_type = 'postgresql_cluster'
            print("导出到PG-Cluster库中")
        else:
            db_type = "postgresql"
            print("导出到PG库中")

    export_master_tb = f"{site_name}_aba_profit_gross"
    suffix = str(date_info).replace("-", "_")
    engine = DBUtil.get_db_engine(db_type, site_name)
    #  30天表进行备份
    if date_type == DateTypes.last30day.name or date_type == DateTypes.month_week.name:
        export_tb_before = f"{site_name}_aba_profit_gross_last30day"
        export_tb_rel = f"{export_tb_before}_copy"
        export_tb = export_tb_rel
        with engine.connect() as connection:
            sql = f"""
                    drop table if  exists {export_tb_rel};
                    create table if not exists {export_tb_rel} 
                    (
                        like {export_tb_before}  including comments
                    );
                    """
            print("================================执行sql================================")
            print(sql)
            connection.execute(sql)


    else:
        export_tb = f"{site_name}_aba_profit_gross_{date_type}_{suffix}"
        next_val = CommonUtil.get_next_val(date_type, date_info)
        with engine.connect() as connection:
            sql = f"""
                    create table if not exists {export_tb} partition of {export_master_tb} for values from ('{date_type}', '{date_info}') to ('{date_type}', '{next_val}');
                    truncate table {export_tb};
                    """
            print("================================执行sql================================")
            print(sql)
            connection.execute(sql)
    if test_flag == "month_append":
        partition_dict = {
            "site_name": site_name,
            "date_type": "month",
            "date_info": date_info
        }
    else:
        partition_dict = {
            "site_name": site_name,
            "date_type": date_type,
            "date_info": date_info
        }


    # 导出表名
    sh = CommonUtil.build_export_sh(
        site_name=site_name,
        db_type=db_type,
        hive_tb="dwd_st_volume_fba",
        export_tb=export_tb,
        col=[
            "search_term_id",
            "search_term",
            "fba_fee",
            "ocean_freight",
            "air_delivery_fee",
            "referral_fee",
            "return_ratio",
            "operating_costs",
            "costs",
            "advertise",
            "gross_profit_fee_air",
            "date_type",
            "date_info",
            "gross_profit_fee_sea",
            "price",
            "category_id",
            "longs",
            "width",
            "high",
            "weight"
        ],
        partition_dict=partition_dict
    )

    client = SSHUtil.get_ssh_client()
    SSHUtil.exec_command_async(client, sh, ignore_err=False)
    client.close()

    if date_type in (DateTypes.month_week.name, DateTypes.last30day.name):
        #  构建索引并交换表名
        DBUtil.exchange_tb(engine,
                           source_tb_name=export_tb_rel,
                           target_tb_name=export_tb_before,
                           cp_index_flag=True)
        update_workflow_sql = f"""
                    replace INTO selection.workflow_everyday
                    (site_name, report_date, status, status_val, table_name, date_type, page, is_end, remark,export_db_type)
                    VALUES('{site_name}', '{datetime.now().date()}', '导出PG数据库完成', 14, 'us_aba_last_30_day', '30_day', 'ABA搜索词', '是', 'ABA搜索词最近30天表','{db_type}');
                """
    else:
        update_workflow_sql = f"""
                UPDATE selection.workflow_everyday SET status='导出PG数据库完成', status_val=14,is_end ='是',export_db_type = '{db_type}'  
                WHERE site_name= '{site_name}' and date_type='{date_type}' and report_date= '{date_info}'  and page ='ABA搜索词'
            """

    # 往导出流程表插入导出完成数据，方便监听导出脚本是否全部完成
    CommonUtil.modify_export_workflow_status(update_workflow_sql, site_name, date_type, date_info)

    print("success")