dwt_asin_title_history_all.py

import os
import sys

sys.path.append(os.path.dirname(sys.path[0]))
from utils.common_util import CommonUtil
from utils.ssh_util import SSHUtil

"""
标题历史数据脚本
"""

if __name__ == '__main__':
    batchMap = {
        "month": 4,
        "week": 4,
        "day": 4,
        "month_week": 4,
    }
    # 计算 uk de 全表数据 提交的脚本
    for site in ['uk', 'de']:
        for date_type in ['week', 'month', 'day']:
            for i in range(0, batchMap.get(date_type)):
                print("提交中.....")
                cmd = f"""
            /opt/module/spark/bin/spark-submit  \\
            --master yarn \\
            --driver-memory 4g \\
            --executor-memory 20g \\
            --executor-cores 4 \\
            --num-executors 20 \\
            --queue spark \\
            /tmp/wjc_py/dwt/dwt_asin_title_history.py {site} all {date_type}
            """
                client = SSHUtil.get_ssh_client()
                SSHUtil.exec_command_async(client, cmd, ignore_err=False)
                client.close()
                print("执行成功！！！")
                pass
            pass
        pass