dwt_asin_title_history_all.py
1.13 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
import os
import sys
sys.path.append(os.path.dirname(sys.path[0]))
from utils.common_util import CommonUtil
from utils.ssh_util import SSHUtil
"""
标题历史数据脚本
"""
if __name__ == '__main__':
batchMap = {
"month": 4,
"week": 4,
"day": 4,
"month_week": 4,
}
# 计算 uk de 全表数据 提交的脚本
for site in ['uk', 'de']:
for date_type in ['week', 'month', 'day']:
for i in range(0, batchMap.get(date_type)):
print("提交中.....")
cmd = f"""
/opt/module/spark/bin/spark-submit \\
--master yarn \\
--driver-memory 4g \\
--executor-memory 20g \\
--executor-cores 4 \\
--num-executors 20 \\
--queue spark \\
/tmp/wjc_py/dwt/dwt_asin_title_history.py {site} all {date_type}
"""
client = SSHUtil.get_ssh_client()
SSHUtil.exec_command_async(client, cmd, ignore_err=False)
client.close()
print("执行成功！！！")
pass
pass
pass