1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
from utils.ssh_util import SSHUtil
# aba 365重新计算脚本
if __name__ == '__main__':
months = [
'2022-12',
'2023-02',
]
client = SSHUtil.get_ssh_client()
for month in months:
sh = f"""
/opt/module/spark/bin/spark-submit \\
--master yarn \\
--driver-memory 2g \\
--executor-memory 10g \\
--executor-cores 3 \\
--num-executors 10 \\
--queue spark \\
/opt/module/spark/demo/py_demo/dwt/dwt_aba_last365.py us month_old {month}
"""
SSHUtil.exec_command_async(client, sh, ignore_err=False)
pass
last_month = max(months)
sh = f"""
/opt/module/anaconda3/envs/pyspark/bin/python3.8 /opt/module/spark/demo/py_demo/sqoop_py/dwt_aba_last365.py us last365day {last_month}
"""
SSHUtil.exec_command_async(client, sh, ignore_err=False)
years = [
'2022'
]
for year in years:
sh = f"""
/opt/module/anaconda3/envs/pyspark/bin/python3.8 /opt/module/spark/demo/py_demo/sqoop_py/dwt_aba_last365.py us year {year}
"""
SSHUtil.exec_command_async(client, sh, ignore_err=False)
pass
client.close()