1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
#! /bin/env bash
# author: ffman
# usage: 同步brand_analytics表的周，月，季度数据
# params: $1-->site_name_array;$2-->date_type;$3-->year;$4-->week
# version: 1.0
# create_date:2022-6-15
# update_date:2022-6-15
date_type=$2
#year=$3
#week=$4
date_info=$3
# 根据site_name来判断数据库名称
if [ $1 == all ];
then
site_name_array=(us uk de es fr it)
else
site_name_array=($1)
fi
echo "site_name_array: ${site_name_array}"
for site_name in ${site_name_array[*]}
do
echo 1. 当前连接的mysql数据库站点: ${site_name}
echo "site_name: ${site_name}, ${date_type}, ${year}, ${week}, 同步数据"
# 同步数据
/opt/module/spark/bin/spark-submit \
--master local[*] \
--driver-memory 40g \
/opt/module/spark/demo/py_demo/temp/es_brand_analytics.py ${site_name} ${date_type} ${date_info}
done
#--driver-memory 10g \
#--executor-memory 10g \
#--executor-cores 3 \
#--num-executors 10 \
#--queue spark \