1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
#! /bin/env bash
# author: huangjian
# user_no: wg4235
# describe: 店铺FeedBack指标统计
# params: 参数1:site_name;参数2:date_type;参数3:date_info;
# version: 1.1
# create_date: 2022-3-22
# update_date: 2022-11-02
if [ $1 == all ];
then
site_name_array=(us uk de es fr it)
else
site_name_array=($1)
fi
date_type=$2
date_info=$3
d1=`echo $date_info | cut -d \- -f 1`
d2=`echo $date_info | cut -d \- -f 2`
flag=1
for site_name in ${site_name_array[*]}
do
echo "site_name: ${site_name}","date_type: ${date_type}","date_info: ${date_info}"
hdfs dfs -rm -r /home/big_data_selection/dwd/dwd_seller_asin_account_agg/site_name=${site_name}/date_type=${date_type}/date_info=${date_info}/*
hdfs dfs -rm -r /home/big_data_selection/dwd/dwd_seller_asin_account_detail/site_name=${site_name}/date_type=${date_type}/date_info=${date_info}/*
# spark-yarn提交指令
/opt/module/spark/bin/spark-submit \
--master yarn \
--driver-memory 2g \
--executor-memory 10g \
--executor-cores 4 \
--num-executors 20 \
--queue spark \
/mnt/Amazon-Selection/Pyspark_job/dwd/dwd_feedback.py ${site_name} ${date_type} ${date_info}
done