#! /bin/env bash # author: huangjian # user_no: wg4235 # describe: 店铺FeedBack指标统计 # params: 参数1:site_name;参数2:date_type;参数3:date_info; # version: 1.1 # create_date: 2022-3-22 # update_date: 2022-11-02 if [ $1 == all ]; then site_name_array=(us uk de es fr it) else site_name_array=($1) fi date_type=$2 date_info=$3 d1=`echo $date_info | cut -d \- -f 1` d2=`echo $date_info | cut -d \- -f 2` flag=1 for site_name in ${site_name_array[*]} do echo "site_name: ${site_name}","date_type: ${date_type}","date_info: ${date_info}" hdfs dfs -rm -r /home/big_data_selection/dwd/dwd_seller_asin_account_agg/site_name=${site_name}/date_type=${date_type}/date_info=${date_info}/* hdfs dfs -rm -r /home/big_data_selection/dwd/dwd_seller_asin_account_detail/site_name=${site_name}/date_type=${date_type}/date_info=${date_info}/* # spark-yarn提交指令 /opt/module/spark/bin/spark-submit \ --master yarn \ --driver-memory 2g \ --executor-memory 10g \ --executor-cores 4 \ --num-executors 20 \ --queue spark \ /mnt/Amazon-Selection/Pyspark_job/dwd/dwd_feedback.py ${site_name} ${date_type} ${date_info} done