export_st_info.sh

#! /bin/env bash
# author: ffman
# user_no: wg3491
# describe: dwd_st_info-->站点_brand_st_info
# params: $1-->site_name_array; $2-->year; $3-->week
# version: 2.0
# create_date: 2022-3-22
# update_date: 2022-5-18


# sqoop执行变量
sqoop=/opt/module/sqoop/bin/sqoop
# 定义脚本传入的变量
year=$2
week=$3
dt=${year}-${week}

# 根据site_name来判断数据库名称
if [ $1 == all ];
then
    site_name_array=(us uk de es fr it)
else
    site_name_array=($1)
fi
echo "site_name_array: ${site_name_array}"


export_data () {
    $sqoop export -D mapred.job.queue.name=spark \
    --connect "jdbc:mysql://rm-wz9yg9bsb2zf01ea4yo.mysql.rds.aliyuncs.com:3306/${db}?useUnicode=true&characterEncoding=utf-8" \
    --username adv_yswg \
    --password HmRCMUjt03M33Lze \
    --table ${mysql_table} \
    --input-fields-terminated-by '\001' \
    --hcatalog-database big_data_selection \
    --hcatalog-table ${hive_table} \
    --hcatalog-partition-keys site_name,dt \
    --hcatalog-partition-values ${site_name},${dt} \
    --input-null-string '\\N' \
    --input-null-non-string '\\N' \
    --columns ${cols} \
    --num-mappers 3
}


for site_name in ${site_name_array[*]}
    do
        if [ $site_name == us ];
        then
            db=selection
        else
            db=selection_$site_name
        fi
        echo "db: ${db}"
        mysql_table=${site_name}_brand_st_info
        hive_table=dwd_st_info
        cols="st_brand_id,st_is_first_text,st_ao_val,week,year"
        echo "当前导出的hive表:${hive_table}, mysql表: ${mysql_table}"
        export_data
    done