1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
#! /bin/env bash
# author: ffman
# user_no: wg3491
# describe: dwd_st_info-->站点_brand_st_info
# params: $1-->site_name_array; $2-->year; $3-->week
# version: 2.0
# create_date: 2022-3-22
# update_date: 2022-5-18
# sqoop执行变量
sqoop=/opt/module/sqoop/bin/sqoop
# 定义脚本传入的变量
year=$2
week=$3
dt=${year}-${week}
# 根据site_name来判断数据库名称
if [ $1 == all ];
then
site_name_array=(us uk de es fr it)
else
site_name_array=($1)
fi
echo "site_name_array: ${site_name_array}"
export_data () {
$sqoop export -D mapred.job.queue.name=spark \
--connect "jdbc:mysql://rm-wz9yg9bsb2zf01ea4yo.mysql.rds.aliyuncs.com:3306/${db}?useUnicode=true&characterEncoding=utf-8" \
--username adv_yswg \
--password HmRCMUjt03M33Lze \
--table ${mysql_table} \
--input-fields-terminated-by '\001' \
--hcatalog-database big_data_selection \
--hcatalog-table ${hive_table} \
--hcatalog-partition-keys site_name,dt \
--hcatalog-partition-values ${site_name},${dt} \
--input-null-string '\\N' \
--input-null-non-string '\\N' \
--columns ${cols} \
--num-mappers 3
}
for site_name in ${site_name_array[*]}
do
if [ $site_name == us ];
then
db=selection
else
db=selection_$site_name
fi
echo "db: ${db}"
mysql_table=${site_name}_brand_st_info
hive_table=dwd_st_info
cols="st_brand_id,st_is_first_text,st_ao_val,week,year"
echo "当前导出的hive表:${hive_table}, mysql表: ${mysql_table}"
export_data
done