1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
#! /bin/env bash
source /mnt/run_shell/sqoop_shell/templates.sh
# author: huangjian
# usage: 同步各个站点的asin款式描述表
# params: 参数1:site_name;
# version: 1.0
# create_date:2022-11-09
# update_date:2022-11-09
# 根据site_name来判断数据库名称
if [ $1 == all ];
then
site_name_array=(us uk de es fr it)
else
site_name_array=($1)
fi
echo "site_name_array: ${site_name_array}"
for site_name in ${site_name_array[*]}
do
echo 1. 当前连接的mysql数据库站点: ${site_name}
if [ $site_name == us ];
then
db=selection
else
db=selection_$site_name
fi
echo "db: ${db}"
echo 2. 先删除已经存在的分区
hive_table=ods_asin_variat
hdfs_path="/home/big_data_selection/ods/${hive_table}/site_name=${site_name}"
hdfs dfs -rm -r ${hdfs_path}/*
echo 3. 导入数据
mysql_table=${site_name}_variat
cols="id,asin,parent_asin,color,size,style,state,column_2,column_1,created_at,updated_at"
query="select ${cols} from ${mysql_table} where 1=1 and \$CONDITIONS"
maps=1
queue=spark
import_data
done