Skip to content

A
Amazon-Selection-Data
Commit 7539ab9a by chenyuanjie
Options

新增etsy店铺计算流程

parent b36b2ca1
Showing with 372 additions and 27 deletions
Pyspark_job/dim/dim_ehunt_store_detail.py 0 → 100644
"""
@Author : CT
@Description : Etsy店铺详情清洗：store_name去重留最新，start_at/is_star/is_raving/country格式统一
@SourceTable : ods_ehunt_store_detail
@SinkTable : dim_ehunt_store_detail
@CreateTime : 2026/11/02
@UpdateTime : 2026/11/02
"""
import os
import sys
from itertools import chain
sys.path.append(os.path.dirname(sys.path[0])) # 上级目录
from utils.templates import Templates
from pyspark.sql.window import Window
from pyspark.sql import functions as F
from utils.common_util import CommonUtil
from utils.hdfs_utils import HdfsUtils
# 国家统一为中文，key为小写去空格后的原值；未命中的保留原值
COUNTRY_MAP = {
# 英文 -> 中文
"united states": "美国", "british": "英国", "united kingdom": "英国", "germany": "德国",
"canada": "加拿大", "india": "印度", "australia": "澳大利亚", "pakistan": "巴基斯坦",
"turkey": "土耳其", "french": "法国", "france": "法国", "ukraine": "乌克兰", "spain": "西班牙",
"thailand": "泰国", "indonesia": "印度尼西亚", "philippines": "菲律宾", "netherlands": "荷兰",
"italy": "意大利", "vietnam": "越南", "hong kong": "中国香港特区", "morocco": "摩洛哥",
"poland": "波兰", "china": "中国", "latvia": "拉脱维亚", "lithuania": "立陶宛", "israel": "以色列",
"portugal": "葡萄牙", "japan": "日本", "bulgaria": "保加利亚", "malaysia": "马来西亚",
"ireland": "爱尔兰", "romania": "罗马尼亚", "sweden": "瑞典", "new zealand": "新西兰",
"austria": "奥地利", "mexico": "墨西哥", "south korea": "韩国", "singapore": "新加坡",
"south africa": "南非", "switzerland": "瑞士", "argentina": "阿根廷", "hungary": "匈牙利",
"brazil": "巴西", "finland": "芬兰", "estonia": "爱沙尼亚", "united arab emirates": "阿联酋",
"belgium": "比利时", "greece": "希腊", "slovak republic": "斯洛伐克", "czech republic": "捷克",
"chile": "智利", "kosovo": "科索沃", "slovenia": "斯洛文尼亚", "norway": "挪威",
"croatia": "克罗地亚", "georgia": "格鲁吉亚", "denmark": "丹麦", "cyprus": "塞浦路斯",
"kazakhstan": "哈萨克斯坦", "peru": "秘鲁", "egypt": "埃及", "luxembourg": "卢森堡",
"nepal": "尼泊尔", "malta": "马耳他", "sri lanka": "斯里兰卡", "taiwan": "中国台湾省",
"armenia": "亚美尼亚", "puerto rico": "波多黎各", "ghana": "加纳", "moldova": "摩尔多瓦",
"u.s. virgin islands": "美属维尔京群岛", "saudi arabia": "沙特阿拉伯", "nigeria": "尼日利亚",
"kenya": "肯尼亚", "venezuela": "委内瑞拉", "colombia": "哥伦比亚", "russia": "俄罗斯",
"costa rica": "哥斯达黎加", "guatemala": "危地马拉", "iceland": "冰岛",
"dominican republic": "多米尼加共和国", "ecuador": "厄瓜多尔", "lebanon": "黎巴嫩",
"vzbekistan": "乌兹别克斯坦", "uzbekistan": "乌兹别克斯坦", "albania": "阿尔巴尼亚",
"jordan": "约旦", "belarus": "白俄罗斯", "monaco": "摩纳哥", "tanzania": "坦桑尼亚",
"kyrgyzstan": "吉尔吉斯斯坦", "uruguay": "乌拉圭", "algeria": "阿尔及利亚", "antarctica": "南极洲",
"belize": "伯利兹", "ethiopia": "埃塞俄比亚", "haiti": "海地", "kuwait": "科威特",
"macedonia": "北马其顿", "nicaragua": "尼加拉瓜", "panama": "巴拿马", "tunisia": "突尼斯",
"niue": "纽埃", "bolivia": "玻利维亚", "montenegro": "黑山", "barbados": "巴巴多斯",
"bahrain": "巴林", "honduras": "洪都拉斯", "azerbaijan": "阿塞拜疆", "turkmenistan": "土库曼斯坦",
"qatar": "卡塔尔", "afghanistan": "阿富汗", "macao": "中国澳门特区", "macau": "中国澳门特区",
"oman": "阿曼", "jamaica": "牙买加", "guyana": "圭亚那", "guam": "关岛", "faroe islands": "法罗群岛",
"cayman islands": "开曼群岛", "paraguay": "巴拉圭", "bosnia and herzegovina": "波黑",
"bangladesh": "孟加拉", "el salvador": "萨尔瓦多", "cameroon": "喀麦隆", "botswana": "博茨瓦纳",
"bermuda": "百慕大", "bahamas": "巴哈马", "andorra": "安道尔", "zambia": "赞比亚",
"uganda": "乌干达", "togo": "多哥", "sudan": "苏丹", "senegal": "塞内加尔", "samoa": "萨摩亚",
"palestinian territory": "巴勒斯坦", "netherlands antilles": "荷属安的列斯", "namibia": "纳米比亚",
"myanmar": "缅甸", "mongolia": "蒙古", "marshall islands": "马绍尔群岛", "maldives": "马尔代夫",
"liechtenstein": "列支敦士登", "laos": "老挝", "greenland": "格陵兰",
# 中文非标准写法 -> 标准写法
"拉托维亚": "拉脱维亚", "斯罗文尼亚": "斯洛文尼亚", "塞普路斯": "塞浦路斯",
"科索沃,塞尔维亚": "科索沃", "法属尼留旺岛": "留尼汪",
}
YES_VALUES = ['yes', 'y', 'true', '1', '是']
NO_VALUES = ['no', 'n', 'false', '0', '否']
class DimEhuntStoreDetail(Templates):
def __init__(self, site_name="us", date_type="month", date_info="2026-10"):
super().__init__()
self.site_name = site_name
self.date_type = date_type
self.date_info = date_info
self.hive_tb = "dim_ehunt_store_detail"
self.partition_dict = {
"site_name": site_name,
"date_type": date_type,
"date_info": date_info
}
self.hdfs_path = CommonUtil.build_hdfs_path(self.hive_tb, partition_dict=self.partition_dict)
self.spark = self.create_spark_object(
app_name=f"{self.hive_tb}: {self.site_name}, {self.date_type}, {self.date_info}")
# 写入、分区初始化
self.df_save = self.spark.sql(f"select 1+1;")
self.partitions_by = ['site_name', 'date_type', 'date_info']
self.reset_partitions(10)
# 初始化全局df
self.df_store = self.spark.sql(f"select 1+1;")
def read_data(self):
sql = f"""
select store_name, category, country, sales, sales_weekly, revenue_total, revenue_7days, reviews,
reviews_weekly, favorites, favorites_weekly, products, start_at, is_star, is_raving, rating, store_url,
social_info, pay_type, created_at
from ods_ehunt_store_detail
where site_name = '{self.site_name}' and date_type = '{self.date_type}' and date_info = '{self.date_info}'
"""
print(sql)
self.df_store = self.spark.sql(sqlQuery=sql).cache()
print("ods数据量为：", self.df_store.count())
@staticmethod
def handle_yes_no(col_name):
val = F.lower(F.trim(F.col(col_name)))
return F.when(val.isin(YES_VALUES), F.lit("是")).when(val.isin(NO_VALUES), F.lit("否")).otherwise(F.lit(None))
# store_name去重，按入库时间取最新一条
def handle_store_distinct(self):
window = Window.partitionBy('store_name').orderBy(F.col('created_at').desc_nulls_last())
self.df_store = self.df_store.filter(
F.col('store_name').isNotNull() & (F.trim(F.col('store_name')) != '')
).withColumn(
'rk', F.row_number().over(window)
).filter(F.col('rk') == 1).drop('rk')
# 开店时间统一为 yyyy-MM，无法解析的置空
def handle_start_at(self):
year_col = F.regexp_extract(F.col('start_at'), r'^\s*(\d{4})[-/.](\d{1,2})', 1)
month_col = F.regexp_extract(F.col('start_at'), r'^\s*(\d{4})[-/.](\d{1,2})', 2)
self.df_store = self.df_store.withColumn(
'start_at',
F.when(
(year_col != '') & month_col.cast('int').between(1, 12),
F.concat_ws('-', year_col, F.lpad(month_col, 2, '0'))
).otherwise(F.lit(None))
)
# 明星卖家/Raving标签统一为 是/否
def handle_flag(self):
self.df_store = self.df_store.withColumn(
'is_star', self.handle_yes_no('is_star')
).withColumn(
'is_raving', self.handle_yes_no('is_raving')
)
# 国家统一为中文
def handle_country(self):
country_map = F.create_map([F.lit(x) for x in chain(*COUNTRY_MAP.items())])
country_key = F.lower(F.trim(F.col('country')))
self.df_store = self.df_store.withColumn(
'country',
F.when(
country_key.isNull() | country_key.isin('', '[]'), F.lit(None)
).otherwise(F.coalesce(country_map[country_key], F.trim(F.col('country'))))
)
def handle_column(self):
self.df_save = self.df_store.select(
'store_name', 'category', 'country', 'sales', 'sales_weekly', 'revenue_total', 'revenue_7days',
'reviews', 'reviews_weekly', 'favorites', 'favorites_weekly', 'products', 'start_at', 'is_star',
'is_raving', 'rating', 'store_url', 'social_info', 'pay_type', 'created_at',
F.lit(self.site_name).alias('site_name'),
F.lit(self.date_type).alias('date_type'),
F.lit(self.date_info).alias('date_info')
).repartition(self.partitions_num)
def handle_data(self):
self.handle_store_distinct()
self.handle_start_at()
self.handle_flag()
self.handle_country()
self.handle_column()
def save_data(self):
print(f"清除hdfs目录中:{self.hdfs_path}")
HdfsUtils.delete_file_in_folder(self.hdfs_path)
print(f"当前存储的表名为：{self.hive_tb},分区为{self.partitions_by}")
self.df_save.write.saveAsTable(name=self.hive_tb, format='hive', mode='append', partitionBy=self.partitions_by)
print("save success")
if __name__ == '__main__':
site_name = sys.argv[1] # 参数1：站点
date_type = sys.argv[2] # 参数2：类型：month
date_info = sys.argv[3] # 参数3：年-月, 比如: 2026-10
assert date_type == 'month', "date_type 只支持 month"
handle_obj = DimEhuntStoreDetail(site_name=site_name, date_type=date_type, date_info=date_info)
handle_obj.run()
Pyspark_job/dwt/dwt_ehunt_store_detail.py 0 → 100644
"""
@Author : CT
@Description : Etsy店铺详情-月维度：计算近7日销量/评论/收藏的环比(_mom)、同比(_yoy)
@SourceTable : dim_ehunt_store_detail(本月、上月、去年同月)
@SinkTable : dwt_ehunt_store_detail
@CreateTime : 2026/11/02
@UpdateTime : 2026/11/02
"""
import os
import sys
sys.path.append(os.path.dirname(sys.path[0])) # 上级目录
from utils.templates import Templates
from pyspark.sql import functions as F
from pyspark.sql.types import DoubleType
from utils.common_util import CommonUtil
from utils.hdfs_utils import HdfsUtils
# 参与同环比计算的字段
CHANGE_COLS = ['sales_weekly', 'reviews_weekly', 'favorites_weekly']
class DwtEhuntStoreDetail(Templates):
def __init__(self, site_name="us", date_type="month", date_info="2026-10"):
super().__init__()
self.site_name = site_name
self.date_type = date_type
self.date_info = date_info
self.hive_tb = "dwt_ehunt_store_detail"
self.partition_dict = {
"site_name": site_name,
"date_type": date_type,
"date_info": date_info
}
self.hdfs_path = CommonUtil.build_hdfs_path(self.hive_tb, partition_dict=self.partition_dict)
self.spark = self.create_spark_object(
app_name=f"{self.hive_tb}: {self.site_name}, {self.date_type}, {self.date_info}")
# 上月、去年同月；对比月早于基准月的指标不计算
self.last_month = CommonUtil.get_month_offset(date_info, -1)
self.last_year_month = CommonUtil.get_month_offset(date_info, -12)
self.base_month = "2026-09"
self.can_mom = self.last_month >= self.base_month
self.can_yoy = self.last_year_month >= self.base_month
print(f"上月: {self.last_month}, 去年同月: {self.last_year_month}, 基准月: {self.base_month}, "
f"是否计算 mom/yoy: {self.can_mom}/{self.can_yoy}")
# 写入、分区初始化
self.df_save = self.spark.sql(f"select 1+1;")
self.partitions_by = ['site_name', 'date_type', 'date_info']
self.reset_partitions(10)
# 初始化全局df
self.df_store = self.spark.sql(f"select 1+1;")
self.df_store_last_month = self.spark.sql(f"select 1+1;")
self.df_store_last_year = self.spark.sql(f"select 1+1;")
@staticmethod
def calculate_change(current_col, previous_col):
curr = F.when(F.col(current_col) < 0, F.lit(None)).otherwise(F.col(current_col))
prev = F.when(F.col(previous_col) < 0, F.lit(None)).otherwise(F.col(previous_col))
return F.when(
curr.isNull() & prev.isNull(), F.lit(None)
).when(
curr.isNull() & (prev == 0), F.lit(0.0)
).when(
curr.isNull(), F.lit(-1000.0)
).when(
(prev.isNull() | (prev == 0)) & (curr == 0), F.lit(0.0)
).when(
prev.isNull() | (prev == 0), F.lit(1000.0)
).otherwise(
F.round((curr - prev) / prev, 4)
)
def read_history(self, month, prefix):
"""读取历史月份的对比字段，字段加前缀"""
sql = f"""
select store_name, {', '.join(CHANGE_COLS)}
from dim_ehunt_store_detail
where site_name = '{self.site_name}' and date_type = '{self.date_type}' and date_info = '{month}'
"""
print(sql)
df = self.spark.sql(sqlQuery=sql)
for col in CHANGE_COLS:
df = df.withColumnRenamed(col, f"{prefix}_{col}")
df = df.cache()
cnt = df.count()
print(f"{month} 数据量为：{cnt}")
assert cnt > 0, f"dim_ehunt_store_detail {month} 分区无数据，请先跑该月份！"
return df
def read_data(self):
sql = f"""
select store_name, category, country, sales, sales_weekly, revenue_total, revenue_7days, reviews,
reviews_weekly, favorites, favorites_weekly, products, start_at, is_star, is_raving, rating, store_url,
social_info, pay_type, created_at
from dim_ehunt_store_detail
where site_name = '{self.site_name}' and date_type = '{self.date_type}' and date_info = '{self.date_info}'
"""
print(sql)
self.df_store = self.spark.sql(sqlQuery=sql).cache()
print("本月数据量为：", self.df_store.count())
if self.can_mom:
self.df_store_last_month = self.read_history(self.last_month, "previous")
if self.can_yoy:
self.df_store_last_year = self.read_history(self.last_year_month, "lastyear")
# 计算同环比，不满足基准月条件的整列置空
def handle_change_rate(self):
if self.can_mom:
self.df_store = self.df_store.join(self.df_store_last_month, on='store_name', how='left')
if self.can_yoy:
self.df_store = self.df_store.join(self.df_store_last_year, on='store_name', how='left')
null_double = F.lit(None).cast(DoubleType())
for col in CHANGE_COLS:
mom_col = self.calculate_change(col, f"previous_{col}") if self.can_mom else null_double
yoy_col = self.calculate_change(col, f"lastyear_{col}") if self.can_yoy else null_double
self.df_store = self.df_store.withColumn(f"{col}_mom", mom_col).withColumn(f"{col}_yoy", yoy_col)
def handle_column(self):
self.df_save = self.df_store.select(
'store_name', 'category', 'country', 'sales', 'sales_weekly', 'revenue_total', 'revenue_7days',
'reviews', 'reviews_weekly', 'favorites', 'favorites_weekly', 'products', 'start_at', 'is_star',
'is_raving', 'rating', 'store_url', 'social_info', 'pay_type', 'created_at',
F.col('sales_weekly_mom').cast('double').alias('sales_weekly_mom'),
F.col('sales_weekly_yoy').cast('double').alias('sales_weekly_yoy'),
F.col('reviews_weekly_mom').cast('double').alias('reviews_weekly_mom'),
F.col('reviews_weekly_yoy').cast('double').alias('reviews_weekly_yoy'),
F.col('favorites_weekly_mom').cast('double').alias('favorites_weekly_mom'),
F.col('favorites_weekly_yoy').cast('double').alias('favorites_weekly_yoy'),
F.lit(self.site_name).alias('site_name'),
F.lit(self.date_type).alias('date_type'),
F.lit(self.date_info).alias('date_info')
).repartition(self.partitions_num)
def handle_data(self):
self.handle_change_rate()
self.handle_column()
def save_data(self):
print(f"清除hdfs目录中:{self.hdfs_path}")
HdfsUtils.delete_file_in_folder(self.hdfs_path)
print(f"当前存储的表名为：{self.hive_tb},分区为{self.partitions_by}")
self.df_save.write.saveAsTable(name=self.hive_tb, format='hive', mode='append', partitionBy=self.partitions_by)
print("save success")
if __name__ == '__main__':
site_name = sys.argv[1] # 参数1：站点
date_type = sys.argv[2] # 参数2：类型：month
date_info = sys.argv[3] # 参数3：年-月, 比如: 2026-10
assert date_type == 'month', "date_type 只支持 month"
handle_obj = DwtEhuntStoreDetail(site_name=site_name, date_type=date_type, date_info=date_info)
handle_obj.run()
Pyspark_job/sqoop_export/ods_ehunt_store_detail_pg.py → Pyspark_job/sqoop_export/dwt_ehunt_store_detail_pg.py
"""
@Author : CT
@Description : eHunt店铺详情 hive ods_ehunt_store_detail -> pg集群 ehunt_store_detail_{yyyy}(按date_info分区的分布式表)
@Description : Etsy店铺详情 hive dwt_ehunt_store_detail -> pg集群 ehunt_store_detail_{yyyy}(按date_info分区的分布式表)
"""
import os
import sys
......@@ -71,27 +71,33 @@ if __name__ == '__main__':
drop table if exists {export_tb_tmp};
create table {export_tb_tmp}
(
store_name varchar,
category varchar,
country varchar,
sales int4,
sales_weekly int4,
revenue_total float8,
revenue_7days float8,
reviews int4,
reviews_weekly int4,
favorites int4,
favorites_weekly int4,
products int4,
start_at varchar,
is_star varchar,
is_raving varchar,
rating float8,
store_url varchar,
social_info varchar,
pay_type varchar,
date_info varchar,
created_at varchar
store_name varchar,
category varchar,
country varchar,
sales int4,
sales_weekly int4,
revenue_total float8,
revenue_7days float8,
reviews int4,
reviews_weekly int4,
favorites int4,
favorites_weekly int4,
products int4,
start_at varchar,
is_star varchar,
is_raving varchar,
rating float8,
store_url varchar,
social_info varchar,
pay_type varchar,
date_info varchar,
created_at varchar,
sales_weekly_mom float8,
sales_weekly_yoy float8,
reviews_weekly_mom float8,
reviews_weekly_yoy float8,
favorites_weekly_mom float8,
favorites_weekly_yoy float8
);
"""
DBUtil.engine_exec_sql(engine, sql)
......@@ -99,7 +105,7 @@ if __name__ == '__main__':
sh = CommonUtil.build_export_sh(
site_name=site_name,
db_type=db_type,
hive_tb="ods_ehunt_store_detail",
hive_tb="dwt_ehunt_store_detail",
export_tb=export_tb_tmp,
col=[
"store_name",
......@@ -122,7 +128,13 @@ if __name__ == '__main__':
"social_info",
"pay_type",
"date_info",
"created_at"
"created_at",
"sales_weekly_mom",
"sales_weekly_yoy",
"reviews_weekly_mom",
"reviews_weekly_yoy",
"favorites_weekly_mom",
"favorites_weekly_yoy"
],
partition_dict={
"site_name": site_name,
......@@ -136,16 +148,19 @@ if __name__ == '__main__':
SSHUtil.exec_command_async(client, sh, ignore_err=False)
client.close()
# 同一事务内清空子表并从中间表写入，保证幂等；id按子表从1开始生成，分布列store_name不能为空
# 同一事务内清空子表并从中间表写入，保证幂等；id按子表从1开始生成
sql = f"""
truncate table {export_tb};
insert into {export_tb}
(id, store_name, category, country, sales, sales_weekly, revenue_total, revenue_7days, reviews, reviews_weekly,
favorites, favorites_weekly, products, start_at, is_star, is_raving, rating, store_url, social_info, pay_type,
date_info, created_at)
date_info, created_at, sales_weekly_mom, sales_weekly_yoy, reviews_weekly_mom, reviews_weekly_yoy,
favorites_weekly_mom, favorites_weekly_yoy)
select row_number() over (order by store_name) as id, store_name, category, country, sales, sales_weekly, revenue_total::numeric, revenue_7days::numeric,
reviews, reviews_weekly, favorites, favorites_weekly, products, start_at, is_star, is_raving, rating::numeric,
store_url, social_info, pay_type, date_info, created_at::timestamp
store_url, social_info, pay_type, date_info, created_at::timestamp,
sales_weekly_mom::numeric, sales_weekly_yoy::numeric, reviews_weekly_mom::numeric, reviews_weekly_yoy::numeric,
favorites_weekly_mom::numeric, favorites_weekly_yoy::numeric
from {export_tb_tmp}
where store_name is not null;
drop table if exists {export_tb_tmp};
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment