Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
Amazon-Selection-Data
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
abel_cjy
Amazon-Selection-Data
Commits
0d9e053c
Commit
0d9e053c
authored
Jul 14, 2026
by
hejiangming
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
常年可卖/峰值月/属性标签时间限制
parent
a42d2f8a
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
48 additions
and
16 deletions
+48
-16
dwt_aba_last365.py
Pyspark_job/dwt/dwt_aba_last365.py
+15
-0
dwt_aba_st_analytics.py
Pyspark_job/dwt/dwt_aba_st_analytics.py
+33
-16
No files found.
Pyspark_job/dwt/dwt_aba_last365.py
View file @
0d9e053c
...
@@ -779,6 +779,21 @@ class DwtAbaLast365(object):
...
@@ -779,6 +779,21 @@ class DwtAbaLast365(object):
"peak_month"
:
""
"peak_month"
:
""
})
.
cache
()
})
.
cache
()
# ============================================================
# 【日期判断】派生字段各自"起算月"(早于此覆盖为默认值,与月表同口径,前端隐藏):
# peak_month / all_year_text_flag:< '2025-07' 填默认
# st_attribute_label:< '2025-01' 填默认(属性单独口径)
# 默认值:peak_month='' / all_year_text_flag=-1 / st_attribute_label='-1'(Java 转 null 返前端)。
# 注:year 类型 date_info 为 '2024-12'/'2025-12',字符串比较对 YYYY-MM 成立。
# ============================================================
if
self
.
date_info
<
'2025-07'
:
self
.
df_base
=
self
.
df_base
\
.
withColumn
(
'peak_month'
,
F
.
lit
(
''
))
\
.
withColumn
(
'all_year_text_flag'
,
F
.
lit
(
-
1
))
if
self
.
date_info
<
'2025-01'
:
self
.
df_base
=
self
.
df_base
\
.
withColumn
(
'st_attribute_label'
,
F
.
lit
(
'-1'
))
def
save_data
(
self
):
def
save_data
(
self
):
# 重新分区
# 重新分区
self
.
df_base
=
self
.
df_base
.
repartition
(
20
)
self
.
df_base
=
self
.
df_base
.
repartition
(
20
)
...
...
Pyspark_job/dwt/dwt_aba_st_analytics.py
View file @
0d9e053c
...
@@ -595,18 +595,21 @@ class DwtAbaStAnalytics(Templates):
...
@@ -595,18 +595,21 @@ class DwtAbaStAnalytics(Templates):
# 【为什么加站点判断】st_attribute_label 只有 us 计算 因为属性是基于us统计 业务要求 uk de不展示 这里填充默认值
# 【为什么加站点判断】st_attribute_label 只有 us 计算 因为属性是基于us统计 业务要求 uk de不展示 这里填充默认值
# 非 us 直接跳过读取,st_attribute_label 在 handle_data 里统一填占位 '-1'(Java 转 null 返前端)。
# 非 us 直接跳过读取,st_attribute_label 在 handle_data 里统一填占位 '-1'(Java 转 null 返前端)。
if
self
.
site_name
==
'us'
:
if
self
.
site_name
==
'us'
:
has_theme_data
=
self
.
spark
.
sql
(
f
"""
# ===== 临时注释(补数用):补数时 dws_st_theme 可能还没跑好,先关掉存在性校验避免阻塞任务 =====
SELECT 1 FROM dws_st_theme
# 影响范围仅 st_attribute_label 一个字段:无数据时它填占位 '-1'(Java 转 null),不影响任何其他字段。
WHERE site_name='{self.site_name}'
# dws_st_theme 调度恢复正常后,请取消下面注释、恢复 assert。
AND date_type='{self.date_type}'
# has_theme_data = self.spark.sql(f"""
AND date_info='{self.date_info}'
# SELECT 1 FROM dws_st_theme
LIMIT 1
# WHERE site_name='{self.site_name}'
"""
)
.
take
(
1
)
# AND date_type='{self.date_type}'
assert
len
(
has_theme_data
)
>
0
,
(
# AND date_info='{self.date_info}'
f
"上游 dws_st_theme 分区无数据 "
# LIMIT 1
f
"site_name={self.site_name}/date_type={self.date_type}/date_info={self.date_info},"
# """).take(1)
f
"请确认dws_st_theme 调度是否完成;若强行跑 dwt_aba_st_analytics 会导致所有词的 st_attribute_label 错填 '-1'"
# assert len(has_theme_data) > 0, (
)
# f"上游 dws_st_theme 分区无数据 "
# f"site_name={self.site_name}/date_type={self.date_type}/date_info={self.date_info},"
# f"请确认dws_st_theme 调度是否完成;若强行跑 dwt_aba_st_analytics 会导致所有词的 st_attribute_label 错填 '-1'"
# )
sql
=
f
"""
sql
=
f
"""
select
select
...
@@ -714,14 +717,28 @@ class DwtAbaStAnalytics(Templates):
...
@@ -714,14 +717,28 @@ class DwtAbaStAnalytics(Templates):
# ============================================================
# ============================================================
if
self
.
date_type
==
'month'
:
if
self
.
date_type
==
'month'
:
self
.
handle_first_ever_flag
()
self
.
handle_first_ever_flag
()
# ============================================================
# 【日期判断】派生字段各自"起算月"(早于此填默认值,前端隐藏):
# peak_month / all_year_text_flag:>= '2025-07' 才算
# (2025-07 前历史 dwt 曾被删 / 老逻辑读空自身历史补过,峰值月/常年可卖口径不可靠)
# st_attribute_label:>= '2025-01' 才算(属性数据单独口径,2025-01 起可靠)
# 默认值与非 month 流程一致:peak_month='' / all_year_text_flag=-1 / st_attribute_label='-1'。
# ============================================================
calc_peak
=
self
.
date_info
>=
'2025-07'
# 峰值月/常年可卖 起算月
calc_attr
=
self
.
date_info
>=
'2025-01'
# 属性标签 起算月(单独口径)
if
calc_peak
:
self
.
handle_peak_month
()
self
.
handle_peak_month
()
# st_attribute_label 只有 us 计算
else
:
# 非 us(uk/de)不 join、直接填占位 '-1',与非 month 流程同款(Java 转 null 返前端)
self
.
df_save
=
self
.
df_save
\
if
self
.
site_name
==
'us'
:
.
withColumn
(
'peak_month'
,
F
.
lit
(
''
))
\
.
withColumn
(
'all_year_text_flag'
,
F
.
lit
(
-
1
))
# st_attribute_label 只有 us 且 date_info>=2025-01 才 join 计算
# 非 us(uk/de)或早于 2025-01:直接填占位 '-1'(Java 转 null 返前端)
if
calc_attr
and
self
.
site_name
==
'us'
:
self
.
df_save
=
self
.
df_save
.
join
(
self
.
df_st_attribute
,
on
=
'search_term'
,
how
=
'left'
)
self
.
df_save
=
self
.
df_save
.
join
(
self
.
df_st_attribute
,
on
=
'search_term'
,
how
=
'left'
)
self
.
df_st_attribute
.
unpersist
()
else
:
else
:
self
.
df_save
=
self
.
df_save
.
withColumn
(
'st_attribute_label'
,
F
.
lit
(
'-1'
))
self
.
df_save
=
self
.
df_save
.
withColumn
(
'st_attribute_label'
,
F
.
lit
(
'-1'
))
self
.
df_st_attribute
.
unpersist
()
# 无论走哪支都释放(非 us/未算时是空 df,unpersist 无害)
self
.
df_save
=
self
.
df_save
.
join
(
self
.
df_st_filter
,
on
=
'search_term'
,
how
=
'left'
)
self
.
df_save
=
self
.
df_save
.
join
(
self
.
df_st_filter
,
on
=
'search_term'
,
how
=
'left'
)
self
.
df_st_filter
.
unpersist
()
self
.
df_st_filter
.
unpersist
()
else
:
else
:
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment