Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
Amazon-Selection-Data
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
abel_cjy
Amazon-Selection-Data
Commits
0af2971a
Commit
0af2971a
authored
Oct 27, 2026
by
chenyuanjie
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
asin_detail新增三个字段
parent
b6d6483f
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
11 additions
and
2 deletions
+11
-2
ods_asin_detail.py
Pyspark_job/sqoop_import/ods_asin_detail.py
+11
-2
No files found.
Pyspark_job/sqoop_import/ods_asin_detail.py
View file @
0af2971a
...
@@ -74,7 +74,7 @@ if __name__ == '__main__':
...
@@ -74,7 +74,7 @@ if __name__ == '__main__':
# 源表(PG爬虫表)这7个字段是分阶段才有的,早于对应月份的源表没有这几列,
# 源表(PG爬虫表)这7个字段是分阶段才有的,早于对应月份的源表没有这几列,
# 直接select会报"列不存在",历史回溯同步时按date_info降级为对应类型的NULL占位
# 直接select会报"列不存在",历史回溯同步时按date_info降级为对应类型的NULL占位
has_2025_cols
=
date_info
>=
'2025-01'
# bundles_this_asins_json/video_m3u8_url/result_list_json/bundle_asin_component_json/review_json_list/fbm_delivery_price
has_2025_cols
=
date_info
>=
'2025-01'
# bundles_this_asins_json/video_m3u8_url/result_list_json/bundle_asin_component_json/review_json_list/fbm_delivery_price
has_2026_cols
=
date_info
>=
'2026-01'
# amazon_label/landing_asin/sub_title
has_2026_cols
=
date_info
>=
'2026-01'
# amazon_label/landing_asin/sub_title
/warning_labels/media_format/voyager_detail_json
bundles_this_asins_json_expr
=
(
"REPLACE(REPLACE(REPLACE(bundles_this_asins_json, E'
\n
',' '), E'
\r
',' '), E'
\t
',' ') AS bundles_this_asins_json"
bundles_this_asins_json_expr
=
(
"REPLACE(REPLACE(REPLACE(bundles_this_asins_json, E'
\n
',' '), E'
\r
',' '), E'
\t
',' ') AS bundles_this_asins_json"
if
has_2025_cols
else
"CAST(NULL AS VARCHAR) AS bundles_this_asins_json"
)
if
has_2025_cols
else
"CAST(NULL AS VARCHAR) AS bundles_this_asins_json"
)
...
@@ -94,6 +94,12 @@ if __name__ == '__main__':
...
@@ -94,6 +94,12 @@ if __name__ == '__main__':
if
has_2026_cols
else
"CAST(NULL AS VARCHAR) AS landing_asin"
)
if
has_2026_cols
else
"CAST(NULL AS VARCHAR) AS landing_asin"
)
sub_title_expr
=
(
"REPLACE(REPLACE(REPLACE(sub_title, E'
\n
',' '), E'
\r
',' '), E'
\t
',' ') AS sub_title"
sub_title_expr
=
(
"REPLACE(REPLACE(REPLACE(sub_title, E'
\n
',' '), E'
\r
',' '), E'
\t
',' ') AS sub_title"
if
has_2026_cols
else
"CAST(NULL AS VARCHAR) AS sub_title"
)
if
has_2026_cols
else
"CAST(NULL AS VARCHAR) AS sub_title"
)
warning_labels_expr
=
(
"REPLACE(REPLACE(REPLACE(warning_labels, E'
\n
',' '), E'
\r
',' '), E'
\t
',' ') AS warning_labels"
if
has_2026_cols
else
"CAST(NULL AS VARCHAR) AS warning_labels"
)
media_format_expr
=
(
"REPLACE(REPLACE(REPLACE(media_format, E'
\n
',' '), E'
\r
',' '), E'
\t
',' ') AS media_format"
if
has_2026_cols
else
"CAST(NULL AS VARCHAR) AS media_format"
)
voyager_detail_json_expr
=
(
"REPLACE(REPLACE(REPLACE(voyager_detail_json, E'
\n
',' '), E'
\r
',' '), E'
\t
',' ') AS voyager_detail_json"
if
has_2026_cols
else
"CAST(NULL AS VARCHAR) AS voyager_detail_json"
)
query
=
f
"""
query
=
f
"""
SELECT
SELECT
...
@@ -170,7 +176,10 @@ if __name__ == '__main__':
...
@@ -170,7 +176,10 @@ if __name__ == '__main__':
{fbm_delivery_price_expr},
{fbm_delivery_price_expr},
{amazon_label_expr},
{amazon_label_expr},
{landing_asin_expr},
{landing_asin_expr},
{sub_title_expr}
{sub_title_expr},
{warning_labels_expr},
{media_format_expr},
{voyager_detail_json_expr}
FROM {import_table}
FROM {import_table}
WHERE 1=1 AND
\
$CONDITIONS
WHERE 1=1 AND
\
$CONDITIONS
"""
"""
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment