Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
Amazon-Selection-Data
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
abel_cjy
Amazon-Selection-Data
Commits
ca996d62
Commit
ca996d62
authored
Apr 22, 2026
by
chenyuanjie
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
fix
parent
53d3100f
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
6 additions
and
4 deletions
+6
-4
kafka_flow_asin_detail.py
Pyspark_job/my_kafka/kafka_flow_asin_detail.py
+3
-2
kafka_rank_asin_detail.py
Pyspark_job/my_kafka/kafka_rank_asin_detail.py
+3
-2
No files found.
Pyspark_job/my_kafka/kafka_flow_asin_detail.py
View file @
ca996d62
...
...
@@ -435,8 +435,9 @@ class KafkaFlowAsinDetail(Templates):
)
print
(
"导出父ASIN最新变体信息到doris:"
)
df_doris
=
df_asin_variat_agg
.
select
(
"parent_asin"
,
F
.
lit
(
self
.
date_info
)
.
alias
(
"date_info"
),
"asin_crawl_date"
,
"variation_info"
,
"attr_info"
)
table_columns
=
"parent_asin, date_info, asin_crawl_date, variation_info, attr_info"
"parent_asin"
,
F
.
lit
(
self
.
date_info
)
.
alias
(
"date_info"
),
"asin_crawl_date"
,
"variation_info"
,
"attr_info"
,
F
.
current_timestamp
()
.
alias
(
"updated_at"
))
table_columns
=
"parent_asin, date_info, asin_crawl_date, variation_info, attr_info, updated_at"
DorisHelper
.
spark_export_with_columns
(
df_save
=
df_doris
,
db_name
=
self
.
doris_db
,
table_name
=
self
.
parent_asin_latest_detail_table
,
table_columns
=
table_columns
)
df_doris
.
unpersist
()
return
df
...
...
Pyspark_job/my_kafka/kafka_rank_asin_detail.py
View file @
ca996d62
...
...
@@ -434,8 +434,9 @@ class KafkaRankAsinDetail(Templates):
)
print
(
"导出父ASIN最新变体信息到doris:"
)
df_doris
=
df_asin_variat_agg
.
select
(
"parent_asin"
,
F
.
lit
(
self
.
date_info
)
.
alias
(
"date_info"
),
"asin_crawl_date"
,
"variation_info"
,
"attr_info"
)
table_columns
=
"parent_asin, date_info, asin_crawl_date, variation_info, attr_info"
"parent_asin"
,
F
.
lit
(
self
.
date_info
)
.
alias
(
"date_info"
),
"asin_crawl_date"
,
"variation_info"
,
"attr_info"
,
F
.
current_timestamp
()
.
alias
(
"updated_at"
))
table_columns
=
"parent_asin, date_info, asin_crawl_date, variation_info, attr_info, updated_at"
DorisHelper
.
spark_export_with_columns
(
df_save
=
df_doris
,
db_name
=
self
.
doris_db
,
table_name
=
self
.
parent_asin_latest_detail_table
,
table_columns
=
table_columns
)
df_doris
.
unpersist
()
return
df
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment