Commit ed081d26 by chenyuanjie

fix

parent 0c315139
......@@ -108,12 +108,13 @@ class EsAsinProfitRate(object):
)
# 从Doris获取asin_crawl_date和asin_price,用profit_key关联
df_crawl_date = DorisHelper.spark_import_with_flight(
# 改用 Doris Spark Connector(HTTP fenodes)读取,规避 JDBC/FlightSQL 在 DATETIME/NULL 列上的兼容问题
df_crawl_date = DorisHelper.spark_import_with_connector(
session=self.spark,
table_identifier=f"selection.{self.site_name}_asin_latest_detail",
read_fields="asin,asin_crawl_date,asin_price"
).withColumn(
"asin_crawl_date", F.substring(F.col("asin_crawl_date"), 1, 10)
"asin_crawl_date", F.substring(F.col("asin_crawl_date").cast("string"), 1, 10)
).withColumn(
"profit_key", F.concat_ws("_", F.col("asin"), F.round(F.col("asin_price"), 2))
).select('profit_key', 'asin_crawl_date') \
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment