Commit ed081d26 by chenyuanjie

fix

parent 0c315139
...@@ -108,12 +108,13 @@ class EsAsinProfitRate(object): ...@@ -108,12 +108,13 @@ class EsAsinProfitRate(object):
) )
# 从Doris获取asin_crawl_date和asin_price,用profit_key关联 # 从Doris获取asin_crawl_date和asin_price,用profit_key关联
df_crawl_date = DorisHelper.spark_import_with_flight( # 改用 Doris Spark Connector(HTTP fenodes)读取,规避 JDBC/FlightSQL 在 DATETIME/NULL 列上的兼容问题
df_crawl_date = DorisHelper.spark_import_with_connector(
session=self.spark, session=self.spark,
table_identifier=f"selection.{self.site_name}_asin_latest_detail", table_identifier=f"selection.{self.site_name}_asin_latest_detail",
read_fields="asin,asin_crawl_date,asin_price" read_fields="asin,asin_crawl_date,asin_price"
).withColumn( ).withColumn(
"asin_crawl_date", F.substring(F.col("asin_crawl_date"), 1, 10) "asin_crawl_date", F.substring(F.col("asin_crawl_date").cast("string"), 1, 10)
).withColumn( ).withColumn(
"profit_key", F.concat_ws("_", F.col("asin"), F.round(F.col("asin_price"), 2)) "profit_key", F.concat_ws("_", F.col("asin"), F.round(F.col("asin_price"), 2))
).select('profit_key', 'asin_crawl_date') \ ).select('profit_key', 'asin_crawl_date') \
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment