Commit 16cb7ae2 by fangxingjun

no message

parent d3a03b84
...@@ -88,9 +88,39 @@ class Integrate_search_term(): ...@@ -88,9 +88,39 @@ class Integrate_search_term():
conn.execute(deletesql) conn.execute(deletesql)
deletesql = f"DELETE from {self.site_name}_search_term_syn where state = 1 and date_info='{self.date_info}'" deletesql = f"DELETE from {self.site_name}_search_term_syn where state = 1 and date_info='{self.date_info}'"
conn.execute(deletesql) conn.execute(deletesql)
with self.engine_mysql.begin() as conn:
update_sql_workflow = f"""
INSERT INTO workflow_manager
(
site_name,
date_type,
date_info,
spider_name,
spider_is_ready,
spider_state
)
VALUES
(
'{self.site_name}',
'month',
'{self.date_info}',
'us_spider_st',
'yes',
1
)
ON DUPLICATE KEY UPDATE
spider_is_ready = VALUES(spider_is_ready),
spider_state = VALUES(spider_state);
"""
print(f"workflow_manager进度表---重置爬虫的搜索词抓取进度: {update_sql_workflow}")
conn.execute(update_sql_workflow)
data_df.to_sql(f'{self.site_name}_search_term_month_syn', con=self.engine_pg, if_exists="append", data_df.to_sql(f'{self.site_name}_search_term_month_syn', con=self.engine_pg, if_exists="append",
index=False) index=False)
else: else:
with self.engine_pg.begin() as conn: with self.engine_pg.begin() as conn:
delete_sql = f"DELETE from {self.site_name}_search_term_syn where date_info<'{self.date_info_pre}' and state not in (1,2)" delete_sql = f"DELETE from {self.site_name}_search_term_syn where date_info<'{self.date_info_pre}' and state not in (1,2)"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment