Skip to content

A
Amazon-Selection-Data
Commit 05c13cdd by fangxingjun
Options

no message

parent 1d3a545e
Showing with 47 additions and 49 deletions
Pyspark_job/sqoop_export/export_dwt_asin_sync.py
...@@ -21,55 +21,53 @@ def update_workflow_manager(site_name, date_type, date_info): ...@@ -21,55 +21,53 @@ def update_workflow_manager(site_name, date_type, date_info):
site_name='us', site_name='us',
db_type='mysql' db_type='mysql'
) )
with engine_mysql.begin() as conn: spider_script = f'ansible dabing_all -f 10 -m shell -a "nohup /usr/local/bin/python3 /mnt/py_spider/threading_spider/Poll_site_spider_month.py {site_name} {date_type} {date_info} >/dev/null 2>&1 &";'
spider_script = f'ansible dabing_all -f 10 -m shell -a "nohup /usr/local/bin/python3 /mnt/py_spider/threading_spider/Poll_site_spider_month.py {site_name} {date_type} {date_info} >/dev/null 2>&1 &";' priority = site_name_pri_dict[site_name]
priority = site_name_pri_dict[site_name] update_sql_workflow_spider = f"""
update_sql_workflow_spider = f""" INSERT INTO selection.workflow_manager
INSERT INTO selection.workflow_manager (
( workflow_name,
workflow_name, site_name,
site_name, date_type,
date_type, date_info,
date_info, priority,
priority, spider_name,
spider_name, spider_script,
spider_script, spider_is_ready,
spider_is_ready, spider_state,
spider_state, bg_name,
bg_name, bg_dol_state,
bg_dol_state, finished_count
finished_count )
) VALUES
VALUES (
( '月全流程',
'月全流程', '{site_name}',
'{site_name}', 'month',
'month', '{date_info}',
'{date_info}', {priority},
{priority}, '{site_name}_spider_asin',
'{site_name}_spider_asin', '{spider_script}',
'{spider_script}', 'yes',
'yes', 1,
1, '{site_name}_all_cal',
'{site_name}_all_cal', 1,
3, 0
0 )
) ON DUPLICATE KEY UPDATE
ON DUPLICATE KEY UPDATE spider_is_ready = VALUES(spider_is_ready),
spider_is_ready = VALUES(spider_is_ready), spider_script = VALUES(spider_script),
spider_script = VALUES(spider_script), spider_state = VALUES(spider_state);
spider_state = VALUES(spider_state), """
bg_dol_state = VALUES(bg_dol_state); print(f"workflow_manager进度表---重置爬虫的asin抓取进度: {update_sql_workflow_spider}")
""" engine_mysql.execute(update_sql_workflow_spider)
print(f"workflow_manager进度表---重置爬虫的asin抓取进度: {update_sql_workflow_spider}")
conn.execute(update_sql_workflow_spider)
update_sql_workflow_bg = f"""update selection.workflow_manager update_sql_workflow_bg = f"""update selection.workflow_manager
set bg_dol_state=3, finished_count=COALESCE(finished_count, 0) + 1 set bg_dol_state=3, finished_count=COALESCE(finished_count, 0) + 1
WHERE workflow_name='月全流程' and site_name='{site_name}' and date_type='{date_type}' and date_info='{date_info}' and priority={priority}""" WHERE workflow_name='月全流程' and site_name='{site_name}' and date_type='{date_type}' and date_info='{date_info}' and priority={priority}"""
print(f"workflow_manager进度表---更新asin导出进度和完成次数: {update_sql_workflow_bg}") print(f"workflow_manager进度表---更新asin导出进度和完成次数: {update_sql_workflow_bg}")
conn.execute(update_sql_workflow_bg) engine_mysql.execute(update_sql_workflow_bg)
break break
except Exception as e: except Exception as e:
print(f"失败：workflow_manager进度表---重置爬虫的asin抓取进度: {update_sql_workflow_spider}, {e}, 报错信息: {traceback.format_exc()}") print(f"失败：workflow_manager进度表---重置爬虫的asin抓取进度: {update_sql_workflow_spider}, {e}, 报错信息: {traceback.format_exc()}")
time.sleep(300) time.sleep(300)
...@@ -112,6 +110,6 @@ if __name__ == '__main__': ...@@ -112,6 +110,6 @@ if __name__ == '__main__':
site_name = sys.argv[1] # 参数1：站点 site_name = sys.argv[1] # 参数1：站点
date_type = sys.argv[2] # 参数2：类型：week/4_week/month/quarter/day date_type = sys.argv[2] # 参数2：类型：week/4_week/month/quarter/day
date_info = sys.argv[3] # 参数3：年-周/年-月/年-季/年-月-日, 比如: 2022-1 date_info = sys.argv[3] # 参数3：年-周/年-月/年-季/年-月-日, 比如: 2022-1
export_data(site_name, date_type, date_info) # export_data(site_name, date_type, date_info)
if site_name in ["us", "uk", "de"] and date_type == "month": if site_name in ["us", "uk", "de"] and date_type == "month":
update_workflow_manager(site_name, date_type, date_info) update_workflow_manager(site_name, date_type, date_info)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment