no message

Pyspark_job/sqoop_import/ods_asin_detail.py

 import os
 import sys
+import time
+import traceback
 
 sys.path.append(os.path.dirname(sys.path[0]))
 
@@ -44,6 +46,28 @@ if __name__ == '__main__':
         db_type=db_type
     )
 
+    def check_syn(engine):
+        while True:
+            try:
+                sql_check_syn = f"select * from {import_table} where state in (1, 2) limit 100"
+                df = engine.read_sql(sql_check_syn)
+                if df.shape[0] > 0:
+                    print(f"爬虫还未抓完, 等待5分钟继续")
+                    time.sleep(300)
+                    continue
+                else:
+                    print("爬虫已经全部抓取完成, 可以同步数据")
+                break
+            except Exception as e:
+                print(f"检查asin是否全部抓取完成报错, 报错信息: {e}, {traceback.format_exc()}")
+                time.sleep(300)
+                engine = get_remote_engine(
+                    site_name=site_name,
+                    db_type=db_type
+                )
+
+    check_syn(engine=engine)
+
     query = f"""
     SELECT
   id,