no message

Pyspark_job/sqoop_import/ods_bs_top100_asin.py

+import os
+import sys
+
+sys.path.append(os.path.dirname(sys.path[0]))
+
+from utils.common_util import CommonUtil
+from utils.secure_db_client import get_remote_engine
+
+if __name__ == '__main__':
+    site_name = CommonUtil.get_sys_arg(1, None)
+    date_info = CommonUtil.get_sys_arg(2, None)
+    assert site_name is not None, "sitename 不能为空！"
+    assert date_info is not None, "date_info 不能为空！"
+
+    db_type = "mysql"
+    if date_info == 'all':
+        query = f"""
+            select 
+                id,
+                asin,
+                cate_1_id,
+                cate_current_id,
+                bsr_rank,
+                rating,
+                total_comments,
+                created_at as updated_at,
+                date_info,
+                category_id
+            from {site_name}_bs_category_top100_asin
+            where 1 = 1
+            and \$CONDITIONS
+        """
+    else:
+        query = f"""
+            select 
+                id,
+                asin,
+                cate_1_id,
+                cate_current_id,
+                bsr_rank,
+                rating,
+                total_comments,
+                created_at as updated_at,
+                date_info,
+                category_id
+            from {site_name}_bs_category_top100_asin
+            where 1 = 1
+            and date_info = '{date_info}'
+            and \$CONDITIONS
+        """
+
+    hive_tb = "ods_bs_category_top100_asin"
+    partition_dict = {
+        "site_name": site_name
+    }
+    hdfs_path = CommonUtil.build_hdfs_path(hive_tb, partition_dict=partition_dict)
+
+    engine = get_remote_engine(
+        site_name=site_name,
+        db_type=db_type
+    )
+
+    engine.sqoop_raw_import(
+        query=query,
+        hive_table=hive_tb,
+        hdfs_path=hdfs_path,
+        partitions=partition_dict
+    )
+
+    pass

Pyspark_job/sqoop_import/ods_nsr_top100_asin.py

+import os
+import sys
+
+sys.path.append(os.path.dirname(sys.path[0]))
+
+from utils.common_util import CommonUtil
+from utils.secure_db_client import get_remote_engine
+
+if __name__ == '__main__':
+    site_name = CommonUtil.get_sys_arg(1, None)
+    date_info = CommonUtil.get_sys_arg(2, None)
+    assert site_name is not None, "sitename 不能为空！"
+    assert date_info is not None, "date_info 不能为空！"
+
+    db_type = "mysql"
+    if date_info == 'all':
+        query = f"""
+            select 
+                id,
+                asin,
+                cate_1_id,
+                cate_current_id,
+                bsr_rank,
+                rating,
+                total_comments,
+                created_at as updated_at,
+                date_info,
+                category_id
+            from {site_name}_bs_category_top100_asin
+            where 1 = 1
+            and \$CONDITIONS
+        """
+    else:
+        query = f"""
+            select 
+                id,
+                asin,
+                cate_1_id,
+                cate_current_id,
+                bsr_rank,
+                rating,
+                total_comments,
+                created_at as updated_at,
+                date_info,
+                category_id
+            from {site_name}_bs_category_top100_asin
+            where 1 = 1
+            and date_info = '{date_info}'
+            and \$CONDITIONS
+        """
+
+    hive_tb = "ods_bs_category_top100_asin"
+    partition_dict = {
+        "site_name": site_name
+    }
+    hdfs_path = CommonUtil.build_hdfs_path(hive_tb, partition_dict=partition_dict)
+
+    engine = get_remote_engine(
+        site_name=site_name,
+        db_type=db_type
+    )
+
+    engine.sqoop_raw_import(
+        query=query,
+        hive_table=hive_tb,
+        hdfs_path=hdfs_path,
+        partitions=partition_dict
+    )
+
+    pass