import logging
import pandas as pd
from queue import Queue
from sqlalchemy.exc import OperationalError
from func_timeout.exceptions import FunctionTimedOut
from amazon_spider.db.mysql_db import df_to_sql
from amazon_spider.db.mysql_db import del_mysql_asin
# useful for handling different item types with a single interface


class AmazonMxSpiderPipeline:
    def __init__(self, site):
        self.site = site
        self.q_dict = {"asin_queue": Queue(), "error_queue": Queue()}
        self.num = 1
        self.save_num = 20

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            site=crawler.spider.site
        )

    def process_item(self, item, spider):
        if item.get("error_asin"):
            self.q_dict.get("error_queue").put((1, item.get("asin"), spider.short_site_to_dict(item.get("site"))))
        else:
            self.q_dict.get("asin_queue").put(list(item.values()))
        sql_up = f"UPDATE `mx_self_asin` set state=(%s)  where asin=(%s) and site=(%s);"
        for k, v in self.q_dict.items():
            if k == "asin_queue" and v.qsize() >= self.num:
                state_num = 3
                dates = [v.get() for i in range(0, self.num)]
                df = pd.DataFrame(dates, columns=spider.col)
                try:
                    df_to_sql("mx_self_asin_detail", df, site=self.site, db="mysql")
                    logging.info(f"入库成功-----{len(dates)}---------{dates}")
                except OperationalError as e:
                    logging.info(f"入库失败  连接错误{e}")
                    state_num = 1
                except FunctionTimedOut as e:
                    logging.info(f"入库-超时{e}----{len(dates)}---------{dates}")
                    state_num = 1
                asin_list = [(state_num, i[0], i[-1]) for i in dates]
                if len(asin_list) == 1:
                    d = asin_list[0]
                else:
                    d = asin_list
                while True:
                    try:
                        if del_mysql_asin(sql_up, data=d, site=self.site):
                            logging.info(f"修改asin状态3-----{len(d)}---------{d}")
                            break
                        else:
                            logging.info(f"修改asin状态3-失败----{len(d)}---------{d}")
                            continue
                    except FunctionTimedOut as e:
                        logging.info(f"修改asin状态3-超时{e}----{len(d)}---------{d}")
                        continue

            elif k == "error_queue" and v.qsize() >= self.num:
                dates = [v.get() for i in range(0, self.num)]
                print(dates)
                if len(dates) == 1:
                    d = dates[0]
                else:
                    d = dates
                while True:
                    try:
                        if del_mysql_asin(sql_up, data=d, site=self.site):
                            logging.info(f"修改asin状态3-----{len(d)}---------{d}")
                            break
                        else:
                            logging.info(f"修改asin状态3-失败----{len(d)}---------{d}")
                            continue
                    except FunctionTimedOut as e:
                        logging.info(f"修改asin状态3-超时{e}----{len(d)}---------{d}")
                        continue

    def close_spider(self, spider):
        print('爬虫结束，存储最后 数据', {k: v.qsize() for k, v in self.q_dict.items()})
        sql_up = f"UPDATE `mx_self_asin` set state=(%s)  where asin=(%s) and site=(%s);"
        for k, v in self.q_dict.items():
            if k == "asin_queue":
                state_num = 3
                dates = [v.get() for i in range(0, v.qsize())]
                df = pd.DataFrame(dates, columns=spider.col)
                try:
                    df_to_sql("mx_self_asin_detail", df, site=self.site, db="mysql")
                    logging.info(f"入库成功-----{len(dates)}---------{dates}")
                except OperationalError as e:
                    logging.info(f"入库失败  连接错误{e}")
                    state_num = 1
                except FunctionTimedOut as e:
                    logging.info(f"入库-超时{e}----{len(dates)}---------{dates}")
                    state_num = 1
                asin_list = [(state_num, i[0]) for i in dates]
                if len(asin_list) == 1:
                    d = asin_list[0]
                else:
                    d = asin_list
                while True:
                    try:
                        if del_mysql_asin(sql_up, data=d, site=self.site):
                            logging.info(f"修改asin状态3-----{len(d)}---------{d}")
                            break
                        else:
                            logging.info(f"修改asin状态3-失败----{len(d)}---------{d}")
                            continue
                    except FunctionTimedOut as e:
                        logging.info(f"修改asin状态3-超时{e}----{len(d)}---------{d}")
                        continue

            elif k == "error_queue" and v.qsize() >= self.num:
                dates = [v.get() for i in range(0, self.num)]
                if len(dates) == 1:
                    d = dates[0]
                else:
                    d = dates
                while True:
                    try:
                        if del_mysql_asin(sql_up, data=d, site=self.site):
                            logging.info(f"修改asin状态3-----{len(d)}---------{d}")
                            break
                        else:
                            logging.info(f"修改asin状态3-失败----{len(d)}---------{d}")
                            continue
                    except FunctionTimedOut as e:
                        logging.info(f"修改asin状态3-超时{e}----{len(d)}---------{d}")
                        continue