Skip to content

S
spider
Commit 0040e69f by Peng
Options

no message

parent fa8ab9c2
Showing with 129 additions and 81 deletions
wangjing_projects/projects/picture_material/stock_summery/new_stock/all_connect.py
......@@ -173,7 +173,7 @@ class ConnectSpider:
try:
table_name = "stock_image_summary_wj"
# 修改查询语句以匹配你的数据表名称和列名称
query = f"""SELECT account_id,account_secret FROM {table_name} where id = {item_id} and state= 1;"""
query = f"""SELECT account_id,account_secret FROM {table_name} where id = {item_id}"""
print(query)
df_status = self.db_engine192.read_sql(query)
if len(df_status) > 0:
......@@ -191,8 +191,8 @@ class ConnectSpider:
def get_all_image_id(self):
for i in range(5):
try:
table_name = "stock_image_detail_wj"
sql_query = f"SELECT image_id FROM {table_name} "
table_name = "stock_image_detail_wj" # 全部下载完成之后删除这个条件
sql_query = f"SELECT image_id FROM {table_name} where created_time>'2025-12-12'"
df_status = self.db_engine192.read_sql(sql_query)
image_id = list(df_status['image_id'].astype(str))
return image_id
......@@ -205,7 +205,8 @@ class ConnectSpider:
try:
with self.db_engine192.begin() as connection:
table_name = "stock_image_detail_wj"
sql_update = f"UPDATE {table_name} SET state = 3 WHERE image_id ='{image_id}' and state = 1"
sql_update = f"UPDATE {table_name} SET state = 3 WHERE image_id ='{image_id}' and state = 2"
print(sql_update,'234343434343434343434343434')
connection.execute(sql_update)
break
except Exception as e:
......@@ -215,9 +216,16 @@ class ConnectSpider:
def get_pic_urls(self, account_id):
pic_data_list = [] # 创建一个空列表来存储结果
table_name = "stock_image_detail_wj"
query =f"""select image_url, image_id, image_title from {table_name} where account_id = '{account_id}' and state = 1"""
query =f"""select image_url, image_id, image_title,id from {table_name} where account_id = '{account_id}' and state = 1"""
print('== query::',query)
try:
result_df = self.db_engine192.read_sql(query)
# result_df = self.db_engine192.read_sql(query)
result_df = self.db_engine192.read_then_update(
select_sql=query,
update_table=table_name,
set_values={"state": 2}, # 把库存清零
where_keys=["id"], # WHERE sku = :sku
)
# print(result_df)
result_list = result_df.values.tolist()
for row in result_list: # 遍历所有的结果行
......@@ -239,4 +247,5 @@ class ConnectSpider:
if __name__ == '__main__':
ConnectSpider().get_cookie_account(10)
# UPDATE stock_image_detail_wj SET state = 3 WHERE image_id ='2668613199' and state = 1
ConnectSpider().get_pic_urls('xiezhihao@yswg.com.cn')
wangjing_projects/projects/picture_material/stock_summery/new_stock/download_pic.py
......@@ -68,7 +68,11 @@ class Download():
else:
# 不包含中文时，按照原策略处理，但总长不超过160字符
pic_name = re.sub(r'[\\/*?:"<>|]', '_', pic_name)[:160] + '.jpg'
print('image_url:',image_url)
res = requests.get(image_url, timeout=600)
content_type = res.headers.get("Content-Type", "")
print('content_type: 查看返回的类型：',content_type)
# 构建完整的文件路径
file_path = os.path.join(save_folder, pic_name)
self.download_image(file_path, res)
......
wangjing_projects/projects/picture_material/stock_summery/new_stock/upload_pic.py
import json
import os
from datetime import datetime
import requests
# import requests
import PIL
from PIL import Image
from all_connect import ConnectSpider
import time
import shutil
# import time
# import shutil
import re
from pathlib import Path
# from pathlib import Path
import shutil
import traceback
from pathlib import Path
import tempfile
import time
import requests
import hashlib
# 最大像素点
PIL.Image.MAX_IMAGE_PIXELS = 933120000
def generate_thumbnail_240(input_image_path, output_image_path):
# 打开图片
print('生成缩略图')
with Image.open(input_image_path) as img:
# 获取原始图片大小
width, height = img.size
......@@ -38,6 +41,7 @@ def generate_thumbnail_240(input_image_path, output_image_path):
def generate_thumbnail_big(input_image_path, output_image_path):
# 打开图片
print('打开图片 input_image_path::',input_image_path)
with Image.open(input_image_path) as img:
# 获取原始图片大小
width, height = img.size
......@@ -57,19 +61,17 @@ def generate_thumbnail_big(input_image_path, output_image_path):
img.save(output_image_path)
def exec_sql(sql: str, paramMap: dict):
from sqlalchemy import text
with db_engine.connect() as connection:
trans = connection.begin()
connection.execute(text(sql), paramMap)
trans.commit()
pass
def exec_sql(sql):
print('exec_sql::',sql)
with db_engine.begin() as conn:
conn.execute(sql)
def read_sql(sql: str):
import pandas as pd
with db_engine.connect() as connection:
return pd.read_sql(sql, con=connection)
# def read_sql(sql: str):
# import pandas as pd
# with db_engine.connect() as connection:
# return pd.read_sql(sql, con=connection)
db_engine = ConnectSpider().db_engine192
......@@ -83,13 +85,41 @@ all_pic_success_dir = "/home/wangjing/picture_material/stock_summery/all_pic_suc
class UploadPicture():
def __init__(self):
self.count = 0
pass
self.get_microservice_token()
def get_microservice_token(self):
for i in range(5):
try:
url = "http://192.168.2.97:6661/microservice-system/system/admin/getToken"
timestamp = str(int(time.time()))
secret = "b5c96034-1099-4dbf-8c72-50d965cca81d"
md5_value = hashlib.md5((secret + timestamp).encode("utf-8")).hexdigest()
response = requests.post(url, json={
"module": "spider",
"weChatId": "pengyanbing",
"secret": md5_value,
"timestamp": timestamp
})
res = response.json()
print(res)
if (res['code'] == 200):
userinfo = res['data']
self.token = userinfo['token']
expireTime = userinfo['expireTime']
print(self.token, expireTime)
else:
raise Exception(res['msg'])
break
except Exception as e:
print('get_microservice_token, 报错',e)
def upload_file(self, img_id, file_path):
print(img_id,' file_path:::',file_path)
# 检查图片是否已上传
success_flag = False
sql = f"SELECT err_msg FROM stock_image_detail_wj WHERE image_id = '{img_id}';"
result_df = read_sql(sql)
# sql = f"-- SELECT err_msg FROM stock_image_detail_wj WHERE image_id = '{img_id}';"
# result_df = read_sql(sql)
# # 检查是否有任何一个 err_msg 等于 'success'
# if 'success' in result_df['err_msg'].values:
......@@ -116,19 +146,17 @@ class UploadPicture():
big_img_path = os.path.join(tmpdir, "thumbnail_image_big" + suffix)
img_240_path = os.path.join(tmpdir, "thumbnail_image_240" + suffix)
try:
print(target_path, os.path.join(tmpdir, big_img_path),'23333333333')
generate_thumbnail_big(target_path, os.path.join(tmpdir, big_img_path))
generate_thumbnail_240(target_path, os.path.join(tmpdir, img_240_path))
except Exception as ex:
print(f"文件{filename}生成缩略图失败:{traceback.format_exc()}")
print(f"文件{filename}生成缩略图失败:",ex)
return False
pass
retry_time = 1
while retry_time < 3:
url = 'http://wx.yswg.com.cn:8000/microservice-visual/visual/fileSystem/shutterstock_upload'
# url = "http://192.168.2.97:6661/microservice-visual/visual/fileSystem/shutterstock_upload"
# url = 'http://localhost:8600/visual/fileSystem/shutterstock_upload'
files = {
'file': open(target_path, 'rb'),
'file_big': open(big_img_path, 'rb'),
......@@ -136,39 +164,43 @@ class UploadPicture():
}
body = {'token': "dacce869-0471-4ec7-ac50-3b3b1ec22c87"}
try:
response = requests.post(url, files=files, data=body, timeout=600)
headers = {
"authorization": self.token
}
response = requests.post(url, files=files, data=body, timeout=600,headers=headers)
print('response:::',response.text)
response_data = json.loads(response.text)
print('response_data::',response_data)
formatted_time = datetime.now().strftime('%Y/%m/%d %H:%M:%S')
if response_data.get('code') == 200:
print(f"{filename} 上传成功 第{self.count}个,{formatted_time}")
exec_sql(
f"""update stock_image_detail_wj set upload_time=:upload_time,err_msg=:err_msg where image_id = :image_id""",
{"upload_time": formatted_time, "err_msg": "success", "image_id": img_id})
f"""update stock_image_detail_wj set upload_time='{formatted_time}',err_msg='success' where image_id = '{img_id}'""")
self.count = self.count + 1
success_flag = True
break
else:
print(f"{filename} 上传失败 第{self.count}个 {formatted_time}: {response_data}")
errMsg = response_data.get("msg")
print('errMsg:::',errMsg)
if errMsg == '您上传的文件已经被上传过~!请勿重复上传!':
exec_sql(
f"""update stock_image_detail_wj set upload_time=:upload_time,err_msg=:err_msg where image_id = :image_id""",
{"upload_time": formatted_time, "err_msg": "success", "image_id": img_id})
f"""update stock_image_detail_wj set upload_time='{formatted_time}',err_msg='success' where image_id ='img_id'""" )
self.count = self.count + 1
success_flag = True
break
else:
exec_sql(
f"""update stock_image_detail_wj set upload_time=:upload_time,err_msg=:err_msg where image_id = :image_id""",
{"upload_time": formatted_time, "err_msg": json.dumps(response_data), "image_id": img_id})
exec_sql(f"""update stock_image_detail_wj set upload_time='{formatted_time}',err_msg='{json.dumps(response_data)}' where image_id = '{img_id}'""")
if "" in errMsg:
# 此处超时等
pass
retry_time = retry_time + 1
except Exception as e:
print('报错：33333',e)
self.get_microservice_token()
finally:
# 关闭文件流
for file in files.values():
......@@ -200,53 +232,51 @@ class UploadPicture():
print(f"文件{file_name}不是正常规则的图片命名！！")
continue
image_id_str = image_id_list[0]
print('upload_dir:',image_id_str)
file_path = os.path.join(folder_path, file_name)
success_flag = self.upload_file(image_id_str, file_path)
if success_flag:
# 转移到上传成功的文件夹中
print('转移到上传成功的文件夹中')
shutil.move(file_path, os.path.join(all_pic_success_dir, file_name))
pass
def run(self):
folder_path_list = [
"xiezhihao@yswg.com.cn",
"24016587887",
"daiting@yswg.com.cn",
"yangzinan@yswg.com.cn",
"danqingmei30",
"352422819@qq.com",
"xuyuanlian@yswg.com.cn",
"496775268@qq.com",
"878097683@qq.com",
"991450084@qq.com",
"yangxia@yswg.com.cn",
"zouxiayan@yswg.com.cn",
"laiyawen@yswg.com.cn",
"1060103503@qq.com",
"1120841348@qq.com",
"1014703046@qq.com",
"shuqian@yswg.com.cn",
"wengdan300",
"jiangbin@yswg.com.cn",
"yangxuanwei54",
"1196846022@QQ.COM",
"liyun1@yswg.com.cn",
"lizhili@yswg.com.cn",
"chenxiaowei@yswg.com.cn",
"1101549094@qq.com",
"liulujun@yswg.com.cn",
"fanglidong@yswg.com.cn",
"jiangyuan1@yswg.com.cn",
"xujianbo@yswg.com.cn",
"xujing@yswg.com.cn",
"lujun@yswg.com.cn",
"liujia@yswg.com.cn",
"weixiaoling@yswg.com.cn",
"1073463224",
'xiezhihao@yswg.com.cn',
'24016587887',
'daiting@yswg.com.cn',
'yangzinan@yswg.com.cn',
'danqingmei30',
'352422819@qq.com',
'xuyuanlian@yswg.com.cn',
'496775268@qq.com',
'878097683@qq.com',
'991450084@qq.com',
'yangxia@yswg.com.cn',
'zouxiayan@yswg.com.cn',
'luhailin@yswg.com.cn',
'1060103503@qq.com',
'1120841348@qq.com',
'1014703046@qq.com',
'shuqian@yswg.com.cn',
'wengdan300',
'jiangbin@yswg.com.cn',
'yangxuanwei54',
'1196846022@QQ.COM',
'liulujun@yswg.com.cn',
'fanglidong@yswg.com.cn',
'jiangyuan1@yswg.com.cn',
'xujing@yswg.com.cn',
'lujun@yswg.com.cn',
'liujia@yswg.com.cn',
'weixiaoling@yswg.com.cn',
'2563838314@qq.com',
]
for account_id in folder_path_list:
if 8 <= datetime.now().hour < 18:
if 9 <= datetime.now().hour < 16:
print(f"当前时间 {datetime.now().hour} 点，超出运行时段 18:00–08:00，退出循环。")
break
print(f"当前时间 {datetime.now().hour} 点，开始处理账户 {account_id}。")
......@@ -255,10 +285,10 @@ class UploadPicture():
else:
account_id_tmp = account_id
folder_path = os.path.join(all_pic_dir, account_id_tmp)
print('folder_path:::',folder_path)
# 带重试地执行上传目录
success = False
for attempt in range(3): # 最多重试3次
for attempt in range(1): # 最多重试3次
try:
self.upload_dir(folder_path)
success = True
......
wangjing_projects/projects/picture_material/stock_summery/new_stock/xnj_url.py
......@@ -435,9 +435,13 @@ class GetSS_details():
sleep(randint(2, 4))
# 查找并输入密码
print("正在等待密码输入框...")
print("正在等待密码输入框...")# placeholder="密码"
# MuiInputBase-input MuiOutlinedInput-input MuiInputBase-inputSizeSmall MuiInputBase-inputAdornedEnd css-18zhmu4
try:
email_input = iframe.ele(
'.MuiInputBase-input MuiInput-input MuiInputBase-inputSizeSmall MuiInputBase-inputAdornedEnd css-186x7cf')
'xpath://input[@placeholder="密码"]')
except:
email_input = iframe.ele('.MuiInputBase-input MuiOutlinedInput-input MuiInputBase-inputSizeSmall MuiInputBase-inputAdornedEnd css-18zhmu4')
email_input.clear() # 清除任何预填充的内容
email_input.input(self.pwd)
print("已输入密码到密码输入框")
......@@ -509,7 +513,7 @@ class GetSS_details():
}
try:
response = requests.post(url, data=data_json,headers=headers)
response = requests.post(url, data=data_json,headers=headers,timeout=120)
if response.status_code == 200:
return response.json()
else:
......@@ -564,6 +568,7 @@ class GetSS_details():
}
response = requests.post('https://www.shutterstock.com/napi/licensees/current/redownload', cookies=cookie,
headers=self.headers, json=json_data, timeout=600)
print('下载返回:',response.text)
image_url = json.loads(response.text)['meta']['licensedContent'][0]['downloadUrl']
return image_url
......@@ -575,7 +580,7 @@ class GetSS_details():
try:
start_time = datetime.now().strftime("%m-%d %H:%M:%S")
all_image_id = Con.get_all_image_id()
# 12月的重新上传。先注释
if str(image_id) in set(all_image_id):
print(f'{image_id}已上传过')
state = 3
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment