Skip to content

S
spider
Commit 0040e69f by Peng
Options

no message

parent fa8ab9c2
Showing with 130 additions and 82 deletions
wangjing_projects/projects/picture_material/stock_summery/new_stock/all_connect.py
...@@ -173,7 +173,7 @@ class ConnectSpider: ...@@ -173,7 +173,7 @@ class ConnectSpider:
try: try:
table_name = "stock_image_summary_wj" table_name = "stock_image_summary_wj"
# 修改查询语句以匹配你的数据表名称和列名称 # 修改查询语句以匹配你的数据表名称和列名称
query = f"""SELECT account_id,account_secret FROM {table_name} where id = {item_id} and state= 1;""" query = f"""SELECT account_id,account_secret FROM {table_name} where id = {item_id}"""
print(query) print(query)
df_status = self.db_engine192.read_sql(query) df_status = self.db_engine192.read_sql(query)
if len(df_status) > 0: if len(df_status) > 0:
...@@ -191,8 +191,8 @@ class ConnectSpider: ...@@ -191,8 +191,8 @@ class ConnectSpider:
def get_all_image_id(self): def get_all_image_id(self):
for i in range(5): for i in range(5):
try: try:
table_name = "stock_image_detail_wj" table_name = "stock_image_detail_wj" # 全部下载完成之后删除这个条件
sql_query = f"SELECT image_id FROM {table_name} " sql_query = f"SELECT image_id FROM {table_name} where created_time>'2025-12-12'"
df_status = self.db_engine192.read_sql(sql_query) df_status = self.db_engine192.read_sql(sql_query)
image_id = list(df_status['image_id'].astype(str)) image_id = list(df_status['image_id'].astype(str))
return image_id return image_id
...@@ -205,7 +205,8 @@ class ConnectSpider: ...@@ -205,7 +205,8 @@ class ConnectSpider:
try: try:
with self.db_engine192.begin() as connection: with self.db_engine192.begin() as connection:
table_name = "stock_image_detail_wj" table_name = "stock_image_detail_wj"
sql_update = f"UPDATE {table_name} SET state = 3 WHERE image_id ='{image_id}' and state = 1" sql_update = f"UPDATE {table_name} SET state = 3 WHERE image_id ='{image_id}' and state = 2"
print(sql_update,'234343434343434343434343434')
connection.execute(sql_update) connection.execute(sql_update)
break break
except Exception as e: except Exception as e:
...@@ -215,9 +216,16 @@ class ConnectSpider: ...@@ -215,9 +216,16 @@ class ConnectSpider:
def get_pic_urls(self, account_id): def get_pic_urls(self, account_id):
pic_data_list = [] # 创建一个空列表来存储结果 pic_data_list = [] # 创建一个空列表来存储结果
table_name = "stock_image_detail_wj" table_name = "stock_image_detail_wj"
query =f"""select image_url, image_id, image_title from {table_name} where account_id = '{account_id}' and state = 1""" query =f"""select image_url, image_id, image_title,id from {table_name} where account_id = '{account_id}' and state = 1"""
print('== query::',query)
try: try:
result_df = self.db_engine192.read_sql(query) # result_df = self.db_engine192.read_sql(query)
result_df = self.db_engine192.read_then_update(
select_sql=query,
update_table=table_name,
set_values={"state": 2}, # 把库存清零
where_keys=["id"], # WHERE sku = :sku
)
# print(result_df) # print(result_df)
result_list = result_df.values.tolist() result_list = result_df.values.tolist()
for row in result_list: # 遍历所有的结果行 for row in result_list: # 遍历所有的结果行
...@@ -239,4 +247,5 @@ class ConnectSpider: ...@@ -239,4 +247,5 @@ class ConnectSpider:
if __name__ == '__main__': if __name__ == '__main__':
ConnectSpider().get_cookie_account(10) # UPDATE stock_image_detail_wj SET state = 3 WHERE image_id ='2668613199' and state = 1
ConnectSpider().get_pic_urls('xiezhihao@yswg.com.cn')
wangjing_projects/projects/picture_material/stock_summery/new_stock/download_pic.py
...@@ -68,7 +68,11 @@ class Download(): ...@@ -68,7 +68,11 @@ class Download():
else: else:
# 不包含中文时，按照原策略处理，但总长不超过160字符 # 不包含中文时，按照原策略处理，但总长不超过160字符
pic_name = re.sub(r'[\\/*?:"<>|]', '_', pic_name)[:160] + '.jpg' pic_name = re.sub(r'[\\/*?:"<>|]', '_', pic_name)[:160] + '.jpg'
print('image_url:',image_url)
res = requests.get(image_url, timeout=600) res = requests.get(image_url, timeout=600)
content_type = res.headers.get("Content-Type", "")
print('content_type: 查看返回的类型：',content_type)
# 构建完整的文件路径 # 构建完整的文件路径
file_path = os.path.join(save_folder, pic_name) file_path = os.path.join(save_folder, pic_name)
self.download_image(file_path, res) self.download_image(file_path, res)
......
wangjing_projects/projects/picture_material/stock_summery/new_stock/upload_pic.py
import json import json
import os import os
from datetime import datetime from datetime import datetime
import requests # import requests
import PIL import PIL
from PIL import Image from PIL import Image
from all_connect import ConnectSpider from all_connect import ConnectSpider
import time # import time
import shutil # import shutil
import re import re
from pathlib import Path # from pathlib import Path
import shutil import shutil
import traceback import traceback
from pathlib import Path from pathlib import Path
import tempfile import tempfile
import time
import requests
import hashlib
# 最大像素点 # 最大像素点
PIL.Image.MAX_IMAGE_PIXELS = 933120000 PIL.Image.MAX_IMAGE_PIXELS = 933120000
def generate_thumbnail_240(input_image_path, output_image_path): def generate_thumbnail_240(input_image_path, output_image_path):
# 打开图片 # 打开图片
print('生成缩略图')
with Image.open(input_image_path) as img: with Image.open(input_image_path) as img:
# 获取原始图片大小 # 获取原始图片大小
width, height = img.size width, height = img.size
...@@ -38,6 +41,7 @@ def generate_thumbnail_240(input_image_path, output_image_path): ...@@ -38,6 +41,7 @@ def generate_thumbnail_240(input_image_path, output_image_path):
def generate_thumbnail_big(input_image_path, output_image_path): def generate_thumbnail_big(input_image_path, output_image_path):
# 打开图片 # 打开图片
print('打开图片 input_image_path::',input_image_path)
with Image.open(input_image_path) as img: with Image.open(input_image_path) as img:
# 获取原始图片大小 # 获取原始图片大小
width, height = img.size width, height = img.size
...@@ -57,19 +61,17 @@ def generate_thumbnail_big(input_image_path, output_image_path): ...@@ -57,19 +61,17 @@ def generate_thumbnail_big(input_image_path, output_image_path):
img.save(output_image_path) img.save(output_image_path)
def exec_sql(sql: str, paramMap: dict): def exec_sql(sql):
from sqlalchemy import text print('exec_sql::',sql)
with db_engine.connect() as connection: with db_engine.begin() as conn:
trans = connection.begin() conn.execute(sql)
connection.execute(text(sql), paramMap)
trans.commit()
pass
def read_sql(sql: str): # def read_sql(sql: str):
import pandas as pd # import pandas as pd
with db_engine.connect() as connection: # with db_engine.connect() as connection:
return pd.read_sql(sql, con=connection) # return pd.read_sql(sql, con=connection)
db_engine = ConnectSpider().db_engine192 db_engine = ConnectSpider().db_engine192
...@@ -83,13 +85,41 @@ all_pic_success_dir = "/home/wangjing/picture_material/stock_summery/all_pic_suc ...@@ -83,13 +85,41 @@ all_pic_success_dir = "/home/wangjing/picture_material/stock_summery/all_pic_suc
class UploadPicture(): class UploadPicture():
def __init__(self): def __init__(self):
self.count = 0 self.count = 0
pass self.get_microservice_token()
def get_microservice_token(self):
for i in range(5):
try:
url = "http://192.168.2.97:6661/microservice-system/system/admin/getToken"
timestamp = str(int(time.time()))
secret = "b5c96034-1099-4dbf-8c72-50d965cca81d"
md5_value = hashlib.md5((secret + timestamp).encode("utf-8")).hexdigest()
response = requests.post(url, json={
"module": "spider",
"weChatId": "pengyanbing",
"secret": md5_value,
"timestamp": timestamp
})
res = response.json()
print(res)
if (res['code'] == 200):
userinfo = res['data']
self.token = userinfo['token']
expireTime = userinfo['expireTime']
print(self.token, expireTime)
else:
raise Exception(res['msg'])
break
except Exception as e:
print('get_microservice_token, 报错',e)
def upload_file(self, img_id, file_path): def upload_file(self, img_id, file_path):
print(img_id,' file_path:::',file_path)
# 检查图片是否已上传 # 检查图片是否已上传
success_flag = False success_flag = False
sql = f"SELECT err_msg FROM stock_image_detail_wj WHERE image_id = '{img_id}';" # sql = f"-- SELECT err_msg FROM stock_image_detail_wj WHERE image_id = '{img_id}';"
result_df = read_sql(sql) # result_df = read_sql(sql)
# # 检查是否有任何一个 err_msg 等于 'success' # # 检查是否有任何一个 err_msg 等于 'success'
# if 'success' in result_df['err_msg'].values: # if 'success' in result_df['err_msg'].values:
...@@ -116,19 +146,17 @@ class UploadPicture(): ...@@ -116,19 +146,17 @@ class UploadPicture():
big_img_path = os.path.join(tmpdir, "thumbnail_image_big" + suffix) big_img_path = os.path.join(tmpdir, "thumbnail_image_big" + suffix)
img_240_path = os.path.join(tmpdir, "thumbnail_image_240" + suffix) img_240_path = os.path.join(tmpdir, "thumbnail_image_240" + suffix)
try: try:
print(target_path, os.path.join(tmpdir, big_img_path),'23333333333')
generate_thumbnail_big(target_path, os.path.join(tmpdir, big_img_path)) generate_thumbnail_big(target_path, os.path.join(tmpdir, big_img_path))
generate_thumbnail_240(target_path, os.path.join(tmpdir, img_240_path)) generate_thumbnail_240(target_path, os.path.join(tmpdir, img_240_path))
except Exception as ex: except Exception as ex:
print(f"文件{filename}生成缩略图失败:{traceback.format_exc()}") print(f"文件{filename}生成缩略图失败:",ex)
return False return False
pass pass
retry_time = 1 retry_time = 1
while retry_time < 3: while retry_time < 3:
url = 'http://wx.yswg.com.cn:8000/microservice-visual/visual/fileSystem/shutterstock_upload' url = 'http://wx.yswg.com.cn:8000/microservice-visual/visual/fileSystem/shutterstock_upload'
# url = "http://192.168.2.97:6661/microservice-visual/visual/fileSystem/shutterstock_upload"
# url = 'http://localhost:8600/visual/fileSystem/shutterstock_upload'
files = { files = {
'file': open(target_path, 'rb'), 'file': open(target_path, 'rb'),
'file_big': open(big_img_path, 'rb'), 'file_big': open(big_img_path, 'rb'),
...@@ -136,39 +164,43 @@ class UploadPicture(): ...@@ -136,39 +164,43 @@ class UploadPicture():
} }
body = {'token': "dacce869-0471-4ec7-ac50-3b3b1ec22c87"} body = {'token': "dacce869-0471-4ec7-ac50-3b3b1ec22c87"}
try: try:
response = requests.post(url, files=files, data=body, timeout=600) headers = {
"authorization": self.token
}
response = requests.post(url, files=files, data=body, timeout=600,headers=headers)
print('response:::',response.text)
response_data = json.loads(response.text) response_data = json.loads(response.text)
print('response_data::',response_data)
formatted_time = datetime.now().strftime('%Y/%m/%d %H:%M:%S') formatted_time = datetime.now().strftime('%Y/%m/%d %H:%M:%S')
if response_data.get('code') == 200: if response_data.get('code') == 200:
print(f"{filename} 上传成功 第{self.count}个,{formatted_time}") print(f"{filename} 上传成功 第{self.count}个,{formatted_time}")
exec_sql( exec_sql(
f"""update stock_image_detail_wj set upload_time=:upload_time,err_msg=:err_msg where image_id = :image_id""", f"""update stock_image_detail_wj set upload_time='{formatted_time}',err_msg='success' where image_id = '{img_id}'""")
{"upload_time": formatted_time, "err_msg": "success", "image_id": img_id})
self.count = self.count + 1 self.count = self.count + 1
success_flag = True success_flag = True
break break
else: else:
print(f"{filename} 上传失败 第{self.count}个 {formatted_time}: {response_data}") print(f"{filename} 上传失败 第{self.count}个 {formatted_time}: {response_data}")
errMsg = response_data.get("msg") errMsg = response_data.get("msg")
print('errMsg:::',errMsg)
if errMsg == '您上传的文件已经被上传过~!请勿重复上传!': if errMsg == '您上传的文件已经被上传过~!请勿重复上传!':
exec_sql( exec_sql(
f"""update stock_image_detail_wj set upload_time=:upload_time,err_msg=:err_msg where image_id = :image_id""", f"""update stock_image_detail_wj set upload_time='{formatted_time}',err_msg='success' where image_id ='img_id'""" )
{"upload_time": formatted_time, "err_msg": "success", "image_id": img_id})
self.count = self.count + 1 self.count = self.count + 1
success_flag = True success_flag = True
break break
else: else:
exec_sql( exec_sql(f"""update stock_image_detail_wj set upload_time='{formatted_time}',err_msg='{json.dumps(response_data)}' where image_id = '{img_id}'""")
f"""update stock_image_detail_wj set upload_time=:upload_time,err_msg=:err_msg where image_id = :image_id""",
{"upload_time": formatted_time, "err_msg": json.dumps(response_data), "image_id": img_id})
if "" in errMsg: if "" in errMsg:
# 此处超时等 # 此处超时等
pass pass
retry_time = retry_time + 1 retry_time = retry_time + 1
except Exception as e:
print('报错：33333',e)
self.get_microservice_token()
finally: finally:
# 关闭文件流 # 关闭文件流
for file in files.values(): for file in files.values():
...@@ -200,53 +232,51 @@ class UploadPicture(): ...@@ -200,53 +232,51 @@ class UploadPicture():
print(f"文件{file_name}不是正常规则的图片命名！！") print(f"文件{file_name}不是正常规则的图片命名！！")
continue continue
image_id_str = image_id_list[0] image_id_str = image_id_list[0]
print('upload_dir:',image_id_str)
file_path = os.path.join(folder_path, file_name) file_path = os.path.join(folder_path, file_name)
success_flag = self.upload_file(image_id_str, file_path) success_flag = self.upload_file(image_id_str, file_path)
if success_flag: if success_flag:
# 转移到上传成功的文件夹中 # 转移到上传成功的文件夹中
print('转移到上传成功的文件夹中')
shutil.move(file_path, os.path.join(all_pic_success_dir, file_name)) shutil.move(file_path, os.path.join(all_pic_success_dir, file_name))
pass pass
def run(self): def run(self):
folder_path_list = [ folder_path_list = [
"xiezhihao@yswg.com.cn",
"24016587887", 'xiezhihao@yswg.com.cn',
"daiting@yswg.com.cn", '24016587887',
"yangzinan@yswg.com.cn", 'daiting@yswg.com.cn',
"danqingmei30", 'yangzinan@yswg.com.cn',
"352422819@qq.com", 'danqingmei30',
"xuyuanlian@yswg.com.cn", '352422819@qq.com',
"496775268@qq.com", 'xuyuanlian@yswg.com.cn',
"878097683@qq.com", '496775268@qq.com',
"991450084@qq.com", '878097683@qq.com',
"yangxia@yswg.com.cn", '991450084@qq.com',
"zouxiayan@yswg.com.cn", 'yangxia@yswg.com.cn',
"laiyawen@yswg.com.cn", 'zouxiayan@yswg.com.cn',
"1060103503@qq.com", 'luhailin@yswg.com.cn',
"1120841348@qq.com", '1060103503@qq.com',
"1014703046@qq.com", '1120841348@qq.com',
"shuqian@yswg.com.cn", '1014703046@qq.com',
"wengdan300", 'shuqian@yswg.com.cn',
"jiangbin@yswg.com.cn", 'wengdan300',
"yangxuanwei54", 'jiangbin@yswg.com.cn',
"1196846022@QQ.COM", 'yangxuanwei54',
"liyun1@yswg.com.cn", '1196846022@QQ.COM',
"lizhili@yswg.com.cn", 'liulujun@yswg.com.cn',
"chenxiaowei@yswg.com.cn", 'fanglidong@yswg.com.cn',
"1101549094@qq.com", 'jiangyuan1@yswg.com.cn',
"liulujun@yswg.com.cn", 'xujing@yswg.com.cn',
"fanglidong@yswg.com.cn", 'lujun@yswg.com.cn',
"jiangyuan1@yswg.com.cn", 'liujia@yswg.com.cn',
"xujianbo@yswg.com.cn", 'weixiaoling@yswg.com.cn',
"xujing@yswg.com.cn", '2563838314@qq.com',
"lujun@yswg.com.cn",
"liujia@yswg.com.cn",
"weixiaoling@yswg.com.cn",
"1073463224",
] ]
for account_id in folder_path_list: for account_id in folder_path_list:
if 8 <= datetime.now().hour < 18: if 9 <= datetime.now().hour < 16:
print(f"当前时间 {datetime.now().hour} 点，超出运行时段 18:00–08:00，退出循环。") print(f"当前时间 {datetime.now().hour} 点，超出运行时段 18:00–08:00，退出循环。")
break break
print(f"当前时间 {datetime.now().hour} 点，开始处理账户 {account_id}。") print(f"当前时间 {datetime.now().hour} 点，开始处理账户 {account_id}。")
...@@ -255,10 +285,10 @@ class UploadPicture(): ...@@ -255,10 +285,10 @@ class UploadPicture():
else: else:
account_id_tmp = account_id account_id_tmp = account_id
folder_path = os.path.join(all_pic_dir, account_id_tmp) folder_path = os.path.join(all_pic_dir, account_id_tmp)
print('folder_path:::',folder_path)
# 带重试地执行上传目录 # 带重试地执行上传目录
success = False success = False
for attempt in range(3): # 最多重试3次 for attempt in range(1): # 最多重试3次
try: try:
self.upload_dir(folder_path) self.upload_dir(folder_path)
success = True success = True
......
wangjing_projects/projects/picture_material/stock_summery/new_stock/xnj_url.py
...@@ -435,9 +435,13 @@ class GetSS_details(): ...@@ -435,9 +435,13 @@ class GetSS_details():
sleep(randint(2, 4)) sleep(randint(2, 4))
# 查找并输入密码 # 查找并输入密码
print("正在等待密码输入框...") print("正在等待密码输入框...")# placeholder="密码"
email_input = iframe.ele( # MuiInputBase-input MuiOutlinedInput-input MuiInputBase-inputSizeSmall MuiInputBase-inputAdornedEnd css-18zhmu4
'.MuiInputBase-input MuiInput-input MuiInputBase-inputSizeSmall MuiInputBase-inputAdornedEnd css-186x7cf') try:
email_input = iframe.ele(
'xpath://input[@placeholder="密码"]')
except:
email_input = iframe.ele('.MuiInputBase-input MuiOutlinedInput-input MuiInputBase-inputSizeSmall MuiInputBase-inputAdornedEnd css-18zhmu4')
email_input.clear() # 清除任何预填充的内容 email_input.clear() # 清除任何预填充的内容
email_input.input(self.pwd) email_input.input(self.pwd)
print("已输入密码到密码输入框") print("已输入密码到密码输入框")
...@@ -509,7 +513,7 @@ class GetSS_details(): ...@@ -509,7 +513,7 @@ class GetSS_details():
} }
try: try:
response = requests.post(url, data=data_json,headers=headers) response = requests.post(url, data=data_json,headers=headers,timeout=120)
if response.status_code == 200: if response.status_code == 200:
return response.json() return response.json()
else: else:
...@@ -564,6 +568,7 @@ class GetSS_details(): ...@@ -564,6 +568,7 @@ class GetSS_details():
} }
response = requests.post('https://www.shutterstock.com/napi/licensees/current/redownload', cookies=cookie, response = requests.post('https://www.shutterstock.com/napi/licensees/current/redownload', cookies=cookie,
headers=self.headers, json=json_data, timeout=600) headers=self.headers, json=json_data, timeout=600)
print('下载返回:',response.text)
image_url = json.loads(response.text)['meta']['licensedContent'][0]['downloadUrl'] image_url = json.loads(response.text)['meta']['licensedContent'][0]['downloadUrl']
return image_url return image_url
...@@ -575,7 +580,7 @@ class GetSS_details(): ...@@ -575,7 +580,7 @@ class GetSS_details():
try: try:
start_time = datetime.now().strftime("%m-%d %H:%M:%S") start_time = datetime.now().strftime("%m-%d %H:%M:%S")
all_image_id = Con.get_all_image_id() all_image_id = Con.get_all_image_id()
# 12月的重新上传。先注释
if str(image_id) in set(all_image_id): if str(image_id) in set(all_image_id):
print(f'{image_id}已上传过') print(f'{image_id}已上传过')
state = 3 state = 3
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment