Merge branch '代码优化/11018-企业文化项目引入分片上传' into 'master'

代码优化/11018 企业文化项目引入分片上传

See merge request !20

package.json

@@ -23,6 +23,7 @@
   },
   "dependencies": {
     "@element-plus/icons-vue": "^2.3.2",
+    "@types/crypto-js": "^4.2.2",
     "@vueuse/components": "^14.0.0",
     "@vueuse/core": "^14.0.0",
     "@wangeditor/editor": "^5.1.23",
@@ -30,6 +31,7 @@
     "@wecom/jssdk": "^2.3.3",
     "archiver": "^7.0.1",
     "axios": "^1.13.0",
+    "crypto-js": "^4.2.0",
     "dayjs": "^1.11.19",
     "element-plus": "^2.11.5",
     "inquirer": "^13.0.2",

pnpm-lock.yaml

@@ -11,6 +11,9 @@ importers:
       '@element-plus/icons-vue':
         specifier: ^2.3.2
         version: 2.3.2(vue@3.5.22(typescript@5.9.3))
+      '@types/crypto-js':
+        specifier: ^4.2.2
+        version: 4.2.2
       '@vueuse/components':
         specifier: ^14.0.0
         version: 14.0.0(vue@3.5.22(typescript@5.9.3))
@@ -32,6 +35,9 @@ importers:
       axios:
         specifier: ^1.13.0
         version: 1.13.0
+      crypto-js:
+        specifier: ^4.2.0
+        version: 4.2.0
       dayjs:
         specifier: ^1.11.19
         version: 1.11.19
@@ -928,6 +934,9 @@ packages:
   '@tsconfig/node22@22.0.2':
     resolution: {integrity: sha512-Kmwj4u8sDRDrMYRoN9FDEcXD8UpBSaPQQ24Gz+Gamqfm7xxn+GBR7ge/Z7pK8OXNGyUzbSwJj+TH6B+DS/epyA==}
 
+  '@types/crypto-js@4.2.2':
+    resolution: {integrity: sha512-sDOLlVbHhXpAUAL0YHDUUwDZf3iN4Bwi4W6a0W0b+QcAezUbRtH4FVb+9J4h+XFPW7l/gQ9F8qC7P+Ec4k8QVQ==}
+
   '@types/estree@1.0.8':
     resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==}
 
@@ -1723,6 +1732,9 @@ packages:
     resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==}
     engines: {node: '>= 8'}
 
+  crypto-js@4.2.0:
+    resolution: {integrity: sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==}
+
   css-select@4.3.0:
     resolution: {integrity: sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ==}
 
@@ -4730,6 +4742,8 @@ snapshots:
 
   '@tsconfig/node22@22.0.2': {}
 
+  '@types/crypto-js@4.2.2': {}
+
   '@types/estree@1.0.8': {}
 
   '@types/event-emitter@0.3.5': {}
@@ -5740,6 +5754,8 @@ snapshots:
       shebang-command: 2.0.0
       which: 2.0.2
 
+  crypto-js@4.2.0: {}
+
   css-select@4.3.0:
     dependencies:
       boolbase: 1.0.0

src/api/common/index.ts

-// 常规的接口
-// import service from '@/utils/request/index'
-// import type { FielItemDto } from './types'
-/**
- * 获取常规的接口
- */
-// export const uploadFile = (file: File, onProgress?: (progress: number) => void) => {
-//   const formData = new FormData()
-//   formData.append('file', file)
-//   return service.request<FielItemDto>({
-//     url: '/mobiles/file-upload/singleUpload',
-//     method: 'POST',
-//     data: formData,
-//     onUploadProgress: (progressEvent) => {
-//       const percentCompleted = Math.round((progressEvent.loaded * 100) / (progressEvent.total || 1))
-//       onProgress?.(percentCompleted)
-//     },
-//   })
-// }
-
-/**
- * 暂时调用oa正式接口
- */
 import axios, { type AxiosRequestConfig } from 'axios'
+import CryptoJS from 'crypto-js'
+
 type UploadFileResponseItem = {
   createTime: string
   createUser: number
@@ -37,7 +16,217 @@ type UploadFileResponseItem = {
   updateUser: string
 }
 
-// 单个文件上传
+type ChunkCheckResponse = {
+  status: number
+  fileUrl?: string
+  chunkNumbers?: number[]
+}
+
+const OA_UPLOAD_BASE_URL = 'http://47.112.96.71:8082'
+const CHUNK_UPLOAD_THRESHOLD = 10 * 1024 * 1024
+const CHUNK_SIZE = 5 * 1024 * 1024
+const MAX_CONCURRENT_UPLOADS = 3
+
+function hashBufferMD5(buffer: ArrayBuffer): string {
+  const wordArray = CryptoJS.lib.WordArray.create(new Uint8Array(buffer))
+  return CryptoJS.MD5(wordArray).toString(CryptoJS.enc.Hex)
+}
+
+function readSlice(file: File, start: number, end: number): Promise<ArrayBuffer> {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader()
+    reader.onload = (event) => resolve(event.target?.result as ArrayBuffer)
+    reader.onerror = () => reject(reader.error ?? new Error('读取文件分片失败'))
+    reader.readAsArrayBuffer(file.slice(start, end))
+  })
+}
+
+// 返回一个由文件名 + 头部 + 尾部 + 中间抽样共同生成的综合指纹 用于检查文件是否存在 hash不耗事件 主要等待异步io耗时间
+// 4GB的视频耗时30s大概
+async function fileHashHeadTailMiddle(file: File, chunkSize = 1024 * 1024) {
+  const fileSize = file.size
+  const finalHashes: string[] = []
+
+  // 对文件名做md5加密
+  const fileName = file.name.split('_').slice(0, -1).join('_') || file.name
+  finalHashes.push(CryptoJS.MD5(fileName).toString(CryptoJS.enc.Hex))
+
+  // 读取文件头做md5加密
+  const headBuffer = await readSlice(file, 0, Math.min(chunkSize, fileSize))
+  finalHashes.push(hashBufferMD5(headBuffer))
+
+  // 读取文件尾做md5加密
+  const tailSize = Math.min(chunkSize, fileSize)
+  const tailBuffer = await readSlice(file, fileSize - tailSize, fileSize)
+  finalHashes.push(hashBufferMD5(tailBuffer))
+
+  // 循环读取文件中间的部分 然后部分做md5加密
+  if (fileSize > 2 * chunkSize) {
+    let offset = chunkSize
+
+    while (offset < fileSize - chunkSize) {
+      const size = Math.min(chunkSize, fileSize - chunkSize - offset)
+      const buffer = await readSlice(file, offset, offset + size)
+      const u8 = new Uint8Array(buffer)
+      const piece = new Uint8Array(10)
+
+      // 取前5个字节和后5个字节做md5加密 计算时间比较短
+      piece.set(u8.slice(0, Math.min(5, u8.length)), 0)
+      piece.set(u8.slice(Math.max(u8.length - 5, 0)), 5)
+      finalHashes.push(hashBufferMD5(piece.buffer))
+      offset += size
+    }
+  }
+
+  // 将所有md5加密后的结果拼接起来 然后做md5加密
+  return CryptoJS.MD5(finalHashes.join('')).toString(CryptoJS.enc.Hex)
+}
+
+// 将文件进行分片 返回一个Blob数组
+function createChunks(file: File, chunkSize: number) {
+  const chunks: Blob[] = []
+  let start = 0
+
+  while (start < file.size) {
+    const end = Math.min(start + chunkSize, file.size)
+    chunks.push(file.slice(start, end))
+    start = end
+  }
+
+  return chunks
+}
+
+// 分片上传文件
+/**
+ * 第 1 步：给文件生成一个 hash
+ * 第 2 步：调用 check 询问服务端当前状态
+ * 第 3 步：根据 check 结果决定是否秒传 或者 断点续传 或者 正常分片上传
+ * 第 4 步：把大文件切成多个分片
+ * 第 5 步：创建多个并发 worker，每个 worker 循环领取分片任务 进行分片上传
+ * 第 6 步：所有分片都上传完成后，通知服务端合并
+ * 第 7 步：如果文件存在，则合并文件
+ */
+async function uploadFileByChunks(
+  file: File,
+  options: {
+    onProgress?: (progress: number) => void
+  },
+  signal: AbortSignal,
+): Promise<UploadFileResponseItem> {
+  const { onProgress } = options
+  const fileHash = await fileHashHeadTailMiddle(file)
+  const fileSuffix = file.name.includes('.') ? file.name.slice(file.name.lastIndexOf('.')) : ''
+
+  const checkResponse = await axios.post<{ data: ChunkCheckResponse }>(
+    `${OA_UPLOAD_BASE_URL}/mobiles/file-upload/check`,
+    {
+      hash: fileHash,
+      fileName: file.name,
+      fileSize: file.size,
+      fileSuffx: fileSuffix,
+    },
+    { signal },
+  )
+
+  // 秒传服务检查文件是否存在
+  const checkData = checkResponse.data.data
+  if (checkData.status === 200 && checkData.fileUrl) {
+    onProgress?.(100)
+    return {
+      fileId: fileHash,
+      fileName: file.name,
+      filePath: checkData.fileUrl,
+      finalName: file.name,
+    } as UploadFileResponseItem
+  }
+
+  const chunks = createChunks(file, CHUNK_SIZE)
+  const totalChunks = chunks.length
+
+  // 断点续传 记录已经上传的片数
+  const uploadedSet = new Set<number>(
+    Array.isArray(checkData.chunkNumbers)
+      ? checkData.chunkNumbers
+          .map((item) => Number(item))
+          .filter((item) => Number.isInteger(item) && item >= 0)
+      : [],
+  )
+
+  const updateProgress = () => {
+    const progress = totalChunks > 0 ? Math.floor((uploadedSet.size / totalChunks) * 100) : 0
+    onProgress?.(progress)
+  }
+
+  // 有可能部分已经上传过了直接更新一次上传进度
+  updateProgress()
+
+  // 上传单个分片
+  const uploadChunk = async (chunk: Blob, index: number) => {
+    const formData = new FormData()
+    formData.append('fileId', fileHash)
+    formData.append('filePart', chunk, file.name)
+    formData.append('chunkNumber', String(index))
+
+    await axios.post(`${OA_UPLOAD_BASE_URL}/mobiles/file-upload/chunk`, formData, {
+      signal,
+      headers: {
+        'Content-Type': 'multipart/form-data',
+      },
+    })
+
+    uploadedSet.add(index)
+    updateProgress()
+  }
+
+  let nextIndex = 0
+  const worker = async () => {
+    // 遍历所有分片 如果已经上传过了就跳过 否则上传
+    // 相当于开了 3个while死循环 同时上传文件  只有等3个线程都上传完了 才结束
+    // 总有某个 worker 在自己的异步任务完成后恢复执行，然后继续进入下一轮 while，领取新的分片任务。
+
+    while (nextIndex < totalChunks) {
+      const currentIndex = nextIndex
+      nextIndex += 1
+
+      if (uploadedSet.has(currentIndex)) continue
+
+      await uploadChunk(chunks[currentIndex]!, currentIndex)
+    }
+  }
+
+  //  同时开了 3 个异步 worker，每个 worker 里面都有一个 while 循环，不断去“领取下一个还没处理的分片”。
+  await Promise.all(
+    Array.from({ length: Math.min(MAX_CONCURRENT_UPLOADS, totalChunks) }, () => worker()),
+  )
+
+  const finishFormData = new FormData()
+  finishFormData.set('fileName', file.name)
+  finishFormData.set('fileId', fileHash)
+  finishFormData.set('chunkTotalCount', String(totalChunks))
+
+  // 上传完成 通知后端文件上传完成
+  const finishResponse = await axios.post<{ data: { fileUrl: string } }>(
+    `${OA_UPLOAD_BASE_URL}/mobiles/file-upload/finish`,
+    finishFormData,
+    {
+      signal,
+      headers: {
+        'Content-Type': 'multipart/form-data',
+      },
+    },
+  )
+
+  onProgress?.(100)
+
+  return {
+    fileId: fileHash,
+    fileName: file.name,
+    filePath: finishResponse.data.data.fileUrl,
+    finalName: file.name,
+  } as UploadFileResponseItem
+}
+
+// 上传文件 10mb以上用nas分片 10mb以下正常上传
 export const uploadFile = (
   file: File,
   options: {
@@ -47,29 +236,39 @@ export const uploadFile = (
   cancel: () => void
   promise: Promise<UploadFileResponseItem>
 } => {
+  const { onProgress } = options
+  const controller = new AbortController()
+  const isChunkUpload = file.size > CHUNK_UPLOAD_THRESHOLD
+
+  const promise = isChunkUpload
+    ? uploadFileByChunks(file, options, controller.signal)
+    : (() => {
+        // 用IIFE包裹直接返回promise
         const formData = new FormData()
         formData.append('fileList', file)
 
-  const { onProgress } = options
-
-  const controller = new AbortController()
-  const axiosOptions: AxiosRequestConfig = {
+        const axiosOptions: AxiosRequestConfig<FormData> = {
           signal: controller.signal,
         }
 
         if (onProgress) {
           axiosOptions.onUploadProgress = (progressEvent) => {
-      const percentCompleted = Math.round((progressEvent.loaded * 100) / (progressEvent.total || 1))
-      onProgress?.(percentCompleted)
+            const percentCompleted = Math.round(
+              (progressEvent.loaded * 100) / (progressEvent.total || 1),
+            )
+            onProgress(percentCompleted)
           }
         }
 
+        return axios
+          .post(`${OA_UPLOAD_BASE_URL}/mobiles/uploadFile`, formData, axiosOptions)
+          .then((data) => data.data.data[0] as UploadFileResponseItem)
+      })()
+
   return {
     cancel: () => {
       controller.abort()
     },
-    promise: axios
-      .post('http://47.112.96.71:8082/mobiles/uploadFile', formData, axiosOptions)
-      .then((data) => data.data.data[0] as UploadFileResponseItem),
+    promise,
   }
 }

src/components/common/UploadVideo/index.vue

@@ -157,18 +157,17 @@ const startUpload = async () => {
     uploadError.value = ''
     uploadProgress.value = 0
 
-    // 调用你的上传方法
-
+    // 先提案获取视频的原信息 用本地blob
+    const blob = URL.createObjectURL(currentFile.value)
+    const metadataPromise = getVideoMetadata(blob)
     const { promise, cancel } = uploadFileApi(currentFile.value, {
       onProgress: (progress) => {
         uploadProgress.value = progress
       },
     })
     cancelUploadController = cancel
-    const data = await promise
-
-    // 获取视频元数据
-    const metadata = await getVideoMetadata(data.filePath)
+    // 并行等待
+    const [data, metadata] = await Promise.all([promise, metadataPromise])
 
     // 根据你的 API 返回结构调整
     const videoData: VideoInfo = {

src/views/publishVideo/index.vue

@@ -309,11 +309,12 @@
       width="800px"
       :close-on-click-modal="false"
     >
-      <!-- 这个用blob是因为 转为canvas 防止跨域污染 -->
+      <!--  转为canvas 防止跨域污染  crossorigin="anonymous" -->
       <div v-if="locationVideoBlolUrl" class="space-y-6">
         <!-- 视频预览 -->
         <div class="relative">
           <video
+            crossorigin="anonymous"
             ref="videoRef"
             :src="locationVideoBlolUrl"
             class="w-full max-h-96 rounded-lg bg-black"
@@ -389,7 +390,7 @@ import SelectTags from '@/components/common/SelectTags/index.vue'
 import type { TagItemDto, AddOrUpdateVideoDto } from '@/api'
 import { useVideoStore, useUserStore } from '@/stores'
 import { storeToRefs } from 'pinia'
-import { selectDepOrUser, fetchOssBlob } from '@/utils'
+import { selectDepOrUser } from '@/utils'
 import BackButton from '@/components/common/BackButton/index.vue'
 import { push } from 'notivue'
 
@@ -627,19 +628,13 @@ const handleSubmit = async (releaseStatus: ReleaseStatusTypeEnum) => {
 }
 
 const handleVideoChange = ({
-  file,
   videoDuration,
 }: {
   file: File
   url: string
   videoDuration: string
 }) => {
-  if (locationVideoBlolUrl.value) {
-    URL.revokeObjectURL(locationVideoBlolUrl.value)
-  }
-  if (file) {
-    locationVideoBlolUrl.value = URL.createObjectURL(file)
-  }
+  locationVideoBlolUrl.value = form.value.videoUrl
   form.value.videoDuration = videoDuration
 }
 
@@ -662,8 +657,9 @@ onDeactivated(() => {
 onActivated(async () => {
   if (route.query.id) {
     const { data } = await getArticleDetail(route.query.id as string)
-    const blobUrl = await fetchOssBlob(data.videoUrl)
-    locationVideoBlolUrl.value = blobUrl
+
+    locationVideoBlolUrl.value = data.videoUrl
+
     form.value = {
       ...form.value,
       ...data,