分布式文件系统ok,fastdfs搞定

build.gradle

@@ -27,7 +27,7 @@ buildscript {
         dubbo_starter_version = '0.1.0'
         dubbo_version = "2.6.1"
         jsqlparser_version = '1.2'
-
+        fastdfs_client_version = '1.26.2'
     }
     repositories {
         mavenCentral()
@@ -160,6 +160,9 @@ subprojects {
         // sql parser
         compile("com.github.jsqlparser:jsqlparser:$jsqlparser_version")
 
+        // fastdfs
+        compile("com.github.tobato:fastdfs-client:$fastdfs_client_version")
+        
         ext.jarTree = fileTree(dir: 'libs', include: '**/*.jar')
         compile jarTree
     }

zen-api/src/main/kotlin/com/gxzc/zen/api/sys/mapper/SysUploadInfoMapper.kt

@@ -1,17 +0,0 @@
-package com.gxzc.zen.api.sys.mapper
-
-import com.gxzc.zen.api.sys.model.SysUploadInfo
-import com.gxzc.zen.common.base.BaseMapper
-import org.springframework.stereotype.Repository
-/**
- * <p>
- * 上传文件信息表 Mapper 接口
- * </p>
- *
- * @author NorthLan
- * @since 2018-06-01
- */
-@Repository
-interface SysUploadInfoMapper : BaseMapper<SysUploadInfo>{
-    fun test()
-}

zen-api/src/main/kotlin/com/gxzc/zen/api/sys/model/SysUploadInfo.kt

@@ -19,10 +19,6 @@ open class SysUploadInfo : BaseModel() {
      * 当前虚拟文件名
      */
     var filename: String? = null
-    /**
-     * 真实文件名
-     */
-    var realFilename: String? = null
     /**
      * 原始文件名
      */
@@ -31,10 +27,6 @@ open class SysUploadInfo : BaseModel() {
      * 虚拟相对路径
      */
     var relativePath: String? = null
-    /**
-     * 上传批次ID
-     */
-    var batchId: String? = null
     /**
      * 最后一次修改时间
      */
@@ -51,6 +43,14 @@ open class SysUploadInfo : BaseModel() {
      * 文件大小(byte)
      */
     var filesize: Long? = null
+    /**
+     * 存储group位
+     */
+    var group: String? = null
+    /**
+     * 保存的path
+     */
+    var path: String? = null
     /**
      * 备用字段
      */

zen-api/src/main/kotlin/com/gxzc/zen/api/sys/service/ISysUploadInfoService.kt

@@ -1,46 +0,0 @@
-package com.gxzc.zen.api.sys.service
-
-import com.gxzc.zen.api.sys.model.SysUploadInfo
-import com.gxzc.zen.common.base.BaseService
-import com.gxzc.zen.common.dto.ZenFileMetadata
-import java.io.File
-
-/**
- * <p>
- * 上传文件信息表 服务类
- * </p>
- *
- * @author NorthLan
- * @since 2018-06-01
- */
-interface ISysUploadInfoService : BaseService<SysUploadInfo> {
-    /**
-     * 添加文件数据
-     */
-    fun addUploadFileInfo(fileMetadata: ZenFileMetadata, file: File)
-
-    /**
-     * 获取某上传批次所有文件数据
-     */
-    fun getUploadInfosByBatchId(batchId: String): MutableList<SysUploadInfo>
-
-    /**
-     * 通过id获取单条 文件数据
-     */
-    fun getUploadInfoById(id: Long): SysUploadInfo?
-
-    /**
-     * 通过ids获取 文件数据 列表
-     */
-    fun getUploadInfoListByIdList(ids: Collection<Long>): MutableList<SysUploadInfo>
-
-    /**
-     * 通过id修改文件信息
-     */
-    fun modifyFileInfo(entity: SysUploadInfo): SysUploadInfo
-
-    /**
-     * 通过id批量修改文件信息
-     */
-    fun modifyBatchFileInfo(entityList: MutableList<SysUploadInfo>): MutableList<SysUploadInfo>
-}

zen-api/src/main/kotlin/com/gxzc/zen/api/sys/service/impl/SysUploadInfoServiceImpl.kt

@@ -1,91 +0,0 @@
-package com.gxzc.zen.api.sys.service.impl
-
-import com.baomidou.mybatisplus.mapper.EntityWrapper
-import com.baomidou.mybatisplus.service.impl.ServiceImpl
-import com.gxzc.zen.api.sys.mapper.SysUploadInfoMapper
-import com.gxzc.zen.api.sys.model.SysUploadInfo
-import com.gxzc.zen.api.sys.service.ISysUploadInfoService
-import com.gxzc.zen.common.dto.ZenFileMetadata
-import org.apache.commons.io.FilenameUtils
-import org.springframework.stereotype.Service
-import java.io.File
-import java.util.*
-
-/**
- * <p>
- * 上传文件信息表 服务实现类
- * </p>
- *
- * @author NorthLan
- * @since 2018-06-01
- */
-@Service
-class SysUploadInfoServiceImpl : ServiceImpl<SysUploadInfoMapper, SysUploadInfo>(), ISysUploadInfoService {
-
-    override fun addUploadFileInfo(fileMetadata: ZenFileMetadata, file: File) {
-        val entity = SysUploadInfo().apply {
-            // filename
-            this.filename = if (fileMetadata.rename.isNullOrEmpty()) {
-                fileMetadata.filename
-            } else {
-                "${FilenameUtils.removeExtension(fileMetadata.rename)}.${FilenameUtils.getExtension(fileMetadata.filename)}"
-            }
-            this.realFilename = file.name
-            this.originFilename = fileMetadata.filename
-            // infomation
-            this.relativePath = if (fileMetadata.repath.isNullOrEmpty()) fileMetadata.relativePath?.replace(this.originFilename!!, "") else fileMetadata.repath
-            this.batchId = fileMetadata.batchId
-            this.lastModified = Date(file.lastModified())
-            this.md5 = fileMetadata.md5
-            this.extension = FilenameUtils.getExtension(file.name)
-            this.filesize = fileMetadata.totalSize
-        }
-        baseMapper.insert(entity)
-    }
-
-    override fun getUploadInfosByBatchId(batchId: String): MutableList<SysUploadInfo> {
-        val condition = SysUploadInfo().apply { this.batchId = batchId }
-        val ret = baseMapper.selectWOLogic(EntityWrapper(condition))
-        ret.forEach {
-            generateURL(it)
-        }
-        return ret
-    }
-
-    private fun generateURL(it: SysUploadInfo?) {
-        if (it != null) {
-            it.url = "/${it.realFilename}"
-//            if (it.relativePath.isNullOrEmpty()) {
-//                it.url = FilenameUtils.normalize("/${it.filename}")
-//            } else {
-//                it.url = FilenameUtils.normalize("/${it.relativePath}/${it.filename}")
-//            }
-        }
-    }
-
-    override fun getUploadInfoById(id: Long): SysUploadInfo? {
-        val condition = SysUploadInfo().apply { this.id = id }
-        return baseMapper.selectOne(condition).apply {
-            generateURL(this)
-        }
-    }
-
-    override fun getUploadInfoListByIdList(ids: Collection<Long>): MutableList<SysUploadInfo> {
-        val ret = baseMapper.selectWOLogic(EntityWrapper<SysUploadInfo>().`in`("id", ids))
-        ret.forEach {
-            generateURL(it)
-        }
-        return ret
-    }
-
-    override fun modifyFileInfo(entity: SysUploadInfo): SysUploadInfo {
-        this.updateById(entity)
-        return getUploadInfoById(entity.id!!)!!
-    }
-
-    override fun modifyBatchFileInfo(entityList: MutableList<SysUploadInfo>): MutableList<SysUploadInfo> {
-        this.updateBatchById(entityList)
-        return getUploadInfoListByIdList(entityList.map { it.id!! }.toList())
-    }
-
-}

zen-api/src/main/kotlin/com/gxzc/zen/rpc/api/sys/RUploadService.kt

@@ -1,7 +1,7 @@
 package com.gxzc.zen.rpc.api.sys
 
 import com.gxzc.zen.api.sys.model.SysUploadInfo
-import com.gxzc.zen.common.dto.ZenFileMetadata
+import com.gxzc.zen.common.util.upload.ZenFileMetadata
 import com.gxzc.zen.rpc.model.sys.RUploadInfo
 import java.io.File
 

zen-api/src/main/kotlin/com/gxzc/zen/umps/config/ShiroConfig.kt

@@ -88,6 +88,7 @@ class ShiroConfig {
                     "/auth/logout" to "logout", // 登出
                     "/test/**" to "canon", // 测试 免登录
                     "/upload/**" to "canon", // 上传免登录
+                    "/fdfsupload/**" to "canon", // 上传免登录
                     "/api/**" to "canon", // api 免登陆
                     "/ws/**" to "canon", // websocket 免登陆
                     ////////////////////// 静态资源 /////////////////////

zen-api/src/main/resources/mapping/sys/SysUploadInfoMapper.xml

@@ -1,31 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
-<mapper namespace="com.gxzc.zen.api.sys.mapper.SysUploadInfoMapper">
-
-    <!-- 通用查询映射结果 -->
-    <resultMap id="BaseResultMap" type="com.gxzc.zen.api.sys.model.SysUploadInfo">
-    <result column="id" property="id" />
-    <result column="enable" property="enable" />
-    <result column="remark" property="remark" />
-    <result column="create_time" property="createTime" />
-    <result column="create_by" property="createBy" />
-    <result column="update_time" property="updateTime" />
-    <result column="update_by" property="updateBy" />
-        <result column="filename" property="filename" />
-        <result column="real_filename" property="realFilename" />
-        <result column="origin_filename" property="originFilename" />
-        <result column="relative_path" property="relativePath" />
-        <result column="batch_id" property="batchId" />
-        <result column="last_modified" property="lastModified" />
-        <result column="md5" property="md5" />
-        <result column="extension" property="extension" />
-        <result column="filesize" property="filesize" />
-        <result column="ext1" property="ext1" />
-        <result column="ext2" property="ext2" />
-        <result column="ext3" property="ext3" />
-        <result column="ext4" property="ext4" />
-    </resultMap>
-<select id="test">
-    select * from sys_upload_info
-</select>
-</mapper>

zen-api/src/test/kotlin/com/gxzc/zen/api/TestCollectionUtil.kt

@@ -9,6 +9,7 @@ import org.junit.Test
  * @date 2018/5/4
  * @url https://noahlan.com
  */
+@Suppress("UNUSED_VARIABLE")
 class TestCollectionUtil {
     @Test
     fun testJiao() {

zen-core/src/main/kotlin/com/gxzc/zen/common/config/FdfsConfig.kt

@@ -0,0 +1,21 @@
+package com.gxzc.zen.common.config
+
+import com.github.tobato.fastdfs.FdfsClientConfig
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
+import org.springframework.context.annotation.Configuration
+import org.springframework.context.annotation.EnableMBeanExport
+import org.springframework.context.annotation.Import
+import org.springframework.jmx.support.RegistrationPolicy
+
+/**
+ *
+ * @author NorthLan
+ * @date 2018/8/29
+ * @url https://noahlan.com
+ */
+@Configuration
+@Import(FdfsClientConfig::class)
+@ConditionalOnProperty(prefix = "fdfs", name = ["enabled"], havingValue = "true")
+@EnableMBeanExport(registration = RegistrationPolicy.IGNORE_EXISTING)
+class FdfsConfig {
+}

zen-core/src/main/kotlin/com/gxzc/zen/common/dto/UploadFileInfo.kt

@@ -1,52 +0,0 @@
-package com.gxzc.zen.common.dto
-
-import java.io.Serializable
-import java.util.*
-
-/**
- * 上传的文件信息(本地)
- * @author NorthLan
- * @date 2018/7/3
- * @url https://noahlan.com
- */
-open class UploadFileInfo : Serializable {
-    companion object {
-        private const val serialVersionUID = 100000000000002L
-    }
-
-    /**
-     * 当前虚拟文件名
-     */
-    var filename: String? = null
-    /**
-     * 真实文件名
-     */
-    var realFilename: String? = null
-    /**
-     * 原始文件名
-     */
-    var originFilename: String? = null
-    /**
-     * 虚拟相对路径
-     */
-    var relativePath: String? = null
-    /**
-     * 最后一次修改时间
-     */
-    var lastModified: Date? = null
-    /**
-     * 文件非完整md5(头+[尾]+修改时间)
-     */
-    var md5: String? = null
-    /**
-     * 后缀名
-     */
-    var extension: String? = null
-    /**
-     * 文件大小(byte)
-     */
-    var filesize: Long? = null
-
-    ////////////////////// url
-    var url: String? = null
-}

zen-core/src/main/kotlin/com/gxzc/zen/common/exception/ZenExceptionEnum.kt

@@ -25,8 +25,9 @@ enum class ZenExceptionEnum(val code: Int, val msg: String) {
      * 文件上传
      */
     FILE_READING_ERROR(400, "FILE_READING_ERROR!"),
-    FILE_NOT_FOUND(400, "FILE_NOT_FOUND!"),
-    FILE_METADATA_VALIDATE_ERROR(401, "文件metadata验证失败"),
+    FILE_NOT_FOUND(401, "FILE_NOT_FOUND!"),
+    FILE_METADATA_VALIDATE_ERROR(402, "文件metadata验证失败"),
+    FILE_FRAG_UPLOAD_FAILURE(403, "文件分片上传失败"),
 
     /**
      * 错误的请求

zen-core/src/main/kotlin/com/gxzc/zen/common/util/KeyLock.kt

@@ -0,0 +1,85 @@
+package com.gxzc.zen.common.util
+
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.Semaphore
+
+/**
+ * 细粒度key锁
+ * 高并发 维护一个ThreadLocal保证每个线程锁定计数一致
+ * 可重入 ReentrantLock(同等),因为使用了ConcurrentHashMap
+ * 公平锁 (竞态状态 须排队)
+ * @author NorthLan
+ * @date 2018/8/30
+ * @url https://noahlan.com
+ */
+open class KeyLock<K> {
+    private val map = ConcurrentHashMap<K, Semaphore>()
+    private val local = object : ThreadLocal<MutableMap<K, LockInfo>>() {
+        override fun initialValue(): MutableMap<K, LockInfo> {
+            return mutableMapOf()
+        }
+    }
+
+    /**
+     * 锁定key, 其他需要使用此key的线程进入等待,直到释放锁 {@link #unlock(K)} <br>
+     * 使用hashcode和equals来判断key是否相同,因此key必须实现{@link #hashCode()}和{@link #equals(Object)}方法
+     */
+    fun lock(key: K?) {
+        if (key == null) {
+            return
+        }
+        val info = local.get()[key]
+        if (info == null) {
+            val current = Semaphore(1, false).apply {
+                acquireUninterruptibly()
+            }
+            map.put(key, current)?.apply {
+                acquireUninterruptibly()
+            }
+            local.get()[key] = LockInfo(current)
+        } else {
+            ++info.lockCount
+        }
+    }
+
+    /**
+     * 释放key锁
+     */
+    fun unlock(key: K?) {
+        if (key == null) {
+            return
+        }
+        val info = local.get()[key]
+        if (info != null && --info.lockCount == 0) {
+            info.current.release()
+            map.remove(key, info.current)
+            local.get().remove(key)
+        }
+    }
+
+    /**
+     * 锁定多个key
+     * 建议在调用此方法前先对keys进行排序,使用相同的锁定顺序,防止死锁发生
+     */
+    fun lock(keys: Array<K>?) {
+        keys?.forEach {
+            lock(it)
+        }
+    }
+
+    /**
+     * 释放多个key
+     */
+    fun unlock(keys: Array<K>?) {
+        keys?.forEach {
+            unlock(it)
+        }
+    }
+
+    /**
+     * 锁信息
+     */
+    private class LockInfo(var current: Semaphore) {
+        var lockCount: Int = 1
+    }
+}

zen-core/src/main/kotlin/com/gxzc/zen/common/util/upload/FdfsFile.kt

@@ -0,0 +1,9 @@
+package com.gxzc.zen.common.util.upload
+
+/**
+ *
+ * @author NorthLan
+ * @date 2018/8/30
+ * @url https://noahlan.com
+ */
+open class FdfsFile(val group: String, val path: String)

zen-core/src/main/kotlin/com/gxzc/zen/common/util/upload/FdfsUploadUtil.kt

@@ -0,0 +1,308 @@
+package com.gxzc.zen.common.util.upload
+
+import com.github.tobato.fastdfs.domain.StorePath
+import com.github.tobato.fastdfs.exception.FdfsServerException
+import com.github.tobato.fastdfs.service.AppendFileStorageClient
+import com.github.tobato.fastdfs.service.FastFileStorageClient
+import com.gxzc.zen.common.exception.ZenException
+import com.gxzc.zen.common.exception.ZenExceptionEnum
+import com.gxzc.zen.common.properties.UploadProperties
+import com.gxzc.zen.common.util.KeyLock
+import com.gxzc.zen.common.util.SpringContextHolder
+import com.gxzc.zen.common.util.upload.constants.CheckStatus
+import com.gxzc.zen.common.util.upload.constants.UploadStatus
+import com.gxzc.zen.common.util.upload.result.CheckResult
+import com.gxzc.zen.common.util.upload.result.UploadResult
+import org.apache.commons.io.FilenameUtils
+import org.slf4j.LoggerFactory
+import org.springframework.web.multipart.MultipartFile
+import java.io.BufferedInputStream
+import java.io.File
+import java.nio.file.Files
+import java.nio.file.Paths
+
+/**
+ *
+ * @author NorthLan
+ * @date 2018/8/29
+ * @url https://noahlan.com
+ */
+object FdfsUploadUtil {
+    private val logger = LoggerFactory.getLogger(FdfsUploadUtil::class.java)
+
+    private var uploadProperties = SpringContextHolder.getBean(UploadProperties::class.java)
+        get() {
+            if (field == null) {
+                field = SpringContextHolder.getBean(UploadProperties::class.java)
+            }
+            return field
+        }
+
+    private val appendFileStorageClient: AppendFileStorageClient by lazy { SpringContextHolder.getBean(AppendFileStorageClient::class.java)!! }
+    private val fastFileStorageClient: FastFileStorageClient by lazy { SpringContextHolder.getBean(FastFileStorageClient::class.java)!! }
+
+    // 针对文件md5加锁,保证上传合并文件时线程安全
+    private val checkMergeLock: KeyLock<Int> = KeyLock()
+    private val mergeLock: KeyLock<Int> = KeyLock()
+    private val uploadLock: KeyLock<Int> = KeyLock()
+    //    private val mergeLock = ReentrantLock(true)
+    /**
+     * 检查文件是否存在,实现文件秒传
+     * 通过MD5查询数据库是否有记录
+     *     有记录: 查询真实文件是否存在
+     *         真实文件存在: 秒传
+     *         真实文件不存在: 数据库记录清理
+     *     无记录: 文件不存在,查询分片存在情况
+     * 状态:
+     * 1. 有数据库记录,有文件
+     * 2. 有数据库记录,无文件
+     * 3. 无数据库记录,文件分片不完整
+     * 4. 无数据库记录,文件分片完整
+     */
+    fun check(fdfsFile: FdfsFile?, fileMetadata: ZenFileMetadata): CheckResult {
+        val ret = CheckResult()
+        if (validateRequest(fileMetadata, null)) {
+            // fdfsFile != null 表示数据库中有记录,再次检查文件在FastDFS中是否存在
+            if (fdfsFile != null) {
+                // 数据库有记录
+                val fileInfo = try {
+                    fastFileStorageClient.queryFileInfo(fdfsFile.group, fdfsFile.path)
+                } catch (e: FdfsServerException) {
+                    null
+                }
+                if (fileInfo != null) {
+                    // 文件存在
+                    ret.checkStatus = CheckStatus.FDFS_DB_FILE
+                    ret.uploadedChunks = mutableListOf()
+                    for (i in 1..fileMetadata.totalChunks!!) {
+                        ret.uploadedChunks!!.add(i)
+                    }
+                } else {
+                    // 真实文件不存在
+                    ret.checkStatus = CheckStatus.FDFS_DB_NO_FILE
+                }
+            } else {
+                // 数据库无记录
+                ret.checkStatus = CheckStatus.FDFS_NO_DB_FRAG_CHUNK
+                // 检查分片存在情况
+                for (i in 1..fileMetadata.totalChunks!!) {
+                    if (chunkExists(i, fileMetadata.md5!!)) {
+                        if (ret.uploadedChunks == null) {
+                            ret.uploadedChunks = mutableListOf()
+                        }
+                        ret.uploadedChunks!!.add(i)
+                    }
+                }
+                if (ret.uploadedChunks != null && ret.uploadedChunks!!.size == fileMetadata.totalChunks) {
+                    ret.checkStatus = CheckStatus.FDFS_NO_DB_FULL_CHUNK
+
+                    val hashedMd5 = fileMetadata.md5!!.hashCode()
+                    val tmpPath = uploadProperties!!.tmpPath!!
+
+                    checkMergeLock.lock(hashedMd5)
+                    try {
+                        // 将分片依次上传到 fastdfs 中(同步操作)
+                        //  logger.warn("checking: 上传所有分片啊!!!")
+                        val storePath = appendFileChunks(tmpPath, fileMetadata)
+                        ret.fdfsFile = FdfsFile(storePath.group, storePath.path)
+                    } finally {
+                        checkMergeLock.unlock(hashedMd5)
+                    }
+                }
+            }
+        } else {
+            throw ZenException(ZenExceptionEnum.FILE_METADATA_VALIDATE_ERROR)
+        }
+        return ret
+    }
+
+    /**
+     * 正常上传<br>
+     * 文件夹结构/文件名
+     */
+    fun upload(fileMetadata: ZenFileMetadata, file: MultipartFile): UploadResult {
+        val tmpPath = uploadProperties!!.tmpPath!!
+        val chunkSize = uploadProperties!!.chunkSize!!
+        var storePath: StorePath? = null
+        if (validateRequest(fileMetadata, file)) {
+            val filename = fileMetadata.filename!!
+            val fileExtName = FilenameUtils.getExtension(filename)
+            val hashedMd5 = fileMetadata.md5!!.hashCode()
+
+            // 如果分片小于chunkSize && totalChunk = 1 表示文件只有一个分片
+            if (fileMetadata.totalChunks == 1 && fileMetadata.chunkSize!! <= chunkSize) {
+                // 这里也需要加锁,避免并发上传失败
+                uploadLock.lock(hashedMd5)
+                try {
+                    file.inputStream.use {
+                        storePath = appendFileStorageClient.uploadAppenderFile(null, it, file.size, fileExtName)
+                    }
+                } finally {
+                    uploadLock.unlock(hashedMd5)
+                }
+            } else {
+                // 保存分片到本地磁盘临时路径
+                val chunkFilename = getChunkFilename(tmpPath, fileMetadata.chunkNumber, fileMetadata.md5)
+                val directory = Paths.get(tmpPath)
+                if (Files.notExists(directory)) {
+                    Files.createDirectories(directory)
+                }
+                file.transferTo(File(chunkFilename))
+
+                mergeLock.lock(hashedMd5)
+                try {
+                    if (checkChunks(tmpPath, fileMetadata)) {
+                        // 将分片依次上传到 fastdfs 中(同步操作)
+                        // logger.warn("上传所有分片啊!!! ${fileMetadata.chunkNumber} ${fileMetadata.totalChunks}")
+                        storePath = appendFileChunks(tmpPath, fileMetadata)
+                    }
+                } finally {
+                    mergeLock.unlock(hashedMd5)
+                }
+
+            }
+        } else {
+            throw ZenException(ZenExceptionEnum.FILE_METADATA_VALIDATE_ERROR)
+        }
+        var status: String = UploadStatus.UPLOADING
+        if (storePath != null) {
+            status = UploadStatus.UPLOADED
+        }
+        return UploadResult().apply {
+            this.status = status
+            this.storePath = storePath
+        }
+    }
+
+    /**
+     * 将单个文件所有分片上传到FastDFS中
+     */
+    private fun appendFileChunks(sourceRootPath: String, fileMetadata: ZenFileMetadata): StorePath {
+        val filename = fileMetadata.filename!!
+        val fileExtName = FilenameUtils.getExtension(filename)
+        val totalChunks = fileMetadata.totalChunks!!
+        val md5 = fileMetadata.md5!!
+        //        val hashedMd5 = md5.hashCode()
+        //        val chunkSize = uploadProperties!!.chunkSize!!
+        var storePath: StorePath? = null
+        for (i in 1..totalChunks) {
+            val sourceFile = File(getChunkFilename(sourceRootPath, i, md5))
+            val inputStream = BufferedInputStream(sourceFile.inputStream())
+            try {
+                if (i == 1) {
+                    // first chunk
+                    try {
+                        storePath = appendFileStorageClient.uploadAppenderFile(null, inputStream, sourceFile.length(), fileExtName)
+                    } catch (e: FdfsServerException) {
+                        if (e.errorCode != 16 && e.errorCode != 2) {
+                            throw ZenException(403, e.message!!)
+                        } else {
+                            throw e // 2:找不到节点或文件 16:服务器忙 可进行重试 ,其它情况均视为失败
+                        }
+                    } finally {
+                        if (storePath == null) {
+                            throw ZenException(ZenExceptionEnum.FILE_FRAG_UPLOAD_FAILURE)
+                        }
+                    }
+                } else {
+                    if (storePath == null) {
+                        throw ZenException(ZenExceptionEnum.FILE_FRAG_UPLOAD_FAILURE)
+                    } else {
+                        appendFileStorageClient.appendFile(storePath.group, storePath.path, inputStream, sourceFile.length())
+                        // appendFileStorageClient.modifyFile(storePath.group, storePath.path, inputStream, sourceFile.length(), i * chunkSize)
+                    }
+                }
+            } finally {
+                inputStream.close()
+                // 删除分片
+                sourceFile.delete()
+            }
+        }
+        return storePath!!
+    }
+
+    /**
+     * 检查分片是否存在 (tmp目录)
+     */
+    private fun chunkExists(chunkNumber: Int?, md5: String): Boolean {
+        val tmpPath = uploadProperties!!.tmpPath!!
+        return Files.exists(Paths.get(getChunkFilename(tmpPath, chunkNumber, md5)))
+    }
+
+    /**
+     * 检查分块是否上传完毕
+     */
+    private fun checkChunks(tmpPath: String, fileMetadata: ZenFileMetadata): Boolean {
+        val totalChunks = fileMetadata.totalChunks!!
+        val md5 = fileMetadata.md5!!
+        for (i in 1..totalChunks) {
+            if (Files.notExists(Paths.get(getChunkFilename(tmpPath, i, md5)))) {
+                return false
+            }
+        }
+        // 不能这样判断,因为chunk是无序的
+        //        return fileMetadata.chunkNumber == fileMetadata.totalChunks
+        return true
+    }
+
+    /**
+     * 生成 分片文件名
+     */
+    private fun getChunkFilename(path: String, chunkNumber: Int?, identifier: String?): String {
+        return "$path/upload-$identifier.$chunkNumber"
+    }
+
+    /**
+     * validate request multipart chunks
+     */
+    private fun validateRequest(fileMetadata: ZenFileMetadata, file: MultipartFile?): Boolean {
+        val md5 = fileMetadata.md5
+        val chunkNumber = fileMetadata.chunkNumber
+        val chunkSize = fileMetadata.chunkSize
+        val totalSize = fileMetadata.totalSize
+        val filename = fileMetadata.filename
+
+        if (chunkNumber == null || chunkNumber <= 0 ||
+                chunkSize == null || chunkSize <= 0 ||
+                totalSize == null || totalSize <= 0 ||
+                md5 == null || md5.isEmpty() ||
+                filename == null || filename.isEmpty()) {
+            return false
+        }
+
+        val numberOfChunks = Math.max(Math.floor(totalSize / (chunkSize * 1.0)), 1.0).toInt()
+        if (chunkNumber > numberOfChunks) {
+            return false
+        }
+
+        // is the file too large?
+        uploadProperties?.let {
+            val maxFileSize = it.maxFileSize
+            if (maxFileSize != null && maxFileSize > 0) {
+                if (totalSize > maxFileSize) {
+                    logger.error("filesize limit: [${maxFileSize / 1024 / 1024} MB], now [${totalSize / 1024 / 1024} MB]")
+                    return false
+                }
+            }
+        }
+
+        if (file != null) {
+            // The chunk in the POST request isn't the correct size
+            if (chunkNumber < numberOfChunks && file.size != chunkSize) {
+                logger.error("The chunk in the POST request isn't the correct size")
+                return false
+            }
+            // The chunks in the POST is the last one, and the fil is not the correct size
+            if (numberOfChunks > 1 && chunkNumber == numberOfChunks && file.size != (totalSize % chunkSize) + chunkSize) {
+                logger.error("The chunks in the POST is the last one, and the fil is not the correct size")
+                return false
+            }
+            // The file is only a single chunk, and the data size does not fit
+            if (numberOfChunks == 1 && file.size != totalSize) {
+                logger.error("The file is only a single chunk, and the data size does not fit")
+                return false
+            }
+        }
+        return true
+    }
+}

zen-core/src/main/kotlin/com/gxzc/zen/common/util/UploadUtil.kt → zen-core/src/main/kotlin/com/gxzc/zen/common/util/upload/UploadUtil.kt

@@ -1,20 +1,21 @@
-package com.gxzc.zen.common.util
+package com.gxzc.zen.common.util.upload
 
-import com.github.benmanes.caffeine.cache.Caffeine
-import com.gxzc.zen.common.dto.ZenFileMetadata
-import com.gxzc.zen.common.dto.ZenFileResponse
 import com.gxzc.zen.common.exception.ZenException
 import com.gxzc.zen.common.exception.ZenExceptionEnum
 import com.gxzc.zen.common.properties.UploadProperties
+import com.gxzc.zen.common.util.FileUtil
+import com.gxzc.zen.common.util.KeyLock
+import com.gxzc.zen.common.util.SpringContextHolder
+import com.gxzc.zen.common.util.upload.constants.CheckStatus
+import com.gxzc.zen.common.util.upload.constants.UploadStatus
+import com.gxzc.zen.common.util.upload.result.CheckResult
+import com.gxzc.zen.common.util.upload.result.UploadResult
 import org.apache.commons.io.FilenameUtils
 import org.slf4j.LoggerFactory
-import org.springframework.cache.caffeine.CaffeineCache
 import org.springframework.web.multipart.MultipartFile
 import java.io.*
 import java.nio.file.Files
 import java.nio.file.Paths
-import java.util.concurrent.ConcurrentHashMap
-import java.util.concurrent.TimeUnit
 
 /**
  * 上传 工具类
@@ -33,44 +34,43 @@ object UploadUtil {
             return field
         }
 
-    private val uploadedStatusMap = ConcurrentHashMap<String, String>() // 文件上传状态缓存
-    private val batchCountMap = CaffeineCache("batchCountMap", Caffeine.newBuilder().expireAfterWrite(1L, TimeUnit.HOURS).build(), true) // 文件批量上传批次文件数量记录
+    private val checkMergeLock = KeyLock<Int>()
+    private val mergeLock = KeyLock<Int>()
+    private val uploadLock = KeyLock<Int>()
 
     private val FILE_SEPARATOR = System.getProperty("file.separator") // 适配操作系统的文件路径
 
-    object STATUS {
-        const val CHECKING = "checking" // 检查中
-        const val MERGING = "merging" // 合并中
-        const val UPLOADED = "uploaded" // 单文件上传完毕
-        const val BATCH_UPLOADED = "batchUploaded" // 批量文件上传完毕
-        const val UPLOADING = "uploading" // 上传中
-    }
-
     /**
      * 正常上传<br>
      * 文件夹结构/文件名
      */
-    fun upload(fileMetadata: ZenFileMetadata, file: MultipartFile): ZenFileResponse {
+    fun upload(fileMetadata: ZenFileMetadata, file: MultipartFile): UploadResult {
         val tmpPath = uploadProperties!!.tmpPath!!
         val dataPath = uploadProperties!!.dataPath!!
         val chunkSize = uploadProperties!!.chunkSize!!
         var retFile: File? = null
         if (validateRequest(fileMetadata, file)) {
+            val filename = fileMetadata.filename!!
+            val md5 = fileMetadata.md5!!
+            val hashedMd5 = md5.hashCode()
             // 文件分片 如果分片小于chunkSize && totalChunk = 1,直接转存
             if (fileMetadata.totalChunks == 1 && fileMetadata.chunkSize!! <= chunkSize) {
-                val filename = fileMetadata.filename!!
-                val md5 = fileMetadata.md5!!
 
-                // 目标文件流 通过文件名来拼接
-                val outputFullFilename = getFullDestFilename(dataPath, filename, md5)
-                val outputPath = Paths.get(FilenameUtils.getFullPath(outputFullFilename))
-                // 不存在则创建文件夹
-                if (Files.notExists(outputPath)) {
-                    Files.createDirectories(outputPath)
+                uploadLock.lock(hashedMd5)
+                try {
+                    // 目标文件流 通过文件名来拼接
+                    val outputFullFilename = getFullDestFilename(dataPath, filename, md5)
+                    val outputPath = Paths.get(FilenameUtils.getFullPath(outputFullFilename))
+                    // 不存在则创建文件夹
+                    if (Files.notExists(outputPath)) {
+                        Files.createDirectories(outputPath)
+                    }
+                    retFile = File(outputFullFilename)
+                    file.transferTo(retFile)
+                    retFile.setLastModified(fileMetadata.lastModified!!)
+                } finally {
+                    uploadLock.unlock(hashedMd5)
                 }
-                retFile = File(outputFullFilename)
-                file.transferTo(retFile)
-                retFile.setLastModified(fileMetadata.lastModified!!)
             } else {
                 val chunkFilename = getChunkFilename(tmpPath, fileMetadata.chunkNumber, fileMetadata.md5)
                 val directory = Paths.get(tmpPath)
@@ -78,59 +78,45 @@ object UploadUtil {
                     Files.createDirectories(directory)
                 }
                 file.transferTo(File(chunkFilename))
-                // 检查分块完整性
-                if (checkChunks(tmpPath, fileMetadata)) {
-                    // 合并
-                    retFile = try {
-                        mergeChunks(tmpPath, dataPath, fileMetadata)
-                    } catch (e: Throwable) {
-                        logger.error("merge file chunks exception, cause ", e)
-                        uploadedStatusMap.remove(fileMetadata.md5!!)
-                        null
+
+                mergeLock.lock(hashedMd5)
+                try {
+                    // 检查分块完整性
+                    if (checkChunks(tmpPath, fileMetadata)) {
+                        // 合并
+                        retFile = try {
+                            mergeChunks(tmpPath, dataPath, fileMetadata)
+                        } catch (e: Throwable) {
+                            logger.error("merge file chunks exception, cause ", e)
+                            null
+                        }
                     }
+                } finally {
+                    mergeLock.unlock(hashedMd5)
                 }
             }
         } else {
             throw ZenException(ZenExceptionEnum.FILE_METADATA_VALIDATE_ERROR)
         }
-        var status: String = STATUS.UPLOADING
+        var status: String = UploadStatus.UPLOADING
         if (retFile != null) {
-            status = setBatch(fileMetadata)
+            status = UploadStatus.UPLOADED
         }
-        return ZenFileResponse().apply {
+        return UploadResult().apply {
             this.status = status
             this.file = retFile
         }
     }
 
-    /**
-     * 设置批次文件数量信息 ++
-     */
-    private fun setBatch(fileMetadata: ZenFileMetadata): String {
-        val batchId = fileMetadata.batchId!!
-        val totalNumber = fileMetadata.totalNumber!!
-        synchronized(batchCountMap) {
-            var count = batchCountMap[batchId]?.get() as? Int ?: 0
-            logger.debug("batchId: $batchId, count: ${count + 1}, totalNumber: $totalNumber")
-            return if (++count >= totalNumber) {
-                batchCountMap.evict(batchId)
-                STATUS.BATCH_UPLOADED
-            } else {
-                batchCountMap.put(batchId, count)
-                STATUS.UPLOADED
-            }
-        }
-    }
-
     /**
      * 检测文件是否存在 实现文件秒传
      * 真实文件已存在，秒传
      * 真实文件不存在，检查分片上传情况
      */
-    fun checkUpload(fileMetadata: ZenFileMetadata): ZenFileResponse {
+    fun checkUpload(fileMetadata: ZenFileMetadata): CheckResult {
         val tmpPath = uploadProperties!!.tmpPath!!
         val dataPath = uploadProperties!!.dataPath!!
-        val ret = ZenFileResponse().apply { status = STATUS.CHECKING }
+        val ret = CheckResult()
         if (validateRequest(fileMetadata, null)) {
             if (fileExists(fileMetadata)) {
                 ret.uploadedChunks = mutableListOf()
@@ -138,8 +124,10 @@ object UploadUtil {
                     ret.uploadedChunks!!.add(i)
                 }
                 ret.file = File(getFullDestFilename(dataPath, fileMetadata.filename!!, fileMetadata.md5!!))
+                ret.checkStatus = CheckStatus.FILE_EXISTS
             } else {
                 // 检查分片存在情况
+                ret.checkStatus = CheckStatus.NO_FILE_FRAG_CHUNK
                 for (i in 1..fileMetadata.totalChunks!!) {
                     if (chunkExists(i, fileMetadata.md5!!)) {
                         if (ret.uploadedChunks == null) {
@@ -150,24 +138,24 @@ object UploadUtil {
                 }
                 if (ret.uploadedChunks != null && ret.uploadedChunks!!.size == fileMetadata.totalChunks) {
                     // 合并
-                    ret.file = try {
-                        mergeChunks(tmpPath, dataPath, fileMetadata)
-                    } catch (e: Throwable) {
-                        logger.error("merge file chunks exception, cause ", e)
-                        uploadedStatusMap.remove(fileMetadata.md5!!)
-                        null
+                    ret.checkStatus = CheckStatus.NO_FILE_FULL_CHUNK
+                    val hashedMd5 = fileMetadata.md5!!.hashCode()
+                    checkMergeLock.lock(hashedMd5)
+                    try {
+                        ret.file = try {
+                            mergeChunks(tmpPath, dataPath, fileMetadata)
+                        } catch (e: Throwable) {
+                            logger.error("merge file chunks exception, cause ", e)
+                            null
+                        }
+                    } finally {
+                        checkMergeLock.unlock(hashedMd5)
                     }
-
-//                    // 所有分块传完 移除其中一个分块,再传一次而后 merge
-//                    ret.uploadedChunks!!.removeAt(0)
                 }
             }
         } else {
             throw ZenException(ZenExceptionEnum.FILE_METADATA_VALIDATE_ERROR)
         }
-        if (ret.file != null) {
-            ret.status = setBatch(fileMetadata)
-        }
         return ret
     }
 
@@ -215,15 +203,6 @@ object UploadUtil {
         val totalChunks = fileMetadata.totalChunks!!
         val md5 = fileMetadata.md5!!
 
-        if (uploadedStatusMap[md5] == STATUS.MERGING) {
-            return null
-        }
-
-        uploadedStatusMap[md5] = STATUS.MERGING
-
-        val t = System.currentTimeMillis()
-        logger.debug("start merging chunks for [$filename], now: $t")
-
         // 目标文件流 通过文件名来拼接
         val outputFullFilename = getFullDestFilename(destRootPath, filename, md5)
         val outputPath = Paths.get(FilenameUtils.getFullPath(outputFullFilename))
@@ -253,13 +232,9 @@ object UploadUtil {
         destOutputStream.flush()
         destOutputStream.close()
 
-
         // 修改文件 修改时间
         outputFile.setLastModified(fileMetadata.lastModified!!)
 
-        uploadedStatusMap.remove(md5)
-
-        logger.debug("merging successful, cost: ${System.currentTimeMillis() - t} ms.")
         return outputFile
     }
 

zen-core/src/main/kotlin/com/gxzc/zen/common/dto/ZenFileMetadata.kt → zen-core/src/main/kotlin/com/gxzc/zen/common/util/upload/ZenFileMetadata.kt

@@ -1,4 +1,4 @@
-package com.gxzc.zen.common.dto
+package com.gxzc.zen.common.util.upload
 
 import java.io.Serializable
 

zen-core/src/main/kotlin/com/gxzc/zen/common/util/upload/constants/CheckStatus.kt

@@ -0,0 +1,20 @@
+package com.gxzc.zen.common.util.upload.constants
+
+/**
+ * 检查状态
+ * @author NorthLan
+ * @date 2018/8/30
+ * @url https://noahlan.com
+ */
+object CheckStatus {
+    // FDFS_CHECK_STATUS
+    const val FDFS_DB_FILE = "FDFS_DB_FILE" // 有数据库记录,有文件
+    const val FDFS_DB_NO_FILE = "FDFS_DB_NO_FILE" // 有数据库记录,无文件
+    const val FDFS_NO_DB_FRAG_CHUNK = "FDFS_NO_DB_FRAG_CHUNK" // 无数据库记录,分片不完整
+    const val FDFS_NO_DB_FULL_CHUNK = "FDFS_NO_DB_FULL_CHUNK" // 无数据库记录,分片完整
+
+    // CHECK_STATUS
+    const val FILE_EXISTS = "FILE_EXISTS" // 真实文件存在
+    const val NO_FILE_FRAG_CHUNK = "NO_FILE_FRAG_CHUNK" // 文件不存在,分片不完整
+    const val NO_FILE_FULL_CHUNK = "NO_FILE_FULL_CHUNK" // 文件不存在,分片完整
+}

zen-core/src/main/kotlin/com/gxzc/zen/common/util/upload/constants/UploadStatus.kt

@@ -0,0 +1,14 @@
+package com.gxzc.zen.common.util.upload.constants
+
+/**
+ * 上传状态
+ * @author NorthLan
+ * @date 2018/8/30
+ * @url https://noahlan.com
+ */
+object UploadStatus {
+//    const val CHECKING = "checking" // 检查中
+    const val UPLOADED = "uploaded" // 单文件上传完毕
+//    const val BATCH_UPLOADED = "batchUploaded" // 批量文件上传完毕
+    const val UPLOADING = "uploading" // 上传中
+}

zen-core/src/main/kotlin/com/gxzc/zen/common/util/upload/result/CheckResult.kt

@@ -0,0 +1,28 @@
+package com.gxzc.zen.common.util.upload.result
+
+import com.fasterxml.jackson.annotation.JsonIgnore
+import com.fasterxml.jackson.annotation.JsonInclude
+import com.gxzc.zen.common.util.upload.FdfsFile
+import java.io.File
+
+/**
+ * 检查结果
+ * @author NorthLan
+ * @date 2018/8/30
+ * @url https://noahlan.com
+ */
+@JsonInclude(JsonInclude.Include.NON_NULL)
+open class CheckResult {
+    var uploadedChunks: MutableList<Int>? = null // 已上传分片数
+
+//    @JsonIgnore
+    var checkStatus: String? = null
+
+    @JsonIgnore
+    var fdfsFile: FdfsFile? = null
+
+    @JsonIgnore
+    var file: File? = null
+
+    var info: Any? = null // 回传信息
+}

zen-core/src/main/kotlin/com/gxzc/zen/common/dto/ZenFileResponse.kt → zen-core/src/main/kotlin/com/gxzc/zen/common/util/upload/result/UploadResult.kt

@@ -1,22 +1,26 @@
-package com.gxzc.zen.common.dto
+package com.gxzc.zen.common.util.upload.result
 
 import com.fasterxml.jackson.annotation.JsonIgnore
 import com.fasterxml.jackson.annotation.JsonInclude
+import com.github.tobato.fastdfs.domain.StorePath
+import com.gxzc.zen.common.util.upload.constants.UploadStatus
 import java.io.File
 
 /**
- * 文件
+ * 上传结果
  * @author NorthLan
  * @date 2018/5/25
  * @url https://noahlan.com
  */
 @JsonInclude(JsonInclude.Include.NON_NULL)
-open class ZenFileResponse {
-    var uploadedChunks: MutableList<Int>? = null // 已上传分片数
-    var status: String? = null // 状态码
+open class UploadResult {
+    var status: String? = null // 上传状态码
 
     @JsonIgnore
     var file: File? = null // 服务端文件实例
 
+    @JsonIgnore
+    var storePath: StorePath? = null
+
     var info: Any? = null // 信息
 }

zen-web/src/main/kotlin/com/gxzc/zen/web/sys/controller/TestController.kt

@@ -1,8 +1,6 @@
 package com.gxzc.zen.web.sys.controller
 
-import com.gxzc.zen.api.sys.service.ISysUploadInfoService
 import com.gxzc.zen.common.base.BaseController
-import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.web.bind.annotation.GetMapping
 import org.springframework.web.bind.annotation.RequestMapping
 import org.springframework.web.bind.annotation.RestController
@@ -18,11 +16,7 @@ import org.springframework.web.bind.annotation.RestController
 @RequestMapping("test")
 class TestController : BaseController() {
 
-    @Autowired
-    private lateinit var uploadService: ISysUploadInfoService
-
     @GetMapping("a")
     fun a() {
-        //        uploadService.test()
     }
 }

zen-web/src/main/kotlin/com/gxzc/zen/web/sys/controller/UploadController.kt

@@ -1,10 +1,12 @@
 package com.gxzc.zen.web.sys.controller
 
 import com.gxzc.zen.common.base.BaseController
-import com.gxzc.zen.common.dto.ZenFileMetadata
 import com.gxzc.zen.common.exception.ZenException
-import com.gxzc.zen.common.util.UploadUtil
-import com.gxzc.zen.web.sys.util.UploadCacheUtil
+import com.gxzc.zen.common.util.upload.UploadUtil
+import com.gxzc.zen.common.util.upload.ZenFileMetadata
+import com.gxzc.zen.common.util.upload.constants.CheckStatus
+import com.gxzc.zen.common.util.upload.constants.UploadStatus
+import com.gxzc.zen.logging.annotation.LogAnnotation
 import io.swagger.annotations.ApiOperation
 import org.slf4j.LoggerFactory
 import org.springframework.http.HttpStatus
@@ -29,19 +31,20 @@ class UploadController : BaseController() {
     }
 
     @ApiOperation("获取已上传分片列表")
+    @LogAnnotation(ignore = true)
     @GetMapping
     fun checkChunk(fileMetadata: ZenFileMetadata): ResponseEntity<*> {
         // 检查已上传文件分片
         val ret = UploadUtil.checkUpload(fileMetadata)
         return if (ret.uploadedChunks != null) {
-            when (ret.status) {
-            // 单文件完成,插入数据库
-                UploadUtil.STATUS.UPLOADED -> UploadCacheUtil.addUploadFileInfo(fileMetadata, ret.file!!)
-            // 批次完成,将最后个文件插入 而后取出所有本批次文件信息
-                UploadUtil.STATUS.BATCH_UPLOADED -> {
-                    UploadCacheUtil.addUploadFileInfo(fileMetadata, ret.file!!)
-                    ret.info = UploadCacheUtil.getUploadInfosByBatchId(fileMetadata.batchId!!)
-                    UploadCacheUtil.evictCache(fileMetadata.batchId!!)
+            when (ret.checkStatus) {
+            // 不入库
+                CheckStatus.FILE_EXISTS,
+                CheckStatus.NO_FILE_FULL_CHUNK -> {
+                    // 文件存在
+                    ret.info = fileMetadata
+                }
+                CheckStatus.NO_FILE_FRAG_CHUNK -> {
                 }
             }
             ResponseEntity.ok(ret)
@@ -51,29 +54,26 @@ class UploadController : BaseController() {
     }
 
     @ApiOperation("上传", notes = "支持小文件上传,大文件分片上传(统一分片)")
+    @LogAnnotation(ignore = true)
     @PostMapping
     fun upload(fileMetadata: ZenFileMetadata, file: MultipartFile?): ResponseEntity<*> {
         if (file == null) {
             return ResponseEntity.status(HttpStatus.NOT_ACCEPTABLE.value()).body(null)
         }
-        val uploadResponse = try {
+        val uploadResult = try {
             UploadUtil.upload(fileMetadata, file)
         } catch (e: ZenException) {
             null
         }
 
-        return if (uploadResponse != null) {
-            when (uploadResponse.status) {
+        return if (uploadResult != null) {
+            when (uploadResult.status) {
             // 单文件完成,插入数据库
-                UploadUtil.STATUS.UPLOADED -> UploadCacheUtil.addUploadFileInfo(fileMetadata, uploadResponse.file!!)
-            // 批次完成,将最后个文件插入 而后取出所有本批次文件信息
-                UploadUtil.STATUS.BATCH_UPLOADED -> {
-                    UploadCacheUtil.addUploadFileInfo(fileMetadata, uploadResponse.file!!)
-                    uploadResponse.info = UploadCacheUtil.getUploadInfosByBatchId(fileMetadata.batchId!!)
-                    UploadCacheUtil.evictCache(fileMetadata.batchId!!)
+                UploadStatus.UPLOADED -> {
+                    uploadResult.info = fileMetadata
                 }
             }
-            ResponseEntity.ok(uploadResponse)
+            ResponseEntity.ok(uploadResult)
         } else {
             ResponseEntity.status(HttpStatus.NOT_ACCEPTABLE.value()).body(null)
         }

zen-web/src/main/kotlin/com/gxzc/zen/web/sys/util/UploadCacheUtil.kt

@@ -1,81 +0,0 @@
-package com.gxzc.zen.web.sys.util
-
-import com.github.benmanes.caffeine.cache.Caffeine
-import com.gxzc.zen.common.dto.UploadFileInfo
-import com.gxzc.zen.common.dto.ZenFileMetadata
-import org.apache.commons.io.FilenameUtils
-import org.springframework.cache.caffeine.CaffeineCache
-import java.io.File
-import java.util.*
-import java.util.concurrent.TimeUnit
-
-/**
- *
- * @author NorthLan
- * @date 2018/7/3
- * @url https://noahlan.com
- */
-@Suppress("UNCHECKED_CAST")
-object UploadCacheUtil {
-    private val batchFileMap = CaffeineCache("batchFileMap", Caffeine.newBuilder().expireAfterWrite(1L, TimeUnit.HOURS).build(), true) // 文件批量上传批次实际文件信息
-
-    private fun getBatchFileList(batchId: String): MutableList<UploadFileInfo>? {
-        return batchFileMap[batchId]?.get() as? MutableList<UploadFileInfo>
-    }
-
-    private fun addFileInfo(fileInfo: UploadFileInfo, batchId: String) {
-        var fileList = getBatchFileList(batchId)
-        if (fileList == null) {
-            fileList = mutableListOf()
-            batchFileMap.put(batchId, fileList)
-        }
-        fileList.add(fileInfo)
-    }
-
-    /**
-     * 添加文件数据
-     */
-    fun addUploadFileInfo(fileMetadata: ZenFileMetadata, file: File) {
-        val entity = UploadFileInfo().apply {
-            // filename
-            this.filename = if (fileMetadata.rename.isNullOrEmpty()) {
-                fileMetadata.filename
-            } else {
-                "${FilenameUtils.removeExtension(fileMetadata.rename)}.${FilenameUtils.getExtension(fileMetadata.filename)}"
-            }
-            this.realFilename = file.name
-            this.originFilename = fileMetadata.filename
-            // infomation
-            this.relativePath = if (fileMetadata.repath.isNullOrEmpty()) fileMetadata.relativePath?.replace(this.originFilename!!, "") else fileMetadata.repath
-            this.lastModified = Date(file.lastModified())
-            this.md5 = fileMetadata.md5
-            this.extension = FilenameUtils.getExtension(file.name)
-            this.filesize = fileMetadata.totalSize
-        }
-        addFileInfo(entity, fileMetadata.batchId!!)
-    }
-
-    /**
-     * 获取某上传批次所有文件数据
-     */
-    fun getUploadInfosByBatchId(batchId: String): MutableList<UploadFileInfo> {
-        val ret = getBatchFileList(batchId) ?: mutableListOf()
-        ret.forEach {
-            generateURL(it)
-        }
-        return ret
-    }
-
-    /**
-     * 清理缓存
-     */
-    fun evictCache(batchId: String) {
-        batchFileMap.evict(batchId)
-    }
-
-    private fun generateURL(it: UploadFileInfo?) {
-        if (it != null) {
-            it.url = "/${it.realFilename}"
-        }
-    }
-}

zen-web/src/main/resources/application-upload.yml → zen-web/src/main/resources/application-fs.yml

@@ -15,6 +15,19 @@ upload:
   maxFileSize: 4294967296 # 单位 byte 为0表示无限制
   chunkSize: 10485760 # 10*1024*1024 = 10MB
 
+fdfs:
+  enabled: true
+  connect-timeout: 3000
+  so-timeout: 3000
+  tracker-list:
+  - 192.168.111.3:22122
+  pool:
+    max-total: 200
+    max-wait-millis: 3000
+    test-on-borrow: true
+    min-idle-per-key: 20
+    max-idle-per-key: 50
+
 ---
 spring:
   profiles: prod
@@ -31,4 +44,17 @@ upload:
   tmpPath: /tmp # 临时文件存放位置 默认 /tmp/zen
   dataPath: /archives/upload
   maxFileSize: 4294967296 # 单位 byte 为0表示无限制
-  chunkSize: 10485760 # 10*1024*1024 = 10MB
+  chunkSize: 10485760 # 10*1024*1024 = 10MB
+
+fdfs:
+  enabled: true
+  connect-timeout: 3000
+  so-timeout: 3000
+  tracker-list:
+  - 192.168.111.3:22122
+  pool:
+    max-total: 200
+    max-wait-millis: 3000
+    test-on-borrow: true
+    min-idle-per-key: 20
+    max-idle-per-key: 50

zen-web/src/main/resources/application.yml

@@ -1,7 +1,7 @@
 spring:
   profiles:
     active: dev
-    include: orm,rpc,msg,cache,umps,platform,job,upload
+    include: orm,rpc,msg,cache,umps,platform,job,fs
 ---
 spring:
   profiles: dev
@@ -31,7 +31,8 @@ logging:
       zen.umps.config: warn
     com.xxl.job.core: warn
     com.atomikos: warn
-    com.maihaoche: debug
+    com.maihaoche: warn
+  #    com.github.tobato.fastdfs: debug
   file: logs/zen.log
 
 ---
@@ -63,4 +64,5 @@ logging:
       zen.umps.config: warn
     com.xxl.job.core: warn
     com.atomikos: warn
+    com.maihaoche: warn
   file: logs/zen.log