From a22ccbdda5e23d1a59686816149fa705b48a745a Mon Sep 17 00:00:00 2001 From: Chopper Date: Tue, 7 Jun 2022 09:35:58 +0800 Subject: [PATCH] =?UTF-8?q?=E5=85=BC=E5=AE=B9MINIO?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/lili/buyer/test/cart/FileTest.java | 4 +- .../controller/common/UploadController.java | 7 +- framework/pom.xml | 7 + .../modules/file/entity/enums/OssEnum.java | 17 ++ ...FileManagerPlugin.java => FilePlugin.java} | 11 +- .../file/plugin/FilePluginFactory.java | 59 +++++++ ...eManagerPlugin.java => AliFilePlugin.java} | 62 ++----- .../file/plugin/impl/MinioFilePlugin.java | 165 ++++++++++++++++++ .../file/serviceimpl/FileServiceImpl.java | 9 +- .../modules/system/entity/dto/OssSetting.java | 38 ++++ pom.xml | 1 + 11 files changed, 320 insertions(+), 60 deletions(-) create mode 100644 framework/src/main/java/cn/lili/modules/file/entity/enums/OssEnum.java rename framework/src/main/java/cn/lili/modules/file/plugin/{FileManagerPlugin.java => FilePlugin.java} (77%) create mode 100644 framework/src/main/java/cn/lili/modules/file/plugin/FilePluginFactory.java rename framework/src/main/java/cn/lili/modules/file/plugin/impl/{AliFileManagerPlugin.java => AliFilePlugin.java} (72%) create mode 100644 framework/src/main/java/cn/lili/modules/file/plugin/impl/MinioFilePlugin.java diff --git a/buyer-api/src/test/java/cn/lili/buyer/test/cart/FileTest.java b/buyer-api/src/test/java/cn/lili/buyer/test/cart/FileTest.java index 1ccf3cffb..b7907bca1 100644 --- a/buyer-api/src/test/java/cn/lili/buyer/test/cart/FileTest.java +++ b/buyer-api/src/test/java/cn/lili/buyer/test/cart/FileTest.java @@ -1,7 +1,7 @@ package cn.lili.buyer.test.cart; -import cn.lili.modules.file.plugin.FileManagerPlugin; +import cn.lili.modules.file.plugin.FilePlugin; import cn.lili.modules.goods.entity.dos.Brand; import cn.lili.modules.goods.service.BrandService; import com.xkcoding.http.util.StringUtil; @@ -27,7 +27,7 @@ class FileTest { @Autowired - private FileManagerPlugin fileManagerPlugin; + private FilePlugin fileManagerPlugin; @Autowired private BrandService brandService; diff --git a/common-api/src/main/java/cn/lili/controller/common/UploadController.java b/common-api/src/main/java/cn/lili/controller/common/UploadController.java index 25001d073..d97fa40ce 100644 --- a/common-api/src/main/java/cn/lili/controller/common/UploadController.java +++ b/common-api/src/main/java/cn/lili/controller/common/UploadController.java @@ -12,7 +12,8 @@ import cn.lili.common.utils.CommonUtil; import cn.lili.common.vo.ResultMessage; import cn.lili.modules.file.entity.File; -import cn.lili.modules.file.plugin.FileManagerPlugin; +import cn.lili.modules.file.plugin.FilePlugin; +import cn.lili.modules.file.plugin.FilePluginFactory; import cn.lili.modules.file.service.FileService; import cn.lili.modules.system.entity.dos.Setting; import cn.lili.modules.system.entity.enums.SettingEnum; @@ -47,7 +48,7 @@ public class UploadController { @Autowired private SettingService settingService; @Autowired - private FileManagerPlugin fileManagerPlugin; + private FilePluginFactory filePluginFactory; @Autowired private Cache cache; @@ -86,7 +87,7 @@ public ResultMessage upload(MultipartFile file, try { InputStream inputStream = file.getInputStream(); //上传至第三方云服务或服务器 - result = fileManagerPlugin.inputStreamUpload(inputStream, fileKey); + result = filePluginFactory.filePlugin().inputStreamUpload(inputStream, fileKey); //保存数据信息至数据库 newFile.setName(file.getOriginalFilename()); newFile.setFileSize(file.getSize()); diff --git a/framework/pom.xml b/framework/pom.xml index 6919e7634..5974c1031 100644 --- a/framework/pom.xml +++ b/framework/pom.xml @@ -420,6 +420,13 @@ + + io.minio + minio + ${minio.version} + + + diff --git a/framework/src/main/java/cn/lili/modules/file/entity/enums/OssEnum.java b/framework/src/main/java/cn/lili/modules/file/entity/enums/OssEnum.java new file mode 100644 index 000000000..748faeca2 --- /dev/null +++ b/framework/src/main/java/cn/lili/modules/file/entity/enums/OssEnum.java @@ -0,0 +1,17 @@ +package cn.lili.modules.file.entity.enums; + +import com.aliyun.oss.OSS; + +/** + * OssEnum + * + * @author Chopper + * @version v1.0 + * 2022-06-06 11:23 + */ +public enum OssEnum { + /** + * + */ + ALI_OSS, MINIO; +} diff --git a/framework/src/main/java/cn/lili/modules/file/plugin/FileManagerPlugin.java b/framework/src/main/java/cn/lili/modules/file/plugin/FilePlugin.java similarity index 77% rename from framework/src/main/java/cn/lili/modules/file/plugin/FileManagerPlugin.java rename to framework/src/main/java/cn/lili/modules/file/plugin/FilePlugin.java index bd279d057..2f6fa799d 100644 --- a/framework/src/main/java/cn/lili/modules/file/plugin/FileManagerPlugin.java +++ b/framework/src/main/java/cn/lili/modules/file/plugin/FilePlugin.java @@ -1,15 +1,22 @@ package cn.lili.modules.file.plugin; +import cn.lili.modules.file.entity.enums.OssEnum; + import java.io.InputStream; import java.util.List; /** - * 文件管理插件 + * 文件插件接口 * * @author Chopper */ -public interface FileManagerPlugin { +public interface FilePlugin { + + /** + * 插件名称 + */ + OssEnum pluginName(); /** * 文件路径上传 diff --git a/framework/src/main/java/cn/lili/modules/file/plugin/FilePluginFactory.java b/framework/src/main/java/cn/lili/modules/file/plugin/FilePluginFactory.java new file mode 100644 index 000000000..63e4da080 --- /dev/null +++ b/framework/src/main/java/cn/lili/modules/file/plugin/FilePluginFactory.java @@ -0,0 +1,59 @@ +package cn.lili.modules.file.plugin; + +import cn.hutool.json.JSONUtil; +import cn.lili.common.exception.ServiceException; +import cn.lili.modules.file.entity.enums.OssEnum; +import cn.lili.modules.file.plugin.impl.AliFilePlugin; +import cn.lili.modules.file.plugin.impl.MinioFilePlugin; +import cn.lili.modules.system.entity.dos.Setting; +import cn.lili.modules.system.entity.dto.OssSetting; +import cn.lili.modules.system.entity.enums.SettingEnum; +import cn.lili.modules.system.service.SettingService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * 文件服务抽象工厂 直接返回操作类 + * + * @author Chopper + * @version v1.0 + * 2022-06-06 11:35 + */ +@Component +public class FilePluginFactory { + + + @Autowired + private SettingService settingService; + + + /** + * 获取oss client + * + * @return + */ + public FilePlugin filePlugin() { + + OssSetting ossSetting = null; + try { + Setting setting = settingService.get(SettingEnum.OSS_SETTING.name()); + + ossSetting = JSONUtil.toBean(setting.getSettingValue(), OssSetting.class); + + + switch (OssEnum.valueOf(ossSetting.getType())) { + + case MINIO: + return new MinioFilePlugin(ossSetting); + case ALI_OSS: + return new AliFilePlugin(ossSetting); + default: + throw new ServiceException(); + } + } catch (Exception e) { + throw new ServiceException(); + } + } + + +} diff --git a/framework/src/main/java/cn/lili/modules/file/plugin/impl/AliFileManagerPlugin.java b/framework/src/main/java/cn/lili/modules/file/plugin/impl/AliFilePlugin.java similarity index 72% rename from framework/src/main/java/cn/lili/modules/file/plugin/impl/AliFileManagerPlugin.java rename to framework/src/main/java/cn/lili/modules/file/plugin/impl/AliFilePlugin.java index 4ddbf8996..e195a2a01 100644 --- a/framework/src/main/java/cn/lili/modules/file/plugin/impl/AliFileManagerPlugin.java +++ b/framework/src/main/java/cn/lili/modules/file/plugin/impl/AliFilePlugin.java @@ -1,23 +1,17 @@ package cn.lili.modules.file.plugin.impl; -import cn.hutool.core.util.StrUtil; import cn.lili.common.enums.ResultCode; import cn.lili.common.exception.ServiceException; -import cn.lili.modules.file.plugin.FileManagerPlugin; -import cn.lili.modules.system.entity.dos.Setting; +import cn.lili.modules.file.entity.enums.OssEnum; +import cn.lili.modules.file.plugin.FilePlugin; import cn.lili.modules.system.entity.dto.OssSetting; -import cn.lili.modules.system.entity.enums.SettingEnum; -import cn.lili.modules.system.service.SettingService; import com.aliyun.oss.ClientException; import com.aliyun.oss.OSS; import com.aliyun.oss.OSSClientBuilder; import com.aliyun.oss.OSSException; import com.aliyun.oss.model.DeleteObjectsRequest; import com.aliyun.oss.model.ObjectMetadata; -import com.google.gson.Gson; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; import java.io.File; import java.io.InputStream; @@ -29,28 +23,19 @@ * @author Chopper */ -@Component @Slf4j -public class AliFileManagerPlugin implements FileManagerPlugin { +public class AliFilePlugin implements FilePlugin { - @Autowired - private SettingService settingService; + private OssSetting ossSetting; - /** - * 下一个初始化配置参数的时间 - * 这里为了防止多次调用redis，减少与redis的交互时间 - */ - private static Long nextInitSetting; - - /** - * 暂时设定3分账请求一次设置 - */ - private static final Long INTERVAL = 60 * 3 * 1000L; + public AliFilePlugin(OssSetting ossSetting) { + this.ossSetting = ossSetting; + } - /** - * 静态设置，最快三分钟更新一次 - */ - private static OssSetting ossSetting; + @Override + public OssEnum pluginName() { + return OssEnum.ALI_OSS; + } /** * 获取oss client @@ -58,32 +43,12 @@ public class AliFileManagerPlugin implements FileManagerPlugin { * @return */ private OSS getOssClient() { - OssSetting ossSetting = getSetting(); - return new OSSClientBuilder().build( ossSetting.getEndPoint(), ossSetting.getAccessKeyId(), ossSetting.getAccessKeySecret()); } - /** - * 获取配置 - * - * @return - */ - private OssSetting getSetting() { - //如果没有配置，或者没有下次刷新时间，或者下次刷新时间小于当前时间，则从redis 更新一次 - if (ossSetting == null || nextInitSetting == null || nextInitSetting < System.currentTimeMillis()) { - Setting setting = settingService.get(SettingEnum.OSS_SETTING.name()); - if (setting == null || StrUtil.isBlank(setting.getSettingValue())) { - throw new ServiceException(ResultCode.OSS_NOT_EXIST); - } - nextInitSetting = System.currentTimeMillis() + INTERVAL; - ossSetting = new Gson().fromJson(setting.getSettingValue(), OssSetting.class); - return ossSetting; - } - return ossSetting; - } /** * 获取配置前缀 @@ -91,7 +56,6 @@ private OssSetting getSetting() { * @return */ private String getUrlPrefix() { - OssSetting ossSetting = getSetting(); return "https://" + ossSetting.getBucketName() + "." + ossSetting.getEndPoint() + "/"; } @@ -130,7 +94,7 @@ public String inputStreamUpload(InputStream inputStream, String key) { try { ObjectMetadata meta = new ObjectMetadata(); meta.setContentType("image/jpg"); - ossClient.putObject(getSetting().getBucketName(), key, inputStream, meta); + ossClient.putObject(ossSetting.getBucketName(), key, inputStream, meta); } catch (OSSException oe) { log.error("Caught an OSSException, which means your request made it to OSS, " + "but was rejected with an error response for some reason."); @@ -161,7 +125,7 @@ public void deleteFile(List key) { try { ossClient.deleteObjects( - new DeleteObjectsRequest(getSetting().getBucketName()).withKeys(key)); + new DeleteObjectsRequest(ossSetting.getBucketName()).withKeys(key)); } catch (OSSException oe) { log.error("Caught an OSSException, which means your request made it to OSS, " + "but was rejected with an error response for some reason."); diff --git a/framework/src/main/java/cn/lili/modules/file/plugin/impl/MinioFilePlugin.java b/framework/src/main/java/cn/lili/modules/file/plugin/impl/MinioFilePlugin.java new file mode 100644 index 000000000..67471641d --- /dev/null +++ b/framework/src/main/java/cn/lili/modules/file/plugin/impl/MinioFilePlugin.java @@ -0,0 +1,165 @@ +package cn.lili.modules.file.plugin.impl; + +import cn.lili.modules.file.entity.enums.OssEnum; +import cn.lili.modules.file.plugin.FilePlugin; +import cn.lili.modules.system.entity.dto.OssSetting; +import io.minio.*; +import io.minio.errors.ErrorResponseException; +import io.minio.messages.DeleteObject; +import lombok.extern.slf4j.Slf4j; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.util.*; +import java.util.stream.Collectors; + + +/** + * MINIO文件插件 + * + * @author liushuai(liushuai711 @ gmail.com) + * @version v4.0 + * @Description: + * @since 2022/6/6 17:45 + */ +@Slf4j +public class MinioFilePlugin implements FilePlugin { + + private OssSetting ossSetting; + + public MinioFilePlugin(OssSetting ossSetting) { + this.ossSetting = ossSetting; + } + + /** + * 桶占位符 + */ + private static final String BUCKET_PARAM = "${bucket}"; + /** + * bucket权限-只读 + */ + private static final String READ_ONLY = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"*\"]},\"Action\":[\"s3:GetBucketLocation\",\"s3:ListBucket\"],\"Resource\":[\"arn:aws:s3:::" + BUCKET_PARAM + "\"]},{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"*\"]},\"Action\":[\"s3:GetObject\"],\"Resource\":[\"arn:aws:s3:::" + BUCKET_PARAM + "/*\"]}]}"; + /** + * bucket权限-只读 + */ + private static final String WRITE_ONLY = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"*\"]},\"Action\":[\"s3:GetBucketLocation\",\"s3:ListBucketMultipartUploads\"],\"Resource\":[\"arn:aws:s3:::" + BUCKET_PARAM + "\"]},{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"*\"]},\"Action\":[\"s3:AbortMultipartUpload\",\"s3:DeleteObject\",\"s3:ListMultipartUploadParts\",\"s3:PutObject\"],\"Resource\":[\"arn:aws:s3:::" + BUCKET_PARAM + "/*\"]}]}"; + /** + * bucket权限-读写 + */ + private static final String READ_WRITE = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"*\"]},\"Action\":[\"s3:GetBucketLocation\",\"s3:ListBucket\",\"s3:ListBucketMultipartUploads\"],\"Resource\":[\"arn:aws:s3:::" + BUCKET_PARAM + "\"]},{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"*\"]},\"Action\":[\"s3:DeleteObject\",\"s3:GetObject\",\"s3:ListMultipartUploadParts\",\"s3:PutObject\",\"s3:AbortMultipartUpload\"],\"Resource\":[\"arn:aws:s3:::" + BUCKET_PARAM + "/*\"]}]}"; + + + private MinioClient minioClient; + + + @Override + public OssEnum pluginName() { + return OssEnum.MINIO; + } + + @Override + public String pathUpload(String filePath, String key) { + try { + return this.inputStreamUpload(new FileInputStream(filePath), key); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + return null; + } + + @Override + public String inputStreamUpload(InputStream inputStream, String key) { + String bucket = ""; + try { + MinioClient client = getOssClient(); + bucket = ossSetting.getM_bucketName(); + PutObjectArgs putObjectArgs = PutObjectArgs.builder() + .bucket(bucket).stream(inputStream, inputStream.available(), 5 * 1024 * 1024) + .object(key) + .contentType("image/png") + .build(); + client.putObject(putObjectArgs); + } catch (ErrorResponseException e) { + e.printStackTrace(); + return null; + } catch (Exception e) { + log.error("上传失败2，", e); + return null; + } + //拼接出可访问的url地址 + return ossSetting.getM_endpoint() + "/" + bucket + "/" + key; + } + + + @Override + public void deleteFile(List key) { + if (key == null || key.isEmpty()) { + return; + } + MinioClient ossClient = getOssClient(); + List objectList = key.stream().map(DeleteObject::new).collect(Collectors.toList()); + ossClient.removeObjects(RemoveObjectsArgs.builder().objects(objectList).build()); + } + + + /** + * 获取oss client + * + * @return + */ + private MinioClient getOssClient() { + if (minioClient != null) { + return this.minioClient; + } + synchronized (this) { + if (minioClient == null) { + //创建客户端 + this.minioClient = MinioClient.builder() + .endpoint(ossSetting.getM_endpoint()) + .credentials(ossSetting.getM_accessKey(), ossSetting.getM_secretKey()) + .build(); + try { + //查看对应的bucket是否已经存在，不存在则创建 + if (!minioClient.bucketExists(BucketExistsArgs.builder().bucket(ossSetting.getM_bucketName()).build())) { + //创建bucket + MakeBucketArgs makeBucketArgs = MakeBucketArgs.builder().bucket(ossSetting.getM_bucketName()).build(); + this.minioClient.makeBucket(makeBucketArgs); + setBucketPolicy(this.minioClient, ossSetting.getM_bucketName(), "read-write"); + log.info("创建minio桶成功{}", ossSetting.getM_bucketName()); + } + } catch (Exception e) { + e.printStackTrace(); + log.error("创建[{}]bucket失败", ossSetting.getM_bucketName()); + } + } + } + return minioClient; + } + + + /** + * 更新桶权限策略 + * + * @param bucket 桶 + * @param policy 权限 + */ + public static void setBucketPolicy(MinioClient client, String bucket, String policy) throws Exception { + switch (policy) { + case "read-only": + client.setBucketPolicy(SetBucketPolicyArgs.builder().bucket(bucket).config(READ_ONLY.replace(BUCKET_PARAM, bucket)).build()); + break; + case "write-only": + client.setBucketPolicy(SetBucketPolicyArgs.builder().bucket(bucket).config(WRITE_ONLY.replace(BUCKET_PARAM, bucket)).build()); + break; + case "read-write": + client.setBucketPolicy(SetBucketPolicyArgs.builder().bucket(bucket).region("public").config(READ_WRITE.replace(BUCKET_PARAM, bucket)).build()); + break; + case "none": + default: + break; + } + } + + +} diff --git a/framework/src/main/java/cn/lili/modules/file/serviceimpl/FileServiceImpl.java b/framework/src/main/java/cn/lili/modules/file/serviceimpl/FileServiceImpl.java index 4d68c0f54..bc830cce7 100644 --- a/framework/src/main/java/cn/lili/modules/file/serviceimpl/FileServiceImpl.java +++ b/framework/src/main/java/cn/lili/modules/file/serviceimpl/FileServiceImpl.java @@ -9,7 +9,8 @@ import cn.lili.modules.file.entity.File; import cn.lili.modules.file.entity.dto.FileOwnerDTO; import cn.lili.modules.file.mapper.FileMapper; -import cn.lili.modules.file.plugin.FileManagerPlugin; +import cn.lili.modules.file.plugin.FilePlugin; +import cn.lili.modules.file.plugin.FilePluginFactory; import cn.lili.modules.file.service.FileService; import cn.lili.mybatis.util.PageUtil; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; @@ -31,7 +32,7 @@ public class FileServiceImpl extends ServiceImpl implements FileService { @Autowired - private FileManagerPlugin fileManagerPlugin; + private FilePluginFactory filePluginFactory; @Override public void batchDelete(List ids) { @@ -42,7 +43,7 @@ public void batchDelete(List ids) { List files = this.list(queryWrapper); List keys = new ArrayList<>(); files.forEach(item -> keys.add(item.getFileKey())); - fileManagerPlugin.deleteFile(keys); + filePluginFactory.filePlugin().deleteFile(keys); this.remove(queryWrapper); } @@ -68,7 +69,7 @@ public void batchDelete(List ids, AuthUser authUser) { List files = this.list(queryWrapper); List keys = new ArrayList<>(); files.forEach(item -> keys.add(item.getFileKey())); - fileManagerPlugin.deleteFile(keys); + filePluginFactory.filePlugin().deleteFile(keys); this.remove(queryWrapper); } diff --git a/framework/src/main/java/cn/lili/modules/system/entity/dto/OssSetting.java b/framework/src/main/java/cn/lili/modules/system/entity/dto/OssSetting.java index 80c5fd075..fd71d3b38 100644 --- a/framework/src/main/java/cn/lili/modules/system/entity/dto/OssSetting.java +++ b/framework/src/main/java/cn/lili/modules/system/entity/dto/OssSetting.java @@ -1,5 +1,7 @@ package cn.lili.modules.system.entity.dto; +import cn.lili.common.utils.StringUtils; +import cn.lili.modules.file.entity.enums.OssEnum; import lombok.Data; import java.io.Serializable; @@ -15,6 +17,12 @@ public class OssSetting implements Serializable { private static final long serialVersionUID = 2975271656230801861L; + + /** + * oss类型 + */ + private String type; + /** * 域名 */ @@ -35,4 +43,34 @@ public class OssSetting implements Serializable { * 密钥 */ private String accessKeySecret = ""; + + + /** + * minio服务地址 + */ + private String m_endpoint; + + /** + * minio用户名 + */ + private String m_accessKey; + + /** + * minio密码 + */ + private String m_secretKey; + + /** + * minio bucket名称 + */ + private String m_bucketName; + + + public String getType() { + //默认给阿里云oss存储类型 + if (StringUtils.isEmpty(type)) { + return OssEnum.ALI_OSS.name(); + } + return type; + } } diff --git a/pom.xml b/pom.xml index a9d92a596..99b15f0a6 100644 --- a/pom.xml +++ b/pom.xml @@ -60,6 +60,7 @@ 1.2.2 2.3.1 20211018.2 + 8.0.3