From a3a11880f78f2ee1ab0a9978a58b6fc66339cf4e Mon Sep 17 00:00:00 2001 From: zlg <482370576@qq.com> Date: Tue, 16 Jul 2024 18:54:01 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=A4=9A=E7=BA=BF=E7=A8=8B=E4=B8=8B?= =?UTF-8?q?=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../SchisandraOssQiniuController.java | 115 +++++++++++------- .../SchisandraOssTencentController.java | 59 ++++++--- .../SchisandraUserOssController.java | 6 +- .../oss/core/qiniu/QiNiuOssClient.java | 55 +++++++-- .../oss/core/tencent/TencentOssClient.java | 50 ++++++-- .../SchisandraFileHeatmapDomainService.java | 2 +- .../SchisandraOssQiniuDomainService.java | 4 + ...chisandraFileHeatmapDomainServiceImpl.java | 14 ++- .../SchisandraOssQiniuDomainServiceImpl.java | 7 ++ .../basic/entity/SchisandraOssQiniu.java | 2 + .../service/SchisandraOssQiniuService.java | 2 + .../impl/SchisandraOssQiniuServiceImpl.java | 5 + 12 files changed, 233 insertions(+), 88 deletions(-) diff --git a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/controller/SchisandraOssQiniuController.java b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/controller/SchisandraOssQiniuController.java index 9166a2c..612829f 100644 --- a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/controller/SchisandraOssQiniuController.java +++ b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/controller/SchisandraOssQiniuController.java @@ -1,10 +1,10 @@ package com.schisandra.oss.application.controller; +import cn.hutool.core.util.ZipUtil; import cn.hutool.extra.spring.SpringUtil; import com.alibaba.fastjson.JSON; import com.google.common.base.Preconditions; import com.schisandra.oss.application.convert.SchisandraOssQiniuDTOConverter; -import com.schisandra.oss.application.convert.SchisandraOssSftpDTOConverter; import com.schisandra.oss.application.dto.SchisandraOssQiniuDTO; import com.schisandra.oss.application.oss.constant.OssConstant; import com.schisandra.oss.application.oss.core.qiniu.QiNiuOssClient; @@ -12,25 +12,28 @@ import com.schisandra.oss.application.oss.core.qiniu.QiNiuOssConfiguration; import com.schisandra.oss.application.oss.exception.OssException; import com.schisandra.oss.application.oss.model.OssInfo; import com.schisandra.oss.common.entity.Result; -import com.schisandra.oss.domain.bo.SchisandraOssMinioBO; import com.schisandra.oss.domain.bo.SchisandraOssQiniuBO; -import com.schisandra.oss.domain.bo.SchisandraOssSftpBO; import com.schisandra.oss.domain.redis.RedisUtil; import com.schisandra.oss.domain.service.SchisandraOssQiniuDomainService; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.springframework.util.StreamUtils; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletResponse; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.net.URLEncoder; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; /** * 七牛云对象存储配置表 controller @@ -43,15 +46,14 @@ import java.util.List; @Slf4j public class SchisandraOssQiniuController { + private final String USER_OSS_PREFIX = "oss:user:heat"; + @Resource + RedisUtil redisUtil; @Resource private SchisandraOssQiniuDomainService schisandraOssQiniuDomainService; @Resource private QiNiuOssConfiguration qiNiuOssConfiguration; - private final String USER_OSS_PREFIX = "oss:user:heat"; - @Resource - RedisUtil redisUtil; - /** * 返回七牛表所有数据 * @@ -60,7 +62,7 @@ public class SchisandraOssQiniuController { @GetMapping("returnAll") public Result returnAll() { List list = schisandraOssQiniuDomainService.selectAll(); - if(list.isEmpty()){ + if (list.isEmpty()) { return Result.fail(); } return Result.ok(SchisandraOssQiniuDTOConverter.INSTANCE.convertBOToDTOList(list)); @@ -75,11 +77,11 @@ public class SchisandraOssQiniuController { @GetMapping("listDir") public Result listQiniuInfo(@RequestParam String userId, @RequestParam String prefix, @RequestParam String bucket) throws Exception { Preconditions.checkNotNull(userId, "不能为空"); - QiNiuOssClient bean = SpringUtil.getBean(OssConstant.OssType.QINIU+ userId); + QiNiuOssClient bean = SpringUtil.getBean(OssConstant.OssType.QINIU + userId); bean.getQiNiuOssConfig().setBucketName(bucket); - if(bean.listfile(bucket, prefix).isEmpty()){ + if (bean.listfile(bucket, prefix).isEmpty()) { return Result.fail(); - }else{ + } else { return Result.ok(bean.listfile(bucket, prefix)); } @@ -98,7 +100,7 @@ public class SchisandraOssQiniuController { Preconditions.checkNotNull(bucket, "不能为空"); Preconditions.checkNotNull(oldFileName, "不能为空"); Preconditions.checkNotNull(newFileName, "不能为空"); - QiNiuOssClient bean = SpringUtil.getBean(OssConstant.OssType.QINIU+ userId); + QiNiuOssClient bean = SpringUtil.getBean(OssConstant.OssType.QINIU + userId); bean.getQiNiuOssConfig().setBucketName(bucket); try { bean.rename(oldFileName, newFileName); @@ -122,7 +124,7 @@ public class SchisandraOssQiniuController { Preconditions.checkNotNull(bucket, "不能为空"); Preconditions.checkNotNull(oldFilePath, "不能为空"); Preconditions.checkNotNull(newFilePath, "不能为空"); - QiNiuOssClient bean = SpringUtil.getBean(OssConstant.OssType.QINIU+ userId); + QiNiuOssClient bean = SpringUtil.getBean(OssConstant.OssType.QINIU + userId); bean.getQiNiuOssConfig().setBucketName(bucket); try { bean.copy(oldFilePath, newFilePath); @@ -141,16 +143,45 @@ public class SchisandraOssQiniuController { * @date: 2024/7/8 13:56 */ @GetMapping("downloadFile") - public void download_open(@RequestParam String userId, @RequestParam String endpoint, @RequestParam String filename, HttpServletResponse response) throws IOException { - QiNiuOssClient bean = SpringUtil.getBean(OssConstant.OssType.QINIU+ userId); - bean.getQiNiuOssConfig().setEndpoint(endpoint); + public Result download_open(@RequestParam String userId, @RequestParam String bucket, @RequestParam List listObjectsArgs, HttpServletResponse response) throws IOException { + QiNiuOssClient bean = SpringUtil.getBean(OssConstant.OssType.QINIU + userId); ServletOutputStream output = response.getOutputStream(); - - + SchisandraOssQiniuBO schisandraOssQiniuBO = schisandraOssQiniuDomainService.getEndpointByuserIdAndBucket(Long.valueOf(userId), bucket); + if (schisandraOssQiniuBO.getEndpoint() == null) { + return Result.fail("请配置该存储桶的相关配置:endpoint"); + } + response.setHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode("下载.zip", "UTF-8")); + response.setContentType("application/octet-stream"); response.setCharacterEncoding("UTF-8"); - response.setContentType("application/vnd.ms-excel"); - response.setHeader("Content-disposition", "attachment;filename=" + URLEncoder.encode(filename, "utf-8")); - bean.downLoad_open(output, filename, endpoint); + try { + HashMap list = bean.downLoadMany(listObjectsArgs, schisandraOssQiniuBO.getEndpoint()); + String[] path = new String[list.size()]; + path = list.keySet().toArray(path); + //记录用户下载 + Date date = new Date(); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + String formattedDate = dateFormat.format(date); + AtomicInteger count = new AtomicInteger(0); + InputStream[] inputStream2 = new InputStream[list.size()]; + list.keySet().forEach(name -> { + try { + byte[] bytes = StreamUtils.copyToByteArray(list.get(name)); + inputStream2[count.get()] = new ByteArrayInputStream(bytes); + String key = redisUtil.buildKey(USER_OSS_PREFIX, "download", formattedDate, userId, name); + HashMap map = new HashMap<>(); + map.put("size", bytes.length); + map.put("time", formattedDate); + redisUtil.setNx(key, map, 30L, TimeUnit.DAYS); + count.incrementAndGet(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + ZipUtil.zip(output, path, inputStream2); + } catch (Exception e) { + return Result.fail(); + } + return Result.ok(); } @@ -165,12 +196,12 @@ public class SchisandraOssQiniuController { Preconditions.checkNotNull(bucket, "不能为空"); Preconditions.checkNotNull(userId, "不能为空"); Preconditions.checkNotNull(filePath, "不能为空"); - QiNiuOssClient bean = SpringUtil.getBean(OssConstant.OssType.QINIU+ userId); + QiNiuOssClient bean = SpringUtil.getBean(OssConstant.OssType.QINIU + userId); bean.getQiNiuOssConfig().setBucketName(bucket); - try{ + try { bean.delete(filePath); return Result.ok(); - }catch(OssException e){ + } catch (OssException e) { return Result.fail(); } } @@ -203,9 +234,9 @@ public class SchisandraOssQiniuController { // 获取文件输入流 InputStream is = file.getInputStream(); - QiNiuOssClient bean = SpringUtil.getBean(OssConstant.OssType.QINIU+ userId); + QiNiuOssClient bean = SpringUtil.getBean(OssConstant.OssType.QINIU + userId); bean.getQiNiuOssConfig().setBucketName(bucket); - if(bean.upLoad(is, fileName, true) != null){ + if (bean.upLoad(is, fileName, true) != null) { return Result.ok(bean.upLoad(is, fileName, true)); } return Result.fail(); @@ -222,11 +253,10 @@ public class SchisandraOssQiniuController { @PostMapping("seleteBucket") public Result seleteBucket(@RequestParam String userId) { Preconditions.checkNotNull(userId, "不能为空"); - QiNiuOssClient bean = SpringUtil.getBean(OssConstant.OssType.QINIU+ userId); - if(bean.selectAllBucket().isEmpty()){ + QiNiuOssClient bean = SpringUtil.getBean(OssConstant.OssType.QINIU + userId); + if (bean.selectAllBucket().isEmpty()) { return Result.fail(); - } - else + } else return Result.ok(bean.selectAllBucket()); } @@ -240,11 +270,11 @@ public class SchisandraOssQiniuController { public Result deleteBucket(@RequestParam String userId, @RequestParam String bucket) { Preconditions.checkNotNull(userId, "不能为空"); Preconditions.checkNotNull(bucket, "不能为空"); - QiNiuOssClient bean = SpringUtil.getBean(OssConstant.OssType.QINIU+ userId); + QiNiuOssClient bean = SpringUtil.getBean(OssConstant.OssType.QINIU + userId); bean.getQiNiuOssConfig().setBucketName(bucket); - if(bean.deleteBucket(bucket) == 1){ + if (bean.deleteBucket(bucket) == 1) { return Result.ok(); - }else + } else return Result.fail("删除桶失败"); } @@ -261,16 +291,15 @@ public class SchisandraOssQiniuController { public Result createBucket(@RequestParam String userId, @RequestParam String bucket, @RequestParam String region) { Preconditions.checkNotNull(userId, "不能为空"); Preconditions.checkNotNull(bucket, "不能为空"); - QiNiuOssClient bean = SpringUtil.getBean(OssConstant.OssType.QINIU+ userId); + QiNiuOssClient bean = SpringUtil.getBean(OssConstant.OssType.QINIU + userId); bean.getQiNiuOssConfig().setBucketName(bucket); bean.getQiNiuOssConfig().setRegion(region); - if(bean.createBucket(bucket, region) == 1){ + if (bean.createBucket(bucket, region) == 1) { return Result.ok(bean.createBucket(bucket, region)); - }else + } else return Result.fail("创建桶失败"); - } /** @@ -280,7 +309,7 @@ public class SchisandraOssQiniuController { * @date: 2024/7/8 13:34 */ @PostMapping("init") - public Result initQiniu(@RequestParam String userId,@RequestParam String Id) { + public Result initQiniu(@RequestParam String userId, @RequestParam String Id) { if (log.isInfoEnabled()) { log.info("SchisandraOssQiniuController.init.userId:{}", userId); } @@ -289,15 +318,15 @@ public class SchisandraOssQiniuController { bo.setId(Long.valueOf(Id)); bo.setUserId(Long.valueOf(userId)); bo.setStatus(String.valueOf(true)); - if(schisandraOssQiniuDomainService.update(bo)){ + if (schisandraOssQiniuDomainService.update(bo)) { return qiNiuOssConfiguration.qiNiuOssClient(userId); - }else{ + } else { return Result.fail("初始化失败"); } } @PostMapping("shutdown") - public Result shutdownQiniu(@RequestParam String userId,@RequestParam String Id){ + public Result shutdownQiniu(@RequestParam String userId, @RequestParam String Id) { if (log.isInfoEnabled()) { log.info("SchisandraOssQiniuController.shutdown.userId:{}", JSON.toJSONString(userId)); } @@ -305,10 +334,10 @@ public class SchisandraOssQiniuController { bo.setId(Long.valueOf(Id)); bo.setUserId(Long.valueOf(userId)); bo.setStatus(String.valueOf(false)); - if(schisandraOssQiniuDomainService.update(bo)){ + if (schisandraOssQiniuDomainService.update(bo)) { SpringUtil.unregisterBean(OssConstant.OssType.QINIU + userId); return Result.ok("关闭成功"); - }else{ + } else { return Result.fail("关闭失败"); } } diff --git a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/controller/SchisandraOssTencentController.java b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/controller/SchisandraOssTencentController.java index e0b5b75..781e258 100644 --- a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/controller/SchisandraOssTencentController.java +++ b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/controller/SchisandraOssTencentController.java @@ -1,25 +1,19 @@ package com.schisandra.oss.application.controller; +import cn.hutool.core.util.ZipUtil; import cn.hutool.extra.spring.SpringUtil; import com.alibaba.fastjson.JSON; import com.google.common.base.Preconditions; - import com.qcloud.cos.model.CopyResult; import com.qcloud.cos.model.DeleteObjectsRequest; import com.qcloud.cos.model.DeleteObjectsResult; import com.schisandra.oss.application.convert.SchisandraOssTencentDTOConverter; -import com.schisandra.oss.application.convert.SchisandraOssUpDTOConverter; import com.schisandra.oss.application.dto.SchisandraOssTencentDTO; - -import com.schisandra.oss.application.dto.SchisandraOssUpDTO; import com.schisandra.oss.application.oss.constant.OssConstant; -import com.schisandra.oss.application.oss.core.minio.MinioOssClient; import com.schisandra.oss.application.oss.core.tencent.TencentOssClient; import com.schisandra.oss.application.oss.core.tencent.TencentOssConfiguration; import com.schisandra.oss.common.entity.Result; -import com.schisandra.oss.domain.bo.SchisandraOssAliBO; import com.schisandra.oss.domain.bo.SchisandraOssTencentBO; -import com.schisandra.oss.domain.bo.SchisandraOssUpBO; import com.schisandra.oss.domain.redis.RedisUtil; import com.schisandra.oss.domain.service.SchisandraOssTencentDomainService; import io.minio.errors.*; @@ -27,17 +21,26 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.Cacheable; import org.springframework.cache.annotation.Caching; +import org.springframework.util.StreamUtils; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; +import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServletResponse; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.net.URLEncoder; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; /** * 腾讯云对象存储配置表 controller @@ -155,7 +158,7 @@ public class SchisandraOssTencentController { * @author zlg * @date: 2024/6/29 13:27 */ - @CacheEvict(value = "TencentBucket", key = "#userId") +// @CacheEvict(value = "TencentBucket", key = "#userId") @PostMapping("createBucket") public Result createBucket(@RequestParam String userId, @RequestParam String bucket, @RequestParam String appId) { Preconditions.checkNotNull(userId, "不能为空"); @@ -232,7 +235,6 @@ public class SchisandraOssTencentController { } return Result.ok(bean.upLoadParts(is, target, file.getOriginalFilename())); } - /** * @description: 腾讯云对象存储下载文件 * @param: [userId, bucket, target] @@ -241,11 +243,43 @@ public class SchisandraOssTencentController { * @date: 2024/7/1 16:02 */ @GetMapping("downloadFile") - public Result downloadTencent(@RequestParam String userId, @RequestParam String bucket, @RequestParam String target) { + public Result downloadTencent(@RequestParam String userId, @RequestParam String bucket, @RequestParam List listObjectsArgs, HttpServletResponse response) throws IOException { Preconditions.checkNotNull(userId, "不能为空"); Preconditions.checkNotNull(bucket, "不能为空"); TencentOssClient bean = SpringUtil.getBean(OssConstant.OssType.TENCENT + userId); - return Result.ok(bean.downloadTencent(bucket, target)); + try { + HashMap list = bean.downloadTencentMany(bucket, listObjectsArgs); + response.setHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode("下载.zip", "UTF-8")); + response.setContentType("application/octet-stream"); + response.setCharacterEncoding("UTF-8"); + ServletOutputStream output = response.getOutputStream(); + String[] path = new String[list.size()]; + path = list.keySet().toArray(path); + //记录用户下载 + Date date = new Date(); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + String formattedDate = dateFormat.format(date); + AtomicInteger count = new AtomicInteger(0); + InputStream[] inputStream2 = new InputStream[list.size()]; + list.keySet().forEach( name -> { + try { + byte[] bytes = StreamUtils.copyToByteArray(list.get(name)); + inputStream2[count.get()] = new ByteArrayInputStream(bytes); + String key = redisUtil.buildKey(USER_OSS_PREFIX, "download", formattedDate, userId, name); + HashMap map = new HashMap<>(); + map.put("size", bytes.length); + map.put("time",formattedDate); + redisUtil.setNx(key, map, 30L, TimeUnit.DAYS); + count.incrementAndGet(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + ZipUtil.zip(output, path, inputStream2); + } catch (Exception e) { + return Result.fail(); + } + return Result.ok(); } /** @@ -306,9 +340,6 @@ public class SchisandraOssTencentController { } - - - /** * 新增腾讯云对象存储配置表 */ diff --git a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/controller/SchisandraUserOssController.java b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/controller/SchisandraUserOssController.java index 6dd09c9..8322db8 100644 --- a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/controller/SchisandraUserOssController.java +++ b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/controller/SchisandraUserOssController.java @@ -228,8 +228,7 @@ public class SchisandraUserOssController { @GetMapping("selectUploadFileDiagramPerMonth") public Result selectFileDiagramPerMonth(@RequestParam Long userId) { Preconditions.checkNotNull(userId, "userId不能为空"); - return Result.ok(schisandraFileHeatmapDomainService.selectFileDiagramPerMonth(String.valueOf(userId),0) - .get("month")); + return Result.ok(schisandraFileHeatmapDomainService.selectFileDiagramPerMonth(String.valueOf(userId),0)); } /** * @description: 查询用户下载文件每月 @@ -241,8 +240,7 @@ public class SchisandraUserOssController { @GetMapping("selectDownloadFileDiagramPerMonth") public Result selectDownloadFileDiagramPerMonth(@RequestParam Long userId) { Preconditions.checkNotNull(userId, "userId不能为空"); - return Result.ok(schisandraFileHeatmapDomainService.selectFileDiagramPerMonth(String.valueOf(userId),1) - .get("month")); + return Result.ok(schisandraFileHeatmapDomainService.selectFileDiagramPerMonth(String.valueOf(userId),1)); } /** * @description: 查询用户最近预览文件 diff --git a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/oss/core/qiniu/QiNiuOssClient.java b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/oss/core/qiniu/QiNiuOssClient.java index d5c58ef..e7aa0bf 100644 --- a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/oss/core/qiniu/QiNiuOssClient.java +++ b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/oss/core/qiniu/QiNiuOssClient.java @@ -10,6 +10,10 @@ import cn.hutool.core.util.ReflectUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.http.HttpResponse; import cn.hutool.http.HttpUtil; +import com.amazonaws.util.IOUtils; +import com.jd.platform.async.executor.Async; +import com.jd.platform.async.worker.WorkResult; +import com.jd.platform.async.wrapper.WorkerWrapper; import com.qiniu.common.QiniuException; import com.qiniu.storage.*; import com.qiniu.storage.model.FileInfo; @@ -39,9 +43,15 @@ import okhttp3.OkHttpClient; import okhttp3.Request; import java.io.*; +import java.net.HttpURLConnection; +import java.net.URL; import java.net.URLEncoder; import java.text.SimpleDateFormat; import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import static com.aliyuncs.utils.Base64Helper.encode; @@ -126,26 +136,51 @@ public class QiNiuOssClient implements StandardOssClient { } return infos; } + public HashMap downLoadMany(List target, String endpoint) throws ExecutionException, InterruptedException { + HashMap list = new HashMap<>(); + List wrappers=new ArrayList<>(); + target.forEach(path -> { + WorkerWrapper wrapper1 = new WorkerWrapper.Builder().worker((String dirname, Map allWrappers) -> { + try { + list.put(dirname,downLoad_open(dirname,endpoint)); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + }).param(path.toString()).id(path).callback((boolean success, String param, WorkResult re)->{ + }).build(); + wrappers.add(wrapper1); + }); + Async.beginWork(3500L,new ThreadPoolExecutor(10, 100, 5L, + TimeUnit.SECONDS, new LinkedBlockingQueue()), wrappers); + return list; + } /** - * 七牛图片下载 + * 七牛下载 * - * @param os targetName endpoint + * @param targetName endpoint * @param endpoint * @return */ @SneakyThrows - public void downLoad_open(OutputStream os, String targetName,String endpoint) { - String filename = URLEncoder.encode(targetName,"UTF-8").replace("+", "%20"); - DownloadUrl downloadUrl = new DownloadUrl(endpoint, false, getKey(filename, false)); + public InputStream downLoad_open(String targetName,String endpoint) { + DownloadUrl downloadUrl = new DownloadUrl(endpoint, false, getKey(targetName, false)); try { - String url = downloadUrl.buildURL(); - HttpUtil.download(url, os, false); + String urlStr = downloadUrl.buildURL(); + URL url = new URL(urlStr); + HttpURLConnection conn = (HttpURLConnection)url.openConnection(); + //设置超时间为3秒 + conn.setConnectTimeout(3*1000); + //防止屏蔽程序抓取而返回403错误 + conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)"); + //得到输入流 + InputStream inputStream = conn.getInputStream(); + return inputStream; } catch (QiniuException e) { - String errorMsg = String.format("%s下载失败", targetName); - log.error(errorMsg, e); - throw new OssException(errorMsg, e); + log.error(String.valueOf(e)); } + return null; } public List >selectAllBucket() throws IOException{ diff --git a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/oss/core/tencent/TencentOssClient.java b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/oss/core/tencent/TencentOssClient.java index 4350f4b..5839d08 100644 --- a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/oss/core/tencent/TencentOssClient.java +++ b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-application/schisandra-cloud-storage-oss-application-controller/src/main/java/com/schisandra/oss/application/oss/core/tencent/TencentOssClient.java @@ -11,6 +11,10 @@ import cn.hutool.core.io.unit.DataSizeUtil; import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.ReflectUtil; import cn.hutool.core.util.StrUtil; +import com.alibaba.druid.util.StringUtils; +import com.jd.platform.async.executor.Async; +import com.jd.platform.async.worker.WorkResult; +import com.jd.platform.async.wrapper.WorkerWrapper; import com.qcloud.cos.COSClient; import com.qcloud.cos.exception.CosClientException; import com.qcloud.cos.exception.CosServiceException; @@ -28,6 +32,7 @@ import com.schisandra.oss.application.oss.model.download.DownloadCheckPoint; import com.schisandra.oss.application.oss.model.download.DownloadObjectStat; import com.schisandra.oss.application.oss.model.upload.*; import com.schisandra.oss.application.oss.utils.OssPathUtil; +import io.minio.ListObjectsArgs; import io.minio.errors.*; import lombok.AllArgsConstructor; import lombok.Data; @@ -35,16 +40,12 @@ import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.io.*; import java.net.URL; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.*; import java.util.stream.Collectors; @Slf4j @@ -140,7 +141,7 @@ public class TencentOssClient implements StandardOssClient { list.forEach(bucket -> { HashMap names = new HashMap<>(); names.put("name",bucket.getName()); - names.put("size",getTencentBucketSize(bucket.getName())); +// names.put("size",getTencentBucketSize(bucket.getName())); list1.add(names); }); return list1; @@ -157,19 +158,44 @@ public class TencentOssClient implements StandardOssClient { } return DataSizeUtil.format(fileSize); } - public Download downloadTencent(String bucket, String target) { + public HashMap downloadTencentMany(String bucket, List target) throws ExecutionException, InterruptedException { + HashMap list = new HashMap<>(); + List wrappers=new ArrayList<>(); + target.forEach(path -> { + WorkerWrapper wrapper1 = new WorkerWrapper.Builder().worker((String dirName, Map allWrappers) -> { + try { + list.put(dirName,downloadTencent(bucket,dirName)); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + }).param(path.toString()).id(path).callback((boolean success, String param, WorkResult re)->{ + }).build(); + wrappers.add(wrapper1); + }); + Async.beginWork(3500L,new ThreadPoolExecutor(10, 100, 5L, + TimeUnit.SECONDS, new LinkedBlockingQueue()), wrappers); + return list; + } + public InputStream downloadTencent(String bucket, String target) { ExecutorService threadPool = Executors.newFixedThreadPool(4); // 传入一个threadpool, 若不传入线程池, 默认TransferManager中会生成一个单线程的线程池。 TransferManager transferManager = new TransferManager(cosClient, threadPool); - try { - Download download = transferManager.download(bucket, target, - new File("D:\\java_project\\schisandra-cloud-storage\\")); +// File file = new File("D:\\java_project\\schisandra-cloud-storage\\"); + File file=File.createTempFile("dsd",".tmp"); + + Download download = transferManager.download(bucket, target,file); download.waitForCompletion(); - return download; + InputStream is = new FileInputStream(file); + return is; } catch (InterruptedException e) { throw new RuntimeException(e); + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); } } diff --git a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-domain/src/main/java/com/schisandra/oss/domain/service/SchisandraFileHeatmapDomainService.java b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-domain/src/main/java/com/schisandra/oss/domain/service/SchisandraFileHeatmapDomainService.java index cddb4c1..3726a39 100644 --- a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-domain/src/main/java/com/schisandra/oss/domain/service/SchisandraFileHeatmapDomainService.java +++ b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-domain/src/main/java/com/schisandra/oss/domain/service/SchisandraFileHeatmapDomainService.java @@ -108,6 +108,6 @@ public interface SchisandraFileHeatmapDomainService { * @date: 2024/7/13 10:33 */ int insertFileHeatmapBatch(List schisandraFileHeatmapBOList); - HashMap selectFileDiagramPerMonth(String userId,int type); + List selectFileDiagramPerMonth(String userId,int type); List> selectRecentPreviewFile(String userId); } diff --git a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-domain/src/main/java/com/schisandra/oss/domain/service/SchisandraOssQiniuDomainService.java b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-domain/src/main/java/com/schisandra/oss/domain/service/SchisandraOssQiniuDomainService.java index f4613e6..1dbe957 100644 --- a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-domain/src/main/java/com/schisandra/oss/domain/service/SchisandraOssQiniuDomainService.java +++ b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-domain/src/main/java/com/schisandra/oss/domain/service/SchisandraOssQiniuDomainService.java @@ -36,4 +36,8 @@ public interface SchisandraOssQiniuDomainService { SchisandraOssQiniuBO getQiniuOssConfig(String userId); List selectAll(); + + SchisandraOssQiniuBO getEndpointByuserIdAndBucket(Long id,String bucket); + + } diff --git a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-domain/src/main/java/com/schisandra/oss/domain/service/impl/SchisandraFileHeatmapDomainServiceImpl.java b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-domain/src/main/java/com/schisandra/oss/domain/service/impl/SchisandraFileHeatmapDomainServiceImpl.java index dfe2018..8af3414 100644 --- a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-domain/src/main/java/com/schisandra/oss/domain/service/impl/SchisandraFileHeatmapDomainServiceImpl.java +++ b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-domain/src/main/java/com/schisandra/oss/domain/service/impl/SchisandraFileHeatmapDomainServiceImpl.java @@ -202,10 +202,16 @@ public class SchisandraFileHeatmapDomainServiceImpl implements SchisandraFileHea } @Override - public HashMap selectFileDiagramPerMonth(String userId,int type) { - HashMap map=new HashMap<>(); - map.put("month",schisandraFileHeatmapService.selectFileDiagramPerMonth(userId, type)); - return map; + public List selectFileDiagramPerMonth(String userId,int type) { + List rows=schisandraFileHeatmapService.selectFileDiagramPerMonth(userId, type); + List list2=new ArrayList<>(); + rows.forEach(obj->{ + List list1=new ArrayList<>(); + list1.add(String.valueOf(obj.get("date"))); + list1.add(String.valueOf(obj.get("count"))); + list2.add(list1); + }); + return list2; } @Override public List> selectRecentPreviewFile(String userId) { diff --git a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-domain/src/main/java/com/schisandra/oss/domain/service/impl/SchisandraOssQiniuDomainServiceImpl.java b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-domain/src/main/java/com/schisandra/oss/domain/service/impl/SchisandraOssQiniuDomainServiceImpl.java index a328a71..2976db8 100644 --- a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-domain/src/main/java/com/schisandra/oss/domain/service/impl/SchisandraOssQiniuDomainServiceImpl.java +++ b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-domain/src/main/java/com/schisandra/oss/domain/service/impl/SchisandraOssQiniuDomainServiceImpl.java @@ -87,4 +87,11 @@ public class SchisandraOssQiniuDomainServiceImpl implements SchisandraOssQiniuDo List list = schisandraOssQiniuService.selectAll(); return SchisandraOssQiniuBOConverter.INSTANCE.convertEntityToBOList(list); } + + @Override + public SchisandraOssQiniuBO getEndpointByuserIdAndBucket(Long userId, String bucket) { + SchisandraOssQiniu schisandraOssQiniu = schisandraOssQiniuService.getEndpointByuserIdAnd(userId,bucket); + SchisandraOssQiniuBO schisandraOssQiniuBO = SchisandraOssQiniuBOConverter.INSTANCE.convertEntityToBO(schisandraOssQiniu); + return schisandraOssQiniuBO; + } } diff --git a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-infra/src/main/java/com/schisandra/oss/infra/basic/entity/SchisandraOssQiniu.java b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-infra/src/main/java/com/schisandra/oss/infra/basic/entity/SchisandraOssQiniu.java index b80131a..9628ef6 100644 --- a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-infra/src/main/java/com/schisandra/oss/infra/basic/entity/SchisandraOssQiniu.java +++ b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-infra/src/main/java/com/schisandra/oss/infra/basic/entity/SchisandraOssQiniu.java @@ -33,6 +33,8 @@ public class SchisandraOssQiniu implements Serializable { @Column("user_id") private Long userId; + @Column("endpoint") + private String endpoint; /** * */ diff --git a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-infra/src/main/java/com/schisandra/oss/infra/basic/service/SchisandraOssQiniuService.java b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-infra/src/main/java/com/schisandra/oss/infra/basic/service/SchisandraOssQiniuService.java index 2ab6684..8c8053f 100644 --- a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-infra/src/main/java/com/schisandra/oss/infra/basic/service/SchisandraOssQiniuService.java +++ b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-infra/src/main/java/com/schisandra/oss/infra/basic/service/SchisandraOssQiniuService.java @@ -13,6 +13,8 @@ import java.util.List; */ public interface SchisandraOssQiniuService { + SchisandraOssQiniu getEndpointByuserIdAnd(Long userId, String bucket); + /** * 通过ID查询单条数据 * diff --git a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-infra/src/main/java/com/schisandra/oss/infra/basic/service/impl/SchisandraOssQiniuServiceImpl.java b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-infra/src/main/java/com/schisandra/oss/infra/basic/service/impl/SchisandraOssQiniuServiceImpl.java index 2014e4a..02f910a 100644 --- a/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-infra/src/main/java/com/schisandra/oss/infra/basic/service/impl/SchisandraOssQiniuServiceImpl.java +++ b/schisandra-cloud-storage-oss/schisandra-cloud-storage-oss-infra/src/main/java/com/schisandra/oss/infra/basic/service/impl/SchisandraOssQiniuServiceImpl.java @@ -21,6 +21,11 @@ public class SchisandraOssQiniuServiceImpl implements SchisandraOssQiniuService @Resource private SchisandraOssQiniuDao schisandraOssQiniuDao; + @Override + public SchisandraOssQiniu getEndpointByuserIdAnd(Long userId, String bucket) { + return schisandraOssQiniuDao.selectOneByCondition(SchisandraOssQiniuTableDef.SCHISANDRA_OSS_QINIU.USER_ID.eq(userId).and(SchisandraOssQiniuTableDef.SCHISANDRA_OSS_QINIU.BUCKET_NAME.eq(bucket))); + } + /** * 通过ID查询单条数据 *