|
@@ -0,0 +1,213 @@
|
|
|
+package com.persagy.dmp.file.service.impl;
|
|
|
+
|
|
|
+import cn.hutool.core.collection.CollUtil;
|
|
|
+import cn.hutool.core.io.IoUtil;
|
|
|
+import cn.hutool.core.lang.TypeReference;
|
|
|
+import cn.hutool.core.map.MapUtil;
|
|
|
+import cn.hutool.core.thread.ThreadUtil;
|
|
|
+import cn.hutool.core.util.CharsetUtil;
|
|
|
+import cn.hutool.core.util.StrUtil;
|
|
|
+import cn.hutool.http.HttpRequest;
|
|
|
+import cn.hutool.http.HttpResponse;
|
|
|
+import cn.hutool.http.HttpStatus;
|
|
|
+import cn.hutool.http.HttpUtil;
|
|
|
+import com.alibaba.fastjson.JSONArray;
|
|
|
+import com.alibaba.fastjson.JSONObject;
|
|
|
+import com.persagy.dmp.common.constant.CommonConstant;
|
|
|
+import com.persagy.dmp.common.constant.ResponseCode;
|
|
|
+import com.persagy.dmp.common.exception.BusinessException;
|
|
|
+import com.persagy.dmp.file.context.OldFileAppContext;
|
|
|
+import com.persagy.dmp.file.service.CompatibleOldFileService;
|
|
|
+import com.persagy.dmp.file.service.FileMigrateService;
|
|
|
+import lombok.Data;
|
|
|
+import lombok.RequiredArgsConstructor;
|
|
|
+import lombok.extern.slf4j.Slf4j;
|
|
|
+import org.springframework.stereotype.Service;
|
|
|
+
|
|
|
+import java.util.List;
|
|
|
+import java.util.Map;
|
|
|
+import java.util.Set;
|
|
|
+
|
|
|
+/***
|
|
|
+ * Description: hdfs转移到Minio服务
|
|
|
+ * @author : lijie
|
|
|
+ * Update By 2021/12/13 15:21
|
|
|
+ */
|
|
|
+@Service
|
|
|
+@Data
|
|
|
+@Slf4j
|
|
|
+@RequiredArgsConstructor
|
|
|
+public class ImageUrlToImageUrlServiceImpl implements FileMigrateService {
|
|
|
+
|
|
|
+ private final CompatibleOldFileService compatibleOldFileService;
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void migrateFiles(Map<String, Object> requestMap) {
|
|
|
+ final String fromImageServiceUrl = MapUtil.getStr(requestMap, "fromImageServiceUrl");
|
|
|
+ final String fromSystemId = MapUtil.getStr(requestMap, "fromSystemId");
|
|
|
+ final String fromSecret = MapUtil.getStr(requestMap, "fromSecret");
|
|
|
+ final String toImageServiceUrl = MapUtil.getStr(requestMap, "toImageServiceUrl");
|
|
|
+ final String toSystemId = MapUtil.getStr(requestMap, "toSystemId");
|
|
|
+ final String toSecret = MapUtil.getStr(requestMap, "toSecret");
|
|
|
+ final Set<String> fileKeys = MapUtil.get(requestMap, "keys", new TypeReference<Set<String>>() {});
|
|
|
+ final boolean deleteOldFileFlag = MapUtil.getBool(requestMap, "deleteOldFileFlag",false);
|
|
|
+ if(StrUtil.isBlank(fromImageServiceUrl)
|
|
|
+ || StrUtil.isBlank(fromSystemId)
|
|
|
+ || StrUtil.isBlank(toImageServiceUrl)
|
|
|
+ || StrUtil.isBlank(toSystemId)
|
|
|
+ || StrUtil.isBlank(toSecret)) {
|
|
|
+ throw new BusinessException(ResponseCode.A0400.getCode(), ResponseCode.A0400.getDesc());
|
|
|
+ }
|
|
|
+ if(deleteOldFileFlag && StrUtil.isBlank(fromSecret)) {
|
|
|
+ throw new BusinessException(ResponseCode.A0400.getCode(), ResponseCode.A0400.getDesc());
|
|
|
+ }
|
|
|
+ // 启动迁移 执行时间较长,后台线程运行
|
|
|
+ // TODO 需要加分布式锁,并支持查看执行情况
|
|
|
+ ThreadUtil.execute(() -> startMigrate(fromImageServiceUrl,fromSystemId,fromSecret,toImageServiceUrl,toSystemId,toSecret,deleteOldFileFlag,fileKeys));
|
|
|
+ }
|
|
|
+
|
|
|
+ /***
|
|
|
+ * Description: 启动数据迁移
|
|
|
+ * @param fromImageServiceUrl: 来源文件的地址
|
|
|
+ * @param fromSystemId:来源文件的用户id
|
|
|
+ * @param fromSecret:来源文件的用户密码
|
|
|
+ * @param toImageServiceUrl:去向文件的地址
|
|
|
+ * @param toSystemId:去向文件的用户id
|
|
|
+ * @param toSecret:去向文件的用户密码
|
|
|
+ * @param deleteOldFileFlag:是否删除旧文件的标记
|
|
|
+ * @param fileKeys:是否只迁移部分文件key列表
|
|
|
+ * @return : void
|
|
|
+ * @author : lijie
|
|
|
+ * @date :2021/9/9 23:31
|
|
|
+ * Update By lijie 2021/9/9 23:31
|
|
|
+ */
|
|
|
+ private void startMigrate(String fromImageServiceUrl, String fromSystemId, String fromSecret, String toImageServiceUrl,
|
|
|
+ String toSystemId, String toSecret, boolean deleteOldFileFlag, Set<String> fileKeys){
|
|
|
+ final Map<String,String> uriMap = MapUtil.of("/common/images_list","/common/image_get");
|
|
|
+ uriMap.put("/common/files_list","/common/file_get");
|
|
|
+ final String urlFormat = "{}{}?systemId={}";
|
|
|
+ for (String listSubUrl : uriMap.keySet()) {
|
|
|
+ String requestListUrl = StrUtil.format(urlFormat,fromImageServiceUrl,listSubUrl,fromSystemId);
|
|
|
+ transferFile(requestListUrl,null,fromImageServiceUrl,fromSystemId,fromSecret,deleteOldFileFlag,
|
|
|
+ fileKeys,uriMap.get(listSubUrl),toImageServiceUrl,toSystemId,toSecret);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ /***
|
|
|
+ * Description: 迁移文件
|
|
|
+ * @param requestListUrl : 查询文件的url
|
|
|
+ * return : void
|
|
|
+ * @author : lijie
|
|
|
+ * Update By 2021/12/13 17:36
|
|
|
+ */
|
|
|
+ private void transferFile(String requestListUrl,String fromKey,String fromImageServiceUrl, String fromSystemId,
|
|
|
+ String fromSecret, Boolean deleteOldFileFlag,Set<String> destFileKeys, String getUrl,
|
|
|
+ String toImageServiceUrl,String toSystemId,String toSecret) {
|
|
|
+ final int pageSize = 1000;
|
|
|
+ JSONObject param = new JSONObject();
|
|
|
+ // 1.如果指定文件列表不为空则直接迁移指定列表的文件
|
|
|
+ if (CollUtil.isNotEmpty(destFileKeys)){
|
|
|
+ // 指定迁移key列表
|
|
|
+ transferFileInternal(destFileKeys,fromImageServiceUrl,fromSystemId,fromSecret,deleteOldFileFlag,getUrl,
|
|
|
+ toImageServiceUrl,toSystemId,toSecret);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ // 2.不指定迁移文件列表则全部迁移
|
|
|
+ // 1.组装参数
|
|
|
+ param.put("maxCount",pageSize);
|
|
|
+ if (StrUtil.isNotBlank(fromKey)){
|
|
|
+ param.put("from",fromKey);
|
|
|
+ }
|
|
|
+ // 2.判断请求参数是否合法
|
|
|
+ String post = HttpUtil.post(requestListUrl, param.toJSONString());
|
|
|
+ if (StrUtil.isBlank(post) || !post.startsWith(StrUtil.DELIM_START)){
|
|
|
+ log.error("请求查询文件列表出错:"+post);
|
|
|
+ }
|
|
|
+ JSONObject postJson = JSONObject.parseObject(post);
|
|
|
+ if (!CommonConstant.QUERY_SUCCESS.equals(postJson.getString(CommonConstant.RESULT))){
|
|
|
+ log.error("请求查询文件列表出错:"+post);
|
|
|
+ }
|
|
|
+ // 3.获得文件key列表
|
|
|
+ List<String> fileKeys = JSONArray.parseArray(postJson.getString("Content"), String.class);
|
|
|
+ if (CollUtil.isEmpty(fileKeys)){
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ // 4.根据文件key列表进行文件传输
|
|
|
+ transferFileInternal(CollUtil.newHashSet(fileKeys),fromImageServiceUrl,fromSystemId,fromSecret,deleteOldFileFlag,getUrl,
|
|
|
+ toImageServiceUrl,toSystemId,toSecret);
|
|
|
+ // 5.如果文件大小等于1000则进行分页查询
|
|
|
+ if (pageSize!=fileKeys.size()){
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ transferFile(requestListUrl,fileKeys.get(fileKeys.size() - 1),fromImageServiceUrl,fromSystemId,fromSecret,
|
|
|
+ deleteOldFileFlag,destFileKeys,getUrl,toImageServiceUrl,toSystemId,toSecret);
|
|
|
+ }
|
|
|
+ /***
|
|
|
+ * Description: 根据key列表迁移文件
|
|
|
+ * @param destFileKeys : key列表
|
|
|
+ * @param fromImageServiceUrl : 文件服务地址
|
|
|
+ * @param fromSystemId : 用户id
|
|
|
+ * @param fromSecret : 密码
|
|
|
+ * @param deleteOldFileFlag : 是否删除旧文件
|
|
|
+ * @param getSubUrl : 下载文件的subUrl
|
|
|
+ * return : void
|
|
|
+ * @author : lijie
|
|
|
+ * Update By 2021/12/13 18:22
|
|
|
+ */
|
|
|
+ private void transferFileInternal(Set<String> destFileKeys,String fromImageServiceUrl, String fromSystemId,
|
|
|
+ String fromSecret, Boolean deleteOldFileFlag, String getSubUrl,
|
|
|
+ String toImageServiceUrl,String toSystemId,String toSecret) {
|
|
|
+ final Map<String,String> deleteUriMap = MapUtil.of("/common/image_get","/common/images_delete");
|
|
|
+ deleteUriMap.put("/common/file_get","/common/files_delete");
|
|
|
+ final Map<String,String> uploadUriMap = MapUtil.of("/common/image_get","/common/image_upload");
|
|
|
+ uploadUriMap.put("/common/file_get","/common/file_upload");
|
|
|
+ final String downloadUrl = "{}{}?systemId={}&key={}";
|
|
|
+ final String uploadUrlFormat = "{}{}?systemId={}&secret={}&key={}";
|
|
|
+ // 1.分批迁移文件
|
|
|
+ for (String destFileKey : destFileKeys) {
|
|
|
+ if (StrUtil.isBlank(destFileKey)){
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ String downUrl = StrUtil.format(downloadUrl, fromImageServiceUrl, getSubUrl, fromSystemId,destFileKey);
|
|
|
+ byte[] bytes = HttpUtil.downloadBytes(downUrl);
|
|
|
+ String str = StrUtil.str(bytes, CharsetUtil.defaultCharset());
|
|
|
+ if ("File not existed".equalsIgnoreCase(str) || "Image not existed".equalsIgnoreCase(str)){
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ String uploadUrl = StrUtil.format(uploadUrlFormat, toImageServiceUrl, uploadUriMap.get(getSubUrl), toSystemId, toSecret, destFileKey);
|
|
|
+ HttpRequest httpRequest = HttpUtil.createPost(uploadUrl);
|
|
|
+ httpRequest.body(bytes);
|
|
|
+ httpRequest.header("Content-Type","application/json");
|
|
|
+ HttpResponse response = httpRequest.execute();
|
|
|
+ if(HttpStatus.HTTP_OK != response.getStatus()){
|
|
|
+ log.debug("上传文件失败,返回的结果:"+JSONObject.toJSONString(response.body()));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ // 2.删除文件
|
|
|
+ deleteFileByKeys(fromImageServiceUrl,fromSystemId,fromSecret,deleteOldFileFlag,destFileKeys,deleteUriMap.get(getSubUrl));
|
|
|
+ }
|
|
|
+
|
|
|
+ /***
|
|
|
+ * Description: 删除文件
|
|
|
+ * @param imageServiceUrl : 文件服务地址
|
|
|
+ * @param systemId : 用户名
|
|
|
+ * @param secret : 密码
|
|
|
+ * @param deleteOldFileFlag : 是否删除旧文件,true-删除,false-不删除
|
|
|
+ * @param destFileKeys : 删除的文件key列表
|
|
|
+ * @param deleteSubUrl : 删除文件的地址
|
|
|
+ * return : void
|
|
|
+ * @author : lijie
|
|
|
+ * Update By 2021/12/13 18:11
|
|
|
+ */
|
|
|
+ private void deleteFileByKeys(String imageServiceUrl, String systemId, String secret, Boolean deleteOldFileFlag, Set<String> destFileKeys, String deleteSubUrl) {
|
|
|
+ if (!deleteOldFileFlag || CollUtil.isEmpty(destFileKeys)){
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ final String deleteUrlFormat = "{}{}?systemId={}&secret={}";
|
|
|
+ JSONObject param = new JSONObject();
|
|
|
+ param.put("keys",destFileKeys);
|
|
|
+ String deleteUrl = StrUtil.format(deleteUrlFormat, imageServiceUrl, deleteSubUrl, systemId, secret);
|
|
|
+ String post = HttpUtil.post(deleteUrl, param.toJSONString());
|
|
|
+ log.debug("批量删除文件结果:"+post);
|
|
|
+ }
|
|
|
+
|
|
|
+}
|