|  | @@ -0,0 +1,213 @@
 | 
	
		
			
				|  |  | +package com.persagy.dmp.file.service.impl;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +import cn.hutool.core.collection.CollUtil;
 | 
	
		
			
				|  |  | +import cn.hutool.core.io.IoUtil;
 | 
	
		
			
				|  |  | +import cn.hutool.core.lang.TypeReference;
 | 
	
		
			
				|  |  | +import cn.hutool.core.map.MapUtil;
 | 
	
		
			
				|  |  | +import cn.hutool.core.thread.ThreadUtil;
 | 
	
		
			
				|  |  | +import cn.hutool.core.util.CharsetUtil;
 | 
	
		
			
				|  |  | +import cn.hutool.core.util.StrUtil;
 | 
	
		
			
				|  |  | +import cn.hutool.http.HttpRequest;
 | 
	
		
			
				|  |  | +import cn.hutool.http.HttpResponse;
 | 
	
		
			
				|  |  | +import cn.hutool.http.HttpStatus;
 | 
	
		
			
				|  |  | +import cn.hutool.http.HttpUtil;
 | 
	
		
			
				|  |  | +import com.alibaba.fastjson.JSONArray;
 | 
	
		
			
				|  |  | +import com.alibaba.fastjson.JSONObject;
 | 
	
		
			
				|  |  | +import com.persagy.dmp.common.constant.CommonConstant;
 | 
	
		
			
				|  |  | +import com.persagy.dmp.common.constant.ResponseCode;
 | 
	
		
			
				|  |  | +import com.persagy.dmp.common.exception.BusinessException;
 | 
	
		
			
				|  |  | +import com.persagy.dmp.file.context.OldFileAppContext;
 | 
	
		
			
				|  |  | +import com.persagy.dmp.file.service.CompatibleOldFileService;
 | 
	
		
			
				|  |  | +import com.persagy.dmp.file.service.FileMigrateService;
 | 
	
		
			
				|  |  | +import lombok.Data;
 | 
	
		
			
				|  |  | +import lombok.RequiredArgsConstructor;
 | 
	
		
			
				|  |  | +import lombok.extern.slf4j.Slf4j;
 | 
	
		
			
				|  |  | +import org.springframework.stereotype.Service;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +import java.util.List;
 | 
	
		
			
				|  |  | +import java.util.Map;
 | 
	
		
			
				|  |  | +import java.util.Set;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +/***
 | 
	
		
			
				|  |  | + * Description: hdfs转移到Minio服务
 | 
	
		
			
				|  |  | + * @author : lijie
 | 
	
		
			
				|  |  | + * Update By 2021/12/13 15:21
 | 
	
		
			
				|  |  | + */
 | 
	
		
			
				|  |  | +@Service
 | 
	
		
			
				|  |  | +@Data
 | 
	
		
			
				|  |  | +@Slf4j
 | 
	
		
			
				|  |  | +@RequiredArgsConstructor
 | 
	
		
			
				|  |  | +public class ImageUrlToImageUrlServiceImpl implements FileMigrateService {
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +    private final CompatibleOldFileService compatibleOldFileService;
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +    @Override
 | 
	
		
			
				|  |  | +    public void migrateFiles(Map<String, Object> requestMap) {
 | 
	
		
			
				|  |  | +        final String fromImageServiceUrl = MapUtil.getStr(requestMap, "fromImageServiceUrl");
 | 
	
		
			
				|  |  | +        final String fromSystemId = MapUtil.getStr(requestMap, "fromSystemId");
 | 
	
		
			
				|  |  | +        final String fromSecret = MapUtil.getStr(requestMap, "fromSecret");
 | 
	
		
			
				|  |  | +        final String toImageServiceUrl = MapUtil.getStr(requestMap, "toImageServiceUrl");
 | 
	
		
			
				|  |  | +        final String toSystemId = MapUtil.getStr(requestMap, "toSystemId");
 | 
	
		
			
				|  |  | +        final String toSecret = MapUtil.getStr(requestMap, "toSecret");
 | 
	
		
			
				|  |  | +        final Set<String> fileKeys = MapUtil.get(requestMap, "keys", new TypeReference<Set<String>>() {});
 | 
	
		
			
				|  |  | +        final boolean deleteOldFileFlag = MapUtil.getBool(requestMap, "deleteOldFileFlag",false);
 | 
	
		
			
				|  |  | +        if(StrUtil.isBlank(fromImageServiceUrl)
 | 
	
		
			
				|  |  | +                || StrUtil.isBlank(fromSystemId)
 | 
	
		
			
				|  |  | +                || StrUtil.isBlank(toImageServiceUrl)
 | 
	
		
			
				|  |  | +                || StrUtil.isBlank(toSystemId)
 | 
	
		
			
				|  |  | +                || StrUtil.isBlank(toSecret)) {
 | 
	
		
			
				|  |  | +            throw new BusinessException(ResponseCode.A0400.getCode(), ResponseCode.A0400.getDesc());
 | 
	
		
			
				|  |  | +        }
 | 
	
		
			
				|  |  | +        if(deleteOldFileFlag && StrUtil.isBlank(fromSecret)) {
 | 
	
		
			
				|  |  | +            throw new BusinessException(ResponseCode.A0400.getCode(), ResponseCode.A0400.getDesc());
 | 
	
		
			
				|  |  | +        }
 | 
	
		
			
				|  |  | +        // 启动迁移 执行时间较长,后台线程运行
 | 
	
		
			
				|  |  | +        // TODO 需要加分布式锁,并支持查看执行情况
 | 
	
		
			
				|  |  | +        ThreadUtil.execute(() -> startMigrate(fromImageServiceUrl,fromSystemId,fromSecret,toImageServiceUrl,toSystemId,toSecret,deleteOldFileFlag,fileKeys));
 | 
	
		
			
				|  |  | +    }
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +    /***
 | 
	
		
			
				|  |  | +     * Description: 启动数据迁移
 | 
	
		
			
				|  |  | +     * @param fromImageServiceUrl: 来源文件的地址
 | 
	
		
			
				|  |  | +     * @param fromSystemId:来源文件的用户id
 | 
	
		
			
				|  |  | +     * @param fromSecret:来源文件的用户密码
 | 
	
		
			
				|  |  | +     * @param toImageServiceUrl:去向文件的地址
 | 
	
		
			
				|  |  | +     * @param toSystemId:去向文件的用户id
 | 
	
		
			
				|  |  | +     * @param toSecret:去向文件的用户密码
 | 
	
		
			
				|  |  | +     * @param deleteOldFileFlag:是否删除旧文件的标记
 | 
	
		
			
				|  |  | +     * @param fileKeys:是否只迁移部分文件key列表
 | 
	
		
			
				|  |  | +     * @return : void
 | 
	
		
			
				|  |  | +     * @author : lijie
 | 
	
		
			
				|  |  | +     * @date :2021/9/9 23:31
 | 
	
		
			
				|  |  | +     * Update By lijie 2021/9/9 23:31
 | 
	
		
			
				|  |  | +     */
 | 
	
		
			
				|  |  | +    private void startMigrate(String fromImageServiceUrl, String fromSystemId, String fromSecret, String toImageServiceUrl,
 | 
	
		
			
				|  |  | +                              String toSystemId, String toSecret, boolean deleteOldFileFlag, Set<String> fileKeys){
 | 
	
		
			
				|  |  | +        final Map<String,String> uriMap = MapUtil.of("/common/images_list","/common/image_get");
 | 
	
		
			
				|  |  | +        uriMap.put("/common/files_list","/common/file_get");
 | 
	
		
			
				|  |  | +        final String urlFormat = "{}{}?systemId={}";
 | 
	
		
			
				|  |  | +        for (String listSubUrl : uriMap.keySet()) {
 | 
	
		
			
				|  |  | +            String requestListUrl = StrUtil.format(urlFormat,fromImageServiceUrl,listSubUrl,fromSystemId);
 | 
	
		
			
				|  |  | +            transferFile(requestListUrl,null,fromImageServiceUrl,fromSystemId,fromSecret,deleteOldFileFlag,
 | 
	
		
			
				|  |  | +                    fileKeys,uriMap.get(listSubUrl),toImageServiceUrl,toSystemId,toSecret);
 | 
	
		
			
				|  |  | +        }
 | 
	
		
			
				|  |  | +    }
 | 
	
		
			
				|  |  | +    /***
 | 
	
		
			
				|  |  | +     * Description: 迁移文件
 | 
	
		
			
				|  |  | +     * @param requestListUrl : 查询文件的url
 | 
	
		
			
				|  |  | +     * return : void
 | 
	
		
			
				|  |  | +     * @author : lijie
 | 
	
		
			
				|  |  | +     * Update By 2021/12/13 17:36
 | 
	
		
			
				|  |  | +     */
 | 
	
		
			
				|  |  | +    private void transferFile(String requestListUrl,String fromKey,String fromImageServiceUrl, String fromSystemId,
 | 
	
		
			
				|  |  | +                              String fromSecret, Boolean deleteOldFileFlag,Set<String> destFileKeys, String getUrl,
 | 
	
		
			
				|  |  | +                              String toImageServiceUrl,String toSystemId,String toSecret) {
 | 
	
		
			
				|  |  | +        final int pageSize = 1000;
 | 
	
		
			
				|  |  | +        JSONObject param = new JSONObject();
 | 
	
		
			
				|  |  | +        // 1.如果指定文件列表不为空则直接迁移指定列表的文件
 | 
	
		
			
				|  |  | +        if (CollUtil.isNotEmpty(destFileKeys)){
 | 
	
		
			
				|  |  | +            // 指定迁移key列表
 | 
	
		
			
				|  |  | +            transferFileInternal(destFileKeys,fromImageServiceUrl,fromSystemId,fromSecret,deleteOldFileFlag,getUrl,
 | 
	
		
			
				|  |  | +                    toImageServiceUrl,toSystemId,toSecret);
 | 
	
		
			
				|  |  | +            return;
 | 
	
		
			
				|  |  | +        }
 | 
	
		
			
				|  |  | +        // 2.不指定迁移文件列表则全部迁移
 | 
	
		
			
				|  |  | +        // 1.组装参数
 | 
	
		
			
				|  |  | +        param.put("maxCount",pageSize);
 | 
	
		
			
				|  |  | +        if (StrUtil.isNotBlank(fromKey)){
 | 
	
		
			
				|  |  | +            param.put("from",fromKey);
 | 
	
		
			
				|  |  | +        }
 | 
	
		
			
				|  |  | +        // 2.判断请求参数是否合法
 | 
	
		
			
				|  |  | +        String post = HttpUtil.post(requestListUrl, param.toJSONString());
 | 
	
		
			
				|  |  | +        if (StrUtil.isBlank(post) || !post.startsWith(StrUtil.DELIM_START)){
 | 
	
		
			
				|  |  | +            log.error("请求查询文件列表出错:"+post);
 | 
	
		
			
				|  |  | +        }
 | 
	
		
			
				|  |  | +        JSONObject postJson = JSONObject.parseObject(post);
 | 
	
		
			
				|  |  | +        if (!CommonConstant.QUERY_SUCCESS.equals(postJson.getString(CommonConstant.RESULT))){
 | 
	
		
			
				|  |  | +            log.error("请求查询文件列表出错:"+post);
 | 
	
		
			
				|  |  | +        }
 | 
	
		
			
				|  |  | +        // 3.获得文件key列表
 | 
	
		
			
				|  |  | +        List<String> fileKeys = JSONArray.parseArray(postJson.getString("Content"), String.class);
 | 
	
		
			
				|  |  | +        if (CollUtil.isEmpty(fileKeys)){
 | 
	
		
			
				|  |  | +            return;
 | 
	
		
			
				|  |  | +        }
 | 
	
		
			
				|  |  | +        // 4.根据文件key列表进行文件传输
 | 
	
		
			
				|  |  | +        transferFileInternal(CollUtil.newHashSet(fileKeys),fromImageServiceUrl,fromSystemId,fromSecret,deleteOldFileFlag,getUrl,
 | 
	
		
			
				|  |  | +                toImageServiceUrl,toSystemId,toSecret);
 | 
	
		
			
				|  |  | +        // 5.如果文件大小等于1000则进行分页查询
 | 
	
		
			
				|  |  | +        if (pageSize!=fileKeys.size()){
 | 
	
		
			
				|  |  | +            return;
 | 
	
		
			
				|  |  | +        }
 | 
	
		
			
				|  |  | +        transferFile(requestListUrl,fileKeys.get(fileKeys.size() - 1),fromImageServiceUrl,fromSystemId,fromSecret,
 | 
	
		
			
				|  |  | +                deleteOldFileFlag,destFileKeys,getUrl,toImageServiceUrl,toSystemId,toSecret);
 | 
	
		
			
				|  |  | +    }
 | 
	
		
			
				|  |  | +    /***
 | 
	
		
			
				|  |  | +     * Description: 根据key列表迁移文件
 | 
	
		
			
				|  |  | +     * @param destFileKeys : key列表
 | 
	
		
			
				|  |  | +     * @param fromImageServiceUrl : 文件服务地址
 | 
	
		
			
				|  |  | +     * @param fromSystemId : 用户id
 | 
	
		
			
				|  |  | +     * @param fromSecret : 密码
 | 
	
		
			
				|  |  | +     * @param deleteOldFileFlag : 是否删除旧文件
 | 
	
		
			
				|  |  | +     * @param getSubUrl : 下载文件的subUrl
 | 
	
		
			
				|  |  | +     * return : void  
 | 
	
		
			
				|  |  | +     * @author : lijie
 | 
	
		
			
				|  |  | +     * Update By 2021/12/13 18:22
 | 
	
		
			
				|  |  | +     */
 | 
	
		
			
				|  |  | +    private void transferFileInternal(Set<String> destFileKeys,String fromImageServiceUrl, String fromSystemId,
 | 
	
		
			
				|  |  | +                                      String fromSecret, Boolean deleteOldFileFlag, String getSubUrl,
 | 
	
		
			
				|  |  | +                                      String toImageServiceUrl,String toSystemId,String toSecret) {
 | 
	
		
			
				|  |  | +        final Map<String,String> deleteUriMap = MapUtil.of("/common/image_get","/common/images_delete");
 | 
	
		
			
				|  |  | +        deleteUriMap.put("/common/file_get","/common/files_delete");
 | 
	
		
			
				|  |  | +        final Map<String,String> uploadUriMap = MapUtil.of("/common/image_get","/common/image_upload");
 | 
	
		
			
				|  |  | +        uploadUriMap.put("/common/file_get","/common/file_upload");
 | 
	
		
			
				|  |  | +        final String downloadUrl = "{}{}?systemId={}&key={}";
 | 
	
		
			
				|  |  | +        final String uploadUrlFormat = "{}{}?systemId={}&secret={}&key={}";
 | 
	
		
			
				|  |  | +        // 1.分批迁移文件
 | 
	
		
			
				|  |  | +        for (String destFileKey : destFileKeys) {
 | 
	
		
			
				|  |  | +            if (StrUtil.isBlank(destFileKey)){
 | 
	
		
			
				|  |  | +                continue;
 | 
	
		
			
				|  |  | +            }
 | 
	
		
			
				|  |  | +            String downUrl = StrUtil.format(downloadUrl, fromImageServiceUrl, getSubUrl, fromSystemId,destFileKey);
 | 
	
		
			
				|  |  | +            byte[] bytes = HttpUtil.downloadBytes(downUrl);
 | 
	
		
			
				|  |  | +            String str = StrUtil.str(bytes, CharsetUtil.defaultCharset());
 | 
	
		
			
				|  |  | +            if ("File not existed".equalsIgnoreCase(str) || "Image not existed".equalsIgnoreCase(str)){
 | 
	
		
			
				|  |  | +                continue;
 | 
	
		
			
				|  |  | +            }
 | 
	
		
			
				|  |  | +            String uploadUrl = StrUtil.format(uploadUrlFormat, toImageServiceUrl, uploadUriMap.get(getSubUrl), toSystemId, toSecret, destFileKey);
 | 
	
		
			
				|  |  | +            HttpRequest httpRequest = HttpUtil.createPost(uploadUrl);
 | 
	
		
			
				|  |  | +            httpRequest.body(bytes);
 | 
	
		
			
				|  |  | +            httpRequest.header("Content-Type","application/json");
 | 
	
		
			
				|  |  | +            HttpResponse response = httpRequest.execute();
 | 
	
		
			
				|  |  | +            if(HttpStatus.HTTP_OK != response.getStatus()){
 | 
	
		
			
				|  |  | +                log.debug("上传文件失败,返回的结果:"+JSONObject.toJSONString(response.body()));
 | 
	
		
			
				|  |  | +            }
 | 
	
		
			
				|  |  | +        }
 | 
	
		
			
				|  |  | +        // 2.删除文件
 | 
	
		
			
				|  |  | +        deleteFileByKeys(fromImageServiceUrl,fromSystemId,fromSecret,deleteOldFileFlag,destFileKeys,deleteUriMap.get(getSubUrl));
 | 
	
		
			
				|  |  | +    }
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +    /***
 | 
	
		
			
				|  |  | +     * Description: 删除文件
 | 
	
		
			
				|  |  | +     * @param imageServiceUrl : 文件服务地址
 | 
	
		
			
				|  |  | +     * @param systemId : 用户名
 | 
	
		
			
				|  |  | +     * @param secret : 密码
 | 
	
		
			
				|  |  | +     * @param deleteOldFileFlag : 是否删除旧文件,true-删除,false-不删除
 | 
	
		
			
				|  |  | +     * @param destFileKeys : 删除的文件key列表
 | 
	
		
			
				|  |  | +     * @param deleteSubUrl : 删除文件的地址
 | 
	
		
			
				|  |  | +     * return : void
 | 
	
		
			
				|  |  | +     * @author : lijie
 | 
	
		
			
				|  |  | +     * Update By 2021/12/13 18:11
 | 
	
		
			
				|  |  | +     */
 | 
	
		
			
				|  |  | +    private void deleteFileByKeys(String imageServiceUrl, String systemId, String secret, Boolean deleteOldFileFlag, Set<String> destFileKeys, String deleteSubUrl) {
 | 
	
		
			
				|  |  | +        if (!deleteOldFileFlag || CollUtil.isEmpty(destFileKeys)){
 | 
	
		
			
				|  |  | +            return;
 | 
	
		
			
				|  |  | +        }
 | 
	
		
			
				|  |  | +        final String deleteUrlFormat = "{}{}?systemId={}&secret={}";
 | 
	
		
			
				|  |  | +        JSONObject param = new JSONObject();
 | 
	
		
			
				|  |  | +        param.put("keys",destFileKeys);
 | 
	
		
			
				|  |  | +        String deleteUrl = StrUtil.format(deleteUrlFormat, imageServiceUrl, deleteSubUrl, systemId, secret);
 | 
	
		
			
				|  |  | +        String post = HttpUtil.post(deleteUrl, param.toJSONString());
 | 
	
		
			
				|  |  | +        log.debug("批量删除文件结果:"+post);
 | 
	
		
			
				|  |  | +    }
 | 
	
		
			
				|  |  | +
 | 
	
		
			
				|  |  | +}
 |