Kaynağa Gözat

fix bug:批量迁移文件时因单个文件失败导致线程中断

lijie 3 yıl önce
ebeveyn
işleme
be8d533058

+ 10 - 6
dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/service/impl/HdfsToMinioServiceImpl.java

@@ -119,13 +119,17 @@ public class HdfsToMinioServiceImpl implements FileMigrateService {
             RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path(fileBasePath), true);
             while (listFiles.hasNext()){
                 LocatedFileStatus fileStatus = listFiles.next();
-                if (fileStatus.isDirectory()){
-                    continue;
+                try {
+                    if (fileStatus.isDirectory()){
+                        continue;
+                    }
+                    Path path = fileStatus.getPath();
+                    FSDataInputStream dataInputStream = fileSystem.open(path);
+                    InputStream wrappedStream = dataInputStream.getWrappedStream();
+                    compatibleOldFileService.commonUploadFile(wrappedStream,bucketName,null);
+                }catch (Exception e){
+                    log.error("迁移单个文件失败,错误的文件key:"+fileStatus.getPath()+",错误结果:",e);
                 }
-                Path path = fileStatus.getPath();
-                FSDataInputStream dataInputStream = fileSystem.open(path);
-                InputStream wrappedStream = dataInputStream.getWrappedStream();
-                compatibleOldFileService.commonUploadFile(wrappedStream,bucketName,null);
             }
             // 处理完文件后直接删除hdfs文件夹
             if (deleteOldFileFlag){

+ 20 - 16
dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/service/impl/ImageUrlToImageUrlServiceImpl.java

@@ -202,22 +202,26 @@ public class ImageUrlToImageUrlServiceImpl implements FileMigrateService {
         final String uploadUrlFormat = "{}{}?systemId={}&secret={}&key={}";
         // 1.分批迁移文件
         for (String destFileKey : destFileKeys) {
-            if (StrUtil.isBlank(destFileKey)){
-                continue;
-            }
-            String downUrl = StrUtil.format(downloadUrl, fromImageServiceUrl, getSubUrl, fromSystemId,destFileKey);
-            byte[] bytes = HttpUtil.downloadBytes(downUrl);
-            String str = StrUtil.str(bytes, CharsetUtil.defaultCharset());
-            if ("File not existed".equalsIgnoreCase(str) || "Image not existed".equalsIgnoreCase(str)){
-                continue;
-            }
-            String uploadUrl = StrUtil.format(uploadUrlFormat, toImageServiceUrl, uploadUriMap.get(getSubUrl), toSystemId, toSecret, destFileKey);
-            HttpRequest httpRequest = HttpUtil.createPost(uploadUrl);
-            httpRequest.body(bytes);
-            httpRequest.header("Content-Type","application/json");
-            HttpResponse response = httpRequest.execute();
-            if(HttpStatus.HTTP_OK != response.getStatus()){
-                log.debug("上传文件失败,返回的结果:"+JSONObject.toJSONString(response.body()));
+            try {
+                if (StrUtil.isBlank(destFileKey)){
+                    continue;
+                }
+                String downUrl = StrUtil.format(downloadUrl, fromImageServiceUrl, getSubUrl, fromSystemId,destFileKey);
+                byte[] bytes = HttpUtil.downloadBytes(downUrl);
+                String str = StrUtil.str(bytes, CharsetUtil.defaultCharset());
+                if ("File not existed".equalsIgnoreCase(str) || "Image not existed".equalsIgnoreCase(str)){
+                    continue;
+                }
+                String uploadUrl = StrUtil.format(uploadUrlFormat, toImageServiceUrl, uploadUriMap.get(getSubUrl), toSystemId, toSecret, destFileKey);
+                HttpRequest httpRequest = HttpUtil.createPost(uploadUrl);
+                httpRequest.body(bytes);
+                httpRequest.header("Content-Type","application/json");
+                HttpResponse response = httpRequest.execute();
+                if(HttpStatus.HTTP_OK != response.getStatus()){
+                    log.debug("上传文件失败,返回的结果:"+JSONObject.toJSONString(response.body()));
+                }
+            }catch (Exception e){
+                log.error("迁移单个文件失败,错误的文件key:"+destFileKey+",错误结果:",e);
             }
         }
         // 2.删除文件

+ 14 - 10
dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/service/impl/ImageUrlToMinioServiceImpl.java

@@ -187,17 +187,21 @@ public class ImageUrlToMinioServiceImpl implements FileMigrateService {
         final String downloadUrl = "{}{}?systemId={}&key={}";
         // 1.分批迁移文件
         for (String destFileKey : destFileKeys) {
-            if (StrUtil.isBlank(destFileKey)){
-                continue;
-            }
-            String downUrl = StrUtil.format(downloadUrl, imageServiceUrl, getSubUrl, systemId,destFileKey);
-            byte[] bytes = HttpUtil.downloadBytes(downUrl);
-            String str = StrUtil.str(bytes, CharsetUtil.defaultCharset());
-            if ("File not existed".equalsIgnoreCase(str) || "Image not existed".equalsIgnoreCase(str)){
-                continue;
+            try {
+                if (StrUtil.isBlank(destFileKey)){
+                    continue;
+                }
+                String downUrl = StrUtil.format(downloadUrl, imageServiceUrl, getSubUrl, systemId,destFileKey);
+                byte[] bytes = HttpUtil.downloadBytes(downUrl);
+                String str = StrUtil.str(bytes, CharsetUtil.defaultCharset());
+                if ("File not existed".equalsIgnoreCase(str) || "Image not existed".equalsIgnoreCase(str)){
+                    continue;
+                }
+                OldFileAppContext.getContext().setKey(destFileKey);
+                compatibleOldFileService.commonUploadFile(IoUtil.toStream(bytes),bucketName,null);
+            }catch (Exception e){
+                log.error("迁移单个文件失败,错误的文件key:"+destFileKey+",错误结果:",e);
             }
-            OldFileAppContext.getContext().setKey(destFileKey);
-            compatibleOldFileService.commonUploadFile(IoUtil.toStream(bytes),bucketName,null);
         }
         // 2.删除文件
         deleteFileByKeys(imageServiceUrl,systemId,secret,deleteOldFileFlag,destFileKeys,deleteUriMap.get(getSubUrl));

+ 0 - 1
dmp-comp/dmp-auth/src/main/java/com/persagy/dmp/auth/aspect/RequestLogAspect.java

@@ -36,7 +36,6 @@ public class RequestLogAspect {
 
 	/** @Pointcut("execution(* *(..))") */
 	@Pointcut("execution(public * com.persagy.dmp..*.controller.*.*(..))")
-
 	public void controllerPoint() {
 	}