Forráskód Böngészése

功能:实现三种文件迁移方案的接口

lijie 3 éve
szülő
commit
3dc866033d

+ 4 - 4
dmp-cloud/dmp-file/pom.xml

@@ -17,6 +17,10 @@
             <groupId>com.persagy</groupId>
             <artifactId>integrated-config-client</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.persagy</groupId>
+            <artifactId>integrated-redis-spring-boot-starter</artifactId>
+        </dependency>
         <!-- 项目启动 -->
         <dependency>
             <groupId>com.persagy</groupId>
@@ -37,17 +41,14 @@
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>log4j-over-slf4j</artifactId>
-            <version>1.7.32</version>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
-            <version>2.5.1</version>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-hdfs</artifactId>
-            <version>2.5.1</version>
         </dependency>
         <dependency>
             <groupId>com.persagy</groupId>
@@ -57,7 +58,6 @@
         <dependency>
             <groupId>io.minio</groupId>
             <artifactId>minio</artifactId>
-            <version>8.3.0</version>
         </dependency>
     </dependencies>
 </project>

+ 42 - 0
dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/constant/FileMigrateEnum.java

@@ -0,0 +1,42 @@
+package com.persagy.dmp.file.constant;
+
+import lombok.Getter;
+/***
+ * Description: 文件迁移服务
+ * @author : lijie
+ * Update By 2021/12/13 15:08
+ */
+@Getter
+public enum FileMigrateEnum {
+
+    /** HDFS_TO_MINIO */
+    HDFS_TO_MINIO("0", "hdfsToMinioServiceImpl"),
+    /** IMAGE_URL_TO_MINIO */
+    IMAGE_URL_TO_MINIO("1", "imageUrlToMinioServiceImpl"),
+    /** HDFS */
+    IMAGE_URL_TO_IMAGE_URL("2", "imageUrlToImageUrlServiceImpl");
+
+    private String index;
+    private String name;
+
+    FileMigrateEnum(String index, String name) {
+        this.index = index;
+        this.name = name;
+    }
+
+    /**
+     * 获取Enum
+     * @param index
+     * @return
+     */
+    public static FileMigrateEnum load(String index) {
+        for (FileMigrateEnum value: values()) {
+            if (value.getIndex().equals(index)) {
+                return value;
+            }
+        }
+        return null;
+    }
+
+
+}

+ 7 - 182
dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/controller/FileMigrateController.java

@@ -4,6 +4,7 @@ import cn.hutool.core.map.MapUtil;
 import cn.hutool.core.thread.ThreadUtil;
 import cn.hutool.core.util.StrUtil;
 import com.baomidou.mybatisplus.core.toolkit.IdWorker;
+import com.persagy.common.lock.Lock;
 import com.persagy.dmp.common.constant.ResponseCode;
 import com.persagy.dmp.common.exception.BusinessException;
 import com.persagy.dmp.common.lang.PsDateTime;
@@ -11,6 +12,8 @@ import com.persagy.dmp.common.model.response.CommonResult;
 import com.persagy.dmp.common.utils.ResultHelper;
 import com.persagy.dmp.file.model.FileInfo;
 import com.persagy.dmp.file.model.FileInfoCreator;
+import com.persagy.dmp.file.service.FileMigrateFactory;
+import com.persagy.dmp.file.service.FileMigrateService;
 import com.persagy.dmp.file.service.IFileService;
 import io.minio.*;
 import lombok.RequiredArgsConstructor;
@@ -40,23 +43,6 @@ import java.util.*;
 @RequiredArgsConstructor
 public class FileMigrateController {
 
-    /**jar同级config目录下的位置*/
-    private static final String FILE_CONFIG_PATH="file:./config/";
-    /**Classpath位置*/
-    private static final String FILE_CLASSPATH_PATH="classpath:./";
-    /**hdfs-site.xml文件名称*/
-    private static final String HDFS_SITE_FILE_NAME ="hdfs-site.xml";
-    /**core-site.xml文件名称*/
-    private static final String HDFS_CORE_FILE_NAME ="core-site.xml";
-    /**hdfs访问地址*/
-    private static final String HDFS_URL_KEY="fs.defaultFS";
-    /** 文件key默认存储路径  */
-    private final static String BASE_FILE_PATH="/test/files";
-    /**hdfs访问用户名*/
-    private static final String HADOOP_USER_NAME="HADOOP_USER_NAME";
-
-    private final IFileService fileService;
-
     /**
      * 启动文件迁移
      * 迁移范围:hbase迁移至minio
@@ -71,173 +57,12 @@ public class FileMigrateController {
      * }
      */
     @PostMapping("/start")
-    public CommonResult<String> query(@RequestBody Map<String, Object> requestMap) {
+    public CommonResult<String> start(@RequestBody Map<String, Object> requestMap) {
         // 处理参数
-        final String hdfsUrl = MapUtil.getStr(requestMap, "hdfsUrl");
-        final String fileBasePath = MapUtil.getStr(requestMap, "fileBasePath");
-        final String hdfsUser = MapUtil.getStr(requestMap, "hdfsUser");
-        final String minioUrl = MapUtil.getStr(requestMap, "minioUrl");
-        final String minioUser = MapUtil.getStr(requestMap, "minioUser");
-        final String minioSecret = MapUtil.getStr(requestMap, "minioSecret");
-        final String groupCode = MapUtil.getStr(requestMap, "groupCode","persagy");
-        if(StrUtil.isBlank(hdfsUrl)
-                || StrUtil.isBlank(fileBasePath)
-                || StrUtil.isBlank(hdfsUser)
-                || StrUtil.isBlank(minioUrl)
-                || StrUtil.isBlank(minioUser)
-                || StrUtil.isBlank(minioSecret)) {
-            throw new BusinessException(ResponseCode.A0400.getCode(), ResponseCode.A0400.getDesc());
-        }
-        // 启动迁移 执行时间较长,后台线程运行 
-        // TODO 需要加分布式锁,并支持查看执行情况
-        ThreadUtil.execute(() -> startMigrate(hdfsUrl,fileBasePath,hdfsUser,minioUrl,minioUser,minioSecret,groupCode));
+        final String migrateType = MapUtil.getStr(requestMap, "migrateType");
+        FileMigrateService service = FileMigrateFactory.getService(migrateType);
+        service.migrateFiles(requestMap);
         return ResultHelper.success();
     }
-    /***
-     * Description: 启动数据迁移
-     * @param hdfsUrl : HDFS的文件路径
-     * @param fileBasePath : 文件路径
-     * @param hdfsUser : HDFS文件服务器访问权限用户
-     * @param minioUrl : minio地址
-     * @param accessKey : minio用户
-     * @param secretKey : minio密码
-     * @param groupCode : 集团编码
-     * @return : void  
-     * @author : lijie
-     * @date :2021/9/9 23:31
-     * Update By lijie 2021/9/9 23:31
-     */
-    private void startMigrate(String hdfsUrl, String fileBasePath, String hdfsUser, String minioUrl, String accessKey,
-                              String secretKey, String groupCode){
-        // 1.创建minio客户端
-        MinioClient minioClient = MinioClient.builder()
-                .endpoint(minioUrl)
-                .region("GMT+8")
-                .credentials(accessKey, secretKey)
-                .build();
-        // 2.创建HDFS文件流
-        FileSystem fileSystem = null;
-        try {
-            if (!minioClient.bucketExists(BucketExistsArgs.builder().bucket(groupCode).build())){
-                minioClient.makeBucket(MakeBucketArgs.builder().bucket(groupCode).region("GMT+8").build());
-            }
-            fileSystem = FileSystem.get(createConfig(hdfsUrl,hdfsUser));
-            RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path(BASE_FILE_PATH + "/" + fileBasePath), true);
-            while (listFiles.hasNext()){
-                LocatedFileStatus fileStatus = listFiles.next();
-                if (fileStatus.isDirectory()){
-                    continue;
-                }
-                Path path = fileStatus.getPath();
-                FileInfo fileInfo = FileInfoCreator.of(groupCode, IdWorker.getIdStr(), groupCode, path.getName());
-                fileInfo.setCreationTime(new PsDateTime());
-                fileService.insertFile(fileInfo);
-                FSDataInputStream dataInputStream = fileSystem.open(path);
-                InputStream wrappedStream = dataInputStream.getWrappedStream();
-                minioClient.putObject(PutObjectArgs.builder()
-                        .bucket(groupCode)
-                        .object(path.getName())
-                        .stream(wrappedStream,-1, 10485760).build());
-                System.out.println(fileStatus.getPath().getName());
-            }
-            // 处理完文件后直接删除hdfs文件夹
-            fileSystem.delete(new Path(BASE_FILE_PATH + "/" + fileBasePath), true);
-        }catch (Exception e){
-            log.error("转移文件失败",e);
-        }finally {
-            if (null!=fileSystem){
-                try {
-                    fileSystem.close();
-                } catch (IOException e) {
-                    log.error("关闭流失败",e);
-                }
-            }
-        }
-    }
-    /***
-     * Description: 创建HDFS配置类
-     * @param url : hdfs的服务器地址
-     * @return : org.apache.hadoop.conf.Configuration
-     * @author : lijie
-     * @date :2021/9/9 23:37
-     * Update By lijie 2021/9/9 23:37
-     */
-    public Configuration createConfig(String url,String accessKey) {
-        Configuration plainConfig = new Configuration();
-        try {
-            System.setProperty(HADOOP_USER_NAME,accessKey);
-            String hbaseSiteConfigFilePath = FILE_CONFIG_PATH+ HDFS_SITE_FILE_NAME;
-            String hbaseSiteClasspathFilePath = FILE_CLASSPATH_PATH+ HDFS_SITE_FILE_NAME;
-            String hbaseCoreConfigFilePath = FILE_CONFIG_PATH+ HDFS_CORE_FILE_NAME;
-            String hbaseCoreClasspathFilePath = FILE_CLASSPATH_PATH+ HDFS_CORE_FILE_NAME;
-            log.info("hdfs-site.xml文件存在情况,config目录:{},classPath:{}",checkResoucesExist(hbaseSiteConfigFilePath),
-                    checkResoucesExist(hbaseSiteClasspathFilePath));
-            log.info("core-site.xml文件存在情况,config目录:{},classPath:{}",checkResoucesExist(hbaseCoreConfigFilePath),
-                    checkResoucesExist(hbaseCoreClasspathFilePath));
-            log.info("classpath下的文件是否存在:"+checkResoucesExist(hbaseSiteClasspathFilePath));
-            // 导入hbase-site.xml文件
-            Resource[] siteResources = getResource(hbaseSiteConfigFilePath, hbaseSiteClasspathFilePath);
-            if (siteResources.length<1){
-                throw new BusinessException(ResponseCode.B0300.getCode(),"hdfs-site.xml文件不存在");
-            }
-            // 导入hbase-core.xml文件
-            Resource[] coreResources = getResource(hbaseCoreConfigFilePath, hbaseCoreClasspathFilePath);
-            if (coreResources.length<1){
-                throw new BusinessException(ResponseCode.B0300.getCode(),"core-site.xml文件不存在");
-            }
-            plainConfig.set(HDFS_URL_KEY,url);
-            Resource siteResource = siteResources[0];
-            plainConfig.addResource(siteResource.getInputStream());
-            Resource coreResource = coreResources[0];
-            plainConfig.addResource(coreResource.getInputStream());
-        }catch (Exception e) {
-            log.error("加载HDFS配置文件失败!",e);
-        }
-        return plainConfig;
-    }
 
-    /***
-     * @Description: 校验资源文件是否存在
-     * @param configFilePath : jar包同级目录下的配置文件
-     * @return : com.alibaba.fastjson.JSONArray
-     * @author: lijie
-     * @Date:2020/6/6 19:43
-     * Update By lijie 2020/6/6 19:43
-     */
-    private static boolean checkResoucesExist(String configFilePath) throws IOException {
-        ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
-        Resource[] resolverResources = resolver.getResources(configFilePath);
-        if (resolverResources.length > 0 && resolverResources[0].exists()){
-            return true;
-        }
-        return false;
-    }
-
-    /***
-     * Description: 获取资源文件方法
-     * @param configFilePath : 配置文件路径
-     * @param classpathFilePath : classpath下的文件
-     * @return : org.springframework.core.io.Resource[]
-     * @author : lijie
-     * @date :2021/3/1 14:25
-     * Update By lijie 2021/3/1 14:25
-     */
-    public static Resource[] getResource(String configFilePath, String classpathFilePath)
-            throws IOException {
-        ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
-        Resource[] resolverResources = resolver.getResources(configFilePath);
-        if (resolverResources.length > 0) {
-            boolean exist = true;
-            for (Resource resource : resolverResources) {
-                if (!resource.exists()) {
-                    exist = false;
-                    break;
-                }
-            }
-            if(exist) {
-                return resolverResources;
-            }
-        }
-        return resolver.getResources(classpathFilePath);
-    }
 }

+ 8 - 0
dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/service/CompatibleOldFileService.java

@@ -75,4 +75,12 @@ public interface CompatibleOldFileService {
      * Update By 2021/12/9 19:39
      */
     String registerMultipartUpload();
+    /***
+     * Description: 通用的文件上传逻辑
+     * @param inputStream : 文件流
+     * @return : java.lang.String
+     * @author : lijie
+     * Update By lijie 2021/12/9 10:17
+     */
+    void commonUploadFile(InputStream inputStream, String bucketName, String businessId);
 }

+ 37 - 0
dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/service/FileMigrateFactory.java

@@ -0,0 +1,37 @@
+package com.persagy.dmp.file.service;
+
+import cn.hutool.core.util.ObjectUtil;
+import cn.hutool.core.util.StrUtil;
+import com.persagy.dmp.common.constant.ResponseCode;
+import com.persagy.dmp.common.exception.BusinessException;
+import com.persagy.dmp.common.helper.SpringHelper;
+import com.persagy.dmp.file.constant.FileMigrateEnum;
+import com.persagy.dmp.file.constant.FileStorageEnum;
+
+/**
+ * 文件存储管理工厂
+ * @author Charlie Yu
+ * @date 2021-05-15
+ */
+public class FileMigrateFactory {
+
+    /**
+     * 获取文件存储服务
+     * @return
+     */
+    public static FileMigrateService getService(String migrateType) {
+        if (StrUtil.isBlank(migrateType)){
+            throw new BusinessException(ResponseCode.A0400.getCode(), ResponseCode.A0400.getDesc());
+        }
+        FileMigrateEnum fileMigrateEnum = FileMigrateEnum.load(migrateType);
+        if (null==fileMigrateEnum){
+            throw new BusinessException(ResponseCode.A0400.getCode(), ResponseCode.A0400.getDesc());
+        }
+        String beanName = fileMigrateEnum.getName();
+        FileMigrateService fileMigrateService = SpringHelper.getBean(beanName, FileMigrateService.class);
+        if (ObjectUtil.isEmpty(fileMigrateService)){
+            throw new BusinessException(ResponseCode.A0400.getCode(), ResponseCode.A0400.getDesc());
+        }
+        return fileMigrateService;
+    }
+}

+ 12 - 0
dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/service/FileMigrateService.java

@@ -0,0 +1,12 @@
+package com.persagy.dmp.file.service;
+
+import java.util.Map;
+
+/***
+ * Description: 文件迁移服务
+ * @author : lijie
+ * Update By 2021/12/13 15:20
+ */
+public interface FileMigrateService {
+    void migrateFiles(Map<String, Object> requestMap);
+}

+ 4 - 3
dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/service/impl/CompatibleOldFileServiceImpl.java

@@ -275,7 +275,7 @@ public class CompatibleOldFileServiceImpl implements CompatibleOldFileService {
                 || StrUtil.isBlank(fileInfo.getFilePath())
                 || !FileCommonConst.UPLOAD_SUCCESS.equals(fileInfo.getUploadStatus())
                 || !service.exists(fileInfo.getFileBucket(),fileInfo.getFilePath())){
-            IoUtil.writeUtf8(response.getOutputStream(),Boolean.FALSE,"file not existed");
+            IoUtil.writeUtf8(response.getOutputStream(),Boolean.FALSE,"File not existed");
             return;
         }
         // 3.下载文件
@@ -283,13 +283,14 @@ public class CompatibleOldFileServiceImpl implements CompatibleOldFileService {
     }
 
     /***
-     * Description: 通用的文件删除逻辑
+     * Description: 通用的文件上传逻辑
      * @param inputStream : 文件流
      * @return : java.lang.String
      * @author : lijie
      * Update By lijie 2021/12/9 10:17
      */
-    private void commonUploadFile(InputStream inputStream,String bucketName,String businessId) {
+    @Override
+    public void commonUploadFile(InputStream inputStream,String bucketName,String businessId) {
         IFileStorageService service = FileStorageFactory.getService();
         // 1.获得文件大小
         byte[] streamBytes = FileStorageHelper.getStreamBytes(inputStream);

+ 230 - 0
dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/service/impl/HdfsToMinioServiceImpl.java

@@ -0,0 +1,230 @@
+package com.persagy.dmp.file.service.impl;
+
+import cn.hutool.core.map.MapUtil;
+import cn.hutool.core.thread.ThreadUtil;
+import cn.hutool.core.util.StrUtil;
+import com.baomidou.mybatisplus.core.toolkit.IdWorker;
+import com.persagy.dmp.common.constant.ResponseCode;
+import com.persagy.dmp.common.exception.BusinessException;
+import com.persagy.dmp.common.lang.PsDateTime;
+import com.persagy.dmp.file.model.FileInfo;
+import com.persagy.dmp.file.model.FileInfoCreator;
+import com.persagy.dmp.file.service.CompatibleOldFileService;
+import com.persagy.dmp.file.service.FileMd5Service;
+import com.persagy.dmp.file.service.FileMigrateService;
+import com.persagy.dmp.file.service.IFileService;
+import io.minio.BucketExistsArgs;
+import io.minio.MakeBucketArgs;
+import io.minio.MinioClient;
+import io.minio.PutObjectArgs;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.*;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
+import org.springframework.core.io.support.ResourcePatternResolver;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Map;
+
+/***
+ * Description: hdfs转移到Minio服务
+ * @author : lijie
+ * Update By 2021/12/13 15:21
+ */
+@Service
+@Data
+@Slf4j
+@RequiredArgsConstructor
+public class HdfsToMinioServiceImpl implements FileMigrateService {
+
+    /**jar同级config目录下的位置*/
+    private static final String FILE_CONFIG_PATH="file:./config/";
+    /**Classpath位置*/
+    private static final String FILE_CLASSPATH_PATH="classpath:./";
+    /**hdfs-site.xml文件名称*/
+    private static final String HDFS_SITE_FILE_NAME ="hdfs-site.xml";
+    /**core-site.xml文件名称*/
+    private static final String HDFS_CORE_FILE_NAME ="core-site.xml";
+    /**hdfs访问地址*/
+    private static final String HDFS_URL_KEY="fs.defaultFS";
+    /** 文件key默认存储路径  */
+    private final static String BASE_FILE_PATH="/test/files";
+    /**hdfs访问用户名*/
+    private static final String HADOOP_USER_NAME="HADOOP_USER_NAME";
+
+    private final IFileService fileService;
+    private final FileMd5Service fileMd5Service;
+    private final CompatibleOldFileService compatibleOldFileService;
+
+    @Override
+    public void migrateFiles(Map<String, Object> requestMap) {
+        final String hdfsUrl = MapUtil.getStr(requestMap, "hdfsUrl");
+        final String fileBasePath = MapUtil.getStr(requestMap, "fileBasePath");
+        final String hdfsUser = MapUtil.getStr(requestMap, "hdfsUser");
+        final String minioUrl = MapUtil.getStr(requestMap, "minioUrl");
+        final String minioUser = MapUtil.getStr(requestMap, "minioUser");
+        final String minioSecret = MapUtil.getStr(requestMap, "minioSecret");
+        final String bucketName = MapUtil.getStr(requestMap, "bucketName","persagy");
+        final boolean deleteOldFileFlag = MapUtil.getBool(requestMap, "deleteOldFileFlag",false);
+        if(StrUtil.isBlank(hdfsUrl)
+                || StrUtil.isBlank(fileBasePath)
+                || StrUtil.isBlank(hdfsUser)
+                || StrUtil.isBlank(minioUrl)
+                || StrUtil.isBlank(minioUser)
+                || StrUtil.isBlank(minioSecret)) {
+            throw new BusinessException(ResponseCode.A0400.getCode(), ResponseCode.A0400.getDesc());
+        }
+        // 启动迁移 执行时间较长,后台线程运行
+        // TODO 需要加分布式锁,并支持查看执行情况
+        ThreadUtil.execute(() -> startMigrate(hdfsUrl,fileBasePath,hdfsUser,minioUrl,minioUser,minioSecret,bucketName,deleteOldFileFlag));
+    }
+
+    /***
+     * Description: 启动数据迁移
+     * @param hdfsUrl : HDFS的文件路径
+     * @param fileBasePath : 文件路径
+     * @param hdfsUser : HDFS文件服务器访问权限用户
+     * @param minioUrl : minio地址
+     * @param accessKey : minio用户
+     * @param secretKey : minio密码
+     * @param bucketName : 集团编码
+     * @return : void
+     * @author : lijie
+     * @date :2021/9/9 23:31
+     * Update By lijie 2021/9/9 23:31
+     */
+    private void startMigrate(String hdfsUrl, String fileBasePath, String hdfsUser, String minioUrl, String accessKey,
+                              String secretKey, String bucketName,Boolean deleteOldFileFlag){
+        // 1.创建minio客户端
+        MinioClient minioClient = MinioClient.builder()
+                .endpoint(minioUrl)
+                .region("GMT+8")
+                .credentials(accessKey, secretKey)
+                .build();
+        // 2.创建HDFS文件流
+        FileSystem fileSystem = null;
+        try {
+            if (!minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())){
+                minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).region("GMT+8").build());
+            }
+            fileSystem = FileSystem.get(createConfig(hdfsUrl,hdfsUser));
+            RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path(fileBasePath), true);
+            while (listFiles.hasNext()){
+                LocatedFileStatus fileStatus = listFiles.next();
+                if (fileStatus.isDirectory()){
+                    continue;
+                }
+                Path path = fileStatus.getPath();
+                FSDataInputStream dataInputStream = fileSystem.open(path);
+                InputStream wrappedStream = dataInputStream.getWrappedStream();
+                compatibleOldFileService.commonUploadFile(wrappedStream,bucketName,null);
+            }
+            // 处理完文件后直接删除hdfs文件夹
+            if (deleteOldFileFlag){
+                fileSystem.delete(new Path(fileBasePath), true);
+            }
+        }catch (Exception e){
+            log.error("转移文件失败",e);
+        }finally {
+            if (null!=fileSystem){
+                try {
+                    fileSystem.close();
+                } catch (IOException e) {
+                    log.error("关闭流失败",e);
+                }
+            }
+        }
+    }
+    /***
+     * Description: 创建HDFS配置类
+     * @param url : hdfs的服务器地址
+     * @return : org.apache.hadoop.conf.Configuration
+     * @author : lijie
+     * @date :2021/9/9 23:37
+     * Update By lijie 2021/9/9 23:37
+     */
+    public Configuration createConfig(String url, String accessKey) {
+        Configuration plainConfig = new Configuration();
+        try {
+            System.setProperty(HADOOP_USER_NAME,accessKey);
+            String hbaseSiteConfigFilePath = FILE_CONFIG_PATH+ HDFS_SITE_FILE_NAME;
+            String hbaseSiteClasspathFilePath = FILE_CLASSPATH_PATH+ HDFS_SITE_FILE_NAME;
+            String hbaseCoreConfigFilePath = FILE_CONFIG_PATH+ HDFS_CORE_FILE_NAME;
+            String hbaseCoreClasspathFilePath = FILE_CLASSPATH_PATH+ HDFS_CORE_FILE_NAME;
+            log.info("hdfs-site.xml文件存在情况,config目录:{},classPath:{}",checkResoucesExist(hbaseSiteConfigFilePath),
+                    checkResoucesExist(hbaseSiteClasspathFilePath));
+            log.info("core-site.xml文件存在情况,config目录:{},classPath:{}",checkResoucesExist(hbaseCoreConfigFilePath),
+                    checkResoucesExist(hbaseCoreClasspathFilePath));
+            log.info("classpath下的文件是否存在:"+checkResoucesExist(hbaseSiteClasspathFilePath));
+            // 导入hbase-site.xml文件
+            Resource[] siteResources = getResource(hbaseSiteConfigFilePath, hbaseSiteClasspathFilePath);
+            if (siteResources.length<1){
+                throw new BusinessException(ResponseCode.B0300.getCode(),"hdfs-site.xml文件不存在");
+            }
+            // 导入hbase-core.xml文件
+            Resource[] coreResources = getResource(hbaseCoreConfigFilePath, hbaseCoreClasspathFilePath);
+            if (coreResources.length<1){
+                throw new BusinessException(ResponseCode.B0300.getCode(),"core-site.xml文件不存在");
+            }
+            plainConfig.set(HDFS_URL_KEY,url);
+            Resource siteResource = siteResources[0];
+            plainConfig.addResource(siteResource.getInputStream());
+            Resource coreResource = coreResources[0];
+            plainConfig.addResource(coreResource.getInputStream());
+        }catch (Exception e) {
+            log.error("加载HDFS配置文件失败!",e);
+        }
+        return plainConfig;
+    }
+
+    /***
+     * @Description: 校验资源文件是否存在
+     * @param configFilePath : jar包同级目录下的配置文件
+     * @return : com.alibaba.fastjson.JSONArray
+     * @author: lijie
+     * @Date:2020/6/6 19:43
+     * Update By lijie 2020/6/6 19:43
+     */
+    private static boolean checkResoucesExist(String configFilePath) throws IOException {
+        ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
+        Resource[] resolverResources = resolver.getResources(configFilePath);
+        if (resolverResources.length > 0 && resolverResources[0].exists()){
+            return true;
+        }
+        return false;
+    }
+
+    /***
+     * Description: 获取资源文件方法
+     * @param configFilePath : 配置文件路径
+     * @param classpathFilePath : classpath下的文件
+     * @return : org.springframework.core.io.Resource[]
+     * @author : lijie
+     * @date :2021/3/1 14:25
+     * Update By lijie 2021/3/1 14:25
+     */
+    public static Resource[] getResource(String configFilePath, String classpathFilePath)
+            throws IOException {
+        ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
+        Resource[] resolverResources = resolver.getResources(configFilePath);
+        if (resolverResources.length > 0) {
+            boolean exist = true;
+            for (Resource resource : resolverResources) {
+                if (!resource.exists()) {
+                    exist = false;
+                    break;
+                }
+            }
+            if(exist) {
+                return resolverResources;
+            }
+        }
+        return resolver.getResources(classpathFilePath);
+    }
+
+}

+ 213 - 0
dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/service/impl/ImageUrlToImageUrlServiceImpl.java

@@ -0,0 +1,213 @@
+package com.persagy.dmp.file.service.impl;
+
+import cn.hutool.core.collection.CollUtil;
+import cn.hutool.core.io.IoUtil;
+import cn.hutool.core.lang.TypeReference;
+import cn.hutool.core.map.MapUtil;
+import cn.hutool.core.thread.ThreadUtil;
+import cn.hutool.core.util.CharsetUtil;
+import cn.hutool.core.util.StrUtil;
+import cn.hutool.http.HttpRequest;
+import cn.hutool.http.HttpResponse;
+import cn.hutool.http.HttpStatus;
+import cn.hutool.http.HttpUtil;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.persagy.dmp.common.constant.CommonConstant;
+import com.persagy.dmp.common.constant.ResponseCode;
+import com.persagy.dmp.common.exception.BusinessException;
+import com.persagy.dmp.file.context.OldFileAppContext;
+import com.persagy.dmp.file.service.CompatibleOldFileService;
+import com.persagy.dmp.file.service.FileMigrateService;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/***
+ * Description: hdfs转移到Minio服务
+ * @author : lijie
+ * Update By 2021/12/13 15:21
+ */
+@Service
+@Data
+@Slf4j
+@RequiredArgsConstructor
+public class ImageUrlToImageUrlServiceImpl implements FileMigrateService {
+
+    private final CompatibleOldFileService compatibleOldFileService;
+
+    @Override
+    public void migrateFiles(Map<String, Object> requestMap) {
+        final String fromImageServiceUrl = MapUtil.getStr(requestMap, "fromImageServiceUrl");
+        final String fromSystemId = MapUtil.getStr(requestMap, "fromSystemId");
+        final String fromSecret = MapUtil.getStr(requestMap, "fromSecret");
+        final String toImageServiceUrl = MapUtil.getStr(requestMap, "toImageServiceUrl");
+        final String toSystemId = MapUtil.getStr(requestMap, "toSystemId");
+        final String toSecret = MapUtil.getStr(requestMap, "toSecret");
+        final Set<String> fileKeys = MapUtil.get(requestMap, "keys", new TypeReference<Set<String>>() {});
+        final boolean deleteOldFileFlag = MapUtil.getBool(requestMap, "deleteOldFileFlag",false);
+        if(StrUtil.isBlank(fromImageServiceUrl)
+                || StrUtil.isBlank(fromSystemId)
+                || StrUtil.isBlank(toImageServiceUrl)
+                || StrUtil.isBlank(toSystemId)
+                || StrUtil.isBlank(toSecret)) {
+            throw new BusinessException(ResponseCode.A0400.getCode(), ResponseCode.A0400.getDesc());
+        }
+        if(deleteOldFileFlag && StrUtil.isBlank(fromSecret)) {
+            throw new BusinessException(ResponseCode.A0400.getCode(), ResponseCode.A0400.getDesc());
+        }
+        // 启动迁移 执行时间较长,后台线程运行
+        // TODO 需要加分布式锁,并支持查看执行情况
+        ThreadUtil.execute(() -> startMigrate(fromImageServiceUrl,fromSystemId,fromSecret,toImageServiceUrl,toSystemId,toSecret,deleteOldFileFlag,fileKeys));
+    }
+
+    /***
+     * Description: 启动数据迁移
+     * @param fromImageServiceUrl: 来源文件的地址
+     * @param fromSystemId:来源文件的用户id
+     * @param fromSecret:来源文件的用户密码
+     * @param toImageServiceUrl:去向文件的地址
+     * @param toSystemId:去向文件的用户id
+     * @param toSecret:去向文件的用户密码
+     * @param deleteOldFileFlag:是否删除旧文件的标记
+     * @param fileKeys:是否只迁移部分文件key列表
+     * @return : void
+     * @author : lijie
+     * @date :2021/9/9 23:31
+     * Update By lijie 2021/9/9 23:31
+     */
+    private void startMigrate(String fromImageServiceUrl, String fromSystemId, String fromSecret, String toImageServiceUrl,
+                              String toSystemId, String toSecret, boolean deleteOldFileFlag, Set<String> fileKeys){
+        final Map<String,String> uriMap = MapUtil.of("/common/images_list","/common/image_get");
+        uriMap.put("/common/files_list","/common/file_get");
+        final String urlFormat = "{}{}?systemId={}";
+        for (String listSubUrl : uriMap.keySet()) {
+            String requestListUrl = StrUtil.format(urlFormat,fromImageServiceUrl,listSubUrl,fromSystemId);
+            transferFile(requestListUrl,null,fromImageServiceUrl,fromSystemId,fromSecret,deleteOldFileFlag,
+                    fileKeys,uriMap.get(listSubUrl),toImageServiceUrl,toSystemId,toSecret);
+        }
+    }
+    /***
+     * Description: 迁移文件
+     * @param requestListUrl : 查询文件的url
+     * return : void
+     * @author : lijie
+     * Update By 2021/12/13 17:36
+     */
+    private void transferFile(String requestListUrl,String fromKey,String fromImageServiceUrl, String fromSystemId,
+                              String fromSecret, Boolean deleteOldFileFlag,Set<String> destFileKeys, String getUrl,
+                              String toImageServiceUrl,String toSystemId,String toSecret) {
+        final int pageSize = 1000;
+        JSONObject param = new JSONObject();
+        // 1.如果指定文件列表不为空则直接迁移指定列表的文件
+        if (CollUtil.isNotEmpty(destFileKeys)){
+            // 指定迁移key列表
+            transferFileInternal(destFileKeys,fromImageServiceUrl,fromSystemId,fromSecret,deleteOldFileFlag,getUrl,
+                    toImageServiceUrl,toSystemId,toSecret);
+            return;
+        }
+        // 2.不指定迁移文件列表则全部迁移
+        // 1.组装参数
+        param.put("maxCount",pageSize);
+        if (StrUtil.isNotBlank(fromKey)){
+            param.put("from",fromKey);
+        }
+        // 2.判断请求参数是否合法
+        String post = HttpUtil.post(requestListUrl, param.toJSONString());
+        if (StrUtil.isBlank(post) || !post.startsWith(StrUtil.DELIM_START)){
+            log.error("请求查询文件列表出错:"+post);
+        }
+        JSONObject postJson = JSONObject.parseObject(post);
+        if (!CommonConstant.QUERY_SUCCESS.equals(postJson.getString(CommonConstant.RESULT))){
+            log.error("请求查询文件列表出错:"+post);
+        }
+        // 3.获得文件key列表
+        List<String> fileKeys = JSONArray.parseArray(postJson.getString("Content"), String.class);
+        if (CollUtil.isEmpty(fileKeys)){
+            return;
+        }
+        // 4.根据文件key列表进行文件传输
+        transferFileInternal(CollUtil.newHashSet(fileKeys),fromImageServiceUrl,fromSystemId,fromSecret,deleteOldFileFlag,getUrl,
+                toImageServiceUrl,toSystemId,toSecret);
+        // 5.如果文件大小等于1000则进行分页查询
+        if (pageSize!=fileKeys.size()){
+            return;
+        }
+        transferFile(requestListUrl,fileKeys.get(fileKeys.size() - 1),fromImageServiceUrl,fromSystemId,fromSecret,
+                deleteOldFileFlag,destFileKeys,getUrl,toImageServiceUrl,toSystemId,toSecret);
+    }
+    /***
+     * Description: 根据key列表迁移文件
+     * @param destFileKeys : key列表
+     * @param fromImageServiceUrl : 文件服务地址
+     * @param fromSystemId : 用户id
+     * @param fromSecret : 密码
+     * @param deleteOldFileFlag : 是否删除旧文件
+     * @param getSubUrl : 下载文件的subUrl
+     * return : void  
+     * @author : lijie
+     * Update By 2021/12/13 18:22
+     */
+    private void transferFileInternal(Set<String> destFileKeys,String fromImageServiceUrl, String fromSystemId,
+                                      String fromSecret, Boolean deleteOldFileFlag, String getSubUrl,
+                                      String toImageServiceUrl,String toSystemId,String toSecret) {
+        final Map<String,String> deleteUriMap = MapUtil.of("/common/image_get","/common/images_delete");
+        deleteUriMap.put("/common/file_get","/common/files_delete");
+        final Map<String,String> uploadUriMap = MapUtil.of("/common/image_get","/common/image_upload");
+        uploadUriMap.put("/common/file_get","/common/file_upload");
+        final String downloadUrl = "{}{}?systemId={}&key={}";
+        final String uploadUrlFormat = "{}{}?systemId={}&secret={}&key={}";
+        // 1.分批迁移文件
+        for (String destFileKey : destFileKeys) {
+            if (StrUtil.isBlank(destFileKey)){
+                continue;
+            }
+            String downUrl = StrUtil.format(downloadUrl, fromImageServiceUrl, getSubUrl, fromSystemId,destFileKey);
+            byte[] bytes = HttpUtil.downloadBytes(downUrl);
+            String str = StrUtil.str(bytes, CharsetUtil.defaultCharset());
+            if ("File not existed".equalsIgnoreCase(str) || "Image not existed".equalsIgnoreCase(str)){
+                continue;
+            }
+            String uploadUrl = StrUtil.format(uploadUrlFormat, toImageServiceUrl, uploadUriMap.get(getSubUrl), toSystemId, toSecret, destFileKey);
+            HttpRequest httpRequest = HttpUtil.createPost(uploadUrl);
+            httpRequest.body(bytes);
+            httpRequest.header("Content-Type","application/json");
+            HttpResponse response = httpRequest.execute();
+            if(HttpStatus.HTTP_OK != response.getStatus()){
+                log.debug("上传文件失败,返回的结果:"+JSONObject.toJSONString(response.body()));
+            }
+        }
+        // 2.删除文件
+        deleteFileByKeys(fromImageServiceUrl,fromSystemId,fromSecret,deleteOldFileFlag,destFileKeys,deleteUriMap.get(getSubUrl));
+    }
+
+    /***
+     * Description: 删除文件
+     * @param imageServiceUrl : 文件服务地址
+     * @param systemId : 用户名
+     * @param secret : 密码
+     * @param deleteOldFileFlag : 是否删除旧文件,true-删除,false-不删除
+     * @param destFileKeys : 删除的文件key列表
+     * @param deleteSubUrl : 删除文件的地址
+     * return : void
+     * @author : lijie
+     * Update By 2021/12/13 18:11
+     */
+    private void deleteFileByKeys(String imageServiceUrl, String systemId, String secret, Boolean deleteOldFileFlag, Set<String> destFileKeys, String deleteSubUrl) {
+        if (!deleteOldFileFlag || CollUtil.isEmpty(destFileKeys)){
+            return;
+        }
+        final String deleteUrlFormat = "{}{}?systemId={}&secret={}";
+        JSONObject param = new JSONObject();
+        param.put("keys",destFileKeys);
+        String deleteUrl = StrUtil.format(deleteUrlFormat, imageServiceUrl, deleteSubUrl, systemId, secret);
+        String post = HttpUtil.post(deleteUrl, param.toJSONString());
+        log.debug("批量删除文件结果:"+post);
+    }
+
+}

+ 193 - 0
dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/service/impl/ImageUrlToMinioServiceImpl.java

@@ -0,0 +1,193 @@
+package com.persagy.dmp.file.service.impl;
+
+import cn.hutool.core.collection.CollUtil;
+import cn.hutool.core.io.IoUtil;
+import cn.hutool.core.lang.Pair;
+import cn.hutool.core.lang.TypeReference;
+import cn.hutool.core.map.MapUtil;
+import cn.hutool.core.thread.ThreadUtil;
+import cn.hutool.core.util.CharsetUtil;
+import cn.hutool.core.util.StrUtil;
+import cn.hutool.http.HttpUtil;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.persagy.dmp.common.constant.CommonConstant;
+import com.persagy.dmp.common.constant.ResponseCode;
+import com.persagy.dmp.common.exception.BusinessException;
+import com.persagy.dmp.file.context.OldFileAppContext;
+import com.persagy.dmp.file.service.CompatibleOldFileService;
+import com.persagy.dmp.file.service.FileMigrateService;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/***
+ * Description: hdfs转移到Minio服务
+ * @author : lijie
+ * Update By 2021/12/13 15:21
+ */
+@Service
+@Data
+@Slf4j
+@RequiredArgsConstructor
+public class ImageUrlToMinioServiceImpl implements FileMigrateService {
+
+    private final CompatibleOldFileService compatibleOldFileService;
+
+    @Override
+    public void migrateFiles(Map<String, Object> requestMap) {
+        final String imageServiceUrl = MapUtil.getStr(requestMap, "imageServiceUrl");
+        final String systemId = MapUtil.getStr(requestMap, "systemId");
+        final Set<String> fileKeys = MapUtil.get(requestMap, "keys", new TypeReference<Set<String>>() {});
+        final String secret = MapUtil.getStr(requestMap, "secret");
+        final String bucketName = MapUtil.getStr(requestMap, "bucketName","persagy");
+        final boolean deleteOldFileFlag = MapUtil.getBool(requestMap, "deleteOldFileFlag",false);
+        if(StrUtil.isBlank(imageServiceUrl)
+                || StrUtil.isBlank(systemId)
+                || StrUtil.isBlank(bucketName)) {
+            throw new BusinessException(ResponseCode.A0400.getCode(), ResponseCode.A0400.getDesc());
+        }
+        if(deleteOldFileFlag && StrUtil.isBlank(secret)) {
+            throw new BusinessException(ResponseCode.A0400.getCode(), ResponseCode.A0400.getDesc());
+        }
+        // 启动迁移 执行时间较长,后台线程运行
+        // TODO 需要加分布式锁,并支持查看执行情况
+        ThreadUtil.execute(() -> startMigrate(imageServiceUrl,systemId,secret,bucketName,deleteOldFileFlag,fileKeys));
+    }
+
+    /***
+     * Description: 启动数据迁移
+     * @param imageServiceUrl : HDFS的文件路径
+     * @param systemId : 文件路径
+     * @param secret : HDFS文件服务器访问权限用户
+     * @param bucketName : minio地址
+     * @param bucketName : 集团编码
+     * @return : void
+     * @author : lijie
+     * @date :2021/9/9 23:31
+     * Update By lijie 2021/9/9 23:31
+     */
+    private void startMigrate(String imageServiceUrl, String systemId, String secret, String bucketName,
+                              Boolean deleteOldFileFlag,Set<String> fileKeys){
+        final Map<String,String> uriMap = MapUtil.of("/common/images_list","/common/image_get");
+        uriMap.put("/common/files_list","/common/file_get");
+        final String urlFormat = "{}{}?systemId={}";
+        for (String listSubUrl : uriMap.keySet()) {
+            String requestListUrl = StrUtil.format(urlFormat,imageServiceUrl,listSubUrl,systemId);
+            transferFile(requestListUrl,null,imageServiceUrl,systemId,secret,bucketName,deleteOldFileFlag,fileKeys,uriMap.get(listSubUrl));
+        }
+    }
+    /***
+     * Description: 迁移文件
+     * @param requestListUrl : 查询文件的url
+     * return : void
+     * @author : lijie
+     * Update By 2021/12/13 17:36
+     */
+    private void transferFile(String requestListUrl,String fromKey,String imageServiceUrl, String systemId,
+                              String secret, String bucketName, Boolean deleteOldFileFlag,Set<String> destFileKeys,
+                              String getUrl) {
+        final int pageSize = 1000;
+        JSONObject param = new JSONObject();
+        // 1.如果指定文件列表不为空则直接迁移指定列表的文件
+        if (CollUtil.isNotEmpty(destFileKeys)){
+            // 指定迁移key列表
+            transferFileInternal(destFileKeys,imageServiceUrl,systemId,secret,bucketName,deleteOldFileFlag,getUrl);
+            return;
+        }
+        // 2.不指定迁移文件列表则全部迁移
+        // 1.组装参数
+        param.put("maxCount",pageSize);
+        if (StrUtil.isNotBlank(fromKey)){
+            param.put("from",fromKey);
+        }
+        // 2.判断请求参数是否合法
+        String post = HttpUtil.post(requestListUrl, param.toJSONString());
+        if (StrUtil.isBlank(post) || !post.startsWith(StrUtil.DELIM_START)){
+            log.error("请求查询文件列表出错:"+post);
+        }
+        JSONObject postJson = JSONObject.parseObject(post);
+        if (!CommonConstant.QUERY_SUCCESS.equals(postJson.getString(CommonConstant.RESULT))){
+            log.error("请求查询文件列表出错:"+post);
+        }
+        // 3.获得文件key列表
+        List<String> fileKeys = JSONArray.parseArray(postJson.getString("Content"), String.class);
+        if (CollUtil.isEmpty(fileKeys)){
+            return;
+        }
+        // 4.根据文件key列表进行文件传输
+        transferFileInternal(CollUtil.newHashSet(fileKeys),imageServiceUrl,systemId,secret,bucketName,deleteOldFileFlag,getUrl);
+        // 5.如果文件大小等于1000则进行分页查询
+        if (pageSize!=fileKeys.size()){
+            return;
+        }
+        transferFile(requestListUrl,fileKeys.get(fileKeys.size() - 1),imageServiceUrl,systemId,secret,bucketName,
+                deleteOldFileFlag,destFileKeys,getUrl);
+    }
+    /***
+     * Description: 根据key列表迁移文件
+     * @param destFileKeys : key列表
+     * @param imageServiceUrl : 文件服务地址
+     * @param systemId : 用户id
+     * @param secret : 密码
+     * @param bucketName : 桶名称
+     * @param deleteOldFileFlag : 是否删除旧文件
+     * @param getSubUrl : 下载文件的subUrl
+     * return : void  
+     * @author : lijie
+     * Update By 2021/12/13 18:22
+     */
+    private void transferFileInternal(Set<String> destFileKeys,String imageServiceUrl, String systemId,
+                                      String secret, String bucketName, Boolean deleteOldFileFlag,
+                                      String getSubUrl) {
+        final Map<String,String> deleteUriMap = MapUtil.of("/common/image_get","/common/images_delete");
+        deleteUriMap.put("/common/file_get","/common/files_delete");
+        final String downloadUrl = "{}{}?systemId={}&key={}";
+        // 1.分批迁移文件
+        for (String destFileKey : destFileKeys) {
+            if (StrUtil.isBlank(destFileKey)){
+                continue;
+            }
+            String downUrl = StrUtil.format(downloadUrl, imageServiceUrl, getSubUrl, systemId,destFileKey);
+            byte[] bytes = HttpUtil.downloadBytes(downUrl);
+            String str = StrUtil.str(bytes, CharsetUtil.defaultCharset());
+            if ("File not existed".equalsIgnoreCase(str) || "Image not existed".equalsIgnoreCase(str)){
+                continue;
+            }
+            OldFileAppContext.getContext().setKey(destFileKey);
+            compatibleOldFileService.commonUploadFile(IoUtil.toStream(bytes),bucketName,null);
+        }
+        // 2.删除文件
+        deleteFileByKeys(imageServiceUrl,systemId,secret,deleteOldFileFlag,destFileKeys,deleteUriMap.get(getSubUrl));
+    }
+
+    /***
+     * Description: 删除文件
+     * @param imageServiceUrl : 文件服务地址
+     * @param systemId : 用户名
+     * @param secret : 密码
+     * @param deleteOldFileFlag : 是否删除旧文件,true-删除,false-不删除
+     * @param destFileKeys : 删除的文件key列表
+     * @param deleteSubUrl : 删除文件的地址
+     * return : void
+     * @author : lijie
+     * Update By 2021/12/13 18:11
+     */
+    private void deleteFileByKeys(String imageServiceUrl, String systemId, String secret, Boolean deleteOldFileFlag, Set<String> destFileKeys, String deleteSubUrl) {
+        if (!deleteOldFileFlag || CollUtil.isEmpty(destFileKeys)){
+            return;
+        }
+        final String deleteUrlFormat = "{}{}?systemId={}&secret={}";
+        JSONObject param = new JSONObject();
+        param.put("keys",destFileKeys);
+        String deleteUrl = StrUtil.format(deleteUrlFormat, imageServiceUrl, deleteSubUrl, systemId, secret);
+        String post = HttpUtil.post(deleteUrl, param.toJSONString());
+        log.debug("批量删除文件结果:"+post);
+    }
+
+}

+ 1 - 1
dmp-cloud/dmp-file/src/main/resources/db/init/schema.sql

@@ -1,5 +1,5 @@
 CREATE TABLE IF NOT EXISTS `dt_file_info` (
-`id` varchar(40) NOT NULL COMMENT '主键',
+`id` varchar(128) NOT NULL COMMENT '主键',
 `group_code` varchar(40) NOT NULL COMMENT '集团编码',
 `file_name` varchar(128) NOT NULL COMMENT '文件名称',
 `file_md5` varchar(64) NULL DEFAULT NULL COMMENT '文件的md5值',

+ 2 - 0
dmp-common/src/main/java/com/persagy/dmp/common/constant/CommonConstant.java

@@ -52,4 +52,6 @@ public interface CommonConstant {
 
     /** remove字段 */
     String REMOVE_FIELD ="$remove";
+    /** result字段 */
+    String RESULT ="result";
 }

+ 18 - 0
dmp-parent/pom.xml

@@ -62,6 +62,8 @@
         <maven-resources-plugin.version>3.0.2</maven-resources-plugin.version>
         <xstream.version>1.4.18</xstream.version>
         <log4j-api.version>2.15.0</log4j-api.version>
+        <hadoop.version>2.5.1</hadoop.version>
+        <log4j-over-slf4j>1.7.32</log4j-over-slf4j>
     </properties>
 
     <dependencyManagement>
@@ -330,6 +332,22 @@
                 <artifactId>minio</artifactId>
                 <version>${minio.version}</version>
             </dependency>
+            <!--hadoop-->
+            <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-common</artifactId>
+                <version>${hadoop.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-hdfs</artifactId>
+                <version>${hadoop.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.slf4j</groupId>
+                <artifactId>log4j-over-slf4j</artifactId>
+                <version>${log4j-over-slf4j}</version>
+            </dependency>
             <dependency>
                 <groupId>com.squareup.okhttp3</groupId>
                 <artifactId>okhttp</artifactId>