使用Java操作hdfs

使用Java操作hdfs

• 下载IDEA

  从官网下载好IDEA(必须是企业版的，社区版没有maven)

• 配置maven

  • 首先创建一个项目

    

  • 配置pom.xml

     <dependencies>
            <!-- https://mvnrepository.com/artifact/org.bytedeco.javacpp-presets/opencv -->
            <dependency>
                <groupId>org.bytedeco.javacpp-presets</groupId>
                <artifactId>opencv</artifactId>
                <version>4.0.1-1.4.4</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-client</artifactId>
                <version>3.1.3</version>
                <scope>provided</scope>  <!--scope依赖包作用范围,相当于java中private-->
            </dependency>
            <dependency>
                <groupId>junit</groupId>
                <artifactId>junit</artifactId>
                <version>4.12</version>
            </dependency>
            <dependency>
                <groupId>org.slf4j</groupId>
                <artifactId>slf4j-log4j12</artifactId>
                <version>1.7.30</version>
            </dependency>
        </dependencies>

• 创建文件目录

  public void mkdir() throws Exception{
          Configuration configuration =new Configuration();
          URI uri = new URI("hdfs://192.168.186.128:9820");
          FileSystem fileSystem = FileSystem.get(uri,configuration,"root");
          Path path  =new Path("/myFile");
          System.out.println(fileSystem.mkdirs(path)?"创建成功!":"创建失败!");
      }

• 上传文件

    public void uploadFile() throws Exception{
          Configuration configuration =new Configuration();
          URI uri = new URI("hdfs://192.168.186.128:9820");
          FileSystem fileSystem = FileSystem.get(uri,configuration,"root");
          Path local = new Path("d:/4k图.zip");
          Path target = new Path("hdfs://192.168.186.128:9820/myFile");
          fileSystem.copyFromLocalFile(false,false,local,target);
          fileSystem.close();
      }

• 查看文件名称

   public void lisFile() throws Exception{
          Configuration configuration =new Configuration();
          URI uri = new URI("hdfs://192.168.186.128:9820");
          FileSystem fileSystem = FileSystem.get(uri,configuration,"root");
          FileStatus[] fileStatuses =fileSystem.listStatus(new Path("/"));
          for(FileStatus f:fileStatuses){
              System.out.println(f.getPath().getName());
          }
      }

• 从hdfs下载文件

  • 首先需要配置windows 的Hadoop环境

  • 首先将之前的安装Hadoop的安装包解压添加到系统变量

    

    

    然后下载文件 ，解压

    

    接着将上述的hadoop.dll文件复制到C:\Windows\System32目录下

    public void downLoadFile() throws Exception{
          Configuration configuration =new Configuration();
          URI uri = new URI("hdfs://192.168.186.128:9820");
          FileSystem fileSystem = FileSystem.get(uri,configuration,"root");
          Path local = new Path("D:/");
          Path downTarget = new Path("hdfs://192.168.186.128:9820/myFile/4k图.zip");
          fileSystem.copyToLocalFile(false,downTarget,local,true);
      }

• 修改文件名

    public void modifyFileName() throws Exception{
          Configuration configuration =new Configuration();
          URI uri = new URI("hdfs://192.168.186.128:9820");
          FileSystem fileSystem = FileSystem.get(uri,configuration,"root");
          boolean flag = fileSystem.rename(new Path("/myFile"),new Path("/myData"));
          System.out.println(flag?"重命名成功":"重命名失败");
      }

• 删除文件

   public void deleteFile() throws Exception{
          Configuration configuration =new Configuration();
          URI uri = new URI("hdfs://192.168.186.128:9820");
          FileSystem fileSystem = FileSystem.get(uri,configuration,"root");
          boolean flag = fileSystem.delete(new Path("/Path"));
          System.out.println(flag?"删除成功":"删除失败");
      }