JavaAPI基础操作单元测试编写并通过

2018-04-08 12:17:24 +08:00
parent 1999ecae7d
commit 8d206f2f59
77 changed files with 1762 additions and 44 deletions
--- a/src/test/java/com/fjy/hadoop/HDFSApp.java
+++ b/src/test/java/com/fjy/hadoop/HDFSApp.java
@@ -0,0 +1,164 @@
+package com.fjy.hadoop;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.Progressable;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.net.URI;
+
+/**
+ * Hadoop HDFS Java API 操作
+ */
+public class HDFSApp {
+
+    public static final String HDFS_PATH = "hdfs://192.168.79.129:8020";
+
+    FileSystem fileSystem = null;
+    Configuration configuration = null;
+
+    /**
+     * 创建HDFS目录
+     *
+     * @throws Exception
+     */
+    @Test
+    public void mkdir() throws Exception {
+        fileSystem.mkdirs(new Path("/hdfsapi/test"));
+    }
+
+    /**
+     * 创建文件
+     *
+     * @throws Exception
+     */
+    @Test
+    public void create() throws Exception {
+        FSDataOutputStream outputStream = fileSystem.create(new Path("/hdfsapi/test/api.txt"));
+        outputStream.write("hello Hadoop API".getBytes());
+        outputStream.flush();
+        outputStream.close();
+    }
+
+    /**
+     * 查看文件内容
+     *
+     * @throws Exception
+     */
+    @Test
+    public void cat() throws Exception {
+        FSDataInputStream inputStream = fileSystem.open(new Path("/hdfsapi/test/api.txt"));
+        IOUtils.copyBytes(inputStream, System.out, 1024);
+        inputStream.close();
+    }
+
+    /**
+     * 文件重命名
+     *
+     * @throws Exception
+     */
+    @Test
+    public void rename() throws Exception {
+        Path oldPath = new Path("/hdfsapi/test/api.txt");
+        Path newPath = new Path("/hdfsapi/test/api_rename.txt");
+        fileSystem.rename(oldPath, newPath);
+    }
+
+    /**
+     * 从本地上传文件
+     *
+     * @throws Exception
+     */
+    @Test
+    public void copyFromLocalFile() throws Exception {
+        Path localPath = new Path("F:\\JAVA Workspace\\Temp\\upload\\windows.txt");
+        Path hdfsPath = new Path("/hdfsapi/test/windows.txt");
+        fileSystem.copyFromLocalFile(localPath, hdfsPath);
+    }
+
+    @Test
+    public void copyFromLocalFileWithProgress() throws Exception {
+        InputStream in = new BufferedInputStream(
+                new FileInputStream(
+                        new File("F:\\JAVA Workspace\\Temp\\upload\\bigFile.MP4")));
+        FSDataOutputStream outputStream = fileSystem.create(new Path("/hdfsapi/test/bigFile.MP4"),
+                new Progressable() {
+                    public void progress() {
+                        System.out.print("#");//传输进度信息
+                    }
+                });
+        IOUtils.copyBytes(in, outputStream, 4096);
+    }
+
+    /**
+     * 从服务器下载文件
+     *
+     * @throws Exception
+     */
+    @Test
+    public void copyToLocalFile() throws Exception {
+        Path localPath = new Path("F:\\JAVA Workspace\\Temp\\upload\\api_rename_from_hdfs.txt");
+        Path hdfsPath = new Path("/hdfsapi/test/api_rename.txt");
+        //windows平台要配置false和true参数，因为没有配置hadoop环境变量的原因
+        fileSystem.copyToLocalFile(false, hdfsPath, localPath, true);
+    }
+
+    /**
+     * 查看所有文件
+     *
+     * @throws Exception
+     */
+    @Test
+    public void listFile() throws Exception {
+        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/hdfsapi/test"));
+        for (FileStatus fileStatus : fileStatuses) {
+            String isDir = fileStatus.isDirectory() ? "文件夹" : "文件";
+            short replication = fileStatus.getReplication();//副本数量
+            long len = fileStatus.getLen();
+            String path = fileStatus.getPath().toString();
+
+            System.out.println(isDir + "\t" + replication + "\t" + len + "\t" + path);
+        }
+    }
+
+    /**
+     * 删除HDFS文件
+     * @throws Exception
+     */
+    @Test
+    public void delete() throws Exception{
+        //true表示递归删除，默认也是递归删除
+        fileSystem.delete(new Path("/hdfsapi/test/api_rename.txt"),true);
+    }
+
+    /**
+     * HDFS初始化
+     *
+     * @throws Exception
+     */
+    @Before
+    public void setUp() throws Exception {
+        System.out.println("HDFSApp.setUp");
+        configuration = new Configuration();
+        fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, "hadoop");
+    }
+
+    /**
+     * 释放资源
+     *
+     * @throws Exception
+     */
+    @After
+    public void tearDown() throws Exception {
+        configuration = null;
+        fileSystem = null;
+        System.out.println("HDFSApp.tearDown");
+    }
+}