大数据培训_Maven构建java工程操作hdfs实例

构建maven工程

导入maven类包

<dependencies>

<dependency>

<groupId>junit</groupId>

<artifactId>junit</artifactId>

<version>4.11</version>

<scope>test</scope>

</dependency>

<dependency>

<groupId>org.apache.hadoop</groupId>

<artifactId>hadoop-client</artifactId>

<version>2.8.1</version>

</dependency>

</dependencies>

编写Junit单元测试类

package net.togogo.hdfsproject;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.Before;
import org.junit.Test;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;

public class HdfsClientTest {
FileSystem fs = null;

@Before
public void init() throws Exception {

// 构造一个配置参数对象，设置一个参数：我们要访问的hdfs的URI
// 从而FileSystem.get()方法就知道应该是去构造一个访问hdfs文件系统的客户端，以及hdfs的访问地址
// new Configuration();的时候，它就会去加载jar包中的hdfs-default.xml
// 然后再加载classpath下的hdfs-site.xml
Configuration conf = new Configuration();
conf.set("fs.defaultFS","hdfs://192.168.75.129:9000");

conf.set("dfs.replication","3");

// 获取一个hdfs的访问客户端，根据参数，这个实例应该是DistributedFileSystem的实例
// fs = FileSystem.get(conf);

// 如果这样去获取，那conf里面就可以不要配"fs.defaultFS"参数，而且，这个客户端的身份标识已经是hadoop用户
fs = FileSystem.get(new URI("hdfs://192.168.75.129:9000"),conf,"hadoop");

}

@Test
public void testAddFileToHdfs(){
// 要上传的文件所在的本地路径
Path src = new Path("D:stoneHCNA-AIimagesidcard.jpg");
// 要上传到hdfs的目标路径
Path dst = new Path("/togogo/idcard.jpg");
try {
fs.copyFromLocalFile(src,dst);
System.out.println("文件上传成功...");
} catch (IOException e) {
e.printStackTrace();
}
}

@Test
public void testDelFileFromHdfs(){
// 要上传到hdfs的目标路径
Path dst = new Path("/togogo/work");
try {
fs.delete(dst,true);
System.out.println("文件删除成功...");
} catch (IOException e) {
e.printStackTrace();
}
}


@Test
public void testListFiles() throws FileNotFoundException,IllegalArgumentException,IOException {

// 思考：为什么返回迭代器，而不是List之类的容器
RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"),true);

while (listFiles.hasNext()) {
LocatedFileStatus fileStatus = listFiles.next();
System.out.println(fileStatus.getPath().getName());
System.out.println(fileStatus.getBlockSize());
System.out.println(fileStatus.getPermission());
System.out.println(fileStatus.getLen());
BlockLocation[] blockLocations = fileStatus.getBlockLocations();
for (BlockLocation bl : blockLocations) {
System.out.println("block-length:" + bl.getLength() + "--" + "block-offset:" + bl.getOffset());
String[] hosts = bl.getHosts();
for (String host : hosts) {
System.out.println(host);
}
}
System.out.println("--------------为angelababy打印的分割线--------------");
}
}


@Test
public void testListAll() throws FileNotFoundException,IllegalArgumentException,IOException {

FileStatus[] listStatus = fs.listStatus(new Path("/"));

String flag = "d-- ";
for (FileStatus fstatus : listStatus) {
if (fstatus.isFile()) flag = "f-- ";
System.out.println(flag + fstatus.getPath().getName());
}
}
}
​