Hadoop 文件系统可以通过 shell 命令 hadoop fs -xx 进行操作, 同时也提供了 Java 编程接口
maven 配置
- <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <groupId>hadoopStudy</groupId>
- <artifactId>demo</artifactId>
- <version>1.0-SNAPSHOT</version>
- <packaging>jar</packaging>
- <name>demo</name>
- <url>http://maven.apache.org</url>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
- <dependencies>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>3.8.1</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>3.0.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>3.0.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>3.0.0</version>
- </dependency>
- </dependencies>
- </project>
代码实现
- package hadoopStudy;
- import org.apache.hadoop.conf.Configuration;
- import org.apache.hadoop.fs.FileSystem;
- import org.apache.hadoop.fs.LocatedFileStatus;
- import org.apache.hadoop.fs.Path;
- import org.apache.hadoop.fs.RemoteIterator;
- import java.io.IOException;
- import java.net.URI;
- import java.net.URISyntaxException;
- public class HdfsUtil {
- private FileSystem fs;
- HdfsUtil(String user){
- Configuration cfg = new Configuration();
- URI uri = null;
- try {
- uri = new URI("hdfs://localhost:9000");
- } catch (URISyntaxException e) {
- e.printStackTrace();
- }
- try {
- // 根据配置文件, 实例化成 DistributedFileSystem
- fs = FileSystem.get(uri, cfg, user); // 得到 fs 句柄
- } catch (IOException e) {
- e.printStackTrace();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- /**
- * 上传文件
- */
- public void upload(String src, String dst){
- try {
- // 上传
- fs.copyFromLocalFile(new Path(src), new Path(dst));
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- /**
- * 下载文件
- */
- public void download(String src, String dst){
- try {
- // 下载
- fs.copyToLocalFile(new Path(src), new Path(dst));
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- /**
- * 创建文件夹
- */
- public void mkdir(String dir){
- try {
- fs.mkdirs(new Path(dir));
- } catch (IOException e) {
- // 创建目录
- e.printStackTrace();
- }
- }
- /**
- * 删除文件
- */
- public void delete(String name, boolean recursive){
- try {
- fs.delete(new Path(name), recursive);
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- /**
- * 重命名
- */
- public void rename(String source, String dst){
- try {
- fs.rename(new Path(source), new Path(dst));
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- /**
- * 列出文件信息
- */
- public void list(String dir, boolean recursive){
- try {
- RemoteIterator<LocatedFileStatus> iter = fs.listFiles(new Path(dir), recursive);
- while (iter.hasNext()){
- LocatedFileStatus file = iter.next();
- System.out.println(file.getPath().getName());
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
小结
Hdfs 的 Java API 提供了优雅的 FileSystem 抽象类, 在客户端使用时与其他文件系统区别不大
来源: http://www.bubuko.com/infodetail-2517029.html