基于Hadoop分布式存储的网盘系统实现(简易粗糙版)

巨匠您好,我是邵乃意,一个闲散的序列猿,一个认实的斜杠青年。起首,人们称号我:被代码耽搁的做家,没才调的书法家,没腔调的歌手,业余跑龙套的演员,各派活泼分子...第二,那几年来,我整理了良多高手的IT技能相关教程,我热爱生活,热爱分享。

第三,若是你觉得你的工做是有效的,请爱护保重它,转发它,攻讦它,感激你!

教程内容​​0x00 教程本色​​​​0x01 基于Hadoop漫衍式存储的网盘细碎完成​​​​1. 预备使命​​​​2. 编写代码​​​​3. 运转后果​​​​4. 完成阐明​​​​0xFF 总结​​0x00教程精华预备使命编写代码运转后果完成阐明

廓清:此完成十分粗拙,仅供参考。完成本教程后,您能够看到操做的后果,并完成与HDFS交互的最根本功用。

基于Hadoop散布式存储的0x01网盘细碎。1.筹办工做

(1) Hadoop需要本身安拆。

(2)启动Hadoop。

2.写代码

留意:那里必需介绍Hadoop依赖。请参考教程:Java API完成HDFS的连接把持。添加了一个新的Maven名称,并引入了依赖关系。

(1)编写页面代码:MainFram

package com.bigdata.mapreduce.swing;import com.bigdata.mapreduce.utils.HDFSUtil;import java.awt.event.ActionEvent;import java.awt.event.ActionListener;import java.util.ArrayList;import java.util.List;import javax.swing.event.TreeSelectionEvent;import javax.swing.event.TreeSelectionListener;import javax.swing.*;import javax.swing.tree.DefaultMutableTreeNode;public class MainFram extends JFrame{ private JTree tree=null; private JButton btnUpload = new JButton("上传"); private JButton btnDownload = new JButton("下载"); private JButton btnCreateDir = new JButton("创建文献夹"); private JButton btndeleteDirOrFiles = new JButton("删除了文献或许文献夹"); private JButton btnRenameDirOrFiles = new JButton("重命名文献或许文献夹"); private JButton btnMoveDir = new JButton("挪动文献/文献夹"); private JButton btnListDir = new JButton("列出文献"); private DefaultMutableTreeNode root=new DefaultMutableTreeNode("我的网盘"); private JPanel jp_center=new JPanel(); // 初始化函数 private void initTree() { tree=new JTree(root); tree.addTreeSelectionListener(new TreeSelectionListener() { @Override public void valueChanged(TreeSelectionEvent e) { tree_ValueChanged(e); } }); } private void tree_ValueChanged(TreeSelectionEvent e) {// JOptionPane.showMessageDialog(this,// tree.getSelectionPath().getLastPathComponent().toString()); btnListDir_Clicked(); } public MainFram() { JPanel jp=(JPanel)this.getContentPane(); initTree(); JScrollPane jsp_tree=new JScrollPane(tree); // 创建Panel JPanel jp_top=new JPanel(); // 增加三个按钮 jp_top.add(btnUpload); jp_top.add(btnDownload); jp_top.add(btnCreateDir); jp_top.add(btndeleteDirOrFiles); jp_top.add(btnRenameDirOrFiles); jp_top.add(btnMoveDir);// jp_top.add(btnListDir); JSplitPane splitPane_right=new JSplitPane(JSplitPane.VERTICAL_SPLIT,jp_top,jp_center); splitPane_right.setDividerLocation(100); splitPane_right.setDividerSize(1); JSplitPane splitPane=new JSplitPane(JSplitPane.HORIZONTAL_SPLIT,jsp_tree,splitPane_right); splitPane.setDividerLocation(300); splitPane.setDividerSize(2); jp.add(splitPane); btnUpload.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent e) { btnUpload_Clicked(); } }); btnDownload.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent e) { btnDownload_Clicked(); } }); btnCreateDir.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent e) { btnCreateDir_Clicked(); } }); btnListDir.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent e) { btnListDir_Clicked(); } }); btndeleteDirOrFiles.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent e) { btndeleteDirOrFiles_Clicked(); } }); btnRenameDirOrFiles.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent e) { btnRenameDirOrFiles_Clicked(); } }); btnMoveDir.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent e) { btnMoveDir_Clicked(); } }); this.setTitle("我的云盘"); this.setSize(1200, 800); this.setVisible(true); this.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); } private void btnDownload_Clicked() { System.out.println("download"); String message = "请输出需求下载的文献,HDFS上的文献:\n"; try { List<String> hdfsDir = HDFSUtil.listRemoteDirAndFiles("/"); for (int k = 0; k < hdfsDir.size(); k++) { message += hdfsDir.get(k) + "\n"; } } catch (Exception e) { e.printStackTrace(); } String srcPath = JOptionPane.showInputDialog(null, message); System.out.println(srcPath); JFileChooser jf = new JFileChooser(); jf.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); jf.setDialogTitle("请挑选要上传的文献夹..."); jf.showDialog(null, null); String destPath = jf.getSelectedFile().getAbsolutePath() + "/";// String[] name = jf.getSelectedFile().getAbsolutePath().split("/");// String destPath = "/user"; if (destPath.isEmpty()) { System.out.println("请挑选本地路子!"); } else { try { HDFSUtil.downloadFromHDFS(srcPath, destPath); } catch (Exception e1) { e1.printStackTrace(); } } System.out.println("下载成功!"); } private void btnUpload_Clicked() { System.out.println("upload"); JFileChooser jf = new JFileChooser(); jf.setFileSelectionMode(JFileChooser.FILES_ONLY); jf.setDialogTitle("请挑选要上传的文献..."); jf.showDialog(null, null); String srcPath = jf.getSelectedFile().getAbsolutePath() + "/";// String destPath = "/"; if (srcPath.isEmpty()) { System.out.println("本地文献路子不能为空!"); } else { String destPath = JOptionPane.showInputDialog(null, "请输出需求上传到HDFS的路子:"); if (!destPath.isEmpty()) { try { HDFSUtil.uploadToHDFS(srcPath, destPath); } catch (Exception e1) { e1.printStackTrace(); } } else { System.out.println("HDFS路子不能为空!"); } } System.out.println("上传成功!"); } private void btnMoveDir_Clicked() { String message1 = "请输出需求挪动的文献或许文献夹名:\n"; String message2 = "请输出思要挪动后的文献夹名:\n"; try { List<String> hdfsDir = HDFSUtil.listRemoteDirAndFiles("/"); for (int k = 0; k < hdfsDir.size(); k++) { message1 += hdfsDir.get(k) + "\n"; message2 += hdfsDir.get(k) + "\n"; } } catch (Exception e) { e.printStackTrace(); } String oldName = JOptionPane.showInputDialog(null, message1); String newName = JOptionPane.showInputDialog(null, message2); System.out.println(oldName); if (oldName.isEmpty() || newName.isEmpty()) { System.out.println("参数过错,请重试!"); } else { try { HDFSUtil.moveDirFromHDFS("/" + oldName, "/" + newName); } catch (Exception e) { e.printStackTrace(); } } System.out.println("挪动文献/文献夹成功!"); } private void btnRenameDirOrFiles_Clicked() { String message1 = "请输出需求重命名的文献或许文献夹名:\n"; String message2 = "请输出念要重命后的文献或许文献夹名:\n"; try { List<String> hdfsDir = HDFSUtil.listRemoteDirAndFiles("/"); for (int k = 0; k < hdfsDir.size(); k++) { message1 += hdfsDir.get(k) + "\n"; message2 += hdfsDir.get(k) + "\n"; } } catch (Exception e) { e.printStackTrace(); } String oldName = JOptionPane.showInputDialog(null, message1); String newName = JOptionPane.showInputDialog(null, message2); System.out.println(oldName); if (oldName.isEmpty() || newName.isEmpty()) { System.out.println("参数过错,请重试!"); } else { try { HDFSUtil.renameFromHDFS("/" + oldName, "/" + newName); } catch (Exception e) { e.printStackTrace(); } } System.out.println("重命名文献/文献夹成功!"); } private void btndeleteDirOrFiles_Clicked() { String message = "请输出需求删除了的文献或许文献夹名:\n"; try { List<String> hdfsDir = HDFSUtil.listRemoteDirAndFiles("/"); for (int k = 0; k < hdfsDir.size(); k++) { message += hdfsDir.get(k) + "\n"; } } catch (Exception e) { e.printStackTrace(); } String deleteName = JOptionPane.showInputDialog(null, message); System.out.println(deleteName); if (deleteName.isEmpty()) { System.out.println("请输出文献或许文献夹名!"); } else { try { HDFSUtil.deleteFromHDFS("/" + deleteName); } catch (Exception e) { e.printStackTrace(); } } System.out.println("删除了文献/文献夹成功!"); } private void btnListDir_Clicked() { int flag = 0; System.out.println("listDir"); try { List<String> hdfsDir = HDFSUtil.listRemoteDirAndFiles("/");// List<String> hdfsDir = HDFSUtil.listRemoteDir("/"); List<String> oneDirList = new ArrayList<>(); DefaultMutableTreeNode oneDir = new DefaultMutableTreeNode(); DefaultMutableTreeNode twoDir = new DefaultMutableTreeNode(); for (int i = 0; i < hdfsDir.size(); i++) { System.out.println(hdfsDir.get(i)); String[] arr = hdfsDir.get(i).split("/"); int length = arr.length; // 今朝只撑持而级目录 if (length == 2) { // length为2透露表示只要一个文献或许空文献夹,如:/user oneDir = new DefaultMutableTreeNode(arr[1]); root.add(oneDir); } else if (length == 3) { // length为3透露表示有两个文献或许两个空文献夹,如:/user/hello.txt // 假设一级目录里曾经有那个目录,则没有增加那个目录 for (int j = 0; j < oneDirList.size(); j++) { if (oneDirList.get(j).equals(arr[1])) { flag = 1; } } if (flag == 0) { oneDir = new DefaultMutableTreeNode(arr[1]); twoDir = new DefaultMutableTreeNode(arr[2]); oneDirList.add(arr[1]); oneDir.add(twoDir); root.add(oneDir); } else if (flag == 1) { twoDir = new DefaultMutableTreeNode(arr[2]); oneDir.add(twoDir); } } } System.out.println("列出文献成功!"); } catch (Exception e) { e.printStackTrace(); } } private void btnCreateDir_Clicked() { System.out.println("createDir"); String remoteDirName = JOptionPane.showInputDialog(null, "请输出需求创建的文献夹名:"); System.out.println(remoteDirName); if (remoteDirName.isEmpty()) { System.out.println("请输出文献名!"); } else { try { HDFSUtil.createDirFromHDFS("/" + remoteDirName); } catch (Exception e) { e.printStackTrace(); } } System.out.println("新修文献夹成功!"); } public static void main(String[] args) { JFrame.setDefaultLookAndFeelDecorated(true); new MainFram(); }}

(2)编写HDFS代码:HDFSUtil

package com.bigdata.mapreduce.utils;import java.text.SimpleDateFormat;import java.util.ArrayList;import java.util.List;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.* ;public class HDFSUtil { private static String hdfsURL="hdfs://localhost:9000"; private static Configuration conf; static { conf = new Configuration(); conf.set("fs.defaultFS", hdfsURL); } // 从HDFS凹凸载文献 public static void downloadFromHDFS(String remoteFile,String localFile) throws Exception { FileSystem fs = FileSystem.get(conf); Path remotePath = new Path(remoteFile); Path localPath = new Path(localFile); fs.copyToLocalFile(remotePath, localPath); fs.close(); } // 上传文献到HDFS public static void uploadToHDFS(String localfile,String remotefile) throws Exception { FileSystem fs = FileSystem.get(conf); Path remotePath = new Path(remotefile); Path localPath = new Path(localfile); fs.copyFromLocalFile(localPath, remotePath); fs.close(); } // 创建HDFS文献夹 public static void createDirFromHDFS(String remoteDir) throws Exception { FileSystem fs = FileSystem.get(conf); Path remotePath = new Path(remoteDir); //调用mkdirs函数创建目录 fs.mkdirs(remotePath); fs.close(); } // 删除了文献以及文献夹 public static void deleteFromHDFS(String remoteDir) throws Exception { FileSystem fs = FileSystem.get(conf); Path remotePath = new Path(remoteDir); //调用mkdirs函数创建目录,true透露表示轮回递归删除了 fs.delete(remotePath, true); fs.close(); } // 重命名文献 public static void renameFromHDFS(String oldName, String newName) throws Exception { FileSystem fs = FileSystem.get(conf); Path hdfsOldName = new Path(oldName); Path hdfsNewName = new Path(newName); fs.rename(hdfsOldName, hdfsNewName); fs.close(); } // 查抄文献夹 public static void ListHDFSDir(String remoteDir) throws Exception { FileSystem fs = FileSystem.get(conf); Path dirPath = new Path(remoteDir); /*递归获得目录下的一切文献*/ RemoteIterator<LocatedFileStatus> remoteIterator = fs.listFiles(dirPath, true); /*输入每一个文献的新闻*/ while (remoteIterator.hasNext()) { FileStatus s = remoteIterator.next(); System.out.println("路子: " + s.getPath().toString()); System.out.println("权限: " + s.getPermission().toString()); System.out.println("巨细: " + s.getLen()); /*前去的是时间戳,转化为时间日期格局*/ Long timeStamp = s.getModificationTime(); SimpleDateFormat format = new SimpleDateFormat( "yyyy-MM-dd HH:妹妹:ss"); String date = format.format(timeStamp); System.out.println("时间: " + date); System.out.println(); } fs.close(); } // 列出文献 public static List<String> listRemoteDirAndFiles(String remoteDir) throws Exception { List<String> remoteDirList = new ArrayList<>(); FileSystem fs = FileSystem.get(conf); Path dirPath = new Path(remoteDir); /*递归获得目录下的一切文献*/ RemoteIterator<LocatedFileStatus> remoteIterator = fs.listFiles(dirPath, true); /*输入每一个文献的讯息*/ while (remoteIterator.hasNext()) { FileStatus s = remoteIterator.next(); String myPath = s.getPath().toString().substring(21); remoteDirList.add(myPath); } fs.close(); return remoteDirList; } // 列出文献夹 public static List<String> listRemoteDir(String remoteDir) throws Exception { List<String> remoteDirList = new ArrayList<>(); FileSystem fs = FileSystem.get(conf); Path dirPath = new Path(remoteDir); FileStatus[] fileStatus = fs.listStatus(dirPath); for (FileStatus file : fileStatus) { if (file.isDirectory()) { listRemoteDir(file.getPath().toString()); remoteDirList.add(file.getPath().toString().substring(21)); } else { remoteDirList.add(file.getPath().toString().substring(21)); } } fs.close(); return remoteDirList; } // 挪动文献夹 public static void moveDirFromHDFS(String oldPath, String newPath) throws Exception { FileSystem fs = FileSystem.get(conf); Path hdfsOldName = new Path(oldPath); Path hdfsNewName = new Path(newPath); fs.rename(hdfsOldName, hdfsNewName); fs.close(); } // 测试// public static void main(String[] args) {// try { // 一、列出文献// List<String> list = HDFSUtil.listRemoteDirAndFiles("/");// for (int i = 0; i < list.size(); i++) {// System.out.println(list.get(i));// } // 二、重名文献// HDFSUtil.renameFromHDFS("/start-dfs.cmd", "/start-dfs.sh"); // 三、创建文献夹// HDFSUtil.createDirFromHDFS("/hive"); // 四、挪动文献夹(把/路子下的hive文献夹,放到/user路子)// HDFSUtil.moveDirFromHDFS("/hive", "/user"); // 五、删除了文献或许文献夹// HDFSUtil.deleteFromHDFS("/user");// HDFSUtil.deleteFromHDFS("/user/hive/start-dfs.sh");// } catch (Exception e) {// e.printStackTrace();// }// }}3.操做后果

许多功用已经完成,虽然没有一个出格适用。

间接运行MainFram类以查看运行成果:

基于Hadoop散布式存储的网盘系统实现(简易粗拙版)_ide

4.完成廓清

(1)若是需要更正HDFS的链接,能够更正代码的值:

private static String hdfsURL="hdfs://localhost:9000";

(2)其实如今的目次树是没有尺度的。只是站在零根和同窗角度的完成方案,需求自求多福。

0xFF摘要颠末本教程,可以扩大巨匠的思惟,其实咱们学了HDFS的API把持,并不是尽善尽美的,咱们现实上是可以本人写一个复杂的小工具来运用的。关注本博主,进修更寡幽默的常识。

邵乃依原创不容易。如转载,请说明原因。教学是一生的奇观。

您可以还会对下面的文章感兴趣:

最新评论

◎欢迎参与讨论,请在这里发表您的看法、交流您的观点。