云备份

项目介绍该项目完成一个自动对指定目录下的文件进行备份的功能客户端对指定目录进行监控对每个文件生成etag信息鉴别是否需要备份客户端将需要备份的文件基于http协议的PUT请求上传到服务器端服务端对于PUT上传的文件进行备份到指定目录下整个通信的过程使用SSL/TLS加密传输服务端设计实现功能提供解析基于https协议的put请求将文件数据进行备份提供浏览器能够查看服务器上文件信息功能提供浏览器能够下载服务器上文件功能提供对后台长时间无访问文件的压缩存储功服务端代码运行环境Linux平台1 #include httplib.h 2 #include fstream 3 #include unistd.h 4 #include fcntl.h 5 #include boost/filesystem.hpp 6 7 #define SERVER_BASE_DIR www 8 #define SERVER_BACKUP_DIR SERVER_BASE_DIR/list/ 9 #define SERVER_ADDR 0.0.0.0 10 #define SERVER_PORT 9527 11 12 namespace bf boost::filesystem; 13 14 class CloudServer 15 { 16 private: 17 httplib::Server srv; 18 public: 19 CloudServer() 20 { 21 bf::path base_path(SERVER_BASE_DIR); 22 if(!bf::exists(base_path)){ 23 bf::create_directory(base_path); 24 } 25 bf::path list_path(SERVER_BACKUP_DIR); 26 if(!bf::exists(list_path)){ 27 bf::create_directory(list_path); 28 } 29 } 30 bool Start(){ 31 srv.set_base_dir(SERVER_BASE_DIR); 32 srv.Get(/(list(/){0,1}){0,1}, GetFileList); 33 srv.Get(/list/(.*), GetFileData); 34 srv.Put(/list/(.*), PutFileData); 35 srv.listen(SERVER_ADDR, SERVER_PORT); 36 return true; 37 } 38 private: 39 static void GetFileList(const httplib::Request req, httplib::Response rsp) 40 { 41 bf::path list(SERVER_BASE_DIR); 42 bf::directory_iterator item_begin(list); 43 bf::directory_iterator item_end; 44 45 std::string body; 46 body htmlbodyolhr /; 47 for(; item_begin ! item_end; item_begin){ 48 if(bf::is_directory(item_begin-status())){ 49 continue; 50 } 51 std::string file item_begin-path().filename().string(); 52 std::string uri /list/ file; 53 body h4li; 54 body a href; 55 body uri; 56 body ; 57 body file; 58 body /a; 59 body /li/h4; 60 } 61 body hr //ol/body/html; 62 rsp.set_content(body[0], text/html); 63 return; 64 } 65 static void GetFileData(const httplib::Request req, httplib::Response rsp) 66 { 67 std::string file SERVER_BASE_DIR req.path; 68 if(!bf::exists(file)){ 69 std::cerr file file is not exists; 70 rsp.status 404; 71 return; 72 } 73 std::ifstream ifile(file, std::ios::binary); 74 if(!ifile.is_open()){ 75 std::cerr open file file error\n; 76 rsp.status 500; 77 return ; 78 } 79 std::string body; 80 int64_t fsize bf::file_size(file); 81 body.resize(fsize); 82 ifile.read(body[0], fsize); 83 if(!ifile.good()){ 84 std::cerr read file file body error\n; 85 rsp.status 500; 86 return ; 87 } 88 rsp.set_content(body, text/html); 89 } 90 static void PutFileData(const httplib:: Request req, httplib::Response rsp) 91 { 92 if(!req.has_header(Range)){ 93 rsp.status 400; 94 return ; 95 } 96 std::string range req.get_header_value(Range); 97 int64_t range_start; 98 if(RangeParse(range, range_start) false){ 99 rsp.status 400; 100 return ; 101 } 102 std::cout backup file:[ req.path ]range:[ range ]\n; 103 std::string real SERVER_BASE_DIR req.path; 104 int fd open(real.c_str(),O_CREAT|O_WRONLY, 0664); 105 if(fd 0){ 106 std::cerr open file real error\n; 107 rsp.status 500; 108 return ; 109 } 110 lseek(fd, range_start, SEEK_SET); 111 int ret write(fd, req.body[0], req.body.size()); 112 if(ret ! req.body.size()){ 113 std::cerr file write body error\n; 114 rsp.status 500; 115 return ; 116 } 117 close(fd); 118 119 return ; 120 } 121 static bool RangeParse(std::string range, int64_t start){ 122 size_t pos1 range.find(); 123 size_t pos2 range.find(-); 124 if(pos1 std::string::npos || pos2 std::string ::npos){ 125 std::cerr Range:[ range ] format error\n; 126 return false; 127 } 128 std::stringstream rs; 129 rs range.substr(pos11, pos2 - pos1 - 1); 130 rs start; 131 return true; 132 } 138 }; 140 int main() 141 { 142 CloudServer srv; 143 srv.Start(); 144 return 0; 145 }客户端设计实现功能提供监控目录的功能能够获取目录下文件信息鉴别文件是否需要备份备份文件基于HTTPS协议PUT请求实现文件多线程分块上传功能文件的信息记录便于文件是否需要备份的鉴别客户端代码运行环境vs2015#ifndef __M_CLOUD_H__ #define __M_CLOUD_H__ #include iostream #include fstream #include sstream #include string #include vector #include thread #include unordered_map #include boost\filesystem.hpp #include boost\algorithm\string.hpp #include httplib.h #define CLIENT_BACKUP_DIR backup #define CLIENT_BACKUP_INFO_FILE back.list #define RANGE_MAX_SIZE (10 20) #define SERVER_IP 192.168.194.129 #define SERVER_PORT 9527 #define BACKUP_URI /list/ namespace bf boost::filesystem; class ThrBackup { private: std::string _file; int64_t _range_start; int64_t _range_len; public: bool _res; public: ThrBackup(const std::string file, int64_t start, int64_t len):_res(true), _file(file), _range_start(start), _range_len(len){} void Start() { //获取文件的range分块数据 std::ifstream path(_file, std::ios::binary); if (!path.is_open()) { std::cerr range backup file _file failed\n; _res false; return; } //跳转到range的起始位置 path.seekg(_range_start, std::ios::beg); std::string body; body.resize(_range_len); //读取文件中range分块的文件数据 path.read(body[0], _range_len); if (!path.good()) { std::cerr read file _file range data failed\n; _res false; return; } path.close(); //上传range数据 bf::path name(_file); //组织上传的url路径 std::string uri BACKUP_URI name.filename().string(); //实例化一个httplib的客户端对象 httplib::Client cli(SERVER_IP, SERVER_PORT); httplib::Headers hdr; hdr.insert(std::make_pair(Content-Length, std::to_string(_range_len))); std::stringstream tmp; tmp bytes _range_start - (_range_start _range_len - 1); hdr.insert(std::make_pair(Range, tmp.str().c_str())); //通过实例化的Client想服务端 auto rsp cli.Put(uri.c_str(), hdr, body, text/plain); if (rsp rsp-status ! 200) { _res false; } std::stringstream ss; ss backup file _file range:[ _range_start - _range_len ] backup success\n; std::cout ss.str(); return ; } }; class CloudClient { private: std::unordered_mapstd::string, std::string _backup_list; private: bool GetBackupInfo() { bf::path path(CLIENT_BACKUP_INFO_FILE); if (!bf::exists(path) ){ std::cerr list file path.string() is not exist std::endl; return false; } int64_t fsize bf::file_size(path); if (fsize 0) { std::cerr have no backup info\n; return false; } std::string body; body.resize(fsize); std::ifstream file(CLIENT_BACKUP_INFO_FILE, std::ios::binary); if (!file.is_open()) { std::cerr list file open error\n; return false; } file.read(body[0], fsize); if (!file.good()) { std::cerr read list file body error\n; return false; } file.close(); std::vectorstd::string list; boost::split(list, body,boost::is_any_of (\n)); for (auto i : list) { size_t pos i.find( ); if (pos std::string::npos) { continue; } std::string key i.substr(0, pos); std::string val i.substr(pos 1); _backup_list[key] val; } return true; } bool SetBackupInfo() { std::string body; for (auto i : _backup_list) { body i.first i.second \n; } std::ofstream file(CLIENT_BACKUP_INFO_FILE, std::ios::binary); if (!file.is_open()) { std::cerr open list file error\n; return false; } file.write(body[0], body.size()); if (!file.good()) { std::cerr set backup info error\n; return false; } file.close(); return true; } bool BackupDirListen(const std::string path) { bf::directory_iterator item_begin(path); bf::directory_iterator item_end; for (; item_begin ! item_end; item_begin) { if (bf::is_directory(item_begin-status())) { BackupDirListen(item_begin-path().string()); continue; } if (FileIsNeedBackup(item_begin-path().string()) false) { continue; } if (PutFileData(item_begin-path().string()) false) { continue; } AddBackupInfo(item_begin-path().string()); } return true; } bool AddBackupInfo(const std::string file) { std::string etag; if (GetFileEtag(file, etag) false) { return false; } _backup_list[file] etag; } bool GetFileEtag(const std::string file, std::string etag) { bf::path path(file); if (!bf::exists(path)) { std::cerr get file file etag error\n; return false; } int64_t fsize bf::file_size(path); int64_t mtime bf::last_write_time(path); std::stringstream tmp; tmp std::hex fsize - std::hex mtime; etag tmp.str(); return 0; } bool PutFileData(const std::string file) { //1.获取文件大小 int64_t fsize bf::file_size(file); if (fsize 0) { std::cerr file file unnecessary backup\n; return false; } //2.计算总共需要分多少块得到每块大小以及起始位置 //3.循环创建线程在线程中上传文件数据 int count fsize / RANGE_MAX_SIZE; std::vectorThrBackup thr_res; std::vectorstd::thread thr_list; std::cerr file:[ file ] fsize:[ fsize ] count:[ count 1 ]\n; for (int i 0; i count; i) { int64_t range_start i * RANGE_MAX_SIZE; int64_t range_end (i 1) * RANGE_MAX_SIZE - 1; if (i count) { range_end fsize - 1; } int64_t range_len range_end - range_start 1; ThrBackup backup_info(file, range_start, range_len); std::cerr file:[ file ]range:[ range_start - range_end ]- range_len \n; thr_res.push_back(backup_info); } for (int i 0; i count; i) { thr_list.push_back(std::thread(thr_start, thr_res[i])); } //4.等待线程退出判断文件上传结果 bool ret true; for (int i 0; i count; i) { thr_list[i].join(); if (thr_res[i]._res true) { continue; } ret false; } //5.上传成功则添加文件的备份信息记录 if (ret false) { return false; } std::cerr file:[ file ] backup success\n; return true; } bool FileIsNeedBackup(const std::string file) { std::string etag; if (GetFileEtag(file, etag) false) { return false; } auto it _backup_list.find(file); if (it ! _backup_list.end() it-second etag) { return false; } return true; } static void thr_start(ThrBackup *backup_info) { backup_info-Start(); return; } public: bool Start() { GetBackupInfo(); while (1) { BackupDirListen(CLIENT_BACKUP_DIR); SetBackupInfo(); Sleep(10); } return true; } }; #endifhttplib基本使用/*test.cpp*/ #include httplib.h static void HelloWorld(const httplib::Request req, httplib::Response rsp) { rsp.set_content(htmlhello world/html, text/html); return; } int main() { httplib::Server srv; srv.set_base_dir(./www); srv.Get(/, HelloWorld); srv.listen(0.0.0.0, 9000); } /*g -stdc0x test.cpp -o test -lpthread -lboost_filesystem -lboost_system*/boost库下载路径boost_1_70_0-msvc-14.1-64.exe