From 0c57edc0106da8a2dbd39c979b41f7e4677c7e53 Mon Sep 17 00:00:00 2001 From: Lucian Petrut Date: Thu, 19 Mar 2020 09:38:40 +0000 Subject: [PATCH] tools,common,msg: Use O_BINARY for win32 compatibility Unless O_BINARY is passed when opening files, Windows will translate newlines and handle CTRL+z (Alt+026) as EOF, which can abruptly end file reads. We'll define O_BINARY as 0 on other platforms and explicitly use O_BINARY when expecting binary files. Signed-off-by: Lucian Petrut --- src/common/buffer.cc | 6 +++--- src/common/safe_io.c | 6 +++--- src/include/compat.h | 7 +++++++ src/include/denc.h | 2 +- src/include/encoding.h | 2 +- src/msg/Message.cc | 2 +- src/rbd_replay/Replayer.cc | 2 +- src/rbd_replay/rbd-replay-prep.cc | 3 ++- src/tools/ceph-dencoder/ceph_dencoder.cc | 2 +- src/tools/ceph_monstore_tool.cc | 4 ++-- src/tools/cephfs/Dumper.cc | 4 ++-- src/tools/rados/rados.cc | 12 ++++++------ src/tools/rbd/action/Export.cc | 4 ++-- src/tools/rbd/action/Import.cc | 4 ++-- src/tools/rbd/action/Journal.cc | 4 ++-- src/tools/rbd/action/MergeDiff.cc | 6 +++--- src/tools/rbd/action/MirrorPool.cc | 2 +- 17 files changed, 40 insertions(+), 32 deletions(-) diff --git a/src/common/buffer.cc b/src/common/buffer.cc index 6dbf9c00671..32de477472d 100644 --- a/src/common/buffer.cc +++ b/src/common/buffer.cc @@ -1685,7 +1685,7 @@ void buffer::list::decode_base64(buffer::list& e) ssize_t buffer::list::pread_file(const char *fn, uint64_t off, uint64_t len, std::string *error) { - int fd = TEMP_FAILURE_RETRY(::open(fn, O_RDONLY|O_CLOEXEC)); + int fd = TEMP_FAILURE_RETRY(::open(fn, O_RDONLY|O_CLOEXEC|O_BINARY)); if (fd < 0) { int err = errno; std::ostringstream oss; @@ -1745,7 +1745,7 @@ ssize_t buffer::list::pread_file(const char *fn, uint64_t off, uint64_t len, std int buffer::list::read_file(const char *fn, std::string *error) { - int fd = TEMP_FAILURE_RETRY(::open(fn, O_RDONLY|O_CLOEXEC)); + int fd = TEMP_FAILURE_RETRY(::open(fn, O_RDONLY|O_CLOEXEC|O_BINARY)); if (fd < 0) { int err = errno; std::ostringstream oss; @@ -1812,7 +1812,7 @@ ssize_t buffer::list::recv_fd(int fd, size_t len) int buffer::list::write_file(const char *fn, int mode) { - int fd = TEMP_FAILURE_RETRY(::open(fn, O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC, mode)); + int fd = TEMP_FAILURE_RETRY(::open(fn, O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC|O_BINARY, mode)); if (fd < 0) { int err = errno; cerr << "bufferlist::write_file(" << fn << "): failed to open file: " diff --git a/src/common/safe_io.c b/src/common/safe_io.c index 9b7dbb77924..80921d0f2e4 100644 --- a/src/common/safe_io.c +++ b/src/common/safe_io.c @@ -237,7 +237,7 @@ int safe_write_file(const char *base, const char *file, snprintf(fn, sizeof(fn), "%s/%s", base, file); snprintf(tmp, sizeof(tmp), "%s/%s.tmp", base, file); - fd = open(tmp, O_WRONLY|O_CREAT|O_TRUNC, mode); + fd = open(tmp, O_WRONLY|O_CREAT|O_TRUNC|O_BINARY, mode); if (fd < 0) { ret = errno; return -ret; @@ -262,7 +262,7 @@ int safe_write_file(const char *base, const char *file, return ret; } - fd = open(base, O_RDONLY); + fd = open(base, O_RDONLY|O_BINARY); if (fd < 0) { ret = -errno; return ret; @@ -281,7 +281,7 @@ int safe_read_file(const char *base, const char *file, int fd, len; snprintf(fn, sizeof(fn), "%s/%s", base, file); - fd = open(fn, O_RDONLY); + fd = open(fn, O_RDONLY|O_BINARY); if (fd < 0) { return -errno; } diff --git a/src/include/compat.h b/src/include/compat.h index a73e7fcac38..d3f829c9be2 100644 --- a/src/include/compat.h +++ b/src/include/compat.h @@ -355,4 +355,11 @@ static inline int ceph_sock_errno() { #endif } +// Needed on Windows when handling binary files. Without it, line +// endings will be replaced and certain characters can be treated as +// EOF. +#ifndef O_BINARY +#define O_BINARY 0 +#endif + #endif /* !CEPH_COMPAT_H */ diff --git a/src/include/denc.h b/src/include/denc.h index a463f7647df..402c5a47169 100644 --- a/src/include/denc.h +++ b/src/include/denc.h @@ -118,7 +118,7 @@ private: ::snprintf(fn, sizeof(fn), ENCODE_STRINGIFY(ENCODE_DUMP_PATH) "/%s__%d.%x", name, getpid(), i++); - int fd = ::open(fn, O_WRONLY|O_TRUNC|O_CREAT|O_CLOEXEC, 0644); + int fd = ::open(fn, O_WRONLY|O_TRUNC|O_CREAT|O_CLOEXEC|O_BINARY, 0644); if (fd < 0) { return; } diff --git a/src/include/encoding.h b/src/include/encoding.h index 5036f8710a4..49f2f77be30 100644 --- a/src/include/encoding.h +++ b/src/include/encoding.h @@ -171,7 +171,7 @@ WRITE_FLTTYPE_ENCODER(double, uint64_t, le64) break; \ char fn[PATH_MAX]; \ snprintf(fn, sizeof(fn), ENCODE_STRINGIFY(ENCODE_DUMP_PATH) "/%s__%d.%x", #cl, getpid(), i++); \ - int fd = ::open(fn, O_WRONLY|O_TRUNC|O_CREAT|O_CLOEXEC, 0644); \ + int fd = ::open(fn, O_WRONLY|O_TRUNC|O_CREAT|O_CLOEXEC|O_BINARY, 0644); \ if (fd >= 0) { \ ::ceph::bufferlist sub; \ sub.substr_of(bl, pre_off, bl.length() - pre_off); \ diff --git a/src/msg/Message.cc b/src/msg/Message.cc index d7a031cc7e4..7f79696ca2d 100644 --- a/src/msg/Message.cc +++ b/src/msg/Message.cc @@ -283,7 +283,7 @@ void Message::encode(uint64_t features, int crcflags, bool skip_header_crc) snprintf(fn, sizeof(fn), ENCODE_STRINGIFY(ENCODE_DUMP) "/%s__%d.%x", abi::__cxa_demangle(typeid(*this).name(), 0, 0, &status), getpid(), i++); - int fd = ::open(fn, O_WRONLY|O_TRUNC|O_CREAT|O_CLOEXEC, 0644); + int fd = ::open(fn, O_WRONLY|O_TRUNC|O_CREAT|O_CLOEXEC|O_BINARY, 0644); if (fd >= 0) { bl.write_fd(fd); ::close(fd); diff --git a/src/rbd_replay/Replayer.cc b/src/rbd_replay/Replayer.cc index b159d8a51ff..11e84900d1a 100644 --- a/src/rbd_replay/Replayer.cc +++ b/src/rbd_replay/Replayer.cc @@ -214,7 +214,7 @@ void Replayer::run(const std::string& replay_file) { m_rbd = new librbd::RBD(); map workers; - int fd = open(replay_file.c_str(), O_RDONLY); + int fd = open(replay_file.c_str(), O_RDONLY|O_BINARY); if (fd < 0) { std::cerr << "Failed to open " << replay_file << ": " << cpp_strerror(errno) << std::endl; diff --git a/src/rbd_replay/rbd-replay-prep.cc b/src/rbd_replay/rbd-replay-prep.cc index d4b4db5379a..977c7c19e39 100644 --- a/src/rbd_replay/rbd-replay-prep.cc +++ b/src/rbd_replay/rbd-replay-prep.cc @@ -15,6 +15,7 @@ // This code assumes that IO IDs and timestamps are related monotonically. // In other words, (a.id < b.id) == (a.timestamp < b.timestamp) for all IOs a and b. +#include "include/compat.h" #include "common/errno.h" #include "rbd_replay/ActionTypes.h" #include @@ -198,7 +199,7 @@ public: struct bt_iter *bt_itr = bt_ctf_get_iter(itr); - int fd = open(output_file_name.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0644); + int fd = open(output_file_name.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0644); ASSERT_EXIT(fd >= 0, "Error opening output file " << output_file_name << ": " << cpp_strerror(errno)); BOOST_SCOPE_EXIT( (fd) ) { diff --git a/src/tools/ceph-dencoder/ceph_dencoder.cc b/src/tools/ceph-dencoder/ceph_dencoder.cc index 2e4940256af..87091561b39 100644 --- a/src/tools/ceph-dencoder/ceph_dencoder.cc +++ b/src/tools/ceph-dencoder/ceph_dencoder.cc @@ -185,7 +185,7 @@ int main(int argc, const char **argv) cerr << "expecting filename" << std::endl; exit(1); } - int fd = ::open(*i, O_WRONLY|O_CREAT|O_TRUNC, 0644); + int fd = ::open(*i, O_WRONLY|O_CREAT|O_TRUNC|O_BINARY, 0644); if (fd < 0) { cerr << "error opening " << *i << " for write: " << cpp_strerror(errno) << std::endl; exit(1); diff --git a/src/tools/ceph_monstore_tool.cc b/src/tools/ceph_monstore_tool.cc index 044fe30d6a4..de26e8aedad 100644 --- a/src/tools/ceph_monstore_tool.cc +++ b/src/tools/ceph_monstore_tool.cc @@ -43,7 +43,7 @@ class TraceIter { MonitorDBStore::TransactionRef t; public: explicit TraceIter(string fname) : fd(-1), idx(-1) { - fd = ::open(fname.c_str(), O_RDONLY); + fd = ::open(fname.c_str(), O_RDONLY|O_BINARY); t.reset(new MonitorDBStore::Transaction); } bool valid() { @@ -879,7 +879,7 @@ int main(int argc, char **argv) { int fd = STDOUT_FILENO; if (!outpath.empty()){ - fd = ::open(outpath.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666); + fd = ::open(outpath.c_str(), O_WRONLY|O_CREAT|O_TRUNC|O_BINARY, 0666); if (fd < 0) { std::cerr << "error opening output file: " << cpp_strerror(errno) << std::endl; diff --git a/src/tools/cephfs/Dumper.cc b/src/tools/cephfs/Dumper.cc index fbfa6ae8fff..f3b07c55181 100644 --- a/src/tools/cephfs/Dumper.cc +++ b/src/tools/cephfs/Dumper.cc @@ -104,7 +104,7 @@ int Dumper::dump(const char *dump_file) cout << "journal is " << start << "~" << len << std::endl; - int fd = ::open(dump_file, O_WRONLY|O_CREAT|O_TRUNC, 0644); + int fd = ::open(dump_file, O_WRONLY|O_CREAT|O_TRUNC|O_BINARY, 0644); if (fd >= 0) { // include an informative header uuid_d fsid = monc->get_fsid(); @@ -213,7 +213,7 @@ int Dumper::undump(const char *dump_file, bool force) derr << "recover_journal failed, try to get header from dump file " << dendl; } - int fd = ::open(dump_file, O_RDONLY); + int fd = ::open(dump_file, O_RDONLY|O_BINARY); if (fd < 0) { r = errno; derr << "couldn't open " << dump_file << ": " << cpp_strerror(r) << dendl; diff --git a/src/tools/rados/rados.cc b/src/tools/rados/rados.cc index 1357f8751ae..ff637e1a520 100644 --- a/src/tools/rados/rados.cc +++ b/src/tools/rados/rados.cc @@ -445,7 +445,7 @@ static int dump_data(std::string const &filename, bufferlist const &data) if (filename == "-") { fd = STDOUT_FILENO; } else { - fd = TEMP_FAILURE_RETRY(::open(filename.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0644)); + fd = TEMP_FAILURE_RETRY(::open(filename.c_str(), O_WRONLY|O_CREAT|O_TRUNC|O_BINARY, 0644)); if (fd < 0) { int err = errno; cerr << "failed to open file: " << cpp_strerror(err) << std::endl; @@ -471,7 +471,7 @@ static int do_get(IoCtx& io_ctx, const char *objname, const char *outfile, unsig if (strcmp(outfile, "-") == 0) { fd = STDOUT_FILENO; } else { - fd = TEMP_FAILURE_RETRY(::open(outfile, O_WRONLY|O_CREAT|O_TRUNC, 0644)); + fd = TEMP_FAILURE_RETRY(::open(outfile, O_WRONLY|O_CREAT|O_TRUNC|O_BINARY, 0644)); if (fd < 0) { int err = errno; cerr << "failed to open file: " << cpp_strerror(err) << std::endl; @@ -568,7 +568,7 @@ static int do_put(IoCtx& io_ctx, int ret = 0; int fd = STDIN_FILENO; if (!stdio) - fd = open(infile, O_RDONLY); + fd = open(infile, O_RDONLY|O_BINARY); if (fd < 0) { cerr << "error reading input file " << infile << ": " << cpp_strerror(errno) << std::endl; return 1; @@ -629,7 +629,7 @@ static int do_append(IoCtx& io_ctx, int ret = 0; int fd = STDIN_FILENO; if (!stdio) - fd = open(infile, O_RDONLY); + fd = open(infile, O_RDONLY|O_BINARY); if (fd < 0) { cerr << "error reading input file " << infile << ": " << cpp_strerror(errno) << std::endl; return 1; @@ -3855,7 +3855,7 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts, if (nargs.size() < 2 || std::string(nargs[1]) == "-") { file_fd = STDOUT_FILENO; } else { - file_fd = open(nargs[1], O_WRONLY|O_CREAT|O_TRUNC, 0666); + file_fd = open(nargs[1], O_WRONLY|O_CREAT|O_TRUNC|O_BINARY, 0666); if (file_fd < 0) { cerr << "Error opening '" << nargs[1] << "': " << cpp_strerror(file_fd) << std::endl; @@ -3904,7 +3904,7 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts, if (filename == "-") { file_fd = STDIN_FILENO; } else { - file_fd = open(filename.c_str(), O_RDONLY); + file_fd = open(filename.c_str(), O_RDONLY|O_BINARY); if (file_fd < 0) { cerr << "Error opening '" << filename << "': " << cpp_strerror(file_fd) << std::endl; diff --git a/src/tools/rbd/action/Export.cc b/src/tools/rbd/action/Export.cc index b5b82f4c0dc..5551c8ee881 100644 --- a/src/tools/rbd/action/Export.cc +++ b/src/tools/rbd/action/Export.cc @@ -229,7 +229,7 @@ int do_export_diff(librbd::Image& image, const char *fromsnapname, if (strcmp(path, "-") == 0) fd = STDOUT_FILENO; else - fd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0644); + fd = open(path, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0644); if (fd < 0) return -errno; @@ -565,7 +565,7 @@ static int do_export(librbd::Image& image, const char *path, bool no_progress, if (to_stdout) { fd = STDOUT_FILENO; } else { - fd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0644); + fd = open(path, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0644); if (fd < 0) { return -errno; } diff --git a/src/tools/rbd/action/Import.cc b/src/tools/rbd/action/Import.cc index 8fca5c71f94..f8d2f936ffa 100644 --- a/src/tools/rbd/action/Import.cc +++ b/src/tools/rbd/action/Import.cc @@ -449,7 +449,7 @@ int do_import_diff(librados::Rados &rados, librbd::Image &image, if (strcmp(path, "-") == 0) { fd = STDIN_FILENO; } else { - fd = open(path, O_RDONLY); + fd = open(path, O_RDONLY|O_BINARY); if (fd < 0) { r = -errno; std::cerr << "rbd: error opening " << path << std::endl; @@ -843,7 +843,7 @@ static int do_import(librados::Rados &rados, librbd::RBD &rbd, fd = STDIN_FILENO; size = 1ULL << order; } else { - if ((fd = open(path, O_RDONLY)) < 0) { + if ((fd = open(path, O_RDONLY|O_BINARY)) < 0) { r = -errno; std::cerr << "rbd: error opening " << path << std::endl; goto done2; diff --git a/src/tools/rbd/action/Journal.cc b/src/tools/rbd/action/Journal.cc index 0379583264b..08606fcc365 100644 --- a/src/tools/rbd/action/Journal.cc +++ b/src/tools/rbd/action/Journal.cc @@ -709,7 +709,7 @@ static int do_export_journal(librados::IoCtx& io_ctx, if (to_stdout) { fd = STDOUT_FILENO; } else { - fd = open(path.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0644); + fd = open(path.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0644); if (fd < 0) { r = -errno; std::cerr << "rbd: error creating " << path << std::endl; @@ -920,7 +920,7 @@ static int do_import_journal(librados::IoCtx& io_ctx, if (from_stdin) { fd = STDIN_FILENO; } else { - if ((fd = open(path.c_str(), O_RDONLY)) < 0) { + if ((fd = open(path.c_str(), O_RDONLY|O_BINARY)) < 0) { r = -errno; std::cerr << "rbd: error opening " << path << std::endl; return r; diff --git a/src/tools/rbd/action/MergeDiff.cc b/src/tools/rbd/action/MergeDiff.cc index 406b23b4d02..d33d1c11abb 100644 --- a/src/tools/rbd/action/MergeDiff.cc +++ b/src/tools/rbd/action/MergeDiff.cc @@ -173,7 +173,7 @@ static int do_merge_diff(const char *first, const char *second, if (first_stdin) { fd = STDIN_FILENO; } else { - fd = open(first, O_RDONLY); + fd = open(first, O_RDONLY|O_BINARY); if (fd < 0) { r = -errno; std::cerr << "rbd: error opening " << first << std::endl; @@ -181,7 +181,7 @@ static int do_merge_diff(const char *first, const char *second, } } - sd = open(second, O_RDONLY); + sd = open(second, O_RDONLY|O_BINARY); if (sd < 0) { r = -errno; std::cerr << "rbd: error opening " << second << std::endl; @@ -191,7 +191,7 @@ static int do_merge_diff(const char *first, const char *second, if (strcmp(path, "-") == 0) { pd = 1; } else { - pd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0644); + pd = open(path, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0644); if (pd < 0) { r = -errno; std::cerr << "rbd: error create " << path << std::endl; diff --git a/src/tools/rbd/action/MirrorPool.cc b/src/tools/rbd/action/MirrorPool.cc index 99902c72be8..052df75f8d8 100644 --- a/src/tools/rbd/action/MirrorPool.cc +++ b/src/tools/rbd/action/MirrorPool.cc @@ -911,7 +911,7 @@ int execute_peer_bootstrap_import( int fd = STDIN_FILENO; if (token_path != "-") { - fd = open(token_path.c_str(), O_RDONLY); + fd = open(token_path.c_str(), O_RDONLY|O_BINARY); if (fd < 0) { r = -errno; std::cerr << "rbd: error opening " << token_path << std::endl;