Skip to content

Commit 4b18c46

Browse files
pdillingerfacebook-github-bot
authored andcommitted
Refactor: add LineFileReader and Status::MustCheck (facebook#8026)
Summary: Removed confusing, awkward, and undocumented internal API ReadOneLine and replaced with very simple LineFileReader. In refactoring backupable_db.cc, this has the side benefit of removing the arbitrary cap on the size of backup metadata files. Also added Status::MustCheck to make it easy to mark a Status as "must check." Using this, I can ensure that after LineFileReader::ReadLine returns false the caller checks GetStatus(). Also removed some excessive conditional compilation in status.h Pull Request resolved: facebook#8026 Test Plan: added unit test, and running tests with ASSERT_STATUS_CHECKED Reviewed By: mrambacher Differential Revision: D26831687 Pulled By: pdillinger fbshipit-source-id: ef749c265a7a26bb13cd44f6f0f97db2955f6f0f
1 parent 847ca9f commit 4b18c46

14 files changed

+392
-257
lines changed

CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -653,6 +653,7 @@ set(SOURCES
653653
file/file_prefetch_buffer.cc
654654
file/file_util.cc
655655
file/filename.cc
656+
file/line_file_reader.cc
656657
file/random_access_file_reader.cc
657658
file/read_write_util.cc
658659
file/readahead_raf.cc

TARGETS

+2
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,7 @@ cpp_library(
222222
"file/file_prefetch_buffer.cc",
223223
"file/file_util.cc",
224224
"file/filename.cc",
225+
"file/line_file_reader.cc",
225226
"file/random_access_file_reader.cc",
226227
"file/read_write_util.cc",
227228
"file/readahead_raf.cc",
@@ -528,6 +529,7 @@ cpp_library(
528529
"file/file_prefetch_buffer.cc",
529530
"file/file_util.cc",
530531
"file/filename.cc",
532+
"file/line_file_reader.cc",
531533
"file/random_access_file_reader.cc",
532534
"file/read_write_util.cc",
533535
"file/readahead_raf.cc",

env/mock_env.cc

+10
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include "file/filename.h"
1616
#include "port/sys_time.h"
1717
#include "rocksdb/file_system.h"
18+
#include "test_util/sync_point.h"
1819
#include "util/cast_util.h"
1920
#include "util/hash.h"
2021
#include "util/random.h"
@@ -105,6 +106,15 @@ class MemFile {
105106

106107
IOStatus Read(uint64_t offset, size_t n, const IOOptions& /*options*/,
107108
Slice* result, char* scratch, IODebugContext* /*dbg*/) const {
109+
{
110+
IOStatus s;
111+
TEST_SYNC_POINT_CALLBACK("MemFile::Read:IOStatus", &s);
112+
if (!s.ok()) {
113+
// with sync point only
114+
*result = Slice();
115+
return s;
116+
}
117+
}
108118
MutexLock lock(&mutex_);
109119
const uint64_t available = Size() - std::min(Size(), offset);
110120
size_t offset_ = static_cast<size_t>(offset);

file/line_file_reader.cc

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
2+
// This source code is licensed under both the GPLv2 (found in the
3+
// COPYING file in the root directory) and Apache 2.0 License
4+
// (found in the LICENSE.Apache file in the root directory).
5+
6+
#include "file/line_file_reader.h"
7+
8+
#include <cstring>
9+
10+
namespace ROCKSDB_NAMESPACE {
11+
12+
Status LineFileReader::Create(const std::shared_ptr<FileSystem>& fs,
13+
const std::string& fname,
14+
const FileOptions& file_opts,
15+
std::unique_ptr<LineFileReader>* reader,
16+
IODebugContext* dbg) {
17+
std::unique_ptr<FSSequentialFile> file;
18+
Status s = fs->NewSequentialFile(fname, file_opts, &file, dbg);
19+
if (s.ok()) {
20+
reader->reset(new LineFileReader(std::move(file), fname));
21+
}
22+
return s;
23+
}
24+
25+
bool LineFileReader::ReadLine(std::string* out) {
26+
assert(out);
27+
if (!status_.ok()) {
28+
// Status should be checked (or permit unchecked) any time we return false.
29+
status_.MustCheck();
30+
return false;
31+
}
32+
out->clear();
33+
for (;;) {
34+
// Look for line delimiter
35+
const char* found = static_cast<const char*>(
36+
std::memchr(buf_begin_, '\n', buf_end_ - buf_begin_));
37+
if (found) {
38+
size_t len = found - buf_begin_;
39+
out->append(buf_begin_, len);
40+
buf_begin_ += len + /*delim*/ 1;
41+
++line_number_;
42+
return true;
43+
}
44+
if (at_eof_) {
45+
status_.MustCheck();
46+
return false;
47+
}
48+
// else flush and reload buffer
49+
out->append(buf_begin_, buf_end_ - buf_begin_);
50+
Slice result;
51+
status_ = sfr_.Read(buf_.size(), &result, buf_.data());
52+
if (!status_.ok()) {
53+
status_.MustCheck();
54+
return false;
55+
}
56+
if (result.size() != buf_.size()) {
57+
// The obscure way of indicating EOF
58+
at_eof_ = true;
59+
}
60+
buf_begin_ = result.data();
61+
buf_end_ = result.data() + result.size();
62+
}
63+
}
64+
65+
} // namespace ROCKSDB_NAMESPACE

file/line_file_reader.h

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
2+
// This source code is licensed under both the GPLv2 (found in the
3+
// COPYING file in the root directory) and Apache 2.0 License
4+
// (found in the LICENSE.Apache file in the root directory).
5+
6+
#pragma once
7+
#include <array>
8+
9+
#include "file/sequence_file_reader.h"
10+
11+
namespace ROCKSDB_NAMESPACE {
12+
13+
// A wrapper on top of Env::SequentialFile for reading text lines from a file.
14+
// Lines are delimited by '\n'. The last line may or may not include a
15+
// trailing newline. Uses SequentialFileReader internally.
16+
class LineFileReader {
17+
private:
18+
std::array<char, 8192> buf_;
19+
SequentialFileReader sfr_;
20+
Status status_;
21+
const char* buf_begin_ = buf_.data();
22+
const char* buf_end_ = buf_.data();
23+
size_t line_number_ = 0;
24+
bool at_eof_ = false;
25+
26+
public:
27+
// See SequentialFileReader constructors
28+
template <typename... Args>
29+
explicit LineFileReader(Args&&... args)
30+
: sfr_(std::forward<Args&&>(args)...) {}
31+
32+
static Status Create(const std::shared_ptr<FileSystem>& fs,
33+
const std::string& fname, const FileOptions& file_opts,
34+
std::unique_ptr<LineFileReader>* reader,
35+
IODebugContext* dbg);
36+
37+
LineFileReader(const LineFileReader&) = delete;
38+
LineFileReader& operator=(const LineFileReader&) = delete;
39+
40+
// Reads another line from the file, returning true on success and saving
41+
// the line to `out`, without delimiter, or returning false on failure. You
42+
// must check GetStatus() to determine whether the failure was just
43+
// end-of-file (OK status) or an I/O error (another status).
44+
bool ReadLine(std::string* out);
45+
46+
// Returns the number of the line most recently returned from ReadLine.
47+
// Return value is unspecified if ReadLine has returned false due to
48+
// I/O error. After ReadLine returns false due to end-of-file, return
49+
// value is the last returned line number, or equivalently the total
50+
// number of lines returned.
51+
size_t GetLineNumber() const { return line_number_; }
52+
53+
// Returns any error encountered during read. The error is considered
54+
// permanent and no retry or recovery is attempted with the same
55+
// LineFileReader.
56+
const Status& GetStatus() const { return status_; }
57+
};
58+
59+
} // namespace ROCKSDB_NAMESPACE

file/read_write_util.cc

-37
Original file line numberDiff line numberDiff line change
@@ -22,43 +22,6 @@ IOStatus NewWritableFile(FileSystem* fs, const std::string& fname,
2222
return s;
2323
}
2424

25-
bool ReadOneLine(std::istringstream* iss, SequentialFileReader* seq_file_reader,
26-
std::string* output, bool* has_data, Status* result) {
27-
const int kBufferSize = 8192;
28-
char buffer[kBufferSize + 1];
29-
Slice input_slice;
30-
31-
std::string line;
32-
bool has_complete_line = false;
33-
while (!has_complete_line) {
34-
if (std::getline(*iss, line)) {
35-
has_complete_line = !iss->eof();
36-
} else {
37-
has_complete_line = false;
38-
}
39-
if (!has_complete_line) {
40-
// if we're not sure whether we have a complete line,
41-
// further read from the file.
42-
if (*has_data) {
43-
*result = seq_file_reader->Read(kBufferSize, &input_slice, buffer);
44-
}
45-
if (input_slice.size() == 0) {
46-
// meaning we have read all the data
47-
*has_data = false;
48-
break;
49-
} else {
50-
iss->str(line + input_slice.ToString());
51-
// reset the internal state of iss so that we can keep reading it.
52-
iss->clear();
53-
*has_data = (input_slice.size() == kBufferSize);
54-
continue;
55-
}
56-
}
57-
}
58-
*output = line;
59-
return *has_data || has_complete_line;
60-
}
61-
6225
#ifndef NDEBUG
6326
bool IsFileSectorAligned(const size_t off, size_t sector_size) {
6427
return off % sector_size == 0;

file/read_write_util.h

-4
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,6 @@ extern IOStatus NewWritableFile(FileSystem* fs, const std::string& fname,
2424
std::unique_ptr<FSWritableFile>* result,
2525
const FileOptions& options);
2626

27-
// Read a single line from a file.
28-
bool ReadOneLine(std::istringstream* iss, SequentialFileReader* seq_file_reader,
29-
std::string* output, bool* has_data, Status* result);
30-
3127
#ifndef NDEBUG
3228
bool IsFileSectorAligned(const size_t off, size_t sector_size);
3329
#endif // NDEBUG

0 commit comments

Comments
 (0)