Skip to content

Commit

Permalink
read xlog.db entries into hash (improves performance significantly)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hamann
hamann committed Oct 21, 2013
1 parent 49e4872 commit abddb56
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 75 deletions.
34 changes: 18 additions & 16 deletions lib/rbarman/cli_command.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,13 +117,13 @@ def servers(opts = {})
def wal_files(server, backup_id)
lines = run_barman_command("list-files --target wal #{server} #{backup_id}")
wal_files = parse_wal_files_list(lines)
xlog_db_lines = file_content("#{@barman_home}/#{server}/wals/xlog.db")
xlog_db = read_xlog_db(server)
wal_files.each do |w|
wal = "#{w.timeline}#{w.xlog}#{w.segment}"
lines = xlog_db_lines.grep(/^#{wal}\t.+/)
raise(RuntimeError, "Found more than one wal file entry in xlog.db for #{wal}") if lines.count > 1
raise(RuntimeError, "Could not find any entry for #{wal} in xlog.db") if lines.count == 0
wal_file_info_from_xlog_db_line(w, lines[0])
entry = xlog_db[wal]
w.size = entry[:size]
w.compression = entry[:compression]
w.created = entry[:created]
end
return wal_files
end
Expand Down Expand Up @@ -253,17 +253,6 @@ def parse_backup_info_file(backup)
end
end

# Assigns size, created and compression values to a {WalFile} by parsing a line from xlog.db
# @param [WalFile] wal_file the wal file
# @param [String] line a string like '00000001000005A9000000BC\\t4684503\t1360568429.0\\tbzip2'
# @return [void]
def wal_file_info_from_xlog_db_line(wal_file, line)
splitted = line.split("\t")
wal_file.size = splitted[1]
wal_file.created = splitted[2].to_i
wal_file.compression = splitted[3].downcase.to_sym
end

# Converts the size according to the unit to bytes
# @param [Numeric] size the size
# @param [String] unit the unit, like `B`, `KiB`, `MiB`, `GiB` or `TiB`
Expand Down Expand Up @@ -356,5 +345,18 @@ def run_barman_command(args, opts = {})
def file_content(path)
return File.readlines(path).map { |l| l.chomp }
end

def read_xlog_db(server)
result = Hash.new
File.readlines("#{@barman_home}/#{server}/wals/xlog.db").each do |line|
splitted = line.chomp.split("\t")
result[splitted[0]] = {
:size => splitted[1],
:created => splitted[2],
:compression => splitted[3].downcase.to_sym
}
end
result
end
end
end
79 changes: 20 additions & 59 deletions spec/cli_command_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,13 +167,15 @@
"/var/lib/barman/test/wals/00000001000005A9/00000001000005A9000000BD"
]

xlog_db_lines = [
"00000001000005A9000000BC\t4684503\t1360568429.0\tbzip2",
"00000001000005A9000000BD\t5099998\t1360568442.0\tnone"
]

xlog_db = Hash.new
xlog_db["00000001000005A9000000BC"] = {:size => 4684503, :created => 1360568429.0, :compression => :bzip2 }
xlog_db["00000001000005A9000000BD"] = {:size => 5099998, :created => 1360568442.0, :compression => :bzip2 }


@cmd.stub!(:run_barman_command).and_return(backup_list, wal_files)
@cmd.stub!(:file_content).and_return(backup_info_lines, xlog_db_lines)
@cmd.stub(:run_barman_command).and_return(backup_list, wal_files)
@cmd.stub(:file_content).and_return(backup_info_lines)
@cmd.stub(:read_xlog_db).and_return(xlog_db)
backups = @cmd.backups("test", { :with_wal_files => true })
expect(backups).to be_an_instance_of Backups
expect(backups.count).to eq(1)
Expand Down Expand Up @@ -211,10 +213,10 @@
"/var/lib/barman/test/wals/00000001000005A9/00000001000005A9000000BD"
]

xlog_db_lines = [
"00000001000005A9000000BC\t4684503\t1360568429.0\tbzip2",
"00000001000005A9000000BD\t5099998\t1360568442.0\tnone"
]
xlog_db = Hash.new
xlog_db["00000001000005A9000000BC"] = {:size => 4684503, :created => 1360568429.0, :compression => :bzip2 }
xlog_db["00000001000005A9000000BD"] = {:size => 5099998, :created => 1360568442.0, :compression => :bzip2 }


backup_info_lines = [
"begin_time=2013-02-25 19:26:54.852814",
Expand All @@ -226,8 +228,9 @@
"begin_wal=0000000100000552000000B6",
"end_wal=000000010000055700000031"
]
@cmd.stub!(:run_barman_command).and_return(backup_list, wal_files)
@cmd.stub!(:file_content).and_return(backup_info_lines, xlog_db_lines)
@cmd.stub(:run_barman_command).and_return(backup_list, wal_files)
@cmd.stub(:file_content).and_return(backup_info_lines)
@cmd.stub(:read_xlog_db).and_return(xlog_db)
backup = @cmd.backup("test", "20130222T080002", { :with_wal_files => false })
expect(backup.id).to eq("20130222T080002")
expect(backup.wal_files).to eq(nil)
Expand Down Expand Up @@ -293,59 +296,17 @@
"/var/lib/barman/test/wals/00000001000005A9/00000001000005A9000000BD"
]

xlog_db_lines = [
"00000001000005A9000000BC\t4684503\t1360568429.0\tbzip2",
"00000001000005A9000000BD\t5099998\t1360568442.0\tnone",
"00000001000005A9000000BD.00000020.backup\t249\t1360562212.0\tnone"
]
xlog_db = Hash.new
xlog_db["00000001000005A9000000BC"] = {:size => 4684503, :created => 1360568429.0, :compression => :bzip2 }
xlog_db["00000001000005A9000000BD"] = {:size => 5099998, :created => 1360568442.0, :compression => :bzip2 }
xlog_db["00000001000005A9000000BD.00000020.backup"] = {:size => 249, :created => 1360562212.0, :compression => :none }

@cmd.stub!(:run_barman_command).and_return(lines)
@cmd.stub!(:file_content).and_return(xlog_db_lines)
@cmd.stub!(:read_xlog_db).and_return(xlog_db)

wal_files = @cmd.wal_files("test", "123")
expect(wal_files.count).to eq(2)
end

it 'should return an RuntimeError if xlog.db does not contain an entry for a wal' do
lines = [
"/var/lib/barman/test/wals/00000001000005A9/00000001000005A9000000BC"
]

xlog_db_lines = [
]

@cmd.stub!(:run_barman_command).and_return(lines)
@cmd.stub!(:file_content).and_return(xlog_db_lines)

expect { @cmd.wal_files("test", "123") }.to raise_error(RuntimeError)
end

it 'should return an RuntimeError if xlog.db does contain more than one entry for a wal' do
lines = [
"/var/lib/barman/test/wals/00000001000005A9/00000001000005A9000000BC"
]

xlog_db_lines = [
"00000001000005A9000000BC\t4684503\t1360568429.0\tbzip2",
"00000001000005A9000000BC\t4682233\t1360568442.0\tbzip2",
]

@cmd.stub!(:run_barman_command).and_return(lines)
@cmd.stub!(:file_content).and_return(xlog_db_lines)

expect { @cmd.wal_files("test", "123") }.to raise_error(RuntimeError)
end
end

describe "wal_file_info_from_xlog_db_line" do
it 'should set wal file infos from a xlog db line' do
line = "000000010000049A000000FA\t4684503\t1360568429.0\tbzip2"
w = WalFile.parse("000000010000049A000000FA")
@cmd.wal_file_info_from_xlog_db_line(w, line)
expect(w.size).to eq(4684503)
expect(w.created).to eq(Time.at(1360568429))
expect(w.compression).to eq("bzip2".to_sym)
end
end

describe "parse_show_server_lines" do
Expand Down

0 comments on commit abddb56

Please sign in to comment.