Skip to content

Commit

Permalink
add compact-interval
Browse files Browse the repository at this point in the history
  • Loading branch information
@KernelMaker
KernelMaker committed Aug 22, 2017
1 parent 4a790c1 commit acddae8
Show file tree
Hide file tree
Showing 6 changed files with 99 additions and 23 deletions.
9 changes: 7 additions & 2 deletions conf/pika.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,14 @@ db-sync-speed : -1
# network-interface : eth1
# replication
# slaveof : master-ip:master-port
# CronTask, format: start:end-ratio, like 02-04/60, pika will check to schedule compaction between 2 to 4 o'clock everyday
# if the freesize/disksize > 60%
#
# CronTask, format: start-end/ratio, like 02-04/60, pika will check to schedule compaction between 2 to 4 o'clock everyday
# if the freesize/disksize > 60%. NOTICE:if compact-interval is set, compact-cron will be mask and disable.
# compact-cron :
#
# Compact-interval, format: interval/ratio, like 6/60, pika will check to schedule compaction every 6 hours,
# if the freesize/disksize > 60%. NOTICE:compact-interval is prior than compact-cron;
# compact-interval :

###################
## Critical Settings
Expand Down
7 changes: 6 additions & 1 deletion include/pika_conf.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ class PikaConf : public slash::BaseConf {
std::string db_sync_path() { RWLock l(&rwlock_, false); return db_sync_path_; }
int db_sync_speed() { RWLock l(&rwlock_, false); return db_sync_speed_; }
std::string compact_cron() { RWLock l(&rwlock_, false); return compact_cron_; }
std::string compact_interval() { RWLock l(&rwlock_, false); return compact_interval_; }
int write_buffer_size() { RWLock l(&rwlock_, false); return write_buffer_size_; }
int timeout() { RWLock l(&rwlock_, false); return timeout_; }

Expand Down Expand Up @@ -146,7 +147,10 @@ class PikaConf : public slash::BaseConf {
void SetCompactCron(const std::string &value) {
RWLock l(&rwlock_, true);
compact_cron_ = value;

}
void SetCompactInterval(const std::string &value) {
RWLock l(&rwlock_, true);
compact_interval_ = value;
}

int Load();
Expand All @@ -164,6 +168,7 @@ class PikaConf : public slash::BaseConf {
int expire_dump_days_;
int db_sync_speed_;
std::string compact_cron_;
std::string compact_interval_;
int write_buffer_size_;
int log_level_;
bool daemonize_;
Expand Down
1 change: 1 addition & 0 deletions include/pika_server.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,7 @@ class PikaServer {

time_t start_time_s_;
bool have_scheduled_crontask_;
struct timeval last_check_compact_time_;

int worker_num_;
PikaDispatchThread* pika_dispatch_thread_;
Expand Down
32 changes: 30 additions & 2 deletions src/pika_admin.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -788,6 +788,10 @@ void ConfigCmd::ConfigGet(std::string &ret) {
ret = "*2\r\n";
EncodeString(&ret, "compact-cron");
EncodeString(&ret, g_pika_conf->compact_cron());
} else if (get_item == "compact-interval") {
ret = "*2\r\n";
EncodeString(&ret, "compact-interval");
EncodeString(&ret, g_pika_conf->compact_interval());
} else if (get_item == "maxmemory") {
ret = "*2\r\n";
EncodeString(&ret, "maxmemory");
Expand Down Expand Up @@ -901,7 +905,7 @@ void ConfigCmd::ConfigGet(std::string &ret) {
EncodeString(&ret, "slaveof");
EncodeString(&ret, g_pika_conf->slaveof());
} else if (get_item == "*") {
ret = "*76\r\n";
ret = "*78\r\n";
EncodeString(&ret, "port");
EncodeInt32(&ret, g_pika_conf->port());
EncodeString(&ret, "thread-num");
Expand Down Expand Up @@ -974,6 +978,8 @@ void ConfigCmd::ConfigGet(std::string &ret) {
EncodeInt32(&ret, g_pika_conf->db_sync_speed());
EncodeString(&ret, "compact-cron");
EncodeString(&ret, g_pika_conf->compact_cron());
EncodeString(&ret, "compact-interval");
EncodeString(&ret, g_pika_conf->compact_interval());
EncodeString(&ret, "network-interface");
EncodeString(&ret, g_pika_conf->network_interface());
EncodeString(&ret, "slaveof");
Expand All @@ -986,7 +992,7 @@ void ConfigCmd::ConfigGet(std::string &ret) {
void ConfigCmd::ConfigSet(std::string& ret) {
std::string set_item = config_args_v_[1];
if (set_item == "*") {
ret = "*16\r\n";
ret = "*18\r\n";
EncodeString(&ret, "loglevel");
EncodeString(&ret, "timeout");
EncodeString(&ret, "requirepass");
Expand All @@ -1003,6 +1009,8 @@ void ConfigCmd::ConfigSet(std::string& ret) {
EncodeString(&ret, "slowlog-log-slower-than");
EncodeString(&ret, "slave-read-only");
EncodeString(&ret, "db-sync-speed");
EncodeString(&ret, "compact-cron");
EncodeString(&ret, "compact-interval");
return;
}
std::string value = config_args_v_[2];
Expand Down Expand Up @@ -1134,6 +1142,26 @@ void ConfigCmd::ConfigSet(std::string& ret) {
g_pika_conf->SetCompactCron(value);
ret = "+OK\r\n";
}
} else if (set_item == "compact-interval") {
bool invalid = false;
std::string::size_type len = value.length();
std::string::size_type slash = value.find("/");
if (slash == std::string::npos || slash + 1 >= len) {
invalid = true;
} else {
int interval = std::atoi(value.substr(0, slash).c_str());
int usage = std::atoi(value.substr(slash+1).c_str());
if (interval <= 0 || usage < 0 || usage > 100) {
invalid = true;
}
}
if (invalid) {
ret = "-ERR invalid compact-interval\r\n";
return;
} else {
g_pika_conf->SetCompactInterval(value);
ret = "+OK\r\n";
}
} else {
ret = "-ERR No such configure item\r\n";
}
Expand Down
16 changes: 16 additions & 0 deletions src/pika_conf.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,21 @@ int PikaConf::Load()
}
}
}
compact_interval_ = "";
GetConfStr("compact-interval", &compact_interval_);
if (compact_interval_ != "") {
std::string::size_type len = compact_interval_.length();
std::string::size_type slash = compact_interval_.find("/");
if (slash == std::string::npos || slash + 1 >= len) {
compact_interval_ = "";
} else {
int interval = std::atoi(compact_interval_.substr(0, slash).c_str());
int usage = std::atoi(compact_interval_.substr(slash+1).c_str());
if (interval <= 0 || usage < 0 || usage > 100) {
compact_interval_ = "";
}
}
}

// write_buffer_size
GetConfInt("write-buffer-size", &write_buffer_size_);
Expand Down Expand Up @@ -232,6 +247,7 @@ int PikaConf::ConfigRewrite() {
SetConfInt("slowlog-log-slower-than", slowlog_log_slower_than_);
SetConfBool("slave-read-only", readonly_);
SetConfStr("compact-cron", compact_cron_);
SetConfStr("compact-interval", compact_interval_);
SetConfStr("network-interface", network_interface_);
SetConfStr("slaveof", slaveof_);

Expand Down
57 changes: 39 additions & 18 deletions src/pika_server.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ PikaServer::PikaServer() :
ping_thread_(NULL),
exit_(false),
have_scheduled_crontask_(false),
last_check_compact_time_({0, 0}),
sid_(0),
master_ip_(""),
master_connection_(0),
Expand Down Expand Up @@ -1109,10 +1110,44 @@ bool PikaServer::GetBinlogFiles(std::map<uint32_t, std::string>& binlogs) {
}

void PikaServer::AutoCompactRange() {
struct statfs disk_info;
int ret = statfs(g_pika_conf->db_path().c_str(), &disk_info);
if (ret == -1) {
LOG(WARNING) << "statfs error: " << strerror(errno);
return;
}

uint64_t total_size = disk_info.f_bsize * disk_info.f_blocks;
uint64_t free_size = disk_info.f_bsize * disk_info.f_bfree;
// LOG(INFO) << "free_size: " << free_size << " disk_size: " << total_size <<
// " cal: " << ((double)free_size / total_size) * 100;
std::string ci = g_pika_conf->compact_interval();
std::string cc = g_pika_conf->compact_cron();
if (cc == "") {

if (ci != "") {
std::string::size_type slash = ci.find("/");
int interval = std::atoi(ci.substr(0, slash).c_str());
int usage = std::atoi(ci.substr(slash+1).c_str());
struct timeval now;
gettimeofday(&now, NULL);
if (last_check_compact_time_.tv_sec == 0 ||
now.tv_sec - last_check_compact_time_.tv_sec
>= interval * 3600) {
gettimeofday(&last_check_compact_time_, NULL);
if (((double)free_size / total_size) * 100 >= usage) {
nemo::Status s = db_->Compact(nemo::kALL);
if (s.ok()) {
LOG(INFO) << "[Interval]schedule compactRange, freesize: " << free_size/1048576 << "MB, disksize: " << total_size/1048576 << "MB";
} else {
LOG(INFO) << "[Interval]schedule compactRange Failed, freesize: " << free_size/1048576 << "MB, disksize: " << total_size/1048576
<< "MB, error: " << s.ToString();
}
}
}
return;
} else {
}

if (cc != "") {
std::string::size_type colon = cc.find("-");
std::string::size_type underline = cc.find("/");
int start = std::atoi(cc.substr(0, colon).c_str());
Expand All @@ -1129,27 +1164,13 @@ void PikaServer::AutoCompactRange() {
} else {
have_scheduled_crontask_ = false;
}
// LOG(INFO) << "start: " << start << " end: " << end << " usage " << usage <<
// " have_scheduled: " << have_scheduled_crontask_ << " in_window: " << in_window;
if (!have_scheduled_crontask_ && in_window) {
struct statfs disk_info;
int ret = statfs(g_pika_conf->db_path().c_str(), &disk_info);
if (ret == -1) {
LOG(WARNING) << "statfs error: " << strerror(errno);
return;
}

uint64_t total_size = disk_info.f_bsize * disk_info.f_blocks;
uint64_t free_size = disk_info.f_bsize * disk_info.f_bfree;

// LOG(INFO) << "free_size: " << free_size << " disk_size: " << total_size <<
// " cal: " << ((double)free_size / total_size) * 100;
if (((double)free_size / total_size) * 100 >= usage) {
nemo::Status s = db_->Compact(nemo::kALL);
if (s.ok()) {
LOG(INFO) << "schedule compactRange, freesize: " << free_size/1048576 << "MB, disksize: " << total_size/1048576 << "MB";
LOG(INFO) << "[Cron]schedule compactRange, freesize: " << free_size/1048576 << "MB, disksize: " << total_size/1048576 << "MB";
} else {
LOG(INFO) << "schedule compactRange Failed, freesize: " << free_size/1048576 << "MB, disksize: " << total_size/1048576
LOG(INFO) << "[Cron]schedule compactRange Failed, freesize: " << free_size/1048576 << "MB, disksize: " << total_size/1048576
<< "MB, error: " << s.ToString();
}
have_scheduled_crontask_ = true;
Expand Down

0 comments on commit acddae8

Please sign in to comment.