Skip to content

Commit

Permalink
optimize ssdb_to_pika
Browse files Browse the repository at this point in the history
  • Loading branch information
@KernelMaker
KernelMaker committed Mar 31, 2017
1 parent 82054c8 commit 23e2475
Showing 1 changed file with 73 additions and 48 deletions.
121 changes: 73 additions & 48 deletions tools/ssdb_to_pika/ssdb_to_pika.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
#include "nemo.h"

const int kBatchLen = 1000;
const int kSplitNum = 1000000;
const int kSplitNum = 500000;

void MigrateKv(const std::string& ip, const int port,
const std::string& password, nemo::Nemo* db,
Expand All @@ -15,11 +15,14 @@ void MigrateKv(const std::string& ip, const int port,
std::cout << "Kv client failed to connect to ssdb" << std::endl;
return;
}
const std::vector<std::string> *resp = client->request("auth", password);
if (!resp || resp->empty() || resp->front() != "ok") {
std::cout << "Kv client auth error" << std::endl;
delete client;
return;
const std::vector<std::string> *resp;
if (password != "") {
resp = client->request("auth", password);
if (!resp || resp->empty() || resp->front() != "ok") {
std::cout << "Kv client auth error" << std::endl;
delete client;
return;
}
}
std::cout << std::this_thread::get_id() << ", Kv client start to migrate, from " << start << " to " << end << std::endl;

Expand Down Expand Up @@ -61,11 +64,14 @@ void MigrateHash(const std::string& ip, const int port,
std::cout << "Hash client failed to connect to ssdb" << std::endl;
return;
}
const std::vector<std::string> *resp = client->request("auth", password);
if (!resp || resp->empty() || resp->front() != "ok") {
std::cout << "Hash client auth error" << std::endl;
delete client;
return;
const std::vector<std::string> *resp;
if (password != "") {
resp = client->request("auth", password);
if (!resp || resp->empty() || resp->front() != "ok") {
std::cout << "Hash client auth error" << std::endl;
delete client;
return;
}
}
std::cout << std::this_thread::get_id() << ", Hash client start to migrate, " << keys.size() << " keys, from " << keys.front() << " to " << keys.back() << std::endl;

Expand Down Expand Up @@ -110,13 +116,16 @@ void MigrateQueue(const std::string& ip, const int port,
std::cout << "Queue client failed to connect to ssdb" << std::endl;
return;
}
const std::vector<std::string> *resp = client->request("auth", password);
if (!resp || resp->empty() || resp->front() != "ok") {
std::cout << "Queue client auth error" << std::endl;
delete client;
return;
const std::vector<std::string> *resp;
if (password != "") {
resp = client->request("auth", password);
if (!resp || resp->empty() || resp->front() != "ok") {
std::cout << "Queue client auth error" << std::endl;
delete client;
return;
}
}
std::cout << std::this_thread::get_id() << ", Queue client start to migrate, from " << keys.front() << " to " << keys.back() << std::endl;
std::cout << std::this_thread::get_id() << ", Queue client start to migrate, " << keys.size() << " keys, from " << keys.front() << " to " << keys.back() << std::endl;

std::vector<std::string> fs;
ssdb::Status status_ssdb;
Expand Down Expand Up @@ -160,13 +169,16 @@ void MigrateZset(const std::string& ip, const int port,
std::cout << "Zset client failed to connect to ssdb" << std::endl;
return;
}
const std::vector<std::string> *resp = client->request("auth", password);
if (!resp || resp->empty() || resp->front() != "ok") {
std::cout << "Zset client auth error" << std::endl;
delete client;
return;
const std::vector<std::string> *resp;
if (password != "") {
resp = client->request("auth", password);
if (!resp || resp->empty() || resp->front() != "ok") {
std::cout << "Zset client auth error" << std::endl;
delete client;
return;
}
}
std::cout << std::this_thread::get_id() << ", Zset client start to migrate, from " << keys.front() << " to " << keys.back() << std::endl;
std::cout << std::this_thread::get_id() << ", Zset client start to migrate, " << keys.size() << " keys, from " << keys.front() << " to " << keys.back() << std::endl;

std::vector<std::string> sms;
ssdb::Status status_ssdb;
Expand Down Expand Up @@ -210,13 +222,15 @@ void DoKv(const std::string& ip, const int port,
std::cout << "Kv center client failed to connect to ssdb" << std::endl;
return;
}
const std::vector<std::string> *resp = client->request("auth", password);
if (!resp || resp->empty() || resp->front() != "ok") {
std::cout << "Kv client auth error" << std::endl;
delete client;
return;
const std::vector<std::string> *resp;
if (password != "") {
resp = client->request("auth", password);
if (!resp || resp->empty() || resp->front() != "ok") {
std::cout << "Kv center client auth error" << std::endl;
delete client;
return;
}
}

std::string start = "";
std::string end = "";

Expand Down Expand Up @@ -259,16 +273,18 @@ void DoHash(const std::string& ip, const int port,

ssdb::Client *client = ssdb::Client::connect(ip, port);
if (client == NULL) {
std::cout << "Kv center client failed to connect to ssdb" << std::endl;
std::cout << "Hash center client failed to connect to ssdb" << std::endl;
return;
}
const std::vector<std::string>* resp = client->request("auth", password);
if (!resp || resp->empty() || resp->front() != "ok") {
std::cout << "Kv client auth error" << std::endl;
delete client;
return;
const std::vector<std::string> *resp;
if (password != "") {
resp = client->request("auth", password);
if (!resp || resp->empty() || resp->front() != "ok") {
std::cout << "Hash center client auth error" << std::endl;
delete client;
return;
}
}

std::string start = "";
std::string end = "";

Expand Down Expand Up @@ -315,14 +331,17 @@ void DoZset(const std::string& ip, const int port,

ssdb::Client *client = ssdb::Client::connect(ip, port);
if (client == NULL) {
std::cout << "Kv center client failed to connect to ssdb" << std::endl;
std::cout << "Zset center client failed to connect to ssdb" << std::endl;
return;
}
const std::vector<std::string>* resp = client->request("auth", password);
if (!resp || resp->empty() || resp->front() != "ok") {
std::cout << "Kv client auth error" << std::endl;
delete client;
return;
const std::vector<std::string> *resp;
if (password != "") {
resp = client->request("auth", password);
if (!resp || resp->empty() || resp->front() != "ok") {
std::cout << "Zset center client auth error" << std::endl;
delete client;
return;
}
}

std::string start = "";
Expand Down Expand Up @@ -374,12 +393,15 @@ void DoQueue(const std::string& ip, const int port,
std::cout << "Queue center client failed to connect to ssdb" << std::endl;
return;
}
const std::vector<std::string>* resp = client->request("auth", password);
const std::vector<std::string> *resp;
if (password != "") {
resp = client->request("auth", password);
if (!resp || resp->empty() || resp->front() != "ok") {
std::cout << "Queue client auth error" << std::endl;
std::cout << "Queue center client auth error" << std::endl;
delete client;
return;
}
}

std::string start = "";
std::string end = "";
Expand Down Expand Up @@ -424,20 +446,23 @@ void DoQueue(const std::string& ip, const int port,

void Usage() {
std::cout << "usage:" << std::endl
<< "./ssdb2pika ssdb_server_ip ssdb_server_port ssdb_server_passwd pika_db_path"
<< "./ssdb2pika ssdb_server_ip ssdb_server_port pika_db_path [ssdb_server_passwd]"
<< std::endl;
}

int main(int argc, char** argv) {
if (argc != 5) {
if (argc != 4 && argc != 5) {
Usage();
return -1;
}

const char *ip = argv[1];
int port = atoi(argv[2]);
std::string password = argv[3];
std::string nemo_path = argv[4];
std::string nemo_path = argv[3];
std::string password = "";
if (argc == 5) {
password = argv[4];
}

nemo::Options option;
option.write_buffer_size = 268435456; //256M
Expand Down

0 comments on commit 23e2475

Please sign in to comment.