当前位置：网站首页>Usage Summary of hiredis and rapidjson Libraries

Usage Summary of hiredis and rapidjson Libraries

2022-04-21 11:57:00 【Maverick cat a】

Hiredis brief introduction

Hiredis yes Redis Officially released C Version client hiredis library .redis It is also used in the source code of hiredis. such as redis-cli and Redis Sentinel mechanism and master-slave mechanism in , Clusters are used hiredis.

hiredis Provides synchronization 、 Asynchronous access , asynchronous API Need to work with some event Libraries .

Its general workflow ：

Establishing a connection -> dispatch orders -> Wait for the results and deal with -> Release the connection .

Hiredis Easy to use

Some pits have also been encountered in use , Here is a summary . Like that one. mset Batch submit data instruction . Look at the code below ：

    // mset key1 value1 key2 value2 ........
	int ret = REDIS_ERR;
	std::ostringstream out;
	for (auto val : keys_vals) {
	   out << " "<< val.first << " " << val.second;
	}
	reply = (redisReply *)redisCommand(context, "MSET %s", out.str().c_str());
	if (reply != nullptr) {
	    serverLog(LL_NOTICE, "mSetWithCommit:%s\n", reply->str);
	    freeReplyObject(reply);
	    //serverLog(LL_NOTICE, "ok\n");
	    //return REDIS_OK;
	}

In fact, there are problems , if value Spaces in , Will report ：Hiredis MSET,error:ERR wrong number of arguments for MSET.

Use hiredis Of API When calling, if it is the following command ：

hmset userid:1001 username 'xiao ming'

This grammar , Use redis-cli There is no problem , But if hiredis There will be problems .

newspaper ERR wrong number of arguments for HMSET error .

The reason is that xiao ming There's a space , He took it for username 'xiao, The other is ming' Missing value later , It's a mistake . There's a pit here .

terms of settlement ： Use redisCommandArgv Splicing .

//! \brief  Batch submit data warehousing 
//! \param keys_vals 
//! \return REDIS_OK/REDIS_ERR
int mSetWithCommit(const std::map<std::string, std::string>& keys_vals) {
	serverLog(LL_NOTICE, "->mSetWithCommit:\n");

	redisReply *reply;
	if (context == nullptr) return REDIS_ERR;

	int ret = REDIS_ERR;
	
	std::vector<std::string> tVec;
	tVec.push_back("MSET");
	for (auto it : keys_vals) {
		tVec.push_back(it.first);
		tVec.push_back(it.second);
	}
	std::vector<const char *> argv(tVec.size());
	std::vector<size_t> argvlen(tVec.size());
	int j = 0;
	for (auto i = tVec.begin(); i != tVec.end(); ++i, ++j){
		argv[j] = i->c_str();
		argvlen[j] = i->length();
	}
	reply = (redisReply *)redisCommandArgv(context, argv.size(), &(argv[0]), &(argvlen[0]));
	if (reply != nullptr) {
		ret = REDIS_OK;
		if (reply->type == REDIS_REPLY_ERROR) {
			serverLog(LL_WARNING, "MSET error,error:%s\n", reply->str);
			ret = REDIS_ERR;
		}
		freeReplyObject(reply);
	}
	if (ret == REDIS_OK) {
		serverLog(LL_NOTICE, "ok\n");
	}
	return ret;
}

Connection correlation

// Redis Connection configuration related 
static const char* HostIP = "127.0.0.1";
static const char* Auth = "XXXXX";
static const int Hostport = 6379;

static redisContext *context;

/* Connect to the server. If force is not zero the connection is performed
* even if there is already a connected socket. */
static int cliConnect(int force) {
	serverLog(LL_NOTICE, "->cliConnect:\n");
	if (context == NULL || force) {
		if (context != NULL) {
			redisFree(context);
		}
		context = redisConnect(HostIP, Hostport);
		if (context->err) {
			fprintf(stderr, "Could not connect to Redis at ");
			fprintf(stderr, "%s: %s\n", HostIP, context->errstr);
			redisFree(context);
			context = NULL;
			return REDIS_ERR;
		}

		// Do AUTH and select the right DB
		if (cliAuth(Auth) != REDIS_OK)
			return REDIS_ERR;
		if (cliSelect(0) != REDIS_OK)
			return REDIS_ERR;
		serverLog(LL_NOTICE, "OK\n");
		return REDIS_OK;
	}
	return REDIS_ERR;
}

/* Send AUTH command to the server */
static int cliAuth(const char* auth) {
	redisReply *reply;
	if (auth == NULL) return REDIS_OK;

	reply = (redisReply *)redisCommand(context, "AUTH %s", auth);
	if (reply != NULL) {
		freeReplyObject(reply);
		return REDIS_OK;
	}
	return REDIS_ERR;
}

/* Send SELECT dbnum to the server */
static int cliSelect(int dbnum) {
	redisReply *reply;
	if (dbnum == 0) return REDIS_OK;
	reply = (redisReply *)redisCommand(context, "SELECT %d", dbnum);
	if (reply != NULL) {
		int result = REDIS_OK;
		if (reply->type == REDIS_REPLY_ERROR) result = REDIS_ERR;
		freeReplyObject(reply);
		return result;
	}
	return REDIS_ERR;
}

Common interfaces encapsulation

//! \brief  Establishing a connection 
//! \param force 
//! \return REDIS_OK/REDIS_ERR
int wrapper_cliConnect(int force) {
	return cliConnect(force);
}

//! \brief  Get a single key-value(String)
//! \param key 
//! \param value 
//! \return REDIS_OK/REDIS_ERR
int getValueString(const std::string &key, std::string &value) {
	redisReply *reply;

	if (context == nullptr) return REDIS_ERR;

	reply = (redisReply *)redisCommand(context, "GET %s", key.c_str());
	if (reply != NULL) {
		int result = REDIS_ERR;
		if (reply->type == REDIS_REPLY_ERROR) result = REDIS_ERR;

		if (reply->type == REDIS_REPLY_STRING) {
			//int len = reply->len;
			//serverLog(LL_NOTICE, "reply->len =%d\n", len);
			//serverLog(LL_NOTICE, "reply->str =%s\n", reply->str);
			value = reply->str;
			//serverLog(LL_NOTICE, "%s\n", reply->str);
			result = REDIS_OK;
		}

		freeReplyObject(reply);
		return result;
	}
	return REDIS_ERR;
}

//! \brief  Set up individual key-value(String)
//! \param key 
//! \param value 
//! \return REDIS_OK/REDIS_ERR
int setValueString(const std::string &key, std::string &value) {
	redisReply *reply;

	if (context == nullptr) return REDIS_ERR;

	reply = (redisReply *)redisCommand(context, "SET %s %s", key.c_str(), value.c_str());
	if (reply != NULL) {
		int result = REDIS_OK;
		if (reply->type == REDIS_REPLY_ERROR) {
			serverLog(LL_WARNING, "SET error,key:%s,val:%s,error:%s\n", key.c_str(), value.c_str(), reply->str);
			result = REDIS_ERR;
		}
		freeReplyObject(reply);
		return result;
	}
	return REDIS_ERR;
}

//! \brief  Get all key value data （ Not recommended for use keys* Have an impact on ）
//! \param keys_vals 
//! \return REDIS_OK/REDIS_ERR
int getAllKeysVals(std::map<std::string, std::string> &keys_vals) {
	redisReply *reply;
	int ret = REDIS_ERR;

	if (context == nullptr) return REDIS_ERR;

	reply = (redisReply *)redisCommand(context, "keys *");
	if (reply != nullptr) {
		redisReply** repy;
		int count = reply->elements;
		repy = reply->element;
		for (int i = 0; i < count; i++) {
			// Storage key-val value 
			std::string val;
			ret = getValueString((*repy)->str,val);
			if (ret == REDIS_OK) {
				keys_vals.emplace((*repy)->str, val);
			}else {
				serverLog(LL_WARNING, "GET error\n");
			}
			repy++;
		}
		freeReplyObject(reply);
		return REDIS_OK;
	}
	return REDIS_ERR;
	
}

//! \brief  Get all key value data (SCAN  Instructions are read in batches   Every time 1000 strip )
//! \param keys_vals 
//! \return REDIS_OK/REDIS_ERR
int getAllKeysVals_S(std::map<std::string, std::string> &keys_vals) {
	redisReply *reply;
	int ret = REDIS_ERR;

	if (context == nullptr) return REDIS_ERR;

	int index = 0;
	do {
		reply = (redisReply *)redisCommand(context, "SCAN %d MATCH * COUNT 1000", index);
		if (reply == nullptr) return REDIS_ERR;

		if (reply->type != REDIS_REPLY_ARRAY) {
			freeReplyObject(reply);
			return ret;
		}

		index = atoi(reply->element[0]->str);
		//serverLog(LL_NOTICE,"index:%d", index);
		if (1 == reply->elements) {
			serverLog(LL_WARNING, "no data\n");
			freeReplyObject(reply);
			return ret;
		}
		if (reply->element[1]->type != REDIS_REPLY_ARRAY) {
			serverLog(LL_WARNING,"redis scan keys reply not array");
			freeReplyObject(reply);
			return ret;
		}
		int count = reply->element[1]->elements;
		//serverLog(LL_NOTICE, "count:%d", count);
		for (int i = 0; i < count; i++) {
			std::string val;
			std::string key = reply->element[1]->element[i]->str;
			//serverLog(LL_NOTICE,"i:%d,key:%s\n", i, key.c_str());
			ret = getValueString(key, val);
			if (ret == REDIS_OK) {
				keys_vals.emplace(key, val);
			}else {
				serverLog(LL_WARNING, "GET error\n");
			}
		}

	} while (0 != index);

	freeReplyObject(reply);
	return REDIS_OK;
}

//! \brief  Batch submit data warehousing 
//! \param keys_vals 
//! \return REDIS_OK/REDIS_ERR
int mSetWithCommit(const std::map<std::string, std::string>& keys_vals) {
	serverLog(LL_NOTICE, "->mSetWithCommit:\n");

	redisReply *reply;
	if (context == nullptr) return REDIS_ERR;

	int ret = REDIS_OK;
	for (auto val : keys_vals) {
		ret = setValueString(val.first, val.second);
		if (ret != REDIS_OK) {
			serverLog(LL_WARNING, "SET error\n");
			return ret;
		}
	}
	serverLog(LL_NOTICE, "ok\n");
	/*
	// mset key1 value1 key2 value2 ........
	int ret = REDIS_ERR;
	std::ostringstream out;
	for (auto val : keys_vals) {
	out << " "<< val.first << " " << val.second;
	}
	reply = (redisReply *)redisCommand(context, "MSET %s", out.str().c_str());
	if (reply != nullptr) {
	serverLog(LL_NOTICE, "mSetWithCommit:%s\n", reply->str);
	freeReplyObject(reply);
	//serverLog(LL_NOTICE, "ok\n");
	//return REDIS_OK;
	}
	*/
	return ret;
}

//! \brief  Get current date   Format  YYYYMMDD 
//! \param  empty  
//! \return string
std::string getNowDate() {
	time_t lt;
	struct tm * now;
	char tmbuf[64];

	lt = time(NULL);
	now = localtime(&lt);
	// %Y%m%d%H%M%S
	strftime(tmbuf, sizeof(tmbuf), "%Y%m%d", now);
	std::string nowDate(tmbuf);
	return std::move(nowDate);
}
//! \brief  Storage json file   Save and read in binary form 
//! \param strjson 
//! \return 0  success   Not 0 Failure 
int saveDumpJson(const std::string &strjson) {
	serverLog(LL_NOTICE, "->saveDumpJson:\n");
	std::ofstream outFile;
	outFile.open(TEMP_FILE_NAME, std::ios::binary);
	if (!outFile.is_open()) {
		outFile.close();
		serverLog(LL_NOTICE, "Error\n");
		return -1;
	}
	outFile << strjson << std::endl;
	outFile.close();
	//  Write a temporary file first , After success, the move operation becomes an official document 
	auto ret = MoveFileExA(TEMP_FILE_NAME.c_str(), (DUMP_DIR_PATH + DUMP_FILE_PRIFIX + getNowDate()).c_str(),
		MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED | MOVEFILE_WRITE_THROUGH);
	if (ret) {
		serverLog(LL_NOTICE, "OK\n");
		return 0;
	}
	return -2;

}

//  Read file operation 
std::string readAll(const std::string& fileName){
	std::ifstream in(fileName, std::ios::binary);
	std::istreambuf_iterator<char> begin(in);
	std::istreambuf_iterator<char> end;
	return std::string{ begin, end };
}

RapidJSON brief introduction

RapidJSON Tencent is an efficient open source C++ JSON Parsers and generators , It's just a header file C++ library .RapidJSON It's cross platform , Support Windows, Linux, Mac OS X And iOS, Android.

Its source code is https://github.com/Tencent/rapidjson/, Stable version is 2016 Published in 1.1.0 edition .

RapidJSON characteristic

(1). RapidJSON Small and complete ： It also supports SAX and DOM Style API,SAX The parser is only about 500 Line code .

(2). RapidJSON fast ： Its performance can be compared with strlen() comparison , Can support SSE2/SSE4.2 Speed up , Use templates and inline functions to reduce the cost of function calls .

(3). RapidJSON Independent ： It doesn't depend on BOOST Wait for the external library , It doesn't even depend on STL.

(4). RapidJSON Memory friendly ： In most 32/64 On the bit machine , Every JSON It's only worth 16 byte ( In addition to strings ), It is preset to use a fast memory allocator , Allows the analyzer to allocate memory compactly .

(5). RapidJSON Yes Unicode friendly ： It supports UTF-8、UTF-16、UTF-32( big-endian / Small end of the sequence ), And internal support for the detection of these codes 、 Check and transcode . for example ,RapidJSON You can analyze a UTF-8 Document to DOM (Document Object Model, File object model ) when , Put the JSON String transcoding to UTF-16. It also supports proxy pairs (surrogate pair) And "\u0000"( Null character ).

Every JSON Values are stored as Value class , and Document Class represents the whole DOM, It stores a DOM The root of the tree Value.RapidJSON All public types and functions of are in rapidjson In the namespace .

Parsing and generation JSON Time consuming （ The lower the better ）：

Resolve to DOM Memory usage after （ The lower the better ）：

Easy to use

std::string objectToString(const rapidjson::Value& valObj)
{
	rapidjson::StringBuffer buffer;
	rapidjson::Writer<rapidjson::StringBuffer> jWriter(buffer);
	valObj.Accept(jWriter);
	return buffer.GetString();
}

//! \brief  Key value pairs are converted to jsonString
//! \param keyVals 
//! \return string
std::string createJsonString(std::map<std::string, std::string> keyVals) {
	/*
	rapidjson::Document doc;
	rapidjson::Document::AllocatorType &allocator = doc.GetAllocator();
	doc.SetObject();
	//rapidjson::Value root(rapidjson::kObjectType);
	rapidjson::Value key(rapidjson::kStringType);
	rapidjson::Value value(rapidjson::kStringType);
	for (auto it : keyVals) {
	key.SetString(it.first.c_str(), allocator);
	value.SetString(it.second.c_str(), allocator);
	doc.AddMember(key, value, allocator);
	}

	rapidjson::StringBuffer buffer;
	rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
	doc.Accept(writer);
	return std::move(std::string(buffer.GetString()));
	*/
	rapidjson::StringBuffer strBuf;
	rapidjson::Writer<rapidjson::StringBuffer> writer(strBuf);
	writer.StartObject();
	for (auto it : keyVals) {
		writer.Key(it.first.c_str());
		writer.RawNumber(it.second.c_str(), it.second.length());
	}
	writer.EndObject();
	return std::move(std::string(strBuf.GetString()));
}

//! \brief  Traversal folder 
//! \brief  Get the prefix of the file name and find the specified file name +sort Sort   Descending , date   From big to small  
//! \param path,prefix,file_list
//! \return 0 success   Not 0 Failure 
int loadFileList(const std::string path, const std::string prefix,
	std::vector<std::string>& file_list) {

	struct _finddata_t c_file;
	intptr_t   hFile;

	if (_chdir(path.c_str())) {
		serverLog(LL_WARNING, "Dir error,not exist:%s\n", path.c_str());
		return -1;
	}
	//  Find the first one first 
	hFile = _findfirst((prefix+"*").c_str(), &c_file);
	if (hFile == -1) {
		serverLog(LL_WARNING,"No files in current directory!\n");
		return -2;
	}
	//serverLog(LL_NOTICE, "c_file.name:%s\n", c_file.name);
	file_list.emplace_back(c_file.name);
	// Find the rest of the files 
	while (_findnext(hFile, &c_file) == 0) {
		//serverLog(LL_NOTICE, "c_file.name:%s\n", c_file.name);
		file_list.emplace_back(c_file.name);
	}
	_findclose(hFile);

	//  Sort  greater-- Descending   date   From big to small  
	sort(file_list.begin(), file_list.end(), std::greater<std::string>());

	return 0;
}


//! \brief  Load data 
//! \brief
//! \param  empty 
//! \return 0 success   Not 0 Failure 
int loadDumpJson() {
	std::vector<std::string> fileList;
	std::string loadFileName;

	//  Traverse all file names 
	loadFileList(DUMP_DIR_PATH, DUMP_FILE_PRIFIX, fileList);
	for (auto f : fileList) {
		serverLog(LL_NOTICE, "file:%s\n", f.c_str());
	}
	if (fileList.empty()) {
		serverLog(LL_WARNING, "fileList empty\n");
		return -1;
	}
	//  Always load and read the latest backup file 
	loadFileName = DUMP_DIR_PATH + fileList[0];
	std::string contents = readAll(loadFileName);
	//  Check the legitimacy of the content 
	rapidjson::Document doc;
	if (doc.Parse(contents.c_str()).HasParseError()) {
		std::ostringstream outmsg;
		outmsg << "loadDumpJson,name:%s,parse json error," << fileList[0] << doc.GetErrorOffset() << ", " << doc.GetParseError() << std::endl;
		serverLog(LL_WARNING, outmsg.str().c_str());
		return -1;
	}
	serverLog(LL_NOTICE, "load %s ok\n", fileList[0].c_str());
	//  Package as key-value Data in form 
	std::map<std::string, std::string> key_values;
	for (auto item = doc.MemberBegin(); item != doc.MemberEnd(); ++item) {
		std::string key = item->name.GetString();
		//!! Note that there can be no more objectToString Pack it once , Otherwise, multiple escape characters will appear 
		//std::string value = objectToString(item->value).c_str();
		std::string value = item->value.GetString();
		key_values.insert(std::make_pair(key.c_str(), value));
	}
	return 0;

}
//! \brief  Storage json file   Save and read in binary form 
//! \param strjson 
//! \return 0  success   Not 0 Failure 
int saveDumpJson(std::string &strjson) {
	serverLog(LL_NOTICE, "->saveDumpJson:\n");
	std::ofstream outFile;
	outFile.open(TEMP_FILE_NAME, std::ios::binary);
	if (!outFile.is_open()) {
		outFile.close();
		serverLog(LL_NOTICE, "Error\n");
		return -1;
	}
	outFile << strjson << std::endl;
	outFile.close();
	//  Write a temporary file first , After success, the move operation becomes an official document 
	auto ret = MoveFileExA(TEMP_FILE_NAME.c_str(), (DUMP_DIR_PATH + DUMP_FILE_PRIFIX + getNowDate()).c_str(),
		MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED | MOVEFILE_WRITE_THROUGH);
	if (ret) {
		serverLog(LL_NOTICE, "OK\n");
		return 0;
	}
	return -2;

}

//! \brief  Backup mechanism worker thread 
//! \brief  Perform connection redis And store the data as dump Backup file operation 
//! \param  empty 
//! \return 
DWORD WINAPI SaveWorkerThread(LPVOID lpParam) {
	serverLog(LL_NOTICE, "SaveWorkerThread: ENTER\n");

	//  Create event object 
	gWorkEvent = CreateEvent(NULL, TRUE, FALSE, TEXT("gWorkEvent"));
	//  Wait for the thread start signal 
	WaitForSingleObject(gWorkEvent,INFINITE);
	//  Clear the signal 
	ResetEvent(gWorkEvent);
	//  Load backup data 
	loadDumpJson();
	serverLog(LL_NOTICE, "SaveWorkerThread: In\n");
	while (true) {
		serverLog(LL_NOTICE, "SaveWorkerThread: Wait...\n");
		DWORD waitResult = WaitForSingleObject(
			gWorkEvent, 
			INFINITE);   
		doSaveWork();
		//  Clear the signal 
		ResetEvent(gWorkEvent);
	}
}


//! \brief  Start the worker thread , External call (server.c in )
//! \param  empty 
//! \return 
void StartSaveWorkerThread() {
	serverLog(LL_NOTICE, "SaveWorkerThread: Start\n");
	if (!SetEvent(gWorkEvent)) {
		serverLog(LL_NOTICE, "Start SaveWorkerThread failed (%d)\n", GetLastError());
	}
}