A C++ LMDB Wrapper

The simple way to handle buffer is to use std::string or std::wstring. No memory to allocate, it’s simple.

class LMDBWRAPPER_API CLMDBWrapperEx
{
public:
	CLMDBWrapperEx() {}
	~CLMDBWrapperEx()
	{
		mdb_env_close(env);
	}

private:
	MDB_env * env;
	MDB_dbi dbi;
	MDB_val key, data;
	MDB_txn *txn;

public:

	void Init(std::wstring db)
	{
		std::string wdb(db.begin(), db.end());
		Init(wdb);
	}

	void Init(std::string db)
	{
		char sz[255];
		sprintf_s(sz, "%s\\%s", Constants::LMDBRootPath.c_str(), db.c_str());
		::CreateDirectoryA(sz, NULL);

		mdb_env_create(&env);
		mdb_env_set_maxreaders(env, 1);
		mdb_env_set_mapsize(env, 10485760 * 1000);
		mdb_env_open(env, sz, MDB_CREATE, 0);
	}

	void Set(std::wstring k, std::wstring v)
	{
		std::string key(k.begin(), k.end());
		std::string value(v.begin(), v.end());
		Set(key, value);
	}

	void Set(std::string k, std::string v)
	{
		mdb_txn_begin(env, NULL, 0, &txn);
		mdb_dbi_open(txn, NULL, 0, &dbi);

		key.mv_size = k.length() + 1;
		key.mv_data = (void *)k.c_str();
		data.mv_size = v.length() + 1;
		data.mv_data = (void *)v.c_str();
		int err = mdb_put(txn, dbi, &key, &data, 0); // MDB_NOOVERWRITE);
		printf("Set err:%d Key:%s Data:%s\n", err, key.mv_data, data.mv_data);

		mdb_txn_commit(txn);
		mdb_dbi_close(env, dbi);
	}

	bool Get(std::wstring k, std::wstring & v)
	{
		std::string key(k.begin(), k.end());
		std::string value;
		bool ret = Get(key, value);

		std::wstring wvalue(value.begin(), value.end());
		v = wvalue;
		return ret;
	}

	bool Get(std::string k, std::string & value)
	{
		mdb_txn_begin(env, NULL, 0, &txn);
		mdb_dbi_open(txn, NULL, 0, &dbi);

		key.mv_size = k.length() + 1;
		key.mv_data = (void *)k.c_str();

		int err = mdb_get(txn, dbi, &key, &data);
		printf("Get err:%d Key:%s Data:%s\n", err, key.mv_data, data.mv_data);
		value = (char *)(data.mv_data);

		mdb_txn_commit(txn);
		mdb_dbi_close(env, dbi);

		return err == 0 ? true : false;
	}
};

 

Advertisements

A Windows Service with C++ NoSQL in Azure Container Instance

The case study is a NoSQL WS that offers Read/Write possibilities for multiple databases using a REST API.

The NoSQL technology is OpenLDAP-LMDB library. It was ported from Linux to Windows and exported as a Windows DLL.

A Windows Service exposes a WS hosted by CPP REST SDK as a stand alone Web Server on a dedicated url/port. In my case, port is 7001, url is /MyServer/LMDB/.

The docker container run in Azure Container Instance using a Window Server 2016 1609 image from microsoft/iis:windowsservercore-ltsc2016.

DockerFile is:

The project is a Windows Service x64 (LMDBService.exe) who loads multiple dll :

  • cpprest141d_2_10.dll
  • LMDBWindowsDllD64.dll
  • LMDBWrapperD64.dll
  • MySharedStuffD64.dll
  • Msvcp140d.dll
  • Vcruntime140d.dll
  • Ucrtbased.dll

The Windows service consume 2 MB of memory and PCU is always 0%. It’s the advantage of C++.

The Windows service needs to be run as administrator because cpp rest sdk require admin privileges to create the url on a port. Else ACCESS_DENIED error.

The docker image is built localy (W10) and uploaded in Azure Container Registry.

  • docker image build –tag myserver6 d:devdocker
  • docker login lmdbreg.azurecr.io -u lmdbreg -p XXXXXXXXXXXXXXXXXXXXXXXXXXXX
  • docker tag myserver6 lmdbreg.azurecr.io/prod6
  • docker push lmdbreg.azurecr.io/prod6

Azure container url : http://137.117.141.0:7001/MyServer/LMDB/request?ping

The container is up & running.

There are multiple verbs in HTTP :

  • GET : get-data, set-data, ping
  • POST : get-data-b64, set-data-b64

Anything else route to about.

The container use IIS to see the private log file (LMDB.txt).

Logs : http://137.117.141.0/Logs/lmdb.txt

The container use the port 7001 and writes/reads data from c:temp and c:templogs

http://137.117.141.0:7001/MyServer/LMDB/?request=ping

http://137.117.141.0:7001/MyServer/LMDB/?request=about

Sample GET requests:

INFO – 7:05:41 PM – http://137.117.141.0:7001/MyServer/LMDB/?request=set-data&key=Key_v0&value=Value_v0&name=cache_NET

INFO – 7:05:41 PM – {"key":"Key_v0","value":"Value_v0"}

INFO – 7:05:41 PM – http://137.117.141.0:7001/MyServer/LMDB/?request=get-data&key=Key_v0&name=cache_NET

INFO – 7:05:42 PM – {"key":"Key_v0","value":"Value_v0"}

For POST requests, you need code C# or JS…

Example :

private static void StoreFile(string url, string path)

{

string cache = "cache_NET";

string base_url = String.Format("http://{0}:7001/MyServer/LMDB/?request=set-data-b64&name={1}", url, cache);

var enc = System.Text.Encoding.UTF8;

string key = path;

string value = String.Empty;

byte[] buffer = File.ReadAllBytes(path);

string buffer2 = enc.GetString(buffer);

value = Base64Helper.Base64Encode(buffer2);

MakePostBuffer(base_url, value);

}

private static void MakePostBuffer(string url, string valueb64)

{

HttpWebRequest r = (HttpWebRequest)WebRequest.Create(url);

string c = """;

string a1 = "{";

string a2 = "}";

string key = String.Format("key_{0}", DateTime.Now.Ticks);

string postData = String.Format("{1}{0}key{0}:{0}{4}{0}, {0}value{0}:{0}{3}{0}{2}", c, a1, a2, valueb64, key); // url;

//Logger.LogInfo(postData);

var data = Encoding.ASCII.GetBytes(postData);

r.Method = "POST";

r.ContentType = "application/json;";

r.ContentLength = data.Length;

r.KeepAlive = false;

using (var stream = r.GetRequestStream())

{

stream.Write(data, 0, data.Length);

}

string str = String.Format("Sending {0} bytes on {1}", data.Length, url);

Logger.LogInfo(str);

WebResponse wr = r.GetResponse();

//Logger.LogInfo(wr.ContentType);

Stream s = wr.GetResponseStream();

Encoding encode = System.Text.Encoding.GetEncoding("utf-8");

StreamReader reader = new StreamReader(s, encode);

string buffer = reader.ReadToEnd();

Logger.LogInfo(buffer);

reader.Close();

wr.Close();

}

Christophe Pichaud

Architecture of the Windows Service LMDBService (updated)

Here is the new diagram:

Architecture_LMDBService_bis

Azure Container Instance

To run my container on ACI, I need a specific image :

FROM microsoft/iis:windowsservercore-ltsc2016

If not, it does not work. The error will be the OS version is not supported.

Envoyé de mon téléphone Windows 10

Windows API vs ISO C++

It’s been a while I am writing softwares for Windows using Win32 API and C runtime. But since some few years, I use ISO C++ and STL features more and more.

My best friends are :

  • std::wstring for string management
  • std::vector for container
  • std::shared_ptr for memory management
  • std::wostringstream for stream

For sure, there is a little overhead compared to C runtime routines but life is easy.

Envoyé de mon téléphone Windows 10

Making a POST call with JSON data using CPPREST C++ SDK

The source code here is the same as the C# version in the previous post. The client makes a simple call like that: 

	std::string value2 = "azertyuiopqsdfghjklmwxcvbn";
	std::string buffer = Base64Helper::base64_encode((const unsigned char*)value2.c_str(), value2.length());
	std::wstring value3(buffer.begin(), buffer.end());
	SetData(key, value3, value3.length(), dbname);

 

Here is the wrapper code for SetData. It’s included in HttpLMDB dll: 

bool HTTPLMDB_API SetData(std::wstring key, std::wstring valueb64, DWORD dwLen, std::wstring name)
{
	std::wstring port = Constants::MasterNodePort;
	std::wstring ip = ServerHelper::GetIP();
	std::wstring url = ServerHelper::BuildURL(ip, port);

	std::wstring contentType = _T("Content-Type");
	std::wstring contentTypeV = _T("application/json");
	std::wstring keepAlive = _T("Keep-Alive");
	std::wstring keepAliveV = _T("false");
	std::wstring contentLength = _T("Content-Length");
	
	std::wostringstream bufLen;
	bufLen << contentType.length() + contentTypeV.length() + keepAlive.length() + keepAliveV.length() + contentLength.length() + 4;
	std::wstring len = bufLen.str().c_str();

	std::wostringstream buf;
	buf << url << '/' << Constants::Request << Constants::VerbSetDataB64
		<< _T("&name=") << name;
	url = buf.str().c_str();	
	//{"key":"key_toto0","value":"value_toto0"}
	std::wostringstream bufjson;
	bufjson << "{" << '"' << "key" << '"' << ":" << '"' << key << '"' << ","
		<< '"' << "value" << '"' << ":" << '"' << valueb64 << '"' << '}';
	std::wstring jsonv = bufjson.str().c_str();
	//wcout << _T("jsonv : ") << jsonv << endl;

	http_client client_lmdb(url);
	http_request request(methods::POST);
	request.headers().add(contentType, contentTypeV);
	request.headers().add(keepAlive, keepAliveV);
	request.headers().add(contentLength, len);
	request.set_body(jsonv);

	http_response response;
	response = client_lmdb.request(request).get();

	wcout << response.to_string() << endl;

	return true;
}

 

Here is the code. It’s not very difficult.

Handling a long url using POST verb

To be able to store data in my LMDB Service, I need to store data as base64 items. To do that, I need to transmit json data from the client to the server. It can’t be passed on the url. So I use post handling. Here is the C++ handler:

void TheServer::handle_post(http_request message)

{
       try
       {
             g_Logger.WriteLog(_T("handle_post"));
 
             PrintRequest(message);

             std::wstring request = ServerHelper::FindParameter(message, _T("request"));
            
              if (request == Constants::VerbSetDataB64)
             {
                 RequestVerbSetData64(message);
                    return;
             }
             else if (request == Constants::VerbGetDataB64)
             {
                    // Does not work yet
                    RequestVerbGetData64(message);
                    return;
             }
       }
       catch (...)
       {
             // an internal problem occured
             g_Logger.WriteLog(_T("handle_post exception..."));
       }
 
       message.reply(status_codes::OK);
};

 

The C++ routine here to analyze is RequestVerbSetData64. Here is the code:

void TheServer::RequestVerbSetData64(http_request message)
{
       USES_CONVERSION;
       CLMDBWrapper lmdb;
       g_Logger.WriteLog(Constants::VerbSetDataB64.c_str());
 
       std::wstring dbNameW = ServerHelper::FindParameter(message, _T("name"));

       std::string dbName(dbNameW.begin(), dbNameW.end());
       std::wstring json;
       web::json::value jsonV = message.extract_json().get();

       Data data = Data::FromJSON(jsonV.as_object());
       TCHAR sz[255];
       _stprintf_s(sz, _T("Data key:%s value:..."), data.key.c_str());
       g_Logger.WriteLog(sz);

       if (lmdb.Init((LPSTR)dbName.c_str()) == false)
       {
             g_Logger.WriteLog(_T("LMDB Init not done !"));

            message.reply(status_codes::OK);
             return;
       }

       LPSTR lpszKey = W2A(data.key.c_str());
       LPSTR lpszValue = W2A(data.value.c_str());
       DWORD dwLen = strlen(lpszValue);

       lmdb.SetData(lpszKey, lpszValue, dwLen);

       message.reply(status_codes::OK);

       lmdb.Uninit((LPSTR)dbName.c_str());
}

 

The source code is simple to write, simple to read. Because it is native code, it is fast and we just need to distribute the dll we use. here, it’s just the C runtime, the C++ runtime and the CPPREST dll. This is the advantage of the native stuff, you don’t need to distribute any framework that size is around 350 MB… It’s lightweight, it’s fast, it’s built on the metal.