LMDBServiceNet coded in C# .NET

On Monday 3, I will make a demo in my company about Docker/Azure and I have decided to make a demo about C# .NET stuff because few people are interested in C++… So I have rewritten my service in C#. It was not very difficult:

• Web Server provided by CPP Rest SDK is migrated with System.Net.Http
• LMDBWindows64.dll is wrapped by LMDBNet.dll with C# bindings
• C++ syntax is the same as C# syntax so all the code rebuild ok at 90%

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using LMDBNet;
using Newtonsoft.Json;

namespace LMDBServiceNet
{
    class WebServer
    {
        HttpListener _listener;
        LMDBWrapper _wrapper;

        public WebServer(string uriPrefix)
        {
            _listener = new HttpListener();
            _listener.Prefixes.Add(uriPrefix);

            _wrapper = new LMDBWrapper();
            _wrapper.Init("cache_db");
        }
        public async void Start()
        {
            _listener.Start();
            while (true)
                try
                {
                    var context = await _listener.GetContextAsync();
                    Task.Run(() => ProcessRequestAsync(context));
                }
                catch (HttpListenerException) { break; } // Listener stopped.
                catch (InvalidOperationException) { break; } // Listener stopped.
        }
        public void Stop() { _listener.Stop(); }

        async void ProcessRequestAsync(HttpListenerContext context)
        {
            try
            {
                string request = Path.GetFileName(context.Request.RawUrl);
                string str = String.Format("you asked for: {0}", request);
                //Console.WriteLine(str);

                Dictionary<string, string> parameters = ExtractParameters(context);

                if (parameters.Count == 0)
                {
                    await WriteResponse(context, String.Empty, HttpStatusCode.OK);
                    return;
                }

                string verb;
                if ( parameters.TryGetValue("request", out verb) == false )
                {
                    await WriteResponse(context, String.Empty, HttpStatusCode.OK);
                    return;
                }

                if (context.Request.HttpMethod == "GET")
                {
                    if (verb == Constants.VerbPing)
                    {
                        await RequestVerbPing(context);
                        return;
                    }
                    else if (verb == Constants.VerbUsage)
                    {
                        await RequestUsage(context);
                        return;
                    }
                    else if (verb == Constants.VerbGetData)
                    {
                        await RequestVerbGetData(context, parameters);
                        return;
                    }
                    else if (verb == Constants.VerbSetData)
                    {
                        await RequestVerbSetData(context, parameters);
                        return;
                    }
                }

                await WriteResponse(context, String.Empty, HttpStatusCode.OK);
            }
            catch (Exception ex) { Console.WriteLine("Request error: " + ex); }
        }

        private Dictionary<string, string> ExtractParameters(HttpListenerContext context)
        {
            string request = Path.GetFileName(context.Request.RawUrl);
            string str = String.Format("Request: {0}", request);
            //Console.WriteLine(str);

            //http://192.168.175.241:7001/MyServer/LMDB/?request=set-data&key=toto0&value=toto1&name=cache2

            Dictionary<string, string> parameters = new Dictionary<string, string>();

            if (String.IsNullOrEmpty(request))
                return parameters;

            var strings = request.Split('&');

            if (strings == null)
                return parameters;

            if (strings.Length >= 1)
            {
                strings[0] = strings[0].Remove(0, 1);
            }
            else
            {
                return parameters;
            }

            foreach (string s in strings)
            {
                //Console.WriteLine(s);
                var data = s.Split('=');
                if( data.Length == 2)
                {
                    string values = String.Format("{0}:{1}", data[0], data[1]);
                    //Console.WriteLine(values);

                    parameters.Add(data[0], data[1]);

                }
            }

            return parameters;

        }

        private async Task WriteResponse(HttpListenerContext context, string message, HttpStatusCode code)
        {
            Logger.LogInfo(message);

            var enc = Encoding.UTF8;
            byte[] msg = enc.GetBytes(message);

            context.Response.StatusCode = (int)code;

            context.Response.ContentLength64 = msg.Length;
            using (Stream s = context.Response.OutputStream)
                await s.WriteAsync(msg, 0, msg.Length);
        }

        private async Task RequestUsage(HttpListenerContext context)
        {

            DataUsage usage = new DataUsage();
            usage.company = "NEOS-SDI";
            usage.developer = "Christophe Pichaud";
            usage.version = "August 2008 BETA 0.3 .NET C#";

            string str = JsonConvert.SerializeObject(usage);

            await WriteResponse(context, str, HttpStatusCode.OK);
        }

        private async Task RequestVerbPing(HttpListenerContext context)
        {
            DataPing ping = new DataPing();
            ping.ip = "localhost";
            ping.port = Constants.MasterNodePort;
            ping.server = Environment.MachineName;

            string str = JsonConvert.SerializeObject(ping);

            await WriteResponse(context, str, HttpStatusCode.OK);
        }

        private async Task RequestVerbSetData(HttpListenerContext context, Dictionary<string, string> parameters)
        {
            string key;
            string value;

            if (parameters.TryGetValue("key", out key) == false ||
                parameters.TryGetValue("value", out value) == false)
            {
                await WriteResponse(context, String.Empty, HttpStatusCode.OK);
                return;
            }

            _wrapper.SetData(key, value);

            Data data = new Data();
            data.Key = key;
            data.Value = value;

            string str = JsonConvert.SerializeObject(data);
            await WriteResponse(context, str, HttpStatusCode.OK);
        }

        private async Task RequestVerbGetData(HttpListenerContext context, Dictionary<string, string> parameters)
        {
            string key;
            string value;

            if (parameters.TryGetValue("key", out key) == false)
            {
                await WriteResponse(context, String.Empty, HttpStatusCode.OK);
                return;
            }

            _wrapper.GetData(key, out value);

            Data data = new Data();
            data.Key = key;
            data.Value = value;

            string str = JsonConvert.SerializeObject(data);
            await WriteResponse(context, str, HttpStatusCode.OK);
        }
    }
}

A C++ LMDB Wrapper

The simple way to handle buffer is to use std::string or std::wstring. No memory to allocate, it’s simple.

class LMDBWRAPPER_API CLMDBWrapperEx
{
public:
	CLMDBWrapperEx() {}
	~CLMDBWrapperEx()
	{
		mdb_env_close(env);
	}

private:
	MDB_env * env;
	MDB_dbi dbi;
	MDB_val key, data;
	MDB_txn *txn;

public:

	void Init(std::wstring db)
	{
		std::string wdb(db.begin(), db.end());
		Init(wdb);
	}

	void Init(std::string db)
	{
		char sz[255];
		sprintf_s(sz, "%s\\%s", Constants::LMDBRootPath.c_str(), db.c_str());
		::CreateDirectoryA(sz, NULL);

		mdb_env_create(&env);
		mdb_env_set_maxreaders(env, 1);
		mdb_env_set_mapsize(env, 10485760 * 1000);
		mdb_env_open(env, sz, MDB_CREATE, 0);
	}

	void Set(std::wstring k, std::wstring v)
	{
		std::string key(k.begin(), k.end());
		std::string value(v.begin(), v.end());
		Set(key, value);
	}

	void Set(std::string k, std::string v)
	{
		mdb_txn_begin(env, NULL, 0, &txn);
		mdb_dbi_open(txn, NULL, 0, &dbi);

		key.mv_size = k.length() + 1;
		key.mv_data = (void *)k.c_str();
		data.mv_size = v.length() + 1;
		data.mv_data = (void *)v.c_str();
		int err = mdb_put(txn, dbi, &key, &data, 0); // MDB_NOOVERWRITE);
		printf("Set err:%d Key:%s Data:%s\n", err, key.mv_data, data.mv_data);

		mdb_txn_commit(txn);
		mdb_dbi_close(env, dbi);
	}

	bool Get(std::wstring k, std::wstring & v)
	{
		std::string key(k.begin(), k.end());
		std::string value;
		bool ret = Get(key, value);

		std::wstring wvalue(value.begin(), value.end());
		v = wvalue;
		return ret;
	}

	bool Get(std::string k, std::string & value)
	{
		mdb_txn_begin(env, NULL, 0, &txn);
		mdb_dbi_open(txn, NULL, 0, &dbi);

		key.mv_size = k.length() + 1;
		key.mv_data = (void *)k.c_str();

		int err = mdb_get(txn, dbi, &key, &data);
		printf("Get err:%d Key:%s Data:%s\n", err, key.mv_data, data.mv_data);
		value = (char *)(data.mv_data);

		mdb_txn_commit(txn);
		mdb_dbi_close(env, dbi);

		return err == 0 ? true : false;
	}
};

 

A Windows Service with C++ NoSQL in Azure Container Instance

The case study is a NoSQL WS that offers Read/Write possibilities for multiple databases using a REST API.

The NoSQL technology is OpenLDAP-LMDB library. It was ported from Linux to Windows and exported as a Windows DLL.

A Windows Service exposes a WS hosted by CPP REST SDK as a stand alone Web Server on a dedicated url/port. In my case, port is 7001, url is /MyServer/LMDB/.

The docker container run in Azure Container Instance using a Window Server 2016 1609 image from microsoft/iis:windowsservercore-ltsc2016.

DockerFile is:

The project is a Windows Service x64 (LMDBService.exe) who loads multiple dll :

• cpprest141d_2_10.dll
• LMDBWindowsDllD64.dll
• LMDBWrapperD64.dll
• MySharedStuffD64.dll
• Msvcp140d.dll
• Vcruntime140d.dll
• Ucrtbased.dll

The Windows service consume 2 MB of memory and PCU is always 0%. It’s the advantage of C++.

The Windows service needs to be run as administrator because cpp rest sdk require admin privileges to create the url on a port. Else ACCESS_DENIED error.

The docker image is built localy (W10) and uploaded in Azure Container Registry.

• docker image build –tag myserver6 d:devdocker
• docker login lmdbreg.azurecr.io -u lmdbreg -p XXXXXXXXXXXXXXXXXXXXXXXXXXXX
• docker tag myserver6 lmdbreg.azurecr.io/prod6
• docker push lmdbreg.azurecr.io/prod6

Azure container url : http://137.117.141.0:7001/MyServer/LMDB/request?ping

The container is up & running.

There are multiple verbs in HTTP :

• GET : get-data, set-data, ping
• POST : get-data-b64, set-data-b64

Anything else route to about.

The container use IIS to see the private log file (LMDB.txt).

Logs : http://137.117.141.0/Logs/lmdb.txt

The container use the port 7001 and writes/reads data from c:temp and c:templogs

http://137.117.141.0:7001/MyServer/LMDB/?request=ping

http://137.117.141.0:7001/MyServer/LMDB/?request=about

Sample GET requests:

INFO – 7:05:41 PM – http://137.117.141.0:7001/MyServer/LMDB/?request=set-data&key=Key_v0&value=Value_v0&name=cache_NET

INFO – 7:05:41 PM – {"key":"Key_v0","value":"Value_v0"}

INFO – 7:05:41 PM – http://137.117.141.0:7001/MyServer/LMDB/?request=get-data&key=Key_v0&name=cache_NET

INFO – 7:05:42 PM – {"key":"Key_v0","value":"Value_v0"}

For POST requests, you need code C# or JS…

Example :

private static void StoreFile(string url, string path)

{

string cache = "cache_NET";

string base_url = String.Format("http://{0}:7001/MyServer/LMDB/?request=set-data-b64&name={1}", url, cache);

var enc = System.Text.Encoding.UTF8;

string key = path;

string value = String.Empty;

byte[] buffer = File.ReadAllBytes(path);

string buffer2 = enc.GetString(buffer);

value = Base64Helper.Base64Encode(buffer2);

MakePostBuffer(base_url, value);

}

private static void MakePostBuffer(string url, string valueb64)

{

HttpWebRequest r = (HttpWebRequest)WebRequest.Create(url);

string c = """;

string a1 = "{";

string a2 = "}";

string key = String.Format("key_{0}", DateTime.Now.Ticks);

string postData = String.Format("{1}{0}key{0}:{0}{4}{0}, {0}value{0}:{0}{3}{0}{2}", c, a1, a2, valueb64, key); // url;

//Logger.LogInfo(postData);

var data = Encoding.ASCII.GetBytes(postData);

r.Method = "POST";

r.ContentType = "application/json;";

r.ContentLength = data.Length;

r.KeepAlive = false;

using (var stream = r.GetRequestStream())

{

stream.Write(data, 0, data.Length);

}

string str = String.Format("Sending {0} bytes on {1}", data.Length, url);

Logger.LogInfo(str);

WebResponse wr = r.GetResponse();

//Logger.LogInfo(wr.ContentType);

Stream s = wr.GetResponseStream();

Encoding encode = System.Text.Encoding.GetEncoding("utf-8");

StreamReader reader = new StreamReader(s, encode);

string buffer = reader.ReadToEnd();

Logger.LogInfo(buffer);

reader.Close();

wr.Close();

}

Christophe Pichaud