Category Archives: Microsoft

LMDBServiceNet coded in C# .NET

On Monday 3, I will make a demo in my company about Docker/Azure and I have decided to make a demo about C# .NET stuff because few people are interested in C++… So I have rewritten my service in C#. It was not very difficult:

  • Web Server provided by CPP Rest SDK is migrated with System.Net.Http
  • LMDBWindows64.dll is wrapped by LMDBNet.dll with C# bindings
  • C++ syntax is the same as C# syntax so all the code rebuild ok at 90%

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using LMDBNet;
using Newtonsoft.Json;

namespace LMDBServiceNet
{
    class WebServer
    {
        HttpListener _listener;
        LMDBWrapper _wrapper;

        public WebServer(string uriPrefix)
        {
            _listener = new HttpListener();
            _listener.Prefixes.Add(uriPrefix);

            _wrapper = new LMDBWrapper();
            _wrapper.Init("cache_db");
        }
        public async void Start()
        {
            _listener.Start();
            while (true)
                try
                {
                    var context = await _listener.GetContextAsync();
                    Task.Run(() => ProcessRequestAsync(context));
                }
                catch (HttpListenerException) { break; } // Listener stopped.
                catch (InvalidOperationException) { break; } // Listener stopped.
        }
        public void Stop() { _listener.Stop(); }

        async void ProcessRequestAsync(HttpListenerContext context)
        {
            try
            {
                string request = Path.GetFileName(context.Request.RawUrl);
                string str = String.Format("you asked for: {0}", request);
                //Console.WriteLine(str);

                Dictionary<string, string> parameters = ExtractParameters(context);

                if (parameters.Count == 0)
                {
                    await WriteResponse(context, String.Empty, HttpStatusCode.OK);
                    return;
                }

                string verb;
                if ( parameters.TryGetValue("request", out verb) == false )
                {
                    await WriteResponse(context, String.Empty, HttpStatusCode.OK);
                    return;
                }

                if (context.Request.HttpMethod == "GET")
                {
                    if (verb == Constants.VerbPing)
                    {
                        await RequestVerbPing(context);
                        return;
                    }
                    else if (verb == Constants.VerbUsage)
                    {
                        await RequestUsage(context);
                        return;
                    }
                    else if (verb == Constants.VerbGetData)
                    {
                        await RequestVerbGetData(context, parameters);
                        return;
                    }
                    else if (verb == Constants.VerbSetData)
                    {
                        await RequestVerbSetData(context, parameters);
                        return;
                    }
                }

                await WriteResponse(context, String.Empty, HttpStatusCode.OK);
            }
            catch (Exception ex) { Console.WriteLine("Request error: " + ex); }
        }

        private Dictionary<string, string> ExtractParameters(HttpListenerContext context)
        {
            string request = Path.GetFileName(context.Request.RawUrl);
            string str = String.Format("Request: {0}", request);
            //Console.WriteLine(str);

            //http://192.168.175.241:7001/MyServer/LMDB/?request=set-data&key=toto0&value=toto1&name=cache2

            Dictionary<string, string> parameters = new Dictionary<string, string>();

            if (String.IsNullOrEmpty(request))
                return parameters;

            var strings = request.Split('&');

            if (strings == null)
                return parameters;

            if (strings.Length >= 1)
            {
                strings[0] = strings[0].Remove(0, 1);
            }
            else
            {
                return parameters;
            }

            foreach (string s in strings)
            {
                //Console.WriteLine(s);
                var data = s.Split('=');
                if( data.Length == 2)
                {
                    string values = String.Format("{0}:{1}", data[0], data[1]);
                    //Console.WriteLine(values);

                    parameters.Add(data[0], data[1]);

                }
            }

            return parameters;

        }

        private async Task WriteResponse(HttpListenerContext context, string message, HttpStatusCode code)
        {
            Logger.LogInfo(message);

            var enc = Encoding.UTF8;
            byte[] msg = enc.GetBytes(message);

            context.Response.StatusCode = (int)code;

            context.Response.ContentLength64 = msg.Length;
            using (Stream s = context.Response.OutputStream)
                await s.WriteAsync(msg, 0, msg.Length);
        }

        private async Task RequestUsage(HttpListenerContext context)
        {

            DataUsage usage = new DataUsage();
            usage.company = "NEOS-SDI";
            usage.developer = "Christophe Pichaud";
            usage.version = "August 2008 BETA 0.3 .NET C#";

            string str = JsonConvert.SerializeObject(usage);

            await WriteResponse(context, str, HttpStatusCode.OK);
        }

        private async Task RequestVerbPing(HttpListenerContext context)
        {
            DataPing ping = new DataPing();
            ping.ip = "localhost";
            ping.port = Constants.MasterNodePort;
            ping.server = Environment.MachineName;

            string str = JsonConvert.SerializeObject(ping);

            await WriteResponse(context, str, HttpStatusCode.OK);
        }

        private async Task RequestVerbSetData(HttpListenerContext context, Dictionary<string, string> parameters)
        {
            string key;
            string value;

            if (parameters.TryGetValue("key", out key) == false ||
                parameters.TryGetValue("value", out value) == false)
            {
                await WriteResponse(context, String.Empty, HttpStatusCode.OK);
                return;
            }

            _wrapper.SetData(key, value);

            Data data = new Data();
            data.Key = key;
            data.Value = value;

            string str = JsonConvert.SerializeObject(data);
            await WriteResponse(context, str, HttpStatusCode.OK);
        }

        private async Task RequestVerbGetData(HttpListenerContext context, Dictionary<string, string> parameters)
        {
            string key;
            string value;

            if (parameters.TryGetValue("key", out key) == false)
            {
                await WriteResponse(context, String.Empty, HttpStatusCode.OK);
                return;
            }

            _wrapper.GetData(key, out value);

            Data data = new Data();
            data.Key = key;
            data.Value = value;

            string str = JsonConvert.SerializeObject(data);
            await WriteResponse(context, str, HttpStatusCode.OK);
        }
    }
}
Advertisements

A C++ LMDB Wrapper

The simple way to handle buffer is to use std::string or std::wstring. No memory to allocate, it’s simple.

class LMDBWRAPPER_API CLMDBWrapperEx
{
public:
	CLMDBWrapperEx() {}
	~CLMDBWrapperEx()
	{
		mdb_env_close(env);
	}

private:
	MDB_env * env;
	MDB_dbi dbi;
	MDB_val key, data;
	MDB_txn *txn;

public:

	void Init(std::wstring db)
	{
		std::string wdb(db.begin(), db.end());
		Init(wdb);
	}

	void Init(std::string db)
	{
		char sz[255];
		sprintf_s(sz, "%s\\%s", Constants::LMDBRootPath.c_str(), db.c_str());
		::CreateDirectoryA(sz, NULL);

		mdb_env_create(&env);
		mdb_env_set_maxreaders(env, 1);
		mdb_env_set_mapsize(env, 10485760 * 1000);
		mdb_env_open(env, sz, MDB_CREATE, 0);
	}

	void Set(std::wstring k, std::wstring v)
	{
		std::string key(k.begin(), k.end());
		std::string value(v.begin(), v.end());
		Set(key, value);
	}

	void Set(std::string k, std::string v)
	{
		mdb_txn_begin(env, NULL, 0, &txn);
		mdb_dbi_open(txn, NULL, 0, &dbi);

		key.mv_size = k.length() + 1;
		key.mv_data = (void *)k.c_str();
		data.mv_size = v.length() + 1;
		data.mv_data = (void *)v.c_str();
		int err = mdb_put(txn, dbi, &key, &data, 0); // MDB_NOOVERWRITE);
		printf("Set err:%d Key:%s Data:%s\n", err, key.mv_data, data.mv_data);

		mdb_txn_commit(txn);
		mdb_dbi_close(env, dbi);
	}

	bool Get(std::wstring k, std::wstring & v)
	{
		std::string key(k.begin(), k.end());
		std::string value;
		bool ret = Get(key, value);

		std::wstring wvalue(value.begin(), value.end());
		v = wvalue;
		return ret;
	}

	bool Get(std::string k, std::string & value)
	{
		mdb_txn_begin(env, NULL, 0, &txn);
		mdb_dbi_open(txn, NULL, 0, &dbi);

		key.mv_size = k.length() + 1;
		key.mv_data = (void *)k.c_str();

		int err = mdb_get(txn, dbi, &key, &data);
		printf("Get err:%d Key:%s Data:%s\n", err, key.mv_data, data.mv_data);
		value = (char *)(data.mv_data);

		mdb_txn_commit(txn);
		mdb_dbi_close(env, dbi);

		return err == 0 ? true : false;
	}
};

 

Making a POST call with JSON data using CPPREST C++ SDK

The source code here is the same as the C# version in the previous post. The client makes a simple call like that: 

	std::string value2 = "azertyuiopqsdfghjklmwxcvbn";
	std::string buffer = Base64Helper::base64_encode((const unsigned char*)value2.c_str(), value2.length());
	std::wstring value3(buffer.begin(), buffer.end());
	SetData(key, value3, value3.length(), dbname);

 

Here is the wrapper code for SetData. It’s included in HttpLMDB dll: 

bool HTTPLMDB_API SetData(std::wstring key, std::wstring valueb64, DWORD dwLen, std::wstring name)
{
	std::wstring port = Constants::MasterNodePort;
	std::wstring ip = ServerHelper::GetIP();
	std::wstring url = ServerHelper::BuildURL(ip, port);

	std::wstring contentType = _T("Content-Type");
	std::wstring contentTypeV = _T("application/json");
	std::wstring keepAlive = _T("Keep-Alive");
	std::wstring keepAliveV = _T("false");
	std::wstring contentLength = _T("Content-Length");
	
	std::wostringstream bufLen;
	bufLen << contentType.length() + contentTypeV.length() + keepAlive.length() + keepAliveV.length() + contentLength.length() + 4;
	std::wstring len = bufLen.str().c_str();

	std::wostringstream buf;
	buf << url << '/' << Constants::Request << Constants::VerbSetDataB64
		<< _T("&name=") << name;
	url = buf.str().c_str();	
	//{"key":"key_toto0","value":"value_toto0"}
	std::wostringstream bufjson;
	bufjson << "{" << '"' << "key" << '"' << ":" << '"' << key << '"' << ","
		<< '"' << "value" << '"' << ":" << '"' << valueb64 << '"' << '}';
	std::wstring jsonv = bufjson.str().c_str();
	//wcout << _T("jsonv : ") << jsonv << endl;

	http_client client_lmdb(url);
	http_request request(methods::POST);
	request.headers().add(contentType, contentTypeV);
	request.headers().add(keepAlive, keepAliveV);
	request.headers().add(contentLength, len);
	request.set_body(jsonv);

	http_response response;
	response = client_lmdb.request(request).get();

	wcout << response.to_string() << endl;

	return true;
}

 

Here is the code. It’s not very difficult.

Handling a long url using POST verb

To be able to store data in my LMDB Service, I need to store data as base64 items. To do that, I need to transmit json data from the client to the server. It can’t be passed on the url. So I use post handling. Here is the C++ handler:

void TheServer::handle_post(http_request message)

{
       try
       {
             g_Logger.WriteLog(_T("handle_post"));
 
             PrintRequest(message);

             std::wstring request = ServerHelper::FindParameter(message, _T("request"));
            
              if (request == Constants::VerbSetDataB64)
             {
                 RequestVerbSetData64(message);
                    return;
             }
             else if (request == Constants::VerbGetDataB64)
             {
                    // Does not work yet
                    RequestVerbGetData64(message);
                    return;
             }
       }
       catch (...)
       {
             // an internal problem occured
             g_Logger.WriteLog(_T("handle_post exception..."));
       }
 
       message.reply(status_codes::OK);
};

 

The C++ routine here to analyze is RequestVerbSetData64. Here is the code:

void TheServer::RequestVerbSetData64(http_request message)
{
       USES_CONVERSION;
       CLMDBWrapper lmdb;
       g_Logger.WriteLog(Constants::VerbSetDataB64.c_str());
 
       std::wstring dbNameW = ServerHelper::FindParameter(message, _T("name"));

       std::string dbName(dbNameW.begin(), dbNameW.end());
       std::wstring json;
       web::json::value jsonV = message.extract_json().get();

       Data data = Data::FromJSON(jsonV.as_object());
       TCHAR sz[255];
       _stprintf_s(sz, _T("Data key:%s value:..."), data.key.c_str());
       g_Logger.WriteLog(sz);

       if (lmdb.Init((LPSTR)dbName.c_str()) == false)
       {
             g_Logger.WriteLog(_T("LMDB Init not done !"));

            message.reply(status_codes::OK);
             return;
       }

       LPSTR lpszKey = W2A(data.key.c_str());
       LPSTR lpszValue = W2A(data.value.c_str());
       DWORD dwLen = strlen(lpszValue);

       lmdb.SetData(lpszKey, lpszValue, dwLen);

       message.reply(status_codes::OK);

       lmdb.Uninit((LPSTR)dbName.c_str());
}

 

The source code is simple to write, simple to read. Because it is native code, it is fast and we just need to distribute the dll we use. here, it’s just the C runtime, the C++ runtime and the CPPREST dll. This is the advantage of the native stuff, you don’t need to distribute any framework that size is around 350 MB… It’s lightweight, it’s fast, it’s built on the metal.

Running LMDBService WS NoSQL in Azure Container Instance

Now that LMDBService runs in a docker image, it is possible to run it into Azure Container Instance.

Here are the steps to follow:

  • Build the docker image locally with its dockerfile
    • docker image build –tag mydocker/myserver d:\dev\docker
  • Create a Repository in Azure Repository
  • Connect in docker shell to the repo
    • docker login lmdbwsprod.azurecr.io -u lmdbwsprod -p xxxxxxxxxxxxxxxxxxxxxxxxxx
  • Tag the prod repository
    • docker tag mydocker/myserver lmdbwsprod.azurecr.io/prod
  • Push the docker image to the Azure repository
  • Go to Azure portal and choose Azure Repository
    • Choose prod repository
    • Check tags and choose latest
    • Click Run Instance in […] button
    • Open port 7001
    • Click OK
  • Go to Azure Container Instance
    • choose the container and click Overview
    • Retrieve the IP address

Test the URL : http://40.114.209.198:7001/MyServer/LMDB/?request=set-data&key=Key_v99&value=Value_v99&name=cache_NET

 

Running a Windows Service with LMDB as a REST Web API Web Server in a Docker container

To make the test, I use the Microsoft/IIS image available at : https://hub.docker.com/r/microsoft/iis/

This image is managed by Microsoft and regulary updated. Size is around 1.8 GB. To get the image in Docker, run:

  • docker pull microsoft/iis

Then, we to create a Docker file to instruct Docker how to setup the environment :

FROM microsoft/iis
COPY *.* c:/
RUN sc create LMDBService start=auto binpath=”C:\LMDBService.exe”
#RUN net start LMDBService
EXPOSE 80
EXPOSE 7001
RUN md c:\temp

Then, we need to build the image :

  • docker image build –tag mydocker/myserver d:\dev\docker

And now, we can run the container as a deamon:

  • docker run -d -p 7001:7001 -it mydocker/myserver

In order to test the Web Server in a browser, we need to know the IP address of the web server… We need to know the IP of the container… We begin by listing the containers :

  • docker ps –a
  • we take the first item of the list

And we run the following command using the id of the container:

  • docker inspect -f “{{ .NetworkSettings.Networks.nat.IPAddress }}” 543e57f54047
  • response is displayed => 172.26.255.203

Now we can test the web server in the container.

iis

The container runs the windows service who host the web server… Let’s use this url :

web

It works !

 

LMDB – I love that stuff !

Since few months, I work on LMDB for Azure stuff. First, I have:

  • migrated the library from Linux to Windows x64
  • adapted the .NET Layer Interop
  • made custom clients code

Now, I have setup the Azure Web App Svc for hosting a simple file uploader where files will be stored in LMDB NoSQL database. The way of using it is so simple :

        private static void StoreFile(string path)
        {
            string key = path;
            string value = String.Empty;
            byte[] buffer = File.ReadAllBytes(path);

            LMDBEnvironment _env;
            string dir = "c:\\temp\\cache_net10B";
            _env = new LMDBEnvironment(dir);
            _env.MaxDatabases = 2;
            _env.MapSize = 10485760 * 100;
            _env.Open();

            DateTime dtStart = DateTime.Now;
            var tx = _env.BeginTransaction();
            var db = tx.OpenDatabase("DB", new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create });
            var enc = System.Text.Encoding.UTF8;
            tx.Put(db, enc.GetBytes(key), buffer);
            tx.Commit();
            db.Dispose();

            DateTime dtStop = DateTime.Now;
            TimeSpan ts = dtStop - dtStart;
            string str = String.Format("Time elapsed for set:{0} ms", ts.TotalMilliseconds);
            Logger.LogInfo(str);

            _env.Dispose();
        }

 

If you want to test LMDB in your environement, download that stuff:

Download the LMDB Windows DLL and console_app.Exe
Download the LMDB Windows DLL, the .NET WRapper DLL and console_app.Exe