Use OpenSSL with Asynchronous Sockets, I/O Completion Ports and Ceritificate Signing

[Update 2016/12/28] A client and server sample program with source code has been added and can be downloaded from here.

Using OpenSSL with blocking sockets on Windows is simple and straightforward, you can add OpenSSL to existing project without much effort, http://h71000.www7.hp.com/doc/83final/ba554_90007/ch04s03.html is one of the most detailed and complete guides. It became a complete different story when you need to apply OpenSSL with asynchronous sockets on Windows, and it is hard to find a complete guide on the web about how to do so. This article intends to demonstrate how to use OpenSSL to implement a SSL client or server with I/O completion ports, it will also talk about few frequently asked questions about digital certificate at the end.

I/O completion ports
I/O completion ports are the best choice on Windows to implement high performance network applications. They use overlapped I/O and allow applications to handle results efficiently with minimized thread context switch. The code below gives a simple but effective implementation of using I/O completion ports.

void session_on_completed_packets(DWORD dwNumberOfBytesTransferred, ULONG_PTR lpCompletionKey, LPOVERLAPPED pOverlapped);

struct iocp_info
{
	HANDLE h_iocp;
	size_t threads_count;
	HANDLE *h_threads;
}iocp;

void iocp_on_completed_packets(DWORD dwNumberOfBytesTransferred, ULONG_PTR lpCompletionKey, LPOVERLAPPED pOverlapped)
{
	session_on_completed_packets(dwNumberOfBytesTransferred, lpCompletionKey, pOverlapped);
}

unsigned __stdcall iocp_proc(void *p)
{
	iocp_info *iocp = (iocp_info*)p;
	while(true)
	{
		DWORD dwNumberOfBytesTransferred = 0;
		ULONG_PTR lpCompletionKey = 0;
		LPOVERLAPPED pOverlapped = NULL;
		GetQueuedCompletionStatus(iocp->h_iocp, &dwNumberOfBytesTransferred, &lpCompletionKey, &pOverlapped, INFINITE);
		if(NULL == lpCompletionKey && NULL == pOverlapped)
		{
			PostQueuedCompletionStatus(iocp->h_iocp, 0, 0, 0);
			break;
		}
		iocp_on_completed_packets(dwNumberOfBytesTransferred, lpCompletionKey, pOverlapped);
	}
	return 0;
}

void iocp_start(iocp_info *iocp)
{
	iocp->h_iocp = CreateIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 0);
	SYSTEM_INFO info = {0};
	GetNativeSystemInfo(&info);
	iocp->threads_count = info.dwNumberOfProcessors * 2;
	iocp->h_threads = (HANDLE*)malloc(sizeof(HANDLE) * iocp->threads_count);
	for(size_t n = 0; n < iocp->threads_count; ++n)
		iocp->h_threads[n] = (HANDLE)_beginthreadex(0, 0, iocp_proc, iocp, 0, 0);
}

void iocp_stop(iocp_info *iocp)
{
	PostQueuedCompletionStatus(iocp->h_iocp, 0, 0, 0);
	WaitForMultipleObjects(iocp->threads_count, iocp->h_threads, TRUE, INFINITE);
	for(size_t n = 0; n < iocp->threads_count; ++n)
		CloseHandle(iocp->h_threads[n]);
	CloseHandle(iocp->h_iocp);
}

void iocp_associate_handle(HANDLE h)
{
	CreateIoCompletionPort(h, iocp.h_iocp, 0, 0);
}

OpenSSL initialization
You need to call few functions to initialize OpenSSL library, some crypto functions also requires OS-specific locking mechanism be to make them to be multiple-thread safe.

#include "openssl/ssl.h"
#include "openssl/err.h"
#include "openssl/x509v3.h"

#pragma comment (lib, "libeay32.lib")
#pragma comment (lib, "ssleay32.lib")

typedef CRITICAL_SECTION	ssl_lock;

struct CRYPTO_dynlock_value {
	ssl_lock lock;
};

int number_of_locks = 0;
ssl_lock *ssl_locks = nullptr;
SSL_CTX *ssl_ctx = nullptr;
ssl_lock lock_connect_ex;
LPFN_CONNECTEX pfn_ConnectEx = nullptr;

void ssl_lock_callback(int mode, int n, const char *file, int line)
{
	if(mode & CRYPTO_LOCK)
		EnterCriticalSection(&ssl_locks[n]);
	else
		LeaveCriticalSection(&ssl_locks[n]);
}

CRYPTO_dynlock_value* ssl_lock_dyn_create_callback(const char *file, int line)
{
	CRYPTO_dynlock_value *l = (CRYPTO_dynlock_value*)malloc(sizeof(CRYPTO_dynlock_value));
	InitializeCriticalSection(&l->lock);
	return l;
}

void ssl_lock_dyn_callback(int mode, CRYPTO_dynlock_value* l, const char *file, int line)
{
	if(mode & CRYPTO_LOCK)
		EnterCriticalSection(&l->lock);
	else
		LeaveCriticalSection(&l->lock);
}

void ssl_lock_dyn_destroy_callback(CRYPTO_dynlock_value* l, const char *file, int line)
{
	DeleteCriticalSection(&l->lock);
	free(l);
}

void ssl_init()
{
	WSADATA wsaData = {0};
	WSAStartup(MAKEWORD(2, 2), &wsaData);

	iocp_start(&iocp);

	number_of_locks = CRYPTO_num_locks();
	if(number_of_locks > 0)
	{
		ssl_locks = (ssl_lock*)malloc(number_of_locks * sizeof(ssl_lock));
		for(int n = 0; n < number_of_locks; ++n)
			InitializeCriticalSection(&ssl_locks[n]);
	}

#ifdef _DEBUG
    CRYPTO_malloc_debug_init();
    CRYPTO_dbg_set_options(V_CRYPTO_MDEBUG_ALL);
    CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
#endif
	
	CRYPTO_set_locking_callback(&ssl_lock_callback);
    CRYPTO_set_dynlock_create_callback(&ssl_lock_dyn_create_callback);
	CRYPTO_set_dynlock_lock_callback(&ssl_lock_dyn_callback);
    CRYPTO_set_dynlock_destroy_callback(&ssl_lock_dyn_destroy_callback);

    SSL_load_error_strings();
    SSL_library_init();

	const SSL_METHOD* meth = SSLv23_method();
	ssl_ctx = SSL_CTX_new(meth);
	SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_NONE, nullptr);

	InitializeCriticalSection(&lock_connect_ex);
}

void ssl_set_ctx_cert_and_key(X509 *cert, EVP_PKEY *pkey)
{
	SSL_CTX_use_certificate(ssl_ctx, cert);
	SSL_CTX_use_PrivateKey(ssl_ctx, pkey);
}

void ssl_deinit()
{
	SSL_CTX_free(ssl_ctx);
	ssl_ctx = nullptr;

	CRYPTO_set_locking_callback(NULL);
	CRYPTO_set_dynlock_create_callback(NULL);
	CRYPTO_set_dynlock_lock_callback(NULL);
	CRYPTO_set_dynlock_destroy_callback(NULL);

    EVP_cleanup();
    CRYPTO_cleanup_all_ex_data();
    ERR_remove_state(0);
    ERR_free_strings();

	if(nullptr != ssl_locks)
	{
		for(int n = 0; n < number_of_locks; ++n)
			DeleteCriticalSection(&ssl_locks[n]);
 		
		free(ssl_locks);
		ssl_locks = nullptr;
		number_of_locks = 0;
	}

	iocp_stop(&iocp);
	DeleteCriticalSection(&lock_connect_ex);
	WSACleanup();
}

Some OpenSSL APIs (especially SSL_read and SSL_write) might set internal error code to WANT_READ or WANT_WRITE. These are not fatal errors but rather than flags telling us that they need to read or write more data in order to continue. The helper function ssl_is_fatal_error is used for distinguish these scenarios with real errors, another function ssl_get_error shows how to get pending errors from a SSL structure. These two functions will be used in sample code showed later.

static bool ssl_is_fatal_error(int ssl_error)
{
	switch(ssl_error)
	{
		case SSL_ERROR_NONE:
		case SSL_ERROR_WANT_READ:
		case SSL_ERROR_WANT_WRITE:
		case SSL_ERROR_WANT_CONNECT:
		case SSL_ERROR_WANT_ACCEPT:
			return false;
	}
	return true;
}

static int ssl_get_error(SSL *ssl, int result)
{
	int error = SSL_get_error(ssl, result);
	if(SSL_ERROR_NONE != error)
	{
		char message[512] = {0};
		int error_log = error;
		while(SSL_ERROR_NONE != error_log)
		{
			ERR_error_string_n(error_log, message, _countof(message));
			if(ssl_is_fatal_error(error_log))
			{
				// print error message to console or logs
			}
			error_log = ERR_get_error();
		}
	}
	return error;
}

Data Structure
We need to define a data structure to hold all the related information together, which include the handle to socket, socket buffer, overlapped structure, WASBUF structure, OpenSSL structures, socket buffers and SSL buffers etc. Each data member is commented with its purpose.

enum OVERLAPPED_TYPE{
	RECV = 0,
	SEND,
	CONNECT
};

enum ADDRESS_TYPE{
	LOCAL = 0,
	REMOTE
};

enum SOCKET_STATUS{
	NONE		= 0x0,
	ACCEPTING	= 0x1,
	CONNECTING	= 0x2,
	HANDSHAKING	= 0x4,
	CONNECTED	= 0x8,
	RECEIVING	= 0x10,
	SENDING		= 0x20,
	CLOSING		= 0x40,
	CLOSED		= 0x80,
	OPERATING	= ACCEPTING | CONNECTING | HANDSHAKING | RECEIVING | SENDING
};

#define ADDR_SZ_SIZE	46
#define BUFFER_SIZE		1024

struct session;

struct session_overlapped
{
	OVERLAPPED overlapped;
	DWORD result;
	session *psession;
};

struct session
{
	SOCKET s; // handle to socket
	SOCKET s_listening; // handle to external listening socket
	sockaddr_storage addresses[2]; // local and remote address
	char addresses_sz[2][ADDR_SZ_SIZE]; // addresses in string format
	char socket_buffer[2][BUFFER_SIZE]; // memory used for read/write from/to socket
	char ssl_buffer[2][BUFFER_SIZE]; // memory used for read/write from/to ssl memory bio
	DWORD ssl_buffer_size[2]; // indicates the bytes of valid data in ssl_buffer
	unsigned int status; // stores current socket status, bit-masked value of one or more of SOCKET_STATUS
	session_overlapped overlapped[3]; // structure for overlapped operations
	WSABUF wsabuf[2]; // structure used for pass buffer to overlapped operations
	DWORD bytes_transferred[2]; // store the bytes of buffer that received/sent from/to the socket
	DWORD wsa_flags[2]; // store the flags send/receive from overlapped operations, not used
	SSL *ssl; // SSL structure used by OpenSSL
	BIO *bio[2]; // memory BIO used by OpenSSL
	ssl_lock lock; // synchronization object for multiple-thread data access
	void *pdata; // user supplied contextual data, not used by openssl processing
};

session_init is used to initialize the data structure, session_close is used to close the socket and free resources, it is normally called when peer socket closed its connection or serious error occurred. One thing to note here is that since we are doing asynchronous I/O, it might take multiple calls of session_close to same object to really close the session, when the status member has value of CLOSED, the pointer to session can now be deleted.

session* session_init(session *psession, void *pdata)
{
	memset(psession, 0, sizeof(session));
	psession->pdata = pdata;
	InitializeCriticalSection(&psession->lock);

	psession->overlapped[CONNECT].psession = psession->overlapped[RECV].psession = psession->overlapped[SEND].psession = psession;
	psession->wsabuf[RECV].buf = psession->socket_buffer[RECV];
	psession->wsabuf[RECV].len = BUFFER_SIZE;
	psession->wsabuf[SEND].buf = psession->socket_buffer[SEND];
	psession->wsabuf[SEND].len = 0;
	psession->ssl = SSL_new(ssl_ctx);
	psession->bio[SEND] = BIO_new(BIO_s_mem());
	psession->bio[RECV] = BIO_new(BIO_s_mem());
	SSL_set_bio(psession->ssl, psession->bio[RECV], psession->bio[SEND]);

	return psession;
}

session* session_new(void *pdata)
{
	session *psession = (session*)malloc(sizeof(session));
	return session_init(psession, pdata);
}

void session_delete(session *psession)
{
	DeleteCriticalSection(&psession->lock);
	free(psession);
}

void session_lock(session *psession)
{
	EnterCriticalSection(&psession->lock);
}

void session_unlock(session *psession)
{
	LeaveCriticalSection(&psession->lock);
}

bool session_is_data_pending(session *psession)
{
	if(psession->bytes_transferred[RECV] > 0)
		return true;

	if(SENDING == (psession->status & SENDING))
		return true;

	if(nullptr != psession->ssl)
	{
		if(BIO_pending(psession->bio[SEND]) > 0 ||	BIO_pending(psession->bio[RECV]) > 0)
			return true;
	}

	return false;
}

void session_close(session *psession)
{
	psession->status |= CLOSING;
	if( INVALID_SOCKET != psession->s && !session_is_data_pending(psession))
	{
		shutdown(psession->s, SD_BOTH);
		closesocket(psession->s);
		psession->s = INVALID_SOCKET;
	}

	if(INVALID_SOCKET == psession->s && 0 == (psession->status & OPERATING))
	{
		if(nullptr != psession->ssl)
		{
			SSL_shutdown(psession->ssl);
			SSL_free(psession->ssl);
			psession->ssl = nullptr;
			psession->bio[SEND] = psession->bio[RECV] = nullptr;
		}
		psession->status = CLOSED;
	}
}

connect, accept, send and receive
These functions shows basic overlapped socket operations. They call ConnectEx, AcceptEx, WSASend and WSARecv respectively with pointer to appropriate overlapped structure. After the operation finished, result will be delivered to iocp and picked up by an arbitrary thread we created. The iocp completion routine will call session_on_completed_packets where we will check the result and process the data received.

void session_connect(session *psession, const sockaddr_storage *remote_addr)
{
	memcpy_s(&psession->addresses[REMOTE], sizeof(sockaddr_storage), remote_addr, sizeof(sockaddr_storage));
	sockaddr_to_string(&psession->addresses[REMOTE], psession->addresses_sz[REMOTE]);

	psession->s = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);

	setsockopt(psession->s, SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, NULL, 0);
	iocp_associate_handle((HANDLE)psession->s);

	addrinfo hints = {0};
	hints.ai_family = remote_addr->ss_family;
	hints.ai_protocol = IPPROTO_TCP;
	hints.ai_socktype = SOCK_STREAM;
	hints.ai_flags = AI_PASSIVE;

	addrinfo* paddrinfo = nullptr;
	getaddrinfo(0, "", &hints, &paddrinfo);
	if(NULL != paddrinfo)
	{
		bind(psession->s, paddrinfo->ai_addr, paddrinfo->ai_addrlen);
		freeaddrinfo(paddrinfo);
		paddrinfo = nullptr;
	}

	EnterCriticalSection(&lock_connect_ex);
	if(nullptr == pfn_ConnectEx)
	{
		DWORD dwRetBytes = 0;
		GUID guid = WSAID_CONNECTEX;
		WSAIoctl(psession->s, SIO_GET_EXTENSION_FUNCTION_POINTER, (void*)&guid, sizeof(guid), (void*)&pfn_ConnectEx, sizeof(pfn_ConnectEx), &dwRetBytes, NULL, NULL);
	}
	LeaveCriticalSection(&lock_connect_ex);

	psession->status |= CONNECTING;
	(*pfn_ConnectEx)(psession->s, (sockaddr*)remote_addr, sizeof(sockaddr_storage), 0, 0, 0, &psession->overlapped[CONNECT].overlapped);
	psession->overlapped[CONNECT].result = WSAGetLastError();
	if(0 != psession->overlapped[CONNECT].result && WSA_IO_PENDING != psession->overlapped[CONNECT].result)
	{
		psession->status &= ~CONNECTING;
		session_close(psession);
	}
}

void session_accept(session *psession)
{
	psession->s = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
	iocp_associate_handle((HANDLE)psession->s);

	psession->status |= ACCEPTING;
	BOOL accepted = AcceptEx(psession->s_listening, psession->s, psession->addresses, 0, sizeof(sockaddr_storage), sizeof(sockaddr_storage), 0, &psession->overlapped[CONNECT].overlapped);
	psession->overlapped[CONNECT].result = WSAGetLastError();
	if(WSA_IO_PENDING != psession->overlapped[CONNECT].result)
	{
		psession->status = NONE;
		session_close(psession);
	}
}

void session_send(session *psession)
{
	if((SENDING != (psession->status & SENDING)) && (psession->wsabuf[SEND].len > 0))
	{
		psession->status |= SENDING;
		psession->bytes_transferred[SEND] = 0;
		WSASend(psession->s, &psession->wsabuf[SEND], 1, &psession->bytes_transferred[SEND], 0, &psession->overlapped[SEND].overlapped, 0);
		psession->overlapped[SEND].result = WSAGetLastError();
		if(0 != psession->overlapped[SEND].result && WSA_IO_PENDING != psession->overlapped[SEND].result)
		{
			psession->status &= ~SENDING;
			session_close(psession);
		}
	}
}

void session_recv(session *psession)
{
	if((CLOSING != (psession->status & CLOSING)) && (RECEIVING != (psession->status & RECEIVING)) && (psession->bytes_transferred[RECV] == 0))
	{
		psession->status |= RECEIVING;
		DWORD size = 0;
		WSARecv(psession->s, &psession->wsabuf[RECV], 1, &size, &psession->wsa_flags[RECV], &psession->overlapped[RECV].overlapped, 0);
		psession->overlapped[RECV].result = WSAGetLastError();
		if(0 != psession->overlapped[RECV].result && WSA_IO_PENDING != psession->overlapped[RECV].result)
		{
			psession->status &= ~RECEIVING;
			session_close(psession);
		}
	}
}

OpenSSL data handling
When use OpenSSL with asynchronous sockets, we need to use memory BIOs instead of socket BIOs and we are responsible for data transmission between sockets and memory BIOs. For data received from sockets, we need to call BIO_write to write them to input BIO, then call SSL_read with input BIO to retrieve decrypted plain text; for data to be sent out, first call SSL_write to write them into output BIO, then call BIO_read on output BIO to retrieve data that needs to be sent to peer over socket. Since we are using memory BIOs, there are some very important points worth mentioning:

  • Any data received from socket should be written to input BIO with BIO_write.
  • Call SSL_read even if you didn’t write any data into input BIO, it is especially critical to call SSL_read after the socket connection was established and SSL_set_connect_state or SSL_set_accept_state was called. We need SSL_read to start hand-shaking with peer here.
  • After calling SSL_read, make sure call BIO_read on output BIO to check whether output BIO has pending data to be sent to peer over socket. This might happen because SSL_read will write data to output BIO when it handles hand-shaking or re-negotiation internally. We need to send data from BIO_read to peer over socket, and before making next call to SSL_read, check whether you have data received from socket, and if yes, use BIO_write to write them into input BIO.
  • SSL/TLS works by records with size can be up to 16KB, it might take several reads from network and writes to input BIO to fill up a SSL/TLS record before SSL_read can decode the record and spit something out. SSL_read still can be called, but it will return error WANT_READ, and we use ssl_is_fatal_error to ignore it.
  • If return value from SSL_read, SSL_write, BIO_read and BIO_write are greater than 0, it means the operation is successful, buffer passed to SSL_read now contains decrypted plain text and buffer passed to BIO_read on output BIO now contains data needs to be sent to peer over socket. If the return value is 0 or less than 0, you need to call ssl_is_fatal_error to check the internal SSL error and determine whether to bail out based on it. One exception here is BIO_write, if it returns 0 or less, you can call BIO_should_retry to determine whether it is fatal error. If BIO_should_retry returned true, you need to call BIO_write again with original parameters. Of course a call to SSL_read needs to be made in-between as it will consume data held in input BIO buffer.

Following function demonstrates how to move data between socket buffer, memory BIOs and SSL buffer.

bool session_process(session *psession)
{
	bool fatal_error_occurred = false;
	if(nullptr != psession->ssl)
	{
		if(psession->bytes_transferred[RECV] > 0)
		{
			int bytes = BIO_write(psession->bio[RECV], psession->socket_buffer[RECV], psession->bytes_transferred[RECV]);
			int ssl_error = ssl_get_error(psession->ssl, bytes);
			if(bytes == psession->bytes_transferred[RECV])
			{
				psession->bytes_transferred[RECV] = 0;
			}
			else if(!BIO_should_retry(psession->bio[RECV]))
			{
				fatal_error_occurred = true;
			}
		}

		if(psession->ssl_buffer_size[RECV] == 0)
		{
			int bytes = 0;
			do
			{
				bytes = SSL_read(psession->ssl, psession->ssl_buffer[RECV], BUFFER_SIZE);
				int ssl_error = ssl_get_error(psession->ssl, bytes);

				if ((HANDSHAKING == (psession->status & HANDSHAKING)) && SSL_is_init_finished(psession->ssl))
				{
					psession->status &= ~HANDSHAKING;
					psession->status |= CONNECTED;

					app_on_session_connect(psession);
				}

				if (bytes > 0)
				{
					psession->ssl_buffer_size[RECV] = bytes;
					app_on_session_recv(psession);
					psession->ssl_buffer_size[RECV] = 0;
				}
				else if (ssl_is_fatal_error(ssl_error))
				{
					fatal_error_occurred = true;
				}
			} while (bytes > 0);
		}

		if(psession->ssl_buffer_size[SEND] > 0)
		{
			int bytes = SSL_write(psession->ssl, psession->ssl_buffer[SEND], psession->ssl_buffer_size[SEND]);
			int ssl_error = ssl_get_error(psession->ssl, bytes);
			if(bytes == psession->ssl_buffer_size[SEND])
			{
				psession->ssl_buffer_size[SEND] = 0;
			}
			else if(ssl_is_fatal_error(ssl_error))
			{
				fatal_error_occurred = true;
			}
		}

		if(psession->wsabuf[SEND].len == 0 && (0 != psession->s_listening || BIO_pending(psession->bio[SEND])))
		{
			int bytes = BIO_read(psession->bio[SEND], psession->socket_buffer[SEND], BUFFER_SIZE);
			int ssl_error = ssl_get_error(psession->ssl, bytes);
			if(bytes > 0)
			{
				psession->wsabuf[SEND].len = bytes;
			}
			else if(ssl_is_fatal_error(ssl_error))
			{
				fatal_error_occurred = true;
			}
		}

		if(fatal_error_occurred)
			session_close(psession);
	}

	session_send(psession);
	session_recv(psession);
	
	return !fatal_error_occurred;
}

I/O completion routine and result handling
session_on_completed_packets is our I/O completion handling routine, here we can cast OVERLAPPED pointer safely to a session_overlapped pointer, from which not only we know which session the pointer belongs to, by comparing the pointer with each element in overlapped array, we also know the type of the finished operation. After that we call one of the three handling functions to process the result. If peer socket has been closed (received 0 bytes), or something went wrong, these functions will call session_close one or multiple times. If the session status is CLOSED, it means there is no more active operations and we can free resources used by this session and finally delete this session.

void session_on_connect(session *psession)
{
	psession->status &= ~CONNECTING;
	if(0 == psession->overlapped[CONNECT].result)
	{
		int size = sizeof(psession->addresses[LOCAL]);
		getsockname(psession->s, (sockaddr*)&psession->addresses[LOCAL], &size);
		sockaddr_to_string(&psession->addresses[LOCAL], psession->addresses_sz[LOCAL]);

		psession->status |= HANDSHAKING;
		SSL_set_connect_state(psession->ssl);

		session_process(psession);
	}
	else
	{
		app_on_session_connect(psession);
		session_close(psession);
	}
}

void session_on_accept(session *psession)
{
	if(0 == psession->overlapped[CONNECT].result)
	{
		session *psession_new = session_new(psession->pdata);
		psession_new->s_listening = psession->s_listening;
		session_accept(psession_new);

		psession->status &= ~ACCEPTING;
		psession->status |= HANDSHAKING;

		int naddrslen[2] = {0, 0};
		sockaddr *paddrs[2] = {0, 0};
		GetAcceptExSockaddrs(psession->addresses, 0, sizeof(sockaddr_storage), sizeof(sockaddr_storage), &paddrs[LOCAL], &naddrslen[LOCAL], &paddrs[REMOTE], &naddrslen[REMOTE]);
		memcpy_s(&psession->addresses[LOCAL], sizeof(sockaddr_storage), paddrs[LOCAL], naddrslen[LOCAL]);
		memcpy_s(&psession->addresses[REMOTE], sizeof(sockaddr_storage), paddrs[REMOTE], naddrslen[REMOTE]);
		setsockopt(psession->s, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT, (char*)&psession->s_listening, sizeof(SOCKET));
		sockaddr_to_string(&psession->addresses[LOCAL], psession->addresses_sz[LOCAL]);
		sockaddr_to_string(&psession->addresses[REMOTE], psession->addresses_sz[REMOTE]);

		SSL_set_accept_state(psession->ssl);
		session_process(psession);
	}
}

void session_on_recv(session *psession)
{
	psession->status &= ~RECEIVING;
	if((psession->bytes_transferred[RECV] > 0))
	{
		session_process(psession);
	}
	else
	{
		session_close(psession);
	}
}

void session_on_send(session *psession)
{
	psession->status &= ~SENDING;
	psession->wsabuf[SEND].len = 0;
	app_on_session_send(psession);
	if(0 == psession->overlapped[SEND].result)
	{
		session_process(psession);
	}
	else
	{
		session_close(psession);
	}
}

void session_on_completed_packets(DWORD dwNumberOfBytesTransferred, ULONG_PTR lpCompletionKey, LPOVERLAPPED pOverlapped)
{
	session_overlapped *p = (session_overlapped*)pOverlapped;
	session_lock(p->psession);

	DWORD dwOverlappeddNumberOfBytesTransferred = 0, dwOverlappedFlags = 0;
	BOOL succeeded = WSAGetOverlappedResult(p->psession->s, pOverlapped, &dwOverlappeddNumberOfBytesTransferred, TRUE, &dwOverlappedFlags);
	p->result = WSAGetLastError();
	
	if(pOverlapped == &p->psession->overlapped[CONNECT].overlapped)
	{
		if(0 == p->psession->s_listening)
			session_on_connect(p->psession);
		else
			session_on_accept(p->psession);
	}
	else if(pOverlapped == &p->psession->overlapped[RECV].overlapped)
	{
		p->psession->bytes_transferred[RECV] = dwNumberOfBytesTransferred;
		session_on_recv(p->psession);
	}
	else if(pOverlapped == &p->psession->overlapped[SEND].overlapped)
	{
		p->psession->bytes_transferred[SEND] = dwNumberOfBytesTransferred;
		session_on_send(p->psession);
	}

	bool close_session = (CLOSED == p->psession->status);
	session_unlock(p->psession);
	
	if(close_session)
		app_on_session_close(p->psession);
}

Use OpenSSL in a server application
To use OpenSSL in a server application, there are some additional steps required. First is to load the server certificate and private key. Here is sample code of loading certificate and private key in PEM format from memory and apply them to a SSL_CTX:

void set_cert()
{
	int length = strlen(server_cert_key_pem);
	BIO *bio_cert = BIO_new_mem_buf((void*)server_cert_key_pem, length);
	X509 *cert = PEM_read_bio_X509(bio_cert, nullptr, nullptr, nullptr);
	printf("Certificate used for server:\n");
	ssl_print_cert_info(cert);
	EVP_PKEY *pkey = PEM_read_bio_PrivateKey(bio_cert, 0, 0, 0);
	ssl_set_ctx_cert_and_key(cert, pkey);
	X509_free(cert);
	EVP_PKEY_free(pkey);
	BIO_free(bio_cert);
}

SSL_CTX holds global configurations. If you need to apply certificate and private key to each individual SSL connection, use SSL_use_certificate and SSL_use_PrivateKey instead.

The next step is to create a socket and set it in listening state:

SOCKET create_listen_socket(int port)
{
	sockaddr_storage addr = {0};
	sockaddr_in *paddrin = (sockaddr_in*)&addr;
	addr.ss_family = AF_INET;
	paddrin->sin_port = htons(port);
	paddrin->sin_addr.s_addr = htonl(INADDR_ANY);

	SOCKET s = WSASocket(addr.ss_family, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
	iocp_associate_handle((HANDLE)s);
	bind(s, (sockaddr*)&addr, sizeof(addr));
	listen(s, SOMAXCONN);
	return s;
}

To accept connections, create an instance of session object and call session_accept. If a connection was accepted, a new instance of session object will be created automatically and passed to a call to session_accept(). The server will handle connection request from multiple clients automatically.

	session *psession = session_new(0);
	psession->s_listening = s;
	session_accept(psession);

How to integrate it with your application
It is easy to integrate the framework with your application. The first step is to implement following callback functions:

// callback functions which are called under the protection of the psession->lock.
extern void app_on_session_connect(session *psession);
extern void app_on_session_send(session *psession);
extern void app_on_session_recv(session *psession);

// callback function which is called when psession is no longer in use and be safely
// deleted by calling session_delete().
extern void app_on_session_close(session *psession);

They will be called from iocp worker threads and except app_on_session_close, they will be called under the protection of session’s lock, which means it is thread-safe to access members of session object. app_on_session_close is used to notify application that framework will no longer touch psession and application can delete the session object now.

To send data out, call function session_send_data, it returns bytes sent, and be careful that it might be less than the size you passed it. Application needs to adjust the buffer pointer with bytes returned and call this function again, the best place might be from the callback function app_on_session_send, which is called when framework sent some data to peer over socket and there might be space freed in internal buffer.

When framework received and decrypted some data, it will call callback function app_on_session_recv, data is in psession->ssl_buffer[RECV] and its size is in psession->ssl_buffer_size[RECV].

When peer closes session first or something went wrong, framework will close session automatically, and after all pending operations have finished or been cancelled, it will call app_on_session_close. Application can also close session explicitly:

session_lock(psession);
session_close(psession);
session_unlock(psession);

app_on_session_close will be called when it is done, and again, it will be called from iocp worker thread.

SSL man-in-the-middle agent
You might realize that you can implement a SSL man-in-the-middle agent by pairing a client session and server session if you can manage redirecting peer traffic to your server, and one potential issue is certificate. After SSL MITM agent received connection from client, before it can perform hand-shake with client, the agent needs to connect to real server and finish hand-shaking with it, extract server certificate, create a new RSA key pair, set issuer to a self-signed CA’s subject, re-sign the certificate with CA’s private key, call SSL_use_certificate and SSL_use_PrivateKey on client SSL and start hand-shaking with client. Following code shows how to make a copy of certificate received from real server and re-sign the certificate with a self-signed CA certificate and its private key:

X509* ssl_copy_cert(X509 *cert)
{
	int length = i2d_X509(cert, nullptr);
	unsigned char *buffer = (unsigned char *)OPENSSL_malloc(length);
	unsigned char *pointer_that_will_be_increased_after_encode = buffer;
	i2d_X509(cert, &pointer_that_will_be_increased_after_encode);
	const unsigned char *pointer_that_will_be_increased_after_decode = (const unsigned char *)buffer;
	X509 *cert_copy = d2i_X509(nullptr, &pointer_that_will_be_increased_after_decode, length);
	OPENSSL_free(buffer);
	return cert_copy;
}

EVP_PKEY* ssl_sign_cert(X509 *cert, X509 *ca_cert, EVP_PKEY *ca_pkey_private)
{
	srand((int)time(0) + rand());
	EVP_PKEY *pkey = EVP_PKEY_new();
	RSA *rsa = RSA_generate_key(2048, RSA_F4, 0, 0);
	EVP_PKEY_assign_RSA(pkey, rsa);
	rsa = nullptr;

	ASN1_INTEGER_set(X509_get_serialNumber(cert), (int)time(0) + rand());
	X509_set_issuer_name(cert, X509_get_subject_name(ca_cert));
	::X509_set_pubkey(cert, pkey);
		
	X509_sign(cert, ca_pkey_private, EVP_sha1());

	return pkey;
}

Note that CA certificate and private key can be loaded using same way as server certificate.

Following function shows how to read important information from X509 certificate:

void ssl_print_cert_info(X509 *cert)
{
	printf("=======================\n");
	X509_NAME *name_subject = X509_get_subject_name(cert);
	if(name_subject > 0)
	{
		BIO *bio = BIO_new(BIO_s_mem());
		X509_NAME_print_ex(bio, name_subject, 0, XN_FLAG_RFC2253);
		char *subject = nullptr;
		long length = BIO_get_mem_data(bio, &subject);
		if(nullptr != subject && length > 0)
		{
			std::string str;
			str.resize(length);
			std::copy(subject, subject + length, str.begin());
			printf("CERT subject: %s\n", str.c_str());
		}
		BIO_free(bio);
	}

	X509_NAME *name_issuer = X509_get_issuer_name(cert);
	if(name_issuer > 0)
	{
		printf("CERT cn: %s\n", ssl_get_cert_issuer_info_by_id(name_issuer, NID_commonName).c_str());
		printf("CERT c: %s\n", ssl_get_cert_issuer_info_by_id(name_issuer, NID_countryName).c_str());
		printf("CERT o: %s\n", ssl_get_cert_issuer_info_by_id(name_issuer, NID_organizationName).c_str());
	}

	int criticality = -1, ext_index = -1;
	ASN1_BIT_STRING *key_usage = (ASN1_BIT_STRING *)X509_get_ext_d2i(cert, NID_key_usage, &criticality, &ext_index);
	if(key_usage > 0)
	{
		const char *usages[] = {"digitalSignature",
								"nonRepudiation",
								"keyEncipherment",
								"dataEncipherment",
								"keyAgreement",
								"keyCertSign",
								"cRLSign",
								"encipherOnly",
								"decipherOnly"};

		printf("CERT key_usage:");
		for (int index = 0; index < 8; index++)
		{
			if(ASN1_BIT_STRING_get_bit(key_usage, index))
				printf(" %s;", usages[index]);
		}
		printf("\n");
	}

	const char *kuValue = NULL;
	STACK_OF(ASN1_OBJECT) *ext_key_usage = (STACK_OF(ASN1_OBJECT) *)X509_get_ext_d2i(cert, NID_ext_key_usage, NULL, NULL);
	if(ext_key_usage > 0)
	{
		printf("CERT ext_key_usage:");
		while(sk_ASN1_OBJECT_num(ext_key_usage) > 0)
		{
			int usage_id = OBJ_obj2nid(sk_ASN1_OBJECT_pop(ext_key_usage));
			const char *usage_value = OBJ_nid2sn(usage_id);
			printf(" %d:%s;", usage_id, usage_value);
		}
		printf("\n");
	}

	STACK_OF(GENERAL_NAME) *alt_name = (STACK_OF(GENERAL_NAME)*) X509_get_ext_d2i(cert, NID_subject_alt_name, NULL, NULL);
	while(sk_GENERAL_NAME_num(alt_name) > 0)
	{
		GENERAL_NAME *name = sk_GENERAL_NAME_pop(alt_name);
		switch(name->type)
		{
			case GEN_DNS:
				printf("CERT GEN_DNS: %s\n", ASN1_STRING_data(name->d.dNSName));
				break;
			case GEN_URI:
				printf("CERT GEN_URI: %s\n", ASN1_STRING_data(name->d.uniformResourceIdentifier));
				break;
			case GEN_EMAIL:
				printf("CERT GEN_EMAIL: %s\n", (char*)ASN1_STRING_data(name->d.rfc822Name));
				break;
		}
	}
	printf("=======================\n");
}

Sample
I created a sample client and server program which demonstrate how to use the code showed above. The sample can be downloaded from here. You can download and build them using VS2010. The client program can be used with server program, or you can run it with a real web server, for example, here is output of running client.exe to connect Amazon on port 443 and sent “get /” to server:

C:\Users\user\Desktop\Cert>client amazon.com
Connected to 205.251.242.54:443
=======================
CERT subject: CN=www.amazon.com,O=Amazon.com Inc.,L=Seattle,ST=Washington,C=US
CERT cn: VeriSign Class 3 Secure Server CA - G2
CERT c: US
CERT o: VeriSign, Inc.
CERT key_usage: digitalSignature; keyEncipherment;
CERT ext_key_usage: 130:clientAuth; 129:serverAuth;
CERT GEN_DNS: www.amazon.com
CERT GEN_DNS: amazon.com
CERT GEN_DNS: uedata.amazon.com
=======================
Type message to be sent to server and press ENTER, empty message to exit
get /
Received 166 bytes from 205.251.242.54:443:<html>
<head><title>400 Bad Request</title></head>
<body bgcolor="white">
<center><h1>400 Bad Request</h1></center>
<hr><center>nginx</center>
</body>
</html>

Session 192.168.241.133:49563 - 205.251.242.54:443 is closed
Advertisements

Posted on February 20, 2013, in openssl, socket. Bookmark the permalink. 38 Comments.

  1. Dear Sir, can you put small examples for server and client?

  2. Dear Sir! I should make a choice between clients. How to store sessions of all clients, and then to select these sessions? How it is more convenient to make it? Can you help me with this. Thank you

  3. Dear sir! I shall make a choice between clients. How to save sessions of all clients, and then to select a session of one (necessary) client? How it is more convenient to make it? You can help me with it? Thanks

    • When iocp accepted a connection, it will call app_on_session_connect with pointer to a session object; when iocp closes this connection later, it will call app_on_session_close with pointer to same session object. So the key to your question is to take advantage of these two callbacks and use the psession to track the clients. For example, you can put psession into a global map or list in app_on_session_connect and remove it from map or list in app_on_session_close. Or you can create your own struct to keep additional information and keep a pointer to session inside the struct.

      • ———- Block 118 at 0x011EE648: 4616 bytes ———-
        Thanks for a prompt reply – I approximately so thought. Now other question of memory leaks which are defined by Visual Lead Detector in case of output from the server. The amount of leaks is equal to the size of structure of session. How it can be avoided? Thanks
        ….
        ….
        Call Stack:
        f:\dd\vctools\crt_bld\self_x86\crt\src\malloc.c (55): server.exe!_heap_alloc_base
        f:\dd\vctools\crt_bld\self_x86\crt\src\dbgheap.c (431): server.exe!_heap_alloc_dbg_impl + 0x9 bytes
        f:\dd\vctools\crt_bld\self_x86\crt\src\dbgheap.c (239): server.exe!_nh_malloc_dbg_impl + 0x19 bytes
        f:\dd\vctools\crt_bld\self_x86\crt\src\dbgheap.c (302): server.exe!_nh_malloc_dbg + 0x1D bytes
        f:\dd\vctools\crt_bld\self_x86\crt\src\dbgmalloc.c (56): server.exe!malloc + 0x15 bytes
        f:\dd\vctools\crt_bld\self_x86\crt\src\new.cpp (59): server.exe!operator new + 0x9 bytes
        f:\dd\vctools\crt_bld\self_x86\crt\src\threadex.c (314): server.exe!_callthreadstartex + 0xF bytes
        f:\dd\vctools\crt_bld\self_x86\crt\src\threadex.c (297): server.exe!_threadstartex
        0x7696339A (File and line number not available): kernel32.dll!BaseThreadInitThunk + 0x12 bytes
        0x77459ED2 (File and line number not available): ntdll.dll!RtlInitializeExceptionChain + 0x63 bytes
        0x77459EA5 (File and line number not available): ntdll.dll!RtlInitializeExceptionChain + 0x36 bytes
        Data:
        C8 01 00 00 E4 00 00 00 02 00 20 FC 51 C3 38 EA …….. ….Q.8.
        00 00 00 00 00 00 00 00 38 EA 00 00 00 00 00 00 …….. 8…….
        00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
        00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
        00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
        00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
        00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
        00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
        00 00 00 00 00 00 00 00 02 00 37 86 51 C3 38 EE …….. ..7.Q.8.
        00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
        00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
        00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
        00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
        00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
        00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
        00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..

        Visual Leak Detector detected 6 memory leaks (41522 bytes).
        Largest number used: 54282 bytes.
        Total allocations: 122070 bytes.
        Visual Leak Detector is now exiting.

      • Well, the session objects are deleted by calling session_delete() from the callback function app_on_session_close(), and app_on_session_close() will be called from session_close() only if the session has reached CLOSED status. Both peer and server can initiate the tear down of the session, and session_close() might be called more than once for same session object before it can set its status to CLOSED, e.g. there is always a recv posted on the socket, closesocket() will cancel this i/o, the cancellation will result a callback from iocp, only in this cancellation back we can set session status to CLOSED safely.

        If you found memory leaks of session objects, I would suspect their status didn’t reach CLOSED, and I would do followings in the order:

        1. Implement a global list of pointers to session, in callback app_on_session_connect(), add psession into the list; in callback app_on_session_close(), remove the psession from list. When program finishes, dump the remaining and leaked sessions, especially the peer addresses and value of status to check whether it has one of the OPERATING bits set;
        2. Print logs about session information from critical functions, especially session_on_completed_packets() and session_close(), monitor the status transition. Compare the logs with the leaked sessions to check what made status not reaching CLOSED.

        Let me know if you found something, thanks!

  4. I made the following. It led to that there was only one memory leak.

    // A vector of object pointers
    CVector vecObjectPtrs;

    void app_on_session_connect(session *psession)
    {
    printf(“Connected from %s\n”, psession->addresses_sz[REMOTE]);
    // Append to end of vector
    vecObjectPtrs.Add(psession);// ( 1 )
    }

    void app_on_session_close(session *psession)
    {
    printf(“Session %s – %s is closed and will be deleted\n”, psession->addresses_sz[LOCAL], psession->addresses_sz[REMOTE]);

    for (int i=vecObjectPtrs.GetSize()-1; i >= 0; i–) // ( 2 )
    {
    if (vecObjectPtrs[i] == psession)
    {
    vecObjectPtrs.RemoveAt(i);
    break;
    }
    }

    // whent this function is called, all pending operations associated with the psession have either finished or been cancelled.
    // the framework won’t touch psession anymore so it is safe to delete psession here directly.
    session_delete(psession);
    }

    int _tmain(int argc, _TCHAR* argv[])
    {

    for (int i=0; i status);
    session_delete( vecObjectPtrs[i]); ( 3 )
    }

    return 0;
    }

    Output:

    Session status 8 ( 1 – app_on_session_connect)

    Session status 128 ( 2 – app_on_session_close)


    Session status 24 ( 3 – _tmain on exit)
    Session status 24
    Session status 24
    Session status 24

    WARNING: Visual Leak Detector detected memory leaks!
    ———- Block 114 at 0x004CB068: 4616 bytes ———-
    Call Stack:
    d:\examples\openssl_iocp_1\shared\openssl_iocp.cpp (261): server.exe!session_new + 0x10 bytes
    d:\examples\openssl_iocp_1\shared\openssl_iocp.cpp (559): server.exe!session_on_accept + 0x5 bytes
    d:\examples\openssl_iocp_1\shared\openssl_iocp.cpp (624): server.exe!session_on_completed_packets + 0xC bytes
    d:\examples\openssl_iocp_1\shared\openssl_iocp.cpp (18): server.exe!iocp_on_completed_packets + 0x11 bytes
    d:\examples\openssl_iocp_1\shared\openssl_iocp.cpp (35): server.exe!iocp_proc + 0x11 bytes
    f:\dd\vctools\crt_bld\self_x86\crt\src\threadex.c (314): server.exe!_callthreadstartex + 0xF bytes
    f:\dd\vctools\crt_bld\self_x86\crt\src\threadex.c (297): server.exe!_threadstartex
    0x7550339A (File and line number not available): kernel32.dll!BaseThreadInitThunk + 0x12 bytes
    0x77D19ED2 (File and line number not available): ntdll.dll!RtlInitializeExceptionChain + 0x63 bytes
    0x77D19EA5 (File and line number not available): ntdll.dll!RtlInitializeExceptionChain + 0x36 bytes
    Data:
    9C 01 00 00 E4 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..

    Visual Leak Detector detected 1 memory leak (22922 bytes).
    Largest number used: 54490 bytes.
    Total allocations: 85198 bytes.
    Visual Leak Detector is now exiting.
    The program ‘[0x3A20] server.exe: Native’ has exited with code 0 (0x0).

  5. Format error..
    I made the following. It led to that there was only one memory leak.

    // A vector of object pointers
    CVector vecObjectPtrs;

    void app_on_session_connect(session *psession)
    {
    printf(“Connected from %s\n”, psession->addresses_sz[REMOTE]);
    // Append to end of vector
    vecObjectPtrs.Add(psession);// ( 1 )
    }

    void app_on_session_close(session *psession)
    {
    printf(“Session %s – %s is closed and will be deleted\n”, psession->addresses_sz[LOCAL], psession->addresses_sz[REMOTE]);

    for (int i=vecObjectPtrs.GetSize()-1; i >= 0; i–) // ( 2 )
    {
    if (vecObjectPtrs[i] == psession)
    {
    vecObjectPtrs.RemoveAt(i);
    break;
    }
    }

    // whent this function is called, all pending operations associated with the psession have either finished or been cancelled.
    // the framework won’t touch psession anymore so it is safe to delete psession here directly.
    session_delete(psession);
    }

    int _tmain(int argc, _TCHAR* argv[])
    {

    for (int i=0; i status);
    session_delete( vecObjectPtrs[i]); ( 3 )
    }

    return 0;
    }

    Output:

    Session status 8 ( 1 – app_on_session_connect)

    Session status 128 ( 2 – app_on_session_close)


    Session status 24 ( 3 – _tmain on exit)
    Session status 24
    Session status 24
    Session status 24

    WARNING: Visual Leak Detector detected memory leaks!
    ———- Block 114 at 0x004CB068: 4616 bytes ———-
    Call Stack:
    d:\examples\openssl_iocp_1\shared\openssl_iocp.cpp (261): server.exe!session_new + 0x10 bytes
    d:\examples\openssl_iocp_1\shared\openssl_iocp.cpp (559): server.exe!session_on_accept + 0x5 bytes
    d:\examples\openssl_iocp_1\shared\openssl_iocp.cpp (624): server.exe!session_on_completed_packets + 0xC bytes
    d:\examples\openssl_iocp_1\shared\openssl_iocp.cpp (18): server.exe!iocp_on_completed_packets + 0x11 bytes
    d:\examples\openssl_iocp_1\shared\openssl_iocp.cpp (35): server.exe!iocp_proc + 0x11 bytes
    f:\dd\vctools\crt_bld\self_x86\crt\src\threadex.c (314): server.exe!_callthreadstartex + 0xF bytes
    f:\dd\vctools\crt_bld\self_x86\crt\src\threadex.c (297): server.exe!_threadstartex
    0x7550339A (File and line number not available): kernel32.dll!BaseThreadInitThunk + 0x12 bytes
    0x77D19ED2 (File and line number not available): ntdll.dll!RtlInitializeExceptionChain + 0x63 bytes
    0x77D19EA5 (File and line number not available): ntdll.dll!RtlInitializeExceptionChain + 0x36 bytes
    Data:
    9C 01 00 00 E4 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..
    00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 …….. ……..

    Visual Leak Detector detected 1 memory leak (22922 bytes).
    Largest number used: 54490 bytes.
    Total allocations: 85198 bytes.
    Visual Leak Detector is now exiting.
    The program ‘[0x3A20] server.exe: Native’ has exited with code 0 (0x0).

    • I don’t know why, there was the wrong formatting of this piece of a code:
      for (int i=0; i status);
      session_delete( vecObjectPtrs[i]); ( 3 )
      }

      • Can you please print out the status of each remaining session in vecObjectPtrs as well as their local and remote address? This list should be empty, unless the peer didn’t close connection properly or there was a bug in the code. The question to you is how is the connection being closed on client side?

        There is always one “leaked” session object by design, when a connection was accepted, a new session will be created automatically in session_on_accept() to accept next connection. You can use a global variable to track it and delete it when shutting down.

        PS. for posting code, you can refer to http://en.support.wordpress.com/code/posting-source-code/.

  6. for (int i=0; i < vecObjectPtrs.GetSize(); ++i)

  7. Sir, I made a simple example of use of your code on MFC. I need to listen to 2 ports.
    1 . The buttons Start 1 and Start 2 I start listening of 2 ports.
    2 . The buttons Stop 1 and Stop 2 I finish listening of these ports.
    3 . Pressing a “x” I finish the program.
    Usually I have one leakage of memory, sometimes – an exception. It is connected, most likely, with synchronization. If you not against, I could send you this project for viewing.

  8. Sir, I have to you a question – whether it is possible to create in principle in one application some clients and servers? I need to listen to two ports and to do redirections of data from one port on another.

    • Its more than possible. I guess you want to redirect outbound connections to a port that your code is listening and wanted to be able to inspect the network traffic. Basically the data flow is like this:

      [any application] [loopback] [outer interface] [remote server]

      If you duplicate the members in session structure, one set for the inner socket (on loopback) and another set for outer socket (on interface used to connect remote server), then you have most pieces done here.

      You will also need to figure out how to redirect connections to your port on loopback. Most security applications use drivers to do that.

      The truth is that the code showed here actually is a simplified version of such proxy project. I had to stripped it down to a single socket.

      • Thanks a lot for the response. Theoretically I understand that such opportunity is available. But how practically to make it in my situation when in one application I need to listen to 2 ports?
        How to make everything for one – I understand.
        1 . ssl_init ();
        2 . set_cert ();
        3 . soc = create_listen_socket (ip, port);
        4 . psession = session_new ();
        5 . psession->s_listening = soc;
        6 . session_accept(psession);
        But how to me now to add listening 2 ports?
        To repeat steps 1 – 6, probably, it won’t be correct?

      • You just need to repeat step 3-6, e.g.:

        SOCKET s1 = create_listen_socket(port1);
        session *psession = session_new();
        psession->s_listening = s1;
        session_accept(psession);

        SOCKET s2 = create_listen_socket(port2);
        session *psession = session_new();
        psession->s_listening = s2;
        session_accept(psession);

        If a connection was established, app_on_session_connect() will be called, and you can check the port in session.addresses to figure out which one it is.

      • Because of not so good knowledge of English, I not understood directly everything. Now it became clear. Thanks a lot

      • If I change structure to such look:

        struct session
        {
        	SOCKET s_1; // handle to socket
        	SOCKET s_listening_1; // handle to external listening socket
        	sockaddr_storage addresses_1[2]; // local and remote address
        	char addresses_sz_1[2][ADDR_SZ_SIZE]; // addresses in string format
        	char socket_buffer_1[2][BUFFER_SIZE]; // memory used for read/write from/to socket
        	char ssl_buffer_1[2][BUFFER_SIZE]; // memory used for read/write from/to ssl memory bio
        	DWORD ssl_buffer_size_1[2]; // indicates the bytes of valid data in ssl_buffer
        	unsigned int status_1; // stores current socket status, bit-masked value of one or more of SOCKET_STATUS
        	session_overlapped overlapped_1[3]; // structure for overlapped operations
        	WSABUF wsabuf_1[2]; // structure used for pass buffer to overlapped operations
        	DWORD bytes_transferred_1[2]; // store the bytes of buffer that received/sent from/to the socket
        	DWORD wsa_flags_1[2]; // store the flags send/receive from overlapped operations, not used
        	SSL *ssl_1; // SSL structure used by OpenSSL
        	BIO *bio_1[2]; // memory BIO used by OpenSSL
        	ssl_lock lock_1; // synchronization object for multiple-thread data access
        	
        	
        	SOCKET s_2; // handle to socket
        	SOCKET s_listening_2; // handle to external listening socket
        	sockaddr_storage addresses_2[2]; // local and remote address
        	char addresses_sz_2[2][ADDR_SZ_SIZE]; // addresses in string format
        	char socket_buffer_2[2][BUFFER_SIZE]; // memory used for read/write from/to socket
        	char ssl_buffer_2[2][BUFFER_SIZE]; // memory used for read/write from/to ssl memory bio
        	DWORD ssl_buffer_size_2[2]; // indicates the bytes of valid data in ssl_buffer
        	unsigned int status_2; // stores current socket status, bit-masked value of one or more of SOCKET_STATUS
        	session_overlapped overlapped_2[3]; // structure for overlapped operations
        	WSABUF wsabuf_2[2]; // structure used for pass buffer to overlapped operations
        	DWORD bytes_transferred_2[2]; // store the bytes of buffer that received/sent from/to the socket
        	DWORD wsa_flags_2[2]; // store the flags send/receive from overlapped operations, not used
        	SSL *ssl_2; // SSL structure used by OpenSSL
        	BIO *bio_2[2]; // memory BIO used by OpenSSL
        	ssl_lock lock_2; // synchronization object for multiple-thread data access
        };
        

        As I will be able to understand, for example, in function session_on_completed_packets (…), what of sockets worked – s_1 or s_2?

  9. famellee | April 16, 2013 at 11:32 am
    Thanks a lot. It just that is necessary

  10. I liked your approach to iocp use.
    Whether it is possible to modify an available code for use it in http (not https)?

    • Absolutely. HTTPS is actually HTTP over SSL which in-turn is over TCP, while HTTP is over TCP directly. Essentially all you need to do is to remove functions have “ssl” in the code. And yes, iocp gives the best performance (in terms of resource usage) on Windows.

      • Sir!
        1. In session structure I commented out SSL * ssl and BIO * bio[2].
        2. I modified the session_process function (session * psession) to a look:

        bool CWrapOpenSslIOCP::session_process(session * psession)
           {
              bool fatal_error_occurred = false;
        	  
        
        if(psession->ssl_buffer_size[SEND] > 0)
        {		
        	psession->ssl_buffer_size[SEND] = 0;	
        	psession->bytes_transferred[RECV] = 0;
        }
        
        if((HANDSHAKING == (psession->status & HANDSHAKING)))
        {
        	psession->status &= ~HANDSHAKING;
        	psession->status |= CONNECTED;
        
        	on_session_connect(psession);
        
        	psession->status &= ~CONNECTED;
        }
        if(psession->bytes_transferred[RECV] > 0)
        {
        	psession->ssl_buffer_size[RECV] = psession->bytes_transferred[RECV];//bytes;
        	strncpy_s(psession->ssl_buffer[RECV], psession->wsabuf[RECV].buf, psession->wsabuf[RECV].len);
        	on_session_recv(psession);
        	psession->ssl_buffer_size[RECV] = 0;
        	psession->wsabuf[RECV].len = 0;
        	psession->bytes_transferred[RECV] = 0;
        }
        
        session_send(psession);
        session_recv(psession);
        
        return !fatal_error_occurred;
           }
        

        3. Everything works, but the HANDSHAKING field obviously superfluous.
        How to get rid of HANDSHAKING in the http-server?
        Thanks very much

      • If you don’t need SSL in your application, simply remove definition of HANDSHAKING, when iocp finishes an accept or connect, set status to CONNECTED directly.

  11. If I so do, I receive program hangup (through any number of connections).

    • Can you please elaborate? E.g. what did you do, and what do you mean “hangup”? Program hang or it doesn’t accept client connections?

      • I do a stress test – simply I send packets to the server. Through any time the program doesn’t accept client connection.

  12. Hi

    i am very glad to see this post . i googled for IOCP ssl client a lot finally found the solution here.

    it will be really helpful l if can you please provide me the C# version of the code . i am not familiar will c++ 😦

    Thanks,
    Vishnu.

  13. Hey Famelee,

    Nice post!

    I was wondering how to pipe content from openssl to another port.

    I would like to connect to a normal port and pipe the content via ssl.

    I saw that i have to:
    create the socket and connect to local service that i want to pipe..
    register the io handle on the completion via CreateIoCompletionPort
    then pipe the data TO and FROM..

    But i am lost on how to change the state machine to read, write and how to deal with state changes and locks on both socketes… one socket with ssl and another pure tcp…

    Do you have any hints on how can i do this?

    Thanks!

    Pucci

    • Famelee,

      I think I am getting there..

      I read all the replies and started to understand.

      What i did not get is this:

      Using iocp, the iocp for sockets will just receive packets.
      To send packets i will use wsasend normally?
      Will the send command activate the iocp too?

      Thanks,

      Pucci

  14. Hi.

    Your sample is very nice.

    I am starting server and use client to connect to it. TCPView shows that connection is esatblished with server but server do not respond on any client queries. It seems that there is no accept notify.

    Can you help with it?

  15. Dear Famelee,

    I am new to iocp and openssl and i have got two doubts.

    1. what is the purpose of WSASend() function in this code, since you are already using SSL_write() and BIO_read() functions.

    2. how do i stop server from echoing back to the client ? i am trying to make it work as simplex connection where client will only send and server will receive.

    Thanks,
    Rishi

  16. Dear Famelee
    http://sdrv.ms/ZBEGiL dont work, where can i get code of Samples?
    thanks

  17. nice code! do you have a mitm example used this framework ?

    • The most difficult part of MITM is getting clients connect to the proxy rather than the real server. There are several ways doing that depending on the location of the proxy, and it is not in the scope of the post.

      Once the proxy received the connection from clients, it is going to be easy to implement the SSL MITM using code showed here, by using two paired sessions to handle connections of clientagent and agentserver. The code posted here is actually stripped from a working MITM prototype with only public information, but they do include the most important parts.

  1. Pingback: OpenSSL tidbits | Some Things Are Obvious

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s

%d bloggers like this: