I'm trying to use a sqlite memory db ala "file:zmem?mode=memory&cache=shared" with multiple threads. To test this i have the following code which is not multi threaded, but just opens a second connection to the same memory db to read the results. Unfortunately i'm not able to find any working code samples on either sqlite.org or any other site on the internet.
I have the following C99 test code which works until another connection tries to access what the first connection created.
#include <sqlite3.h>
#include <stdio.h>
int exec(sqlite3 *db, char* sql) {
sqlite3_stmt *stmt;
int ret;
do {
ret = sqlite3_prepare_v2(db, sql, -1, &stmt, 0);
if (ret != SQLITE_OK) {
printf("prep error: %s", sqlite3_errmsg(db));
break;
}
ret = sqlite3_step(stmt);
if (ret != SQLITE_DONE) {
printf("step error: %s", sqlite3_errmsg(db));
break;
}
ret = SQLITE_OK;
} while(0);
sqlite3_finalize(stmt);
return ret;
}
sqlite3* getCon(char *name) {
sqlite3 *db = 0;
int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE |
SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_MEMORY | SQLITE_OPEN_SHAREDCACHE;
int ret = sqlite3_open_v2(name, &db, flags, 0);
if (ret != SQLITE_OK) {
printf("no open");
}
return db;
}
void query(sqlite3 *db) {
sqlite3_stmt *stmt = 0;
do {
int ret = sqlite3_prepare_v2(db, "SELECT * FROM MyTable", -1, &stmt, 0);
if (ret != SQLITE_OK) {
printf("prep error: %s\n", sqlite3_errmsg(db));
break;
}
while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) {
char *name = (char *) sqlite3_column_text(stmt, 0);
printf("%s\n", name);
}
if (ret != SQLITE_DONE) {
printf("step error: %s\n", sqlite3_errmsg(db));
}
} while(0);
sqlite3_finalize(stmt);
}
int main() {
sqlite3 *db = 0, *db2 = 0;
char *label = "zmem";
char bf[64];
char *buf = &bf[0];
int ret, i;
do {
db = getCon(label);
if (!db) break;
printf("connected\n");
ret = exec(db, "CREATE TABLE MyTable (test VARCHAR(255));");
if (ret != SQLITE_OK) break;
for (i = 1; i <= 3; i++) {
sprintf(buf,"insert into MyTable (test) values('Hello Row %d')", i);
ret = exec(db, buf);
if (ret != SQLITE_OK) break;
}
query(db);
db2 = getCon(label);
if (!db2) break;
printf("connected 2\n");
query(db2);
} while(0);
if (db) sqlite3_close(db);
if (db2) sqlite3_close(db2);
}
When i run this code it produces the following for me
connected
Hello Row 1
Hello Row 2
Hello Row 3
connected 2
prep error: no such table: MyTable
Could someone clue me in on how to properly do this?
Related
I have written a sample program to understand the effects of GPU/CPU pinned memory and Heap memory. The following code illustrates this. I have allocated three buffers of dimensions say 1280x720. I have filled buffers 1 & 2 with some data and in turn used these buffers to fill buffer 3. The mathematical operatio involved in filling buffer 3 is insignificant. In case 1, the memory allocated from these buffers are from heap (malloc call). In case 2, the memory for these buffers are allocated from OpenCL API calls (clCreateBuffer()). There is a performance difference between these 2 cases. I tested it on Intel integrated GPU's. I am unable to explain this difference in performance. Does it have some thing to do with cacheable properties of CPU/GPU pinned memory vs Heap memory.
Have you encountered such behavior before or am i doing something wrong?
#include <stdio.h>
#include <malloc.h>
#include <string.h>
#include <stdlib.h>
#include <inttypes.h>
#define OPENCL
#if defined(_WIN32)
/*
* Win32 specific includes
*/
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <windows.h>
#else
#include <sys/time.h>
/* timersub is not provided by msys at this time. */
#ifndef timersub
#define timersub(a, b, result) \
do { \
(result)->tv_sec = (a)->tv_sec - (b)->tv_sec; \
(result)->tv_usec = (a)->tv_usec - (b)->tv_usec; \
if ((result)->tv_usec < 0) { \
--(result)->tv_sec; \
(result)->tv_usec += 1000000; \
} \
} while (0)
#endif
#endif
struct usec_timer {
#if defined(_WIN32)
LARGE_INTEGER begin, end;
#else
struct timeval begin, end;
#endif
};
static void usec_timer_start(struct usec_timer *t) {
#if defined(_WIN32)
QueryPerformanceCounter(&t->begin);
#else
gettimeofday(&t->begin, NULL);
#endif
}
static void usec_timer_mark(struct usec_timer *t) {
#if defined(_WIN32)
QueryPerformanceCounter(&t->end);
#else
gettimeofday(&t->end, NULL);
#endif
}
static int64_t usec_timer_elapsed(struct usec_timer *t) {
#if defined(_WIN32)
LARGE_INTEGER freq, diff;
diff.QuadPart = t->end.QuadPart - t->begin.QuadPart;
QueryPerformanceFrequency(&freq);
return diff.QuadPart * 1000000 / freq.QuadPart;
#else
struct timeval diff;
timersub(&t->end, &t->begin, &diff);
return diff.tv_sec * 1000000 + diff.tv_usec;
#endif
}
#ifdef OPENCL
#include ".\CL\cl.h"
int opencl_init(cl_context *context, cl_command_queue *cmd_queue) {
cl_int status;
cl_uint num_platforms = 0;
cl_platform_id platform;
cl_uint num_devices = 0;
cl_device_id device;
cl_command_queue_properties command_queue_properties = 0;
// Get the number of platforms in the system.
status = clGetPlatformIDs(0, NULL, &num_platforms);
if (status != CL_SUCCESS || num_platforms == 0)
goto fail;
// Get the platform ID for one platform
status = clGetPlatformIDs(1, &platform, NULL);
if (status != CL_SUCCESS)
goto fail;
// Get the number of devices available on the platform
status = clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, 0, NULL, &num_devices);
if (status != CL_SUCCESS || num_devices == 0)
goto fail;
// Get the device ID for one device
status = clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, 1, &device, NULL);
if (status != CL_SUCCESS)
goto fail;
// Create OpenCL context for one device
*context = clCreateContext(NULL, 1, &device, NULL, NULL, &status);
if (status != CL_SUCCESS || *context == NULL)
goto fail;
// Create command queues for the device
*cmd_queue = clCreateCommandQueue(*context, device, command_queue_properties, &status);
if (status != CL_SUCCESS || *cmd_queue == NULL)
goto fail;
return 0;
fail:
return 1;
}
#endif
int main(int argc, char **argv) {
int x, y, z;
int width = 1280, height = 720;
unsigned char *buffer[3];
int use_gpu;
cl_mem opencl_mem[3];
cl_context context;
cl_command_queue cmd_queue;
cl_int status;
if (argc != 2)
return 0;
use_gpu = atoi(argv[1]);
if (use_gpu) {
if (opencl_init(&context, &cmd_queue))
printf("OpenCL init failure");
}
if (use_gpu) {
for (x = 0; x < 3; x++) {
opencl_mem[x] = clCreateBuffer(context,
CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR,
width * height * sizeof(*buffer[x]), NULL,
&status);
if (status != CL_SUCCESS)
return 0;
buffer[x] = clEnqueueMapBuffer(cmd_queue, opencl_mem[x], CL_TRUE,
CL_MAP_READ | CL_MAP_WRITE, 0,
width * height * sizeof(*buffer[x]), 0,
NULL, NULL, &status);
if (status != CL_SUCCESS) {
clReleaseMemObject(opencl_mem[x]);
opencl_mem[x] = NULL;
return 0;
}
}
} else {
for (x = 0; x < 3; x++) {
buffer[x] = malloc(width * height * sizeof(*buffer[x]));
if (buffer[x] == NULL) {
printf("Unable to alloc memory");
}
}
}
memset(buffer[0], 1, width * height * sizeof(*buffer[0]));
memset(buffer[1], 2, width * height * sizeof(*buffer[1]));
memset(buffer[2], 0, width * height * sizeof(*buffer[2]));
{
struct usec_timer emr_timer;
usec_timer_start(&emr_timer);
for (z = 0; z < 600; z++) {
for (y = 0; y < height; y++) {
for (x = 0; x < width; x++) {
// don't worry about overflows
buffer[2][y * width + x] += buffer[0][y * width + x]
+ buffer[1][y * width + x];
}
}
}
usec_timer_mark(&emr_timer);
printf("Elapsed time %"PRIu64"\n", usec_timer_elapsed(&emr_timer));
}
if (use_gpu) {
for (x = 0; x < 3; x++) {
if (buffer[x] != NULL) {
status = clEnqueueUnmapMemObject(cmd_queue, opencl_mem[0], buffer[0], 0,
NULL, NULL);
status |= clFinish(cmd_queue);
if (status != CL_SUCCESS)
return 0;
buffer[0] = NULL;
}
if (opencl_mem[0] != NULL) {
status = clReleaseMemObject(opencl_mem[0]);
if (status != CL_SUCCESS)
return 0;
opencl_mem[0] = NULL;
}
}
clReleaseCommandQueue(cmd_queue);
clReleaseContext(context);
} else {
for (x = 0; x < 3; x++) {
free(buffer[x]);
buffer[x] = NULL;
}
}
return 0;
}
If you use malloc + operation + free you are using only CPU resources.
If you use OpenCL you are using CPU + GPU, and you involve in syncronization and data copy penalties.
Alloc in GPU
Map to CPU space (allocs another buffer in CPU)
Operate CPU buffer
Unmap (pinned copy to the GPU buffer + deallocate the CPU one).
Destroy GPU buffer
What makes you think it should have the same speed? Of course is more costly, and will always be. You are doing the same CPU operation + some extra OpenCL operations.
Pinned memory is faster than non-pinned memory in transfers, but it is never faster than non copy, because you simply are not copying anything!
Also for a memory benchmark, doing operation with 3*1280*720 = 2.6MB, is completely silly. It would take just microseconds in common systems. And anyway, that part should be the same for both cases.
The overhead will dominate your results, rather than the throughput.
When coding http requests, i used Visual Studio for c++, now my code goes like this,
#define WIN32_LEAN_AND_MEAN
#include "stdafx.h"
#include <cstdio>
#include <cstdlib>
#include <Winsock2.h>
#include <WS2tcpip.h>
#include <windows.h>
#pragma comment (lib,"ws2_32")
int _tmain(int argc, _TCHAR* argv[])
{
WSADATA wsaData;
int iResult = WSAStartup(MAKEWORD(2,2),&wsaData);
SOCKET m_socket;
m_socket = socket(AF_INET,SOCK_STREAM,IPPROTO_TCP);
if(m_socket==INVALID_SOCKET)
{
printf("Invalid Socket :WSAGetLastError()");
}
sockaddr_in clientService;
clientService.sin_family = AF_INET;
clientService.sin_addr.s_addr = inet_addr("127.0.0.1");
clientService.sin_port = htons(5357);
LPHOSTENT host = gethostbyname("72.144.89.32");
if(connect(m_socket,(SOCKADDR*)&clientService,sizeof(clientService))==SOCKET_ERROR)
{
printf("Connection Failure");
WSACleanup();
return 1;
}
char buffer[2048];
strcpy(buffer,"POST /dbarea.php HTTP/1.1\n");
strcat(buffer,"Content - Type:application/x-www-form-urlencoded\n");
strcat(buffer,"Host: localost\n");
strcat(buffer,"content-Length:32\n");
strcat(buffer,"\n");
strcat(buffer,"username=emeka1&password=laikan112");
//int n = write(SOCKADDR*,buffer,strlen(buffer));
printf("Data Sent Successfully..");
return 0;
}
Now my php is like this
<?php
session_start();
$username = urldecode($_POST['username']);
$password = urldecode($_POST['password']);
echo "Username: $username\nPassword:$password";
?>
Now i have an issue on the php area , its not receiving information and printing it again for us to see, what could possibly be the problem?
There are several errors in your HTTP request.
You need to use \r\n instead of \n.
Content - Type needs to be Content-Type.
localost needs to be localhost.
content-Length:32 needs to be Content-Length: 34
Try this:
int _tmain(int argc, _TCHAR* argv[])
{
WSADATA wsaData;
int iResult = WSAStartup(MAKEWORD(2,2), &wsaData);
if (iResult != 0)
{
printf("WinSock startup error: %d\n", iResult);
return 1;
}
SOCKET m_socket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (m_socket == INVALID_SOCKET)
{
printf("Socket creation error: %d\n", WSAGetLastError());
WSACleanup();
return 1;
}
sockaddr_in clientService = {0};
clientService.sin_family = AF_INET;
clientService.sin_addr.s_addr = inet_addr("127.0.0.1");
clientService.sin_port = htons(5357);
if (connect(m_socket, (SOCKADDR*)&clientService, sizeof(clientService)) == SOCKET_ERROR)
{
printf("Connection error: %d\n", WSAGetLastError());
closesocket(m_socket);
WSACleanup();
return 1;
}
char buffer[2048];
strcpy(buffer, "POST /dbarea.php HTTP/1.1\r\n");
strcat(buffer, "Content-Type: application/x-www-form-urlencoded\r\n");
strcat(buffer, "Host: localhost:5357\r\n");
strcat(buffer, "Content-Length: 34\r\n");
strcat(buffer, "\r\n");
strcat(buffer, "username=emeka1&password=laikan112");
char ptr = buffer;
int len = strlen(buffer);
do
{
int n = send(m_socket, ptr, len, 0);
if (n < 1)
{
if (n == SOCKET_ERROR)
printf("Send error: %d\n", WSAGetLastError());
else
printf("disconnected by server\n");
closesocket(m_socket);
WSACleanup();
return 1;
}
ptr += n;
len -= n;
}
while (len > 0);
printf("Data Sent Successfully..\n");
closesocket(m_socket);
WSACleanup();
return 0;
}
You really should not write your own HTTP code from scratch like this, when there are better alternatives available, such as Microsoft's WinInet/WinHTTP APIs, the curl library, etc. Let them do the hard work for you.
#include <WinInet.h>
#include <tchar.h>
void ReportWinInetError(const char *operation)
{
DWORD dwErr = GetLastError();
if (dwErr == ERROR_INTERNET_EXTENDED_ERROR)
{
LPTSTR szBuffer = NULL;
DWORD dwLength = 0;
BOOL bRet = InternetGetLastResponseInfo(&dwErr, NULL, &dwLength);
if ((bRet == FALSE) && (GetLastError() == ERROR_INSUFFICIENT_BUFFER))
{
szBuffer = new TCHAR[dwLength+1];
bRet = InternetGetLastResponseInfo(&dwErr, szBuffer, &dwLength);
}
if (bRet)
_tprintf(_T("WinInet error while %hs: %u %*s"), operation, dwErr, dwLength, szBuffer);
else
printf("Unknown WinInet error while %s", operation);
delete[] szBuffer;
}
else
printf("WinInet error while %s: %u", operation, dwErr);
}
int _tmain(int argc, _TCHAR* argv[])
{
HINTERNET hInternet = InternetOpen(TEXT("MyApp"), INTERNET_OPEN_TYPE_DIRECT, NULL, NULL, 0);
if (!hInternet)
{
ReportWinInetError("opening session");
return 1;
}
HINTERNET hConnect = InternetConnect(hInternet, TEXT("127.0.0.1"), 5357, NULL, NULL, INTERNET_SERVICE_HTTP, 0, 0);
if (!hConnect)
{
ReportWinInetError("connecting session");
InternetCloseHandle(hInternet);
return 1;
}
HINTERNET hRequest = HttpOpenRequest(hConnect, TEXT("POST"), TEXT("/dbarea.php"), TEXT("HTTP/1.1"), NULL, NULL, 0, 0);
if (!hRequest)
{
ReportWinInetError("opening request");
InternetCloseHandle(hConnect);
InternetCloseHandle(hInternet);
return 1;
}
if (!HttpSendRequest(hRequest, TEXT("Content-Type: application/x-www-form-urlencoded"), -1, "username=emeka1&password=laikan112", 34))
{
ReportWinInetError("sending request");
InternetCloseHandle(hRequest);
InternetCloseHandle(hConnect);
InternetCloseHandle(hInternet);
return 1;
}
printf("Data Sent Successfully..\n");
InternetCloseHandle(hRequest);
InternetCloseHandle(hConnect);
InternetCloseHandle(hInternet);
return 0;
}
#include <curl/curl.h>
int _tmain(int argc, _TCHAR* argv[])
{
CURLCode ret = curl_global_init(CURL_GLOBAL_DEFAULT);
if (ret != CURLE_OK)
{
printf("Curl init error: %d\n", ret);
return 1;
}
CURL *curl = curl_easy_init();
if (!curl)
{
printf("Curl request creation error\n");
curl_global_cleanup();
return 1;
}
curl_easy_setopt(curl, CURLOPT_URL, "http://127.0.0.1:5357/dbarea.php");
curl_easy_setopt(curl, CURLOPT_POST, 1);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "username=emeka1&password=laikan112");
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, -1);
ret = curl_easy_perform(curl);
if (ret != CURLE_OK)
{
printf("Curl send error: %d\n", ret);
curl_easy_cleanup(curl);
curl_global_cleanup();
return 1;
}
printf("Data Sent Successfully..\n");
curl_easy_cleanup(curl);
curl_global_cleanup();
}
#include <stdio.h>
#include <iostream>
#include <Windows.h>
#include <C:\Program Files\Microsoft MPI\Inc\mpi.h>
using namespace std;
#define BUFSIZE 128
int main (int argc, char *argv[])
{
int err;
int rank;
int size;
double start_time = 0.0;
double end_time;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_File file;
char cbuf[BUFSIZE];
for(int i = 0; i < BUFSIZE; i++)
{
cbuf[i] = 'a' + i;
}
if(err = MPI_Init(&argc, &argv))
{
printf("%s \n", "Error! MPI is halted!");
MPI_Abort(comm, err);
}
MPI_Comm_size(comm, &size);
MPI_Comm_rank(comm, &rank);
if(rank == 0)
{
start_time = MPI_Wtime();
}
err = MPI_File_open(comm, "testfile", MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &file);
if(err != MPI_SUCCESS)
{
printf("Error %d! Can't open the file!\n", err);
MPI_Abort(comm, err);
return EXIT_FAILURE;
}
err = MPI_File_set_view(file, (MPI_Offset) (rank * BUFSIZE * sizeof(char)), MPI_CHAR, MPI_CHAR, "native", MPI_INFO_NULL);
if(err != MPI_SUCCESS)
{
printf("%s \n", "Error! Can't set the view!");
MPI_Abort(comm, err);
return EXIT_FAILURE;
}
err = MPI_File_write(file, cbuf, BUFSIZE, MPI_CHAR, MPI_STATUSES_IGNORE);
if(err != MPI_SUCCESS)
{
printf("%s \n", "Error! Problems with writing!");
MPI_Abort(comm, err);
return EXIT_FAILURE;
}
MPI_File_close(&file);
if(rank == 0)
{
end_time = MPI_Wtime();
printf("Time elapsed : %f seconds", (end_time - start_time) * 1000);
}
MPI_Finalize();
return EXIT_SUCCESS;
}
I'm trying to write some symbols to a file with MPI. When I do that, I get an errorcode of 288 and the file can't be opened. I used command line: mpiexec -n 10 myapp.exe. I was searching for the errorcode but didn't find anything at all.
Go one step further. Your error code doesn't mean anything by itself. But, you can feed that code to MPI_Error_string and get something more human readable. I have this function in every MPI-IO code I write:
static void handle_error(int errcode, char *str)
{
char msg[MPI_MAX_ERROR_STRING];
int resultlen;
MPI_Error_string(errcode, msg, &resultlen);
fprintf(stderr, "%s: %s\n", str, msg);
MPI_Abort(MPI_COMM_WORLD, 1);
}
And then define this macro:
#define MPI_CHECK(fn) { int errcode; errcode = (fn);\
if (errcode != MPI_SUCCESS) handle_error (errcode, #fn ); }
So I can call routines like this:
CHECK(MPI_File_open(comm, "testfile",
MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &file) );
I am currently making a client and server program in the Unix Environment. I have made it so the client is able to upload the contents of a file to the client. I am now in the process of adding options and error handlers to the server example the client must enter the file name. To do this i wanted to send a message saying OK if all the options checked out however if i do this it seems to cause my file reading and sending to go Crazy and I have no idea why.
I have uploaded the functions for doing this
Client Code
int putFile (char path[256], int fd)
{
char mystring[1000];
char buffer[100];
int i , n;
FILE * pFile;
n = read(fd,buffer,100);
printf("%s", buffer);
if (strcmp(buffer, "OK") == 0)
{
pFile = fopen(path, "r");
if(pFile != NULL)
{
while(fgets(mystring, sizeof(mystring), pFile) != NULL)
{
//fputs(mystring, fd);
write(fd,mystring,strlen(mystring));
}
}
else
{
printf("Invalid File or Address \n");
}
fclose(pFile);
}
else
{
printf("%s \n", buffer);
}
}
Server Code for reading the socket
int putRequest(int fd, char buf[], char str[])
{
char data[256];
int number;
char * ptr;
char results[100];
int total = 0;
char *arguments[1024];
char temp[10];
int i;
ptr = strtok(buf," ");
while (ptr != NULL)
{
char * temp;
temp = (char *)malloc(sizeof(ptr));
temp = ptr;
arguments[total] = temp;
total++;
ptr = strtok (NULL, " ");
}
if(total == 1)
{
strcat(str, "Invaild Arguments \n");
return 1;
}
write(fd, "OK", 256);
FILE * pFile;
pFile = fopen ("myfile.txt","w");
if (pFile!=NULL)
{
while(read(fd, data, 256) != NULL)
{
fputs(data, pFile);
}
fclose (pFile);
}
else
{
strcat(str, "Invaild File");
return 0;
}
strcat(str, "Done");
return 1;
}
Thanks in advance and just post something if you need to see more code. I just placed the code which should be causing the problem.
I am trying to record the time between 'http request' package and 'http response' package.
I write an socket client using winsock. The code is below
if (send(sock, request.c_str(), request.length(), 0) != request.length())
die_with_error("send() sent a different number of bytes than expected");
// Record the time of httpRequestSent
::QueryPerformanceCounter(&httpRequestSent);
::QueryPerformanceFrequency(&frequency);
//get response
response = "";
resp_leng= BUFFERSIZE;
http_leng= 381;
while(resp_leng==BUFFERSIZE||http_leng>0)
{
resp_leng= recv(sock, (char*)&buffer, BUFFERSIZE, 0);
http_leng= http_leng - resp_leng;
if (resp_leng>0)
response+= string(buffer).substr(0,resp_leng);
//note: download lag is not handled in this code
}
::QueryPerformanceCounter(&httpResponseGot);
//display response
cout << response << endl;
// Display the HTTP duration
httpDuration = (double)(httpResponseGot.QuadPart - httpRequestSent.QuadPart) / (double)frequency.QuadPart;
printf("The HTTP duration is %lf\n", httpDuration);
The code works nicely except one situation: HTTP Retransmition. I used wireshark to monitor packages and found out once there is a retransmition the code seems block on recv(), but cannot get any data from the socket buffer. I wonder why would this happen. Could somebody explain the reasons?
Any help will be appreciated.
Here is a second answer with more dynamic buffer handling and more error checking:
void send_data(SOCKET sock, void *data, unsigned int data_len)
{
unsigned char *ptr = (unsigned char*) data;
while (data_len > 0)
{
int num_to_send = (int) std::min(1024*1024, data_len);
int num_sent = send(sock, ptr, num_to_send, 0);
if (num_sent < 0)
{
if ((num_sent == SOCKET_ERROR) && (WSAGetLastError() == WSAEWOULDBLOCK))
continue;
die_with_error("send() failed");
}
if (num_sent == 0)
die_with_error("socket disconnected");
ptr += num_sent;
data_len -= num_sent;
}
}
unsigned int recv_data(SOCKET sock, void *data, unsigned int data_len, bool error_on_disconnect = true)
{
unsigned char *ptr = (unsigned char*) data;
unsigned int total = 0;
while (data_len > 0)
{
int num_to_recv = (int) std::min(1024*1024, data_len);
int num_recvd = recv(sock, ptr, num_to_recv, 0);
if (num_recvd < 0)
{
if ((num_recvd == SOCKET_ERROR) && (WSAGetLastError() == WSAEWOULDBLOCK))
continue;
die_with_error("recv() failed");
}
if (num_recvd == 0)
{
if (error_on_disconnect)
die_with_error("socket disconnected");
break;
}
ptr += num_recvd;
datalen -= num_recvd;
total += num_recvd;
}
while (true);
return total;
}
std::string recv_line(SOCKET sock)
{
std::string line;
char c;
do
{
recv_data(sock, &c, 1);
if (c == '\r')
{
recv_data(sock, &c, 1);
if (c == '\n')
break;
line += '\r';
}
else if (c == '\n')
break;
line += c;
}
return line;
}
void recv_headers(SOCKET sock, std::vector<std::string> *hdrs)
{
do
{
std::string line = recv_line(sock);
if (line.length() == 0)
return;
if (hdrs)
hdrs->push_back(line);
}
while (true);
}
unsigned int recv_chunk_size(SOCKET sock)
{
std::string line = recv_line(sock);
size_t pos = line.find(";");
if (pos != std::string::npos)
line.erase(pos);
char *endptr;
unsigned int value = strtoul(line.c_str(), &endptr, 16);
if (*endptr != '\0')
die_with_error("bad Chunk Size received");
return value;
}
std::string find_header(const std::vector<std::string> &hrds, const std::string &hdr_name)
{
std::string value;
for(size_t i = 0; i < hdrs.size(); ++i)
{
const std::string hdr = hdrs[i];
size_t pos = hdr.find(":");
if (pos != std::string::npos)
{
if (hdr.compare(0, pos-1, name) == 0)
{
pos = hdr.find_first_not_of(" ", pos+1);
if (pos != std::string::npos)
return hdr.substr(pos);
break;
}
}
}
return "";
}
{
// send request ...
std::string request = ...;
send_data(sock, request.c_str(), request.length());
// Record the time of httpRequestSent
::QueryPerformanceCounter(&httpRequestSent);
::QueryPerformanceFrequency(&frequency);
// get response ...
std::vector<std::string> resp_headers;
std::vector<unsigned char> resp_data;
recv_headers(sock, &resp_headers);
std::string transfer_encoding = find_header(resp_headers, "Transfer-Encoding");
if (transfer_encoding.find("chunked") != std::string::npos)
{
unsigned int chunk_len = recv_chunk_size(sock);
while (chunk_len != 0)
{
size_t offset = resp_data.size();
resp_data.resize(offset + chunk_len);
recv_data(sock, &resp_data[offset], chunk_len);
recv_line(sock);
chunk_len = recv_chunk_size(sock);
}
recv_headers(sock, NULL);
}
else
{
std::string content_length = find_header(resp_headers, "Content-Length");
if (content_length.length() != 0)
{
char *endptr;
unsigned int content_length_value = strtoul(content_length.c_str(), &endptr, 10);
if (*endptr != '\0')
die_with_error("bad Content-Length value received");
if (content_length_value > 0)
{
resp_data.resize(content_length_value);
recv_data(sock, &resp_data[0], content_length_value);
}
}
else
{
unsigned char buffer[BUFFERSIZE];
do
{
unsigned int buffer_len = recv_data(sock, buffer, BUFFERSIZE, false);
if (buffer_len == 0)
break;
size_t offset = resp_data.size();
resp_data.resize(offset + buffer_len);
memcpy(&resp_data[offset], buffer, buffer_len);
}
while (true)
}
}
::QueryPerformanceCounter(&httpResponseGot);
// process resp_data as needed
// may be compressed, encoded, etc...
// Display the HTTP duration
httpDuration = (double)(httpResponseGot.QuadPart - httpRequestSent.QuadPart) / (double)frequency.QuadPart;
printf("The HTTP duration is %lf\n", httpDuration);
}
You are not doing adequate error checking on the calls to send() and recv(). Try something like this instead:
char *req_ptr = request.c_str();
int req_leng = request.length();
int req_index = 0;
do
{
int req_sent = send(sock, req_ptr, req_leng, 0);
if (req_sent < 1)
{
if ((req_sent == SOCKET_ERROR) && (WSAGetLastError() == WSAEWOULDBLOCK))
continue;
die_with_error("send() failed");
}
req_ptr += req_sent;
req_leng -= req_sent;
}
while (req_leng > 0);
// Record the time of httpRequestSent
::QueryPerformanceCounter(&httpRequestSent);
::QueryPerformanceFrequency(&frequency);
//get response
std::string response;
int resp_leng = BUFFERSIZE;
int http_leng = -1;
bool http_leng_needed = true;
do
{
if (http_leng_needed)
{
std::string::size_type pos = response.find("\r\n\r\n");
if (pos != std::string::npos)
{
std::string resp_hdrs = response.substr(0, pos);
// look for a "Content-Length" header to see
// if the server sends who many bytes are
// being sent after the headers. Note that
// the server may use "Transfer-Encoding: chunked"
// instead, which has no "Content-Length" header...
//
// I will leave this as an excercise for you to figure out...
http_leng = ...;
// in case body bytes have already been received...
http_leng -= (response.length() - (pos+4));
http_leng_needed = false;
}
}
if (http_leng_needed)
resp_leng = BUFFERSIZE;
else
resp_leng = min(http_leng, BUFFERSIZE);
if (resp_leng == 0)
break;
resp_leng = recv(sock, buffer, resp_leng, 0);
if (resp_leng < 1)
{
if ((resp_leng == SOCKET_ERROR) && (WSAGetLastError() == WSAEWOULDBLOCK))
continue;
die_with_error("send() failed");
}
response += string(buffer, resp_leng);
if (!http_leng_needed)
http_leng -= resp_leng;
}
while ((http_leng_needed) || (http_leng > 0));
::QueryPerformanceCounter(&httpResponseGot);
//display response
cout << response << endl;
// Display the HTTP duration
httpDuration = (double)(httpResponseGot.QuadPart - httpRequestSent.QuadPart) / (double)frequency.QuadPart;
printf("The HTTP duration is %lf\n", httpDuration);
With this said, the "correct" way to handle HTTP in general is to read the inbound data line by line, rather than buffer by buffer, until you encounter the end of the response headers, then you can read the rest of the data buffer by buffer based on the data length indicated by the headers.