HTTP POST in Arduino MKGSM to Eventhub - http

I'm using the Arduino library MKGSM in order to make an HTTP POST to my Azure Eventhub. I am confident with the parameters provided as I have tried them by hand with curl and the HTTP POST did work, so the problem is for sure in my Android syntax. Here is my approach:
#include <MKRGSM.h>
#include "arduino_secrets.h"
// Please enter your sensitive data in the Secret tab or arduino_secrets.h
// PIN Number
const char PINNUMBER[] = SECRET_PINNUMBER;
// APN data
const char GPRS_APN[] = SECRET_GPRS_APN;
const char GPRS_LOGIN[] = SECRET_GPRS_LOGIN;
const char GPRS_PASSWORD[] = SECRET_GPRS_PASSWORD;
// initialize the library instance
GSMSSLClient client;
GPRS gprs;
GSM gsmAccess;
// URL, path and port (for example: arduino.cc)
char server[] = "<namespace>.servicebus.windows.net";
char path[] = "/<myeventhubname>/messages";
int port = 443; // port 443 is the default for HTTPS
void setup() {
// initialize serial communications and wait for port to open:
Serial.begin(9600);
while (!Serial) {
; // wait for serial port to connect. Needed for native USB port only
}
Serial.println("Starting Arduino web client.");
// connection state
boolean connected = false;
// After starting the modem with GSM.begin()
// attach the shield to the GPRS network with the APN, login and password
while (!connected) {
if ((gsmAccess.begin(PINNUMBER) == GSM_READY) &&
(gprs.attachGPRS(GPRS_APN, GPRS_LOGIN, GPRS_PASSWORD) == GPRS_READY)) {
connected = true;
} else {
Serial.println("Not connected");
delay(1000);
}
}
Serial.println("connecting...");
// if you get a connection, report back via serial:
if (client.connect(server, port)) {
Serial.println("connected");
// Make a HTTP request:
client.print("POST ");
client.print(server);
client.print(path);
client.println(" HTTP/1.1");
client.println("Authorization: SharedAccessSignature sr=https%3A%2F%2F<namespace>.servicebus.windows.net%2F<myeventhubname>&sig=<mysig>&se=<myse>&skn=RootManageSharedAccessKey");
client.println();
client.println("{\"HELLO\"}");
} else {
// if you didn't get a connection to the server:
Serial.println("connection failed");
}
}
void loop() {
// if there are incoming bytes available
// from the server, read them and print them:
if (client.available()) {
char c = client.read();
Serial.print(c);
}
// if the server's disconnected, stop the client:
if (!client.available() && !client.connected()) {
Serial.println();
Serial.println("disconnecting.");
client.stop();
// do nothing forevermore:
for (;;)
;
}
}
Basically, the original example was a GET that obtained the ascii symbol of Arduino and I am guessing if it is possible to use the same code to make an http post (in my case, to an Eventhub).
Update:
After changing GSM gsmAccess to GSM gsmAccess(true) I start getting an output which is unreadable. I attach it bellow:
AT
OK
AT+IPR=921600
OK
AT
OK
AT+UPSV=3
OK
AT+CPIN?
ERROR
AT+CPIN?
ERROR
AT+CPIN?
ERROR
AT+CPIN?
+CPIN: READY
OK
AT+CMGF=1
OK
AT+UDCONF=1,1
OK
AT+CTZU=1
OK
AT+UDTMFD=1,2
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
+UMWI: 0,1
+UMWI: 0,2
+UMWI: 0,3
+UMWI: 0,4
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,0
OK
AT+CREG?
+CREG: 0,1
OK
AT+UCALLSTAT=1
OK
AT+CGATT=1
OK
AT+UPSD=0,1,""
OK
AT+UPSD=0,6,3
OK
AT+UPSD=0,2,""
OK
AT+UPSD=0,3,""
OK
AT+UPSD=0,7,"0.0.0.0"
OK
AT+UPSDA=0,3
OK
AT+UPSND=0,8
+UPSND: 0,8,1
OK
AT+USOCR=6
+USOCR: 0
OK
AT+USOCO=0,"<server>.servicebus.windows.net",443
OK
connected
AT+USOWR=0,5,"504F535420"
+USOWR: 0,5
OK
AT+USOWR=0,60,"2F6D796576656E746875622F6D657373616765733F74696D656F75743D3630266170692D76657273696F6E3D323031342D303120485454502F312E31"
+USOWR: 0,60
OK
AT+USOWR=0,193,"417574686F72697A6174696F6E3A205368617265644163636573735369676E61747572652073723D687474707325334125324625324665666F7230312E736572766963656275732E77696E646F77732E6E65742532466D796576656E74687562267369673D683977624C78673467306E50764E6E347977696F462532426C623244446E6556306863353833757A496B7462302533442673653D3135373633313930323826736B6E3D526F6F744D616E6167655368617265644163636573734B6579"
+USOWR: 0,193
OK
AT+USOWR=0,59,"436F6E74656E742D547970653A206170706C69636174696F6E2F61746F6D2B786D6C3B747970653D656E7472793B636861727365743D7574662D38"
+USOWR: 0,59
OK
AT+USOWR=0,2,"0D0A"
+USOWR: 0,2
OK
AT+USOWR=0,35,"486F73743A2065666F7230312E736572766963656275732E77696E646F77732E6E6574"
+USOWR: 0,35
OK
AT+USOWR=0,2,"0D0A"
+USOWR: 0,2
OK
AT+USOWR=0,2,"0D0A"
+USOWR: 0,2
OK
AT+USOWR=0,2,"0D0A"
+USOWR: 0,2
OK
AT+USOWR=0,22,"7B2276616C7565223A2048656C6C6F20576F726C647D"
+USOWR: 0,22
OK
AT+USOWR=0,2,"0D0A"
+USOWR: 0,2
OK
disconnecting.
AT+USOCL=0
OK

Disclaimer - this is a suggestion only. I do not have the equipment or accounts to test/verify this.
Have you tried adding "https://" between POST and server ?
there might also be some parameters required before "HTTP/1.1"
namely
?timeout=60&api-version=2014-01
also,
client.println("Authorization: SharedAccessSignature sr=https%3A%2F%2F<namespace>.servicebus.windows.net%2F<myeventhubname>&sig=<mysig>&se=<myse>&skn=RootManageSharedAccessKey");
might not be right. try removing the https%3A%2F%2F in front of < namespace >
finally, you may also need to send :
Content-Type: application/atom+xml;type=entry;charset=utf-8
Host: your-namespace.servicebus.windows.net
in all:
// Make a HTTP request:
client.print("POST ");
client.print("https://");
client.print(server);
client.print(path);
client.print("?timeout=60&api-version=2014-01");
client.println(" HTTP/1.1");
client.println("Authorization: SharedAccessSignature sr=<namespace>.servicebus.windows.net/<myeventhubname>&sig=<mysig>&se=<myse>&skn=RootManageSharedAccessKey");
client.println("Content-Type: application/atom+xml;type=entry;charset=utf-8");
client.println("Host: <namespace>.servicebus.windows.net");
client.println();
client.println("{\"HELLO\"}");
} else {
.......
Cheers
So with the addition of the output by qwerty, I do not have much more to add yet.
The modem is not (it appears) sending out everything that is outlined in the code snippet I provided
If you scroll down the output until you reach
AT+USOCO=0,"<server>.servicebus.windows.net",443
the POST sequence starts next.
AT+USOWR=0,5,"504F535420"
sends "POST"
the next output from the modem skips the "https://" and server (ie it's as if the following two lines are missing from the code)
client.print("https://");
client.print(server);
but instead skips to sending out the path, timeout and HTTP/1.1. The rest of the output is also missing CR/LF from several lines.
It behaves as if the code in fact was
// Make a HTTP request:
client.print("POST ");
client.print(path);
client.print("?timeout=60&api-version=2014-01");
client.print(" HTTP/1.1");
client.print("Authorization: SharedAccessSignature sr=<namespace>.servicebus.windows.net/<myeventhubname>&sig=<mysig>&se=<myse>&skn=RootManageSharedAccessKey");
client.println("Content-Type: application/atom+xml;type=entry;charset=utf-8");
client.println("Host: <namespace>.servicebus.windows.net");
client.println();
client.println();
client.println("{\"value\": Hello World}");
I'm not sure what is causing this behaviour, but introducing delays might help
(Additionally, if server and path are constant strings, not likely to change for you application, you can concatenate the first 6 client.print() statements)
Giving you:
// Make a HTTP request:
client.print("POST https://<myserver>/<mypath>/?timeout=60&api-version=2014-01 HTTP/1.1");
client.println();
delay(5);
client.print("Authorization: SharedAccessSignature sr=<namespace>.servicebus.windows.net/<myeventhubname>&sig=<mysig>&se=<myse>&skn=RootManageSharedAccessKey");
client.println();
delay(5);
client.print("Content-Type: application/atom+xml;type=entry;charset=utf-8");
client.println();
delay(5);
client.print("Host: <namespace>.servicebus.windows.net");
client.println();
delay(5);
client.println();
delay(5);
client.print("{\"value\": Hello World}");
client.println();
}

Related

ESP8266 constantly resetting, but blink script works

Hello I have been using the Wemos D1 mini lite for awhile now with this ArduCAM setup. Everything was working perfectly, but now my esp8266 is constantly resetting.
I unplugged the ArduCAM and saw I can ran the blink script successfully, but when I try to run a script where it attempts to connect to wifi, it constantly resets.
Here is my code:
#include <ESP8266WiFi.h> // Include the Wi-Fi library
const char* ssid = "ssid"; // The SSID (name) of the Wi-Fi network you want to connect to
const char* password = ""; // The password of the Wi-Fi network
void setup() {
Serial.begin(115200); // Start the Serial communication to send messages to the computer
delay(10);
Serial.println('\n');
WiFi.begin(ssid, password); // Connect to the network
Serial.print("Connecting to ");
Serial.print(ssid); Serial.println(" ...");
int i = 0;
while (WiFi.status() != WL_CONNECTED) { // Wait for the Wi-Fi to connect
delay(1000);
Serial.print(++i); Serial.print(' ');
}
Serial.println('\n');
Serial.println("Connection established!");
Serial.print("IP address:\t");
Serial.println(WiFi.localIP()); // Send the IP address of the ESP8266 to the computer
}
void loop() { }
When I run this, I get the following:
load 0x4010f000, len 1392, room 16
tail 0
chksum 0xd0
csum 0xd0
v3d128e5c
~ld
Exception (3):
epc1=0x40100710 epc2=0x00000000 epc3=0x00000000 excvaddr=0x4006e989 depc=0x00000000
>>>stack>>>
ctx: cont
sp: 3ffffb90 end: 3fffffc0 offset: 01a0
3ffffd30: feefeffe feefeffe feefeffe feefeffe
3ffffd40: feefeffe feefeffe feefeffe 3ffffef0
3ffffd50: 0000049c 0000049c 00000020 40100900
3ffffd60: feefeffe feefeffe feefeffe feefeffe
3ffffd70: 00000002 400042db 000000fd 40100b58
3ffffd80: 40004b31 00001000 000000fd 40100274
3ffffd90: 40105ae0 feefeffe feefeffe 4022da8d
3ffffda0: 40105c9d 4022db77 3ffef25c 0000049c
3ffffdb0: 000000fd 3ffffef0 3ffef25c 4022db5a
3ffffdc0: ffffff00 55aa55aa 0000000e 00000020
3ffffdd0: 00000020 00000078 00000012 aa55aa55
3ffffde0: 000000ff 4022e05a 3ffef25c 3ffef25c
3ffffdf0: 000000ff 00000119 00000119 40100640
3ffffe00: 40105c9d 00000001 3ffef26c 4022e27a
3ffffe10: 00000005 3ffef25c 000000ff 3ffffef0
3ffffe20: 3fffff10 3ffef293 0000000e 00000020
3ffffe30: 3ffef31c 3fffff51 00000001 4022e32a
3ffffe40: 3ffffef0 40239860 00000000 00000000
3ffffe50: 3ffef65c 3fffff10 3fff5594 4022e2f9
3ffffe60: 3ffef25c 4022e360 3ffe84d4 3ffe8642
3ffffe70: 40201946 3ffe8642 3ffe8663 4020189b
3ffffe80: 76696e55 69737265 6f207974 61572066
3ffffe90: 6e696873 6e6f7467 3ffffe00 40204cbc
3ffffea0: 3ffee400 00000000 3ffffe60 40204ef9
3ffffeb0: 00000f98 000001f3 000001f3 40100640
3ffffec0: 000006e8 000000dd 00000014 3ffeebc4
3ffffed0: 007a1200 44e0a632 00000000 401008cb
3ffffee0: adb5c801 fe20b34c feefeffe 00000100
3ffffef0: 76696e55 69737265 6f207974 61572066
3fffff00: 6e696873 6e6f7467 feefef00 feefeffe
3fffff10: 40203500 3ffeefcc 3ffeef4c 402035af
3fffff20: 0000001c 0001c200 00000000 00000000
3fffff30: 00000003 40203771 ffffffff 00000001
3fffff40: 40105065 00000001 3ffee35c 3ffee3d4
3fffff50: 00000000 00000001 3ffee381 00000000
3fffff60: 00000004 00000000 3ffee328 00000001
3fffff70: 0001c200 0000001c 00000000 3ffee3d4
3fffff80: 3fffdad0 3ffee328 3ffee35c 40201073
3fffff90: feefeffe feefeffe feefeffe feefeffe
3fffffa0: feefeffe 00000000 3ffee394 402024b4
3fffffb0: feefeffe feefeffe 3ffe84f0 40100b8d
<<<stack<<<
over and over and over.
I don't understand what is happening. Did I possibly short it? It couldn't have been the ArduCAM, since this was working fine already.
edit:
I found out that the Wifi.begin(ssid,password) command causes the constant reboots. When I remove this, it doesn't reboot. How can I fix this?
I think your main loop is empty. Add while(true) in the main loop function
and it´s better to set the esp in sation mode.
WiFi.mode(WIFI_STA);

Cant send data to Firebase using AT commands

I'm using an Arduino with a GSM modem to try to send data to firebase.
However, when I try to, I get the following error:
SEND OK HTTP/1.1 400 Bad Request Server: nginx Date: Mon, 27 May 2019
22:34:09 GMT Content-Type: text/html Content-Length: 166 Connection:
close Strict-Transport-Security: max-age=31556926; includeSubDomains;
preload
400 Bad Request 400 Bad Request
nginx
CLOSED
The AT commands I issue are:
AT+QIOPEN="TCP", "drone-polution.firebaseio.com", 443
OK
CONNECT OK
AT+QISEND
>
POST /NewDB/.json
Accept: application/json
Content-Type: application/json
Content-Length: 9
{"a":"b"}
The last line is the actual payload.
Any help appreciated.
First, try this to program SAM chip to create an interface between Modem and your console PC (I found that there are good ready functions in MC20_Arduino_Interface.h which you can setup the modem.)
A simple program looks like this:
#include "MC20_Arduino_Interface.h"
// set serial port that connects to MC20
//#define serialMC20 Serial1
void setup()
{
//Begin serial comunication with Arduino and Arduino IDE (Serial Monitor)
SerialUSB.begin(115200);
while (!Serial);
//Being serial communication witj Arduino and MC20
serialMC20.begin(115200);
delay(1000);
SerialUSB.println("Setup Complete!");
}
void loop()
{
//Read MC20 output (if available) and print it in Arduino IDE Serial Monitor
if (serialMC20.available())
{
SerialUSB.write(serialMC20.read());
}
//Read Arduino IDE Serial Monitor inputs (if available) and send them to MC20
if (SerialUSB.available())
{
serialMC20.write(SerialUSB.read());
}
}
Also, I suggest you use Arduino's serial monitor for communication.
If the Modem starts successfully you will see SMS Ready and Call Ready in the serial monitor.
According to Quectel HTTP docs for a POST request:
3.2. Send POST Request to HTTP Server
AT+QIFGCNT=0
OK
AT+QICSGP=1,"CMNET" //Set APN
OK
AT+QIREGAPP //Optional
OK
AT+QIACT //Optional
OK
AT+QHTTPURL=58,30 //Set URL
CONNECT
<Input data>
//For example, input 58 bytes:
http://api.efxnow.com/DEMOWebServices2.8/Service.asmx/Echo
OK
//POST the data whose size is 18 bytes and the maximum latency time for inputting is 50s.
//It is recommended to set the latency time as long as enough to download all the data in the latency time.
AT+QHTTPPOST=18,50,10
CONNECT
//This means module enters into data mode and is ready to receive data from UART.
//For example, input 18 bytes: Message=helloworld.
OK
//This means all data has been received, and DCD is set to high.
AT+QHTTPREAD=30 //Read the response of HTTP server.
CONNECT
<Output data> //Output the response data of HTTP server to UART.
//For example, UART outputs:
<?xml version="1.0" encoding="utf-8"?>
<string xmlns="https://api.efxnow.com/webservices2.3">Message='helloworld' ASCII:104 101 108 108
111 119 111 114 108 100 </string>
OK
AT+QIDEACT //Deactivate PDP context.
DEACT OK
For example for httpbin.org/post it would become something like this:
16:45:56.416 -> AT+QIFGCNT=0
16:45:56.416 -> OK
16:46:02.918 -> AT+QICSGP=1,"mtnirancell"
16:46:02.918 -> OK
16:46:07.850 -> AT+QIREGAPP
16:46:07.850 -> OK
16:46:12.275 -> AT+QIACT
16:46:12.275 -> OK
16:46:27.467 -> AT+QHTTPURL=23,60
16:46:27.467 -> CONNECT
16:46:27.467 -> <http://httpbin.org/post>
16:46:36.965 -> OK
16:46:36.965 ->
16:46:48.786 -> AT+QHTTPPOST=18,50,10
16:46:48.786 -> CONNECT
16:46:48.786 -> <message=helloworld>
16:47:02.094 -> OK
16:47:02.094 ->
16:47:06.569 -> AT+QHTTPREAD=30
16:47:06.569 -> CONNECT
16:47:06.569 -> {
16:47:06.569 -> "args": {},
16:47:06.569 -> "data": "",
16:47:06.569 -> "files": {},
16:47:06.569 -> "form": {
16:47:06.569 -> "message": "helloworld"
16:47:06.569 -> },
16:47:06.569 -> "headers": {
16:47:06.569 -> "Accept": "*/*",
16:47:06.569 -> "Content-Length": "18",
16:47:06.569 -> "Content-Type": "application/x-www-form-urlencoded",
16:47:06.602 -> "Host": "httpbin.org",
16:47:06.602 -> "User-Agent": "QUECTEL_MODULE"
16:47:06.602 -> },
16:47:06.602 -> "json": null,
16:47:06.602 -> "origin": "*******, ********",
16:47:06.602 -> "url": "https://httpbin.org/post"
16:47:06.602 -> }
16:47:06.602 -> OK

<< "[read] I/O error: Read timed out" immediately upon sending headers

We see time-outs during some calls to external REST service from within a Spring Boot application. They do not seem to occur when we connect to the REST service directly. Debug logging on org.apache.http has given us a very peculiar aspect of the failing requests: it contains an inbound log entry '<< "[read] I/O error: Read timed out"' in the middle of sending headers - the same millisecond the first headers were sent.
How can we see an inbound 'Read timed out' a few milliseconds after sending the first headers? And why does it not immediately interrupt the request/connection with a time-out, but instead waits the full 4500ms until it times out with an exception?
Here is our production log for a failing request, redacted. Note the 4500ms delay between lines two and three. My question is about the occurrence of http-outgoing-104 << "[read] I/O error: Read timed out" at 16:55:08.258, not the first one on line 2.
16:55:12.764 Connection released: [id: 104][route: {s}-><<website-redacted>>:443][total kept alive: 0; route allocated: 0 of 2; total allocated: 0 of 20]
16:55:12.763 http-outgoing-104 << "[read] I/O error: Read timed out"
16:55:08.259 http-outgoing-104 >> "<<POST Body Redacted>>"
16:55:08.259 http-outgoing-104 >> "[\r][\n]"
16:55:08.258 http-outgoing-104: set socket timeout to 4500
16:55:08.258 Executing request POST <<Endpoint Redacted>> HTTP/1.1
16:55:08.258 Target auth state: UNCHALLENGED
16:55:08.258 Proxy auth state: UNCHALLENGED
16:55:08.258 Connection leased: [id: 104][route: {s}-><<website-redacted>>:443][total kept alive: 0; route allocated: 1 of 2; total allocated: 1 of 20]
....
16:55:08.258 http-outgoing-104 >> "POST <<Endpoint Redacted>> HTTP/1.1[\r][\n]"
16:55:08.258 http-outgoing-104 >> "Accept: text/plain, application/json, application/*+json, */*[\r][\n]"
16:55:08.258 http-outgoing-104 >> Cookie: <<Redacted>>
16:55:08.258 http-outgoing-104 >> "Content-Type: application/json[\r][\n]"
16:55:08.258 http-outgoing-104 >> "Connection: close[\r][\n]"
16:55:08.258 http-outgoing-104 >> "X-B3-SpanId: <<ID>>[\r][\n]"
16:55:08.258 http-outgoing-104 << "[read] I/O error: Read timed out"
16:55:08.258 http-outgoing-104 >> "X-Span-Name: https:<<Endpoint Redacted>>[\r][\n]"
16:55:08.258 http-outgoing-104 >> "X-B3-TraceId: <<ID>>[\r][\n]"
16:55:08.258 http-outgoing-104 >> "X-B3-ParentSpanId: <<ID>>[\r][\n]"
16:55:08.258 http-outgoing-104 >> "Content-Length: 90[\r][\n]"
16:55:08.258 http-outgoing-104 >> "User-Agent: Apache-HttpClient/4.5.3 (Java/1.8.0_172)[\r][\n]"
16:55:08.258 http-outgoing-104 >> "Cookie: <<Redacted>>"
16:55:08.258 http-outgoing-104 >> "Host: <<Host redacted>>[\r][\n]"
16:55:08.258 http-outgoing-104 >> "Accept-Encoding: gzip,deflate[\r][\n]"
16:55:08.258 http-outgoing-104 >> "X-B3-Sampled: 1[\r][\n
Update 1: a second occurrence:
In another request that timed out the same behavior roughly occurs, but the timeout message is logged even before sending headers and eventually receiving the actual timeout. Note: this request is actually older, after it I have configured the request to include 'Connection: close' to circumvent a firewall dropping the connection under 'Keep Alive'.
19:28:08.102 http-outgoing-36 << "[read] I/O error: Read timed out"
19:28:08.102 http-outgoing-36: Shutdown connection
19:28:08.102 http-outgoing-36: Close connection
19:28:03.598 http-outgoing-36 >> "Connection: Keep-Alive[\r][\n]"
19:28:03.598 http-outgoing-36 >> "Content-Type: application/json;charset=UTF-8[\r][\n]"
...
19:28:03.598 http-outgoing-36 >> "Accept-Encoding: gzip,deflate[\r][\n]"
...
19:28:03.597 http-outgoing-36 >> Cookie: ....
19:28:03.597 http-outgoing-36 >> Accept-Encoding: gzip,deflate
19:28:03.597 http-outgoing-36 >> User-Agent: Apache-HttpClient/4.5.3 (Java/1.8.0_172)
19:28:03.596 Connection leased: [id: 36][route: {s}-><< Site redacted >>:443][total kept alive: 0; route allocated: 1 of 2; total allocated: 1 of 20]
19:28:03.596 http-outgoing-36: set socket timeout to 4500
19:28:03.596 Executing request POST HTTP/1.1
19:28:03.596 Target auth state: UNCHALLENGED
19:28:03.596 http-outgoing-36 << "[read] I/O error: Read timed out"
19:28:03.594 Connection request: [route: {s}-><< Site redacted >>:443][total kept alive: 1; route allocated: 1 of 2; total allocated: 1 of 20]
19:28:03.594 Auth cache not set in the context
Update 2: added HttpClientBuilder configuration
RequestConfig.Builder requestBuilder = RequestConfig.custom()
.setSocketTimeout(socketTimeout)
.setConnectTimeout(connectTimeout);
CloseableHttpClient httpClient = HttpClientBuilder.create()
.setDefaultRequestConfig(requestBuilder.build())
.build();
HttpComponentsClientHttpRequestFactory rf = new HttpComponentsClientHttpRequestFactory(httpClient);
return new RestTemplate(rf);

libcurl write callback is not called for post http message

Intro
I'm sending POST request to server which responses with chunked messages.
So I'm trying to make writecallback being called on each received chunked http message.
Code
#include <iostream>
#include <string>
#include <curl/curl.h>
using namespace std;
size_t write_callback(char *d, size_t n, size_t l, void *userp)
{
cerr << ""--- Called once" << endl;
return n*l;
}
string xml_msg()
{
return "<<some request data>>";
}
curl_slist* get_header(size_t content_length)
{
auto list = curl_slist_append(nullptr, "<<protocol version>>");
list = curl_slist_append(list, "Content-Type: text/xml");
list = curl_slist_append(list, "Content-Length: " + content_length);
return list;
}
void main()
{
auto xml = xml_msg();
curl_global_init(CURL_GLOBAL_ALL);
auto curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_URL, "<<server url>>");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, nullptr);
curl_easy_setopt(curl, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 6.0)");
curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
curl_easy_setopt(curl, CURLOPT_USERPWD, "<<user credentials>>");
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, get_header(xml.size()));
curl_easy_setopt(curl, CURLOPT_POST, 1);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, xml.data());
curl_easy_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, 0L);
curl_easy_perform(curl);
curl_easy_cleanup(curl);
curl_global_cleanup();
}
Verbose log
* STATE: INIT => CONNECT handle 0x15c4de0; line 1422 (connection #-5000)
* Added connection 0. The cache now contains 1 members
* STATE: CONNECT => WAITRESOLVE handle 0x15c4de0; line 1458 (connection #0)
* Trying xxx.xxx.xxx.xxx...
* TCP_NODELAY set
* STATE: WAITRESOLVE => WAITCONNECT handle 0x15c4de0; line 1539 (connection #0)
* Connected to <<host>> (xxx.xxx.xxx.xxx) port 80 (#0)
* STATE: WAITCONNECT => SENDPROTOCONNECT handle 0x15c4de0; line 1591 (connection #0)
* Marked for [keep alive]: HTTP default
* STATE: SENDPROTOCONNECT => PROTOCONNECT handle 0x15c4de0; line 1605 (connection #0)
* STATE: PROTOCONNECT => DO handle 0x15c4de0; line 1626 (connection #0)
* Server auth using Basic with user '<<credentials>>'
> POST <<URL>>
Host: <<host>>
Authorization: Basic <<base64 credentials>>
User-Agent: Mozilla/4.0 (compatible; MSIE 6.0)
Accept: */*
Content-Type: text/xml
Content-Length: 204
* upload completely sent off: 204 out of 204 bytes
* STATE: DO => DO_DONE handle 0x15c4de0; line 1688 (connection #0)
* STATE: DO_DONE => WAITPERFORM handle 0x15c4de0; line 1813 (connection #0)
* STATE: WAITPERFORM => PERFORM handle 0x15c4de0; line 1823 (connection #0)
* HTTP 1.1 or later with persistent connection, pipelining supported
< HTTP/1.1 200 OK
< Date: Tue, 08 May 2018 12:29:49 GMT
* Server is not blacklisted
< Server: <<server>>
< Expires: Thu, 01 Jan 1970 00:00:00 GMT
< Content-Language: en-US
< Cache-Control: no-cache, no-store
< Pragma: no-cache
< Content-Type: application/xml;charset=UTF-8
< Set-Cookie: <<cookie>>
< Transfer-Encoding: chunked
<
--- Called once
* STATE: PERFORM => DONE handle 0x15c4de0; line 1992 (connection #0)
* multi_done
* Connection #0 to <<server>> left intact
Problem
Writecallback has been called when connection had been closed by server due to timeout with FIN tcp packet instead of moment when chunked http response has been received.
It is about 30 secs interval between this events.
Question
What am I doing wrong?
Update 1
Server returns a tcp segment with PUSH flag and http message with chunked transfer encoding containing XML. Message ends with CLRF. Meanwhile Win API socket does not allow to read it and select() returns 0, which means that there is nothing to read/write on this socket.
After 30 secs delay before closing connection due to heartbeat timeout (that is internal implementation of the server), server sends finalizing http message with chunked transfer encoding, which contains 0 and CLRF. After that message select() displays new socket state and libcurl calls write callback with returning chunked message content.
That is what I see after debuging libcurl. I need to find out the way to get chunked http message returned by libcurl once it is received, not after getting final http message.
Ok, I was able to find out that the problem is with Win Api sockets. On linux builds libcurl calls writecallback right after receiving chuncked message. I'm not sure how to fix that issue with Win builds, but at least I found rootcause of problem.

Puzzled about the `telnet localhost` and `telnet 0.0.0.0`

I wrote a simple GO program which listens to 0.0.0.0:9999 and 127.0.0.1:9999:
func main() {
go bind("0.0.0.0:9999", "111 ")
go func() {
time.Sleep(2 * time.Second)
bind("127.0.0.1:9999", "222 ")
}()
time.Sleep(time.Hour)
}
func bind(address string, content string) {
fmt.Println("-------------", address, "-----------------")
listener, err := net.Listen("tcp", address)
if err != nil {
panic(err)
return
}
fmt.Println(listener.Addr().String())
conn, _ := listener.Accept()
for {
_, err := conn.Write([]byte(content))
if err != nil {
panic(err)
}
time.Sleep(1 * time.Second)
}
}
The meaning of the code:
It binds two addresses, and gives different responses to the clients of them
binding "0.0.0.0:9999": will send "111 " repeat to client
binding "127.0.0.1:9999": will send "222 " repeat to client
And then I use telnet to try different addresses, and the responses are:
telnet 127.0.0.1 9999: 222 (OK)
telnet localhost 9999: 111 (WHY?!)
telnet 0.0.0.0 9999: 222 (WHY?!)
telnet <my-internal-ip> 9999: 111 (OK)
I'm quite confused about some of them:
telnet localhost 9999: 111 (WHY?!)
localhost should point to 127.0.0.1, so I think it's same to telnet 127.0.0.1 9999 and the response should be 222, but the actual one is 111
telnet 0.0.0.0 9999: 222 (WHY?!)
I think 0.0.0.0 is not same to 127.0.0.1, I expect to get response of 111, but get 222
I also have a demo project: https://github.com/golang-demos/go-bind-0.0.0.0-127.0.0.1-demo
Update: My os is OSX
Both localhost and 0.0.0.0 are resolved to 127.0.0.1 by the OS
$ ping 0.0.0.0
PING 0.0.0.0 (127.0.0.1) 56(84) bytes of data.
64 bytes from 127.0.0.1: icmp_seq=1 ttl=64 time=0.024 ms
$ping localhost
PING localhost (127.0.0.1) 56(84) bytes of data.
64 bytes from localhost (127.0.0.1): icmp_seq=1 ttl=64 time=0.035 ms`
localhost could resolve to something else depending on /etc/hosts file.
An excellent explanation for Linux ping 0.0.0.0 behavior is here.

Resources