Any solution for file upload problem on Arduino ESP32 via http - http

Question

First the problem.
The user can upload file from the web with ajax. If the file is relatively big, the uploading takes a while. If the user's connection is lost or something happens during the uploading process, the file is going to be damaged or empty.
How should I secure the upload process so the file remains the same if it fails for some reason?
I'm using the following libraries on the Arduino ESP32:
ESPAsyncWebServer
LITTLEFS
I have a basic file upload handler on my esp32 which looks like this:
server.on("/uploading", HTTP_POST, [](AsyncWebServerRequest * request) {
}, handleFileUpload);
void handleFileUpload(AsyncWebServerRequest * request, String filename,size_t index, uint8_t *data, size_t len, bool final) {
if (!index) {
if (!filename.startsWith("/"))
filename = "/" + filename;
if (LITTLEFS.exists(filename)) {
LITTLEFS.remove(filename);
}
uploadFile = LITTLEFS.open(filename, "w");
}
for (size_t i = 0; i < len; i++) {
uploadFile.write(data[i]);
}
if (final) {
uploadFile.close();
if(filename == "/myHomeProgram.json"){initProgram = true;}
AsyncWebServerResponse *response = request->beginResponse(200, "text/plain", "File Uploaded;"+filename);
response->addHeader("Access-Control-Allow-Origin","*");
request->send(response);
}
}
This is working pretty well, the files are uploaded correctly 99% of the cases, but if it fails I lost the file data, or if some other part of the program wants to open the same file it fails too.
Should I write to a temporary file and after if it succeeded write the content to the intended file somehow?
Here is an example from client ( JS ) side:
// Example call:
saveFile(JSON.stringify(places),"/myHomeProgram.json","application/json");
function saveFile(data, filename, type) {
var file = new Blob([data], {type: type});
form = new FormData();
form.append("blob", file, filename);
$.ajax({
url: '/uploading',
type: 'POST',
data: form,
processData: false,
contentType: false
}).done(function(resp){
var response = resp.split(";");
$(".saveIconGraph").removeClass("fas fa-spinner fa-spin");
$(".saveIconGraph").addClass("far fa-save");
if(response[1] == "/myHomeProgram.json"){
toast("success","saveOk","progInfo",3500);
showSaved();
setTimeout(() => {
$("#saveMe").fadeOut( "slow", function() {
showSave();
});
}, 1000);
initPlaces();
}
}).fail(function(resp){
var response = resp.split(";");
$(".saveIconGraph").removeClass("fas fa-spinner fa-spin");
$(".saveIconGraph").addClass("far fa-save");
if(response[1] == "/myHomeProgram.json"){
toast("error","saveNotOk","progInfo",3500);
showSaveError();
$("#saveMeBtn").addClass("shakeEffect");
setTimeout(() => {
$("#saveMeBtn").removeClass("shakeEffect");
showSave();
}, 4500);
}
});
}
I could save the file in a temporary char variable before write, and on the final I could match the size of the file and the temporary variable size and if it is not the same, roll back to the previous. Is this manageable?
Something like this:
String uploadTemp = "";
inline boolean saveFileToTemp(String fileName){
uploadTemp = "";
File f = LITTLEFS.open(fileName, "r");
if (!f) {
f.close();
return false;
}else{
for (int i = 0; i < f.size(); i++){
uploadTemp += (char)f.read();
}
}
f.close();
return true;
}
inline boolean revertBackFile(String fileName){
File g = LITTLEFS.open(fileName, "w");
if (!g) {
g.close();
return false;
}else{
g.print(uploadTemp);
}
g.close();
return true;
}
inline boolean matchFileSizes(String fileName,boolean isFileExists){
boolean isCorrect = false;
if(isFileExists){
File writedFile = LITTLEFS.open(fileName, "w");
if( writedFile.size() == uploadTemp.length()){
isCorrect = true;
}else{
isCorrect = false;
}
writedFile.close();
return isCorrect;
}else{
return true;
}
}
void handleFileUpload(AsyncWebServerRequest * request, String filename,size_t index, uint8_t *data, size_t len, bool final) {
String webResponse;
boolean error = false,isFileExists = false;
if (!index) {
if (!filename.startsWith("/"))
filename = "/" + filename;
if (LITTLEFS.exists(filename)) {
isFileExists = true;
// Save the file to a temporary String if it success we continue.
if( saveFileToTemp(filename) ){
LITTLEFS.remove(filename);
}else{
// If the file save was fail we abort everything.
webResponse = "File NOT Uploaded " + filename;
final = true;
error = true;
}
}
if( !error ){
uploadFile = LITTLEFS.open(filename, "w");
}
}
if( !error ){
// Copy content to the actual file
for (size_t i = 0; i < len; i++) {
uploadFile.write(data[i]);
}
}
if (final) {
uploadFile.close();
if( !error ){
if( matchFileSizes(filename,isFileExists) ){
if(filename == "/myHomeProgram.json"){initProgram = true;}
webResponse = "File Uploaded " + filename;
}else{
error = true;
webResponse = "File length mismatch";
}
}
if( error ){
revertBackFile(filename);
}
Serial.println(webResponse);
AsyncWebServerResponse *response = request->beginResponse(200, "text/plain", webResponse);
response->addHeader("Access-Control-Allow-Origin","*");
request->send(response);
}
}

Answer 1

It seems to me that the problem solved.
I have managed to replace the String buffer with a char one in external memory. It seems stable but requires more testing. I'll post the solution but if anyone has a better approach feel free to comment here.
Thanks.
char * uploadTemp;
inline boolean saveFileToTemp(String fileName){
File f = LITTLEFS.open(fileName, "r");
if (!f) {
f.close();
return false;
}else{
size_t fileSize = f.size();
uploadTemp = (char*)ps_malloc(fileSize + 1);
for (int i = 0; i < fileSize; i++){
uploadTemp[i] = (char)f.read();
}
uploadTemp[fileSize] = '\0';
}
f.close();
return true;
}
inline boolean revertBackFile(String fileName){
File g = LITTLEFS.open(fileName, "w");
if (!g) {
g.close();
return false;
}else{
g.print(uploadTemp);
}
g.close();
return true;
}
inline boolean matchFileSizes(String fileName,boolean isFileExists){
boolean isCorrect = false;
if(isFileExists){
File writedFile = LITTLEFS.open(fileName, "w");
if( writedFile.size() == sizeof(uploadTemp)){
isCorrect = true;
}else{
isCorrect = false;
}
writedFile.close();
return isCorrect;
}else{
return true;
}
}
void handleFileUpload(AsyncWebServerRequest * request, String filename,size_t index, uint8_t *data, size_t len, bool final) {
boolean isFileExists = false,error = false;
String webResponse = "";
int httpStatus = 200;
// Start of the file upload
if (!index) {
// Make sure that there is a / char at the start of the string
if (!filename.startsWith("/")){ filename = "/" + filename; }
// Check if the file exists
if (LITTLEFS.exists(filename)) {
isFileExists = true;
// Get the file contents for safety reasons
// If it succeded we can create a new file in the palce
if( saveFileToTemp(filename) ){
uploadFile = LITTLEFS.open(filename, "w");
}else{
// If we can not save it abort the upload process.
webResponse = "File NOT Uploaded " + filename;
final = true;error = true;
}
}
}
// If we have no error at this point, we can start to copy the content to the file.
if( !error ){
for (size_t i = 0; i < len; i++) {
uploadFile.write(data[i]);
}
}
// If no more data we can start responding back to the client
if (final) {
uploadFile.close();
// Check if we got any error before.
if( !error && matchFileSizes(filename,isFileExists) ){
// Copyed file is the same, upload success.
if(filename == "/myHomeProgram.json"){initProgram = true;}
webResponse = "File Uploaded " + filename;
}else{
webResponse = "File length mismatch";
revertBackFile(filename);
httpStatus = 500;
}
free(uploadTemp);
AsyncWebServerResponse *response = request->beginResponse(httpStatus, "text/plain", webResponse);
response->addHeader("Access-Control-Allow-Origin","*");
request->send(response);
}
}
EDIT:
Yeah, so it was completely wrong.
I have to do the following things:
Save the file we want to upload if it exist into a temporary char array.
Get the uploaded file into a temporary file on upload.
If everything was a success, copy the contents of the temporary file to the intended file.
If something fails, revert back the saved file to the original and report an error.
Something like this ( still in test ):
char * prevFileTemp;
inline boolean saveFileToTemp(String fileName){
File f = LITTLEFS.open(fileName, "r");
if (!f) {
f.close();
return false;
}else{
size_t fileSize = f.size();
prevFileTemp = (char*)ps_malloc(fileSize + 1);
for (int i = 0; i < fileSize; i++){
prevFileTemp[i] = (char)f.read();
}
}
f.close();
return true;
}
inline boolean revertBackFile(String fileName){
if (LITTLEFS.exists(fileName)) {
Serial.println("Reverting back the file");
File g = LITTLEFS.open(fileName, "w");
if (!g) {
g.close();
return false;
}else{
g.print(prevFileTemp);
}
g.close();
}
return true;
}
static const inline boolean copyContent(String fileName){
File arrivedFile = LITTLEFS.open(uploadTemp, "r");
File newFile = LITTLEFS.open(fileName, "w");
// Check if we can open the files as intended.
if( !arrivedFile || !newFile){
revertBackFile(fileName);
return false;
}
// Copy one file content to another.
for (size_t i = 0; i < arrivedFile.size(); i++) { newFile.write( (char)arrivedFile.read() ); }
// Check the sizes, if no match, abort mission.
if( newFile.size() != arrivedFile.size()){ return false; }
arrivedFile.close();newFile.close();
return true;
}
boolean isFileExists = false,uploadError = false,newFileArrived = false;
String webResponse = "",newArrivalFileName = "";
int httpStatus = 200;
inline void resetVariables(){
isFileExists = false;
uploadError = false;
webResponse = "";
httpStatus = 200;
}
void handleFileUpload(AsyncWebServerRequest * request, String filename,size_t index, uint8_t *data, size_t len, bool final) {
// Start file upload process
if (!index) {
// Reset all the variables
resetVariables();
// Make sure that there is a '/' char at the start of the string
if (!filename.startsWith("/")){ filename = "/" + filename; }
// Open the temporary file for content copy if it is exist
if (LITTLEFS.exists(filename)) {
if( saveFileToTemp(filename) ){
uploadFile = LITTLEFS.open(uploadTemp, "w");
}else{
// If we can not save it abort the upload process.
webResponse = "File NOT Uploaded " + filename;
final = true;uploadError = true;
}
}
}
// If we have no error at this point, we can start to copy the content to the temporary file.
if( !uploadError ){
for (size_t i = 0; i < len; i++) {
uploadFile.write(data[i]);
}
}
// If no more data we can start responding back to the client
if (final) {
if (!filename.startsWith("/")){ filename = "/" + filename; }
uploadFile.close();
if( !uploadError && copyContent(filename) ){
webResponse = "File Uploaded " + filename;
}else{
webResponse = "File length mismatch";
revertBackFile(filename);
httpStatus = 500;
}
free(prevFileTemp);
AsyncWebServerResponse *response = request->beginResponse(httpStatus, "text/plain", webResponse);
response->addHeader("Access-Control-Allow-Origin","*");
request->send(response);
}
}