How to decrypt a file in virtual memory in C - encryption

Question

I'm trying to decrypt a file in memory and this file was encrypted with openssl.
For doing that, i use mmap for loading my encrypted file in memory like that:
void* src = mmap(0, statbuf.st_size,PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
and duplicate it because i don't want to modify my original file
void* dst = mmap(0, statbuf.st_size,PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
memcpy (dst, src, statbuf.st_size);
At this step all is ok but i don't know what to do next.
Initialy for testing purpose, i encrypt my file with the openssl command :
system("openssl enc -aes-256-cbc -salt -in my_encryptedfile -out my_encryptedfile.enc -pass")
and decrypt it with this command:
system("openssl enc -d -aes-256-cbc -in my_encryptedfile.enc -out my_encryptedfile -pass pass:")
But i can't use dst in this case, so i search and discovered EVP Symmetric Encryption and Decryption.
link here
Then i encrypted and decrypted my file with that code github code
I try using Key and IV and the decryption in memory and it seem working but i have a problem that i don't understand. When i dump the buffer of my decripted file, i saw "SPACES/NULS" at the end of the file and i don't figure out why it display that. When i try to execute my binany in memory by calling this function :
func()
i got a segmentation fault
Any clues?
typedef void (*JittedFunc)(void);
void* alloc_writable_memory(void *ptr, size_t size) {
ptr = mmap(0, size,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (ptr == (void*)-1) {
perror("mmap");
return NULL;
}
return ptr;
}
int make_memory_executable(void* m, size_t size) {
if (mprotect(m, size, PROT_READ |PROT_WRITE | PROT_EXEC) == -1) {
perror("mprotect");
return -1;
}
return 0;
}
int do_crypt(char *in, char *out, int do_encrypt, int inlen)
{
/* Allow enough space in output buffer for additional block */
unsigned char outbuf[inlen + EVP_MAX_BLOCK_LENGTH];
int outlen;
EVP_CIPHER_CTX *ctx;
/* Bogus key and IV: we'd normally set these from
* another source.
*/
unsigned char key[] = "0123456789abcdeF";
unsigned char iv[] = "1234567887654321";
//int n;
printf("step1\n");
/* Don't set key or IV right away; we want to check lengths */
ctx = EVP_CIPHER_CTX_new();
printf("step2\n");
EVP_CipherInit_ex(ctx, EVP_aes_128_cbc(), NULL, NULL, NULL,do_encrypt);
printf("step3\n");
OPENSSL_assert(EVP_CIPHER_CTX_key_length(ctx) == 16);
OPENSSL_assert(EVP_CIPHER_CTX_iv_length(ctx) == 16);
/* Now we can set key and IV */
EVP_CipherInit_ex(ctx, NULL, NULL, key, iv, do_encrypt);
printf("step4\n");
if(!EVP_CipherUpdate(ctx, outbuf,&outlen, in, inlen))
{
printf("test 2.1: %d %d\n", inlen, outlen);
printf("step8\n");
/* Error */
EVP_CIPHER_CTX_free(ctx);
return 0;
}
//BIO_dump_fp (stdout, (const char *)outbuf, outlen);
printf(" test 2: %d %d\n", inlen, outlen);
if(!EVP_CipherFinal_ex(ctx, outbuf, &outlen))
{
printf("step11\n");
EVP_CIPHER_CTX_free(ctx);
return 0;
}
//copy the decryted buffer in another memory space
memcpy(out, outbuf, outlen);
printf(" test 3: %d %d\n", inlen, outlen);
//BIO_dump_fp (stdout, (const char *)outbuf, outlen);
printf("step12\n");
//fwrite(outbuf, 1, outlen, out);
printf("step13\n");
EVP_CIPHER_CTX_free(ctx);
return 1;
}
int main()
{
FILE *src, *dst;
char *src_mem, *dst_mem, *dst2_mem = NULL;
struct stat statbuf;
int fd;
src = fopen("hello_encrypted", "rb");
if (!src) {
/* Unable to open file for reading */
fprintf(stderr, "ERROR: fopen error: %s\n", strerror(errno));
return errno;
}
/*get the file des from a file*/
fd = fileno(src);
/* find size of input file */
if (fstat (fd,&statbuf) < 0)
{printf ("fstat error");
return 0;
}
/* go to the location corresponding to the last byte */
if (lseek (fd, statbuf.st_size - 1, SEEK_SET) == -1)
{printf ("lseek error");
return 0;
}
if ((src_mem = mmap (0, statbuf.st_size, PROT_READ, MAP_SHARED, fd, 0)) == (caddr_t) -1)
{
printf ("mmap error for input");
return 0;
}
if ((dst_mem = mmap (0, statbuf.st_size, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS , -1, 0)) == (caddr_t) -1)
{
printf ("mmap error for output");
return 0;
}
if ((dst2_mem = mmap (0, statbuf.st_size , PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS , -1, 0)) == (caddr_t) -1)
{
printf ("mmap error for output");
return 0;
}
memcpy(dst_mem, src_mem, statbuf.st_size);
int n;
/* 0 for decrypting or 1 for encrypting*/
n = do_crypt(dst_mem,dst2_mem, 0, statbuf.st_size);
printf("%d\n", n);
make_memory_executable(dst2_mem, statbuf.st_size);
//dump of the decrypt binary
BIO_dump_fp (stdout, (const char *)dst2_mem, statbuf.st_size);
//try to launch the decrypted binary ==> segmentation fault
JittedFunc func = dst2_mem;
func();
fclose(src);
return 0;
}

Answer 1

I would remove from the first mmap the flag PROT_WRITE, because you dont want to modify the original file, I think you can also use PROT_PRIV, but have a look to what says the man of mmap.
On the other hand for decrypt your buffer you can use many libraries (a lot of them based on openssl), I specially like CryptoPP but there are many others. Here is an example of how your code will look like on CryptoPP
try {
// CryptoPP::ECB_Mode< CryptoPP::AES >::Decryption d;
// CrypoPP::CBC_Mode< CryptoPP::AES >::Decryption d;
CrypoPP::CBC_CTS_Mode< CryptoPP::AES >::Decryption d;
// What ever mode is best for you
d.SetKey(key.data(), key.size());
// The StreamTransformationFilter removes
// padding as required.
CryptoPP::StringSource s((const uint8_t*)dst, length_dst, true,
new CryptoPP::StreamTransformationFilter(d,
new CryptoPP::StringSink(recovered)
) // StreamTransformationFilter
); // StringSource
} catch(const CryptoPP::Exception& e) {
std::cout << "ERROR decrypting:" << e.what() << std::endl;
return;
}