stdlib: fix potential UB and integer overflow with huge allocations using malloc-simple allocator

Two things are fixed by this commit:

1/ It is wrong to allocate an object of size > PTRDIFF_MAX.
It is explained in this thread: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303

2/ There was a possible integer overflow in both malloc() and memalign() implementations
of stdlib/malloc-simple.
The malloc() integer overflow issue is fixed by the side effect of fixing the PTRDIFF_MAX issue.
The memalign() one is fixed by adding a comparison.

Signed-off-by: Yann Sionneau <yann@sionneau.net>

1 files changed, 20 insertions, 0 deletions

diff --git a/libc/stdlib/malloc-simple/alloc.c b/libc/stdlib/malloc-simple/alloc.c
index 3baf75fdd..757a05ecf 100644
--- a/libc/stdlib/malloc-simple/alloc.c
+++ b/libc/stdlib/malloc-simple/alloc.c
@@ -13,6 +13,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <stdint.h>
 #include <errno.h>
 #include <sys/mman.h>
 #include <malloc.h>
@@ -28,6 +29,15 @@ void *malloc(size_t size)
 		size++;
 	}
 
+	/* prevent Undefined Behaviour for pointer arithmetic (substract) of too big pointers
+	 * see: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303
+	 * No need to check for size + sizeof(size_t) integer overflow since we already check for PTRDIFF_MAX
+	 */
+	if (unlikely(size > PTRDIFF_MAX)) {
+		__set_errno(ENOMEM);
+		return 0;
+	}
+
 #ifdef __ARCH_USE_MMU__
 # define MMAP_FLAGS MAP_PRIVATE | MAP_ANONYMOUS
 #else
@@ -148,6 +158,16 @@ void * memalign (size_t alignment, size_t size)
 	void * result;
 	unsigned long int adj;
 
+	if (unlikely(size > PTRDIFF_MAX)) {
+		__set_errno(ENOMEM);
+		return NULL;
+	}
+
+	if (unlikely((size + alignment - 1 < size) && (alignment != 0))) {
+		__set_errno(ENOMEM);
+		return NULL;
+	}
+
 	result = malloc (size + alignment - 1);
 	if (result == NULL)
 		return NULL;