setrlimit/getrlimit: fix prlimit64 syscall use for 32-bit CPUs

Commit 95e38b37 ("add support for systems without legacy setrlimit/getrlimit
syscalls") has added use of the prlimit64 syscall in getrlimit and setrlimit
functions. This change causes memory corruption on getrlimit call for 32-bit
CPUs like ARC, as ARC doesn't have ugetrlimit syscall and uses prlimit64.
Also, setrlimit has been broken by prlimit64 call on 32-bit CPUs like, i386,
ARM, ARC.

For the prlimit64 syscall the kernel expects an rlimit struct with 64-bit fields,
but on 32-bit CPUs without _FILE_OFFSET_BITS=64 the struct rlimit has 32-bit
fields.

Add safe implementations of getrlimit, setrlimit, prlimit for 32-bit CPUs with a
local struct rlimit64 variable for use in the prlimit64 syscall.
For 64-bit CPUs and configurations with _FILE_OFFSET_BITS=64 use
getrlimit, setrlimit, prlimit as aliases to getrlimit64, setrlimit64 and
prlimit64. Add a new function prlimit64.

Tested on aarch64, arm, i386, arc.

Fixes: 95e38b37 ("add support for systems without legacy setrlimit/getrlimit syscalls")
Signed-off-by: Pavel Kozlov <pavel.kozlov@synopsys.com>

7 files changed, 197 insertions, 42 deletions

diff --git a/include/sys/resource.h b/include/sys/resource.h
index 00c63ff0f..e9fac2c65 100644
--- a/include/sys/resource.h
+++ b/include/sys/resource.h
@@ -106,6 +106,10 @@ libc_hidden_proto(setpriority)
 extern int prlimit (__pid_t __pid, enum __rlimit_resource __resource,
 		      const struct rlimit *__new_limit,
 		      struct rlimit *__old_limit) __THROW;
+
+extern int prlimit64 (__pid_t __pid, enum __rlimit_resource __resource,
+		      const struct rlimit64 *__new_limit,
+		      struct rlimit64 *__old_limit) __THROW;
 #endif
 
 __END_DECLS
diff --git a/libc/sysdeps/linux/common/getrlimit.c b/libc/sysdeps/linux/common/getrlimit.c
index ad3f4a0e4..46726fcbd 100644
--- a/libc/sysdeps/linux/common/getrlimit.c
+++ b/libc/sysdeps/linux/common/getrlimit.c
@@ -24,21 +24,53 @@ int getrlimit(__rlimit_resource_t resource, struct rlimit *rlimits)
 {
 	return __syscall_ugetrlimit(resource, rlimits);
 }
+libc_hidden_def(getrlimit)
 
-#else
-
-# if !defined(__UCLIBC_HANDLE_OLDER_RLIMIT__)
+#elif defined(__NR_prlimit64)
+/* Use prlimit64 if present, the prlimit64 syscall is free from a back 
+   compatibility stuff for an old getrlimit */
 
-#  if defined(__NR_prlimit64)
+# if __WORDSIZE == 32 && !defined(__USE_FILE_OFFSET64)
+/* If struct rlimit has 64-bit fields (if __WORDSIZE == 64 or __USE_FILE_OFFSET64
+   is defined), then use getrlimit as an alias to getrlimit64, see getrlimit64.c */
 int getrlimit(__rlimit_resource_t resource, struct rlimit *rlimits)
 {
-	return INLINE_SYSCALL (prlimit64, 4, 0, resource, NULL, rlimits);
+	struct rlimit64 rlimits64;
+	int res = INLINE_SYSCALL (prlimit64, 4, 0, resource, NULL, &rlimits64);
+
+	if (res == 0) {
+		/* If the syscall succeeds but the values do not fit into a
+		   rlimit structure set EOVERFLOW errno and retrun -1. */
+		rlimits->rlim_cur = rlimits64.rlim_cur;
+ 		if (rlimits64.rlim_cur != rlimits->rlim_cur) {
+			if (rlimits64.rlim_cur != RLIM64_INFINITY) {
+				__set_errno(EOVERFLOW);
+				return -1;
+			}
+			rlimits->rlim_cur = RLIM_INFINITY;
+		}
+
+		rlimits->rlim_max = rlimits64.rlim_max;
+		if (rlimits64.rlim_max != rlimits->rlim_max) {
+			if (rlimits64.rlim_max != RLIM64_INFINITY) {
+				__set_errno(EOVERFLOW);
+				return -1;
+			}
+			rlimits->rlim_max = RLIM_INFINITY;
+		}
+	}
+	return res;
 }
-#  else
+libc_hidden_def(getrlimit)
+# endif
+
+#else
+
+# if !defined(__UCLIBC_HANDLE_OLDER_RLIMIT__)
+
 /* We don't need to wrap getrlimit() */
 _syscall2(int, getrlimit, __rlimit_resource_t, resource,
 	  struct rlimit *, rlim)
-#  endif
 
 # else
 
@@ -51,11 +83,7 @@ int getrlimit(__rlimit_resource_t resource, struct rlimit *rlimits)
 {
 	int result;
 
-#  if defined(__NR_prlimit64)
-	result = INLINE_SYSCALL (prlimit64, 4, 0, resource, NULL, rlimits);
-#  else
 	result = __syscall_getrlimit(resource, rlimits);
-#  endif
 
 	if (result == -1)
 		return result;
@@ -69,9 +97,6 @@ int getrlimit(__rlimit_resource_t resource, struct rlimit *rlimits)
 	return result;
 }
 # endif
-#endif
-libc_hidden_def(getrlimit)
 
-#if __WORDSIZE == 64
-strong_alias_untyped(getrlimit, getrlimit64)
+libc_hidden_def(getrlimit)
 #endif
diff --git a/libc/sysdeps/linux/common/getrlimit64.c b/libc/sysdeps/linux/common/getrlimit64.c
index be98098a1..47f1410fb 100644
--- a/libc/sysdeps/linux/common/getrlimit64.c
+++ b/libc/sysdeps/linux/common/getrlimit64.c
@@ -17,14 +17,31 @@
 
 #include <_lfs_64.h>
 #include <bits/wordsize.h>
+#include <sys/resource.h>
+#include <sys/syscall.h>
+#include <stddef.h> // needed for NULL to be defined
 
-/* the regular getrlimit will work just fine for 64bit users */
-#if __WORDSIZE == 32
 
-# include <sys/resource.h>
+#if defined(__NR_prlimit64)
+
+/* the regular prlimit64 will work just fine for 64-bit users */
+int getrlimit64 (__rlimit_resource_t resource, struct rlimit64 *rlimits)
+{
+	return INLINE_SYSCALL (prlimit64, 4, 0, resource, NULL, rlimits);
+}
+
+# if !defined(__NR_ugetrlimit) && (__WORDSIZE == 64 || defined (__USE_FILE_OFFSET64))
+/* If getrlimit is not implemented through the __NR_ugetrlimit and size of
+   rlimit_t == rlimit64_t then use getrlimit as an alias to getrlimit64 */
+strong_alias_untyped(getrlimit64, getrlimit)
+libc_hidden_def(getrlimit)
+# endif
+
+#else
 
 /* Put the soft and hard limits for RESOURCE in *RLIMITS.
-   Returns 0 if successful, -1 if not (and sets errno).  */
+   Returns 0 if successful, -1 if not (and sets errno).  
+   The regular getrlimit will work just fine for 64-bit users */
 int getrlimit64 (__rlimit_resource_t resource, struct rlimit64 *rlimits)
 {
     struct rlimit rlimits32;
@@ -44,3 +61,4 @@ int getrlimit64 (__rlimit_resource_t resource, struct rlimit64 *rlimits)
     return 0;
 }
 #endif
+
diff --git a/libc/sysdeps/linux/common/prlimit.c b/libc/sysdeps/linux/common/prlimit.c
index f44dc1664..f59ade3a3 100644
--- a/libc/sysdeps/linux/common/prlimit.c
+++ b/libc/sysdeps/linux/common/prlimit.c
@@ -17,14 +17,57 @@
 
 #include <sys/resource.h>
 #include <sysdep.h>
-#include <bits/kernel-features.h>
+#include <stddef.h> // needed for NULL to be defined
 
-#if defined __ASSUME_PRLIMIT64
+#if defined(__NR_prlimit64) && __WORDSIZE == 32 && !defined(__USE_FILE_OFFSET64)
 int
 prlimit (__pid_t pid, enum __rlimit_resource resource,
-	     const struct rlimit *new_rlimit, struct rlimit *old_rlimit)
+	 const struct rlimit *new_rlimit, struct rlimit *old_rlimit)
 {
-  return INLINE_SYSCALL (prlimit64, 4, pid, resource, new_rlimit,
-			      old_rlimit);
+	struct rlimit64 new_rlimit64;
+	struct rlimit64 old_rlimit64;
+	int res;
+
+	if (new_rlimit != NULL) {
+		if (new_rlimit->rlim_cur == RLIM_INFINITY)
+			new_rlimit64.rlim_cur = RLIM64_INFINITY;
+		else
+			new_rlimit64.rlim_cur = new_rlimit->rlim_cur;
+		if (new_rlimit->rlim_max == RLIM_INFINITY)
+			new_rlimit64.rlim_max = RLIM64_INFINITY;
+		else
+			new_rlimit64.rlim_max = new_rlimit->rlim_max;
+	}
+
+	res = INLINE_SYSCALL (prlimit64, 4, pid, resource, &new_rlimit64,
+			      &old_rlimit64);
+
+	if (res == 0 && old_rlimit != NULL) {
+		/* If the syscall succeeds but the values do not fit into a
+		   rlimit structure set EOVERFLOW errno and retrun -1.
+		   With current Linux implementation of the prlimit64 syscall,
+		   overflow can't happen. An extra condition has been added to get 
+		   the same behavior as in glibc for future potential overflows. */
+		old_rlimit->rlim_cur = old_rlimit64.rlim_cur;
+		if (old_rlimit64.rlim_cur != old_rlimit->rlim_cur) {
+			if (new_rlimit == NULL && 
+			    old_rlimit64.rlim_cur != RLIM64_INFINITY) {
+				__set_errno(EOVERFLOW);
+				return -1;
+			}
+			old_rlimit->rlim_cur = RLIM_INFINITY;
+		}
+		old_rlimit->rlim_max = old_rlimit64.rlim_max;
+		if (old_rlimit64.rlim_max != old_rlimit->rlim_max) {
+			if (new_rlimit == NULL &&
+			    old_rlimit64.rlim_max != RLIM64_INFINITY) {
+				__set_errno(EOVERFLOW);
+				return -1;
+			}
+			old_rlimit->rlim_cur = RLIM_INFINITY;
+		}
+	}
+
+	return res;
 }
 #endif
diff --git a/libc/sysdeps/linux/common/prlimit64.c b/libc/sysdeps/linux/common/prlimit64.c
new file mode 100644
index 000000000..6f57b939e
--- /dev/null
+++ b/libc/sysdeps/linux/common/prlimit64.c
@@ -0,0 +1,36 @@
+/*  Copyright (C) 2023 uClibc-ng
+ *  An prlimit64() - get/set resource limits Linux specific syscall.
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Library General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Library General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Library General Public
+ *  License along with this library; if not, see
+ *  <http://www.gnu.org/licenses/>.
+ */
+
+#include <sys/resource.h>
+#include <sysdep.h>
+
+#if defined(__NR_prlimit64)
+
+int
+prlimit64 (__pid_t pid, enum __rlimit_resource resource,
+	   const struct rlimit64 *new_rlimit, struct rlimit64 *old_rlimit)
+{
+	return INLINE_SYSCALL (prlimit64, 4, pid, resource, new_rlimit,
+			       old_rlimit);
+}
+
+# if __WORDSIZE == 64 || defined (__USE_FILE_OFFSET64)
+strong_alias_untyped(prlimit64, prlimit)
+# endif
+
+#endif
\ No newline at end of file
diff --git a/libc/sysdeps/linux/common/setrlimit.c b/libc/sysdeps/linux/common/setrlimit.c
index 8381afc61..9c6707235 100644
--- a/libc/sysdeps/linux/common/setrlimit.c
+++ b/libc/sysdeps/linux/common/setrlimit.c
@@ -23,21 +23,41 @@ int setrlimit(__rlimit_resource_t resource, const struct rlimit *rlimits)
 {
 	return __syscall_usetrlimit(resource, rlimits);
 }
+libc_hidden_def(setrlimit)
 
-#else
+#elif defined(__NR_prlimit64)
 
-# if !defined(__UCLIBC_HANDLE_OLDER_RLIMIT__)
+/* Use prlimit64 if present, the prlimit64 syscall is free from a back
+   compatibility stuff for setrlimit */
 
-#  if defined(__NR_prlimit64)
+ # if __WORDSIZE == 32 && !defined(__USE_FILE_OFFSET64)
+/* If struct rlimit has 64-bit fields (if __WORDSIZE == 64 or __USE_FILE_OFFSET64
+   is defined), then use setrlimit as an alias to setrlimit64, see setrlimit64.c */
 int setrlimit(__rlimit_resource_t resource, const struct rlimit *rlimits)
 {
-	return INLINE_SYSCALL (prlimit64, 4, 0, resource, rlimits, NULL);
+	struct rlimit64 rlimits64;
+
+	if (rlimits->rlim_cur == RLIM_INFINITY)
+		rlimits64.rlim_cur = RLIM64_INFINITY;
+	else
+		rlimits64.rlim_cur = rlimits->rlim_cur;
+	if (rlimits->rlim_max == RLIM_INFINITY)
+		rlimits64.rlim_max = RLIM64_INFINITY;
+	else
+		rlimits64.rlim_max = rlimits->rlim_max;
+
+	return INLINE_SYSCALL (prlimit64, 4, 0, resource, &rlimits64, NULL);
 }
-#  else
+libc_hidden_def(setrlimit)
+# endif
+
+#else
+
+# if !defined(__UCLIBC_HANDLE_OLDER_RLIMIT__)
+
 /* We don't need to wrap setrlimit() */
 _syscall2(int, setrlimit, __rlimit_resource_t, resource,
 		const struct rlimit *, rlim)
-#  endif
 
 # else
 
@@ -66,16 +86,9 @@ int setrlimit(__rlimit_resource_t resource, const struct rlimit *rlimits)
 								  RLIM_INFINITY >> 1);
 	rlimits_small.rlim_max = MIN((unsigned long int) rlimits->rlim_max,
 								  RLIM_INFINITY >> 1);
-#  if defined(__NR_prlimit64)
-	return INLINE_SYSCALL (prlimit64, 4, 0, resource, &rlimits_small, NULL);
-#  else
 	return __syscall_setrlimit(resource, &rlimits_small);
-#  endif
 }
 # endif
-#endif
-libc_hidden_def(setrlimit)
 
-#if __WORDSIZE == 64
-strong_alias_untyped(setrlimit, setrlimit64)
+libc_hidden_def(setrlimit)
 #endif
diff --git a/libc/sysdeps/linux/common/setrlimit64.c b/libc/sysdeps/linux/common/setrlimit64.c
index fee14f4ad..3446c58fe 100644
--- a/libc/sysdeps/linux/common/setrlimit64.c
+++ b/libc/sysdeps/linux/common/setrlimit64.c
@@ -17,15 +17,31 @@
 
 #include <_lfs_64.h>
 #include <bits/wordsize.h>
+#include <sys/resource.h>
+#include <sys/syscall.h>
+#include <stddef.h> // needed for NULL to be defined
 
-/* the regular setrlimit will work just fine for 64bit users */
-#if __WORDSIZE == 32
 
-# include <sys/resource.h>
+#if defined(__NR_prlimit64)
+
+int setrlimit64 (__rlimit_resource_t resource, const struct rlimit64 *rlimits)
+{
+	return INLINE_SYSCALL (prlimit64, 4, 0, resource, rlimits, NULL);
+}
+
+# if !defined(__NR_usetrlimit) && (__WORDSIZE == 64 || defined (__USE_FILE_OFFSET64))
+/* If setrlimit is not implemented through the __NR_usetrlimit and size of
+   rlimit_t == rlimit64_t then use setrlimit as an alias to setrlimit64 */
+strong_alias_untyped(setrlimit64, setrlimit)
+libc_hidden_def(setrlimit)
+# endif
+
+#else
 
 /* Set the soft and hard limits for RESOURCE to *RLIMITS.
    Only the super-user can increase hard limits.
-   Return 0 if successful, -1 if not (and sets errno).  */
+   Return 0 if successful, -1 if not (and sets errno).
+   The regular setrlimit will work just fine for 64bit users  */
 int setrlimit64 (__rlimit_resource_t resource, const struct rlimit64 *rlimits)
 {
     struct rlimit rlimits32;