- Cleanup dl-vdso.c code.
- Pass `void *` as first arg to `load_vdso()`, using 32-bit type
is completely wrong on 64bit archutectures.
- Split libc code and vDSO-related code.
Move arch-specific implementations into separate files.
The performance improvement is for example 50-60 times on ARMv7
and about 4 times on x86_64.
Signed-off-by: Dmitry Chestnykh <dm.chestnykh(a)gmail.com>
---
extra/Configs/Config.arm | 1 +
ldso/include/dl-syscall.h | 3 +
ldso/include/ldso.h | 6 +-
ldso/ldso/arm/dl-syscalls.h | 23 +++++-
ldso/ldso/dl-startup.c | 2 +-
ldso/ldso/dl-vdso-calls.h | 68 ++++++++++++++++++
ldso/ldso/dl-vdso.c | 86 ++++++++++++-----------
ldso/ldso/mips/dl-syscalls.h | 23 +++++-
ldso/ldso/x86_64/dl-syscalls.h | 23 +++++-
libc/sysdeps/linux/common/clock_gettime.c | 22 +++++-
libc/sysdeps/linux/common/gettimeofday.c | 34 +++------
librt/clock_gettime.c | 18 ++++-
12 files changed, 230 insertions(+), 79 deletions(-)
create mode 100644 ldso/ldso/dl-vdso-calls.h
diff --git a/extra/Configs/Config.arm b/extra/Configs/Config.arm
index 0d02e3f10..ea8ab4895 100644
--- a/extra/Configs/Config.arm
+++ b/extra/Configs/Config.arm
@@ -13,6 +13,7 @@ config FORCE_OPTIONS_FOR_ARCH
select ARCH_ANY_ENDIAN
select ARCH_HAS_UCONTEXT
select ARCH_HAS_DEPRECATED_SYSCALLS
+ select ARCH_VDSO_SUPPORT
config CONFIG_ARM_EABI
bool "Build for EABI"
diff --git a/ldso/include/dl-syscall.h b/ldso/include/dl-syscall.h
index b40f58b10..4f41034ad 100644
--- a/ldso/include/dl-syscall.h
+++ b/ldso/include/dl-syscall.h
@@ -11,8 +11,11 @@
* been dynamicly linked in yet. */
#include "sys/syscall.h"
extern int _dl_errno;
+
+#ifdef UCLIBC_LDSO
#undef __set_errno
#define __set_errno(X) {(_dl_errno) = (X);}
+#endif
/* Pull in the arch specific syscall implementation */
#include <dl-syscalls.h>
diff --git a/ldso/include/ldso.h b/ldso/include/ldso.h
index 80d5d5dd5..a397cce61 100755
--- a/ldso/include/ldso.h
+++ b/ldso/include/ldso.h
@@ -194,10 +194,6 @@ extern void *_dl_get_ready_to_run(struct elf_resolve *tpnt, DL_LOADADDR_TYPE loa
#define AUX_MAX_AT_ID 40
extern ElfW(auxv_t) _dl_auxvt[AUX_MAX_AT_ID];
-void load_vdso( uint32_t sys_info_ehdr, char **envp );
-
-#ifdef __VDSO_SUPPORT__
-extern void* _dl__vdso_gettimeofday;
-#endif
+void load_vdso(void *sys_info_ehdr, char **envp );
#endif /* _LDSO_H */
diff --git a/ldso/ldso/arm/dl-syscalls.h b/ldso/ldso/arm/dl-syscalls.h
index f40c4fd31..5cbb94d30 100644
--- a/ldso/ldso/arm/dl-syscalls.h
+++ b/ldso/ldso/arm/dl-syscalls.h
@@ -1 +1,22 @@
-/* stub for arch-specific syscall issues */
+/* stub for arch-specific syscall issues/specific implementations */
+
+#if defined(__VDSO_SUPPORT__) && !defined(UCLIBC_LDSO)
+
+#include "../dl-vdso-calls.h"
+
+static int __attribute__ ((used)) __arm_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp);
+static int __attribute__ ((used)) __arm_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp)
+{
+ return __generic_vdso_clock_gettime(clock_id, tp);
+}
+
+static int __attribute__ ((used)) __arm_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz);
+static int __attribute__ ((used)) __arm_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz)
+{
+ return __generic_vdso_gettimeofday(tv, tz);
+}
+
+#define ARCH_VDSO_GETTIMEOFDAY(tv, tz) __arm_vdso_gettimeofday(tv, tz)
+#define ARCH_VDSO_CLOCK_GETTIME(clock_id, tp) __arm_vdso_clock_gettime(clock_id, tp)
+
+#endif /* defined(__VDSO_SUPPORT__) && !defined(UCLIBC_LDSO) */
\ No newline at end of file
diff --git a/ldso/ldso/dl-startup.c b/ldso/ldso/dl-startup.c
index 5b992a3a2..e7000bd87 100644
--- a/ldso/ldso/dl-startup.c
+++ b/ldso/ldso/dl-startup.c
@@ -373,7 +373,7 @@ DL_START(unsigned long args)
DL_GET_READY_TO_RUN_EXTRA_ARGS);
- load_vdso(_dl_auxvt[AT_SYSINFO_EHDR].a_un.a_val, envp);
+ load_vdso((void *)_dl_auxvt[AT_SYSINFO_EHDR].a_un.a_val, envp);
/* Transfer control to the application. */
SEND_STDERR_DEBUG("transfering control to application @ ");
diff --git a/ldso/ldso/dl-vdso-calls.h b/ldso/ldso/dl-vdso-calls.h
new file mode 100644
index 000000000..c72f2dadf
--- /dev/null
+++ b/ldso/ldso/dl-vdso-calls.h
@@ -0,0 +1,68 @@
+#ifndef _DL_VDSO_CALLS_H
+#define _DL_VDSO_CALLS_H
+
+#include <sys/time.h>
+#include <sys/syscall.h>
+#include <errno.h>
+#include <time.h>
+
+void __attribute__((weak)) *_get__dl__vdso_clock_gettime(void);
+#if defined(__UCLIBC_USE_TIME64__)
+#include "internal/time64_helpers.h"
+void __attribute__((weak)) *_get__dl__vdso_clock_gettime64(void);
+typedef int (*clock_gettime_func)(clockid_t clock_id, struct __ts64_struct *tp);
+#else
+typedef int (*clock_gettime_func)(clockid_t clock_id, struct timespec *tp);
+#endif
+
+extern int __libc_clock_gettime(clockid_t clock_id, struct timespec *tp);
+
+static int __attribute__ ((used)) __generic_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp);
+static int __attribute__ ((used)) __generic_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp)
+{
+ void *impl = NULL;
+#if defined(__UCLIBC_USE_TIME64__)
+ if (&_get__dl__vdso_clock_gettime64 && (impl = _get__dl__vdso_clock_gettime64())) {
+ struct __ts64_struct __ts64;
+ int __ret = ((clock_gettime_func)impl)(clock_id, &__ts64);
+ if (__ret != 0) {
+ __set_errno(-__ret);
+ return -1;
+ }
+
+ if (tp) {
+ tp->tv_sec = __ts64.tv_sec;
+ tp->tv_nsec = __ts64.tv_nsec;
+ }
+ return 0;
+ }
+
+ /* fallback to syscall */
+ return __libc_clock_gettime(clock_id, tp);
+#else
+ if (&_get__dl__vdso_clock_gettime && (impl = _get__dl__vdso_clock_gettime())) {
+ int __ret = ((clock_gettime_func)impl)(clock_id, tp);
+ if (__ret != 0) {
+ __set_errno(-__ret);
+ return -1;
+ }
+
+ return 0;
+ }
+
+ /* fallback to syscall */
+ return __libc_clock_gettime(clock_id, tp);
+#endif
+}
+
+static int __attribute__ ((used)) __generic_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz)
+{
+ struct timespec ts;
+ int __res = __generic_vdso_clock_gettime(CLOCK_REALTIME, &ts);
+ tv->tv_sec = ts.tv_sec;
+ tv->tv_usec = (suseconds_t)ts.tv_nsec / 1000;
+
+ return __res;
+}
+
+#endif /* _DL_VDSO_CALLS_H */
\ No newline at end of file
diff --git a/ldso/ldso/dl-vdso.c b/ldso/ldso/dl-vdso.c
index c8d724cd0..c23fd8b72 100755
--- a/ldso/ldso/dl-vdso.c
+++ b/ldso/ldso/dl-vdso.c
@@ -1,11 +1,7 @@
#include <elf.h>
-//#include <stdio.h>
#include <string.h>
-
#include "sys/auxv.h"
-//#include <linux/time.h>
-//#include <time.h>
#include "ldso.h"
#include "generated/autoconf.h"
@@ -24,7 +20,7 @@
#ifndef __VDSO_SUPPORT__
- void load_vdso( uint32_t sys_info_ehdr, char **envp ){
+ void load_vdso(void *sys_info_ehdr, char **envp ){
#ifdef __SUPPORT_LD_DEBUG__
if ( _dl_debug_vdso != 0 ){
_dl_dprintf(2,"_dl_vdso support not enabled\n" );
@@ -34,21 +30,34 @@
}
#else
+void *_dl__vdso_gettimeofday = 0;
+void *_dl__vdso_clock_gettime = 0;
+#if defined(__UCLIBC_USE_TIME64__)
+void *_dl__vdso_clock_gettime64 = 0;
+#endif
-
-
-
-//typedef long (*gtod_t)(struct timeval *tv, struct timezone *tz);
-void* _dl__vdso_gettimeofday = 0;
-
-
-//typedef long (*clock_gettime_t)(int clk_id, struct timespec *tp);
-void* _dl__vdso_clock_gettime = 0;
+void *_get__dl__vdso_clock_gettime(void);
+void *_get__dl__vdso_clock_gettime(void)
+{
+ return _dl__vdso_clock_gettime;
+}
+#if defined(__UCLIBC_USE_TIME64__)
+void *_get__dl__vdso_clock_gettime64(void);
+void *_get__dl__vdso_clock_gettime64(void)
+{
+ return _dl__vdso_clock_gettime64;
+}
+#endif
+void *_get__dl__vdso_gettimeofday(void);
+void *_get__dl__vdso_gettimeofday(void)
+{
+ return _dl__vdso_gettimeofday;
+}
-typedef struct{
+typedef struct {
void* base_addr;
@@ -75,7 +84,7 @@ typedef struct{
char* vers_strings[10];
-}elf_infos;
+} elf_infos;
/*
* the raise() dummy function is needed because of divisons in this code
@@ -111,13 +120,10 @@ static ELF(Shdr) *vdso_get_sec_header( elf_infos* elf, int index ){
}
-
-void load_vdso( uint32_t sys_info_ehdr, char **envp ){
+void load_vdso(void *sys_info_ehdr, char **envp ){
elf_infos vdso_infos;
-
-
-
+
if ( sys_info_ehdr == 0 ){
#ifdef __SUPPORT_LD_DEBUG__
if ( _dl_debug_vdso != 0 ){
@@ -137,22 +143,16 @@ void load_vdso( uint32_t sys_info_ehdr, char **envp ){
#endif
return;
}
-
-
+
_dl_memset( &vdso_infos, 0 , sizeof( elf_infos ) );
-
-
+
vdso_infos.base_addr = (void*)sys_info_ehdr;
vdso_infos.hdr = (ELF(Ehdr)*)vdso_infos.base_addr;
-
- //printf("base : %p\n",vdso_infos.base_addr);
-
+
if ( 0 != vdso_check_elf_header( &vdso_infos ) ){
return;
}
-
-
-
+
ELF(Shdr) *sec_header = vdso_get_sec_header( &vdso_infos, vdso_infos.hdr->e_shstrndx);
vdso_infos.section_header_strtab = ( vdso_infos.base_addr + sec_header->sh_offset );
@@ -307,10 +307,7 @@ void load_vdso( uint32_t sys_info_ehdr, char **envp ){
if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
continue;
-
-
char* name = vdso_infos.dynstr_table + sym->st_name;
- char* vers = vdso_infos.vers_strings[ vdso_infos.versym_table[i] ];
void* func_addr = (void*)( vdso_infos.base_addr + sym->st_value );
// the function name is patched to zero if the kernel has no timer which is
@@ -324,9 +321,6 @@ void load_vdso( uint32_t sys_info_ehdr, char **envp ){
continue;
}
- //printf(" %s@@%s\n", name , vers );
-
- //print_sym( sym );
if ( 0 == _dl_strcmp( name, "__vdso_gettimeofday" ) ){
_dl__vdso_gettimeofday = func_addr;
#ifdef __SUPPORT_LD_DEBUG__
@@ -336,7 +330,6 @@ void load_vdso( uint32_t sys_info_ehdr, char **envp ){
#endif
continue;
}
-
if ( 0 == _dl_strcmp( name, "__vdso_clock_gettime" ) ){
_dl__vdso_clock_gettime = func_addr;
#ifdef __SUPPORT_LD_DEBUG__
@@ -346,17 +339,26 @@ void load_vdso( uint32_t sys_info_ehdr, char **envp ){
#endif
continue;
}
+
+#if defined(__UCLIBC_USE_TIME64__)
+ if ( 0 == _dl_strcmp( name, "__vdso_clock_gettime64" ) ){
+ _dl__vdso_clock_gettime64 = func_addr;
+#ifdef __SUPPORT_LD_DEBUG__
+ if ( _dl_debug_vdso != 0 ){
+ _dl_dprintf(2," %s at address %p\n", name, func_addr );
+ }
+#endif
+ continue;
+ }
+#endif /* defined(__UCLIBC_USE_TIME64__) */
#ifdef __SUPPORT_LD_DEBUG__
if ( _dl_debug_vdso != 0 ){
_dl_dprintf(2," <%s> not handled\n", name );
}
#endif
-
-
- }
-
+ }
}
#endif // __VDSO_SUPPORT__
diff --git a/ldso/ldso/mips/dl-syscalls.h b/ldso/ldso/mips/dl-syscalls.h
index f40c4fd31..46fd07bfa 100644
--- a/ldso/ldso/mips/dl-syscalls.h
+++ b/ldso/ldso/mips/dl-syscalls.h
@@ -1 +1,22 @@
-/* stub for arch-specific syscall issues */
+/* stub for arch-specific syscall issues/specific implementations */
+
+#if defined(__VDSO_SUPPORT__) && !defined(UCLIBC_LDSO)
+
+#include "../dl-vdso-calls.h"
+
+static int __attribute__ ((used)) __mips_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp);
+static int __attribute__ ((used)) __mips_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp)
+{
+ return __generic_vdso_clock_gettime(clock_id, tp);
+}
+
+static int __attribute__ ((used)) __mips_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz);
+static int __attribute__ ((used)) __mips_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz)
+{
+ return __generic_vdso_gettimeofday(tv, tz);
+}
+
+#define ARCH_VDSO_GETTIMEOFDAY(tv, tz) __mips_vdso_gettimeofday(tv, tz)
+#define ARCH_VDSO_CLOCK_GETTIME(clock_id, tp) __mips_vdso_clock_gettime(clock_id, tp)
+
+#endif /* defined(__VDSO_SUPPORT__) && !defined(UCLIBC_LDSO) */
\ No newline at end of file
diff --git a/ldso/ldso/x86_64/dl-syscalls.h b/ldso/ldso/x86_64/dl-syscalls.h
index f40c4fd31..0ec7a7dbb 100644
--- a/ldso/ldso/x86_64/dl-syscalls.h
+++ b/ldso/ldso/x86_64/dl-syscalls.h
@@ -1 +1,22 @@
-/* stub for arch-specific syscall issues */
+/* stub for arch-specific syscall issues/specific implementations */
+
+#if defined(__VDSO_SUPPORT__) && !defined(UCLIBC_LDSO)
+
+#include "../dl-vdso-calls.h"
+
+static int __attribute__ ((used)) __x86_64_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp);
+static int __attribute__ ((used)) __x86_64_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp)
+{
+ return __generic_vdso_clock_gettime(clock_id, tp);
+}
+
+static int __attribute__ ((used)) __x86_64_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz);
+static int __attribute__ ((used)) __x86_64_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz)
+{
+ return __generic_vdso_gettimeofday(tv, tz);
+}
+
+#define ARCH_VDSO_GETTIMEOFDAY(tv, tz) __x86_64_vdso_gettimeofday(tv, tz)
+#define ARCH_VDSO_CLOCK_GETTIME(clock_id, tp) __x86_64_vdso_clock_gettime(clock_id, tp)
+
+#endif /* defined(__VDSO_SUPPORT__) && !defined(UCLIBC_LDSO) */
\ No newline at end of file
diff --git a/libc/sysdeps/linux/common/clock_gettime.c b/libc/sysdeps/linux/common/clock_gettime.c
index 4d787b9b7..b924d860b 100644
--- a/libc/sysdeps/linux/common/clock_gettime.c
+++ b/libc/sysdeps/linux/common/clock_gettime.c
@@ -11,10 +11,14 @@
#include <sys/syscall.h>
#include <time.h>
+#ifdef __VDSO_SUPPORT__
+#include "ldso.h"
+#endif
+
#if defined(__UCLIBC_USE_TIME64__) && defined(__NR_clock_gettime64)
#include "internal/time64_helpers.h"
-int clock_gettime(clockid_t clock_id, struct timespec *tp)
+int __libc_clock_gettime(clockid_t clock_id, struct timespec *tp)
{
struct __ts64_struct __ts64;
int __ret = INLINE_SYSCALL(clock_gettime64, 2, clock_id, &__ts64);
@@ -26,11 +30,14 @@ int clock_gettime(clockid_t clock_id, struct timespec *tp)
return __ret;
}
#elif defined(__NR_clock_gettime)
-_syscall2(int, clock_gettime, clockid_t, clock_id, struct timespec*, tp)
+int __libc_clock_gettime(clockid_t clock_id, struct timespec *tp)
+{
+ return INLINE_SYSCALL(clock_gettime, 2, clock_id, tp);
+}
#else
# include <sys/time.h>
-int clock_gettime(clockid_t clock_id, struct timespec* tp)
+int __libc_clock_gettime(clockid_t clock_id, struct timespec* tp)
{
struct timeval tv;
int retval = -1;
@@ -53,3 +60,12 @@ int clock_gettime(clockid_t clock_id, struct timespec* tp)
return retval;
}
#endif
+
+int clock_gettime(clockid_t clock_id, struct timespec *tp)
+{
+#if defined(__VDSO_SUPPORT__) && defined(ARCH_VDSO_CLOCK_GETTIME)
+ return ARCH_VDSO_CLOCK_GETTIME(clock_id, tp);
+#else
+ return __libc_clock_gettime(clock_id, tp);
+#endif
+}
diff --git a/libc/sysdeps/linux/common/gettimeofday.c b/libc/sysdeps/linux/common/gettimeofday.c
index ed18b2be5..9b29fe5a0 100755
--- a/libc/sysdeps/linux/common/gettimeofday.c
+++ b/libc/sysdeps/linux/common/gettimeofday.c
@@ -14,30 +14,18 @@
#include "ldso.h"
#endif
-#ifdef __VDSO_SUPPORT__
-typedef int (*gettimeofday_func)(struct timeval * tv, __timezone_ptr_t tz);
-#endif
-
int gettimeofday(struct timeval * tv, __timezone_ptr_t tz) {
-
- #ifdef __VDSO_SUPPORT__
- if ( _dl__vdso_gettimeofday != 0 ){
- gettimeofday_func func= _dl__vdso_gettimeofday;
- return func( tv, tz );
-
- }else{
- _syscall2_body(int, gettimeofday, struct timeval *, tv, __timezone_ptr_t, tz)
- }
- #else
- if (!tv)
- return 0;
-
- struct timespec __ts;
- int __ret = clock_gettime(CLOCK_REALTIME, &__ts);
- tv->tv_sec = __ts.tv_sec;
- tv->tv_usec = (suseconds_t)__ts.tv_nsec / 1000;
- return __ret;
- #endif
+ if (!tv)
+ return 0;
+#if defined(__VDSO_SUPPORT__) && defined(ARCH_VDSO_GETTIMEOFDAY)
+ return ARCH_VDSO_GETTIMEOFDAY(tv, tz);
+#else
+ struct timespec __ts;
+ int __ret = clock_gettime(CLOCK_REALTIME, &__ts);
+ tv->tv_sec = __ts.tv_sec;
+ tv->tv_usec = (suseconds_t)__ts.tv_nsec / 1000;
+ return __ret;
+#endif
}
diff --git a/librt/clock_gettime.c b/librt/clock_gettime.c
index eab9a3343..dd1f0c514 100644
--- a/librt/clock_gettime.c
+++ b/librt/clock_gettime.c
@@ -22,6 +22,10 @@
#include <sys/time.h>
#include "kernel-posix-cpu-timers.h"
+#ifdef __VDSO_SUPPORT__
+#include "ldso.h"
+#endif
+
#if defined(__UCLIBC_USE_TIME64__)
#include "internal/time64_helpers.h"
#endif
@@ -68,9 +72,8 @@ realtime_gettime (struct timespec *tp)
return retval;
}
-/* Get current value of CLOCK and store it in TP. */
int
-clock_gettime (clockid_t clock_id, struct timespec *tp)
+__libc_clock_gettime (clockid_t clock_id, struct timespec *tp)
{
int retval = -1;
#ifndef HANDLED_REALTIME
@@ -101,3 +104,14 @@ clock_gettime (clockid_t clock_id, struct timespec *tp)
return retval;
}
+
+/* Get current value of CLOCK and store it in TP. */
+int
+clock_gettime (clockid_t clock_id, struct timespec *tp)
+{
+#if defined(__VDSO_SUPPORT__) && defined(ARCH_VDSO_CLOCK_GETTIME)
+ return ARCH_VDSO_CLOCK_GETTIME(clock_id, tp);
+#else
+ return __libc_clock_gettime(clock_id, tp);
+#endif
+}
--
2.44.0
