- Cleanup dl-vdso.c code. - Pass `void *` as first arg to `load_vdso()`, using 32-bit type is completely wrong on 64bit archutectures. - Split libc code and vDSO-related code. Move arch-specific implementations into separate files. The performance improvement is for example 50-60 times on ARMv7 and about 4 times on x86_64. Signed-off-by: Dmitry Chestnykh &lt;dm.chestnykh(a)gmail.com&gt; --- extra/Configs/Config.arm | 1 + ldso/include/dl-syscall.h | 3 + ldso/include/ldso.h | 6 +- ldso/ldso/arm/dl-syscalls.h | 23 +++++- ldso/ldso/dl-startup.c | 2 +- ldso/ldso/dl-vdso-calls.h | 68 ++++++++++++++++++ ldso/ldso/dl-vdso.c | 86 ++++++++++++----------- ldso/ldso/mips/dl-syscalls.h | 23 +++++- ldso/ldso/x86_64/dl-syscalls.h | 23 +++++- libc/sysdeps/linux/common/clock_gettime.c | 22 +++++- libc/sysdeps/linux/common/gettimeofday.c | 34 +++------ librt/clock_gettime.c | 18 ++++- 12 files changed, 230 insertions(+), 79 deletions(-) create mode 100644 ldso/ldso/dl-vdso-calls.h diff --git a/extra/Configs/Config.arm b/extra/Configs/Config.arm index 0d02e3f10..ea8ab4895 100644 --- a/extra/Configs/Config.arm +++ b/extra/Configs/Config.arm @@ -13,6 +13,7 @@ config FORCE_OPTIONS_FOR_ARCH select ARCH_ANY_ENDIAN select ARCH_HAS_UCONTEXT select ARCH_HAS_DEPRECATED_SYSCALLS + select ARCH_VDSO_SUPPORT config CONFIG_ARM_EABI bool "Build for EABI" diff --git a/ldso/include/dl-syscall.h b/ldso/include/dl-syscall.h index b40f58b10..4f41034ad 100644 --- a/ldso/include/dl-syscall.h +++ b/ldso/include/dl-syscall.h @@ -11,8 +11,11 @@ * been dynamicly linked in yet. */ #include "sys/syscall.h" extern int _dl_errno; + +#ifdef UCLIBC_LDSO #undef __set_errno #define __set_errno(X) {(_dl_errno) = (X);} +#endif /* Pull in the arch specific syscall implementation */ #include <dl-syscalls.h> diff --git a/ldso/include/ldso.h b/ldso/include/ldso.h index 80d5d5dd5..a397cce61 100755 --- a/ldso/include/ldso.h +++ b/ldso/include/ldso.h @@ -194,10 +194,6 @@ extern void *_dl_get_ready_to_run(struct elf_resolve *tpnt, DL_LOADADDR_TYPE loa #define AUX_MAX_AT_ID 40 extern ElfW(auxv_t) _dl_auxvt[AUX_MAX_AT_ID]; -void load_vdso( uint32_t sys_info_ehdr, char **envp ); - -#ifdef __VDSO_SUPPORT__ -extern void* _dl__vdso_gettimeofday; -#endif +void load_vdso(void *sys_info_ehdr, char **envp ); #endif /* _LDSO_H */ diff --git a/ldso/ldso/arm/dl-syscalls.h b/ldso/ldso/arm/dl-syscalls.h index f40c4fd31..5cbb94d30 100644 --- a/ldso/ldso/arm/dl-syscalls.h +++ b/ldso/ldso/arm/dl-syscalls.h @@ -1 +1,22 @@ -/* stub for arch-specific syscall issues */ +/* stub for arch-specific syscall issues/specific implementations */ + +#if defined(__VDSO_SUPPORT__) && !defined(UCLIBC_LDSO) + +#include "../dl-vdso-calls.h" + +static int __attribute__ ((used)) __arm_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp); +static int __attribute__ ((used)) __arm_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp) +{ + return __generic_vdso_clock_gettime(clock_id, tp); +} + +static int __attribute__ ((used)) __arm_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz); +static int __attribute__ ((used)) __arm_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz) +{ + return __generic_vdso_gettimeofday(tv, tz); +} + +#define ARCH_VDSO_GETTIMEOFDAY(tv, tz) __arm_vdso_gettimeofday(tv, tz) +#define ARCH_VDSO_CLOCK_GETTIME(clock_id, tp) __arm_vdso_clock_gettime(clock_id, tp) + +#endif /* defined(__VDSO_SUPPORT__) && !defined(UCLIBC_LDSO) */ \ No newline at end of file diff --git a/ldso/ldso/dl-startup.c b/ldso/ldso/dl-startup.c index 5b992a3a2..e7000bd87 100644 --- a/ldso/ldso/dl-startup.c +++ b/ldso/ldso/dl-startup.c @@ -373,7 +373,7 @@ DL_START(unsigned long args) DL_GET_READY_TO_RUN_EXTRA_ARGS); - load_vdso(_dl_auxvt[AT_SYSINFO_EHDR].a_un.a_val, envp); + load_vdso((void *)_dl_auxvt[AT_SYSINFO_EHDR].a_un.a_val, envp); /* Transfer control to the application. */ SEND_STDERR_DEBUG("transfering control to application @ "); diff --git a/ldso/ldso/dl-vdso-calls.h b/ldso/ldso/dl-vdso-calls.h new file mode 100644 index 000000000..c72f2dadf --- /dev/null +++ b/ldso/ldso/dl-vdso-calls.h @@ -0,0 +1,68 @@ +#ifndef _DL_VDSO_CALLS_H +#define _DL_VDSO_CALLS_H + +#include <sys/time.h> +#include <sys/syscall.h> +#include <errno.h> +#include <time.h> + +void __attribute__((weak)) *_get__dl__vdso_clock_gettime(void); +#if defined(__UCLIBC_USE_TIME64__) +#include "internal/time64_helpers.h" +void __attribute__((weak)) *_get__dl__vdso_clock_gettime64(void); +typedef int (*clock_gettime_func)(clockid_t clock_id, struct __ts64_struct *tp); +#else +typedef int (*clock_gettime_func)(clockid_t clock_id, struct timespec *tp); +#endif + +extern int __libc_clock_gettime(clockid_t clock_id, struct timespec *tp); + +static int __attribute__ ((used)) __generic_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp); +static int __attribute__ ((used)) __generic_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp) +{ + void *impl = NULL; +#if defined(__UCLIBC_USE_TIME64__) + if (&_get__dl__vdso_clock_gettime64 && (impl = _get__dl__vdso_clock_gettime64())) { + struct __ts64_struct __ts64; + int __ret = ((clock_gettime_func)impl)(clock_id, &__ts64); + if (__ret != 0) { + __set_errno(-__ret); + return -1; + } + + if (tp) { + tp->tv_sec = __ts64.tv_sec; + tp->tv_nsec = __ts64.tv_nsec; + } + return 0; + } + + /* fallback to syscall */ + return __libc_clock_gettime(clock_id, tp); +#else + if (&_get__dl__vdso_clock_gettime && (impl = _get__dl__vdso_clock_gettime())) { + int __ret = ((clock_gettime_func)impl)(clock_id, tp); + if (__ret != 0) { + __set_errno(-__ret); + return -1; + } + + return 0; + } + + /* fallback to syscall */ + return __libc_clock_gettime(clock_id, tp); +#endif +} + +static int __attribute__ ((used)) __generic_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz) +{ + struct timespec ts; + int __res = __generic_vdso_clock_gettime(CLOCK_REALTIME, &ts); + tv->tv_sec = ts.tv_sec; + tv->tv_usec = (suseconds_t)ts.tv_nsec / 1000; + + return __res; +} + +#endif /* _DL_VDSO_CALLS_H */ \ No newline at end of file diff --git a/ldso/ldso/dl-vdso.c b/ldso/ldso/dl-vdso.c index c8d724cd0..c23fd8b72 100755 --- a/ldso/ldso/dl-vdso.c +++ b/ldso/ldso/dl-vdso.c @@ -1,11 +1,7 @@ #include <elf.h> -//#include <stdio.h> #include <string.h> - #include "sys/auxv.h" -//#include <linux/time.h> -//#include <time.h> #include "ldso.h" #include "generated/autoconf.h" @@ -24,7 +20,7 @@ #ifndef __VDSO_SUPPORT__ - void load_vdso( uint32_t sys_info_ehdr, char **envp ){ + void load_vdso(void *sys_info_ehdr, char **envp ){ #ifdef __SUPPORT_LD_DEBUG__ if ( _dl_debug_vdso != 0 ){ _dl_dprintf(2,"_dl_vdso support not enabled\n" ); @@ -34,21 +30,34 @@ } #else +void *_dl__vdso_gettimeofday = 0; +void *_dl__vdso_clock_gettime = 0; +#if defined(__UCLIBC_USE_TIME64__) +void *_dl__vdso_clock_gettime64 = 0; +#endif - - - -//typedef long (*gtod_t)(struct timeval *tv, struct timezone *tz); -void* _dl__vdso_gettimeofday = 0; - - -//typedef long (*clock_gettime_t)(int clk_id, struct timespec *tp); -void* _dl__vdso_clock_gettime = 0; +void *_get__dl__vdso_clock_gettime(void); +void *_get__dl__vdso_clock_gettime(void) +{ + return _dl__vdso_clock_gettime; +} +#if defined(__UCLIBC_USE_TIME64__) +void *_get__dl__vdso_clock_gettime64(void); +void *_get__dl__vdso_clock_gettime64(void) +{ + return _dl__vdso_clock_gettime64; +} +#endif +void *_get__dl__vdso_gettimeofday(void); +void *_get__dl__vdso_gettimeofday(void) +{ + return _dl__vdso_gettimeofday; +} -typedef struct{ +typedef struct { void* base_addr; @@ -75,7 +84,7 @@ typedef struct{ char* vers_strings[10]; -}elf_infos; +} elf_infos; /* * the raise() dummy function is needed because of divisons in this code @@ -111,13 +120,10 @@ static ELF(Shdr) *vdso_get_sec_header( elf_infos* elf, int index ){ } - -void load_vdso( uint32_t sys_info_ehdr, char **envp ){ +void load_vdso(void *sys_info_ehdr, char **envp ){ elf_infos vdso_infos; - - - + if ( sys_info_ehdr == 0 ){ #ifdef __SUPPORT_LD_DEBUG__ if ( _dl_debug_vdso != 0 ){ @@ -137,22 +143,16 @@ void load_vdso( uint32_t sys_info_ehdr, char **envp ){ #endif return; } - - + _dl_memset( &vdso_infos, 0 , sizeof( elf_infos ) ); - - + vdso_infos.base_addr = (void*)sys_info_ehdr; vdso_infos.hdr = (ELF(Ehdr)*)vdso_infos.base_addr; - - //printf("base : %p\n",vdso_infos.base_addr); - + if ( 0 != vdso_check_elf_header( &vdso_infos ) ){ return; } - - - + ELF(Shdr) *sec_header = vdso_get_sec_header( &vdso_infos, vdso_infos.hdr->e_shstrndx); vdso_infos.section_header_strtab = ( vdso_infos.base_addr + sec_header->sh_offset ); @@ -307,10 +307,7 @@ void load_vdso( uint32_t sys_info_ehdr, char **envp ){ if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC) continue; - - char* name = vdso_infos.dynstr_table + sym->st_name; - char* vers = vdso_infos.vers_strings[ vdso_infos.versym_table[i] ]; void* func_addr = (void*)( vdso_infos.base_addr + sym->st_value ); // the function name is patched to zero if the kernel has no timer which is @@ -324,9 +321,6 @@ void load_vdso( uint32_t sys_info_ehdr, char **envp ){ continue; } - //printf(" %s@@%s\n", name , vers ); - - //print_sym( sym ); if ( 0 == _dl_strcmp( name, "__vdso_gettimeofday" ) ){ _dl__vdso_gettimeofday = func_addr; #ifdef __SUPPORT_LD_DEBUG__ @@ -336,7 +330,6 @@ void load_vdso( uint32_t sys_info_ehdr, char **envp ){ #endif continue; } - if ( 0 == _dl_strcmp( name, "__vdso_clock_gettime" ) ){ _dl__vdso_clock_gettime = func_addr; #ifdef __SUPPORT_LD_DEBUG__ @@ -346,17 +339,26 @@ void load_vdso( uint32_t sys_info_ehdr, char **envp ){ #endif continue; } + +#if defined(__UCLIBC_USE_TIME64__) + if ( 0 == _dl_strcmp( name, "__vdso_clock_gettime64" ) ){ + _dl__vdso_clock_gettime64 = func_addr; +#ifdef __SUPPORT_LD_DEBUG__ + if ( _dl_debug_vdso != 0 ){ + _dl_dprintf(2," %s at address %p\n", name, func_addr ); + } +#endif + continue; + } +#endif /* defined(__UCLIBC_USE_TIME64__) */ #ifdef __SUPPORT_LD_DEBUG__ if ( _dl_debug_vdso != 0 ){ _dl_dprintf(2," <%s> not handled\n", name ); } #endif - - - } - + } } #endif // __VDSO_SUPPORT__ diff --git a/ldso/ldso/mips/dl-syscalls.h b/ldso/ldso/mips/dl-syscalls.h index f40c4fd31..46fd07bfa 100644 --- a/ldso/ldso/mips/dl-syscalls.h +++ b/ldso/ldso/mips/dl-syscalls.h @@ -1 +1,22 @@ -/* stub for arch-specific syscall issues */ +/* stub for arch-specific syscall issues/specific implementations */ + +#if defined(__VDSO_SUPPORT__) && !defined(UCLIBC_LDSO) + +#include "../dl-vdso-calls.h" + +static int __attribute__ ((used)) __mips_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp); +static int __attribute__ ((used)) __mips_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp) +{ + return __generic_vdso_clock_gettime(clock_id, tp); +} + +static int __attribute__ ((used)) __mips_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz); +static int __attribute__ ((used)) __mips_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz) +{ + return __generic_vdso_gettimeofday(tv, tz); +} + +#define ARCH_VDSO_GETTIMEOFDAY(tv, tz) __mips_vdso_gettimeofday(tv, tz) +#define ARCH_VDSO_CLOCK_GETTIME(clock_id, tp) __mips_vdso_clock_gettime(clock_id, tp) + +#endif /* defined(__VDSO_SUPPORT__) && !defined(UCLIBC_LDSO) */ \ No newline at end of file diff --git a/ldso/ldso/x86_64/dl-syscalls.h b/ldso/ldso/x86_64/dl-syscalls.h index f40c4fd31..0ec7a7dbb 100644 --- a/ldso/ldso/x86_64/dl-syscalls.h +++ b/ldso/ldso/x86_64/dl-syscalls.h @@ -1 +1,22 @@ -/* stub for arch-specific syscall issues */ +/* stub for arch-specific syscall issues/specific implementations */ + +#if defined(__VDSO_SUPPORT__) && !defined(UCLIBC_LDSO) + +#include "../dl-vdso-calls.h" + +static int __attribute__ ((used)) __x86_64_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp); +static int __attribute__ ((used)) __x86_64_vdso_clock_gettime(clockid_t clock_id, struct timespec *tp) +{ + return __generic_vdso_clock_gettime(clock_id, tp); +} + +static int __attribute__ ((used)) __x86_64_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz); +static int __attribute__ ((used)) __x86_64_vdso_gettimeofday(struct timeval *tv, __timezone_ptr_t tz) +{ + return __generic_vdso_gettimeofday(tv, tz); +} + +#define ARCH_VDSO_GETTIMEOFDAY(tv, tz) __x86_64_vdso_gettimeofday(tv, tz) +#define ARCH_VDSO_CLOCK_GETTIME(clock_id, tp) __x86_64_vdso_clock_gettime(clock_id, tp) + +#endif /* defined(__VDSO_SUPPORT__) && !defined(UCLIBC_LDSO) */ \ No newline at end of file diff --git a/libc/sysdeps/linux/common/clock_gettime.c b/libc/sysdeps/linux/common/clock_gettime.c index 4d787b9b7..b924d860b 100644 --- a/libc/sysdeps/linux/common/clock_gettime.c +++ b/libc/sysdeps/linux/common/clock_gettime.c @@ -11,10 +11,14 @@ #include <sys/syscall.h> #include <time.h> +#ifdef __VDSO_SUPPORT__ +#include "ldso.h" +#endif + #if defined(__UCLIBC_USE_TIME64__) && defined(__NR_clock_gettime64) #include "internal/time64_helpers.h" -int clock_gettime(clockid_t clock_id, struct timespec *tp) +int __libc_clock_gettime(clockid_t clock_id, struct timespec *tp) { struct __ts64_struct __ts64; int __ret = INLINE_SYSCALL(clock_gettime64, 2, clock_id, &__ts64); @@ -26,11 +30,14 @@ int clock_gettime(clockid_t clock_id, struct timespec *tp) return __ret; } #elif defined(__NR_clock_gettime) -_syscall2(int, clock_gettime, clockid_t, clock_id, struct timespec*, tp) +int __libc_clock_gettime(clockid_t clock_id, struct timespec *tp) +{ + return INLINE_SYSCALL(clock_gettime, 2, clock_id, tp); +} #else # include <sys/time.h> -int clock_gettime(clockid_t clock_id, struct timespec* tp) +int __libc_clock_gettime(clockid_t clock_id, struct timespec* tp) { struct timeval tv; int retval = -1; @@ -53,3 +60,12 @@ int clock_gettime(clockid_t clock_id, struct timespec* tp) return retval; } #endif + +int clock_gettime(clockid_t clock_id, struct timespec *tp) +{ +#if defined(__VDSO_SUPPORT__) && defined(ARCH_VDSO_CLOCK_GETTIME) + return ARCH_VDSO_CLOCK_GETTIME(clock_id, tp); +#else + return __libc_clock_gettime(clock_id, tp); +#endif +} diff --git a/libc/sysdeps/linux/common/gettimeofday.c b/libc/sysdeps/linux/common/gettimeofday.c index ed18b2be5..9b29fe5a0 100755 --- a/libc/sysdeps/linux/common/gettimeofday.c +++ b/libc/sysdeps/linux/common/gettimeofday.c @@ -14,30 +14,18 @@ #include "ldso.h" #endif -#ifdef __VDSO_SUPPORT__ -typedef int (*gettimeofday_func)(struct timeval * tv, __timezone_ptr_t tz); -#endif - int gettimeofday(struct timeval * tv, __timezone_ptr_t tz) { - - #ifdef __VDSO_SUPPORT__ - if ( _dl__vdso_gettimeofday != 0 ){ - gettimeofday_func func= _dl__vdso_gettimeofday; - return func( tv, tz ); - - }else{ - _syscall2_body(int, gettimeofday, struct timeval *, tv, __timezone_ptr_t, tz) - } - #else - if (!tv) - return 0; - - struct timespec __ts; - int __ret = clock_gettime(CLOCK_REALTIME, &__ts); - tv->tv_sec = __ts.tv_sec; - tv->tv_usec = (suseconds_t)__ts.tv_nsec / 1000; - return __ret; - #endif + if (!tv) + return 0; +#if defined(__VDSO_SUPPORT__) && defined(ARCH_VDSO_GETTIMEOFDAY) + return ARCH_VDSO_GETTIMEOFDAY(tv, tz); +#else + struct timespec __ts; + int __ret = clock_gettime(CLOCK_REALTIME, &__ts); + tv->tv_sec = __ts.tv_sec; + tv->tv_usec = (suseconds_t)__ts.tv_nsec / 1000; + return __ret; +#endif } diff --git a/librt/clock_gettime.c b/librt/clock_gettime.c index eab9a3343..dd1f0c514 100644 --- a/librt/clock_gettime.c +++ b/librt/clock_gettime.c @@ -22,6 +22,10 @@ #include <sys/time.h> #include "kernel-posix-cpu-timers.h" +#ifdef __VDSO_SUPPORT__ +#include "ldso.h" +#endif + #if defined(__UCLIBC_USE_TIME64__) #include "internal/time64_helpers.h" #endif @@ -68,9 +72,8 @@ realtime_gettime (struct timespec *tp) return retval; } -/* Get current value of CLOCK and store it in TP. */ int -clock_gettime (clockid_t clock_id, struct timespec *tp) +__libc_clock_gettime (clockid_t clock_id, struct timespec *tp) { int retval = -1; #ifndef HANDLED_REALTIME @@ -101,3 +104,14 @@ clock_gettime (clockid_t clock_id, struct timespec *tp) return retval; } + +/* Get current value of CLOCK and store it in TP. */ +int +clock_gettime (clockid_t clock_id, struct timespec *tp) +{ +#if defined(__VDSO_SUPPORT__) && defined(ARCH_VDSO_CLOCK_GETTIME) + return ARCH_VDSO_CLOCK_GETTIME(clock_id, tp); +#else + return __libc_clock_gettime(clock_id, tp); +#endif +} -- 2.44.0