diff --git a/doc/libffi.texi b/doc/libffi.texi index b9887a8c..ff72e583 100644 --- a/doc/libffi.texi +++ b/doc/libffi.texi @@ -152,7 +152,7 @@ If the function being called is variadic (varargs) then @code{ffi_prep_cif_var} must be used instead of @code{ffi_prep_cif}. @findex ffi_prep_cif_var -@defun ffi_status ffi_prep_cif_var (ffi_cif *@var{cif}, ffi_abi var{abi}, unsigned int @var{nfixedargs}, unsigned int var{ntotalargs}, ffi_type *@var{rtype}, ffi_type **@var{argtypes}) +@defun ffi_status ffi_prep_cif_var (ffi_cif *@var{cif}, ffi_abi @var{abi}, unsigned int @var{nfixedargs}, unsigned int @var{ntotalargs}, ffi_type *@var{rtype}, ffi_type **@var{argtypes}) This initializes @var{cif} according to the given parameters for a call to a variadic function. In general it's operation is the same as for @code{ffi_prep_cif} except that: @@ -161,7 +161,7 @@ same as for @code{ffi_prep_cif} except that: variadic arguments. It must be greater than zero. @var{ntotalargs} the total number of arguments, including variadic -and fixed arguments. +and fixed arguments. @var{argtypes} must have this many elements. Note that, different cif's must be prepped for calls to the same function when different numbers of arguments are passed. diff --git a/src/aarch64/ffi.c b/src/aarch64/ffi.c index f79602bb..cf33abc2 100644 --- a/src/aarch64/ffi.c +++ b/src/aarch64/ffi.c @@ -22,6 +22,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include #include #include +#include #include #include #include "internal.h" @@ -70,6 +71,14 @@ ffi_clear_cache (void *start, void *end) #endif } +#if FFI_EXEC_TRAMPOLINE_TABLE + +#ifdef __MACH__ +#include +#endif + +#endif + /* A subroutine of is_vfp_type. Given a structure type, return the type code of the first non-structure element. Recurse for structure elements. Return -1 if the structure is in fact empty, i.e. no nested elements. */ @@ -725,240 +734,6 @@ ffi_call_go (ffi_cif *cif, void (*fn) (void), void *rvalue, extern void ffi_closure_SYSV (void) FFI_HIDDEN; extern void ffi_closure_SYSV_V (void) FFI_HIDDEN; -#if FFI_EXEC_TRAMPOLINE_TABLE - -#include -#include -#include -#include - -extern void *ffi_closure_trampoline_table_page; - -typedef struct ffi_trampoline_table ffi_trampoline_table; -typedef struct ffi_trampoline_table_entry ffi_trampoline_table_entry; - -struct ffi_trampoline_table -{ - /* contiguous writable and executable pages */ - vm_address_t config_page; - vm_address_t trampoline_page; - - /* free list tracking */ - uint16_t free_count; - ffi_trampoline_table_entry *free_list; - ffi_trampoline_table_entry *free_list_pool; - - ffi_trampoline_table *prev; - ffi_trampoline_table *next; -}; - -struct ffi_trampoline_table_entry -{ - void *(*trampoline) (); - ffi_trampoline_table_entry *next; -}; - -/* The trampoline configuration is placed a page prior to the trampoline's entry point */ -#define FFI_TRAMPOLINE_CODELOC_CONFIG(codeloc) ((void **) (((uint8_t *) codeloc) - PAGE_SIZE)); - -/* Total number of trampolines that fit in one trampoline table */ -#define FFI_TRAMPOLINE_COUNT (PAGE_SIZE / FFI_TRAMPOLINE_SIZE) - -static pthread_mutex_t ffi_trampoline_lock = PTHREAD_MUTEX_INITIALIZER; -static ffi_trampoline_table *ffi_trampoline_tables = NULL; - -static ffi_trampoline_table * -ffi_trampoline_table_alloc () -{ - ffi_trampoline_table *table = NULL; - - /* Loop until we can allocate two contiguous pages */ - while (table == NULL) - { - vm_address_t config_page = 0x0; - kern_return_t kt; - - /* Try to allocate two pages */ - kt = - vm_allocate (mach_task_self (), &config_page, PAGE_SIZE * 2, - VM_FLAGS_ANYWHERE); - if (kt != KERN_SUCCESS) - { - fprintf (stderr, "vm_allocate() failure: %d at %s:%d\n", kt, - __FILE__, __LINE__); - break; - } - - /* Now drop the second half of the allocation to make room for the trampoline table */ - vm_address_t trampoline_page = config_page + PAGE_SIZE; - kt = vm_deallocate (mach_task_self (), trampoline_page, PAGE_SIZE); - if (kt != KERN_SUCCESS) - { - fprintf (stderr, "vm_deallocate() failure: %d at %s:%d\n", kt, - __FILE__, __LINE__); - break; - } - - /* Remap the trampoline table to directly follow the config page */ - vm_prot_t cur_prot; - vm_prot_t max_prot; - - kt = - vm_remap (mach_task_self (), &trampoline_page, PAGE_SIZE, 0x0, FALSE, - mach_task_self (), - (vm_address_t) & ffi_closure_trampoline_table_page, FALSE, - &cur_prot, &max_prot, VM_INHERIT_SHARE); - - /* If we lost access to the destination trampoline page, drop our config allocation mapping and retry */ - if (kt != KERN_SUCCESS) - { - /* Log unexpected failures */ - if (kt != KERN_NO_SPACE) - { - fprintf (stderr, "vm_remap() failure: %d at %s:%d\n", kt, - __FILE__, __LINE__); - } - - vm_deallocate (mach_task_self (), config_page, PAGE_SIZE); - continue; - } - - /* We have valid trampoline and config pages */ - table = calloc (1, sizeof (ffi_trampoline_table)); - table->free_count = FFI_TRAMPOLINE_COUNT; - table->config_page = config_page; - table->trampoline_page = trampoline_page; - - /* Create and initialize the free list */ - table->free_list_pool = - calloc (FFI_TRAMPOLINE_COUNT, sizeof (ffi_trampoline_table_entry)); - - uint16_t i; - for (i = 0; i < table->free_count; i++) - { - ffi_trampoline_table_entry *entry = &table->free_list_pool[i]; - entry->trampoline = - (void *) (table->trampoline_page + (i * FFI_TRAMPOLINE_SIZE)); - - if (i < table->free_count - 1) - entry->next = &table->free_list_pool[i + 1]; - } - - table->free_list = table->free_list_pool; - } - - return table; -} - -void * -ffi_closure_alloc (size_t size, void **code) -{ - /* Create the closure */ - ffi_closure *closure = malloc (size); - if (closure == NULL) - return NULL; - - pthread_mutex_lock (&ffi_trampoline_lock); - - /* Check for an active trampoline table with available entries. */ - ffi_trampoline_table *table = ffi_trampoline_tables; - if (table == NULL || table->free_list == NULL) - { - table = ffi_trampoline_table_alloc (); - if (table == NULL) - { - free (closure); - return NULL; - } - - /* Insert the new table at the top of the list */ - table->next = ffi_trampoline_tables; - if (table->next != NULL) - table->next->prev = table; - - ffi_trampoline_tables = table; - } - - /* Claim the free entry */ - ffi_trampoline_table_entry *entry = ffi_trampoline_tables->free_list; - ffi_trampoline_tables->free_list = entry->next; - ffi_trampoline_tables->free_count--; - entry->next = NULL; - - pthread_mutex_unlock (&ffi_trampoline_lock); - - /* Initialize the return values */ - *code = entry->trampoline; - closure->trampoline_table = table; - closure->trampoline_table_entry = entry; - - return closure; -} - -void -ffi_closure_free (void *ptr) -{ - ffi_closure *closure = ptr; - - pthread_mutex_lock (&ffi_trampoline_lock); - - /* Fetch the table and entry references */ - ffi_trampoline_table *table = closure->trampoline_table; - ffi_trampoline_table_entry *entry = closure->trampoline_table_entry; - - /* Return the entry to the free list */ - entry->next = table->free_list; - table->free_list = entry; - table->free_count++; - - /* If all trampolines within this table are free, and at least one other table exists, deallocate - * the table */ - if (table->free_count == FFI_TRAMPOLINE_COUNT - && ffi_trampoline_tables != table) - { - /* Remove from the list */ - if (table->prev != NULL) - table->prev->next = table->next; - - if (table->next != NULL) - table->next->prev = table->prev; - - /* Deallocate pages */ - kern_return_t kt; - kt = vm_deallocate (mach_task_self (), table->config_page, PAGE_SIZE); - if (kt != KERN_SUCCESS) - fprintf (stderr, "vm_deallocate() failure: %d at %s:%d\n", kt, - __FILE__, __LINE__); - - kt = - vm_deallocate (mach_task_self (), table->trampoline_page, PAGE_SIZE); - if (kt != KERN_SUCCESS) - fprintf (stderr, "vm_deallocate() failure: %d at %s:%d\n", kt, - __FILE__, __LINE__); - - /* Deallocate free list */ - free (table->free_list_pool); - free (table); - } - else if (ffi_trampoline_tables != table) - { - /* Otherwise, bump this table to the top of the list */ - table->prev = NULL; - table->next = ffi_trampoline_tables; - if (ffi_trampoline_tables != NULL) - ffi_trampoline_tables->prev = table; - - ffi_trampoline_tables = table; - } - - pthread_mutex_unlock (&ffi_trampoline_lock); - - /* Free the closure */ - free (closure); -} - -#endif - ffi_status ffi_prep_closure_loc (ffi_closure *closure, ffi_cif* cif, @@ -977,9 +752,11 @@ ffi_prep_closure_loc (ffi_closure *closure, start = ffi_closure_SYSV; #if FFI_EXEC_TRAMPOLINE_TABLE - void **config = FFI_TRAMPOLINE_CODELOC_CONFIG (codeloc); +#ifdef __MACH__ + void **config = (void **)((uint8_t *)codeloc - PAGE_MAX_SIZE); config[0] = closure; config[1] = start; +#endif #else static const unsigned char trampoline[16] = { 0x90, 0x00, 0x00, 0x58, /* ldr x16, tramp+16 */ diff --git a/src/aarch64/ffitarget.h b/src/aarch64/ffitarget.h index 2862ec7f..5ded0e43 100644 --- a/src/aarch64/ffitarget.h +++ b/src/aarch64/ffitarget.h @@ -48,14 +48,21 @@ typedef enum ffi_abi /* ---- Definitions for closures ----------------------------------------- */ #define FFI_CLOSURES 1 -#if defined (__APPLE__) -#define FFI_TRAMPOLINE_SIZE 20 +#define FFI_NATIVE_RAW_API 0 + +#if defined (FFI_EXEC_TRAMPOLINE_TABLE) && FFI_EXEC_TRAMPOLINE_TABLE + +#ifdef __MACH__ +#define FFI_TRAMPOLINE_SIZE 16 #define FFI_TRAMPOLINE_CLOSURE_OFFSET 16 +#else +#error "No trampoline table implementation" +#endif + #else #define FFI_TRAMPOLINE_SIZE 24 #define FFI_TRAMPOLINE_CLOSURE_OFFSET FFI_TRAMPOLINE_SIZE #endif -#define FFI_NATIVE_RAW_API 0 /* ---- Internal ---- */ diff --git a/src/aarch64/sysv.S b/src/aarch64/sysv.S index c1bf9b98..7936c0e5 100644 --- a/src/aarch64/sysv.S +++ b/src/aarch64/sysv.S @@ -356,16 +356,18 @@ CNAME(ffi_closure_SYSV): #endif #if FFI_EXEC_TRAMPOLINE_TABLE - .align 12 + +#ifdef __MACH__ +#include + .align PAGE_MAX_SHIFT CNAME(ffi_closure_trampoline_table_page): - .rept 16384 / FFI_TRAMPOLINE_SIZE - adr x17, -16384 - adr x16, -16380 - ldr x16, [x16] - ldr x17, [x17] - br x16 + .rept PAGE_MAX_SIZE / FFI_TRAMPOLINE_SIZE + adr x16, -PAGE_MAX_SIZE + ldp x17, x16, [x16] + br x16 + nop /* each entry in the trampoline config page is 2*sizeof(void*) so the trampoline itself cannot be smaller that 16 bytes */ .endr - + .globl CNAME(ffi_closure_trampoline_table_page) #ifdef __ELF__ .type CNAME(ffi_closure_trampoline_table_page), #function @@ -374,6 +376,8 @@ CNAME(ffi_closure_trampoline_table_page): #endif #endif +#endif /* FFI_EXEC_TRAMPOLINE_TABLE */ + #ifdef FFI_GO_CLOSURES .align 4 CNAME(ffi_go_closure_SYSV_V): diff --git a/src/arm/ffi.c b/src/arm/ffi.c index 9c8732d1..c24085d3 100644 --- a/src/arm/ffi.c +++ b/src/arm/ffi.c @@ -28,11 +28,22 @@ DEALINGS IN THE SOFTWARE. ----------------------------------------------------------------------- */ +#include #include #include #include #include "internal.h" +#if FFI_EXEC_TRAMPOLINE_TABLE + +#ifdef __MACH__ +#include +#endif + +#else +extern unsigned int ffi_arm_trampoline[2] FFI_HIDDEN; +#endif + /* Forward declares. */ static int vfp_type_p (const ffi_type *); static void layout_vfp_args (ffi_cif *); @@ -530,252 +541,6 @@ void ffi_closure_VFP (void) FFI_HIDDEN; void ffi_go_closure_SYSV (void) FFI_HIDDEN; void ffi_go_closure_VFP (void) FFI_HIDDEN; -#if FFI_EXEC_TRAMPOLINE_TABLE - -#include -#include -#include -#include - -extern void *ffi_closure_trampoline_table_page; - -typedef struct ffi_trampoline_table ffi_trampoline_table; -typedef struct ffi_trampoline_table_entry ffi_trampoline_table_entry; - -struct ffi_trampoline_table -{ - /* contiguous writable and executable pages */ - vm_address_t config_page; - vm_address_t trampoline_page; - - /* free list tracking */ - uint16_t free_count; - ffi_trampoline_table_entry *free_list; - ffi_trampoline_table_entry *free_list_pool; - - ffi_trampoline_table *prev; - ffi_trampoline_table *next; -}; - -struct ffi_trampoline_table_entry -{ - void *(*trampoline) (); - ffi_trampoline_table_entry *next; -}; - -/* Override the standard architecture trampoline size */ -// XXX TODO - Fix -#undef FFI_TRAMPOLINE_SIZE -#define FFI_TRAMPOLINE_SIZE 12 - -/* The trampoline configuration is placed at 4080 bytes prior to the trampoline's entry point */ -#define FFI_TRAMPOLINE_CODELOC_CONFIG(codeloc) ((void **) (((uint8_t *) codeloc) - 4080)); - -/* The first 16 bytes of the config page are unused, as they are unaddressable from the trampoline page. */ -#define FFI_TRAMPOLINE_CONFIG_PAGE_OFFSET 16 - -/* Total number of trampolines that fit in one trampoline table */ -#define FFI_TRAMPOLINE_COUNT ((PAGE_SIZE - FFI_TRAMPOLINE_CONFIG_PAGE_OFFSET) / FFI_TRAMPOLINE_SIZE) - -static pthread_mutex_t ffi_trampoline_lock = PTHREAD_MUTEX_INITIALIZER; -static ffi_trampoline_table *ffi_trampoline_tables = NULL; - -static ffi_trampoline_table * -ffi_trampoline_table_alloc () -{ - ffi_trampoline_table *table = NULL; - - /* Loop until we can allocate two contiguous pages */ - while (table == NULL) - { - vm_address_t config_page = 0x0; - kern_return_t kt; - - /* Try to allocate two pages */ - kt = - vm_allocate (mach_task_self (), &config_page, PAGE_SIZE * 2, - VM_FLAGS_ANYWHERE); - if (kt != KERN_SUCCESS) - { - fprintf (stderr, "vm_allocate() failure: %d at %s:%d\n", kt, - __FILE__, __LINE__); - break; - } - - /* Now drop the second half of the allocation to make room for the trampoline table */ - vm_address_t trampoline_page = config_page + PAGE_SIZE; - kt = vm_deallocate (mach_task_self (), trampoline_page, PAGE_SIZE); - if (kt != KERN_SUCCESS) - { - fprintf (stderr, "vm_deallocate() failure: %d at %s:%d\n", kt, - __FILE__, __LINE__); - break; - } - - /* Remap the trampoline table to directly follow the config page */ - vm_prot_t cur_prot; - vm_prot_t max_prot; - - kt = - vm_remap (mach_task_self (), &trampoline_page, PAGE_SIZE, 0x0, FALSE, - mach_task_self (), - (vm_address_t) & ffi_closure_trampoline_table_page, FALSE, - &cur_prot, &max_prot, VM_INHERIT_SHARE); - - /* If we lost access to the destination trampoline page, drop our config allocation mapping and retry */ - if (kt != KERN_SUCCESS) - { - /* Log unexpected failures */ - if (kt != KERN_NO_SPACE) - { - fprintf (stderr, "vm_remap() failure: %d at %s:%d\n", kt, - __FILE__, __LINE__); - } - - vm_deallocate (mach_task_self (), config_page, PAGE_SIZE); - continue; - } - - /* We have valid trampoline and config pages */ - table = calloc (1, sizeof (ffi_trampoline_table)); - table->free_count = FFI_TRAMPOLINE_COUNT; - table->config_page = config_page; - table->trampoline_page = trampoline_page; - - /* Create and initialize the free list */ - table->free_list_pool = - calloc (FFI_TRAMPOLINE_COUNT, sizeof (ffi_trampoline_table_entry)); - - uint16_t i; - for (i = 0; i < table->free_count; i++) - { - ffi_trampoline_table_entry *entry = &table->free_list_pool[i]; - entry->trampoline = - (void *) (table->trampoline_page + (i * FFI_TRAMPOLINE_SIZE)); - - if (i < table->free_count - 1) - entry->next = &table->free_list_pool[i + 1]; - } - - table->free_list = table->free_list_pool; - } - - return table; -} - -void * -ffi_closure_alloc (size_t size, void **code) -{ - /* Create the closure */ - ffi_closure *closure = malloc (size); - if (closure == NULL) - return NULL; - - pthread_mutex_lock (&ffi_trampoline_lock); - - /* Check for an active trampoline table with available entries. */ - ffi_trampoline_table *table = ffi_trampoline_tables; - if (table == NULL || table->free_list == NULL) - { - table = ffi_trampoline_table_alloc (); - if (table == NULL) - { - free (closure); - return NULL; - } - - /* Insert the new table at the top of the list */ - table->next = ffi_trampoline_tables; - if (table->next != NULL) - table->next->prev = table; - - ffi_trampoline_tables = table; - } - - /* Claim the free entry */ - ffi_trampoline_table_entry *entry = ffi_trampoline_tables->free_list; - ffi_trampoline_tables->free_list = entry->next; - ffi_trampoline_tables->free_count--; - entry->next = NULL; - - pthread_mutex_unlock (&ffi_trampoline_lock); - - /* Initialize the return values */ - *code = entry->trampoline; - closure->trampoline_table = table; - closure->trampoline_table_entry = entry; - - return closure; -} - -void -ffi_closure_free (void *ptr) -{ - ffi_closure *closure = ptr; - - pthread_mutex_lock (&ffi_trampoline_lock); - - /* Fetch the table and entry references */ - ffi_trampoline_table *table = closure->trampoline_table; - ffi_trampoline_table_entry *entry = closure->trampoline_table_entry; - - /* Return the entry to the free list */ - entry->next = table->free_list; - table->free_list = entry; - table->free_count++; - - /* If all trampolines within this table are free, and at least one other table exists, deallocate - * the table */ - if (table->free_count == FFI_TRAMPOLINE_COUNT - && ffi_trampoline_tables != table) - { - /* Remove from the list */ - if (table->prev != NULL) - table->prev->next = table->next; - - if (table->next != NULL) - table->next->prev = table->prev; - - /* Deallocate pages */ - kern_return_t kt; - kt = vm_deallocate (mach_task_self (), table->config_page, PAGE_SIZE); - if (kt != KERN_SUCCESS) - fprintf (stderr, "vm_deallocate() failure: %d at %s:%d\n", kt, - __FILE__, __LINE__); - - kt = - vm_deallocate (mach_task_self (), table->trampoline_page, PAGE_SIZE); - if (kt != KERN_SUCCESS) - fprintf (stderr, "vm_deallocate() failure: %d at %s:%d\n", kt, - __FILE__, __LINE__); - - /* Deallocate free list */ - free (table->free_list_pool); - free (table); - } - else if (ffi_trampoline_tables != table) - { - /* Otherwise, bump this table to the top of the list */ - table->prev = NULL; - table->next = ffi_trampoline_tables; - if (ffi_trampoline_tables != NULL) - ffi_trampoline_tables->prev = table; - - ffi_trampoline_tables = table; - } - - pthread_mutex_unlock (&ffi_trampoline_lock); - - /* Free the closure */ - free (closure); -} - -#else - -extern unsigned int ffi_arm_trampoline[2] FFI_HIDDEN; - -#endif - /* the cif must already be prep'ed */ ffi_status @@ -796,7 +561,7 @@ ffi_prep_closure_loc (ffi_closure * closure, return FFI_BAD_ABI; #if FFI_EXEC_TRAMPOLINE_TABLE - void **config = FFI_TRAMPOLINE_CODELOC_CONFIG (codeloc); + void **config = (void **)((uint8_t *)codeloc - PAGE_MAX_SIZE); config[0] = closure; config[1] = closure_func; #else diff --git a/src/arm/ffitarget.h b/src/arm/ffitarget.h index 4f473f92..1cf10364 100644 --- a/src/arm/ffitarget.h +++ b/src/arm/ffitarget.h @@ -63,7 +63,20 @@ typedef enum ffi_abi { #define FFI_CLOSURES 1 #define FFI_GO_CLOSURES 1 -#define FFI_TRAMPOLINE_SIZE 12 #define FFI_NATIVE_RAW_API 0 +#if defined (FFI_EXEC_TRAMPOLINE_TABLE) && FFI_EXEC_TRAMPOLINE_TABLE + +#ifdef __MACH__ +#define FFI_TRAMPOLINE_SIZE 12 +#define FFI_TRAMPOLINE_CLOSURE_OFFSET 8 +#else +#error "No trampoline table implementation" +#endif + +#else +#define FFI_TRAMPOLINE_SIZE 12 +#define FFI_TRAMPOLINE_CLOSURE_OFFSET FFI_TRAMPOLINE_SIZE +#endif + #endif diff --git a/src/arm/sysv.S b/src/arm/sysv.S index 9398081d..379dcdf6 100644 --- a/src/arm/sysv.S +++ b/src/arm/sysv.S @@ -228,9 +228,13 @@ ARM_FUNC_START(ffi_closure_SYSV) cfi_startproc stmdb sp!, {r0-r3} @ save argument regs cfi_adjust_cfa_offset(16) - ldr r0, [ip, #FFI_TRAMPOLINE_SIZE] @ load cif - ldr r1, [ip, #FFI_TRAMPOLINE_SIZE+4] @ load fun - ldr r2, [ip, #FFI_TRAMPOLINE_SIZE+8] @ load user_data + +#if FFI_EXEC_TRAMPOLINE_TABLE + ldr ip, [ip] @ ip points to the config page, dereference to get the ffi_closure* +#endif + ldr r0, [ip, #FFI_TRAMPOLINE_CLOSURE_OFFSET] @ load cif + ldr r1, [ip, #FFI_TRAMPOLINE_CLOSURE_OFFSET+4] @ load fun + ldr r2, [ip, #FFI_TRAMPOLINE_CLOSURE_OFFSET+8] @ load user_data 0: add ip, sp, #16 @ compute entry sp sub sp, sp, #64+32 @ allocate frame @@ -271,9 +275,13 @@ ARM_FUNC_START(ffi_closure_VFP) cfi_startproc stmdb sp!, {r0-r3} @ save argument regs cfi_adjust_cfa_offset(16) - ldr r0, [ip, #FFI_TRAMPOLINE_SIZE] @ load cif - ldr r1, [ip, #FFI_TRAMPOLINE_SIZE+4] @ load fun - ldr r2, [ip, #FFI_TRAMPOLINE_SIZE+8] @ load user_data + +#if FFI_EXEC_TRAMPOLINE_TABLE + ldr ip, [ip] @ ip points to the config page, dereference to get the ffi_closure* +#endif + ldr r0, [ip, #FFI_TRAMPOLINE_CLOSURE_OFFSET] @ load cif + ldr r1, [ip, #FFI_TRAMPOLINE_CLOSURE_OFFSET+4] @ load fun + ldr r2, [ip, #FFI_TRAMPOLINE_CLOSURE_OFFSET+8] @ load user_data 0: add ip, sp, #16 sub sp, sp, #64+32 @ allocate frame @@ -347,23 +355,18 @@ ARM_FUNC_END(ffi_closure_ret) #if FFI_EXEC_TRAMPOLINE_TABLE -/* ??? The iOS support should be updated. The first insn used to - be STMFD, but that's been moved into ffi_closure_SYSV. If the - writable page is put after this one we can make use of the - pc+8 feature of the architecture. We can also reduce the size - of the thunk to 8 and pack more of these into the page. +#ifdef __MACH__ +#include - In the meantime, simply replace the STMFD with a NOP so as to - keep all the magic numbers the same within ffi.c. */ - - .align 12 +.align PAGE_MAX_SHIFT ARM_FUNC_START(ffi_closure_trampoline_table_page) -.rept 4096 / 12 - nop - ldr ip, [pc, #-4092] - ldr pc, [pc, #-4092] +.rept PAGE_MAX_SIZE / FFI_TRAMPOLINE_SIZE + adr ip, #-PAGE_MAX_SIZE @ the config page is PAGE_MAX_SIZE behind the trampoline page + sub ip, #8 @ account for pc bias + ldr pc, [ip, #4] @ jump to ffi_closure_SYSV or ffi_closure_VFP .endr ARM_FUNC_END(ffi_closure_trampoline_table_page) +#endif #else diff --git a/src/closures.c b/src/closures.c index 721ff00e..3dec0e31 100644 --- a/src/closures.c +++ b/src/closures.c @@ -30,6 +30,7 @@ #define _GNU_SOURCE 1 #endif +#include #include #include @@ -64,11 +65,247 @@ #if FFI_CLOSURES -# if FFI_EXEC_TRAMPOLINE_TABLE +#if FFI_EXEC_TRAMPOLINE_TABLE + +#ifdef __MACH__ + +#include +#include +#include +#include + +extern void *ffi_closure_trampoline_table_page; + +typedef struct ffi_trampoline_table ffi_trampoline_table; +typedef struct ffi_trampoline_table_entry ffi_trampoline_table_entry; + +struct ffi_trampoline_table +{ + /* contiguous writable and executable pages */ + vm_address_t config_page; + vm_address_t trampoline_page; + + /* free list tracking */ + uint16_t free_count; + ffi_trampoline_table_entry *free_list; + ffi_trampoline_table_entry *free_list_pool; + + ffi_trampoline_table *prev; + ffi_trampoline_table *next; +}; + +struct ffi_trampoline_table_entry +{ + void *(*trampoline) (); + ffi_trampoline_table_entry *next; +}; + +/* Total number of trampolines that fit in one trampoline table */ +#define FFI_TRAMPOLINE_COUNT (PAGE_MAX_SIZE / FFI_TRAMPOLINE_SIZE) + +static pthread_mutex_t ffi_trampoline_lock = PTHREAD_MUTEX_INITIALIZER; +static ffi_trampoline_table *ffi_trampoline_tables = NULL; + +static ffi_trampoline_table * +ffi_trampoline_table_alloc () +{ + ffi_trampoline_table *table = NULL; + + /* Loop until we can allocate two contiguous pages */ + while (table == NULL) + { + vm_address_t config_page = 0x0; + kern_return_t kt; + + /* Try to allocate two pages */ + kt = + vm_allocate (mach_task_self (), &config_page, PAGE_MAX_SIZE * 2, + VM_FLAGS_ANYWHERE); + if (kt != KERN_SUCCESS) + { + fprintf (stderr, "vm_allocate() failure: %d at %s:%d\n", kt, + __FILE__, __LINE__); + break; + } + + /* Now drop the second half of the allocation to make room for the trampoline table */ + vm_address_t trampoline_page = config_page + PAGE_MAX_SIZE; + kt = vm_deallocate (mach_task_self (), trampoline_page, PAGE_MAX_SIZE); + if (kt != KERN_SUCCESS) + { + fprintf (stderr, "vm_deallocate() failure: %d at %s:%d\n", kt, + __FILE__, __LINE__); + break; + } + + /* Remap the trampoline table to directly follow the config page */ + vm_prot_t cur_prot; + vm_prot_t max_prot; + + vm_address_t trampoline_page_template = (vm_address_t)&ffi_closure_trampoline_table_page; +#ifdef __arm__ + /* ffi_closure_trampoline_table_page can be thumb-biased on some ARM archs */ + trampoline_page_template &= ~1UL; +#endif + + kt = + vm_remap (mach_task_self (), &trampoline_page, PAGE_MAX_SIZE, 0x0, FALSE, + mach_task_self (), trampoline_page_template, FALSE, + &cur_prot, &max_prot, VM_INHERIT_SHARE); + + /* If we lost access to the destination trampoline page, drop our config allocation mapping and retry */ + if (kt != KERN_SUCCESS) + { + /* Log unexpected failures */ + if (kt != KERN_NO_SPACE) + { + fprintf (stderr, "vm_remap() failure: %d at %s:%d\n", kt, + __FILE__, __LINE__); + } + + vm_deallocate (mach_task_self (), config_page, PAGE_SIZE); + continue; + } + + /* We have valid trampoline and config pages */ + table = calloc (1, sizeof (ffi_trampoline_table)); + table->free_count = FFI_TRAMPOLINE_COUNT; + table->config_page = config_page; + table->trampoline_page = trampoline_page; + + /* Create and initialize the free list */ + table->free_list_pool = + calloc (FFI_TRAMPOLINE_COUNT, sizeof (ffi_trampoline_table_entry)); + + uint16_t i; + for (i = 0; i < table->free_count; i++) + { + ffi_trampoline_table_entry *entry = &table->free_list_pool[i]; + entry->trampoline = + (void *) (table->trampoline_page + (i * FFI_TRAMPOLINE_SIZE)); + + if (i < table->free_count - 1) + entry->next = &table->free_list_pool[i + 1]; + } + + table->free_list = table->free_list_pool; + } + + return table; +} + +void * +ffi_closure_alloc (size_t size, void **code) +{ + /* Create the closure */ + ffi_closure *closure = malloc (size); + if (closure == NULL) + return NULL; + + pthread_mutex_lock (&ffi_trampoline_lock); + + /* Check for an active trampoline table with available entries. */ + ffi_trampoline_table *table = ffi_trampoline_tables; + if (table == NULL || table->free_list == NULL) + { + table = ffi_trampoline_table_alloc (); + if (table == NULL) + { + free (closure); + return NULL; + } + + /* Insert the new table at the top of the list */ + table->next = ffi_trampoline_tables; + if (table->next != NULL) + table->next->prev = table; + + ffi_trampoline_tables = table; + } + + /* Claim the free entry */ + ffi_trampoline_table_entry *entry = ffi_trampoline_tables->free_list; + ffi_trampoline_tables->free_list = entry->next; + ffi_trampoline_tables->free_count--; + entry->next = NULL; + + pthread_mutex_unlock (&ffi_trampoline_lock); + + /* Initialize the return values */ + *code = entry->trampoline; + closure->trampoline_table = table; + closure->trampoline_table_entry = entry; + + return closure; +} + +void +ffi_closure_free (void *ptr) +{ + ffi_closure *closure = ptr; + + pthread_mutex_lock (&ffi_trampoline_lock); + + /* Fetch the table and entry references */ + ffi_trampoline_table *table = closure->trampoline_table; + ffi_trampoline_table_entry *entry = closure->trampoline_table_entry; + + /* Return the entry to the free list */ + entry->next = table->free_list; + table->free_list = entry; + table->free_count++; + + /* If all trampolines within this table are free, and at least one other table exists, deallocate + * the table */ + if (table->free_count == FFI_TRAMPOLINE_COUNT + && ffi_trampoline_tables != table) + { + /* Remove from the list */ + if (table->prev != NULL) + table->prev->next = table->next; + + if (table->next != NULL) + table->next->prev = table->prev; + + /* Deallocate pages */ + kern_return_t kt; + kt = vm_deallocate (mach_task_self (), table->config_page, PAGE_SIZE); + if (kt != KERN_SUCCESS) + fprintf (stderr, "vm_deallocate() failure: %d at %s:%d\n", kt, + __FILE__, __LINE__); + + kt = + vm_deallocate (mach_task_self (), table->trampoline_page, PAGE_SIZE); + if (kt != KERN_SUCCESS) + fprintf (stderr, "vm_deallocate() failure: %d at %s:%d\n", kt, + __FILE__, __LINE__); + + /* Deallocate free list */ + free (table->free_list_pool); + free (table); + } + else if (ffi_trampoline_tables != table) + { + /* Otherwise, bump this table to the top of the list */ + table->prev = NULL; + table->next = ffi_trampoline_tables; + if (ffi_trampoline_tables != NULL) + ffi_trampoline_tables->prev = table; + + ffi_trampoline_tables = table; + } + + pthread_mutex_unlock (&ffi_trampoline_lock); + + /* Free the closure */ + free (closure); +} + +#endif // Per-target implementation; It's unclear what can reasonable be shared between two OS/architecture implementations. -# elif FFI_MMAP_EXEC_WRIT /* !FFI_EXEC_TRAMPOLINE_TABLE */ +#elif FFI_MMAP_EXEC_WRIT /* !FFI_EXEC_TRAMPOLINE_TABLE */ #define USE_LOCKS 1 #define USE_DL_PREFIX 1