diff --git a/.gdbinit b/.gdbinit index 4477828265e0..df4cbffb5fa0 100644 --- a/.gdbinit +++ b/.gdbinit @@ -3,18 +3,22 @@ define set_ts end document set_ts - set the ts resource, it is impossible for gdb to - call ts_resource_ex while no process is running, - but we could get the resource from the argument - of frame info. + set a legacy TSRM resource cache pointer when it is available + from a frame argument. Pass 0 to use the current thread's + direct executor and compiler globals. end define ____executor_globals if basic_functions_module.zts - set $tsrm_ls = _tsrm_ls_cache - set $eg = ((zend_executor_globals*) (*((void ***) $tsrm_ls))[executor_globals_id-1]) - set $cg = ((zend_compiler_globals*) (*((void ***) $tsrm_ls))[compiler_globals_id-1]) - set $eg_ptr = $eg + if !$tsrm_ls + set $eg = _tsrm_ls_cache.eg + set $cg = _tsrm_ls_cache.cg + set $eg_ptr = (zend_executor_globals*) &_tsrm_ls_cache.eg + else + set $eg = ((zend_executor_globals*) (*((void ***) $tsrm_ls))[executor_globals_id-1]) + set $cg = ((zend_compiler_globals*) (*((void ***) $tsrm_ls))[compiler_globals_id-1]) + set $eg_ptr = $eg + end else set $eg = executor_globals set $cg = compiler_globals diff --git a/TSRM/TSRM.c b/TSRM/TSRM.c index 089520041e33..0cc192473116 100644 --- a/TSRM/TSRM.c +++ b/TSRM/TSRM.c @@ -59,7 +59,6 @@ static int resource_types_table_size; /* Reserved space for fast globals access */ static size_t tsrm_reserved_pos = 0; static size_t tsrm_reserved_size = 0; -static size_t tsrm_reserved_front = 0; static MUTEX_T tsmm_mutex; /* thread-safe memory manager mutex */ static MUTEX_T tsrm_env_mutex; /* tsrm environ mutex */ @@ -168,7 +167,6 @@ TSRM_API bool tsrm_startup(int expected_threads, int expected_resources, int deb tsrm_reserved_pos = 0; tsrm_reserved_size = 0; - tsrm_reserved_front = 0; tsrm_env_mutex = tsrm_mutex_alloc(); tsrm_atexit_init(); @@ -229,7 +227,7 @@ TSRM_API void tsrm_shutdown(void) } else { free(p->storage); } - free((char *) p - tsrm_reserved_front); + free(p); p = next_p; } } @@ -261,7 +259,6 @@ TSRM_API void tsrm_shutdown(void) tsrm_reserved_pos = 0; tsrm_reserved_size = 0; - tsrm_reserved_front = 0; }/*}}}*/ /* {{{ */ @@ -354,19 +351,10 @@ TSRM_API void tsrm_reserve(size_t size) }/*}}}*/ -/* Carve a fixed-offset front region out of the reserved space. It is placed - * before the TLS entry, so the hot globals get compile-time-constant negative - * offsets from the cache pointer. */ -TSRM_API void tsrm_reserve_fast_front(size_t size) -{ - tsrm_reserved_front = TSRM_ALIGNED_SIZE(size); - tsrm_reserved_size -= tsrm_reserved_front; -} - - /* allocates a new fast thread-safe-resource id */ TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor) {/*{{{*/ + TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Obtaining a new fast resource id, %d bytes", size)); tsrm_mutex_lock(tsmm_mutex); size = TSRM_ALIGNED_SIZE(size); if (tsrm_reserved_size - tsrm_reserved_pos < size) { @@ -376,27 +364,13 @@ TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, siz tsrm_mutex_unlock(tsmm_mutex); return 0; } - ptrdiff_t fixed_offset = TSRM_ALIGNED_SIZE(sizeof(tsrm_tls_entry)) + tsrm_reserved_pos; + *offset = TSRM_ALIGNED_SIZE(sizeof(tsrm_tls_entry)) + tsrm_reserved_pos; tsrm_reserved_pos += size; - tsrm_mutex_unlock(tsmm_mutex); - - return ts_allocate_fast_id_at(rsrc_id, offset, fixed_offset, size, ctor, dtor); -}/*}}}*/ - - -TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset, ptrdiff_t fixed_offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor) -{ - TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Obtaining a new fast resource id, %d bytes", size)); - - tsrm_mutex_lock(tsmm_mutex); /* obtain a resource id */ *rsrc_id = TSRM_SHUFFLE_RSRC_ID(id_count++); TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Obtained resource id %d", *rsrc_id)); - size = TSRM_ALIGNED_SIZE(size); - *offset = (size_t) fixed_offset; - /* store the new resource type in the resource sizes table */ if (resource_types_table_size < id_count) { tsrm_resource_type *_tmp; @@ -423,7 +397,7 @@ TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset, TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Successfully allocated new resource id %d", *rsrc_id)); return *rsrc_id; -} +}/*}}}*/ /* allocates a resource id whose per-thread storage is a native __thread block */ TSRM_API ts_rsrc_id ts_allocate_tls_id(ts_rsrc_id *rsrc_id, void *(*tls_addr)(void), size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor) @@ -471,10 +445,8 @@ static void set_thread_local_storage_resource_to(tsrm_tls_entry *thread_resource static void allocate_new_resource(tsrm_tls_entry **thread_resources_ptr, THREAD_T thread_id) {/*{{{*/ TSRM_ERROR((TSRM_ERROR_LEVEL_CORE, "Creating data structures for thread %x", thread_id)); - /* The entry follows the fixed-offset front region. - * hot globals live at negative offsets from the TLS cache pointer. */ - char *block = (char *) malloc(tsrm_reserved_front + TSRM_ALIGNED_SIZE(sizeof(tsrm_tls_entry)) + tsrm_reserved_size); - (*thread_resources_ptr) = (tsrm_tls_entry *) (block + tsrm_reserved_front); + /* Fast resources live in the reserved space right behind the entry. */ + (*thread_resources_ptr) = (tsrm_tls_entry *) malloc(TSRM_ALIGNED_SIZE(sizeof(tsrm_tls_entry)) + tsrm_reserved_size); (*thread_resources_ptr)->storage = NULL; if (id_count > 0) { (*thread_resources_ptr)->storage = (void **) malloc(sizeof(void *)*id_count); @@ -587,7 +559,7 @@ TSRM_API void *ts_resource_ex(ts_rsrc_id id, THREAD_T *th_id) /* The dead thread's native TLS is gone, and its remaining resource dtors * may depend on those globals. Only release directly owned allocations. */ ts_free_resources(thread_resources, false, false); - free((char *) thread_resources - tsrm_reserved_front); + free(thread_resources); /* Allocate a new resource at the same point in the linked list, and relink the next pointer */ allocate_new_resource(last_thread_resources, thread_id); thread_resources = *last_thread_resources; @@ -640,7 +612,7 @@ void ts_free_thread(void) tsrm_atexit_set_data(NULL); TSRMLS_CACHE = NULL; if (thread_resources) { - free((char *) thread_resources - tsrm_reserved_front); + free(thread_resources); } tsrm_mutex_unlock(tsmm_mutex); }/*}}}*/ diff --git a/TSRM/TSRM.h b/TSRM/TSRM.h index 90e4b0720772..1c66c74a5d0e 100644 --- a/TSRM/TSRM.h +++ b/TSRM/TSRM.h @@ -95,11 +95,6 @@ TSRM_API ts_rsrc_id ts_allocate_id(ts_rsrc_id *rsrc_id, size_t size, ts_allocate TSRM_API void tsrm_reserve(size_t size); TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor); -/* Fast resources at caller-chosen, compile-time-constant offsets. The fixed - * front region must be reserved after tsrm_reserve() and before any fast id. */ -TSRM_API void tsrm_reserve_fast_front(size_t size); -TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset, ptrdiff_t fixed_offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor); - /* Resource whose per-thread storage is a native __thread block. * Must be called at startup before any other thread exists. */ TSRM_API ts_rsrc_id ts_allocate_tls_id(ts_rsrc_id *rsrc_id, void *(*tls_addr)(void), size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor); @@ -168,7 +163,7 @@ TSRM_API bool tsrm_is_managed_thread(void); || defined(__MUSL__) || defined(__HAIKU__) || defined(_AIX) # define TSRM_TLS_MODEL_ATTR # define TSRM_TLS_MODEL_DEFAULT -#elif defined(__PIC__) && !defined(__PIE__) +#elif (defined(__PIC__) && !defined(__PIE__)) || !defined(ZEND_ENABLE_STATIC_TSRMLS_CACHE) # if defined(TSRM_TLS_MODEL_USE_DEFAULT) # define TSRM_TLS_MODEL_ATTR # define TSRM_TLS_MODEL_DEFAULT @@ -193,17 +188,41 @@ TSRM_API bool tsrm_is_managed_thread(void); #define TSRMG_BULK_STATIC(id, type) ((type) (*((void ***) TSRMLS_CACHE))[TSRM_UNSHUFFLE_RSRC_ID(id)]) #define TSRMG_FAST_STATIC(offset, type, element) (TSRMG_FAST_BULK_STATIC(offset, type)->element) #define TSRMG_FAST_BULK_STATIC(offset, type) ((type) (((char*) TSRMLS_CACHE)+(offset))) + +/* Windows can't dllexport __declspec(thread) symbols, so outside Zend each module + * keeps a per-module `void *` pointer and reaches EG/CG via the resource-id indirection. */ +#ifdef ZEND_WIN32 +# define ZEND_TLS_API +# ifdef LIBZEND_EXPORTS +# define ZEND_TLS_DIRECT 1 +# endif +#else +# define ZEND_TLS_API ZEND_API +# define ZEND_TLS_DIRECT 1 +#endif + +typedef struct _zend_tsrm_ls_cache zend_tsrm_ls_cache; + +#ifdef ZEND_TLS_DIRECT +# define ZEND_TSRMLS_CACHE_T zend_tsrm_ls_cache +# define TSRMLS_CACHE_DEFINE() +extern ZEND_TLS_API TSRM_TLS TSRM_TLS_MODEL_ATTR zend_tsrm_ls_cache _tsrm_ls_cache; +# define ZEND_TSRM_CACHE_PTR (&_tsrm_ls_cache) +# define ZEND_TSRMG_DIRECT(id, type, member, element) (ZEND_TSRM_CACHE_PTR->member.element) +#else +# define ZEND_TSRMLS_CACHE_T void * +# define TSRMLS_CACHE_DEFINE() TSRM_TLS void *_tsrm_ls_cache = NULL; +# define ZEND_TSRMG_DIRECT(id, type, member, element) ZEND_TSRMG(id, type, element) +#endif #ifdef __cplusplus -#define TSRMLS_MAIN_CACHE_EXTERN() extern "C" { extern TSRM_TLS void *TSRMLS_CACHE TSRM_TLS_MODEL_ATTR; } -#define TSRMLS_CACHE_EXTERN() extern "C" { extern TSRM_TLS void *TSRMLS_CACHE; } +#define TSRMLS_MAIN_CACHE_EXTERN() extern "C" { extern TSRM_TLS ZEND_TSRMLS_CACHE_T _tsrm_ls_cache TSRM_TLS_MODEL_ATTR; } +#define TSRMLS_CACHE_EXTERN() extern "C" { extern TSRM_TLS ZEND_TSRMLS_CACHE_T _tsrm_ls_cache; } #else -#define TSRMLS_MAIN_CACHE_EXTERN() extern TSRM_TLS void *TSRMLS_CACHE TSRM_TLS_MODEL_ATTR; -#define TSRMLS_CACHE_EXTERN() extern TSRM_TLS void *TSRMLS_CACHE; +#define TSRMLS_MAIN_CACHE_EXTERN() extern TSRM_TLS ZEND_TSRMLS_CACHE_T _tsrm_ls_cache TSRM_TLS_MODEL_ATTR; +#define TSRMLS_CACHE_EXTERN() extern TSRM_TLS ZEND_TSRMLS_CACHE_T _tsrm_ls_cache; #endif -#define TSRMLS_MAIN_CACHE_DEFINE() TSRM_TLS void *TSRMLS_CACHE TSRM_TLS_MODEL_ATTR = NULL; -#define TSRMLS_CACHE_DEFINE() TSRM_TLS void *TSRMLS_CACHE = NULL; #define TSRMLS_CACHE_UPDATE() TSRMLS_CACHE = tsrm_get_ls_cache() -#define TSRMLS_CACHE _tsrm_ls_cache +#define TSRMLS_CACHE (*(void **) &_tsrm_ls_cache) #ifdef __cplusplus } @@ -216,7 +235,6 @@ TSRM_API bool tsrm_is_managed_thread(void); #define TSRMG_STATIC(id, type, element) #define TSRMLS_MAIN_CACHE_EXTERN() -#define TSRMLS_MAIN_CACHE_DEFINE() #define TSRMLS_CACHE_EXTERN() #define TSRMLS_CACHE_DEFINE() #define TSRMLS_CACHE_UPDATE() diff --git a/UPGRADING.INTERNALS b/UPGRADING.INTERNALS index b8ed6552d4f2..9eae85403517 100644 --- a/UPGRADING.INTERNALS +++ b/UPGRADING.INTERNALS @@ -51,6 +51,8 @@ PHP 8.7 INTERNALS UPGRADE NOTES allowed anymore. User-facing HashTables must have pDestructor = ZVAL_PTR_DTOR or NULL, and must be freed with zend_array_destroy(). Other HashTables can be freed with zend_hash_destroy() + pefree(). +- The ZTS symbols compiler_globals_offset and executor_globals_offset have been + removed: CG()/EG() now reach the globals through a __thread struct. ======================== 2. Build system changes diff --git a/Zend/zend.c b/Zend/zend.c index 17bc4ba0cbad..cd970984f7bc 100644 --- a/Zend/zend.c +++ b/Zend/zend.c @@ -52,16 +52,16 @@ static bool startup_done = false; #ifdef ZTS ZEND_API int compiler_globals_id; ZEND_API int executor_globals_id; -ZEND_API size_t compiler_globals_offset; -ZEND_API size_t executor_globals_offset; +ZEND_TLS_API TSRM_TLS TSRM_TLS_MODEL_ATTR zend_tsrm_ls_cache _tsrm_ls_cache = {0}; /* ts_allocate_tls_id takes a callback so each thread resolves its own block. * A plain &language_scanner_globals would capture only the registering thread's address. */ +static void *executor_globals_tls_addr(void) { return &_tsrm_ls_cache.eg; } +static void *compiler_globals_tls_addr(void) { return &_tsrm_ls_cache.cg; } static void *language_scanner_globals_tls_addr(void) { return &language_scanner_globals; } static HashTable *global_function_table = NULL; static HashTable *global_class_table = NULL; static HashTable *global_constants_table = NULL; static HashTable *global_auto_globals_table = NULL; -TSRMLS_MAIN_CACHE_DEFINE() # define GLOBAL_FUNCTION_TABLE global_function_table # define GLOBAL_CLASS_TABLE global_class_table # define GLOBAL_CONSTANTS_TABLE global_constants_table @@ -1031,11 +1031,9 @@ void zend_startup(zend_utility_functions *utility_functions) /* {{{ */ zend_init_rsrc_list_dtors(); #ifdef ZTS - ts_allocate_fast_id_at(&compiler_globals_id, &compiler_globals_offset, ZEND_CG_OFFSET, sizeof(zend_compiler_globals), (ts_allocate_ctor) compiler_globals_ctor, (ts_allocate_dtor) compiler_globals_dtor); - ts_allocate_fast_id_at(&executor_globals_id, &executor_globals_offset, ZEND_EG_OFFSET, sizeof(zend_executor_globals), (ts_allocate_ctor) executor_globals_ctor, (ts_allocate_dtor) executor_globals_dtor); + ts_allocate_tls_id(&compiler_globals_id, compiler_globals_tls_addr, sizeof(zend_compiler_globals), (ts_allocate_ctor) compiler_globals_ctor, (ts_allocate_dtor) compiler_globals_dtor); + ts_allocate_tls_id(&executor_globals_id, executor_globals_tls_addr, sizeof(zend_executor_globals), (ts_allocate_ctor) executor_globals_ctor, (ts_allocate_dtor) executor_globals_dtor); ts_allocate_tls_id(&language_scanner_globals_id, language_scanner_globals_tls_addr, sizeof(zend_php_scanner_globals), (ts_allocate_ctor) php_scanner_globals_ctor, NULL); - ZEND_ASSERT(compiler_globals_offset == ZEND_CG_OFFSET); - ZEND_ASSERT(executor_globals_offset == ZEND_EG_OFFSET); ts_allocate_fast_id(&ini_scanner_globals_id, &ini_scanner_globals_offset, sizeof(zend_ini_scanner_globals), (ts_allocate_ctor) ini_scanner_globals_ctor, NULL); compiler_globals = ts_resource(compiler_globals_id); executor_globals = ts_resource(executor_globals_id); diff --git a/Zend/zend_gc.c b/Zend/zend_gc.c index cd7c3c512daf..f57e69de7408 100644 --- a/Zend/zend_gc.c +++ b/Zend/zend_gc.c @@ -306,12 +306,12 @@ typedef struct _zend_gc_globals { #ifdef ZTS static int gc_globals_id; -static size_t gc_globals_offset; -#define GC_G(v) ZEND_TSRMG_FAST(gc_globals_offset, zend_gc_globals *, v) +static TSRM_TLS TSRM_TLS_MODEL_ATTR zend_gc_globals gc_globals; +static void *gc_globals_tls_addr(void) { return &gc_globals; } #else -#define GC_G(v) (gc_globals.v) static zend_gc_globals gc_globals; #endif +#define GC_G(v) (gc_globals.v) #if GC_BENCH # define GC_BENCH_INC(counter) GC_G(counter)++ @@ -549,7 +549,7 @@ static void gc_globals_ctor_ex(zend_gc_globals *gc_globals) void gc_globals_ctor(void) { #ifdef ZTS - ts_allocate_fast_id(&gc_globals_id, &gc_globals_offset, sizeof(zend_gc_globals), (ts_allocate_ctor) gc_globals_ctor_ex, (ts_allocate_dtor) root_buffer_dtor); + ts_allocate_tls_id(&gc_globals_id, gc_globals_tls_addr, sizeof(zend_gc_globals), (ts_allocate_ctor) gc_globals_ctor_ex, (ts_allocate_dtor) root_buffer_dtor); #else gc_globals_ctor_ex(&gc_globals); #endif @@ -2322,13 +2322,6 @@ void gc_bench_print(void) } #endif -#ifdef ZTS -size_t zend_gc_globals_size(void) -{ - return sizeof(zend_gc_globals); -} -#endif - static ZEND_FUNCTION(gc_destructor_fiber) { uint32_t idx, end; diff --git a/Zend/zend_gc.h b/Zend/zend_gc.h index b72b90ccd5ef..55b12070b99d 100644 --- a/Zend/zend_gc.h +++ b/Zend/zend_gc.h @@ -69,10 +69,6 @@ void gc_globals_ctor(void); void gc_globals_dtor(void); void gc_reset(void); -#ifdef ZTS -size_t zend_gc_globals_size(void); -#endif - #define GC_REMOVE_FROM_BUFFER(p) do { \ zend_refcounted *_p = (zend_refcounted*)(p); \ if (GC_TYPE_INFO(_p) & GC_INFO_MASK) { \ diff --git a/Zend/zend_globals.h b/Zend/zend_globals.h index 419f9cc07c74..ff025343b7af 100644 --- a/Zend/zend_globals.h +++ b/Zend/zend_globals.h @@ -52,8 +52,6 @@ BEGIN_EXTERN_C() ZEND_API extern int compiler_globals_id; ZEND_API extern int executor_globals_id; -ZEND_API extern size_t compiler_globals_offset; -ZEND_API extern size_t executor_globals_offset; END_EXTERN_C() #endif @@ -342,9 +340,11 @@ struct _zend_executor_globals { }; #ifdef ZTS -/* Compile-time offsets of the hot globals, in a reserved region just before *_tsrm_ls_cache. */ -# define ZEND_CG_OFFSET (-(ptrdiff_t) TSRM_ALIGNED_SIZE(sizeof(zend_compiler_globals))) -# define ZEND_EG_OFFSET (ZEND_CG_OFFSET - (ptrdiff_t) TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals))) +struct _zend_tsrm_ls_cache { + void *cache; + zend_executor_globals eg; + zend_compiler_globals cg; +}; #endif #define EG_FLAGS_INITIAL (0) diff --git a/Zend/zend_globals_macros.h b/Zend/zend_globals_macros.h index adb3913ece71..684ed696adf1 100644 --- a/Zend/zend_globals_macros.h +++ b/Zend/zend_globals_macros.h @@ -30,7 +30,7 @@ BEGIN_EXTERN_C() /* Compiler */ #ifdef ZTS -# define CG(v) ZEND_TSRMG_FAST(ZEND_CG_OFFSET, zend_compiler_globals *, v) +# define CG(v) ZEND_TSRMG_DIRECT(compiler_globals_id, zend_compiler_globals *, cg, v) #else # define CG(v) (compiler_globals.v) extern ZEND_API struct _zend_compiler_globals compiler_globals; @@ -40,7 +40,7 @@ ZEND_API int zendparse(void); /* Executor */ #ifdef ZTS -# define EG(v) ZEND_TSRMG_FAST(ZEND_EG_OFFSET, zend_executor_globals *, v) +# define EG(v) ZEND_TSRMG_DIRECT(executor_globals_id, zend_executor_globals *, eg, v) #else # define EG(v) (executor_globals.v) extern ZEND_API zend_executor_globals executor_globals; @@ -49,15 +49,15 @@ extern ZEND_API zend_executor_globals executor_globals; /* Language Scanner */ #ifdef ZTS extern ZEND_API ts_rsrc_id language_scanner_globals_id; -# if defined(ZEND_WIN32) && !defined(LIBZEND_EXPORTS) -# define LANG_SCNG(v) TSRMG(language_scanner_globals_id, zend_php_scanner_globals *, v) -# else +# ifdef ZEND_TLS_DIRECT # ifdef ZEND_WIN32 extern TSRM_TLS zend_php_scanner_globals language_scanner_globals; # else -extern ZEND_API TSRM_TLS TSRM_TLS_MODEL_ATTR zend_php_scanner_globals language_scanner_globals; +extern ZEND_TLS_API TSRM_TLS TSRM_TLS_MODEL_ATTR zend_php_scanner_globals language_scanner_globals; # endif # define LANG_SCNG(v) (language_scanner_globals.v) +# else +# define LANG_SCNG(v) TSRMG(language_scanner_globals_id, zend_php_scanner_globals *, v) # endif #else extern ZEND_API zend_php_scanner_globals language_scanner_globals; diff --git a/ext/opcache/jit/tls/testing/user.c b/ext/opcache/jit/tls/testing/user.c index 19e9f06c0806..446b876033d2 100644 --- a/ext/opcache/jit/tls/testing/user.c +++ b/ext/opcache/jit/tls/testing/user.c @@ -3,8 +3,6 @@ #include "../zend_jit_tls.h" -TSRMLS_MAIN_CACHE_EXTERN() - int test(void) { size_t tcb_offset = 0; @@ -12,7 +10,7 @@ int test(void) size_t module_offset = -1; /* Ensure the slot is allocated */ - _tsrm_ls_cache = NULL; + void *cache = &_tsrm_ls_cache; zend_result result = zend_jit_resolve_tsrm_ls_cache_offsets( &tcb_offset, &module_index, &module_offset); @@ -24,5 +22,5 @@ int test(void) return 0; } - return zend_jit_tsrm_ls_cache_address(tcb_offset, module_index, module_offset) == &_tsrm_ls_cache; + return zend_jit_tsrm_ls_cache_address(tcb_offset, module_index, module_offset) == cache; } diff --git a/ext/opcache/jit/tls/zend_jit_tls_win.c b/ext/opcache/jit/tls/zend_jit_tls_win.c index 5646f3dcba0b..9bd79571eea3 100644 --- a/ext/opcache/jit/tls/zend_jit_tls_win.c +++ b/ext/opcache/jit/tls/zend_jit_tls_win.c @@ -33,30 +33,20 @@ zend_result zend_jit_resolve_tsrm_ls_cache_offsets( size_t *module_index, size_t *module_offset ) { - /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ - /* Probably, it might be better solution */ + /* Offset of _tsrm_ls_cache within this module's TLS block. */ #ifdef _WIN64 void ***tls_mem = ((void****)__readgsqword(0x58))[_tls_index]; #else void ***tls_mem = ((void****)__readfsdword(0x2c))[_tls_index]; #endif - void *val = _tsrm_ls_cache; - size_t offset = 0; size_t size = (char*)&_tls_end - (char*)&_tls_start; - - while (offset < size) { - if (*tls_mem == val) { - *module_index = _tls_index * sizeof(void*); - *module_offset = offset; - return SUCCESS; - } - tls_mem++; - offset += sizeof(void*); - } + size_t offset = (size_t)((char*)&_tsrm_ls_cache - (char*)tls_mem); if (offset >= size) { zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size"); } - return FAILURE; + *module_index = _tls_index * sizeof(void*); + *module_offset = offset; + return SUCCESS; } diff --git a/ext/opcache/jit/zend_jit_ir.c b/ext/opcache/jit/zend_jit_ir.c index d7380b66e1c3..176c452a637d 100644 --- a/ext/opcache/jit/zend_jit_ir.c +++ b/ext/opcache/jit/zend_jit_ir.c @@ -204,16 +204,16 @@ static size_t tsrm_tls_index = -1; static size_t tsrm_tls_offset = -1; # define EG_TLS_OFFSET(field) \ - (executor_globals_offset + offsetof(zend_executor_globals, field)) + (offsetof(zend_tsrm_ls_cache, eg) + offsetof(zend_executor_globals, field)) # define CG_TLS_OFFSET(field) \ - (compiler_globals_offset + offsetof(zend_compiler_globals, field)) + (offsetof(zend_tsrm_ls_cache, cg) + offsetof(zend_compiler_globals, field)) # define jit_EG(_field) \ - ir_ADD_OFFSET(jit_TLS(jit), EG_TLS_OFFSET(_field)) + jit_TLS_ADDR(jit, EG_TLS_OFFSET(_field)) # define jit_CG(_field) \ - ir_ADD_OFFSET(jit_TLS(jit), CG_TLS_OFFSET(_field)) + jit_TLS_ADDR(jit, CG_TLS_OFFSET(_field)) #else @@ -504,20 +504,24 @@ static const char* zend_reg_name(int8_t reg) #ifdef ZTS static void * ZEND_FASTCALL zend_jit_get_tsrm_ls_cache(void) { - return _tsrm_ls_cache; + return &_tsrm_ls_cache; } -static ir_ref jit_TLS(zend_jit_ctx *jit) +static ir_ref jit_TLS_ADDR(zend_jit_ctx *jit, size_t offset) { ZEND_ASSERT(jit->ctx.control); + if (tsrm_ls_cache_tcb_offset) { + return ir_TLS_ADDR(-1, tsrm_ls_cache_tcb_offset + offset); + } + if (jit->tls) { - /* Emit "TLS" once for basic block */ + /* Reuse the TLS lookup within a basic block. */ ir_insn *insn; ir_ref ref = jit->ctx.control; while (1) { if (ref == jit->tls) { - return jit->tls; + return ir_ADD_OFFSET(jit->tls, offset); } insn = &jit->ctx.ir_base[ref]; if (insn->op >= IR_START || insn->op == IR_CALL) { @@ -527,9 +531,7 @@ static ir_ref jit_TLS(zend_jit_ctx *jit) } } - if (tsrm_ls_cache_tcb_offset == 0 && tsrm_tls_index == -1) { - jit->tls = ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_get_tsrm_ls_cache)); - } else { + if (tsrm_tls_index != -1) { # ifdef ZEND_OPCACHE_SHM_REATTACHMENT /* Different processes may load TLS variables in a different order, so * they may have a different module index. Fetch the module index at @@ -546,15 +548,15 @@ static ir_ref jit_TLS(zend_jit_ctx *jit) ir_ref tls_array = ir_TLS(-1, offset); ir_ref index = ir_LOAD_A(ir_CONST_ADDR(&tsrm_tls_index)); - jit->tls = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(ir_ADD_A(tls_array, index)), tsrm_tls_offset)); + jit->tls = ir_ADD_OFFSET(ir_LOAD_A(ir_ADD_A(tls_array, index)), tsrm_tls_offset); # else - jit->tls = ir_TLS( - tsrm_ls_cache_tcb_offset ? tsrm_ls_cache_tcb_offset : tsrm_tls_index, - tsrm_ls_cache_tcb_offset ? IR_NULL : tsrm_tls_offset); + jit->tls = ir_TLS_ADDR(tsrm_tls_index, tsrm_tls_offset); # endif + } else { + jit->tls = ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_get_tsrm_ls_cache)); } - return jit->tls; + return ir_ADD_OFFSET(jit->tls, offset); } #endif diff --git a/main/debug_gdb_scripts.c b/main/debug_gdb_scripts.c index 0fb9450388dc..57d204b8d92a 100644 --- a/main/debug_gdb_scripts.c +++ b/main/debug_gdb_scripts.c @@ -16,18 +16,22 @@ __asm__( ".ascii \"end\\n\"\n" ".ascii \"\\n\"\n" ".ascii \"document set_ts\\n\"\n" - ".ascii \" set the ts resource, it is impossible for gdb to\\n\"\n" - ".ascii \" call ts_resource_ex while no process is running,\\n\"\n" - ".ascii \" but we could get the resource from the argument\\n\"\n" - ".ascii \" of frame info.\\n\"\n" + ".ascii \" set a legacy TSRM resource cache pointer when it is available\\n\"\n" + ".ascii \" from a frame argument. Pass 0 to use the current thread\\\\'s\\n\"\n" + ".ascii \" direct executor and compiler globals.\\n\"\n" ".ascii \"end\\n\"\n" ".ascii \"\\n\"\n" ".ascii \"define ____executor_globals\\n\"\n" ".ascii \" if basic_functions_module.zts\\n\"\n" - ".ascii \" set $tsrm_ls = _tsrm_ls_cache\\n\"\n" - ".ascii \" set $eg = ((zend_executor_globals*) (*((void ***) $tsrm_ls))[executor_globals_id-1])\\n\"\n" - ".ascii \" set $cg = ((zend_compiler_globals*) (*((void ***) $tsrm_ls))[compiler_globals_id-1])\\n\"\n" - ".ascii \" set $eg_ptr = $eg\\n\"\n" + ".ascii \" if !$tsrm_ls\\n\"\n" + ".ascii \" set $eg = _tsrm_ls_cache.eg\\n\"\n" + ".ascii \" set $cg = _tsrm_ls_cache.cg\\n\"\n" + ".ascii \" set $eg_ptr = (zend_executor_globals*) &_tsrm_ls_cache.eg\\n\"\n" + ".ascii \" else\\n\"\n" + ".ascii \" set $eg = ((zend_executor_globals*) (*((void ***) $tsrm_ls))[executor_globals_id-1])\\n\"\n" + ".ascii \" set $cg = ((zend_compiler_globals*) (*((void ***) $tsrm_ls))[compiler_globals_id-1])\\n\"\n" + ".ascii \" set $eg_ptr = $eg\\n\"\n" + ".ascii \" end\\n\"\n" ".ascii \" else\\n\"\n" ".ascii \" set $eg = executor_globals\\n\"\n" ".ascii \" set $cg = compiler_globals\\n\"\n" diff --git a/main/main.c b/main/main.c index 1e4acfc8aa98..ee7d1cca69b5 100644 --- a/main/main.c +++ b/main/main.c @@ -2797,15 +2797,14 @@ PHPAPI zend_result php_lint_script(zend_file_handle *file) /* {{{ php_reserve_tsrm_memory */ PHPAPI void php_reserve_tsrm_memory(void) { + /* CG/EG/GC globals live in native __thread storage and need no + * reserved TSRM space. */ tsrm_reserve( - TSRM_ALIGNED_SIZE(sizeof(zend_compiler_globals)) + - TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals)) + TSRM_ALIGNED_SIZE(sizeof(zend_ini_scanner_globals)) + TSRM_ALIGNED_SIZE(sizeof(virtual_cwd_globals)) + #ifdef ZEND_SIGNALS TSRM_ALIGNED_SIZE(sizeof(zend_signal_globals_t)) + #endif - TSRM_ALIGNED_SIZE(zend_gc_globals_size()) + TSRM_ALIGNED_SIZE(sizeof(php_core_globals)) + TSRM_ALIGNED_SIZE(sizeof(sapi_globals_struct)) + TSRM_ALIGNED_SIZE(sizeof(zend_accel_globals)) + @@ -2821,10 +2820,6 @@ PHPAPI bool php_tsrm_startup_ex(int expected_threads) { bool ret = tsrm_startup(expected_threads, 1, 0, NULL); php_reserve_tsrm_memory(); - /* Must cover the total size of every ZEND_*_OFFSET global, or the furthest underflows the block. */ - tsrm_reserve_fast_front( - TSRM_ALIGNED_SIZE(sizeof(zend_compiler_globals)) + - TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals))); (void)ts_resource(0); return ret; }