diff --git a/.github/scripts/download-bundled/jit-ir.sh b/.github/scripts/download-bundled/jit-ir.sh index 594203b70218..74a9789f35ad 100755 --- a/.github/scripts/download-bundled/jit-ir.sh +++ b/.github/scripts/download-bundled/jit-ir.sh @@ -5,7 +5,7 @@ cd "$(dirname "$0")/../../.." tmp_dir=/tmp/php-src-download-bundled/jit-ir rm -rf "$tmp_dir" -revision=bb044213cc1bc835613d0172ee95c52360a00524 +revision=30a40633b0b8f3e8e66da0a25941c4199d69d38a git clone --depth 1 --revision="$revision" https://github.com/dstogov/ir.git "$tmp_dir" diff --git a/ext/opcache/jit/ir/ir.c b/ext/opcache/jit/ir/ir.c index fd56dd8067cc..734e9086648f 100644 --- a/ext/opcache/jit/ir/ir.c +++ b/ext/opcache/jit/ir/ir.c @@ -635,6 +635,26 @@ static IR_NEVER_INLINE void ir_const_hash_rehash(ir_ctx *ctx) } } +#ifdef IR_DEBUG +/* Narrow integer constants are kept sign/zero extended to 64 bits, FLOAT has zero upper half */ +static bool ir_const_is_canonical(ir_val val, uint8_t type) +{ + uint32_t shift; + + if (type == IR_FLOAT) { + return val.u32_hi == 0; + } + if (!IR_IS_TYPE_INT(type) || ir_type_size[type] == 8) { + return 1; + } + shift = (8 - ir_type_size[type]) * 8; + if (IR_IS_TYPE_SIGNED(type)) { + return val.i64 == (int64_t)(val.u64 << shift) >> shift; + } + return val.u64 == (val.u64 << shift) >> shift; +} +#endif + ir_ref ir_const_ex(ir_ctx *ctx, ir_val val, uint8_t type, uint32_t optx) { ir_insn *insn; @@ -647,6 +667,8 @@ ir_ref ir_const_ex(ir_ctx *ctx, ir_val val, uint8_t type, uint32_t optx) return IR_NULL; } + IR_ASSERT(IR_IS_SYM_CONST(optx & IR_OPT_OP_MASK) || ir_const_is_canonical(val, type)); + hash = ir_const_hash(val, optx) & ctx->const_hash_mask; ref = ctx->const_hash[hash]; while (ref) { @@ -1278,12 +1300,7 @@ ir_ref ir_folding(ir_ctx *ctx, uint32_t opt, ir_ref op1, ir_ref op2, ir_ref op3, /* Extend a narrow integer result to its type width so no * fold rule sees garbage in the upper bits. */ if (IR_IS_TYPE_INT(type) && ir_type_size[type] < 8) { - uint32_t shift = (8 - ir_type_size[type]) * 8; - if (IR_IS_TYPE_SIGNED(type)) { - val.i64 = (int64_t)(val.u64 << shift) >> shift; - } else { - val.u64 = (val.u64 << shift) >> shift; - } + val.u64 = ir_fix_int(type, val.u64); } if (!ctx->use_lists) { diff --git a/ext/opcache/jit/ir/ir_aarch64.dasc b/ext/opcache/jit/ir/ir_aarch64.dasc index a29e97c4d3e2..b9fdbc2e7961 100644 --- a/ext/opcache/jit/ir/ir_aarch64.dasc +++ b/ext/opcache/jit/ir/ir_aarch64.dasc @@ -10019,7 +10019,7 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) if (ctx->code_buffer) { if (ctx->code_buffer->pos == (char*)entry + size) { /* rollback */ - ctx->code_buffer->pos = (char*)entry - size; + ctx->code_buffer->pos = (char*)entry; } } else { ir_mem_unmap(entry, size); @@ -10142,7 +10142,7 @@ const void *ir_emit_exitgroup(uint32_t first_exit_point, uint32_t exit_points_pe dasm_free(&dasm_state); if (code_buffer->pos == (char*)entry + size) { /* rollback */ - code_buffer->pos = (char*)entry - size; + code_buffer->pos = (char*)entry; } return NULL; } diff --git a/ext/opcache/jit/ir/ir_fold.h b/ext/opcache/jit/ir/ir_fold.h index bd73a3cbf413..16dc6f536b8c 100644 --- a/ext/opcache/jit/ir/ir_fold.h +++ b/ext/opcache/jit/ir/ir_fold.h @@ -2685,6 +2685,7 @@ IR_FOLD(DIV(NEG, C_FLOAT)) { op1 = op1_insn->op1; val.f = -op2_insn->val.f; + val.u32_hi = 0; op2 = ir_const(ctx, val, op2_insn->type); IR_FOLD_RESTART; } @@ -3695,7 +3696,8 @@ IR_FOLD(ADD(ADD, C_ADDR)) { if (IR_IS_CONST_REF(op1_insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op2].op)) { /* (x + c1) + c2 => x + (c1 + c2) */ - val.u64 = ctx->ir_base[op1_insn->op2].val.u64 + op2_insn->val.u64; + val.u64 = ir_fix_unsigned(IR_OPT_TYPE(opt), + ctx->ir_base[op1_insn->op2].val.u64 + op2_insn->val.u64); op1 = op1_insn->op1; op2 = ir_const(ctx, val, op2_insn->type); IR_FOLD_RESTART; @@ -3710,7 +3712,8 @@ IR_FOLD(ADD(ADD, C_I64)) { if (IR_IS_CONST_REF(op1_insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op2].op)) { /* (x + c1) + c2 => x + (c1 + c2) */ - val.i64 = ctx->ir_base[op1_insn->op2].val.u64 + op2_insn->val.u64; + val.u64 = ir_fix_signed(IR_OPT_TYPE(opt), + ctx->ir_base[op1_insn->op2].val.u64 + op2_insn->val.u64); op1 = op1_insn->op1; op2 = ir_const(ctx, val, op2_insn->type); IR_FOLD_RESTART; @@ -3726,13 +3729,15 @@ IR_FOLD(ADD(SUB, C_ADDR)) { if (IR_IS_CONST_REF(op1_insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op2].op)) { /* (x - c1) + c2 => x + (c2 - c1) */ - val.u64 = op2_insn->val.u64 - ctx->ir_base[op1_insn->op2].val.u64; + val.u64 = ir_fix_unsigned(IR_OPT_TYPE(opt), + op2_insn->val.u64 - ctx->ir_base[op1_insn->op2].val.u64); op1 = op1_insn->op1; op2 = ir_const(ctx, val, op2_insn->type); IR_FOLD_RESTART; } else if (IR_IS_CONST_REF(op1_insn->op1) && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op1].op)) { /* (c1 - x) + c2 => (c1 + c2) - x */ - val.u64 = ctx->ir_base[op1_insn->op1].val.u64 + op2_insn->val.u64; + val.u64 = ir_fix_unsigned(IR_OPT_TYPE(opt), + ctx->ir_base[op1_insn->op1].val.u64 + op2_insn->val.u64); opt++; /* ADD -> SUB */ op2 = op1_insn->op2; op1 = ir_const(ctx, val, op2_insn->type); @@ -3748,8 +3753,9 @@ IR_FOLD(ADD(SUB, C_I64)) { if (IR_IS_CONST_REF(op1_insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op2].op)) { /* (x - c1) + c2 => x + (c2 - c1) */ - val.i64 = op2_insn->val.u64 - ctx->ir_base[op1_insn->op2].val.u64; - if (val.i64 < 0 && val.i64 != INT64_MIN) { + val.u64 = ir_fix_signed(IR_OPT_TYPE(opt), + op2_insn->val.u64 - ctx->ir_base[op1_insn->op2].val.u64); + if (val.i64 < 0 && ir_neg_is_safe(IR_OPT_TYPE(opt), val.i64)) { val.i64 = -val.i64; opt++; /* ADD -> SUB */ } @@ -3758,7 +3764,8 @@ IR_FOLD(ADD(SUB, C_I64)) IR_FOLD_RESTART; } else if (IR_IS_CONST_REF(op1_insn->op1) && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op1].op)) { /* (c1 - x) + c2 => (c1 + c2) - x */ - val.i64 = ctx->ir_base[op1_insn->op1].val.u64 + op2_insn->val.u64; + val.u64 = ir_fix_signed(IR_OPT_TYPE(opt), + ctx->ir_base[op1_insn->op1].val.u64 + op2_insn->val.u64); opt++; /* ADD -> SUB */ op2 = op1_insn->op2; op1 = ir_const(ctx, val, op2_insn->type); @@ -3775,7 +3782,8 @@ IR_FOLD(SUB(ADD, C_ADDR)) { if (IR_IS_CONST_REF(op1_insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op2].op)) { /* (x + c1) - c2 => x + (c1 - c2) */ - val.u64 = ctx->ir_base[op1_insn->op2].val.u64 - op2_insn->val.u64; + val.u64 = ir_fix_unsigned(IR_OPT_TYPE(opt), + ctx->ir_base[op1_insn->op2].val.u64 - op2_insn->val.u64); opt--; /* SUB -> ADD */ op1 = op1_insn->op1; op2 = ir_const(ctx, val, op2_insn->type); @@ -3791,8 +3799,9 @@ IR_FOLD(SUB(ADD, C_I64)) { if (IR_IS_CONST_REF(op1_insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op2].op)) { /* (x + c1) - c2 => x + (c1 - c2) */ - val.i64 = ctx->ir_base[op1_insn->op2].val.u64 - op2_insn->val.u64; - if (val.i64 < 0 && val.i64 != INT64_MIN) { + val.u64 = ir_fix_signed(IR_OPT_TYPE(opt), + ctx->ir_base[op1_insn->op2].val.u64 - op2_insn->val.u64); + if (val.i64 < 0 && ir_neg_is_safe(IR_OPT_TYPE(opt), val.i64)) { val.i64 = -val.i64; } else { opt--; /* SUB -> ADD */ @@ -3812,7 +3821,8 @@ IR_FOLD(SUB(C_ADDR, ADD)) { if (IR_IS_CONST_REF(op2_insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[op2_insn->op2].op)) { /* c1 - (x + c2) => (c1 - c2) - x */ - val.u64 = op1_insn->val.u64 - ctx->ir_base[op2_insn->op2].val.u64; + val.u64 = ir_fix_unsigned(IR_OPT_TYPE(opt), + op1_insn->val.u64 - ctx->ir_base[op2_insn->op2].val.u64); op2 = op2_insn->op1; op1 = ir_const(ctx, val, op2_insn->type); IR_FOLD_RESTART; @@ -3827,7 +3837,8 @@ IR_FOLD(SUB(C_I64, ADD)) { if (IR_IS_CONST_REF(op2_insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[op2_insn->op2].op)) { /* c1 - (x + c2) => (c1 - c2) - x */ - val.i64 = op1_insn->val.u64 - ctx->ir_base[op2_insn->op2].val.u64; + val.u64 = ir_fix_signed(IR_OPT_TYPE(opt), + op1_insn->val.u64 - ctx->ir_base[op2_insn->op2].val.u64); op2 = op2_insn->op1; op1 = ir_const(ctx, val, op2_insn->type); IR_FOLD_RESTART; @@ -3843,7 +3854,8 @@ IR_FOLD(SUB(SUB, C_ADDR)) { if (IR_IS_CONST_REF(op1_insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op2].op)) { /* (x - c1) - c2 => x - (c1 + c2) */ - val.u64 = ctx->ir_base[op1_insn->op2].val.u64 + op2_insn->val.u64; + val.u64 = ir_fix_unsigned(IR_OPT_TYPE(opt), + ctx->ir_base[op1_insn->op2].val.u64 + op2_insn->val.u64); if (val.i64 < 0 && val.i64 != INT64_MIN) { val.i64 = -val.i64; opt--; /* SUB -> ADD */ @@ -3853,7 +3865,8 @@ IR_FOLD(SUB(SUB, C_ADDR)) IR_FOLD_RESTART; } else if (IR_IS_CONST_REF(op1_insn->op1) && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op1].op)) { /* (c1 - x) - c2 => (c1 - c2) - x */ - val.u64 = ctx->ir_base[op1_insn->op1].val.u64 - op2_insn->val.u64; + val.u64 = ir_fix_unsigned(IR_OPT_TYPE(opt), + ctx->ir_base[op1_insn->op1].val.u64 - op2_insn->val.u64); op2 = op1_insn->op2; op1 = ir_const(ctx, val, op2_insn->type); IR_FOLD_RESTART; @@ -3868,8 +3881,9 @@ IR_FOLD(SUB(SUB, C_I64)) { if (IR_IS_CONST_REF(op1_insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op2].op)) { /* (x - c1) - c2 => x - (c1 + c2) */ - val.i64 = ctx->ir_base[op1_insn->op2].val.u64 + op2_insn->val.u64; - if (val.i64 < 0 && val.i64 != INT64_MIN) { + val.u64 = ir_fix_signed(IR_OPT_TYPE(opt), + ctx->ir_base[op1_insn->op2].val.u64 + op2_insn->val.u64); + if (val.i64 < 0 && ir_neg_is_safe(IR_OPT_TYPE(opt), val.i64)) { val.i64 = -val.i64; opt--; /* SUB -> ADD */ } @@ -3878,7 +3892,8 @@ IR_FOLD(SUB(SUB, C_I64)) IR_FOLD_RESTART; } else if (IR_IS_CONST_REF(op1_insn->op1) && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op1].op)) { /* (c1 - x) - c2 => (c1 - c2) - x */ - val.i64 = ctx->ir_base[op1_insn->op1].val.u64 - op2_insn->val.u64; + val.u64 = ir_fix_signed(IR_OPT_TYPE(opt), + ctx->ir_base[op1_insn->op1].val.u64 - op2_insn->val.u64); op2 = op1_insn->op2; op1 = ir_const(ctx, val, op2_insn->type); IR_FOLD_RESTART; @@ -3894,16 +3909,19 @@ IR_FOLD(SUB(C_ADDR, SUB)) { if (IR_IS_CONST_REF(op2_insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[op2_insn->op2].op)) { /* c1 - (x - c2) => (c1 + c2) - x */ - val.u64 = op1_insn->val.u64 + ctx->ir_base[op2_insn->op2].val.u64; + val.u64 = ir_fix_unsigned(IR_OPT_TYPE(opt), + op1_insn->val.u64 + ctx->ir_base[op2_insn->op2].val.u64); op2 = op2_insn->op1; op1 = ir_const(ctx, val, op1_insn->type); IR_FOLD_RESTART; } else if (IR_IS_CONST_REF(op2_insn->op1) && !IR_IS_SYM_CONST(ctx->ir_base[op2_insn->op1].op)) { /* c1 - (c2 - x) => x + (c1 - c2) */ - val.u64 = op1_insn->val.u64 - ctx->ir_base[op2_insn->op1].val.u64; + val.u64 = ir_fix_unsigned(IR_OPT_TYPE(opt), + op1_insn->val.u64 - ctx->ir_base[op2_insn->op1].val.u64); if (val.i64 < 0 && val.i64 != INT64_MIN) { val.i64 = -val.i64; - opt++; /* ADD -> SUB */ + } else { + opt--; /* SUB -> ADD */ } op1 = op2_insn->op2; op2 = ir_const(ctx, val, op1_insn->type); @@ -3919,16 +3937,19 @@ IR_FOLD(SUB(C_I64, SUB)) { if (IR_IS_CONST_REF(op2_insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[op2_insn->op2].op)) { /* c1 - (x - c2) => (c1 + c2) - x */ - val.i64 = op1_insn->val.u64 + ctx->ir_base[op2_insn->op2].val.u64; + val.u64 = ir_fix_signed(IR_OPT_TYPE(opt), + op1_insn->val.u64 + ctx->ir_base[op2_insn->op2].val.u64); op2 = op2_insn->op1; op1 = ir_const(ctx, val, op1_insn->type); IR_FOLD_RESTART; } else if (IR_IS_CONST_REF(op2_insn->op1) && !IR_IS_SYM_CONST(ctx->ir_base[op2_insn->op1].op)) { /* c1 - (c2 - x) => x + (c1 - c2) */ - val.i64 = op1_insn->val.u64 - ctx->ir_base[op2_insn->op1].val.u64; - if (val.i64 < 0 && val.i64 != INT64_MIN) { + val.u64 = ir_fix_signed(IR_OPT_TYPE(opt), + op1_insn->val.u64 - ctx->ir_base[op2_insn->op1].val.u64); + if (val.i64 < 0 && ir_neg_is_safe(IR_OPT_TYPE(opt), val.i64)) { val.i64 = -val.i64; - opt++; /* ADD -> SUB */ + } else { + opt--; /* SUB -> ADD */ } op1 = op2_insn->op2; op2 = ir_const(ctx, val, op1_insn->type); @@ -3945,9 +3966,8 @@ IR_FOLD(MUL(MUL, C_ADDR)) { if (IR_IS_CONST_REF(op1_insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op2].op)) { /* (x * c1) * c2 => x * (c1 * c2) */ - uint32_t shift = (8 - ir_type_size[op2_insn->type]) * 8; - val.u64 = ctx->ir_base[op1_insn->op2].val.u64 * op2_insn->val.u64; - val.u64 = (val.u64 << shift) >> shift; + val.u64 = ir_fix_unsigned(IR_OPT_TYPE(opt), + ctx->ir_base[op1_insn->op2].val.u64 * op2_insn->val.u64); op1 = op1_insn->op1; op2 = ir_const(ctx, val, op2_insn->type); IR_FOLD_RESTART; @@ -3962,9 +3982,8 @@ IR_FOLD(MUL(MUL, C_I64)) { if (IR_IS_CONST_REF(op1_insn->op2) && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op2].op)) { /* (x * c1) * c2 => x * (c1 * c2) */ - uint32_t shift = (8 - ir_type_size[op2_insn->type]) * 8; - val.u64 = ctx->ir_base[op1_insn->op2].val.u64 * op2_insn->val.u64; - val.i64 = (int64_t)(val.u64 << shift) >> shift; + val.u64 = ir_fix_signed(IR_OPT_TYPE(opt), + ctx->ir_base[op1_insn->op2].val.u64 * op2_insn->val.u64); op1 = op1_insn->op1; op2 = ir_const(ctx, val, op2_insn->type); IR_FOLD_RESTART; diff --git a/ext/opcache/jit/ir/ir_private.h b/ext/opcache/jit/ir/ir_private.h index 028acd66cb7d..6fa8efa9cf71 100644 --- a/ext/opcache/jit/ir/ir_private.h +++ b/ext/opcache/jit/ir/ir_private.h @@ -958,6 +958,39 @@ IR_ALWAYS_INLINE bool ir_ref_is_true(const ir_ctx *ctx, ir_ref ref) } } +IR_ALWAYS_INLINE bool ir_neg_is_safe(ir_type type, int64_t val) +{ + return val != (INT64_MIN >> ((8 - ir_type_size[type]) * 8)); +} + +/* Clear the "uvisible" high bits */ +IR_ALWAYS_INLINE uint64_t ir_fix_unsigned(ir_type type, uint64_t val) +{ + uint64_t shift; + + shift = (8 - ir_type_size[type]) * 8; + return (val << shift) >> shift; +} + +/* Sign extend the value into the "uvisible" high bits */ +IR_ALWAYS_INLINE uint64_t ir_fix_signed(ir_type type, uint64_t val) +{ + uint64_t shift; + + shift = (8 - ir_type_size[type]) * 8; + return (int64_t)(val << shift) >> shift; +} + +IR_ALWAYS_INLINE uint64_t ir_fix_int(ir_type type, uint64_t val) +{ + IR_ASSERT(IR_IS_TYPE_INT(type)); + if (IR_IS_TYPE_SIGNED(type)) { + return ir_fix_signed(type, val); + } else { + return ir_fix_unsigned(type, val); + } +} + /* IR OP flags */ #define IR_OP_FLAG_OPERANDS_SHIFT 3 diff --git a/ext/opcache/jit/ir/ir_sccp.c b/ext/opcache/jit/ir/ir_sccp.c index 27bb9163dccf..3ad6997b071e 100644 --- a/ext/opcache/jit/ir/ir_sccp.c +++ b/ext/opcache/jit/ir/ir_sccp.c @@ -3761,6 +3761,9 @@ static ir_ref ir_iter_optimize_condition(ir_ctx *ctx, ir_ref control, ir_ref con default: IR_ASSERT(0); } + if (IR_IS_TYPE_SIGNED(condition_insn->type)) { + val.u64 = ir_fix_signed(condition_insn->type, val.u64); + } ir_ref c = ir_const(ctx, val, condition_insn->type); condition_insn = &ctx->ir_base[condition]; @@ -3785,7 +3788,11 @@ static ir_ref ir_iter_optimize_condition(ir_ctx *ctx, ir_ref control, ir_ref con CLEAR_USES(condition_insn->op1); condition_insn->type = ctx->ir_base[op1_insn->op1].type; condition_insn->op1 = op1_insn->op1; - ir_ref c = ir_const(ctx, val_insn->val, condition_insn->type); + ir_val val = val_insn->val; + if (IR_IS_TYPE_SIGNED(condition_insn->type)) { + val.u64 = ir_fix_signed(condition_insn->type, val.u64); + } + ir_ref c = ir_const(ctx, val, condition_insn->type); condition_insn = &ctx->ir_base[condition]; condition_insn->op2 = c; ir_bitqueue_add(ctx->iter_worklist, condition); diff --git a/ext/opcache/jit/ir/ir_x86.dasc b/ext/opcache/jit/ir/ir_x86.dasc index 312a72ff175b..534f7cd55d74 100644 --- a/ext/opcache/jit/ir/ir_x86.dasc +++ b/ext/opcache/jit/ir/ir_x86.dasc @@ -27314,7 +27314,7 @@ next_block:; if (ctx->code_buffer) { if (ctx->code_buffer->pos == (char*)entry + size) { /* rollback */ - ctx->code_buffer->pos = (char*)entry - size; + ctx->code_buffer->pos = (char*)entry; } } else { ir_mem_unmap(entry, size); @@ -27440,7 +27440,7 @@ const void *ir_emit_exitgroup(uint32_t first_exit_point, uint32_t exit_points_pe dasm_free(&dasm_state); if (code_buffer->pos == (char*)entry + size) { /* rollback */ - code_buffer->pos = (char*)entry - size; + code_buffer->pos = (char*)entry; } return NULL; }