json - fix OOM error message on hook branch

NEWS

@@ -102,6 +102,11 @@ PHP                                                                        NEWS
   . Fixed IntlDateFormatter parse functions truncating an $offset below
     INT_MIN. (Marc Bennewitz)
 
+- JSON:
+  . Fixed json_encode() leaving the recursion guard set on objects with hooked
+    properties after running out of memory while escaping a property name.
+    (adapik)
+
 - MBString:
   . Fixed bug GH-23958 (truncated encoding names resolving to full names). (Kamil Tekiela)
   . Fixed out-of-bounds read in php_mb_rfc1867_getword() on a multibyte lead

ext/json/json_encoder.c

@@ -309,8 +309,9 @@ static zend_result php_json_encode_array(smart_str *buf, zval *val, int options,
 					if (php_json_escape_string(buf, ZSTR_VAL(key), ZSTR_LEN(key),
 								options & ~PHP_JSON_NUMERIC_CHECK, encoder) == FAILURE) {
 						if (ZEND_HAS_OOM()) {
-							PHP_JSON_HASH_UNPROTECT_RECURSION(myht);
+							PHP_JSON_HASH_UNPROTECT_RECURSION(recursion_rc);
 							zend_release_properties(prop_ht);
+							zval_ptr_dtor(&tmp);
 							return FAILURE;
 						}
 						if ((options & PHP_JSON_PARTIAL_OUTPUT_ON_ERROR) && buf->s) {

ext/json/tests/json_encode_oom_hooked_property_recursion_guard.phpt

@@ -0,0 +1,28 @@
+--TEST--
+json_encode() OOM while escaping a property name releases the recursion guard of an object with hooked properties
+--FILE--
+<?php
+
+#[AllowDynamicProperties]
+class C {
+    public string $a { get => str_repeat('v', 100); }
+}
+
+$o = new C;
+$o->{str_repeat('k', 3 * 1024 * 1024)} = 1;
+
+register_shutdown_function(function () use ($o) {
+    ini_set('memory_limit', '-1');
+    $json = json_encode($o);
+    var_dump($json !== false, json_last_error_msg());
+});
+
+// Leave less than the 3 MiB property name needs, so that pre-allocating it
+// fails while escaping the key, after the object was marked as being encoded.
+ini_set('memory_limit', memory_get_usage(true) + 1024 * 1024);
+json_encode($o);
+?>
+--EXPECTF--
+Fatal error: Allowed memory size of %d bytes exhausted %S(tried to allocate %d bytes) in %s on line %d
+%Abool(true)
+string(8) "No error"