From d413d45a9bbf11579fdaf8347ee6eb16f60f567b Mon Sep 17 00:00:00 2001 From: Sergey Vidyuk Date: Wed, 3 Sep 2025 12:50:17 +0700 Subject: [PATCH 1/9] Turn rapidjson.Decoder into heap type Each static type is uniq and global for all subinterpreters and can't access cached PyObjects. Decoder do use cached PyObjecs and must access cache stored in module rather than in static variables. Static type it can't access "proper module" due to its "per process singleton nature". The recomended way is to turn static types into heap types in order to use cached PyObjects. --- rapidjson.cpp | 123 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 78 insertions(+), 45 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index 9c8297a..81ba613 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -28,6 +29,53 @@ using namespace rapidjson; +#if PY_VERSION_HEX < 0x030A0000 +// Fallback for Py_TPFLAGS_IMMUTABLETYPE which is added in 3.10 +#define Py_TPFLAGS_IMMUTABLETYPE 0 +#endif + + +static uint32_t +py_version_hex() noexcept { +#if PY_VERSION_HEX < 0x030B0000 + static const uint32_t once_fetched_ver = [] { + constexpr uint32_t ver_fallback = 0x03000000; + + auto* ver_tuple = PySys_GetObject("version_info"); + if (!ver_tuple) + return ver_fallback; + + long major, minor, micro, serial; + const char* releaselevel; + if (!PyArg_ParseTuple(ver_tuple, "lllsl", &major, &minor, µ, &releaselevel, &serial)) + return ver_fallback; + + return uint32_t(((major & 0xFF) << 24) | ((minor & 0xFF) << 16) | ((micro & 0xFF) << 8)); + }(); + return once_fetched_ver; +#else + return Py_Version; +#endif +} + + +struct PyDerefer { + void operator() (PyObject* obj) const noexcept {Py_DecRef(obj);} +}; +using PyStrongRef = std::unique_ptr; + + +static PyStrongRef +inline from_module_and_spec(PyObject& module, PyType_Spec& spec) noexcept { +#if PY_VERSION_HEX >= 0x03090000 + PyStrongRef type{PyType_FromModuleAndSpec(&module, &spec, NULL)}; +#else + PyStrongRef type{PyType_FromSpec(&spec)}; +#endif + return type; +} + + /* On some MacOS combo, using Py_IS_XXX() macros does not work (see https://github.com/python-rapidjson/python-rapidjson/issues/78). OTOH, MSVC < 2015 does not have std::isxxx() (see @@ -208,6 +256,7 @@ static PyObject* do_decode(PyObject* decoder, unsigned uuidMode, unsigned parseMode); static PyObject* decoder_call(PyObject* self, PyObject* args, PyObject* kwargs); static PyObject* decoder_new(PyTypeObject* type, PyObject* args, PyObject* kwargs); +static int decoder_traverse(PyObject *op, visitproc visit, void *arg); static PyObject* do_encode(PyObject* value, PyObject* defaultFn, bool ensureAscii, @@ -1893,46 +1942,22 @@ static PyMemberDef decoder_members[] = { }; -static PyTypeObject Decoder_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "rapidjson.Decoder", /* tp_name */ - sizeof(DecoderObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - (ternaryfunc) decoder_call, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - decoder_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - decoder_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - decoder_new, /* tp_new */ - PyObject_Del, /* tp_free */ +static PyType_Slot Decoder_Type_Slot[] = { + {Py_tp_doc, const_cast(decoder_doc)}, + {Py_tp_call, reinterpret_cast(decoder_call)}, + {Py_tp_members, decoder_members}, + {Py_tp_new, reinterpret_cast(decoder_new)}, + {Py_tp_traverse, reinterpret_cast(decoder_traverse)}, + {0, NULL} +}; + + +static PyType_Spec Decoder_Type_Spec = { + "rapidjson.Decoder", /* name */ + sizeof(DecoderObject), /* basicsize */ + 0, /* itemsize */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE, /* flags */ + Decoder_Type_Slot /* slots */ }; @@ -2294,6 +2319,16 @@ decoder_new(PyTypeObject* type, PyObject* args, PyObject* kwargs) } +static int +decoder_traverse(PyObject *op, visitproc visit, void *arg) +{ + if (py_version_hex() >= 0x03090000) + Py_VISIT(Py_TYPE(op)); + return 0; +} + + + ///////////// // Encoder // ///////////// @@ -3881,7 +3916,8 @@ module_exec(PyObject* m) PyObject* decimalModule; PyObject* uuidModule; - if (PyType_Ready(&Decoder_Type) < 0) + auto decoder_type = from_module_and_spec(*m, Decoder_Type_Spec); + if (!decoder_type) return -1; if (PyType_Ready(&Encoder_Type) < 0) @@ -4064,11 +4100,8 @@ module_exec(PyObject* m) ) return -1; - Py_INCREF(&Decoder_Type); - if (PyModule_AddObject(m, "Decoder", (PyObject*) &Decoder_Type) < 0) { - Py_DECREF(&Decoder_Type); + if (PyModule_AddObject(m, "Decoder", decoder_type.get()) < 0) return -1; - } Py_INCREF(&Encoder_Type); if (PyModule_AddObject(m, "Encoder", (PyObject*) &Encoder_Type) < 0) { From 3ec06e7b379d1787b31f742a606ee82258a82052 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Mon, 24 Aug 2026 07:52:29 +0200 Subject: [PATCH 2/9] Properly release the stolen StrongRef See https://github.com/python-rapidjson/python-rapidjson/pull/227#pullrequestreview-4980034622 for an explanation. --- rapidjson.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/rapidjson.cpp b/rapidjson.cpp index 81ba613..3c99145 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -4102,6 +4102,7 @@ module_exec(PyObject* m) if (PyModule_AddObject(m, "Decoder", decoder_type.get()) < 0) return -1; + decoder_type.release(); Py_INCREF(&Encoder_Type); if (PyModule_AddObject(m, "Encoder", (PyObject*) &Encoder_Type) < 0) { From 428b629235deae3eebf0fc0402b1e5b053dad699 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Mon, 24 Aug 2026 08:43:08 +0200 Subject: [PATCH 3/9] Simplify, assuming Python >= 3.10 --- rapidjson.cpp | 37 +------------------------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index 3c99145..641d9c0 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -29,36 +29,6 @@ using namespace rapidjson; -#if PY_VERSION_HEX < 0x030A0000 -// Fallback for Py_TPFLAGS_IMMUTABLETYPE which is added in 3.10 -#define Py_TPFLAGS_IMMUTABLETYPE 0 -#endif - - -static uint32_t -py_version_hex() noexcept { -#if PY_VERSION_HEX < 0x030B0000 - static const uint32_t once_fetched_ver = [] { - constexpr uint32_t ver_fallback = 0x03000000; - - auto* ver_tuple = PySys_GetObject("version_info"); - if (!ver_tuple) - return ver_fallback; - - long major, minor, micro, serial; - const char* releaselevel; - if (!PyArg_ParseTuple(ver_tuple, "lllsl", &major, &minor, µ, &releaselevel, &serial)) - return ver_fallback; - - return uint32_t(((major & 0xFF) << 24) | ((minor & 0xFF) << 16) | ((micro & 0xFF) << 8)); - }(); - return once_fetched_ver; -#else - return Py_Version; -#endif -} - - struct PyDerefer { void operator() (PyObject* obj) const noexcept {Py_DecRef(obj);} }; @@ -67,11 +37,7 @@ using PyStrongRef = std::unique_ptr; static PyStrongRef inline from_module_and_spec(PyObject& module, PyType_Spec& spec) noexcept { -#if PY_VERSION_HEX >= 0x03090000 PyStrongRef type{PyType_FromModuleAndSpec(&module, &spec, NULL)}; -#else - PyStrongRef type{PyType_FromSpec(&spec)}; -#endif return type; } @@ -2322,8 +2288,7 @@ decoder_new(PyTypeObject* type, PyObject* args, PyObject* kwargs) static int decoder_traverse(PyObject *op, visitproc visit, void *arg) { - if (py_version_hex() >= 0x03090000) - Py_VISIT(Py_TYPE(op)); + Py_VISIT(Py_TYPE(op)); return 0; } From 683e65dbdf1b926af54c2ab4cbbf3a4fc5131355 Mon Sep 17 00:00:00 2001 From: Sergey Vidyuk Date: Sun, 30 Aug 2026 10:19:51 +0200 Subject: [PATCH 4/9] Turn rapidjson.Encoder into heap type --- rapidjson.cpp | 85 ++++++++++++++++++--------------------------------- 1 file changed, 29 insertions(+), 56 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index 641d9c0..568310f 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -42,6 +42,14 @@ inline from_module_and_spec(PyObject& module, PyType_Spec& spec) noexcept { } +static int +heap_type_traverse(PyObject *op, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(op)); + return 0; +} + + /* On some MacOS combo, using Py_IS_XXX() macros does not work (see https://github.com/python-rapidjson/python-rapidjson/issues/78). OTOH, MSVC < 2015 does not have std::isxxx() (see @@ -222,7 +230,6 @@ static PyObject* do_decode(PyObject* decoder, unsigned uuidMode, unsigned parseMode); static PyObject* decoder_call(PyObject* self, PyObject* args, PyObject* kwargs); static PyObject* decoder_new(PyTypeObject* type, PyObject* args, PyObject* kwargs); -static int decoder_traverse(PyObject *op, visitproc visit, void *arg); static PyObject* do_encode(PyObject* value, PyObject* defaultFn, bool ensureAscii, @@ -1913,7 +1920,7 @@ static PyType_Slot Decoder_Type_Slot[] = { {Py_tp_call, reinterpret_cast(decoder_call)}, {Py_tp_members, decoder_members}, {Py_tp_new, reinterpret_cast(decoder_new)}, - {Py_tp_traverse, reinterpret_cast(decoder_traverse)}, + {Py_tp_traverse, reinterpret_cast(heap_type_traverse)}, {0, NULL} }; @@ -2285,15 +2292,6 @@ decoder_new(PyTypeObject* type, PyObject* args, PyObject* kwargs) } -static int -decoder_traverse(PyObject *op, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(op)); - return 0; -} - - - ///////////// // Encoder // ///////////// @@ -3348,46 +3346,22 @@ static PyGetSetDef encoder_props[] = { {NULL} }; -static PyTypeObject Encoder_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "rapidjson.Encoder", /* tp_name */ - sizeof(EncoderObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - (ternaryfunc) encoder_call, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - encoder_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - encoder_members, /* tp_members */ - encoder_props, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - encoder_new, /* tp_new */ - PyObject_Del, /* tp_free */ +static PyType_Slot Encoder_Type_Slot[] = { + {Py_tp_doc, const_cast(encoder_doc)}, + {Py_tp_call, reinterpret_cast(encoder_call)}, + {Py_tp_members, encoder_members}, + {Py_tp_getset, encoder_props}, + {Py_tp_new, reinterpret_cast(encoder_new)}, + {Py_tp_traverse, reinterpret_cast(heap_type_traverse)}, + {0, NULL} +}; + +static PyType_Spec Encoder_Type_Spec = { + "rapidjson.Encoder", /* name */ + sizeof(EncoderObject), /* basicsize */ + 0, /* itemsize */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE, /* flags */ + Encoder_Type_Slot /* slots */ }; @@ -3885,7 +3859,8 @@ module_exec(PyObject* m) if (!decoder_type) return -1; - if (PyType_Ready(&Encoder_Type) < 0) + auto encoder_type = from_module_and_spec(*m, Encoder_Type_Spec); + if (!encoder_type) return -1; if (PyType_Ready(&Validator_Type) < 0) @@ -4069,11 +4044,9 @@ module_exec(PyObject* m) return -1; decoder_type.release(); - Py_INCREF(&Encoder_Type); - if (PyModule_AddObject(m, "Encoder", (PyObject*) &Encoder_Type) < 0) { - Py_DECREF(&Encoder_Type); + if (PyModule_AddObject(m, "Encoder", encoder_type.get()) < 0) return -1; - } + encoder_type.release(); Py_INCREF(&Validator_Type); if (PyModule_AddObject(m, "Validator", (PyObject*) &Validator_Type) < 0) { From e6ecc070ce63b85eadcd5ff8afb4920156fde244 Mon Sep 17 00:00:00 2001 From: Sergey Vidyuk Date: Sun, 30 Aug 2026 10:24:57 +0200 Subject: [PATCH 5/9] Turn rapidjson.Validator into heap type --- rapidjson.cpp | 65 ++++++++++++++++----------------------------------- 1 file changed, 20 insertions(+), 45 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index 568310f..a2e9175 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -3646,46 +3646,22 @@ PyDoc_STRVAR(validator_doc, " string."); -static PyTypeObject Validator_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "rapidjson.Validator", /* tp_name */ - sizeof(ValidatorObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor) validator_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - (ternaryfunc) validator_call, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - validator_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - validator_new, /* tp_new */ - PyObject_Del, /* tp_free */ +static PyType_Slot Validator_Type_Slot[] = { + {Py_tp_doc, const_cast(validator_doc)}, + {Py_tp_call, reinterpret_cast(validator_call)}, + {Py_tp_new, reinterpret_cast(validator_new)}, + {Py_tp_traverse, reinterpret_cast(heap_type_traverse)}, + {Py_tp_dealloc, reinterpret_cast(validator_dealloc)}, + {0, NULL} +}; + + +static PyType_Spec Validator_Type_Spec = { + "rapidjson.Validator", /* name */ + sizeof(ValidatorObject), /* basicsize */ + 0, /* itemsize */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE, /* flags */ + Validator_Type_Slot /* slots */ }; @@ -3863,7 +3839,8 @@ module_exec(PyObject* m) if (!encoder_type) return -1; - if (PyType_Ready(&Validator_Type) < 0) + auto validator_type = from_module_and_spec(*m, Validator_Type_Spec); + if (!validator_type) return -1; if (PyType_Ready(&RawJSON_Type) < 0) @@ -4048,11 +4025,9 @@ module_exec(PyObject* m) return -1; encoder_type.release(); - Py_INCREF(&Validator_Type); - if (PyModule_AddObject(m, "Validator", (PyObject*) &Validator_Type) < 0) { - Py_DECREF(&Validator_Type); + if (PyModule_AddObject(m, "Validator", validator_type.get()) < 0) return -1; - } + validator_type.release(); Py_INCREF(&RawJSON_Type); if (PyModule_AddObject(m, "RawJSON", (PyObject*) &RawJSON_Type) < 0) { From 0786768ad8155fa8e91c07583f35c5d66f63b562 Mon Sep 17 00:00:00 2001 From: Sergey Vidyuk Date: Sun, 30 Aug 2026 10:29:36 +0200 Subject: [PATCH 6/9] Turn rapidjson.RawJSON into heap type --- rapidjson.cpp | 68 +++++++++++++++++---------------------------------- 1 file changed, 23 insertions(+), 45 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index a2e9175..3b72785 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -69,6 +69,7 @@ static PyObject* decimal_type = NULL; static PyObject* timezone_type = NULL; static PyObject* timezone_utc = NULL; static PyObject* uuid_type = NULL; +static PyObject* rawjson_type = NULL; static PyObject* validation_error = NULL; static PyObject* decode_error = NULL; @@ -521,45 +522,22 @@ PyDoc_STRVAR(rawjson_doc, "'{\"already\": \"serialized\"}'"); -static PyTypeObject RawJSON_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "rapidjson.RawJSON", /* tp_name */ - sizeof(RawJSON), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor) RawJSON_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - rawjson_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - RawJSON_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - RawJSON_new, /* tp_new */ +static PyType_Slot RawJSON_Type_Slot[] = { + {Py_tp_dealloc, reinterpret_cast(RawJSON_dealloc)}, + {Py_tp_doc, const_cast(rawjson_doc)}, + {Py_tp_members, reinterpret_cast(RawJSON_members)}, + {Py_tp_new, reinterpret_cast(RawJSON_new)}, + {Py_tp_traverse, reinterpret_cast(heap_type_traverse)}, + {0, NULL} +}; + + +static PyType_Spec RawJSON_Type_Spec = { + "rapidjson.RawJSON", /* name */ + sizeof(RawJSON), /* basicsize */ + 0, /* itemsize */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE, /* flags */ + RawJSON_Type_Slot /* slots */ }; @@ -2977,7 +2955,7 @@ dumps_internal( return false; writer->EndArray(); - } else if (PyObject_TypeCheck(object, &RawJSON_Type)) { + } else if (PyObject_TypeCheck(object, (PyTypeObject*) rawjson_type)) { const char* jsonStr; Py_ssize_t l; jsonStr = PyUnicode_AsUTF8AndSize(((RawJSON*) object)->value, &l); @@ -3843,8 +3821,10 @@ module_exec(PyObject* m) if (!validator_type) return -1; - if (PyType_Ready(&RawJSON_Type) < 0) + auto rawjson_type_local_ref = from_module_and_spec(*m, RawJSON_Type_Spec); + if (!rawjson_type_local_ref) return -1; + rawjson_type = rawjson_type_local_ref.get(); PyDateTime_IMPORT; if(!PyDateTimeAPI) @@ -4029,11 +4009,9 @@ module_exec(PyObject* m) return -1; validator_type.release(); - Py_INCREF(&RawJSON_Type); - if (PyModule_AddObject(m, "RawJSON", (PyObject*) &RawJSON_Type) < 0) { - Py_DECREF(&RawJSON_Type); + if (PyModule_AddObject(m, "RawJSON", rawjson_type_local_ref.get()) < 0) return -1; - } + rawjson_type_local_ref.release(); validation_error = PyErr_NewException("rapidjson.ValidationError", PyExc_ValueError, NULL); From 905df23c2123dc84923630a43150579475870a78 Mon Sep 17 00:00:00 2001 From: Sergey Vidyuk Date: Sun, 30 Aug 2026 10:34:01 +0200 Subject: [PATCH 7/9] Move PyObject caches into a single variable of struct type. This is a first step in the direction of subinterpreter compatibility. Is's recommended to keep such caches in the module state rather then process wide globals: https://docs.python.org/3/howto/isolating-extensions.html#managing-per-module-state Dedicated struct was created which can later on be stored in the module state. It can't be done right now since those members are accessed from static types initialization code which doesn't have access to the module. Turning such static types into heap types is also subinterpreter compatibility prerequisite: https://docs.python.org/3/howto/isolating-extensions.html#heap-types --- rapidjson.cpp | 298 +++++++++++++++++++++++++++----------------------- 1 file changed, 161 insertions(+), 137 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index 3b72785..ff4daf3 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -64,14 +64,19 @@ heap_type_traverse(PyObject *op, visitproc visit, void *arg) #define IS_INF(x) std::isinf(x) #endif +struct Errors { + PyObject* validation = NULL; + PyObject* decode = NULL; +}; + -static PyObject* decimal_type = NULL; -static PyObject* timezone_type = NULL; -static PyObject* timezone_utc = NULL; -static PyObject* uuid_type = NULL; -static PyObject* rawjson_type = NULL; -static PyObject* validation_error = NULL; -static PyObject* decode_error = NULL; +struct Types { + PyObject* decimal = NULL; + PyObject* timezone = NULL; + PyObject* uuid = NULL; + PyObject* rawjson = NULL; + Errors errors; +}; /* These are the names of often used methods or literal values, interned in the module @@ -81,25 +86,44 @@ static PyObject* decode_error = NULL; We cannot use _Py_IDENTIFIER() because that upsets the GNU C++ compiler in -pedantic mode. */ -static PyObject* astimezone_name = NULL; -static PyObject* hex_name = NULL; -static PyObject* timestamp_name = NULL; -static PyObject* total_seconds_name = NULL; -static PyObject* utcoffset_name = NULL; -static PyObject* is_infinite_name = NULL; -static PyObject* is_nan_name = NULL; -static PyObject* start_object_name = NULL; -static PyObject* end_object_name = NULL; -static PyObject* default_name = NULL; -static PyObject* end_array_name = NULL; -static PyObject* string_name = NULL; -static PyObject* read_name = NULL; -static PyObject* write_name = NULL; -static PyObject* encoding_name = NULL; - -static PyObject* minus_inf_string_value = NULL; -static PyObject* nan_string_value = NULL; -static PyObject* plus_inf_string_value = NULL; +struct Names { + PyObject* astimezone = NULL; + PyObject* hex = NULL; + PyObject* timestamp = NULL; + PyObject* total_seconds = NULL; + PyObject* utcoffset = NULL; + PyObject* is_infinite = NULL; + PyObject* is_nan = NULL; + PyObject* start_object = NULL; + PyObject* end_object = NULL; + PyObject* default_name = NULL; + PyObject* end_array = NULL; + PyObject* string = NULL; + PyObject* read = NULL; + PyObject* write = NULL; + PyObject* encoding = NULL; +}; + + +struct Strings { + PyObject* minus_inf = NULL; + PyObject* nan = NULL; + PyObject* plus_inf = NULL; +}; + + +struct Consts { + PyObject* timezone_utc = NULL; + Names names; + Strings strings; +}; + + +struct Cache { + Types types; + Consts consts; +}; +static Cache cache; struct HandlerContext { @@ -321,7 +345,7 @@ class PyReadStreamWrapper { void Read() { Py_CLEAR(chunk); - chunk = PyObject_CallMethodObjArgs(stream, read_name, chunkSize, NULL); + chunk = PyObject_CallMethodObjArgs(stream, cache.consts.names.read, chunkSize, NULL); if (chunk == NULL) { eof = true; @@ -371,7 +395,7 @@ class PyWriteStreamWrapper { bufferEnd = buffer + size; cursor = buffer; multiByteChar = NULL; - isBinary = !PyObject_HasAttr(stream, encoding_name); + isBinary = !PyObject_HasAttr(stream, cache.consts.names.encoding); } ~PyWriteStreamWrapper() { @@ -418,7 +442,7 @@ class PyWriteStreamWrapper { if (c == NULL) { // Propagate the error state, it will be caught by dumps_internal() } else { - PyObject* res = PyObject_CallMethodObjArgs(stream, write_name, c, NULL); + PyObject* res = PyObject_CallMethodObjArgs(stream, cache.consts.names.write, c, NULL); if (res == NULL) { // Likewise } else { @@ -829,17 +853,17 @@ struct PyHandler { stack.reserve(128); if (decoder != NULL) { assert(!objectHook); - if (PyObject_HasAttr(decoder, start_object_name)) { - decoderStartObject = PyObject_GetAttr(decoder, start_object_name); + if (PyObject_HasAttr(decoder, cache.consts.names.start_object)) { + decoderStartObject = PyObject_GetAttr(decoder, cache.consts.names.start_object); } - if (PyObject_HasAttr(decoder, end_object_name)) { - decoderEndObject = PyObject_GetAttr(decoder, end_object_name); + if (PyObject_HasAttr(decoder, cache.consts.names.end_object)) { + decoderEndObject = PyObject_GetAttr(decoder, cache.consts.names.end_object); } - if (PyObject_HasAttr(decoder, end_array_name)) { - decoderEndArray = PyObject_GetAttr(decoder, end_array_name); + if (PyObject_HasAttr(decoder, cache.consts.names.end_array)) { + decoderEndArray = PyObject_GetAttr(decoder, cache.consts.names.end_array); } - if (PyObject_HasAttr(decoder, string_name)) { - decoderString = PyObject_GetAttr(decoder, string_name); + if (PyObject_HasAttr(decoder, cache.consts.names.string)) { + decoderString = PyObject_GetAttr(decoder, cache.consts.names.string); } } sharedKeys = PyDict_New(); @@ -1201,9 +1225,9 @@ struct PyHandler { PyObject* value; if (numberMode & NM_DECIMAL) { - value = PyObject_CallFunctionObjArgs(decimal_type, nan_string_value, NULL); + value = PyObject_CallFunctionObjArgs(cache.types.decimal, cache.consts.strings.nan, NULL); } else { - value = PyFloat_FromString(nan_string_value); + value = PyFloat_FromString(cache.consts.strings.nan); } if (value == NULL) @@ -1221,14 +1245,14 @@ struct PyHandler { PyObject* value; if (numberMode & NM_DECIMAL) { - value = PyObject_CallFunctionObjArgs(decimal_type, + value = PyObject_CallFunctionObjArgs(cache.types.decimal, minus - ? minus_inf_string_value - : plus_inf_string_value, NULL); + ? cache.consts.strings.minus_inf + : cache.consts.strings.plus_inf, NULL); } else { value = PyFloat_FromString(minus - ? minus_inf_string_value - : plus_inf_string_value); + ? cache.consts.strings.minus_inf + : cache.consts.strings.plus_inf); } if (value == NULL) @@ -1295,7 +1319,7 @@ struct PyHandler { PyObject* pystr = PyUnicode_FromStringAndSize(str, length); if (pystr == NULL) return false; - value = PyObject_CallFunctionObjArgs(decimal_type, pystr, NULL); + value = PyObject_CallFunctionObjArgs(cache.types.decimal, pystr, NULL); Py_DECREF(pystr); } else { std::string zstr(str, length); @@ -1450,11 +1474,11 @@ struct PyHandler { if ((datetimeMode & DM_NAIVE_IS_UTC || isZ) && !hasOffset) { if (hasDate) { value = PyDateTimeAPI->DateTime_FromDateAndTime( - year, month, day, hours, mins, secs, usecs, timezone_utc, + year, month, day, hours, mins, secs, usecs, cache.consts.timezone_utc, PyDateTimeAPI->DateTimeType); } else { value = PyDateTimeAPI->Time_FromTime( - hours, mins, secs, usecs, timezone_utc, PyDateTimeAPI->TimeType); + hours, mins, secs, usecs, cache.consts.timezone_utc, PyDateTimeAPI->TimeType); } } else if (datetimeMode & DM_IGNORE_TZ || (!hasOffset && !isZ)) { if (hasDate) { @@ -1469,14 +1493,14 @@ struct PyHandler { value = NULL; } else if (!hasDate && datetimeMode & DM_SHIFT_TO_UTC) { value = PyDateTimeAPI->Time_FromTime( - hours, mins, secs, usecs, timezone_utc, PyDateTimeAPI->TimeType); + hours, mins, secs, usecs, cache.consts.timezone_utc, PyDateTimeAPI->TimeType); } else { PyObject* offset = PyDateTimeAPI->Delta_FromDelta(0, tzoff, 0, 1, PyDateTimeAPI->DeltaType); if (offset == NULL) { value = NULL; } else { - PyObject* tz = PyObject_CallFunctionObjArgs(timezone_type, offset, NULL); + PyObject* tz = PyObject_CallFunctionObjArgs(cache.types.timezone, offset, NULL); Py_DECREF(offset); if (tz == NULL) { value = NULL; @@ -1487,7 +1511,7 @@ struct PyHandler { PyDateTimeAPI->DateTimeType); if (value != NULL && datetimeMode & DM_SHIFT_TO_UTC) { PyObject* asUTC = PyObject_CallMethodObjArgs( - value, astimezone_name, timezone_utc, NULL); + value, cache.consts.names.astimezone, cache.consts.timezone_utc, NULL); Py_DECREF(value); if (asUTC == NULL) { value = NULL; @@ -1534,7 +1558,7 @@ struct PyHandler { if (pystr == NULL) return false; - PyObject* value = PyObject_CallFunctionObjArgs(uuid_type, pystr, NULL); + PyObject* value = PyObject_CallFunctionObjArgs(cache.types.uuid, pystr, NULL); Py_DECREF(pystr); if (value == NULL) @@ -1752,7 +1776,7 @@ load(PyObject* self, PyObject* args, PyObject* kwargs) &allowNan)) return NULL; - if (!PyObject_HasAttr(jsonObject, read_name)) { + if (!PyObject_HasAttr(jsonObject, cache.consts.names.read)) { PyErr_SetString(PyExc_TypeError, "Expected file-like object"); return NULL; } @@ -2064,7 +2088,7 @@ do_decode(PyObject* decoder, const char* jsonStr, Py_ssize_t jsonStrLen, PyErr_Restore(etype, evalue, etraceback); } else - PyErr_Format(decode_error, "Parse error at offset %zu: %s", + PyErr_Format(cache.types.errors.decode, "Parse error at offset %zu: %s", offset, GetParseError_En(reader.GetParseErrorCode())); Py_XDECREF(handler.root); @@ -2131,7 +2155,7 @@ decoder_call(PyObject* self, PyObject* args, PyObject* kwargs) Py_DECREF(asUnicode); return NULL; } - } else if (PyObject_HasAttr(jsonObject, read_name)) { + } else if (PyObject_HasAttr(jsonObject, cache.consts.names.read)) { jsonStr = NULL; jsonStrLen = 0; } else { @@ -2334,14 +2358,14 @@ dumps_internal( } else if (PyBool_Check(object)) { writer->Bool(object == Py_True); } else if (numberMode & NM_DECIMAL - && (is_decimal = PyObject_IsInstance(object, decimal_type))) { + && (is_decimal = PyObject_IsInstance(object, cache.types.decimal))) { if (is_decimal == -1) { return false; } if (!(numberMode & NM_NAN)) { bool is_inf_or_nan; - PyObject* is_inf = PyObject_CallMethodObjArgs(object, is_infinite_name, + PyObject* is_inf = PyObject_CallMethodObjArgs(object, cache.consts.names.is_infinite, NULL); if (is_inf == NULL) { @@ -2351,7 +2375,7 @@ dumps_internal( Py_DECREF(is_inf); if (!is_inf_or_nan) { - PyObject* is_nan = PyObject_CallMethodObjArgs(object, is_nan_name, + PyObject* is_nan = PyObject_CallMethodObjArgs(object, cache.consts.names.is_nan, NULL); if (is_nan == NULL) { @@ -2627,9 +2651,9 @@ dumps_internal( char timeZone[TIMEZONE_LEN] = { 0 }; if (!(datetimeMode & DM_IGNORE_TZ) - && PyObject_HasAttr(object, utcoffset_name)) { + && PyObject_HasAttr(object, cache.consts.names.utcoffset)) { PyObject* utcOffset = PyObject_CallMethodObjArgs(object, - utcoffset_name, + cache.consts.names.utcoffset, NULL); if (utcOffset == NULL) @@ -2649,7 +2673,7 @@ dumps_internal( asUTC = PyDateTimeAPI->DateTime_FromDateAndTime( year, month, day, hour, min, sec, microsec, - timezone_utc, PyDateTimeAPI->DateTimeType); + cache.consts.timezone_utc, PyDateTimeAPI->DateTimeType); } else { hour = PyDateTime_TIME_GET_HOUR(dtObject); min = PyDateTime_TIME_GET_MINUTE(dtObject); @@ -2657,7 +2681,7 @@ dumps_internal( microsec = PyDateTime_TIME_GET_MICROSECOND(dtObject); asUTC = PyDateTimeAPI->Time_FromTime( hour, min, sec, microsec, - timezone_utc, PyDateTimeAPI->TimeType); + cache.consts.timezone_utc, PyDateTimeAPI->TimeType); } if (asUTC == NULL) { @@ -2675,8 +2699,8 @@ dumps_internal( if (datetimeMode & DM_SHIFT_TO_UTC) { // If it's not already in UTC, shift the value if (PyObject_IsTrue(utcOffset)) { - asUTC = PyObject_CallMethodObjArgs(object, astimezone_name, - timezone_utc, NULL); + asUTC = PyObject_CallMethodObjArgs(object, cache.consts.names.astimezone, + cache.consts.timezone_utc, NULL); if (asUTC == NULL) { Py_DECREF(utcOffset); @@ -2693,7 +2717,7 @@ dumps_internal( if (PyObject_IsTrue(utcOffset)) { PyObject* tsObj = PyObject_CallMethodObjArgs(utcOffset, - total_seconds_name, + cache.consts.names.total_seconds, NULL); if (tsObj == NULL) { @@ -2773,7 +2797,7 @@ dumps_internal( } else /* if (datetimeMode & DM_UNIX_TIME) */ { if (PyDateTime_Check(dtObject)) { PyObject* timestampObj = PyObject_CallMethodObjArgs(dtObject, - timestamp_name, + cache.consts.names.timestamp, NULL); if (timestampObj == NULL) { @@ -2850,7 +2874,7 @@ dumps_internal( if (datetimeMode & (DM_SHIFT_TO_UTC | DM_NAIVE_IS_UTC)) midnightObj = PyDateTimeAPI->DateTime_FromDateAndTime( year, month, day, 0, 0, 0, 0, - timezone_utc, PyDateTimeAPI->DateTimeType); + cache.consts.timezone_utc, PyDateTimeAPI->DateTimeType); else midnightObj = PyDateTime_FromDateAndTime(year, month, day, 0, 0, 0, 0); @@ -2859,7 +2883,7 @@ dumps_internal( return false; } - timestampObj = PyObject_CallMethodObjArgs(midnightObj, timestamp_name, + timestampObj = PyObject_CallMethodObjArgs(midnightObj, cache.consts.names.timestamp, NULL); Py_DECREF(midnightObj); @@ -2897,12 +2921,12 @@ dumps_internal( } } } else if (uuidMode != UM_NONE - && PyObject_TypeCheck(object, (PyTypeObject*) uuid_type)) { + && PyObject_TypeCheck(object, (PyTypeObject*) cache.types.uuid)) { PyObject* hexval; if (uuidMode == UM_CANONICAL) hexval = PyObject_Str(object); else - hexval = PyObject_GetAttr(object, hex_name); + hexval = PyObject_GetAttr(object, cache.consts.names.hex); if (hexval == NULL) return false; @@ -2955,7 +2979,7 @@ dumps_internal( return false; writer->EndArray(); - } else if (PyObject_TypeCheck(object, (PyTypeObject*) rawjson_type)) { + } else if (PyObject_TypeCheck(object, (PyTypeObject*) cache.types.rawjson)) { const char* jsonStr; Py_ssize_t l; jsonStr = PyUnicode_AsUTF8AndSize(((RawJSON*) object)->value, &l); @@ -3470,7 +3494,7 @@ encoder_call(PyObject* self, PyObject* args, PyObject* kwargs) EncoderObject* e = (EncoderObject*) self; if (stream != NULL && stream != Py_None) { - if (!PyObject_HasAttr(stream, write_name)) { + if (!PyObject_HasAttr(stream, cache.consts.names.write)) { PyErr_SetString(PyExc_TypeError, "Expected a writable stream"); return NULL; } @@ -3478,8 +3502,8 @@ encoder_call(PyObject* self, PyObject* args, PyObject* kwargs) if (!accept_chunk_size_arg(chunkSizeObj, chunkSize)) return NULL; - if (PyObject_HasAttr(self, default_name)) { - defaultFn = PyObject_GetAttr(self, default_name); + if (PyObject_HasAttr(self, cache.consts.names.default_name)) { + defaultFn = PyObject_GetAttr(self, cache.consts.names.default_name); } result = do_stream_encode(value, stream, chunkSize, defaultFn, e->ensureAscii, @@ -3487,8 +3511,8 @@ encoder_call(PyObject* self, PyObject* args, PyObject* kwargs) e->numberMode, e->datetimeMode, e->uuidMode, e->bytesMode, e->iterableMode, e->mappingMode); } else { - if (PyObject_HasAttr(self, default_name)) { - defaultFn = PyObject_GetAttr(self, default_name); + if (PyObject_HasAttr(self, cache.consts.names.default_name)) { + defaultFn = PyObject_GetAttr(self, cache.consts.names.default_name); } result = do_encode(value, defaultFn, e->ensureAscii, e->writeMode, e->indentChar, @@ -3682,7 +3706,7 @@ static PyObject* validator_call(PyObject* self, PyObject* args, PyObject* kwargs if (error) { if (asUnicode != NULL) Py_DECREF(asUnicode); - PyErr_SetString(decode_error, "Invalid JSON"); + PyErr_SetString(cache.types.errors.decode, "Invalid JSON"); return NULL; } @@ -3707,7 +3731,7 @@ static PyObject* validator_call(PyObject* self, PyObject* args, PyObject* kwargs PyObject* error = Py_BuildValue("sss", validator.GetInvalidSchemaKeyword(), sptr.GetString(), dptr.GetString()); - PyErr_SetObject(validation_error, error); + PyErr_SetObject(cache.types.errors.validation, error); if (error != NULL) Py_DECREF(error); @@ -3770,7 +3794,7 @@ static PyObject* validator_new(PyTypeObject* type, PyObject* args, PyObject* kwa Py_DECREF(asUnicode); if (error) { - PyErr_SetString(decode_error, "Invalid JSON"); + PyErr_SetString(cache.types.errors.decode, "Invalid JSON"); return NULL; } @@ -3821,10 +3845,10 @@ module_exec(PyObject* m) if (!validator_type) return -1; - auto rawjson_type_local_ref = from_module_and_spec(*m, RawJSON_Type_Spec); - if (!rawjson_type_local_ref) + auto rawjson_type = from_module_and_spec(*m, RawJSON_Type_Spec); + if (!rawjson_type) return -1; - rawjson_type = rawjson_type_local_ref.get(); + cache.types.rawjson = rawjson_type.get(); PyDateTime_IMPORT; if(!PyDateTimeAPI) @@ -3838,102 +3862,102 @@ module_exec(PyObject* m) if (decimalModule == NULL) return -1; - decimal_type = PyObject_GetAttrString(decimalModule, "Decimal"); + cache.types.decimal = PyObject_GetAttrString(decimalModule, "Decimal"); Py_DECREF(decimalModule); - if (decimal_type == NULL) + if (cache.types.decimal == NULL) return -1; - timezone_type = PyObject_GetAttrString(datetimeModule, "timezone"); + cache.types.timezone = PyObject_GetAttrString(datetimeModule, "timezone"); Py_DECREF(datetimeModule); - if (timezone_type == NULL) + if (cache.types.timezone == NULL) return -1; - timezone_utc = PyObject_GetAttrString(timezone_type, "utc"); - if (timezone_utc == NULL) + cache.consts.timezone_utc = PyObject_GetAttrString(cache.types.timezone, "utc"); + if (cache.consts.timezone_utc == NULL) return -1; uuidModule = PyImport_ImportModule("uuid"); if (uuidModule == NULL) return -1; - uuid_type = PyObject_GetAttrString(uuidModule, "UUID"); + cache.types.uuid = PyObject_GetAttrString(uuidModule, "UUID"); Py_DECREF(uuidModule); - if (uuid_type == NULL) + if (cache.types.uuid == NULL) return -1; - astimezone_name = PyUnicode_InternFromString("astimezone"); - if (astimezone_name == NULL) + cache.consts.names.astimezone = PyUnicode_InternFromString("astimezone"); + if (cache.consts.names.astimezone == NULL) return -1; - hex_name = PyUnicode_InternFromString("hex"); - if (hex_name == NULL) + cache.consts.names.hex = PyUnicode_InternFromString("hex"); + if (cache.consts.names.hex == NULL) return -1; - timestamp_name = PyUnicode_InternFromString("timestamp"); - if (timestamp_name == NULL) + cache.consts.names.timestamp = PyUnicode_InternFromString("timestamp"); + if (cache.consts.names.timestamp == NULL) return -1; - total_seconds_name = PyUnicode_InternFromString("total_seconds"); - if (total_seconds_name == NULL) + cache.consts.names.total_seconds = PyUnicode_InternFromString("total_seconds"); + if (cache.consts.names.total_seconds == NULL) return -1; - utcoffset_name = PyUnicode_InternFromString("utcoffset"); - if (utcoffset_name == NULL) + cache.consts.names.utcoffset = PyUnicode_InternFromString("utcoffset"); + if (cache.consts.names.utcoffset == NULL) return -1; - is_infinite_name = PyUnicode_InternFromString("is_infinite"); - if (is_infinite_name == NULL) + cache.consts.names.is_infinite = PyUnicode_InternFromString("is_infinite"); + if (cache.consts.names.is_infinite == NULL) return -1; - is_nan_name = PyUnicode_InternFromString("is_nan"); - if (is_infinite_name == NULL) + cache.consts.names.is_nan = PyUnicode_InternFromString("is_nan"); + if (cache.consts.names.is_infinite == NULL) return -1; - minus_inf_string_value = PyUnicode_InternFromString("-Infinity"); - if (minus_inf_string_value == NULL) + cache.consts.strings.minus_inf = PyUnicode_InternFromString("-Infinity"); + if (cache.consts.strings.minus_inf == NULL) return -1; - nan_string_value = PyUnicode_InternFromString("nan"); - if (nan_string_value == NULL) + cache.consts.strings.nan = PyUnicode_InternFromString("nan"); + if (cache.consts.strings.nan == NULL) return -1; - plus_inf_string_value = PyUnicode_InternFromString("+Infinity"); - if (plus_inf_string_value == NULL) + cache.consts.strings.plus_inf = PyUnicode_InternFromString("+Infinity"); + if (cache.consts.strings.plus_inf == NULL) return -1; - start_object_name = PyUnicode_InternFromString("start_object"); - if (start_object_name == NULL) + cache.consts.names.start_object = PyUnicode_InternFromString("start_object"); + if (cache.consts.names.start_object == NULL) return -1; - end_object_name = PyUnicode_InternFromString("end_object"); - if (end_object_name == NULL) + cache.consts.names.end_object = PyUnicode_InternFromString("end_object"); + if (cache.consts.names.end_object == NULL) return -1; - default_name = PyUnicode_InternFromString("default"); - if (default_name == NULL) + cache.consts.names.default_name = PyUnicode_InternFromString("default"); + if (cache.consts.names.default_name == NULL) return -1; - end_array_name = PyUnicode_InternFromString("end_array"); - if (end_array_name == NULL) + cache.consts.names.end_array = PyUnicode_InternFromString("end_array"); + if (cache.consts.names.end_array == NULL) return -1; - string_name = PyUnicode_InternFromString("string"); - if (string_name == NULL) + cache.consts.names.string = PyUnicode_InternFromString("string"); + if (cache.consts.names.string == NULL) return -1; - read_name = PyUnicode_InternFromString("read"); - if (read_name == NULL) + cache.consts.names.read = PyUnicode_InternFromString("read"); + if (cache.consts.names.read == NULL) return -1; - write_name = PyUnicode_InternFromString("write"); - if (write_name == NULL) + cache.consts.names.write = PyUnicode_InternFromString("write"); + if (cache.consts.names.write == NULL) return -1; - encoding_name = PyUnicode_InternFromString("encoding"); - if (encoding_name == NULL) + cache.consts.names.encoding = PyUnicode_InternFromString("encoding"); + if (cache.consts.names.encoding == NULL) return -1; #define STRINGIFY(x) XSTRINGIFY(x) @@ -4009,27 +4033,27 @@ module_exec(PyObject* m) return -1; validator_type.release(); - if (PyModule_AddObject(m, "RawJSON", rawjson_type_local_ref.get()) < 0) + if (PyModule_AddObject(m, "RawJSON", rawjson_type.get()) < 0) return -1; - rawjson_type_local_ref.release(); + rawjson_type.release(); - validation_error = PyErr_NewException("rapidjson.ValidationError", + cache.types.errors.validation = PyErr_NewException("rapidjson.ValidationError", PyExc_ValueError, NULL); - if (validation_error == NULL) + if (cache.types.errors.validation == NULL) return -1; - Py_INCREF(validation_error); - if (PyModule_AddObject(m, "ValidationError", validation_error) < 0) { - Py_DECREF(validation_error); + Py_INCREF(cache.types.errors.validation); + if (PyModule_AddObject(m, "ValidationError", cache.types.errors.validation) < 0) { + Py_DECREF(cache.types.errors.validation); return -1; } - decode_error = PyErr_NewException("rapidjson.JSONDecodeError", + cache.types.errors.decode = PyErr_NewException("rapidjson.JSONDecodeError", PyExc_ValueError, NULL); - if (decode_error == NULL) + if (cache.types.errors.decode == NULL) return -1; - Py_INCREF(decode_error); - if (PyModule_AddObject(m, "JSONDecodeError", decode_error) < 0) { - Py_DECREF(decode_error); + Py_INCREF(cache.types.errors.decode); + if (PyModule_AddObject(m, "JSONDecodeError", cache.types.errors.decode) < 0) { + Py_DECREF(cache.types.errors.decode); return -1; } From bda97129823b30add632bfa7069e3e578c426a43 Mon Sep 17 00:00:00 2001 From: Sergey Vidyuk Date: Sun, 30 Aug 2026 10:39:42 +0200 Subject: [PATCH 8/9] Reduce usage of cache global var by passing required parts as args --- rapidjson.cpp | 116 +++++++++++++++++++++++++++----------------------- 1 file changed, 63 insertions(+), 53 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index ff4daf3..5ae00dd 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -247,7 +247,8 @@ enum MappingMode { ////////////////////////// -static PyObject* do_decode(PyObject* decoder, +static PyObject* do_decode(const Consts& consts, const Types& types, + PyObject* decoder, const char* jsonStr, Py_ssize_t jsonStrlen, PyObject* jsonStream, size_t chunkSize, PyObject* objectHook, @@ -262,7 +263,7 @@ static PyObject* do_encode(PyObject* value, PyObject* defaultFn, bool ensureAsci unsigned numberMode, unsigned datetimeMode, unsigned uuidMode, unsigned bytesMode, unsigned iterableMode, unsigned mappingMode); -static PyObject* do_stream_encode(PyObject* value, PyObject* stream, size_t chunkSize, +static PyObject* do_stream_encode(const Names& names, PyObject* value, PyObject* stream, size_t chunkSize, PyObject* defaultFn, bool ensureAscii, unsigned writeMode, char indentChar, unsigned indentCount, unsigned numberMode, @@ -287,8 +288,8 @@ class PyReadStreamWrapper { public: typedef char Ch; - PyReadStreamWrapper(PyObject* stream, size_t size) - : stream(stream) { + PyReadStreamWrapper(const Names& names, PyObject* stream, size_t size) + : names(&names), stream(stream) { Py_INCREF(stream); chunkSize = PyLong_FromUnsignedLong(size); buffer = NULL; @@ -345,7 +346,7 @@ class PyReadStreamWrapper { void Read() { Py_CLEAR(chunk); - chunk = PyObject_CallMethodObjArgs(stream, cache.consts.names.read, chunkSize, NULL); + chunk = PyObject_CallMethodObjArgs(stream, names->read, chunkSize, NULL); if (chunk == NULL) { eof = true; @@ -372,6 +373,7 @@ class PyReadStreamWrapper { } } + const Names* names; PyObject* stream; PyObject* chunkSize; PyObject* chunk; @@ -387,15 +389,15 @@ class PyWriteStreamWrapper { public: typedef char Ch; - PyWriteStreamWrapper(PyObject* stream, size_t size) - : stream(stream) { + PyWriteStreamWrapper(const Names& names, PyObject* stream, size_t size) + : names(&names), stream(stream) { Py_INCREF(stream); buffer = (char*) PyMem_Malloc(size); assert(buffer); bufferEnd = buffer + size; cursor = buffer; multiByteChar = NULL; - isBinary = !PyObject_HasAttr(stream, cache.consts.names.encoding); + isBinary = !PyObject_HasAttr(stream, names.encoding); } ~PyWriteStreamWrapper() { @@ -442,7 +444,7 @@ class PyWriteStreamWrapper { if (c == NULL) { // Propagate the error state, it will be caught by dumps_internal() } else { - PyObject* res = PyObject_CallMethodObjArgs(stream, cache.consts.names.write, c, NULL); + PyObject* res = PyObject_CallMethodObjArgs(stream, names->write, c, NULL); if (res == NULL) { // Likewise } else { @@ -476,6 +478,7 @@ class PyWriteStreamWrapper { } private: + const Names* names; PyObject* stream; Ch* buffer; Ch* bufferEnd; @@ -822,6 +825,8 @@ float_from_string(const char* s, Py_ssize_t len) struct PyHandler { + const Consts* consts; + const Types* types; PyObject* decoderStartObject; PyObject* decoderEndObject; PyObject* decoderEndArray; @@ -835,12 +840,16 @@ struct PyHandler { unsigned recursionLimit; std::vector stack; - PyHandler(PyObject* decoder, + PyHandler(const Consts& consts, + const Types& types, + PyObject* decoder, PyObject* hook, unsigned dm, unsigned um, unsigned nm) - : decoderStartObject(NULL), + : consts(&consts), + types(&types), + decoderStartObject(NULL), decoderEndObject(NULL), decoderEndArray(NULL), decoderString(NULL), @@ -853,17 +862,17 @@ struct PyHandler { stack.reserve(128); if (decoder != NULL) { assert(!objectHook); - if (PyObject_HasAttr(decoder, cache.consts.names.start_object)) { - decoderStartObject = PyObject_GetAttr(decoder, cache.consts.names.start_object); + if (PyObject_HasAttr(decoder, consts.names.start_object)) { + decoderStartObject = PyObject_GetAttr(decoder, consts.names.start_object); } - if (PyObject_HasAttr(decoder, cache.consts.names.end_object)) { - decoderEndObject = PyObject_GetAttr(decoder, cache.consts.names.end_object); + if (PyObject_HasAttr(decoder, consts.names.end_object)) { + decoderEndObject = PyObject_GetAttr(decoder, consts.names.end_object); } - if (PyObject_HasAttr(decoder, cache.consts.names.end_array)) { - decoderEndArray = PyObject_GetAttr(decoder, cache.consts.names.end_array); + if (PyObject_HasAttr(decoder, consts.names.end_array)) { + decoderEndArray = PyObject_GetAttr(decoder, consts.names.end_array); } - if (PyObject_HasAttr(decoder, cache.consts.names.string)) { - decoderString = PyObject_GetAttr(decoder, cache.consts.names.string); + if (PyObject_HasAttr(decoder, consts.names.string)) { + decoderString = PyObject_GetAttr(decoder, consts.names.string); } } sharedKeys = PyDict_New(); @@ -1225,9 +1234,9 @@ struct PyHandler { PyObject* value; if (numberMode & NM_DECIMAL) { - value = PyObject_CallFunctionObjArgs(cache.types.decimal, cache.consts.strings.nan, NULL); + value = PyObject_CallFunctionObjArgs(types->decimal, consts->strings.nan, NULL); } else { - value = PyFloat_FromString(cache.consts.strings.nan); + value = PyFloat_FromString(consts->strings.nan); } if (value == NULL) @@ -1245,14 +1254,14 @@ struct PyHandler { PyObject* value; if (numberMode & NM_DECIMAL) { - value = PyObject_CallFunctionObjArgs(cache.types.decimal, + value = PyObject_CallFunctionObjArgs(types->decimal, minus - ? cache.consts.strings.minus_inf - : cache.consts.strings.plus_inf, NULL); + ? consts->strings.minus_inf + : consts->strings.plus_inf, NULL); } else { value = PyFloat_FromString(minus - ? cache.consts.strings.minus_inf - : cache.consts.strings.plus_inf); + ? consts->strings.minus_inf + : consts->strings.plus_inf); } if (value == NULL) @@ -1319,7 +1328,7 @@ struct PyHandler { PyObject* pystr = PyUnicode_FromStringAndSize(str, length); if (pystr == NULL) return false; - value = PyObject_CallFunctionObjArgs(cache.types.decimal, pystr, NULL); + value = PyObject_CallFunctionObjArgs(types->decimal, pystr, NULL); Py_DECREF(pystr); } else { std::string zstr(str, length); @@ -1474,11 +1483,11 @@ struct PyHandler { if ((datetimeMode & DM_NAIVE_IS_UTC || isZ) && !hasOffset) { if (hasDate) { value = PyDateTimeAPI->DateTime_FromDateAndTime( - year, month, day, hours, mins, secs, usecs, cache.consts.timezone_utc, + year, month, day, hours, mins, secs, usecs, consts->timezone_utc, PyDateTimeAPI->DateTimeType); } else { value = PyDateTimeAPI->Time_FromTime( - hours, mins, secs, usecs, cache.consts.timezone_utc, PyDateTimeAPI->TimeType); + hours, mins, secs, usecs, consts->timezone_utc, PyDateTimeAPI->TimeType); } } else if (datetimeMode & DM_IGNORE_TZ || (!hasOffset && !isZ)) { if (hasDate) { @@ -1493,14 +1502,14 @@ struct PyHandler { value = NULL; } else if (!hasDate && datetimeMode & DM_SHIFT_TO_UTC) { value = PyDateTimeAPI->Time_FromTime( - hours, mins, secs, usecs, cache.consts.timezone_utc, PyDateTimeAPI->TimeType); + hours, mins, secs, usecs, consts->timezone_utc, PyDateTimeAPI->TimeType); } else { PyObject* offset = PyDateTimeAPI->Delta_FromDelta(0, tzoff, 0, 1, PyDateTimeAPI->DeltaType); if (offset == NULL) { value = NULL; } else { - PyObject* tz = PyObject_CallFunctionObjArgs(cache.types.timezone, offset, NULL); + PyObject* tz = PyObject_CallFunctionObjArgs(types->timezone, offset, NULL); Py_DECREF(offset); if (tz == NULL) { value = NULL; @@ -1511,7 +1520,7 @@ struct PyHandler { PyDateTimeAPI->DateTimeType); if (value != NULL && datetimeMode & DM_SHIFT_TO_UTC) { PyObject* asUTC = PyObject_CallMethodObjArgs( - value, cache.consts.names.astimezone, cache.consts.timezone_utc, NULL); + value, consts->names.astimezone, consts->timezone_utc, NULL); Py_DECREF(value); if (asUTC == NULL) { value = NULL; @@ -1558,7 +1567,7 @@ struct PyHandler { if (pystr == NULL) return false; - PyObject* value = PyObject_CallFunctionObjArgs(cache.types.uuid, pystr, NULL); + PyObject* value = PyObject_CallFunctionObjArgs(types->uuid, pystr, NULL); Py_DECREF(pystr); if (value == NULL) @@ -1714,7 +1723,7 @@ loads(PyObject* self, PyObject* args, PyObject* kwargs) return NULL; } - PyObject* result = do_decode(NULL, jsonStr, jsonStrLen, NULL, 0, objectHook, + PyObject* result = do_decode(cache.consts, cache.types, NULL, jsonStr, jsonStrLen, NULL, 0, objectHook, numberMode, datetimeMode, uuidMode, parseMode); if (asUnicode != NULL) @@ -1888,7 +1897,7 @@ load(PyObject* self, PyObject* args, PyObject* kwargs) } } - return do_decode(NULL, NULL, 0, jsonObject, chunkSize, objectHook, + return do_decode(cache.consts, cache.types, NULL, NULL, 0, jsonObject, chunkSize, objectHook, numberMode, datetimeMode, uuidMode, parseMode); } @@ -2039,12 +2048,13 @@ static PyType_Spec Decoder_Type_Spec = { static PyObject* -do_decode(PyObject* decoder, const char* jsonStr, Py_ssize_t jsonStrLen, - PyObject* jsonStream, size_t chunkSize, PyObject* objectHook, - unsigned numberMode, unsigned datetimeMode, unsigned uuidMode, - unsigned parseMode) +do_decode(const Consts& consts, const Types& types, PyObject* decoder, + const char* jsonStr, Py_ssize_t jsonStrLen, PyObject* jsonStream, + size_t chunkSize, PyObject* objectHook, unsigned numberMode, + unsigned datetimeMode, unsigned uuidMode, unsigned parseMode) { - PyHandler handler(decoder, objectHook, datetimeMode, uuidMode, numberMode); + PyHandler handler(consts, types, decoder, objectHook, + datetimeMode, uuidMode, numberMode); Reader reader; if (jsonStr != NULL) { @@ -2061,7 +2071,7 @@ do_decode(PyObject* decoder, const char* jsonStr, Py_ssize_t jsonStrLen, PyMem_Free(jsonStrCopy); } else { - PyReadStreamWrapper sw(jsonStream, chunkSize); + PyReadStreamWrapper sw(consts.names, jsonStream, chunkSize); DECODE(reader, kParseNoFlags, sw, handler); } @@ -2088,7 +2098,7 @@ do_decode(PyObject* decoder, const char* jsonStr, Py_ssize_t jsonStrLen, PyErr_Restore(etype, evalue, etraceback); } else - PyErr_Format(cache.types.errors.decode, "Parse error at offset %zu: %s", + PyErr_Format(types.errors.decode, "Parse error at offset %zu: %s", offset, GetParseError_En(reader.GetParseErrorCode())); Py_XDECREF(handler.root); @@ -2167,9 +2177,9 @@ decoder_call(PyObject* self, PyObject* args, PyObject* kwargs) DecoderObject* d = (DecoderObject*) self; - PyObject* result = do_decode(self, jsonStr, jsonStrLen, jsonObject, chunkSize, NULL, - d->numberMode, d->datetimeMode, d->uuidMode, - d->parseMode); + PyObject* result = do_decode(cache.consts, cache.types, self, jsonStr, jsonStrLen, + jsonObject, chunkSize, NULL, d->numberMode, d->datetimeMode, + d->uuidMode, d->parseMode); if (asUnicode != NULL) Py_DECREF(asUnicode); @@ -3276,7 +3286,7 @@ dump(PyObject* self, PyObject* args, PyObject* kwargs) if (sortKeys) mappingMode |= MM_SORT_KEYS; - return do_stream_encode(value, stream, chunkSize, defaultFn, + return do_stream_encode(cache.consts.names, value, stream, chunkSize, defaultFn, ensureAscii ? true : false, writeMode, indentChar, indentCount, numberMode, datetimeMode, uuidMode, bytesMode, iterableMode, mappingMode); @@ -3434,13 +3444,13 @@ do_encode(PyObject* value, PyObject* defaultFn, bool ensureAscii, unsigned write static PyObject* -do_stream_encode(PyObject* value, PyObject* stream, size_t chunkSize, PyObject* defaultFn, - bool ensureAscii, unsigned writeMode, char indentChar, - unsigned indentCount, unsigned numberMode, unsigned datetimeMode, - unsigned uuidMode, unsigned bytesMode, unsigned iterableMode, - unsigned mappingMode) +do_stream_encode(const Names& names, PyObject* value, PyObject* stream, size_t chunkSize, + PyObject* defaultFn, bool ensureAscii, unsigned writeMode, + char indentChar, unsigned indentCount, unsigned numberMode, + unsigned datetimeMode, unsigned uuidMode, unsigned bytesMode, + unsigned iterableMode, unsigned mappingMode) { - PyWriteStreamWrapper os(stream, chunkSize); + PyWriteStreamWrapper os(names, stream, chunkSize); if (writeMode == WM_COMPACT) { if (ensureAscii) { @@ -3506,7 +3516,7 @@ encoder_call(PyObject* self, PyObject* args, PyObject* kwargs) defaultFn = PyObject_GetAttr(self, cache.consts.names.default_name); } - result = do_stream_encode(value, stream, chunkSize, defaultFn, e->ensureAscii, + result = do_stream_encode(cache.consts.names, value, stream, chunkSize, defaultFn, e->ensureAscii, e->writeMode, e->indentChar, e->indentCount, e->numberMode, e->datetimeMode, e->uuidMode, e->bytesMode, e->iterableMode, e->mappingMode); From ec591c83e995b1b639e578349914d26dec6ad804 Mon Sep 17 00:00:00 2001 From: Sergey Vidyuk Date: Sun, 30 Aug 2026 10:41:47 +0200 Subject: [PATCH 9/9] Enable subinterpreters on Python >= 3.12 --- rapidjson.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rapidjson.cpp b/rapidjson.cpp index 5ae00dd..fae6080 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -4073,6 +4073,9 @@ module_exec(PyObject* m) static struct PyModuleDef_Slot slots[] = { {Py_mod_exec, (void*) module_exec}, +#if PY_VERSION_HEX >= 0x030C0000 + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, +#endif {0, NULL} };