diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index fb5622c99f7a13..c1a5c5a3f3260d 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -217,6 +217,7 @@ _PyLong_BothAreCompact(const PyLongObject* a, const PyLongObject* b) { static inline bool _PyLong_IsZero(const PyLongObject *op) { + assert(op != NULL); return (op->long_value.lv_tag & SIGN_MASK) == SIGN_ZERO; } @@ -346,6 +347,87 @@ _PyLong_CheckExactAndCompact(PyObject *op) return PyLong_CheckExact(op) && _PyLong_IsCompact((const PyLongObject *)op); } +/* A cheap guard used by Tier 2 / JIT integer fast paths. + * + * "Compact" ints are single-digit. Non-compact ints may still fit in int64_t, + * but are limited to a small number of digits (3 for 30-bit digits, 5 for + * 15-bit digits). This is an intentionally cheap filter: callers must still + * do an exact range check during extraction. + */ +#define _PY_LONG_MAX_DIGITS_FOR_INT64 ((64 + PyLong_SHIFT - 1) / PyLong_SHIFT) + +static inline int +_PyLong_CheckExactAndMightFitInt64(PyObject *op) +{ + if (!PyLong_CheckExact(op)) { + return 0; + } + const PyLongObject *v = (const PyLongObject *)op; + if (_PyLong_IsCompact(v)) { + return 1; + } + Py_ssize_t ndigits = _PyLong_DigitCount(v); + if (ndigits > _PY_LONG_MAX_DIGITS_FOR_INT64) { + return 0; + } + if (ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; /* abs(INT64_MIN) */ + uint64_t max_top = ((v->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + return (uint64_t)v->long_value.ob_digit[ndigits - 1] <= max_top; + } + return 1; +} + +/* Extract an exact int to int64_t without raising. + * + * Returns true on success and writes to *out; returns false if the value is + * out of int64_t range. Never sets an exception. + */ +static inline bool +_PyLong_TryAsInt64Exact(PyLongObject *v, int64_t *out) +{ + assert(PyLong_CheckExact((PyObject *)v)); + if (_PyLong_IsCompact(v)) { + *out = (int64_t)_PyLong_CompactValue(v); + return true; + } + Py_ssize_t ndigits = _PyLong_DigitCount(v); + if (ndigits == 0) { + *out = 0; + return true; + } + if (ndigits > _PY_LONG_MAX_DIGITS_FOR_INT64) { + return false; + } + uint64_t abs_val = 0; + unsigned int shift = 0; + for (Py_ssize_t i = 0; i < ndigits; i++) { + uint64_t d = (uint64_t)v->long_value.ob_digit[i]; + if (shift >= 64) { + return false; + } + if (shift != 0 && (d >> (64 - shift)) != 0) { + return false; + } + abs_val |= d << shift; + shift += PyLong_SHIFT; + } + int sign = 1 - (v->long_value.lv_tag & SIGN_MASK); + if (abs_val <= (uint64_t)INT64_MAX) { + *out = sign < 0 ? -(int64_t)abs_val : (int64_t)abs_val; + return true; + } + if (sign < 0 && abs_val == (uint64_t)INT64_MAX + 1) { + *out = INT64_MIN; + return true; + } + return false; +} + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index d2e29a1b95ede2..341cd60a83c060 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1114,12 +1114,12 @@ PyAPI_DATA(const struct opcode_metadata) _PyOpcode_opcode_metadata[267]; const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, @@ -1128,7 +1128,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_USTR_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_INTERPOLATION] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1174,7 +1174,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, - [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, + [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 7c44c0a430b731..98183473ac0167 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -873,578 +873,566 @@ extern "C" { #define _GUARD_NOS_FLOAT_r12 1085 #define _GUARD_NOS_FLOAT_r22 1086 #define _GUARD_NOS_FLOAT_r33 1087 -#define _GUARD_NOS_INT_r02 1088 -#define _GUARD_NOS_INT_r12 1089 -#define _GUARD_NOS_INT_r22 1090 -#define _GUARD_NOS_INT_r33 1091 -#define _GUARD_NOS_ITER_VIRTUAL_r02 1092 -#define _GUARD_NOS_ITER_VIRTUAL_r12 1093 -#define _GUARD_NOS_ITER_VIRTUAL_r22 1094 -#define _GUARD_NOS_ITER_VIRTUAL_r33 1095 -#define _GUARD_NOS_LIST_r02 1096 -#define _GUARD_NOS_LIST_r12 1097 -#define _GUARD_NOS_LIST_r22 1098 -#define _GUARD_NOS_LIST_r33 1099 -#define _GUARD_NOS_NOT_NULL_r02 1100 -#define _GUARD_NOS_NOT_NULL_r12 1101 -#define _GUARD_NOS_NOT_NULL_r22 1102 -#define _GUARD_NOS_NOT_NULL_r33 1103 -#define _GUARD_NOS_NULL_r02 1104 -#define _GUARD_NOS_NULL_r12 1105 -#define _GUARD_NOS_NULL_r22 1106 -#define _GUARD_NOS_NULL_r33 1107 -#define _GUARD_NOS_OVERFLOWED_r02 1108 -#define _GUARD_NOS_OVERFLOWED_r12 1109 -#define _GUARD_NOS_OVERFLOWED_r22 1110 -#define _GUARD_NOS_OVERFLOWED_r33 1111 -#define _GUARD_NOS_TUPLE_r02 1112 -#define _GUARD_NOS_TUPLE_r12 1113 -#define _GUARD_NOS_TUPLE_r22 1114 -#define _GUARD_NOS_TUPLE_r33 1115 -#define _GUARD_NOS_TYPE_VERSION_r02 1116 -#define _GUARD_NOS_TYPE_VERSION_r12 1117 -#define _GUARD_NOS_TYPE_VERSION_r22 1118 -#define _GUARD_NOS_TYPE_VERSION_r33 1119 -#define _GUARD_NOS_UNICODE_r02 1120 -#define _GUARD_NOS_UNICODE_r12 1121 -#define _GUARD_NOS_UNICODE_r22 1122 -#define _GUARD_NOS_UNICODE_r33 1123 -#define _GUARD_NOT_EXHAUSTED_LIST_r02 1124 -#define _GUARD_NOT_EXHAUSTED_LIST_r12 1125 -#define _GUARD_NOT_EXHAUSTED_LIST_r22 1126 -#define _GUARD_NOT_EXHAUSTED_LIST_r33 1127 -#define _GUARD_NOT_EXHAUSTED_RANGE_r02 1128 -#define _GUARD_NOT_EXHAUSTED_RANGE_r12 1129 -#define _GUARD_NOT_EXHAUSTED_RANGE_r22 1130 -#define _GUARD_NOT_EXHAUSTED_RANGE_r33 1131 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r02 1132 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r12 1133 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 1134 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 1135 -#define _GUARD_THIRD_NULL_r03 1136 -#define _GUARD_THIRD_NULL_r13 1137 -#define _GUARD_THIRD_NULL_r23 1138 -#define _GUARD_THIRD_NULL_r33 1139 -#define _GUARD_TOS_ANY_DICT_r01 1140 -#define _GUARD_TOS_ANY_DICT_r11 1141 -#define _GUARD_TOS_ANY_DICT_r22 1142 -#define _GUARD_TOS_ANY_DICT_r33 1143 -#define _GUARD_TOS_ANY_SET_r01 1144 -#define _GUARD_TOS_ANY_SET_r11 1145 -#define _GUARD_TOS_ANY_SET_r22 1146 -#define _GUARD_TOS_ANY_SET_r33 1147 -#define _GUARD_TOS_DICT_r01 1148 -#define _GUARD_TOS_DICT_r11 1149 -#define _GUARD_TOS_DICT_r22 1150 -#define _GUARD_TOS_DICT_r33 1151 -#define _GUARD_TOS_FLOAT_r01 1152 -#define _GUARD_TOS_FLOAT_r11 1153 -#define _GUARD_TOS_FLOAT_r22 1154 -#define _GUARD_TOS_FLOAT_r33 1155 -#define _GUARD_TOS_FROZENDICT_r01 1156 -#define _GUARD_TOS_FROZENDICT_r11 1157 -#define _GUARD_TOS_FROZENDICT_r22 1158 -#define _GUARD_TOS_FROZENDICT_r33 1159 -#define _GUARD_TOS_FROZENSET_r01 1160 -#define _GUARD_TOS_FROZENSET_r11 1161 -#define _GUARD_TOS_FROZENSET_r22 1162 -#define _GUARD_TOS_FROZENSET_r33 1163 -#define _GUARD_TOS_INT_r01 1164 -#define _GUARD_TOS_INT_r11 1165 -#define _GUARD_TOS_INT_r22 1166 -#define _GUARD_TOS_INT_r33 1167 -#define _GUARD_TOS_IS_NONE_r01 1168 -#define _GUARD_TOS_IS_NONE_r11 1169 -#define _GUARD_TOS_IS_NONE_r22 1170 -#define _GUARD_TOS_IS_NONE_r33 1171 -#define _GUARD_TOS_LIST_r01 1172 -#define _GUARD_TOS_LIST_r11 1173 -#define _GUARD_TOS_LIST_r22 1174 -#define _GUARD_TOS_LIST_r33 1175 -#define _GUARD_TOS_NOT_NULL_r01 1176 -#define _GUARD_TOS_NOT_NULL_r11 1177 -#define _GUARD_TOS_NOT_NULL_r22 1178 -#define _GUARD_TOS_NOT_NULL_r33 1179 -#define _GUARD_TOS_OVERFLOWED_r01 1180 -#define _GUARD_TOS_OVERFLOWED_r11 1181 -#define _GUARD_TOS_OVERFLOWED_r22 1182 -#define _GUARD_TOS_OVERFLOWED_r33 1183 -#define _GUARD_TOS_SET_r01 1184 -#define _GUARD_TOS_SET_r11 1185 -#define _GUARD_TOS_SET_r22 1186 -#define _GUARD_TOS_SET_r33 1187 -#define _GUARD_TOS_SLICE_r01 1188 -#define _GUARD_TOS_SLICE_r11 1189 -#define _GUARD_TOS_SLICE_r22 1190 -#define _GUARD_TOS_SLICE_r33 1191 -#define _GUARD_TOS_TUPLE_r01 1192 -#define _GUARD_TOS_TUPLE_r11 1193 -#define _GUARD_TOS_TUPLE_r22 1194 -#define _GUARD_TOS_TUPLE_r33 1195 -#define _GUARD_TOS_UNICODE_r01 1196 -#define _GUARD_TOS_UNICODE_r11 1197 -#define _GUARD_TOS_UNICODE_r22 1198 -#define _GUARD_TOS_UNICODE_r33 1199 -#define _GUARD_TYPE_r01 1200 -#define _GUARD_TYPE_r11 1201 -#define _GUARD_TYPE_r22 1202 -#define _GUARD_TYPE_r33 1203 -#define _GUARD_TYPE_ITER_r02 1204 -#define _GUARD_TYPE_ITER_r12 1205 -#define _GUARD_TYPE_ITER_r22 1206 -#define _GUARD_TYPE_ITER_r33 1207 -#define _GUARD_TYPE_VERSION_r01 1208 -#define _GUARD_TYPE_VERSION_r11 1209 -#define _GUARD_TYPE_VERSION_r22 1210 -#define _GUARD_TYPE_VERSION_r33 1211 -#define _GUARD_TYPE_VERSION_LOCKED_r01 1212 -#define _GUARD_TYPE_VERSION_LOCKED_r11 1213 -#define _GUARD_TYPE_VERSION_LOCKED_r22 1214 -#define _GUARD_TYPE_VERSION_LOCKED_r33 1215 -#define _HANDLE_PENDING_AND_DEOPT_r00 1216 -#define _HANDLE_PENDING_AND_DEOPT_r10 1217 -#define _HANDLE_PENDING_AND_DEOPT_r20 1218 -#define _HANDLE_PENDING_AND_DEOPT_r30 1219 -#define _IMPORT_FROM_r12 1220 -#define _IMPORT_NAME_r21 1221 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 1222 -#define _INIT_CALL_PY_EXACT_ARGS_r01 1223 -#define _INIT_CALL_PY_EXACT_ARGS_0_r01 1224 -#define _INIT_CALL_PY_EXACT_ARGS_1_r01 1225 -#define _INIT_CALL_PY_EXACT_ARGS_2_r01 1226 -#define _INIT_CALL_PY_EXACT_ARGS_3_r01 1227 -#define _INIT_CALL_PY_EXACT_ARGS_4_r01 1228 -#define _INSERT_NULL_r10 1229 -#define _INSTRUMENTED_FOR_ITER_r23 1230 -#define _INSTRUMENTED_INSTRUCTION_r00 1231 -#define _INSTRUMENTED_JUMP_FORWARD_r00 1232 -#define _INSTRUMENTED_JUMP_FORWARD_r11 1233 -#define _INSTRUMENTED_JUMP_FORWARD_r22 1234 -#define _INSTRUMENTED_JUMP_FORWARD_r33 1235 -#define _INSTRUMENTED_LINE_r00 1236 -#define _INSTRUMENTED_NOT_TAKEN_r00 1237 -#define _INSTRUMENTED_NOT_TAKEN_r11 1238 -#define _INSTRUMENTED_NOT_TAKEN_r22 1239 -#define _INSTRUMENTED_NOT_TAKEN_r33 1240 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 1241 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 1242 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 1243 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 1244 -#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 1245 -#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 1246 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 1247 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 1248 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 1249 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 1250 -#define _IS_NONE_r11 1251 -#define _IS_OP_r03 1252 -#define _IS_OP_r13 1253 -#define _IS_OP_r23 1254 -#define _ITER_CHECK_LIST_r02 1255 -#define _ITER_CHECK_LIST_r12 1256 -#define _ITER_CHECK_LIST_r22 1257 -#define _ITER_CHECK_LIST_r33 1258 -#define _ITER_CHECK_RANGE_r02 1259 -#define _ITER_CHECK_RANGE_r12 1260 -#define _ITER_CHECK_RANGE_r22 1261 -#define _ITER_CHECK_RANGE_r33 1262 -#define _ITER_CHECK_TUPLE_r02 1263 -#define _ITER_CHECK_TUPLE_r12 1264 -#define _ITER_CHECK_TUPLE_r22 1265 -#define _ITER_CHECK_TUPLE_r33 1266 -#define _ITER_JUMP_LIST_r02 1267 -#define _ITER_JUMP_LIST_r12 1268 -#define _ITER_JUMP_LIST_r22 1269 -#define _ITER_JUMP_LIST_r33 1270 -#define _ITER_JUMP_RANGE_r02 1271 -#define _ITER_JUMP_RANGE_r12 1272 -#define _ITER_JUMP_RANGE_r22 1273 -#define _ITER_JUMP_RANGE_r33 1274 -#define _ITER_JUMP_TUPLE_r02 1275 -#define _ITER_JUMP_TUPLE_r12 1276 -#define _ITER_JUMP_TUPLE_r22 1277 -#define _ITER_JUMP_TUPLE_r33 1278 -#define _ITER_NEXT_INLINE_r23 1279 -#define _ITER_NEXT_LIST_r23 1280 -#define _ITER_NEXT_LIST_TIER_TWO_r23 1281 -#define _ITER_NEXT_RANGE_r03 1282 -#define _ITER_NEXT_RANGE_r13 1283 -#define _ITER_NEXT_RANGE_r23 1284 -#define _ITER_NEXT_TUPLE_r03 1285 -#define _ITER_NEXT_TUPLE_r13 1286 -#define _ITER_NEXT_TUPLE_r23 1287 -#define _JUMP_BACKWARD_NO_INTERRUPT_r00 1288 -#define _JUMP_BACKWARD_NO_INTERRUPT_r11 1289 -#define _JUMP_BACKWARD_NO_INTERRUPT_r22 1290 -#define _JUMP_BACKWARD_NO_INTERRUPT_r33 1291 -#define _JUMP_TO_TOP_r00 1292 -#define _LIST_APPEND_r10 1293 -#define _LIST_EXTEND_r11 1294 -#define _LOAD_ATTR_r10 1295 -#define _LOAD_ATTR_CLASS_r11 1296 -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11 1297 -#define _LOAD_ATTR_INSTANCE_VALUE_r02 1298 -#define _LOAD_ATTR_INSTANCE_VALUE_r12 1299 -#define _LOAD_ATTR_INSTANCE_VALUE_r23 1300 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 1301 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 1302 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 1303 -#define _LOAD_ATTR_METHOD_NO_DICT_r02 1304 -#define _LOAD_ATTR_METHOD_NO_DICT_r12 1305 -#define _LOAD_ATTR_METHOD_NO_DICT_r23 1306 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1307 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1308 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1309 -#define _LOAD_ATTR_MODULE_r12 1310 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1311 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1312 -#define _LOAD_ATTR_PROPERTY_FRAME_r01 1313 -#define _LOAD_ATTR_PROPERTY_FRAME_r11 1314 -#define _LOAD_ATTR_PROPERTY_FRAME_r22 1315 -#define _LOAD_ATTR_PROPERTY_FRAME_r33 1316 -#define _LOAD_ATTR_SLOT_r02 1317 -#define _LOAD_ATTR_SLOT_r12 1318 -#define _LOAD_ATTR_SLOT_r23 1319 -#define _LOAD_ATTR_WITH_HINT_r12 1320 -#define _LOAD_BUILD_CLASS_r01 1321 -#define _LOAD_BYTECODE_r00 1322 -#define _LOAD_COMMON_CONSTANT_r01 1323 -#define _LOAD_COMMON_CONSTANT_r12 1324 -#define _LOAD_COMMON_CONSTANT_r23 1325 -#define _LOAD_CONST_r01 1326 -#define _LOAD_CONST_r12 1327 -#define _LOAD_CONST_r23 1328 -#define _LOAD_CONST_INLINE_r01 1329 -#define _LOAD_CONST_INLINE_r12 1330 -#define _LOAD_CONST_INLINE_r23 1331 -#define _LOAD_CONST_INLINE_BORROW_r01 1332 -#define _LOAD_CONST_INLINE_BORROW_r12 1333 -#define _LOAD_CONST_INLINE_BORROW_r23 1334 -#define _LOAD_DEREF_r01 1335 -#define _LOAD_FAST_r01 1336 -#define _LOAD_FAST_r12 1337 -#define _LOAD_FAST_r23 1338 -#define _LOAD_FAST_0_r01 1339 -#define _LOAD_FAST_0_r12 1340 -#define _LOAD_FAST_0_r23 1341 -#define _LOAD_FAST_1_r01 1342 -#define _LOAD_FAST_1_r12 1343 -#define _LOAD_FAST_1_r23 1344 -#define _LOAD_FAST_2_r01 1345 -#define _LOAD_FAST_2_r12 1346 -#define _LOAD_FAST_2_r23 1347 -#define _LOAD_FAST_3_r01 1348 -#define _LOAD_FAST_3_r12 1349 -#define _LOAD_FAST_3_r23 1350 -#define _LOAD_FAST_4_r01 1351 -#define _LOAD_FAST_4_r12 1352 -#define _LOAD_FAST_4_r23 1353 -#define _LOAD_FAST_5_r01 1354 -#define _LOAD_FAST_5_r12 1355 -#define _LOAD_FAST_5_r23 1356 -#define _LOAD_FAST_6_r01 1357 -#define _LOAD_FAST_6_r12 1358 -#define _LOAD_FAST_6_r23 1359 -#define _LOAD_FAST_7_r01 1360 -#define _LOAD_FAST_7_r12 1361 -#define _LOAD_FAST_7_r23 1362 -#define _LOAD_FAST_AND_CLEAR_r01 1363 -#define _LOAD_FAST_AND_CLEAR_r12 1364 -#define _LOAD_FAST_AND_CLEAR_r23 1365 -#define _LOAD_FAST_BORROW_r01 1366 -#define _LOAD_FAST_BORROW_r12 1367 -#define _LOAD_FAST_BORROW_r23 1368 -#define _LOAD_FAST_BORROW_0_r01 1369 -#define _LOAD_FAST_BORROW_0_r12 1370 -#define _LOAD_FAST_BORROW_0_r23 1371 -#define _LOAD_FAST_BORROW_1_r01 1372 -#define _LOAD_FAST_BORROW_1_r12 1373 -#define _LOAD_FAST_BORROW_1_r23 1374 -#define _LOAD_FAST_BORROW_2_r01 1375 -#define _LOAD_FAST_BORROW_2_r12 1376 -#define _LOAD_FAST_BORROW_2_r23 1377 -#define _LOAD_FAST_BORROW_3_r01 1378 -#define _LOAD_FAST_BORROW_3_r12 1379 -#define _LOAD_FAST_BORROW_3_r23 1380 -#define _LOAD_FAST_BORROW_4_r01 1381 -#define _LOAD_FAST_BORROW_4_r12 1382 -#define _LOAD_FAST_BORROW_4_r23 1383 -#define _LOAD_FAST_BORROW_5_r01 1384 -#define _LOAD_FAST_BORROW_5_r12 1385 -#define _LOAD_FAST_BORROW_5_r23 1386 -#define _LOAD_FAST_BORROW_6_r01 1387 -#define _LOAD_FAST_BORROW_6_r12 1388 -#define _LOAD_FAST_BORROW_6_r23 1389 -#define _LOAD_FAST_BORROW_7_r01 1390 -#define _LOAD_FAST_BORROW_7_r12 1391 -#define _LOAD_FAST_BORROW_7_r23 1392 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1393 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1394 -#define _LOAD_FAST_CHECK_r01 1395 -#define _LOAD_FAST_CHECK_r12 1396 -#define _LOAD_FAST_CHECK_r23 1397 -#define _LOAD_FAST_LOAD_FAST_r02 1398 -#define _LOAD_FAST_LOAD_FAST_r13 1399 -#define _LOAD_FROM_DICT_OR_DEREF_r11 1400 -#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1401 -#define _LOAD_GLOBAL_r00 1402 -#define _LOAD_GLOBAL_BUILTINS_r01 1403 -#define _LOAD_GLOBAL_MODULE_r01 1404 -#define _LOAD_LOCALS_r01 1405 -#define _LOAD_LOCALS_r12 1406 -#define _LOAD_LOCALS_r23 1407 -#define _LOAD_NAME_r01 1408 -#define _LOAD_SMALL_INT_r01 1409 -#define _LOAD_SMALL_INT_r12 1410 -#define _LOAD_SMALL_INT_r23 1411 -#define _LOAD_SMALL_INT_0_r01 1412 -#define _LOAD_SMALL_INT_0_r12 1413 -#define _LOAD_SMALL_INT_0_r23 1414 -#define _LOAD_SMALL_INT_1_r01 1415 -#define _LOAD_SMALL_INT_1_r12 1416 -#define _LOAD_SMALL_INT_1_r23 1417 -#define _LOAD_SMALL_INT_2_r01 1418 -#define _LOAD_SMALL_INT_2_r12 1419 -#define _LOAD_SMALL_INT_2_r23 1420 -#define _LOAD_SMALL_INT_3_r01 1421 -#define _LOAD_SMALL_INT_3_r12 1422 -#define _LOAD_SMALL_INT_3_r23 1423 -#define _LOAD_SPECIAL_r00 1424 -#define _LOAD_SUPER_ATTR_ATTR_r31 1425 -#define _LOAD_SUPER_ATTR_METHOD_r32 1426 -#define _LOCK_OBJECT_r01 1427 -#define _LOCK_OBJECT_r11 1428 -#define _LOCK_OBJECT_r22 1429 -#define _LOCK_OBJECT_r33 1430 -#define _MAKE_CALLARGS_A_TUPLE_r33 1431 -#define _MAKE_CELL_r00 1432 -#define _MAKE_FUNCTION_r12 1433 -#define _MAKE_HEAP_SAFE_r01 1434 -#define _MAKE_HEAP_SAFE_r11 1435 -#define _MAKE_HEAP_SAFE_r22 1436 -#define _MAKE_HEAP_SAFE_r33 1437 -#define _MAKE_WARM_r00 1438 -#define _MAKE_WARM_r11 1439 -#define _MAKE_WARM_r22 1440 -#define _MAKE_WARM_r33 1441 -#define _MAP_ADD_r20 1442 -#define _MATCH_CLASS_r33 1443 -#define _MATCH_KEYS_r23 1444 -#define _MATCH_MAPPING_r02 1445 -#define _MATCH_MAPPING_r12 1446 -#define _MATCH_MAPPING_r23 1447 -#define _MATCH_SEQUENCE_r02 1448 -#define _MATCH_SEQUENCE_r12 1449 -#define _MATCH_SEQUENCE_r23 1450 -#define _MAYBE_EXPAND_METHOD_r00 1451 -#define _MAYBE_EXPAND_METHOD_KW_r11 1452 -#define _MONITOR_CALL_r00 1453 -#define _MONITOR_CALL_KW_r11 1454 -#define _MONITOR_JUMP_BACKWARD_r00 1455 -#define _MONITOR_JUMP_BACKWARD_r11 1456 -#define _MONITOR_JUMP_BACKWARD_r22 1457 -#define _MONITOR_JUMP_BACKWARD_r33 1458 -#define _MONITOR_RESUME_r00 1459 -#define _NOP_r00 1460 -#define _NOP_r11 1461 -#define _NOP_r22 1462 -#define _NOP_r33 1463 -#define _POP_EXCEPT_r10 1464 -#define _POP_ITER_r20 1465 -#define _POP_JUMP_IF_FALSE_r00 1466 -#define _POP_JUMP_IF_FALSE_r10 1467 -#define _POP_JUMP_IF_FALSE_r21 1468 -#define _POP_JUMP_IF_FALSE_r32 1469 -#define _POP_JUMP_IF_TRUE_r00 1470 -#define _POP_JUMP_IF_TRUE_r10 1471 -#define _POP_JUMP_IF_TRUE_r21 1472 -#define _POP_JUMP_IF_TRUE_r32 1473 -#define _POP_TOP_r10 1474 -#define _POP_TOP_FLOAT_r00 1475 -#define _POP_TOP_FLOAT_r10 1476 -#define _POP_TOP_FLOAT_r21 1477 -#define _POP_TOP_FLOAT_r32 1478 -#define _POP_TOP_INT_r00 1479 -#define _POP_TOP_INT_r10 1480 -#define _POP_TOP_INT_r21 1481 -#define _POP_TOP_INT_r32 1482 -#define _POP_TOP_NOP_r00 1483 -#define _POP_TOP_NOP_r10 1484 -#define _POP_TOP_NOP_r21 1485 -#define _POP_TOP_NOP_r32 1486 -#define _POP_TOP_OPARG_r00 1487 -#define _POP_TOP_UNICODE_r00 1488 -#define _POP_TOP_UNICODE_r10 1489 -#define _POP_TOP_UNICODE_r21 1490 -#define _POP_TOP_UNICODE_r32 1491 -#define _PUSH_EXC_INFO_r02 1492 -#define _PUSH_EXC_INFO_r12 1493 -#define _PUSH_EXC_INFO_r23 1494 -#define _PUSH_FRAME_r10 1495 -#define _PUSH_NULL_r01 1496 -#define _PUSH_NULL_r12 1497 -#define _PUSH_NULL_r23 1498 -#define _PUSH_NULL_CONDITIONAL_r00 1499 -#define _PUSH_TAGGED_ZERO_r01 1500 -#define _PUSH_TAGGED_ZERO_r12 1501 -#define _PUSH_TAGGED_ZERO_r23 1502 -#define _PY_FRAME_EX_r31 1503 -#define _PY_FRAME_GENERAL_r01 1504 -#define _PY_FRAME_KW_r11 1505 -#define _REPLACE_WITH_TRUE_r02 1506 -#define _REPLACE_WITH_TRUE_r12 1507 -#define _REPLACE_WITH_TRUE_r23 1508 -#define _RESUME_CHECK_r00 1509 -#define _RESUME_CHECK_r11 1510 -#define _RESUME_CHECK_r22 1511 -#define _RESUME_CHECK_r33 1512 -#define _RETURN_GENERATOR_r01 1513 -#define _RETURN_VALUE_r11 1514 -#define _RROT_3_r03 1515 -#define _RROT_3_r13 1516 -#define _RROT_3_r23 1517 -#define _RROT_3_r33 1518 -#define _SAVE_RETURN_OFFSET_r00 1519 -#define _SAVE_RETURN_OFFSET_r11 1520 -#define _SAVE_RETURN_OFFSET_r22 1521 -#define _SAVE_RETURN_OFFSET_r33 1522 -#define _SEND_ASYNC_GEN_r33 1523 -#define _SEND_ASYNC_GEN_TIER_TWO_r33 1524 -#define _SEND_GEN_FRAME_r33 1525 -#define _SEND_VIRTUAL_r33 1526 -#define _SEND_VIRTUAL_TIER_TWO_r03 1527 -#define _SEND_VIRTUAL_TIER_TWO_r13 1528 -#define _SEND_VIRTUAL_TIER_TWO_r23 1529 -#define _SEND_VIRTUAL_TIER_TWO_r33 1530 -#define _SETUP_ANNOTATIONS_r00 1531 -#define _SET_ADD_r10 1532 -#define _SET_FUNCTION_ATTRIBUTE_r01 1533 -#define _SET_FUNCTION_ATTRIBUTE_r11 1534 -#define _SET_FUNCTION_ATTRIBUTE_r21 1535 -#define _SET_FUNCTION_ATTRIBUTE_r32 1536 -#define _SET_IP_r00 1537 -#define _SET_IP_r11 1538 -#define _SET_IP_r22 1539 -#define _SET_IP_r33 1540 -#define _SET_UPDATE_r11 1541 -#define _SPILL_OR_RELOAD_r01 1542 -#define _SPILL_OR_RELOAD_r02 1543 -#define _SPILL_OR_RELOAD_r03 1544 -#define _SPILL_OR_RELOAD_r10 1545 -#define _SPILL_OR_RELOAD_r12 1546 -#define _SPILL_OR_RELOAD_r13 1547 -#define _SPILL_OR_RELOAD_r20 1548 -#define _SPILL_OR_RELOAD_r21 1549 -#define _SPILL_OR_RELOAD_r23 1550 -#define _SPILL_OR_RELOAD_r30 1551 -#define _SPILL_OR_RELOAD_r31 1552 -#define _SPILL_OR_RELOAD_r32 1553 -#define _START_EXECUTOR_r00 1554 -#define _STORE_ATTR_r20 1555 -#define _STORE_ATTR_INSTANCE_VALUE_r21 1556 -#define _STORE_ATTR_SLOT_r21 1557 -#define _STORE_ATTR_WITH_HINT_r21 1558 -#define _STORE_DEREF_r10 1559 -#define _STORE_FAST_LOAD_FAST_r11 1560 -#define _STORE_FAST_STORE_FAST_r20 1561 -#define _STORE_GLOBAL_r10 1562 -#define _STORE_NAME_r10 1563 -#define _STORE_SLICE_r30 1564 -#define _STORE_SUBSCR_r30 1565 -#define _STORE_SUBSCR_DICT_r31 1566 -#define _STORE_SUBSCR_DICT_KNOWN_HASH_r31 1567 -#define _STORE_SUBSCR_LIST_INT_r32 1568 -#define _SWAP_r11 1569 -#define _SWAP_2_r02 1570 -#define _SWAP_2_r12 1571 -#define _SWAP_2_r22 1572 -#define _SWAP_2_r33 1573 -#define _SWAP_3_r03 1574 -#define _SWAP_3_r13 1575 -#define _SWAP_3_r23 1576 -#define _SWAP_3_r33 1577 -#define _SWAP_FAST_r01 1578 -#define _SWAP_FAST_r11 1579 -#define _SWAP_FAST_r22 1580 -#define _SWAP_FAST_r33 1581 -#define _SWAP_FAST_0_r01 1582 -#define _SWAP_FAST_0_r11 1583 -#define _SWAP_FAST_0_r22 1584 -#define _SWAP_FAST_0_r33 1585 -#define _SWAP_FAST_1_r01 1586 -#define _SWAP_FAST_1_r11 1587 -#define _SWAP_FAST_1_r22 1588 -#define _SWAP_FAST_1_r33 1589 -#define _SWAP_FAST_2_r01 1590 -#define _SWAP_FAST_2_r11 1591 -#define _SWAP_FAST_2_r22 1592 -#define _SWAP_FAST_2_r33 1593 -#define _SWAP_FAST_3_r01 1594 -#define _SWAP_FAST_3_r11 1595 -#define _SWAP_FAST_3_r22 1596 -#define _SWAP_FAST_3_r33 1597 -#define _SWAP_FAST_4_r01 1598 -#define _SWAP_FAST_4_r11 1599 -#define _SWAP_FAST_4_r22 1600 -#define _SWAP_FAST_4_r33 1601 -#define _SWAP_FAST_5_r01 1602 -#define _SWAP_FAST_5_r11 1603 -#define _SWAP_FAST_5_r22 1604 -#define _SWAP_FAST_5_r33 1605 -#define _SWAP_FAST_6_r01 1606 -#define _SWAP_FAST_6_r11 1607 -#define _SWAP_FAST_6_r22 1608 -#define _SWAP_FAST_6_r33 1609 -#define _SWAP_FAST_7_r01 1610 -#define _SWAP_FAST_7_r11 1611 -#define _SWAP_FAST_7_r22 1612 -#define _SWAP_FAST_7_r33 1613 -#define _TIER2_RESUME_CHECK_r00 1614 -#define _TIER2_RESUME_CHECK_r11 1615 -#define _TIER2_RESUME_CHECK_r22 1616 -#define _TIER2_RESUME_CHECK_r33 1617 -#define _TO_BOOL_r11 1618 -#define _TO_BOOL_BOOL_r01 1619 -#define _TO_BOOL_BOOL_r11 1620 -#define _TO_BOOL_BOOL_r22 1621 -#define _TO_BOOL_BOOL_r33 1622 -#define _TO_BOOL_INT_r02 1623 -#define _TO_BOOL_INT_r12 1624 -#define _TO_BOOL_INT_r23 1625 -#define _TO_BOOL_LIST_r02 1626 -#define _TO_BOOL_LIST_r12 1627 -#define _TO_BOOL_LIST_r23 1628 -#define _TO_BOOL_NONE_r01 1629 -#define _TO_BOOL_NONE_r11 1630 -#define _TO_BOOL_NONE_r22 1631 -#define _TO_BOOL_NONE_r33 1632 -#define _TO_BOOL_STR_r02 1633 -#define _TO_BOOL_STR_r12 1634 -#define _TO_BOOL_STR_r23 1635 -#define _TRACE_RECORD_r00 1636 -#define _UNARY_INVERT_r12 1637 -#define _UNARY_NEGATIVE_r12 1638 -#define _UNARY_NEGATIVE_FLOAT_INPLACE_r02 1639 -#define _UNARY_NEGATIVE_FLOAT_INPLACE_r12 1640 -#define _UNARY_NEGATIVE_FLOAT_INPLACE_r23 1641 -#define _UNARY_NOT_r01 1642 -#define _UNARY_NOT_r11 1643 -#define _UNARY_NOT_r22 1644 -#define _UNARY_NOT_r33 1645 -#define _UNPACK_EX_r10 1646 -#define _UNPACK_SEQUENCE_r10 1647 -#define _UNPACK_SEQUENCE_LIST_r10 1648 -#define _UNPACK_SEQUENCE_TUPLE_r10 1649 -#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1650 -#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 1651 -#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 1652 -#define _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 1653 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 1654 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 1655 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 1656 -#define _WITH_EXCEPT_START_r33 1657 -#define _YIELD_VALUE_r11 1658 -#define MAX_UOP_REGS_ID 1658 +#define _GUARD_NOS_INT_r22 1088 +#define _GUARD_NOS_ITER_VIRTUAL_r02 1089 +#define _GUARD_NOS_ITER_VIRTUAL_r12 1090 +#define _GUARD_NOS_ITER_VIRTUAL_r22 1091 +#define _GUARD_NOS_ITER_VIRTUAL_r33 1092 +#define _GUARD_NOS_LIST_r02 1093 +#define _GUARD_NOS_LIST_r12 1094 +#define _GUARD_NOS_LIST_r22 1095 +#define _GUARD_NOS_LIST_r33 1096 +#define _GUARD_NOS_NOT_NULL_r02 1097 +#define _GUARD_NOS_NOT_NULL_r12 1098 +#define _GUARD_NOS_NOT_NULL_r22 1099 +#define _GUARD_NOS_NOT_NULL_r33 1100 +#define _GUARD_NOS_NULL_r02 1101 +#define _GUARD_NOS_NULL_r12 1102 +#define _GUARD_NOS_NULL_r22 1103 +#define _GUARD_NOS_NULL_r33 1104 +#define _GUARD_NOS_OVERFLOWED_r22 1105 +#define _GUARD_NOS_TUPLE_r02 1106 +#define _GUARD_NOS_TUPLE_r12 1107 +#define _GUARD_NOS_TUPLE_r22 1108 +#define _GUARD_NOS_TUPLE_r33 1109 +#define _GUARD_NOS_TYPE_VERSION_r02 1110 +#define _GUARD_NOS_TYPE_VERSION_r12 1111 +#define _GUARD_NOS_TYPE_VERSION_r22 1112 +#define _GUARD_NOS_TYPE_VERSION_r33 1113 +#define _GUARD_NOS_UNICODE_r02 1114 +#define _GUARD_NOS_UNICODE_r12 1115 +#define _GUARD_NOS_UNICODE_r22 1116 +#define _GUARD_NOS_UNICODE_r33 1117 +#define _GUARD_NOT_EXHAUSTED_LIST_r02 1118 +#define _GUARD_NOT_EXHAUSTED_LIST_r12 1119 +#define _GUARD_NOT_EXHAUSTED_LIST_r22 1120 +#define _GUARD_NOT_EXHAUSTED_LIST_r33 1121 +#define _GUARD_NOT_EXHAUSTED_RANGE_r02 1122 +#define _GUARD_NOT_EXHAUSTED_RANGE_r12 1123 +#define _GUARD_NOT_EXHAUSTED_RANGE_r22 1124 +#define _GUARD_NOT_EXHAUSTED_RANGE_r33 1125 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r02 1126 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r12 1127 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 1128 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 1129 +#define _GUARD_THIRD_NULL_r03 1130 +#define _GUARD_THIRD_NULL_r13 1131 +#define _GUARD_THIRD_NULL_r23 1132 +#define _GUARD_THIRD_NULL_r33 1133 +#define _GUARD_TOS_ANY_DICT_r01 1134 +#define _GUARD_TOS_ANY_DICT_r11 1135 +#define _GUARD_TOS_ANY_DICT_r22 1136 +#define _GUARD_TOS_ANY_DICT_r33 1137 +#define _GUARD_TOS_ANY_SET_r01 1138 +#define _GUARD_TOS_ANY_SET_r11 1139 +#define _GUARD_TOS_ANY_SET_r22 1140 +#define _GUARD_TOS_ANY_SET_r33 1141 +#define _GUARD_TOS_DICT_r01 1142 +#define _GUARD_TOS_DICT_r11 1143 +#define _GUARD_TOS_DICT_r22 1144 +#define _GUARD_TOS_DICT_r33 1145 +#define _GUARD_TOS_FLOAT_r01 1146 +#define _GUARD_TOS_FLOAT_r11 1147 +#define _GUARD_TOS_FLOAT_r22 1148 +#define _GUARD_TOS_FLOAT_r33 1149 +#define _GUARD_TOS_FROZENDICT_r01 1150 +#define _GUARD_TOS_FROZENDICT_r11 1151 +#define _GUARD_TOS_FROZENDICT_r22 1152 +#define _GUARD_TOS_FROZENDICT_r33 1153 +#define _GUARD_TOS_FROZENSET_r01 1154 +#define _GUARD_TOS_FROZENSET_r11 1155 +#define _GUARD_TOS_FROZENSET_r22 1156 +#define _GUARD_TOS_FROZENSET_r33 1157 +#define _GUARD_TOS_INT_r11 1158 +#define _GUARD_TOS_IS_NONE_r01 1159 +#define _GUARD_TOS_IS_NONE_r11 1160 +#define _GUARD_TOS_IS_NONE_r22 1161 +#define _GUARD_TOS_IS_NONE_r33 1162 +#define _GUARD_TOS_LIST_r01 1163 +#define _GUARD_TOS_LIST_r11 1164 +#define _GUARD_TOS_LIST_r22 1165 +#define _GUARD_TOS_LIST_r33 1166 +#define _GUARD_TOS_NOT_NULL_r01 1167 +#define _GUARD_TOS_NOT_NULL_r11 1168 +#define _GUARD_TOS_NOT_NULL_r22 1169 +#define _GUARD_TOS_NOT_NULL_r33 1170 +#define _GUARD_TOS_OVERFLOWED_r11 1171 +#define _GUARD_TOS_SET_r01 1172 +#define _GUARD_TOS_SET_r11 1173 +#define _GUARD_TOS_SET_r22 1174 +#define _GUARD_TOS_SET_r33 1175 +#define _GUARD_TOS_SLICE_r01 1176 +#define _GUARD_TOS_SLICE_r11 1177 +#define _GUARD_TOS_SLICE_r22 1178 +#define _GUARD_TOS_SLICE_r33 1179 +#define _GUARD_TOS_TUPLE_r01 1180 +#define _GUARD_TOS_TUPLE_r11 1181 +#define _GUARD_TOS_TUPLE_r22 1182 +#define _GUARD_TOS_TUPLE_r33 1183 +#define _GUARD_TOS_UNICODE_r01 1184 +#define _GUARD_TOS_UNICODE_r11 1185 +#define _GUARD_TOS_UNICODE_r22 1186 +#define _GUARD_TOS_UNICODE_r33 1187 +#define _GUARD_TYPE_r01 1188 +#define _GUARD_TYPE_r11 1189 +#define _GUARD_TYPE_r22 1190 +#define _GUARD_TYPE_r33 1191 +#define _GUARD_TYPE_ITER_r02 1192 +#define _GUARD_TYPE_ITER_r12 1193 +#define _GUARD_TYPE_ITER_r22 1194 +#define _GUARD_TYPE_ITER_r33 1195 +#define _GUARD_TYPE_VERSION_r01 1196 +#define _GUARD_TYPE_VERSION_r11 1197 +#define _GUARD_TYPE_VERSION_r22 1198 +#define _GUARD_TYPE_VERSION_r33 1199 +#define _GUARD_TYPE_VERSION_LOCKED_r01 1200 +#define _GUARD_TYPE_VERSION_LOCKED_r11 1201 +#define _GUARD_TYPE_VERSION_LOCKED_r22 1202 +#define _GUARD_TYPE_VERSION_LOCKED_r33 1203 +#define _HANDLE_PENDING_AND_DEOPT_r00 1204 +#define _HANDLE_PENDING_AND_DEOPT_r10 1205 +#define _HANDLE_PENDING_AND_DEOPT_r20 1206 +#define _HANDLE_PENDING_AND_DEOPT_r30 1207 +#define _IMPORT_FROM_r12 1208 +#define _IMPORT_NAME_r21 1209 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 1210 +#define _INIT_CALL_PY_EXACT_ARGS_r01 1211 +#define _INIT_CALL_PY_EXACT_ARGS_0_r01 1212 +#define _INIT_CALL_PY_EXACT_ARGS_1_r01 1213 +#define _INIT_CALL_PY_EXACT_ARGS_2_r01 1214 +#define _INIT_CALL_PY_EXACT_ARGS_3_r01 1215 +#define _INIT_CALL_PY_EXACT_ARGS_4_r01 1216 +#define _INSERT_NULL_r10 1217 +#define _INSTRUMENTED_FOR_ITER_r23 1218 +#define _INSTRUMENTED_INSTRUCTION_r00 1219 +#define _INSTRUMENTED_JUMP_FORWARD_r00 1220 +#define _INSTRUMENTED_JUMP_FORWARD_r11 1221 +#define _INSTRUMENTED_JUMP_FORWARD_r22 1222 +#define _INSTRUMENTED_JUMP_FORWARD_r33 1223 +#define _INSTRUMENTED_LINE_r00 1224 +#define _INSTRUMENTED_NOT_TAKEN_r00 1225 +#define _INSTRUMENTED_NOT_TAKEN_r11 1226 +#define _INSTRUMENTED_NOT_TAKEN_r22 1227 +#define _INSTRUMENTED_NOT_TAKEN_r33 1228 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 1229 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 1230 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 1231 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 1232 +#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 1233 +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 1234 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 1235 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 1236 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 1237 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 1238 +#define _IS_NONE_r11 1239 +#define _IS_OP_r03 1240 +#define _IS_OP_r13 1241 +#define _IS_OP_r23 1242 +#define _ITER_CHECK_LIST_r02 1243 +#define _ITER_CHECK_LIST_r12 1244 +#define _ITER_CHECK_LIST_r22 1245 +#define _ITER_CHECK_LIST_r33 1246 +#define _ITER_CHECK_RANGE_r02 1247 +#define _ITER_CHECK_RANGE_r12 1248 +#define _ITER_CHECK_RANGE_r22 1249 +#define _ITER_CHECK_RANGE_r33 1250 +#define _ITER_CHECK_TUPLE_r02 1251 +#define _ITER_CHECK_TUPLE_r12 1252 +#define _ITER_CHECK_TUPLE_r22 1253 +#define _ITER_CHECK_TUPLE_r33 1254 +#define _ITER_JUMP_LIST_r02 1255 +#define _ITER_JUMP_LIST_r12 1256 +#define _ITER_JUMP_LIST_r22 1257 +#define _ITER_JUMP_LIST_r33 1258 +#define _ITER_JUMP_RANGE_r02 1259 +#define _ITER_JUMP_RANGE_r12 1260 +#define _ITER_JUMP_RANGE_r22 1261 +#define _ITER_JUMP_RANGE_r33 1262 +#define _ITER_JUMP_TUPLE_r02 1263 +#define _ITER_JUMP_TUPLE_r12 1264 +#define _ITER_JUMP_TUPLE_r22 1265 +#define _ITER_JUMP_TUPLE_r33 1266 +#define _ITER_NEXT_INLINE_r23 1267 +#define _ITER_NEXT_LIST_r23 1268 +#define _ITER_NEXT_LIST_TIER_TWO_r23 1269 +#define _ITER_NEXT_RANGE_r03 1270 +#define _ITER_NEXT_RANGE_r13 1271 +#define _ITER_NEXT_RANGE_r23 1272 +#define _ITER_NEXT_TUPLE_r03 1273 +#define _ITER_NEXT_TUPLE_r13 1274 +#define _ITER_NEXT_TUPLE_r23 1275 +#define _JUMP_BACKWARD_NO_INTERRUPT_r00 1276 +#define _JUMP_BACKWARD_NO_INTERRUPT_r11 1277 +#define _JUMP_BACKWARD_NO_INTERRUPT_r22 1278 +#define _JUMP_BACKWARD_NO_INTERRUPT_r33 1279 +#define _JUMP_TO_TOP_r00 1280 +#define _LIST_APPEND_r10 1281 +#define _LIST_EXTEND_r11 1282 +#define _LOAD_ATTR_r10 1283 +#define _LOAD_ATTR_CLASS_r11 1284 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11 1285 +#define _LOAD_ATTR_INSTANCE_VALUE_r02 1286 +#define _LOAD_ATTR_INSTANCE_VALUE_r12 1287 +#define _LOAD_ATTR_INSTANCE_VALUE_r23 1288 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 1289 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 1290 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 1291 +#define _LOAD_ATTR_METHOD_NO_DICT_r02 1292 +#define _LOAD_ATTR_METHOD_NO_DICT_r12 1293 +#define _LOAD_ATTR_METHOD_NO_DICT_r23 1294 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1295 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1296 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1297 +#define _LOAD_ATTR_MODULE_r12 1298 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1299 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1300 +#define _LOAD_ATTR_PROPERTY_FRAME_r01 1301 +#define _LOAD_ATTR_PROPERTY_FRAME_r11 1302 +#define _LOAD_ATTR_PROPERTY_FRAME_r22 1303 +#define _LOAD_ATTR_PROPERTY_FRAME_r33 1304 +#define _LOAD_ATTR_SLOT_r02 1305 +#define _LOAD_ATTR_SLOT_r12 1306 +#define _LOAD_ATTR_SLOT_r23 1307 +#define _LOAD_ATTR_WITH_HINT_r12 1308 +#define _LOAD_BUILD_CLASS_r01 1309 +#define _LOAD_BYTECODE_r00 1310 +#define _LOAD_COMMON_CONSTANT_r01 1311 +#define _LOAD_COMMON_CONSTANT_r12 1312 +#define _LOAD_COMMON_CONSTANT_r23 1313 +#define _LOAD_CONST_r01 1314 +#define _LOAD_CONST_r12 1315 +#define _LOAD_CONST_r23 1316 +#define _LOAD_CONST_INLINE_r01 1317 +#define _LOAD_CONST_INLINE_r12 1318 +#define _LOAD_CONST_INLINE_r23 1319 +#define _LOAD_CONST_INLINE_BORROW_r01 1320 +#define _LOAD_CONST_INLINE_BORROW_r12 1321 +#define _LOAD_CONST_INLINE_BORROW_r23 1322 +#define _LOAD_DEREF_r01 1323 +#define _LOAD_FAST_r01 1324 +#define _LOAD_FAST_r12 1325 +#define _LOAD_FAST_r23 1326 +#define _LOAD_FAST_0_r01 1327 +#define _LOAD_FAST_0_r12 1328 +#define _LOAD_FAST_0_r23 1329 +#define _LOAD_FAST_1_r01 1330 +#define _LOAD_FAST_1_r12 1331 +#define _LOAD_FAST_1_r23 1332 +#define _LOAD_FAST_2_r01 1333 +#define _LOAD_FAST_2_r12 1334 +#define _LOAD_FAST_2_r23 1335 +#define _LOAD_FAST_3_r01 1336 +#define _LOAD_FAST_3_r12 1337 +#define _LOAD_FAST_3_r23 1338 +#define _LOAD_FAST_4_r01 1339 +#define _LOAD_FAST_4_r12 1340 +#define _LOAD_FAST_4_r23 1341 +#define _LOAD_FAST_5_r01 1342 +#define _LOAD_FAST_5_r12 1343 +#define _LOAD_FAST_5_r23 1344 +#define _LOAD_FAST_6_r01 1345 +#define _LOAD_FAST_6_r12 1346 +#define _LOAD_FAST_6_r23 1347 +#define _LOAD_FAST_7_r01 1348 +#define _LOAD_FAST_7_r12 1349 +#define _LOAD_FAST_7_r23 1350 +#define _LOAD_FAST_AND_CLEAR_r01 1351 +#define _LOAD_FAST_AND_CLEAR_r12 1352 +#define _LOAD_FAST_AND_CLEAR_r23 1353 +#define _LOAD_FAST_BORROW_r01 1354 +#define _LOAD_FAST_BORROW_r12 1355 +#define _LOAD_FAST_BORROW_r23 1356 +#define _LOAD_FAST_BORROW_0_r01 1357 +#define _LOAD_FAST_BORROW_0_r12 1358 +#define _LOAD_FAST_BORROW_0_r23 1359 +#define _LOAD_FAST_BORROW_1_r01 1360 +#define _LOAD_FAST_BORROW_1_r12 1361 +#define _LOAD_FAST_BORROW_1_r23 1362 +#define _LOAD_FAST_BORROW_2_r01 1363 +#define _LOAD_FAST_BORROW_2_r12 1364 +#define _LOAD_FAST_BORROW_2_r23 1365 +#define _LOAD_FAST_BORROW_3_r01 1366 +#define _LOAD_FAST_BORROW_3_r12 1367 +#define _LOAD_FAST_BORROW_3_r23 1368 +#define _LOAD_FAST_BORROW_4_r01 1369 +#define _LOAD_FAST_BORROW_4_r12 1370 +#define _LOAD_FAST_BORROW_4_r23 1371 +#define _LOAD_FAST_BORROW_5_r01 1372 +#define _LOAD_FAST_BORROW_5_r12 1373 +#define _LOAD_FAST_BORROW_5_r23 1374 +#define _LOAD_FAST_BORROW_6_r01 1375 +#define _LOAD_FAST_BORROW_6_r12 1376 +#define _LOAD_FAST_BORROW_6_r23 1377 +#define _LOAD_FAST_BORROW_7_r01 1378 +#define _LOAD_FAST_BORROW_7_r12 1379 +#define _LOAD_FAST_BORROW_7_r23 1380 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1381 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1382 +#define _LOAD_FAST_CHECK_r01 1383 +#define _LOAD_FAST_CHECK_r12 1384 +#define _LOAD_FAST_CHECK_r23 1385 +#define _LOAD_FAST_LOAD_FAST_r02 1386 +#define _LOAD_FAST_LOAD_FAST_r13 1387 +#define _LOAD_FROM_DICT_OR_DEREF_r11 1388 +#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1389 +#define _LOAD_GLOBAL_r00 1390 +#define _LOAD_GLOBAL_BUILTINS_r01 1391 +#define _LOAD_GLOBAL_MODULE_r01 1392 +#define _LOAD_LOCALS_r01 1393 +#define _LOAD_LOCALS_r12 1394 +#define _LOAD_LOCALS_r23 1395 +#define _LOAD_NAME_r01 1396 +#define _LOAD_SMALL_INT_r01 1397 +#define _LOAD_SMALL_INT_r12 1398 +#define _LOAD_SMALL_INT_r23 1399 +#define _LOAD_SMALL_INT_0_r01 1400 +#define _LOAD_SMALL_INT_0_r12 1401 +#define _LOAD_SMALL_INT_0_r23 1402 +#define _LOAD_SMALL_INT_1_r01 1403 +#define _LOAD_SMALL_INT_1_r12 1404 +#define _LOAD_SMALL_INT_1_r23 1405 +#define _LOAD_SMALL_INT_2_r01 1406 +#define _LOAD_SMALL_INT_2_r12 1407 +#define _LOAD_SMALL_INT_2_r23 1408 +#define _LOAD_SMALL_INT_3_r01 1409 +#define _LOAD_SMALL_INT_3_r12 1410 +#define _LOAD_SMALL_INT_3_r23 1411 +#define _LOAD_SPECIAL_r00 1412 +#define _LOAD_SUPER_ATTR_ATTR_r31 1413 +#define _LOAD_SUPER_ATTR_METHOD_r32 1414 +#define _LOCK_OBJECT_r01 1415 +#define _LOCK_OBJECT_r11 1416 +#define _LOCK_OBJECT_r22 1417 +#define _LOCK_OBJECT_r33 1418 +#define _MAKE_CALLARGS_A_TUPLE_r33 1419 +#define _MAKE_CELL_r00 1420 +#define _MAKE_FUNCTION_r12 1421 +#define _MAKE_HEAP_SAFE_r01 1422 +#define _MAKE_HEAP_SAFE_r11 1423 +#define _MAKE_HEAP_SAFE_r22 1424 +#define _MAKE_HEAP_SAFE_r33 1425 +#define _MAKE_WARM_r00 1426 +#define _MAKE_WARM_r11 1427 +#define _MAKE_WARM_r22 1428 +#define _MAKE_WARM_r33 1429 +#define _MAP_ADD_r20 1430 +#define _MATCH_CLASS_r33 1431 +#define _MATCH_KEYS_r23 1432 +#define _MATCH_MAPPING_r02 1433 +#define _MATCH_MAPPING_r12 1434 +#define _MATCH_MAPPING_r23 1435 +#define _MATCH_SEQUENCE_r02 1436 +#define _MATCH_SEQUENCE_r12 1437 +#define _MATCH_SEQUENCE_r23 1438 +#define _MAYBE_EXPAND_METHOD_r00 1439 +#define _MAYBE_EXPAND_METHOD_KW_r11 1440 +#define _MONITOR_CALL_r00 1441 +#define _MONITOR_CALL_KW_r11 1442 +#define _MONITOR_JUMP_BACKWARD_r00 1443 +#define _MONITOR_JUMP_BACKWARD_r11 1444 +#define _MONITOR_JUMP_BACKWARD_r22 1445 +#define _MONITOR_JUMP_BACKWARD_r33 1446 +#define _MONITOR_RESUME_r00 1447 +#define _NOP_r00 1448 +#define _NOP_r11 1449 +#define _NOP_r22 1450 +#define _NOP_r33 1451 +#define _POP_EXCEPT_r10 1452 +#define _POP_ITER_r20 1453 +#define _POP_JUMP_IF_FALSE_r00 1454 +#define _POP_JUMP_IF_FALSE_r10 1455 +#define _POP_JUMP_IF_FALSE_r21 1456 +#define _POP_JUMP_IF_FALSE_r32 1457 +#define _POP_JUMP_IF_TRUE_r00 1458 +#define _POP_JUMP_IF_TRUE_r10 1459 +#define _POP_JUMP_IF_TRUE_r21 1460 +#define _POP_JUMP_IF_TRUE_r32 1461 +#define _POP_TOP_r10 1462 +#define _POP_TOP_FLOAT_r00 1463 +#define _POP_TOP_FLOAT_r10 1464 +#define _POP_TOP_FLOAT_r21 1465 +#define _POP_TOP_FLOAT_r32 1466 +#define _POP_TOP_INT_r00 1467 +#define _POP_TOP_INT_r10 1468 +#define _POP_TOP_INT_r21 1469 +#define _POP_TOP_INT_r32 1470 +#define _POP_TOP_NOP_r00 1471 +#define _POP_TOP_NOP_r10 1472 +#define _POP_TOP_NOP_r21 1473 +#define _POP_TOP_NOP_r32 1474 +#define _POP_TOP_OPARG_r00 1475 +#define _POP_TOP_UNICODE_r00 1476 +#define _POP_TOP_UNICODE_r10 1477 +#define _POP_TOP_UNICODE_r21 1478 +#define _POP_TOP_UNICODE_r32 1479 +#define _PUSH_EXC_INFO_r02 1480 +#define _PUSH_EXC_INFO_r12 1481 +#define _PUSH_EXC_INFO_r23 1482 +#define _PUSH_FRAME_r10 1483 +#define _PUSH_NULL_r01 1484 +#define _PUSH_NULL_r12 1485 +#define _PUSH_NULL_r23 1486 +#define _PUSH_NULL_CONDITIONAL_r00 1487 +#define _PUSH_TAGGED_ZERO_r01 1488 +#define _PUSH_TAGGED_ZERO_r12 1489 +#define _PUSH_TAGGED_ZERO_r23 1490 +#define _PY_FRAME_EX_r31 1491 +#define _PY_FRAME_GENERAL_r01 1492 +#define _PY_FRAME_KW_r11 1493 +#define _REPLACE_WITH_TRUE_r02 1494 +#define _REPLACE_WITH_TRUE_r12 1495 +#define _REPLACE_WITH_TRUE_r23 1496 +#define _RESUME_CHECK_r00 1497 +#define _RESUME_CHECK_r11 1498 +#define _RESUME_CHECK_r22 1499 +#define _RESUME_CHECK_r33 1500 +#define _RETURN_GENERATOR_r01 1501 +#define _RETURN_VALUE_r11 1502 +#define _RROT_3_r03 1503 +#define _RROT_3_r13 1504 +#define _RROT_3_r23 1505 +#define _RROT_3_r33 1506 +#define _SAVE_RETURN_OFFSET_r00 1507 +#define _SAVE_RETURN_OFFSET_r11 1508 +#define _SAVE_RETURN_OFFSET_r22 1509 +#define _SAVE_RETURN_OFFSET_r33 1510 +#define _SEND_ASYNC_GEN_r33 1511 +#define _SEND_ASYNC_GEN_TIER_TWO_r33 1512 +#define _SEND_GEN_FRAME_r33 1513 +#define _SEND_VIRTUAL_r33 1514 +#define _SEND_VIRTUAL_TIER_TWO_r03 1515 +#define _SEND_VIRTUAL_TIER_TWO_r13 1516 +#define _SEND_VIRTUAL_TIER_TWO_r23 1517 +#define _SEND_VIRTUAL_TIER_TWO_r33 1518 +#define _SETUP_ANNOTATIONS_r00 1519 +#define _SET_ADD_r10 1520 +#define _SET_FUNCTION_ATTRIBUTE_r01 1521 +#define _SET_FUNCTION_ATTRIBUTE_r11 1522 +#define _SET_FUNCTION_ATTRIBUTE_r21 1523 +#define _SET_FUNCTION_ATTRIBUTE_r32 1524 +#define _SET_IP_r00 1525 +#define _SET_IP_r11 1526 +#define _SET_IP_r22 1527 +#define _SET_IP_r33 1528 +#define _SET_UPDATE_r11 1529 +#define _SPILL_OR_RELOAD_r01 1530 +#define _SPILL_OR_RELOAD_r02 1531 +#define _SPILL_OR_RELOAD_r03 1532 +#define _SPILL_OR_RELOAD_r10 1533 +#define _SPILL_OR_RELOAD_r12 1534 +#define _SPILL_OR_RELOAD_r13 1535 +#define _SPILL_OR_RELOAD_r20 1536 +#define _SPILL_OR_RELOAD_r21 1537 +#define _SPILL_OR_RELOAD_r23 1538 +#define _SPILL_OR_RELOAD_r30 1539 +#define _SPILL_OR_RELOAD_r31 1540 +#define _SPILL_OR_RELOAD_r32 1541 +#define _START_EXECUTOR_r00 1542 +#define _STORE_ATTR_r20 1543 +#define _STORE_ATTR_INSTANCE_VALUE_r21 1544 +#define _STORE_ATTR_SLOT_r21 1545 +#define _STORE_ATTR_WITH_HINT_r21 1546 +#define _STORE_DEREF_r10 1547 +#define _STORE_FAST_LOAD_FAST_r11 1548 +#define _STORE_FAST_STORE_FAST_r20 1549 +#define _STORE_GLOBAL_r10 1550 +#define _STORE_NAME_r10 1551 +#define _STORE_SLICE_r30 1552 +#define _STORE_SUBSCR_r30 1553 +#define _STORE_SUBSCR_DICT_r31 1554 +#define _STORE_SUBSCR_DICT_KNOWN_HASH_r31 1555 +#define _STORE_SUBSCR_LIST_INT_r32 1556 +#define _SWAP_r11 1557 +#define _SWAP_2_r02 1558 +#define _SWAP_2_r12 1559 +#define _SWAP_2_r22 1560 +#define _SWAP_2_r33 1561 +#define _SWAP_3_r03 1562 +#define _SWAP_3_r13 1563 +#define _SWAP_3_r23 1564 +#define _SWAP_3_r33 1565 +#define _SWAP_FAST_r01 1566 +#define _SWAP_FAST_r11 1567 +#define _SWAP_FAST_r22 1568 +#define _SWAP_FAST_r33 1569 +#define _SWAP_FAST_0_r01 1570 +#define _SWAP_FAST_0_r11 1571 +#define _SWAP_FAST_0_r22 1572 +#define _SWAP_FAST_0_r33 1573 +#define _SWAP_FAST_1_r01 1574 +#define _SWAP_FAST_1_r11 1575 +#define _SWAP_FAST_1_r22 1576 +#define _SWAP_FAST_1_r33 1577 +#define _SWAP_FAST_2_r01 1578 +#define _SWAP_FAST_2_r11 1579 +#define _SWAP_FAST_2_r22 1580 +#define _SWAP_FAST_2_r33 1581 +#define _SWAP_FAST_3_r01 1582 +#define _SWAP_FAST_3_r11 1583 +#define _SWAP_FAST_3_r22 1584 +#define _SWAP_FAST_3_r33 1585 +#define _SWAP_FAST_4_r01 1586 +#define _SWAP_FAST_4_r11 1587 +#define _SWAP_FAST_4_r22 1588 +#define _SWAP_FAST_4_r33 1589 +#define _SWAP_FAST_5_r01 1590 +#define _SWAP_FAST_5_r11 1591 +#define _SWAP_FAST_5_r22 1592 +#define _SWAP_FAST_5_r33 1593 +#define _SWAP_FAST_6_r01 1594 +#define _SWAP_FAST_6_r11 1595 +#define _SWAP_FAST_6_r22 1596 +#define _SWAP_FAST_6_r33 1597 +#define _SWAP_FAST_7_r01 1598 +#define _SWAP_FAST_7_r11 1599 +#define _SWAP_FAST_7_r22 1600 +#define _SWAP_FAST_7_r33 1601 +#define _TIER2_RESUME_CHECK_r00 1602 +#define _TIER2_RESUME_CHECK_r11 1603 +#define _TIER2_RESUME_CHECK_r22 1604 +#define _TIER2_RESUME_CHECK_r33 1605 +#define _TO_BOOL_r11 1606 +#define _TO_BOOL_BOOL_r01 1607 +#define _TO_BOOL_BOOL_r11 1608 +#define _TO_BOOL_BOOL_r22 1609 +#define _TO_BOOL_BOOL_r33 1610 +#define _TO_BOOL_INT_r02 1611 +#define _TO_BOOL_INT_r12 1612 +#define _TO_BOOL_INT_r23 1613 +#define _TO_BOOL_LIST_r02 1614 +#define _TO_BOOL_LIST_r12 1615 +#define _TO_BOOL_LIST_r23 1616 +#define _TO_BOOL_NONE_r01 1617 +#define _TO_BOOL_NONE_r11 1618 +#define _TO_BOOL_NONE_r22 1619 +#define _TO_BOOL_NONE_r33 1620 +#define _TO_BOOL_STR_r02 1621 +#define _TO_BOOL_STR_r12 1622 +#define _TO_BOOL_STR_r23 1623 +#define _TRACE_RECORD_r00 1624 +#define _UNARY_INVERT_r12 1625 +#define _UNARY_NEGATIVE_r12 1626 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r02 1627 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r12 1628 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r23 1629 +#define _UNARY_NOT_r01 1630 +#define _UNARY_NOT_r11 1631 +#define _UNARY_NOT_r22 1632 +#define _UNARY_NOT_r33 1633 +#define _UNPACK_EX_r10 1634 +#define _UNPACK_SEQUENCE_r10 1635 +#define _UNPACK_SEQUENCE_LIST_r10 1636 +#define _UNPACK_SEQUENCE_TUPLE_r10 1637 +#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1638 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 1639 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 1640 +#define _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 1641 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 1642 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 1643 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 1644 +#define _WITH_EXCEPT_START_r33 1645 +#define _YIELD_VALUE_r11 1646 +#define MAX_UOP_REGS_ID 1646 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 6713e9bc95f942..bfe22ef0bdadcc 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -105,15 +105,15 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_TOS_INT] = HAS_EXIT_FLAG, [_GUARD_NOS_OVERFLOWED] = HAS_EXIT_FLAG, [_GUARD_TOS_OVERFLOWED] = HAS_EXIT_FLAG, - [_BINARY_OP_MULTIPLY_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_INT_INPLACE] = HAS_EXIT_FLAG, - [_BINARY_OP_SUBTRACT_INT_INPLACE] = HAS_EXIT_FLAG, - [_BINARY_OP_MULTIPLY_INT_INPLACE] = HAS_EXIT_FLAG, - [_BINARY_OP_ADD_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG, - [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG, - [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG, + [_BINARY_OP_MULTIPLY_INT] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_INT] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_INT] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_INT_INPLACE] = HAS_EXIT_FLAG | HAS_ERROR_FLAG, + [_BINARY_OP_SUBTRACT_INT_INPLACE] = HAS_EXIT_FLAG | HAS_ERROR_FLAG, + [_BINARY_OP_MULTIPLY_INT_INPLACE] = HAS_EXIT_FLAG | HAS_ERROR_FLAG, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG | HAS_ERROR_FLAG, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG | HAS_ERROR_FLAG, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG | HAS_ERROR_FLAG, [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, @@ -242,7 +242,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_ATTR_SLOT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG, - [_COMPARE_OP_INT] = HAS_ARG_FLAG, + [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP_STR] = HAS_ARG_FLAG, [_IS_OP] = HAS_ARG_FLAG, [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, @@ -1044,39 +1044,39 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_GUARD_NOS_INT] = { - .best = { 0, 1, 2, 3 }, + .best = { 2, 2, 2, 2 }, .entries = { - { 2, 0, _GUARD_NOS_INT_r02 }, - { 2, 1, _GUARD_NOS_INT_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, { 2, 2, _GUARD_NOS_INT_r22 }, - { 3, 3, _GUARD_NOS_INT_r33 }, + { -1, -1, -1 }, }, }, [_GUARD_TOS_INT] = { - .best = { 0, 1, 2, 3 }, + .best = { 1, 1, 1, 1 }, .entries = { - { 1, 0, _GUARD_TOS_INT_r01 }, + { -1, -1, -1 }, { 1, 1, _GUARD_TOS_INT_r11 }, - { 2, 2, _GUARD_TOS_INT_r22 }, - { 3, 3, _GUARD_TOS_INT_r33 }, + { -1, -1, -1 }, + { -1, -1, -1 }, }, }, [_GUARD_NOS_OVERFLOWED] = { - .best = { 0, 1, 2, 3 }, + .best = { 2, 2, 2, 2 }, .entries = { - { 2, 0, _GUARD_NOS_OVERFLOWED_r02 }, - { 2, 1, _GUARD_NOS_OVERFLOWED_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, { 2, 2, _GUARD_NOS_OVERFLOWED_r22 }, - { 3, 3, _GUARD_NOS_OVERFLOWED_r33 }, + { -1, -1, -1 }, }, }, [_GUARD_TOS_OVERFLOWED] = { - .best = { 0, 1, 2, 3 }, + .best = { 1, 1, 1, 1 }, .entries = { - { 1, 0, _GUARD_TOS_OVERFLOWED_r01 }, + { -1, -1, -1 }, { 1, 1, _GUARD_TOS_OVERFLOWED_r11 }, - { 2, 2, _GUARD_TOS_OVERFLOWED_r22 }, - { 3, 3, _GUARD_TOS_OVERFLOWED_r33 }, + { -1, -1, -1 }, + { -1, -1, -1 }, }, }, [_BINARY_OP_MULTIPLY_INT] = { @@ -4137,22 +4137,10 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_REPLACE_WITH_TRUE_r12] = _REPLACE_WITH_TRUE, [_REPLACE_WITH_TRUE_r23] = _REPLACE_WITH_TRUE, [_UNARY_INVERT_r12] = _UNARY_INVERT, - [_GUARD_NOS_INT_r02] = _GUARD_NOS_INT, - [_GUARD_NOS_INT_r12] = _GUARD_NOS_INT, [_GUARD_NOS_INT_r22] = _GUARD_NOS_INT, - [_GUARD_NOS_INT_r33] = _GUARD_NOS_INT, - [_GUARD_TOS_INT_r01] = _GUARD_TOS_INT, [_GUARD_TOS_INT_r11] = _GUARD_TOS_INT, - [_GUARD_TOS_INT_r22] = _GUARD_TOS_INT, - [_GUARD_TOS_INT_r33] = _GUARD_TOS_INT, - [_GUARD_NOS_OVERFLOWED_r02] = _GUARD_NOS_OVERFLOWED, - [_GUARD_NOS_OVERFLOWED_r12] = _GUARD_NOS_OVERFLOWED, [_GUARD_NOS_OVERFLOWED_r22] = _GUARD_NOS_OVERFLOWED, - [_GUARD_NOS_OVERFLOWED_r33] = _GUARD_NOS_OVERFLOWED, - [_GUARD_TOS_OVERFLOWED_r01] = _GUARD_TOS_OVERFLOWED, [_GUARD_TOS_OVERFLOWED_r11] = _GUARD_TOS_OVERFLOWED, - [_GUARD_TOS_OVERFLOWED_r22] = _GUARD_TOS_OVERFLOWED, - [_GUARD_TOS_OVERFLOWED_r33] = _GUARD_TOS_OVERFLOWED, [_BINARY_OP_MULTIPLY_INT_r03] = _BINARY_OP_MULTIPLY_INT, [_BINARY_OP_MULTIPLY_INT_r13] = _BINARY_OP_MULTIPLY_INT, [_BINARY_OP_MULTIPLY_INT_r23] = _BINARY_OP_MULTIPLY_INT, @@ -5504,10 +5492,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_NOS_FLOAT_r22] = "_GUARD_NOS_FLOAT_r22", [_GUARD_NOS_FLOAT_r33] = "_GUARD_NOS_FLOAT_r33", [_GUARD_NOS_INT] = "_GUARD_NOS_INT", - [_GUARD_NOS_INT_r02] = "_GUARD_NOS_INT_r02", - [_GUARD_NOS_INT_r12] = "_GUARD_NOS_INT_r12", [_GUARD_NOS_INT_r22] = "_GUARD_NOS_INT_r22", - [_GUARD_NOS_INT_r33] = "_GUARD_NOS_INT_r33", [_GUARD_NOS_ITER_VIRTUAL] = "_GUARD_NOS_ITER_VIRTUAL", [_GUARD_NOS_ITER_VIRTUAL_r02] = "_GUARD_NOS_ITER_VIRTUAL_r02", [_GUARD_NOS_ITER_VIRTUAL_r12] = "_GUARD_NOS_ITER_VIRTUAL_r12", @@ -5529,10 +5514,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_NOS_NULL_r22] = "_GUARD_NOS_NULL_r22", [_GUARD_NOS_NULL_r33] = "_GUARD_NOS_NULL_r33", [_GUARD_NOS_OVERFLOWED] = "_GUARD_NOS_OVERFLOWED", - [_GUARD_NOS_OVERFLOWED_r02] = "_GUARD_NOS_OVERFLOWED_r02", - [_GUARD_NOS_OVERFLOWED_r12] = "_GUARD_NOS_OVERFLOWED_r12", [_GUARD_NOS_OVERFLOWED_r22] = "_GUARD_NOS_OVERFLOWED_r22", - [_GUARD_NOS_OVERFLOWED_r33] = "_GUARD_NOS_OVERFLOWED_r33", [_GUARD_NOS_TUPLE] = "_GUARD_NOS_TUPLE", [_GUARD_NOS_TUPLE_r02] = "_GUARD_NOS_TUPLE_r02", [_GUARD_NOS_TUPLE_r12] = "_GUARD_NOS_TUPLE_r12", @@ -5599,10 +5581,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_TOS_FROZENSET_r22] = "_GUARD_TOS_FROZENSET_r22", [_GUARD_TOS_FROZENSET_r33] = "_GUARD_TOS_FROZENSET_r33", [_GUARD_TOS_INT] = "_GUARD_TOS_INT", - [_GUARD_TOS_INT_r01] = "_GUARD_TOS_INT_r01", [_GUARD_TOS_INT_r11] = "_GUARD_TOS_INT_r11", - [_GUARD_TOS_INT_r22] = "_GUARD_TOS_INT_r22", - [_GUARD_TOS_INT_r33] = "_GUARD_TOS_INT_r33", [_GUARD_TOS_IS_NONE] = "_GUARD_TOS_IS_NONE", [_GUARD_TOS_IS_NONE_r01] = "_GUARD_TOS_IS_NONE_r01", [_GUARD_TOS_IS_NONE_r11] = "_GUARD_TOS_IS_NONE_r11", @@ -5619,10 +5598,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_TOS_NOT_NULL_r22] = "_GUARD_TOS_NOT_NULL_r22", [_GUARD_TOS_NOT_NULL_r33] = "_GUARD_TOS_NOT_NULL_r33", [_GUARD_TOS_OVERFLOWED] = "_GUARD_TOS_OVERFLOWED", - [_GUARD_TOS_OVERFLOWED_r01] = "_GUARD_TOS_OVERFLOWED_r01", [_GUARD_TOS_OVERFLOWED_r11] = "_GUARD_TOS_OVERFLOWED_r11", - [_GUARD_TOS_OVERFLOWED_r22] = "_GUARD_TOS_OVERFLOWED_r22", - [_GUARD_TOS_OVERFLOWED_r33] = "_GUARD_TOS_OVERFLOWED_r33", [_GUARD_TOS_SET] = "_GUARD_TOS_SET", [_GUARD_TOS_SET_r01] = "_GUARD_TOS_SET_r01", [_GUARD_TOS_SET_r11] = "_GUARD_TOS_SET_r11", diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 2248920c266aef..538bcf2df4a964 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -4245,6 +4245,52 @@ def testfunc(args): # Verify small int singletons are not corrupted self.assertEqual(7, 3 + 4) + def test_int_add_inplace_noncompact_unique_lhs(self): + # a + b produces a unique non-compact int that still fits in int64_t. + def testfunc(args): + a, b, c, n = args + total = 0 + for _ in range(n): + total += (a + b) + c + return total + + res, ex = self._run_with_optimizer( + testfunc, (5_000_000_000, 6_000_000_000, 7, TIER2_THRESHOLD)) + self.assertEqual(res, TIER2_THRESHOLD * 11_000_000_007) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_ADD_INT_INPLACE", uops) + + def test_int_add_inplace_noncompact_int64_boundary(self): + def testfunc(args): + a, b, n = args + total = 0 + for _ in range(n): + total += (a + b) + 1 + return total + + res, ex = self._run_with_optimizer( + testfunc, (4_611_686_018_427_387_903, 4_611_686_018_427_387_903, + TIER2_THRESHOLD)) + self.assertEqual(res, TIER2_THRESHOLD * 9_223_372_036_854_775_807) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_ADD_INT_INPLACE", uops) + + def test_int_add_inplace_noncompact_overflow_falls_back(self): + def testfunc(args): + a, b, n = args + total = 0 + for _ in range(n): + total += (a + b) + 1 + return total + + res, ex = self._run_with_optimizer( + testfunc, (4_611_686_018_427_387_904, 4_611_686_018_427_387_904, + TIER2_THRESHOLD)) + self.assertEqual(res, TIER2_THRESHOLD * 9_223_372_036_854_775_809) + self.assertIsNotNone(ex) + def test_int_subtract_inplace_unique_lhs(self): # a * b produces a unique compact int; subtracting c reuses it def testfunc(args): @@ -4275,6 +4321,38 @@ def testfunc(args): uops = get_opnames(ex) self.assertIn("_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT", uops) + def test_int_subtract_inplace_noncompact_unique_lhs(self): + # a + b produces a unique non-compact int that still fits in int64_t. + def testfunc(args): + a, b, c, n = args + total = 0 + for _ in range(n): + total += (a + b) - c + return total + + res, ex = self._run_with_optimizer( + testfunc, (5_000_000_000, 6_000_000_000, 7, TIER2_THRESHOLD)) + self.assertEqual(res, TIER2_THRESHOLD * 10_999_999_993) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_SUBTRACT_INT_INPLACE", uops) + + def test_int_subtract_inplace_noncompact_int64_min_boundary(self): + def testfunc(args): + a, b, n = args + total = 0 + for _ in range(n): + total += (a + b) - 1 + return total + + res, ex = self._run_with_optimizer( + testfunc, (-4_611_686_018_427_387_904, -4_611_686_018_427_387_903, + TIER2_THRESHOLD)) + self.assertEqual(res, TIER2_THRESHOLD * (-9_223_372_036_854_775_808)) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_SUBTRACT_INT_INPLACE", uops) + def test_int_multiply_inplace_unique_lhs(self): # (a + b) produces a unique compact int; multiplying by c reuses it def testfunc(args): @@ -4305,6 +4383,69 @@ def testfunc(args): uops = get_opnames(ex) self.assertIn("_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT", uops) + def test_int_multiply_inplace_noncompact_unique_lhs(self): + # a + b produces a unique non-compact int that still fits in int64_t. + def testfunc(args): + a, b, c, n = args + total = 0 + for _ in range(n): + total += (a + b) * c + return total + + res, ex = self._run_with_optimizer( + testfunc, (5_000_000_000, 6_000_000_000, 3, TIER2_THRESHOLD)) + self.assertEqual(res, TIER2_THRESHOLD * 33_000_000_000) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_MULTIPLY_INT_INPLACE", uops) + + def test_int_multiply_inplace_noncompact_overflow_falls_back(self): + def testfunc(args): + a, b, c, n = args + total = 0 + for _ in range(n): + total += (a + b) * c + return total + + res, ex = self._run_with_optimizer( + testfunc, (4_611_686_018_427_387_904, 4_611_686_018_427_387_904, 2, + TIER2_THRESHOLD)) + self.assertEqual(res, TIER2_THRESHOLD * 18_446_744_073_709_551_616) + self.assertIsNotNone(ex) + + def test_compare_int_noncompact(self): + def testfunc(args): + a, b, n = args + total = 0 + for _ in range(n): + if a < b: + total += 1 + return total + + res, ex = self._run_with_optimizer( + testfunc, (5_000_000_000, 6_000_000_000, TIER2_THRESHOLD)) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + + def test_compare_int_noncompact_all_predicates(self): + def testfunc(args): + a, b, n = args + total = 0 + for _ in range(n): + total += (a < b) + total += (a <= b) + total += (a == a) + total += (b != a) + total += (b > a) + total += (b >= a) + return total + + res, ex = self._run_with_optimizer( + testfunc, (9_223_372_036_854_775_006, 9_223_372_036_854_775_007, + TIER2_THRESHOLD)) + self.assertEqual(res, TIER2_THRESHOLD * 6) + self.assertIsNotNone(ex) + def test_int_inplace_chain_propagation(self): # a * b + c * d: both products are unique, the + reuses one; # result of + is also unique for the subsequent += diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 7946550ec0db63..aa356cd8809cb1 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1375,7 +1375,7 @@ def binary_op_add_int(): self.assert_specialized(binary_op_add_int, "BINARY_OP_ADD_INT") self.assert_no_opcode(binary_op_add_int, "BINARY_OP") - def binary_op_int_non_compact(): + def binary_op_int_non_compact_int64(): for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a, b = 10000000000, 1 c = a + b @@ -1385,10 +1385,25 @@ def binary_op_int_non_compact(): c = a * b self.assertEqual(c, 10000000000) - binary_op_int_non_compact() - self.assert_no_opcode(binary_op_int_non_compact, "BINARY_OP_ADD_INT") - self.assert_no_opcode(binary_op_int_non_compact, "BINARY_OP_SUBTRACT_INT") - self.assert_no_opcode(binary_op_int_non_compact, "BINARY_OP_MULTIPLY_INT") + binary_op_int_non_compact_int64() + self.assert_specialized(binary_op_int_non_compact_int64, "BINARY_OP_ADD_INT") + self.assert_specialized(binary_op_int_non_compact_int64, "BINARY_OP_SUBTRACT_INT") + self.assert_specialized(binary_op_int_non_compact_int64, "BINARY_OP_MULTIPLY_INT") + + def binary_op_int_too_large(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 1 << 200, 1 + c = a + b + self.assertEqual(c, (1 << 200) + 1) + c = a - b + self.assertEqual(c, (1 << 200) - 1) + c = a * b + self.assertEqual(c, 1 << 200) + + binary_op_int_too_large() + self.assert_no_opcode(binary_op_int_too_large, "BINARY_OP_ADD_INT") + self.assert_no_opcode(binary_op_int_too_large, "BINARY_OP_SUBTRACT_INT") + self.assert_no_opcode(binary_op_int_too_large, "BINARY_OP_MULTIPLY_INT") def binary_op_add_unicode(): for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index b463bb18b16056..eb0c8a611fcf0a 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -188,7 +188,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -198,7 +213,22 @@ { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { + PyLongObject *left_l = (PyLongObject *)left_o; + int ok = PyLong_CheckExact(left_o); + if (ok && !_PyLong_IsCompact(left_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(left_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((left_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)left_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -212,7 +242,6 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { @@ -222,6 +251,9 @@ } l = left; r = right; + if (PyStackRef_IsError(res)) { + JUMP_TO_LABEL(pop_2_error); + } } // _POP_TOP_INT { @@ -574,7 +606,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -584,7 +631,22 @@ { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { + PyLongObject *left_l = (PyLongObject *)left_o; + int ok = PyLong_CheckExact(left_o); + if (ok && !_PyLong_IsCompact(left_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(left_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((left_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)left_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -598,7 +660,6 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { @@ -608,6 +669,9 @@ } l = left; r = right; + if (PyStackRef_IsError(res)) { + JUMP_TO_LABEL(pop_2_error); + } } // _POP_TOP_INT { @@ -807,7 +871,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -981,7 +1060,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1070,7 +1164,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1162,7 +1271,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1324,7 +1448,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1334,7 +1473,22 @@ { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { + PyLongObject *left_l = (PyLongObject *)left_o; + int ok = PyLong_CheckExact(left_o); + if (ok && !_PyLong_IsCompact(left_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(left_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((left_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)left_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1348,7 +1502,6 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { @@ -1358,6 +1511,9 @@ } l = left; r = right; + if (PyStackRef_IsError(res)) { + JUMP_TO_LABEL(pop_2_error); + } } // _POP_TOP_INT { @@ -5151,7 +5307,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); @@ -5161,7 +5332,22 @@ { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { + PyLongObject *left_l = (PyLongObject *)left_o; + int ok = PyLong_CheckExact(left_o); + if (ok && !_PyLong_IsCompact(left_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(left_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((left_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)left_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); @@ -5173,13 +5359,13 @@ right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(_PyLong_IsCompact((PyLongObject *)left_o)); - assert(_PyLong_IsCompact((PyLongObject *)right_o)); STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && - _PyLong_DigitCount((PyLongObject *)right_o) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + int64_t ileft; + int64_t iright; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(_PyLong_TryAsInt64Exact((PyLongObject *)left_o, &ileft)); + assert(_PyLong_TryAsInt64Exact((PyLongObject *)right_o, &iright)); + stack_pointer = _PyFrame_GetStackPointer(frame); int sign_ish = COMPARISON_BIT(ileft, iright); l = left; r = right; @@ -12189,7 +12375,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); JUMP_TO_PREDICTED(STORE_SUBSCR); @@ -12426,7 +12627,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); JUMP_TO_PREDICTED(TO_BOOL); diff --git a/Objects/longobject.c b/Objects/longobject.c index 6e6011cb19aab5..3e395e9419b852 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -322,31 +322,65 @@ _PyLong_FromSTwoDigits(stwodigits x) return (PyLongObject*)_PyLong_FromLarge(x); } -/* Create a new medium int object from a medium int. - * Do not raise. Return NULL if not medium or can't allocate. */ -static inline _PyStackRef -medium_from_stwodigits(stwodigits x) +static inline bool +_Py_i64_add_overflow(int64_t a, int64_t b, int64_t *out) { - if (IS_SMALL_INT(x)) { - return PyStackRef_FromPyObjectBorrow(get_small_int((sdigit)x)); + if ((b > 0 && a > INT64_MAX - b) || (b < 0 && a < INT64_MIN - b)) { + return true; } - assert(x != 0); - if(!is_medium_int(x)) { - return PyStackRef_NULL; + *out = a + b; + return false; +} + +static inline bool +_Py_i64_sub_overflow(int64_t a, int64_t b, int64_t *out) +{ + if ((b > 0 && a < INT64_MIN + b) || (b < 0 && a > INT64_MAX + b)) { + return true; } - PyLongObject *v = (PyLongObject *)_Py_FREELIST_POP(PyLongObject, ints); - if (v == NULL) { - v = PyObject_Malloc(sizeof(PyLongObject)); - if (v == NULL) { - return PyStackRef_NULL; + *out = a - b; + return false; +} + +#if !defined(__SIZEOF_INT128__) +static inline uint64_t +_Py_uabs_i64(int64_t x) +{ + return x < 0 ? (uint64_t)(0 - (uint64_t)x) : (uint64_t)x; +} +#endif + +static inline bool +_Py_i64_mul_overflow(int64_t a, int64_t b, int64_t *out) +{ +#if defined(__SIZEOF_INT128__) + __int128 prod = (__int128)a * (__int128)b; + if (prod < INT64_MIN || prod > INT64_MAX) { + return true; + } + *out = (int64_t)prod; + return false; +#else + uint64_t ua = _Py_uabs_i64(a); + uint64_t ub = _Py_uabs_i64(b); + uint64_t limit = ((a < 0) ^ (b < 0)) ? (uint64_t)INT64_MAX + 1 : (uint64_t)INT64_MAX; + if (ua != 0 && ub > limit / ua) { + return true; + } + uint64_t uprod = ua * ub; + if ((a < 0) ^ (b < 0)) { + if (uprod == (uint64_t)INT64_MAX + 1) { + *out = INT64_MIN; + } + else { + *out = -(int64_t)uprod; } - _PyObject_Init((PyObject*)v, &PyLong_Type); - _PyLong_InitTag(v); } - digit abs_x = x < 0 ? (digit)(-x) : (digit)x; - _PyLong_SetSignAndDigitCount(v, x<0?-1:1, 1); - v->long_value.ob_digit[0] = abs_x; - return PyStackRef_FromPyObjectStealMortal((PyObject *)v); + else { + *out = (int64_t)uprod; + } + return false; +#endif } @@ -1324,7 +1358,6 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int flags) } if (_PyLong_IsCompact(v)) { - res = 0; cv.v = _PyLong_CompactValue(v); /* Most paths result in res = sizeof(compact value). Only the case * where 0 < n < sizeof(compact value) do we need to check and adjust @@ -2601,6 +2634,7 @@ long_from_binary_base(const char *start, const char *end, Py_ssize_t digits, int /* n <- the number of Python digits needed, = ceiling((digits * bits_per_char) / PyLong_SHIFT). */ + assert(bits_per_char > 0); if (digits > (PY_SSIZE_T_MAX - (PyLong_SHIFT - 1)) / bits_per_char) { PyErr_SetString(PyExc_ValueError, "int string too large to convert"); @@ -3866,9 +3900,32 @@ long_add(PyLongObject *a, PyLongObject *b) _PyStackRef _PyCompactLong_Add(PyLongObject *a, PyLongObject *b) { - assert(_PyLong_BothAreCompact(a, b)); - stwodigits v = medium_value(a) + medium_value(b); - return medium_from_stwodigits(v); + if (_PyLong_BothAreCompact(a, b)) { + stwodigits va = medium_value(a); + stwodigits vb = medium_value(b); + int64_t v64; + if (_Py_i64_add_overflow((int64_t)va, (int64_t)vb, &v64)) { + return PyStackRef_NULL; + } + PyLongObject *result = (PyLongObject *)PyLong_FromInt64(v64); + if (result == NULL) { + return PyStackRef_ERROR; + } + return PyStackRef_FromPyObjectSteal((PyObject *)result); + } + int64_t va, vb; + if (_PyLong_TryAsInt64Exact(a, &va) && _PyLong_TryAsInt64Exact(b, &vb)) { + int64_t v; + if (_Py_i64_add_overflow(va, vb, &v)) { + return PyStackRef_NULL; + } + PyLongObject *result = (PyLongObject *)PyLong_FromInt64(v); + if (result == NULL) { + return PyStackRef_ERROR; + } + return PyStackRef_FromPyObjectSteal((PyObject *)result); + } + return PyStackRef_NULL; } static PyObject * @@ -3911,9 +3968,32 @@ long_sub(PyLongObject *a, PyLongObject *b) _PyStackRef _PyCompactLong_Subtract(PyLongObject *a, PyLongObject *b) { - assert(_PyLong_BothAreCompact(a, b)); - stwodigits v = medium_value(a) - medium_value(b); - return medium_from_stwodigits(v); + if (_PyLong_BothAreCompact(a, b)) { + stwodigits va = medium_value(a); + stwodigits vb = medium_value(b); + int64_t v64; + if (_Py_i64_sub_overflow((int64_t)va, (int64_t)vb, &v64)) { + return PyStackRef_NULL; + } + PyLongObject *result = (PyLongObject *)PyLong_FromInt64(v64); + if (result == NULL) { + return PyStackRef_ERROR; + } + return PyStackRef_FromPyObjectSteal((PyObject *)result); + } + int64_t va, vb; + if (_PyLong_TryAsInt64Exact(a, &va) && _PyLong_TryAsInt64Exact(b, &vb)) { + int64_t v; + if (_Py_i64_sub_overflow(va, vb, &v)) { + return PyStackRef_NULL; + } + PyLongObject *result = (PyLongObject *)PyLong_FromInt64(v); + if (result == NULL) { + return PyStackRef_ERROR; + } + return PyStackRef_FromPyObjectSteal((PyObject *)result); + } + return PyStackRef_NULL; } static PyObject * @@ -4353,14 +4433,37 @@ long_mul(PyLongObject *a, PyLongObject *b) return z; } -/* This function returns NULL if the result is not compact, +/* This function returns NULL if the result does not fit in int64 range, * or if it fails to allocate, but never raises */ _PyStackRef _PyCompactLong_Multiply(PyLongObject *a, PyLongObject *b) { - assert(_PyLong_BothAreCompact(a, b)); - stwodigits v = medium_value(a) * medium_value(b); - return medium_from_stwodigits(v); + if (_PyLong_BothAreCompact(a, b)) { + stwodigits va = medium_value(a); + stwodigits vb = medium_value(b); + int64_t v64; + if (_Py_i64_mul_overflow((int64_t)va, (int64_t)vb, &v64)) { + return PyStackRef_NULL; + } + PyLongObject *result = (PyLongObject *)PyLong_FromInt64(v64); + if (result == NULL) { + return PyStackRef_ERROR; + } + return PyStackRef_FromPyObjectSteal((PyObject *)result); + } + int64_t va, vb; + if (_PyLong_TryAsInt64Exact(a, &va) && _PyLong_TryAsInt64Exact(b, &vb)) { + int64_t v; + if (_Py_i64_mul_overflow(va, vb, &v)) { + return PyStackRef_NULL; + } + PyLongObject *result = (PyLongObject *)PyLong_FromInt64(v); + if (result == NULL) { + return PyStackRef_ERROR; + } + return PyStackRef_FromPyObjectSteal((PyObject *)result); + } + return PyStackRef_NULL; } static PyObject * diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 300b7da753c2ba..df5cf801721077 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -634,24 +634,84 @@ dummy_func( op(_GUARD_NOS_INT, (left, unused -- left, unused)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - EXIT_IF(!_PyLong_CheckExactAndCompact(left_o)); + PyLongObject *left_l = (PyLongObject *)left_o; + int ok = PyLong_CheckExact(left_o); + if (ok && !_PyLong_IsCompact(left_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(left_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((left_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)left_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + EXIT_IF(!ok); } op(_GUARD_TOS_INT, (value -- value)) { PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - EXIT_IF(!_PyLong_CheckExactAndCompact(value_o)); + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + EXIT_IF(!ok); } op(_GUARD_NOS_OVERFLOWED, (left, unused -- left, unused)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); assert(Py_TYPE(left_o) == &PyLong_Type); - EXIT_IF(!_PyLong_IsCompact((PyLongObject *)left_o)); + PyLongObject *left_l = (PyLongObject *)left_o; + int ok = _PyLong_IsCompact(left_l); + if (!ok) { + Py_ssize_t ndigits = _PyLong_DigitCount(left_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((left_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)left_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + EXIT_IF(!ok); } op(_GUARD_TOS_OVERFLOWED, (value -- value)) { PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); assert(Py_TYPE(value_o) == &PyLong_Type); - EXIT_IF(!_PyLong_IsCompact((PyLongObject *)value_o)); + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = _PyLong_IsCompact(value_l); + if (!ok) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + EXIT_IF(!ok); } pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res, l, r)) { @@ -659,7 +719,6 @@ dummy_func( PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); @@ -667,6 +726,7 @@ dummy_func( l = left; r = right; INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsError(res)); } pure op(_BINARY_OP_ADD_INT, (left, right -- res, l, r)) { @@ -674,7 +734,6 @@ dummy_func( PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); @@ -682,6 +741,7 @@ dummy_func( l = left; r = right; INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsError(res)); } pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res, l, r)) { @@ -689,7 +749,6 @@ dummy_func( PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); @@ -697,6 +756,7 @@ dummy_func( l = left; r = right; INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsError(res)); } macro(BINARY_OP_MULTIPLY_INT) = @@ -718,6 +778,7 @@ dummy_func( l = left; r = right; INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsError(_int_inplace_res)); } tier2 op(_BINARY_OP_SUBTRACT_INT_INPLACE, (left, right -- res, l, r)) { @@ -727,6 +788,7 @@ dummy_func( l = left; r = right; INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsError(_int_inplace_res)); } tier2 op(_BINARY_OP_MULTIPLY_INT_INPLACE, (left, right -- res, l, r)) { @@ -736,6 +798,7 @@ dummy_func( l = left; r = right; INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsError(_int_inplace_res)); } tier2 op(_BINARY_OP_ADD_INT_INPLACE_RIGHT, (left, right -- res, l, r)) { @@ -745,6 +808,7 @@ dummy_func( l = left; r = right; INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsError(_int_inplace_res)); } tier2 op(_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, (left, right -- res, l, r)) { @@ -754,6 +818,7 @@ dummy_func( l = left; r = right; INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsError(_int_inplace_res)); } tier2 op(_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, (left, right -- res, l, r)) { @@ -763,6 +828,7 @@ dummy_func( l = left; r = right; INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsError(_int_inplace_res)); } op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { @@ -3314,13 +3380,11 @@ dummy_func( PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(_PyLong_IsCompact((PyLongObject *)left_o)); - assert(_PyLong_IsCompact((PyLongObject *)right_o)); STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && - _PyLong_DigitCount((PyLongObject *)right_o) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + int64_t ileft; + int64_t iright; + assert(_PyLong_TryAsInt64Exact((PyLongObject *)left_o, &ileft)); + assert(_PyLong_TryAsInt64Exact((PyLongObject *)right_o, &iright)); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); l = left; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index c61690e8bd7240..b4c94a2a53d05b 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -618,10 +618,17 @@ gen_try_set_executing(PyGenObject *gen) break; \ } \ assert(_PyObject_IsUniquelyReferenced(target_o)); \ + PyLongObject *left_long = \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(left); \ + PyLongObject *right_long = \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(right); \ + if (!_PyLong_BothAreCompact(left_long, right_long)) { \ + break; \ + } \ Py_ssize_t left_val = _PyLong_CompactValue( \ - (PyLongObject *)PyStackRef_AsPyObjectBorrow(left)); \ + left_long); \ Py_ssize_t right_val = _PyLong_CompactValue( \ - (PyLongObject *)PyStackRef_AsPyObjectBorrow(right)); \ + right_long); \ Py_ssize_t result = left_val OP right_val; \ if (!_PY_IS_SMALL_INT(result) \ && ((twodigits)((stwodigits)result) + PyLong_MASK \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 952860f01b8a68..d2c827ec9472b2 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3821,48 +3821,6 @@ break; } - case _GUARD_NOS_INT_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_INT_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - _PyStackRef _stack_item_0 = _tos_cache0; - left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_0; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - case _GUARD_NOS_INT_r22: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -3871,7 +3829,22 @@ _PyStackRef _stack_item_1 = _tos_cache1; left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { + PyLongObject *left_l = (PyLongObject *)left_o; + int ok = PyLong_CheckExact(left_o); + if (ok && !_PyLong_IsCompact(left_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(left_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((left_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)left_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = _stack_item_1; _tos_cache0 = left; @@ -3880,55 +3853,12 @@ } _tos_cache1 = _stack_item_1; _tos_cache0 = left; + _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _GUARD_NOS_INT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - left = _stack_item_1; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_INT_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - case _GUARD_TOS_INT_r11: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -3936,109 +3866,35 @@ _PyStackRef _stack_item_0 = _tos_cache0; value = _stack_item_0; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UOP_STAT_INC(uopcode, miss); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _GUARD_TOS_INT_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_INT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - value = _stack_item_2; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_OVERFLOWED_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - assert(Py_TYPE(left_o) == &PyLong_Type); - if (!_PyLong_IsCompact((PyLongObject *)left_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_OVERFLOWED_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - _PyStackRef _stack_item_0 = _tos_cache0; - left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - assert(Py_TYPE(left_o) == &PyLong_Type); - if (!_PyLong_IsCompact((PyLongObject *)left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_0; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - case _GUARD_NOS_OVERFLOWED_r22: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -4048,7 +3904,22 @@ left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); assert(Py_TYPE(left_o) == &PyLong_Type); - if (!_PyLong_IsCompact((PyLongObject *)left_o)) { + PyLongObject *left_l = (PyLongObject *)left_o; + int ok = _PyLong_IsCompact(left_l); + if (!ok) { + Py_ssize_t ndigits = _PyLong_DigitCount(left_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((left_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)left_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = _stack_item_1; _tos_cache0 = left; @@ -4057,57 +3928,12 @@ } _tos_cache1 = _stack_item_1; _tos_cache0 = left; + _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _GUARD_NOS_OVERFLOWED_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef left; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - left = _stack_item_1; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - assert(Py_TYPE(left_o) == &PyLong_Type); - if (!_PyLong_IsCompact((PyLongObject *)left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_OVERFLOWED_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - assert(Py_TYPE(value_o) == &PyLong_Type); - if (!_PyLong_IsCompact((PyLongObject *)value_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - case _GUARD_TOS_OVERFLOWED_r11: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -4116,67 +3942,35 @@ value = _stack_item_0; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); assert(Py_TYPE(value_o) == &PyLong_Type); - if (!_PyLong_IsCompact((PyLongObject *)value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = _PyLong_IsCompact(value_l); + if (!ok) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UOP_STAT_INC(uopcode, miss); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _GUARD_TOS_OVERFLOWED_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - assert(Py_TYPE(value_o) == &PyLong_Type); - if (!_PyLong_IsCompact((PyLongObject *)value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_TOS_OVERFLOWED_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - value = _stack_item_2; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - assert(Py_TYPE(value_o) == &PyLong_Type); - if (!_PyLong_IsCompact((PyLongObject *)value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - case _BINARY_OP_MULTIPLY_INT_r03: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -4191,7 +3985,6 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { @@ -4201,6 +3994,15 @@ } l = left; r = right; + if (PyStackRef_IsError(res)) { + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4226,7 +4028,6 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { @@ -4237,6 +4038,15 @@ } l = left; r = right; + if (PyStackRef_IsError(res)) { + stack_pointer[-1] = res; + stack_pointer[0] = l; + stack_pointer[1] = r; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4263,7 +4073,6 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { @@ -4275,6 +4084,10 @@ } l = left; r = right; + if (PyStackRef_IsError(res)) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4297,7 +4110,6 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { @@ -4307,6 +4119,15 @@ } l = left; r = right; + if (PyStackRef_IsError(res)) { + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4332,7 +4153,6 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { @@ -4343,6 +4163,15 @@ } l = left; r = right; + if (PyStackRef_IsError(res)) { + stack_pointer[-1] = res; + stack_pointer[0] = l; + stack_pointer[1] = r; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4369,7 +4198,6 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { @@ -4381,6 +4209,10 @@ } l = left; r = right; + if (PyStackRef_IsError(res)) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4403,7 +4235,6 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { @@ -4413,6 +4244,15 @@ } l = left; r = right; + if (PyStackRef_IsError(res)) { + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4438,7 +4278,6 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { @@ -4449,6 +4288,15 @@ } l = left; r = right; + if (PyStackRef_IsError(res)) { + stack_pointer[-1] = res; + stack_pointer[0] = l; + stack_pointer[1] = r; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4475,7 +4323,6 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { @@ -4487,6 +4334,10 @@ } l = left; r = right; + if (PyStackRef_IsError(res)) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4514,6 +4365,15 @@ res = _int_inplace_res; l = left; r = right; + if (PyStackRef_IsError(_int_inplace_res)) { + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4545,6 +4405,15 @@ res = _int_inplace_res; l = left; r = right; + if (PyStackRef_IsError(_int_inplace_res)) { + stack_pointer[-1] = res; + stack_pointer[0] = l; + stack_pointer[1] = r; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4578,6 +4447,10 @@ res = _int_inplace_res; l = left; r = right; + if (PyStackRef_IsError(_int_inplace_res)) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4605,6 +4478,15 @@ res = _int_inplace_res; l = left; r = right; + if (PyStackRef_IsError(_int_inplace_res)) { + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4636,6 +4518,15 @@ res = _int_inplace_res; l = left; r = right; + if (PyStackRef_IsError(_int_inplace_res)) { + stack_pointer[-1] = res; + stack_pointer[0] = l; + stack_pointer[1] = r; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4669,6 +4560,10 @@ res = _int_inplace_res; l = left; r = right; + if (PyStackRef_IsError(_int_inplace_res)) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4696,6 +4591,15 @@ res = _int_inplace_res; l = left; r = right; + if (PyStackRef_IsError(_int_inplace_res)) { + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4727,6 +4631,15 @@ res = _int_inplace_res; l = left; r = right; + if (PyStackRef_IsError(_int_inplace_res)) { + stack_pointer[-1] = res; + stack_pointer[0] = l; + stack_pointer[1] = r; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4760,6 +4673,10 @@ res = _int_inplace_res; l = left; r = right; + if (PyStackRef_IsError(_int_inplace_res)) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4787,6 +4704,15 @@ res = _int_inplace_res; l = left; r = right; + if (PyStackRef_IsError(_int_inplace_res)) { + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4818,6 +4744,15 @@ res = _int_inplace_res; l = left; r = right; + if (PyStackRef_IsError(_int_inplace_res)) { + stack_pointer[-1] = res; + stack_pointer[0] = l; + stack_pointer[1] = r; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4851,6 +4786,10 @@ res = _int_inplace_res; l = left; r = right; + if (PyStackRef_IsError(_int_inplace_res)) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4878,6 +4817,15 @@ res = _int_inplace_res; l = left; r = right; + if (PyStackRef_IsError(_int_inplace_res)) { + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4909,6 +4857,15 @@ res = _int_inplace_res; l = left; r = right; + if (PyStackRef_IsError(_int_inplace_res)) { + stack_pointer[-1] = res; + stack_pointer[0] = l; + stack_pointer[1] = r; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4942,6 +4899,10 @@ res = _int_inplace_res; l = left; r = right; + if (PyStackRef_IsError(_int_inplace_res)) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -4969,6 +4930,15 @@ res = _int_inplace_res; l = left; r = right; + if (PyStackRef_IsError(_int_inplace_res)) { + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -5000,6 +4970,15 @@ res = _int_inplace_res; l = left; r = right; + if (PyStackRef_IsError(_int_inplace_res)) { + stack_pointer[-1] = res; + stack_pointer[0] = l; + stack_pointer[1] = r; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -5033,6 +5012,10 @@ res = _int_inplace_res; l = left; r = right; + if (PyStackRef_IsError(_int_inplace_res)) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } _tos_cache2 = r; _tos_cache1 = l; _tos_cache0 = res; @@ -13259,13 +13242,17 @@ left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(_PyLong_IsCompact((PyLongObject *)left_o)); - assert(_PyLong_IsCompact((PyLongObject *)right_o)); STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && - _PyLong_DigitCount((PyLongObject *)right_o) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + int64_t ileft; + int64_t iright; + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(_PyLong_TryAsInt64Exact((PyLongObject *)left_o, &ileft)); + assert(_PyLong_TryAsInt64Exact((PyLongObject *)right_o, &iright)); + stack_pointer = _PyFrame_GetStackPointer(frame); int sign_ish = COMPARISON_BIT(ileft, iright); l = left; r = right; @@ -13274,6 +13261,8 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 83051cf41cc043..474bd91ba98f69 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -188,7 +188,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -198,7 +213,22 @@ { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { + PyLongObject *left_l = (PyLongObject *)left_o; + int ok = PyLong_CheckExact(left_o); + if (ok && !_PyLong_IsCompact(left_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(left_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((left_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)left_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -212,7 +242,6 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { @@ -222,6 +251,9 @@ } l = left; r = right; + if (PyStackRef_IsError(res)) { + JUMP_TO_LABEL(pop_2_error); + } } // _POP_TOP_INT { @@ -574,7 +606,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -584,7 +631,22 @@ { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { + PyLongObject *left_l = (PyLongObject *)left_o; + int ok = PyLong_CheckExact(left_o); + if (ok && !_PyLong_IsCompact(left_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(left_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((left_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)left_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -598,7 +660,6 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { @@ -608,6 +669,9 @@ } l = left; r = right; + if (PyStackRef_IsError(res)) { + JUMP_TO_LABEL(pop_2_error); + } } // _POP_TOP_INT { @@ -807,7 +871,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -981,7 +1060,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1070,7 +1164,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1162,7 +1271,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1324,7 +1448,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1334,7 +1473,22 @@ { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { + PyLongObject *left_l = (PyLongObject *)left_o; + int ok = PyLong_CheckExact(left_o); + if (ok && !_PyLong_IsCompact(left_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(left_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((left_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)left_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1348,7 +1502,6 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { @@ -1358,6 +1511,9 @@ } l = left; r = right; + if (PyStackRef_IsError(res)) { + JUMP_TO_LABEL(pop_2_error); + } } // _POP_TOP_INT { @@ -5151,7 +5307,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); @@ -5161,7 +5332,22 @@ { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { + PyLongObject *left_l = (PyLongObject *)left_o; + int ok = PyLong_CheckExact(left_o); + if (ok && !_PyLong_IsCompact(left_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(left_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((left_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)left_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); @@ -5173,13 +5359,13 @@ right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(_PyLong_IsCompact((PyLongObject *)left_o)); - assert(_PyLong_IsCompact((PyLongObject *)right_o)); STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && - _PyLong_DigitCount((PyLongObject *)right_o) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + int64_t ileft; + int64_t iright; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(_PyLong_TryAsInt64Exact((PyLongObject *)left_o, &ileft)); + assert(_PyLong_TryAsInt64Exact((PyLongObject *)right_o, &iright)); + stack_pointer = _PyFrame_GetStackPointer(frame); int sign_ish = COMPARISON_BIT(ileft, iright); l = left; r = right; @@ -12186,7 +12372,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); JUMP_TO_PREDICTED(STORE_SUBSCR); @@ -12423,7 +12624,22 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + PyLongObject *value_l = (PyLongObject *)value_o; + int ok = PyLong_CheckExact(value_o); + if (ok && !_PyLong_IsCompact(value_l)) { + Py_ssize_t ndigits = _PyLong_DigitCount(value_l); + ok = (ndigits <= _PY_LONG_MAX_DIGITS_FOR_INT64); + if (ok && ndigits == _PY_LONG_MAX_DIGITS_FOR_INT64) { + unsigned int shift = PyLong_SHIFT * (unsigned int)(ndigits - 1); + uint64_t max_pos_top = (uint64_t)INT64_MAX >> shift; + uint64_t max_neg_top = ((uint64_t)INT64_MAX + 1) >> shift; + uint64_t max_top = ((value_l->long_value.lv_tag & SIGN_MASK) == SIGN_NEGATIVE) + ? max_neg_top + : max_pos_top; + ok = (uint64_t)value_l->long_value.ob_digit[ndigits - 1] <= max_top; + } + } + if (!ok) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); JUMP_TO_PREDICTED(TO_BOOL); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index d52ebb9804197d..e354242855e862 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -622,7 +622,6 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res_stackref = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res_stackref )) { @@ -631,6 +630,9 @@ } l_stackref = left; r_stackref = right; + if (PyStackRef_IsError(res_stackref )) { + goto error; + } /* End of uop copied from bytecodes for constant evaluation */ (void)l_stackref; (void)r_stackref; @@ -693,7 +695,6 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res_stackref = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res_stackref )) { @@ -702,6 +703,9 @@ } l_stackref = left; r_stackref = right; + if (PyStackRef_IsError(res_stackref )) { + goto error; + } /* End of uop copied from bytecodes for constant evaluation */ (void)l_stackref; (void)r_stackref; @@ -764,7 +768,6 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res_stackref = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res_stackref )) { @@ -773,6 +776,9 @@ } l_stackref = left; r_stackref = right; + if (PyStackRef_IsError(res_stackref )) { + goto error; + } /* End of uop copied from bytecodes for constant evaluation */ (void)l_stackref; (void)r_stackref; @@ -3067,13 +3073,11 @@ /* Start of uop copied from bytecodes for constant evaluation */ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(_PyLong_IsCompact((PyLongObject *)left_o)); - assert(_PyLong_IsCompact((PyLongObject *)right_o)); STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && - _PyLong_DigitCount((PyLongObject *)right_o) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + int64_t ileft; + int64_t iright; + assert(_PyLong_TryAsInt64Exact((PyLongObject *)left_o, &ileft)); + assert(_PyLong_TryAsInt64Exact((PyLongObject *)right_o, &iright)); int sign_ish = COMPARISON_BIT(ileft, iright); l_stackref = left; r_stackref = right; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 79f81482d247e3..70b45f4e4962d0 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -272,7 +272,7 @@ _Py_uop_sym_is_safe_const(JitOptContext *ctx, JitOptRef sym) if (const_val == NULL) { return false; } - if (_PyLong_CheckExactAndCompact(const_val)) { + if (_PyLong_CheckExactAndMightFitInt64(const_val)) { return true; } PyTypeObject *typ = Py_TYPE(const_val); @@ -566,7 +566,7 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) make_const(sym, const_val); return; case JIT_SYM_COMPACT_INT: - if (_PyLong_CheckExactAndCompact(const_val)) { + if (_PyLong_CheckExactAndMightFitInt64(const_val)) { make_const(sym, const_val); } else { @@ -970,7 +970,7 @@ _Py_uop_sym_is_compact_int(JitOptRef ref) { JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { - return (bool)_PyLong_CheckExactAndCompact(sym->value.value); + return (bool)_PyLong_CheckExactAndMightFitInt64(sym->value.value); } return sym->tag == JIT_SYM_COMPACT_INT; } @@ -1008,7 +1008,7 @@ _Py_uop_sym_set_compact_int(JitOptContext *ctx, JitOptRef ref) } return; case JIT_SYM_KNOWN_VALUE_TAG: - if (!_PyLong_CheckExactAndCompact(sym->value.value)) { + if (!_PyLong_CheckExactAndMightFitInt64(sym->value.value)) { Py_CLEAR(sym->value.value); sym_set_bottom(ctx, sym); } @@ -1954,7 +1954,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == true, "value is not constant"); TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == one_obj, "value is not 1"); - val_big = PyNumber_Lshift(_PyLong_GetOne(), PyLong_FromLong(66)); + val_big = PyNumber_Lshift(_PyLong_GetOne(), PyLong_FromLong(200)); if (val_big == NULL) { goto fail; } @@ -1963,7 +1963,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) JitOptRef ref_big = _Py_uop_sym_new_const(ctx, val_big); JitOptRef ref_int = _Py_uop_sym_new_compact_int(ctx); TEST_PREDICATE(_Py_uop_sym_is_compact_int(ref_42), "42 is not a compact int"); - TEST_PREDICATE(!_Py_uop_sym_is_compact_int(ref_big), "(1 << 66) is a compact int"); + TEST_PREDICATE(!_Py_uop_sym_is_compact_int(ref_big), "(1 << 200) is a compact int"); TEST_PREDICATE(_Py_uop_sym_is_compact_int(ref_int), "compact int is not a compact int"); TEST_PREDICATE(_Py_uop_sym_matches_type(ref_int, &PyLong_Type), "compact int is not an int"); diff --git a/Python/specialize.c b/Python/specialize.c index 2ff0a9d0072cec..276a2512d7ba4c 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2350,7 +2350,9 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in specialize(instr, BINARY_OP_ADD_UNICODE); return; } - if (_PyLong_CheckExactAndCompact(lhs) && _PyLong_CheckExactAndCompact(rhs)) { + if (_PyLong_CheckExactAndMightFitInt64(lhs) && + _PyLong_CheckExactAndMightFitInt64(rhs)) + { specialize(instr, BINARY_OP_ADD_INT); return; } @@ -2364,7 +2366,9 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { break; } - if (_PyLong_CheckExactAndCompact(lhs) && _PyLong_CheckExactAndCompact(rhs)) { + if (_PyLong_CheckExactAndMightFitInt64(lhs) && + _PyLong_CheckExactAndMightFitInt64(rhs)) + { specialize(instr, BINARY_OP_MULTIPLY_INT); return; } @@ -2378,7 +2382,9 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { break; } - if (_PyLong_CheckExactAndCompact(lhs) && _PyLong_CheckExactAndCompact(rhs)) { + if (_PyLong_CheckExactAndMightFitInt64(lhs) && + _PyLong_CheckExactAndMightFitInt64(rhs)) + { specialize(instr, BINARY_OP_SUBTRACT_INT); return; } diff --git a/Tools/scripts/jit_int_benchmark.py b/Tools/scripts/jit_int_benchmark.py new file mode 100644 index 00000000000000..d049171dcf1c0d --- /dev/null +++ b/Tools/scripts/jit_int_benchmark.py @@ -0,0 +1,139 @@ +""" +Benchmark for JIT int fast path optimization. + +Measures performance of integer arithmetic in hot loops +where intermediate results overflow the 30-bit compact range. + +Usage: + python Tools/scripts/jit_int_benchmark.py + python -X jit Tools/scripts/jit_int_benchmark.py + PYTHON_JIT_STRESS=1 python -X jit Tools/scripts/jit_int_benchmark.py + +For statistically rigorous comparison: + python -m pyperf timeit "..." # microbenchmark + pyperformance run -b # application benchmarks +""" + +import sys +import time +import statistics + + +# Benchmark configurations +N = 5_000_000 +WARMUP = 3 +RUNS = 7 + + +def bench_intermediate_overflow(): + """a + b exceeds 30-bit compact range, but (a+b)-c fits back in. + Tests the case where JIT would exit on the intermediate result.""" + a = (1 << 30) - 1 + b = (1 << 30) - 1000 + c = 1 << 30 + total = 0 + for i in range(N): + total += a + b - c + return total + + +def bench_double_add(): + """Two adds in a row: (a+b)+c where intermediate is non-compact.""" + a = (1 << 30) - 1 + b = (1 << 30) - 1000 + c = -500 + total = 0 + for i in range(N): + total += a + b + c + return total + + +def bench_accumulate(): + """Values slowly grow through the compact boundary. + Tests the case where a non-compact value is accumulated.""" + total = 0 + for i in range(N): + total += 1000000 + return total + + +def bench_always_large(): + """Both inputs are compact, but the result always exceeds compact range. + Tests the result widening optimization.""" + total = 0 + a = 1 << 29 + b = 1 << 29 + for i in range(N): + total += a + b + total &= (1 << 62) - 1 + return total + + +def bench_mixed(): + """Mix of small and medium ints alternating. + Tests ability to stay in JIT trace across varying int sizes.""" + total = 0 + for i in range(N): + if i & 1: + total += (1 << 30) + (i & 0xFF) + else: + total += i & 0xFFF + total &= (1 << 61) - 1 + return total + + +def bench_chain(): + """Chained operations: a + b + c + d. + Tests consecutive int ops without boxing between them.""" + total = 0 + for i in range(N // 4): + total += i + i + i + i + return total + + +def run_benchmark(name, fn, runs=RUNS, warmup=WARMUP): + # Warmup + for _ in range(warmup): + fn() + + # Timed runs + times = [] + last_result = None + for _ in range(runs): + start = time.perf_counter() + last_result = fn() + elapsed = time.perf_counter() - start + times.append(elapsed) + + mean = statistics.mean(times) + stdev = statistics.stdev(times) if len(times) > 1 else 0 + ns_per = mean / N * 1e9 + print( + f" {name:30s} {mean * 1000:7.2f} ms {ns_per:5.1f} ns/iter" + f" ±{stdev / mean * 100:4.1f}% (verify={str(last_result)[:12]})" + ) + + +if __name__ == "__main__": + jit = "jit" in sys._xoptions or "PYTHON_JIT" in __import__("os").environ + print( + f"Python {sys.version.split()[0]} ({sys.platform})" + f" JIT={'on' if jit else 'off'} PYTHON_JIT_STRESS=" + f"{__import__('os').environ.get('PYTHON_JIT_STRESS', '0')}" + ) + print() + + benchmarks = [ + ("intermediate_overflow", bench_intermediate_overflow), + ("double_add", bench_double_add), + ("always_large", bench_always_large), + ("accumulate", bench_accumulate), + ("mixed", bench_mixed), + ("chain", bench_chain), + ] + + for name, fn in benchmarks: + run_benchmark(name, fn) + + print() + print("Done.") diff --git a/Tools/scripts/jit_int_benchmark_pyperf.py b/Tools/scripts/jit_int_benchmark_pyperf.py new file mode 100644 index 00000000000000..779be0f77e0310 --- /dev/null +++ b/Tools/scripts/jit_int_benchmark_pyperf.py @@ -0,0 +1,282 @@ +"""pyperf microbenchmark for JIT int fast path. + +Measures int arithmetic performance in hot JIT loops where +intermediate results overflow the 30-bit compact range. + +Usage: + python Tools/scripts/jit_int_benchmark_pyperf.py + python Tools/scripts/jit_int_benchmark_pyperf.py -o result.json --rigorous +""" + +import pyperf +import time + + +def bench_intermediate_overflow(loops): + """a + b exceeds 30-bit range, (a+b)-c fits back in.""" + a = (1 << 30) - 1 + b = (1 << 30) - 1000 + c = 1 << 30 + t0 = time.perf_counter() + for _ in range(loops): + # inner loop: 50 ops to amplify the signal + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + x = a + b - c + return time.perf_counter() - t0 + + +def bench_double_add(loops): + """(a+b)+c where a+b is non-compact but a,b,c are compact.""" + a = (1 << 30) - 1 + b = (1 << 30) - 1000 + c = -500 + t0 = time.perf_counter() + for _ in range(loops): + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + x = a + b + c + return time.perf_counter() - t0 + + +def bench_accumulate(loops): + """Values grow through compact boundary (2^30).""" + total = 0 + t0 = time.perf_counter() + for _ in range(loops): + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + total += 1000000 + return time.perf_counter() - t0 + + +def bench_always_large(loops): + """Compact inputs, non-compact result.""" + total = 0 + va = 1 << 29 + vb = 1 << 29 + t0 = time.perf_counter() + for _ in range(loops): + total += va + vb + total &= (1 << 62) - 1 + total += va + vb + total &= (1 << 62) - 1 + total += va + vb + total &= (1 << 62) - 1 + total += va + vb + total &= (1 << 62) - 1 + total += va + vb + total &= (1 << 62) - 1 + total += va + vb + total &= (1 << 62) - 1 + total += va + vb + total &= (1 << 62) - 1 + total += va + vb + total &= (1 << 62) - 1 + total += va + vb + total &= (1 << 62) - 1 + total += va + vb + total &= (1 << 62) - 1 + return time.perf_counter() - t0 + + +def bench_mixed(loops): + """Alternating small/medium ints.""" + total = 0 + t0 = time.perf_counter() + for _ in range(loops): + total += (1 << 30) + 1 + total += 100 + total += (1 << 30) + 2 + total += 200 + total += (1 << 30) + 3 + total += 300 + total += (1 << 30) + 4 + total += 400 + total += (1 << 30) + 5 + total += 500 + return time.perf_counter() - t0 + + +def bench_small(loops): + """All small ints (baseline).""" + total = 0 + t0 = time.perf_counter() + for _ in range(loops): + total += 1 + total += 2 + total += 3 + total += 4 + total += 5 + total += 6 + total += 7 + total += 8 + total += 9 + total += 10 + total += 11 + total += 12 + total += 13 + total += 14 + total += 15 + total += 16 + total += 17 + total += 18 + total += 19 + total += 20 + return time.perf_counter() - t0 + + +BENCHMARKS = [ + ("jit_int_small", bench_small), + ("jit_int_intermediate_overflow", bench_intermediate_overflow), + ("jit_int_double_add", bench_double_add), + ("jit_int_accumulate", bench_accumulate), + ("jit_int_always_large", bench_always_large), + ("jit_int_mixed", bench_mixed), +] + +if __name__ == "__main__": + runner = pyperf.Runner() + for name, fn in BENCHMARKS: + runner.bench_time_func(name, fn)