211 lines
17 KiB
Python
211 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Check C89 declarations and logic order in unicode equality and dict checks."""
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
if len(sys.argv) != 3:
|
|
raise SystemExit("usage: vc6-dict-consistency-c89.test.py eq.h dictobject.c")
|
|
eq_source = Path(sys.argv[1]).read_text()
|
|
dict_source = Path(sys.argv[2]).read_text()
|
|
|
|
def body_for(source, signature):
|
|
start = source.index(signature)
|
|
brace = source.index("{", start)
|
|
depth = 0
|
|
for pos in range(brace, len(source)):
|
|
if source[pos] == "{":
|
|
depth += 1
|
|
elif source[pos] == "}":
|
|
depth -= 1
|
|
if depth == 0:
|
|
return source[brace + 1:pos]
|
|
raise SystemExit("unterminated function: " + signature)
|
|
|
|
body = body_for(eq_source, "unicode_eq(PyObject *aa, PyObject *bb)")
|
|
for declaration in ("PyUnicodeObject *a;", "PyUnicodeObject *b;"):
|
|
if declaration not in body or body.index(declaration) > body.index("assert(PyUnicode_Check(aa));"):
|
|
raise SystemExit("unicode_eq local missing or late: " + declaration)
|
|
if "PyUnicodeObject *a =" in body or "PyUnicodeObject *b =" in body:
|
|
raise SystemExit("unicode_eq object casts must remain after readiness assertions")
|
|
order = ("assert(PyUnicode_Check(aa));", "assert(PyUnicode_Check(bb));", "assert(PyUnicode_IS_READY(aa));", "assert(PyUnicode_IS_READY(bb));", "a = (PyUnicodeObject *)aa;", "b = (PyUnicodeObject *)bb;", "if (PyUnicode_GET_LENGTH(a) !=", "return memcmp(")
|
|
if [body.index(item) for item in order] != sorted(body.index(item) for item in order):
|
|
raise SystemExit("unicode_eq validation/cast/comparison order changed")
|
|
|
|
body = body_for(dict_source, "_PyDict_CheckConsistency(PyObject *op, int check_content)")
|
|
declarations = ("PyDictObject *mp;", "PyDictKeysObject *keys;", "int splitted;", "Py_ssize_t usable;", "Py_ssize_t i;", "Py_ssize_t ix;", "PyDictKeyEntry *key_entries;", "PyDictKeyEntry *dict_entry;", "PyDictUnicodeEntry *unicode_entries;", "PyDictUnicodeEntry *unicode_entry;", "PyObject *key;", "Py_hash_t hash;", "int duplicate_check;", "int index;")
|
|
for declaration in declarations:
|
|
if declaration not in body or body.index(declaration) > body.index("assert(op != NULL);"):
|
|
raise SystemExit("_PyDict_CheckConsistency local missing or late: " + declaration)
|
|
for declaration in declarations:
|
|
prefix = declaration.split(";")[0]
|
|
if prefix + " =" in body:
|
|
raise SystemExit("_PyDict_CheckConsistency initializer must remain at execution point: " + declaration)
|
|
if "for (Py_ssize_t i" in body:
|
|
raise SystemExit("_PyDict_CheckConsistency loop index must be declared before the loops")
|
|
order = ("assert(op != NULL);", "CHECK(PyDict_Check(op));", "mp = (PyDictObject *)op;", "keys = mp->ma_keys;", "splitted = _PyDict_HasSplitTable(mp);", "usable = USABLE_FRACTION", "if (check_content)", "for (i=0; i < DK_SIZE(keys); i++)", "ix = dictkeys_get_index", "key_entries = DK_ENTRIES(keys);", "for (i=0; i < usable; i++)", "dict_entry = &key_entries[i];", "unicode_entries = DK_UNICODE_ENTRIES(keys);", "unicode_entry = &unicode_entries[i];", "duplicate_check = 0;", "for (i=0; i < mp->ma_used; i++)", "index = get_index_from_order(mp, i);", "return 1;")
|
|
if [body.index(item) for item in order] != sorted(body.index(item) for item in order):
|
|
raise SystemExit("_PyDict_CheckConsistency validation/table traversal order changed")
|
|
body = body_for(dict_source, "new_keys_object(uint8_t log2_size, bool unicode)")
|
|
if "struct _Py_dict_state *state;" not in body or body.index("struct _Py_dict_state *state;") > body.index("assert(log2_size >= PyDict_LOG_MINSIZE);"):
|
|
raise SystemExit("new_keys_object freelist state must be declared at function entry")
|
|
if "struct _Py_dict_state *state =" in body:
|
|
raise SystemExit("new_keys_object state lookup must remain after size calculation")
|
|
order = ("assert(log2_size >= PyDict_LOG_MINSIZE);", "usable = USABLE_FRACTION", "if (log2_size < 8)", "state = get_dict_state();", "assert(state->keys_numfree != -1);", "if (log2_size == PyDict_LOG_MINSIZE", "dk = PyObject_Malloc", "dk->dk_refcnt = 1;")
|
|
if [body.index(item) for item in order] != sorted(body.index(item) for item in order):
|
|
raise SystemExit("new_keys_object size/freelist/allocation initialization order changed")
|
|
body = body_for(dict_source, "static void\nfree_keys_object(PyDictKeysObject *keys)")
|
|
for declaration in ("PyDictUnicodeEntry *unicode_entries;", "PyDictKeyEntry *key_entries;", "Py_ssize_t i, n;", "struct _Py_dict_state *state;"):
|
|
if declaration not in body or body.index(declaration) > body.index("assert(keys != Py_EMPTY_KEYS);"):
|
|
raise SystemExit("free_keys_object local missing or late: " + declaration)
|
|
if "struct _Py_dict_state *state =" in body or "for (Py_ssize_t i" in body:
|
|
raise SystemExit("free_keys_object assignments and indices must remain C89-compatible")
|
|
order = ("assert(keys != Py_EMPTY_KEYS);", "if (DK_IS_UNICODE(keys))", "unicode_entries = DK_UNICODE_ENTRIES(keys);", "for (i = 0, n = keys->dk_nentries;", "else {", "key_entries = DK_ENTRIES(keys);", "for (i = 0, n = keys->dk_nentries;", "state = get_dict_state();", "assert(state->keys_numfree != -1);", "if (DK_LOG_SIZE(keys) ==", "PyObject_Free(keys);")
|
|
positions = []
|
|
for item in order:
|
|
positions.append(body.index(item, positions[-1] + 1 if positions else 0))
|
|
if positions != sorted(positions):
|
|
raise SystemExit("free_keys_object entry traversal/freelist/free order changed")
|
|
|
|
body = body_for(dict_source, "new_values(Py_ssize_t size)")
|
|
for declaration in ("size_t prefix_size;", "size_t n;", "uint8_t *mem;"):
|
|
if declaration not in body or body.index(declaration) > body.index("assert(size > 0);"):
|
|
raise SystemExit("new_values local missing or late: " + declaration)
|
|
if "size_t prefix_size =" in body or "size_t n =" in body or "uint8_t *mem =" in body:
|
|
raise SystemExit("new_values arithmetic/allocation must remain after its assertions")
|
|
order = ("assert(size > 0);", "prefix_size = _Py_SIZE_ROUND_UP", "assert(prefix_size < 256);", "n = prefix_size +", "mem = PyMem_Malloc(n);", "if (mem == NULL)", "mem[prefix_size-1] =", "return (PyDictValues*)")
|
|
if [body.index(item) for item in order] != sorted(body.index(item) for item in order):
|
|
raise SystemExit("new_values size/assert/allocation/initialization order changed")
|
|
|
|
body = body_for(dict_source, "new_dict(PyDictKeysObject *keys,")
|
|
if "struct _Py_dict_state *state;" not in body or body.index("struct _Py_dict_state *state;") > body.index("assert(keys != NULL);"):
|
|
raise SystemExit("new_dict freelist state must be declared at function entry")
|
|
if "struct _Py_dict_state *state =" in body:
|
|
raise SystemExit("new_dict state lookup must remain at its execution point")
|
|
order = ("assert(keys != NULL);", "state = get_dict_state();", "assert(state->numfree != -1);", "if (state->numfree)", "mp = state->free_list", "mp = PyObject_GC_New", "mp->ma_keys = keys;", "ASSERT_CONSISTENT(mp);", "return (PyObject *)mp;")
|
|
if [body.index(item) for item in order] != sorted(body.index(item) for item in order):
|
|
raise SystemExit("new_dict freelist/allocation/object initialization order changed")
|
|
body = body_for(dict_source, "clone_combined_dict_keys(PyDictObject *orig)")
|
|
declarations = ("size_t keys_size;", "PyDictKeysObject *keys;", "PyObject **pkey;", "PyObject **pvalue;", "size_t offs;", "Py_ssize_t n;", "Py_ssize_t i;", "PyDictUnicodeEntry *unicode_ep0;", "PyDictKeyEntry *key_ep0;")
|
|
for declaration in declarations:
|
|
if declaration not in body or body.index(declaration) > body.index("assert(PyDict_Check(orig));"):
|
|
raise SystemExit("clone_combined_dict_keys local missing or late: " + declaration)
|
|
for initializer in ("size_t keys_size =", "PyDictKeysObject *keys =", "PyObject **pkey,", "size_t offs;\n if", "Py_ssize_t n =", "for (Py_ssize_t i ="):
|
|
if initializer in body:
|
|
raise SystemExit("clone_combined_dict_keys local initializer/declaration remains late: " + initializer)
|
|
order = ("assert(PyDict_Check(orig));", "keys_size = _PyDict_KeysSize", "keys = PyObject_Malloc(keys_size);", "memcpy(keys, orig->ma_keys, keys_size);", "if (DK_IS_UNICODE(orig->ma_keys))", "unicode_ep0 = DK_UNICODE_ENTRIES(keys);", "pkey = &unicode_ep0->me_key;", "else {", "key_ep0 = DK_ENTRIES(keys);", "pkey = &key_ep0->me_key;", "n = keys->dk_nentries;", "for (i = 0; i < n; i++)", "Py_INCREF(value);", "pvalue += offs;", "_Py_RefTotal++;", "return keys;")
|
|
positions = []
|
|
for item in order:
|
|
positions.append(body.index(item, positions[-1] + 1 if positions else 0))
|
|
if positions != sorted(positions):
|
|
raise SystemExit("clone_combined_dict_keys assertions/copy/reference transfer order changed")
|
|
body = body_for(dict_source, "unicodekeys_lookup_generic(PyDictObject *mp,")
|
|
if "int cmp;" not in body or body.index("int cmp;") > body.index("for (;;)"):
|
|
raise SystemExit("unicodekeys_lookup_generic comparison result must be declared before probing")
|
|
if "int cmp =" in body:
|
|
raise SystemExit("unicodekeys_lookup_generic compare must remain after retaining the key")
|
|
order = ("if (unicode_get_hash(ep->me_key) == hash)", "PyObject *startkey = ep->me_key;", "Py_INCREF(startkey);", "cmp = PyObject_RichCompareBool(startkey, key, Py_EQ);", "Py_DECREF(startkey);", "if (cmp < 0)", "if (dk == mp->ma_keys && ep->me_key == startkey)")
|
|
if [body.index(item) for item in order] != sorted(body.index(item) for item in order):
|
|
raise SystemExit("unicodekeys_lookup_generic retain/compare/decref/restart order changed")
|
|
body = body_for(dict_source, "dictkeys_generic_lookup(PyDictObject *mp,")
|
|
if "int cmp;" not in body or body.index("int cmp;") > body.index("for (;;)"):
|
|
raise SystemExit("dictkeys_generic_lookup comparison result must be declared before probing")
|
|
if "int cmp =" in body:
|
|
raise SystemExit("dictkeys_generic_lookup compare must remain after retaining the key")
|
|
order = ("if (ep->me_hash == hash)", "PyObject *startkey = ep->me_key;", "Py_INCREF(startkey);", "cmp = PyObject_RichCompareBool(startkey, key, Py_EQ);", "Py_DECREF(startkey);", "if (cmp < 0)", "if (dk == mp->ma_keys && ep->me_key == startkey)")
|
|
if [body.index(item) for item in order] != sorted(body.index(item) for item in order):
|
|
raise SystemExit("dictkeys_generic_lookup retain/compare/decref/restart order changed")
|
|
body = body_for(dict_source, "_PyDictKeys_StringLookup(PyDictKeysObject* dk,")
|
|
for declaration in ("DictKeysKind kind;", "Py_hash_t hash;"):
|
|
if declaration not in body or body.index(declaration) > body.index("kind = dk->dk_kind;"):
|
|
raise SystemExit("_PyDictKeys_StringLookup local missing or late: " + declaration)
|
|
if "Py_hash_t hash =" in body:
|
|
raise SystemExit("_PyDictKeys_StringLookup hash calculation must remain after key validation")
|
|
order = ("kind = dk->dk_kind;", "if (!PyUnicode_CheckExact(key)", "hash = unicode_get_hash(key);", "if (hash == -1)", "return unicodekeys_lookup_unicode(dk, key, hash);")
|
|
if [body.index(item) for item in order] != sorted(body.index(item) for item in order):
|
|
raise SystemExit("_PyDictKeys_StringLookup validation/hash/fallback order changed")
|
|
|
|
body = body_for(dict_source, "find_empty_slot(PyDictKeysObject *keys,")
|
|
for declaration in ("size_t mask;", "size_t i;", "size_t perturb;", "Py_ssize_t ix;"):
|
|
if declaration not in body or body.index(declaration) > body.index("assert(keys != NULL);"):
|
|
raise SystemExit("find_empty_slot local missing or late: " + declaration)
|
|
if "for (size_t perturb" in body:
|
|
raise SystemExit("find_empty_slot perturb index must be declared before probing")
|
|
order = ("assert(keys != NULL);", "mask = DK_MASK(keys);", "i = hash & mask;", "ix = dictkeys_get_index(keys, i);", "for (perturb = hash;", "return i;")
|
|
if [body.index(item) for item in order] != sorted(body.index(item) for item in order):
|
|
raise SystemExit("find_empty_slot validation/probe order changed")
|
|
|
|
body = body_for(dict_source, "insert_into_dictkeys(PyDictKeysObject *keys,")
|
|
for declaration in ("Py_hash_t hash;", "Py_ssize_t ix;", "Py_ssize_t hashpos;", "PyDictUnicodeEntry *ep;"):
|
|
if declaration not in body or body.index(declaration) > body.index("assert(PyUnicode_CheckExact(name));"):
|
|
raise SystemExit("insert_into_dictkeys local missing or late: " + declaration)
|
|
if any(token in body for token in ("Py_hash_t hash =", "Py_ssize_t ix =", "Py_ssize_t hashpos =", "PyDictUnicodeEntry *ep =")):
|
|
raise SystemExit("insert_into_dictkeys insertion state must be assigned at its execution point")
|
|
order = ("assert(PyUnicode_CheckExact(name));", "hash = unicode_get_hash(name);", "ix = unicodekeys_lookup_unicode", "Py_INCREF(name);", "hashpos = find_empty_slot(keys, hash);", "dictkeys_set_index(keys, hashpos, ix);", "ep->me_key = name;")
|
|
if [body.index(item) for item in order] != sorted(body.index(item) for item in order):
|
|
raise SystemExit("insert_into_dictkeys hash/refcount/table-update order changed")
|
|
|
|
body = body_for(dict_source, "insert_to_emptydict(PyDictObject *mp,")
|
|
for declaration in ("int unicode;", "PyDictKeysObject *newkeys;", "size_t hashpos;", "PyDictUnicodeEntry *unicode_ep;", "PyDictKeyEntry *key_ep;"):
|
|
if declaration not in body or body.index(declaration) > body.index("assert(mp->ma_keys == Py_EMPTY_KEYS);"):
|
|
raise SystemExit("insert_to_emptydict local missing or late: " + declaration)
|
|
order = ("assert(mp->ma_keys == Py_EMPTY_KEYS);", "unicode = PyUnicode_CheckExact(key);", "newkeys = new_keys_object", "if (newkeys == NULL)", "dictkeys_decref(Py_EMPTY_KEYS);", "MAINTAIN_TRACKING(mp, key, value);", "hashpos = (size_t)hash", "dictkeys_set_index", "if (unicode)", "mp->ma_used++;")
|
|
if [body.index(item) for item in order] != sorted(body.index(item) for item in order):
|
|
raise SystemExit("insert_to_emptydict validation/allocation/tracking/insertion order changed")
|
|
|
|
for signature in ("build_indices_generic(PyDictKeysObject *keys,", "build_indices_unicode(PyDictKeysObject *keys,"):
|
|
body = body_for(dict_source, signature)
|
|
for declaration in ("size_t mask;", "size_t i;", "size_t perturb;", "Py_ssize_t ix;", "Py_hash_t hash;"):
|
|
if declaration not in body or body.index(declaration) > body.index("mask = DK_MASK(keys);"):
|
|
raise SystemExit("build_indices local missing or late: " + declaration)
|
|
if any(token in body for token in ("for (Py_ssize_t ix", "for (size_t perturb", "size_t i = hash")):
|
|
raise SystemExit("build_indices loop locals must be declared at function entry")
|
|
if body.index("for (ix = 0; ix != n") > body.index("dictkeys_set_index(keys, i, ix);"):
|
|
raise SystemExit("build_indices table index insertion order changed")
|
|
|
|
body = body_for(dict_source, "dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode)")
|
|
declarations = ("Py_ssize_t numentries;", "Py_ssize_t i;", "int index;", "PyDictUnicodeEntry *old_unicode_entries;", "PyDictUnicodeEntry *new_unicode_entries;", "PyDictUnicodeEntry *unicode_entry;", "PyDictKeyEntry *old_key_entries;", "PyDictKeyEntry *new_key_entries;", "PyDictKeyEntry *key_entry;", "struct _Py_dict_state *state;")
|
|
for declaration in declarations:
|
|
if declaration not in body or body.index(declaration) > body.index("if (log2_newsize >="):
|
|
raise SystemExit("dictresize local missing or late: " + declaration)
|
|
if any(token in body for token in ("for (Py_ssize_t i", "PyDictKeyEntry *newentries =", "PyDictUnicodeEntry *newentries =", "struct _Py_dict_state *state =")):
|
|
raise SystemExit("dictresize branch/loop locals must be declared at function entry")
|
|
order = ("if (log2_newsize >=", "oldkeys = mp->ma_keys;", "mp->ma_keys = new_keys_object", "numentries = mp->ma_used;", "if (oldvalues != NULL)", "old_unicode_entries = DK_UNICODE_ENTRIES(oldkeys);", "dictkeys_decref(oldkeys);", "else { // oldkeys is combined.", "old_key_entries = DK_ENTRIES(oldkeys);", "old_unicode_entries = DK_UNICODE_ENTRIES(oldkeys);", "state = get_dict_state();", "mp->ma_keys->dk_usable -= numentries;", "ASSERT_CONSISTENT(mp);", "return 0;")
|
|
positions = []
|
|
for item in order:
|
|
positions.append(body.index(item, positions[-1] + 1 if positions else 0))
|
|
if positions != sorted(positions):
|
|
raise SystemExit("dictresize allocation/conversion/freelist/commit order changed")
|
|
body = body_for(dict_source, "dict_keys(PyDictObject *mp)")
|
|
opening, execution = body.split("again:", 1)
|
|
if opening.split() != "PyObject *v; Py_ssize_t n; Py_ssize_t j, pos; PyObject *key;".split():
|
|
raise SystemExit("dict_keys locals must be declared at function entry without initializers")
|
|
expected_execution = """
|
|
n = mp->ma_used;
|
|
v = PyList_New(n);
|
|
if (v == NULL)
|
|
return NULL;
|
|
if (n != mp->ma_used) {
|
|
/* Durnit. The allocations caused the dict to resize.
|
|
* Just start over, this shouldn't normally happen.
|
|
*/
|
|
Py_DECREF(v);
|
|
goto again;
|
|
}
|
|
|
|
/* Nothing we do below makes any function calls. */
|
|
j = 0;
|
|
pos = 0;
|
|
while (_PyDict_Next((PyObject*)mp, &pos, &key, NULL, NULL)) {
|
|
assert(j < n);
|
|
Py_INCREF(key);
|
|
PyList_SET_ITEM(v, j, key);
|
|
j++;
|
|
}
|
|
assert(j == n);
|
|
return v;
|
|
"""
|
|
if execution.split() != expected_execution.split():
|
|
raise SystemExit("dict_keys allocation/retry/initialization/iteration/refcount order changed")
|
|
|
|
print("dict consistency C89 declarations and logic order: PASS")
|