fix: only normalize values for hashing with operators that match
DeepHash._hash ran normalize_value_for_hashing of every registered custom operator on every value, without ever consulting the operator's match method. An operator scoped to one type therefore rewrote values of every other type. When such an operator returns None for the values it was never meant to see, all of those values hash to the same thing and unequal iterables compare as equal under ignore_order=True: [1, 2] and [2, 3] were reported as identical. Ask the operator whether it applies before normalizing. DeepHash walks a single object at a time and has no t1/t2 pair, so the value is passed to match as both sides via a small level-like view, which keeps operators scoped by type or by path behaving as they do while diffing. Operators that define no match method are still applied to everything. Fixes #606
This commit is contained in:
1 parent
25afc42a2c
commit
1476898aa3
2 files changed
+86
No files matched your search
@@ -426,3 +426,47 @@ class TestOperators:
|
||||
diff3 = DeepDiff(deepcopy(d1), deepcopy(d2), ignore_order=True, custom_operators=[RemoveGUIDsOperatorIgnoreOrderReady(types=[str])])
|
||||
assert not diff3, "We shouldn't have a diff because we have normalized the string values to be all the same vlues."
|
||||
|
||||
def test_normalize_value_for_hashing_is_scoped_by_match(self):
|
||||
normalized = []
|
||||
|
||||
class StrOnlyOperator(BaseOperatorPlus):
|
||||
def match(self, level) -> bool:
|
||||
return isinstance(level.t1, str) and isinstance(level.t2, str)
|
||||
|
||||
def give_up_diffing(self, level, diff_instance) -> bool:
|
||||
return level.t1 == level.t2
|
||||
|
||||
def normalize_value_for_hashing(self, parent: Any, obj: Any) -> Any:
|
||||
normalized.append(obj)
|
||||
return obj
|
||||
|
||||
t1 = [1, 2]
|
||||
t2 = [2, 3]
|
||||
expected = DeepDiff(t1, t2, ignore_order=True)
|
||||
|
||||
diff = DeepDiff(t1, t2, ignore_order=True, custom_operators=[StrOnlyOperator()])
|
||||
assert expected == diff, "An operator that only matches strings must not change how integers are diffed."
|
||||
assert [] == normalized, (
|
||||
"normalize_value_for_hashing must not be called for values the operator does not match.")
|
||||
|
||||
def test_normalize_value_for_hashing_not_called_for_unmatched_types(self):
|
||||
normalized = []
|
||||
|
||||
class LowerCaseStringsOperator(BaseOperatorPlus):
|
||||
def match(self, level) -> bool:
|
||||
return isinstance(level.t1, str) and isinstance(level.t2, str)
|
||||
|
||||
def give_up_diffing(self, level, diff_instance) -> bool:
|
||||
return False
|
||||
|
||||
def normalize_value_for_hashing(self, parent: Any, obj: Any) -> Any:
|
||||
# Raises AttributeError if it is handed anything other than a string.
|
||||
normalized.append(obj)
|
||||
return obj.lower()
|
||||
|
||||
DeepDiff(["A", 1], ["a", 1], ignore_order=True,
|
||||
custom_operators=[LowerCaseStringsOperator()])
|
||||
assert normalized, "The operator should still be applied to the values it matches."
|
||||
assert all(isinstance(obj, str) for obj in normalized), (
|
||||
"A string-only operator must never be handed a non-string to normalize.")
|
||||
|
||||
Reference in new issue
Block a user