Upstream seperman/deepdiff state of the touched files before a416378437
This commit is contained in:
upstream committed
1 parent
4179793c2b
commit
25afc42a2c
2 files changed
+1182
No files matched your search
@@ -0,0 +1,754 @@
|
||||
#!/usr/bin/env python
|
||||
import logging
|
||||
import datetime
|
||||
import uuid
|
||||
from typing import Union, Optional, Any, List, TYPE_CHECKING, Dict, Tuple, Set, Callable, Generator
|
||||
from collections.abc import Iterable, MutableMapping
|
||||
from collections import defaultdict
|
||||
from hashlib import sha1, sha256
|
||||
from pathlib import Path
|
||||
from enum import Enum
|
||||
import re
|
||||
from deepdiff.helper import (strings, numbers, only_numbers, times, unprocessed, not_hashed, add_to_frozen_set,
|
||||
convert_item_or_items_into_set_else_none, get_doc, ipranges,
|
||||
convert_item_or_items_into_compiled_regexes_else_none,
|
||||
get_id, type_is_subclass_of_type_group, type_in_type_group,
|
||||
number_to_string, datetime_normalize, KEY_TO_VAL_STR,
|
||||
get_truncate_datetime, dict_, add_root_to_paths, PydanticBaseModel,
|
||||
separate_wildcard_and_exact_paths,
|
||||
SetOrdered)
|
||||
|
||||
from deepdiff.base import Base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pytz.tzinfo import BaseTzInfo
|
||||
import numpy as np
|
||||
|
||||
# Type aliases for better readability
|
||||
HashableType = Union[str, int, float, bytes, bool, tuple, frozenset, type(None)]
|
||||
HashResult = Union[str, Any] # Can be string hash or unprocessed marker
|
||||
HashTuple = Tuple[HashResult, int] # (hash_result, count)
|
||||
HashesDict = Dict[Any, Union[HashTuple, List[Any]]] # Special case for UNPROCESSED_KEY
|
||||
PathType = Union[str, List[str], Set[str]]
|
||||
RegexType = Union[str, re.Pattern[str], List[Union[str, re.Pattern[str]]]]
|
||||
NumberToStringFunc = Callable[..., str] # More flexible for different number_to_string implementations
|
||||
|
||||
|
||||
try:
|
||||
import pandas
|
||||
except ImportError:
|
||||
pandas = False # type: ignore
|
||||
|
||||
try:
|
||||
import polars
|
||||
except ImportError:
|
||||
polars = False # type: ignore
|
||||
try:
|
||||
import numpy as np
|
||||
booleanTypes: Tuple[type, ...] = (bool, np.bool_) # type: ignore
|
||||
except ImportError:
|
||||
booleanTypes = (bool,) # type: ignore
|
||||
|
||||
logger: logging.Logger = logging.getLogger(__name__)
|
||||
|
||||
UNPROCESSED_KEY: object = object()
|
||||
|
||||
EMPTY_FROZENSET: frozenset = frozenset()
|
||||
|
||||
INDEX_VS_ATTRIBUTE: Tuple[str, str] = ('[%s]', '.%s')
|
||||
|
||||
|
||||
HASH_LOOKUP_ERR_MSG: str = '{} is not one of the hashed items.'
|
||||
|
||||
|
||||
def sha256hex(obj: Union[str, bytes]) -> str:
|
||||
"""Use Sha256 as a cryptographic hash."""
|
||||
if isinstance(obj, str):
|
||||
obj = obj.encode('utf-8')
|
||||
return sha256(obj).hexdigest()
|
||||
|
||||
|
||||
def sha1hex(obj: Union[str, bytes]) -> str:
|
||||
"""Use Sha1 as a cryptographic hash."""
|
||||
if isinstance(obj, str):
|
||||
obj = obj.encode('utf-8')
|
||||
return sha1(obj).hexdigest()
|
||||
|
||||
|
||||
default_hasher: Callable[[Union[str, bytes]], str] = sha256hex
|
||||
|
||||
|
||||
def combine_hashes_lists(items: List[List[str]], prefix: Union[str, bytes]) -> str:
|
||||
"""
|
||||
Combines lists of hashes into one hash
|
||||
This can be optimized in future.
|
||||
It needs to work with both murmur3 hashes (int) and sha256 (str)
|
||||
Although murmur3 is not used anymore.
|
||||
"""
|
||||
if isinstance(prefix, bytes):
|
||||
prefix = prefix.decode('utf-8')
|
||||
hashes_bytes = b''
|
||||
for item in items:
|
||||
# In order to make sure the order of hashes in each item does not affect the hash
|
||||
# we resort them.
|
||||
hashes_bytes += (''.join(map(str, sorted(item))) + '--').encode('utf-8')
|
||||
return prefix + str(default_hasher(hashes_bytes))
|
||||
|
||||
|
||||
class BoolObj(Enum):
|
||||
TRUE = 1
|
||||
FALSE = 0
|
||||
|
||||
|
||||
def prepare_string_for_hashing(
|
||||
obj: Union[str, bytes, memoryview],
|
||||
ignore_string_type_changes: bool = False,
|
||||
ignore_string_case: bool = False,
|
||||
encodings: Optional[List[str]] = None,
|
||||
ignore_encoding_errors: bool = False,
|
||||
) -> str:
|
||||
"""
|
||||
Clean type conversions
|
||||
"""
|
||||
original_type = obj.__class__.__name__
|
||||
# https://docs.python.org/3/library/codecs.html#codecs.decode
|
||||
errors_mode = 'ignore' if ignore_encoding_errors else 'strict'
|
||||
if isinstance(obj, memoryview):
|
||||
obj = obj.tobytes()
|
||||
if isinstance(obj, bytes):
|
||||
err = None
|
||||
encodings = ['utf-8'] if encodings is None else encodings
|
||||
encoded = False
|
||||
for encoding in encodings:
|
||||
try:
|
||||
obj = obj.decode(encoding, errors=errors_mode)
|
||||
encoded = True
|
||||
break
|
||||
except UnicodeDecodeError as er:
|
||||
err = er
|
||||
if not encoded and err is not None:
|
||||
obj_decoded = obj.decode('utf-8', errors='ignore') # type: ignore
|
||||
start = max(err.start - 20, 0)
|
||||
start_prefix = ''
|
||||
if start > 0:
|
||||
start_prefix = '...'
|
||||
end = err.end + 20
|
||||
end_suffix = '...'
|
||||
if end >= len(obj):
|
||||
end = len(obj)
|
||||
end_suffix = ''
|
||||
raise UnicodeDecodeError(
|
||||
err.encoding,
|
||||
err.object,
|
||||
err.start,
|
||||
err.end,
|
||||
f"{err.reason} in '{start_prefix}{obj_decoded[start:end]}{end_suffix}'. Please either pass ignore_encoding_errors=True or pass the encoding via encodings=['utf-8', '...']."
|
||||
) from None
|
||||
if not ignore_string_type_changes:
|
||||
obj = KEY_TO_VAL_STR.format(original_type, obj)
|
||||
if ignore_string_case:
|
||||
obj = obj.lower()
|
||||
return str(obj)
|
||||
|
||||
|
||||
doc = get_doc('deephash_doc.rst')
|
||||
|
||||
|
||||
class DeepHash(Base):
|
||||
__doc__ = doc
|
||||
|
||||
# Class attributes
|
||||
hashes: Dict[Any, Any]
|
||||
exclude_types_tuple: Tuple[type, ...]
|
||||
ignore_repetition: bool
|
||||
exclude_paths: Optional[SetOrdered]
|
||||
include_paths: Optional[SetOrdered]
|
||||
exclude_regex_paths: Optional[List[re.Pattern[str]]]
|
||||
hasher: Callable[[Union[str, bytes]], str]
|
||||
use_enum_value: bool
|
||||
default_timezone: Union[datetime.timezone, "BaseTzInfo"]
|
||||
significant_digits: Optional[int]
|
||||
truncate_datetime: Optional[str]
|
||||
number_format_notation: str
|
||||
ignore_type_in_groups: Any
|
||||
ignore_string_type_changes: bool
|
||||
ignore_numeric_type_changes: bool
|
||||
ignore_string_case: bool
|
||||
exclude_obj_callback: Optional[Callable[[Any, str], bool]]
|
||||
apply_hash: bool
|
||||
type_check_func: Callable[[type, Any], bool]
|
||||
number_to_string: Any
|
||||
ignore_private_variables: bool
|
||||
encodings: Optional[List[str]]
|
||||
ignore_encoding_errors: bool
|
||||
ignore_iterable_order: bool
|
||||
custom_operators: Optional[List[Any]]
|
||||
|
||||
def __init__(self,
|
||||
obj: Any,
|
||||
*,
|
||||
apply_hash: bool = True,
|
||||
custom_operators: Optional[List[Any]] = None,
|
||||
default_timezone: Union[datetime.timezone, "BaseTzInfo"] = datetime.timezone.utc,
|
||||
encodings: Optional[List[str]] = None,
|
||||
exclude_glob_paths: Optional[List[Any]] = None,
|
||||
exclude_obj_callback: Optional[Callable[[Any, str], bool]] = None,
|
||||
exclude_paths: Optional[PathType] = None,
|
||||
exclude_regex_paths: Optional[RegexType] = None,
|
||||
exclude_types: Optional[Union[List[type], Set[type], Tuple[type, ...]]] = None,
|
||||
hasher: Optional[Callable[[Union[str, bytes]], str]] = None,
|
||||
hashes: Optional[Union[Dict[Any, Any], "DeepHash"]] = None,
|
||||
ignore_encoding_errors: bool = False,
|
||||
ignore_iterable_order: bool = True,
|
||||
ignore_numeric_type_changes: bool = False,
|
||||
ignore_private_variables: bool = True,
|
||||
ignore_repetition: bool = True,
|
||||
ignore_string_case: bool = False,
|
||||
ignore_string_type_changes: bool = False,
|
||||
ignore_type_in_groups: Any = None,
|
||||
ignore_type_subclasses: bool = False,
|
||||
ignore_uuid_types: bool = False,
|
||||
include_glob_paths: Optional[List[Any]] = None,
|
||||
include_paths: Optional[PathType] = None,
|
||||
number_format_notation: str = "f",
|
||||
number_to_string_func: Optional[NumberToStringFunc] = None,
|
||||
parent: str = "root",
|
||||
significant_digits: Optional[int] = None,
|
||||
truncate_datetime: Optional[str] = None,
|
||||
use_enum_value: bool = False,
|
||||
**kwargs) -> None:
|
||||
if kwargs:
|
||||
raise ValueError(
|
||||
("The following parameter(s) are not valid: %s\n"
|
||||
"The valid parameters are obj, hashes, exclude_types, significant_digits, truncate_datetime,"
|
||||
"exclude_paths, include_paths, exclude_regex_paths, hasher, ignore_repetition, "
|
||||
"number_format_notation, apply_hash, ignore_type_in_groups, ignore_string_type_changes, "
|
||||
"ignore_numeric_type_changes, ignore_type_subclasses, ignore_string_case, ignore_uuid_types, "
|
||||
"number_to_string_func, ignore_private_variables, parent, use_enum_value, default_timezone "
|
||||
"encodings, ignore_encoding_errors") % ', '.join(kwargs.keys()))
|
||||
if isinstance(hashes, MutableMapping):
|
||||
self.hashes = hashes
|
||||
elif isinstance(hashes, DeepHash):
|
||||
self.hashes = hashes.hashes
|
||||
else:
|
||||
self.hashes = dict_()
|
||||
exclude_types = set() if exclude_types is None else set(exclude_types)
|
||||
self.exclude_types_tuple = tuple(exclude_types) # we need tuple for checking isinstance
|
||||
self.ignore_repetition = ignore_repetition
|
||||
_exclude_set = convert_item_or_items_into_set_else_none(exclude_paths)
|
||||
_exclude_exact, _exclude_globs = separate_wildcard_and_exact_paths(_exclude_set)
|
||||
self.exclude_paths = add_root_to_paths(_exclude_exact)
|
||||
self.exclude_glob_paths = exclude_glob_paths or _exclude_globs
|
||||
_include_set = convert_item_or_items_into_set_else_none(include_paths)
|
||||
_include_exact, _include_globs = separate_wildcard_and_exact_paths(_include_set)
|
||||
self.include_paths = add_root_to_paths(_include_exact)
|
||||
self.include_glob_paths = include_glob_paths or _include_globs
|
||||
self.exclude_regex_paths = convert_item_or_items_into_compiled_regexes_else_none(exclude_regex_paths)
|
||||
self.hasher = default_hasher if hasher is None else hasher
|
||||
self.hashes[UNPROCESSED_KEY] = [] # type: ignore
|
||||
self.use_enum_value = use_enum_value
|
||||
self.default_timezone = default_timezone
|
||||
self.significant_digits = self.get_significant_digits(significant_digits, ignore_numeric_type_changes)
|
||||
self.truncate_datetime = get_truncate_datetime(truncate_datetime)
|
||||
self.number_format_notation = number_format_notation
|
||||
self.ignore_type_in_groups = self.get_ignore_types_in_groups(
|
||||
ignore_type_in_groups=ignore_type_in_groups,
|
||||
ignore_string_type_changes=ignore_string_type_changes,
|
||||
ignore_numeric_type_changes=ignore_numeric_type_changes,
|
||||
ignore_type_subclasses=ignore_type_subclasses,
|
||||
ignore_uuid_types=ignore_uuid_types,
|
||||
)
|
||||
self.ignore_string_type_changes = ignore_string_type_changes
|
||||
self.ignore_numeric_type_changes = ignore_numeric_type_changes
|
||||
self.ignore_string_case = ignore_string_case
|
||||
self.exclude_obj_callback = exclude_obj_callback
|
||||
# makes the hash return constant size result if true
|
||||
# the only time it should be set to False is when
|
||||
# testing the individual hash functions for different types of objects.
|
||||
self.apply_hash = apply_hash
|
||||
self.type_check_func = type_in_type_group if ignore_type_subclasses else type_is_subclass_of_type_group
|
||||
# self.type_check_func = type_is_subclass_of_type_group if ignore_type_subclasses else type_in_type_group
|
||||
self.number_to_string = number_to_string_func or number_to_string
|
||||
self.ignore_private_variables = ignore_private_variables
|
||||
self.encodings = encodings
|
||||
self.ignore_encoding_errors = ignore_encoding_errors
|
||||
self.ignore_iterable_order = ignore_iterable_order
|
||||
self.custom_operators = custom_operators
|
||||
|
||||
self._hash(obj, parent=parent, parents_ids=frozenset({get_id(obj)}))
|
||||
|
||||
if self.hashes[UNPROCESSED_KEY]:
|
||||
logger.warning("Can not hash the following items: {}.".format(self.hashes[UNPROCESSED_KEY]))
|
||||
else:
|
||||
del self.hashes[UNPROCESSED_KEY]
|
||||
|
||||
sha256hex: Callable[[Union[str, bytes]], str] = sha256hex
|
||||
sha1hex: Callable[[Union[str, bytes]], str] = sha1hex
|
||||
|
||||
def __getitem__(self, obj: Any, extract_index: Optional[int] = 0) -> Any:
|
||||
return self._getitem(self.hashes, obj, extract_index=extract_index,
|
||||
use_enum_value=self.use_enum_value,
|
||||
ignore_numeric_type_changes=self.ignore_numeric_type_changes)
|
||||
|
||||
@staticmethod
|
||||
def _get_slots_dict(obj: Any) -> Dict[str, Any]:
|
||||
"""Get a dict of initialized slot attributes.
|
||||
|
||||
Uses object.__getattribute__ to check each slot directly, bypassing
|
||||
__getattr__. For uninitialized slots on classes that define __getattr__,
|
||||
falls back to getattr — letting it raise if the object is truly broken.
|
||||
"""
|
||||
result = {}
|
||||
has_getattr = hasattr(type(obj), '__getattr__')
|
||||
for slot in obj.__slots__:
|
||||
try:
|
||||
result[slot] = object.__getattribute__(obj, slot)
|
||||
except AttributeError:
|
||||
if has_getattr:
|
||||
# The slot isn't initialized, but the class defines __getattr__.
|
||||
# Try the normal getattr to let __getattr__ provide a value or
|
||||
# raise — if it raises, we propagate to fail the strategy.
|
||||
result[slot] = getattr(obj, slot)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _getitem(hashes: Dict[Any, Any], obj: Any, extract_index: Optional[int] = 0,
|
||||
use_enum_value: bool = False, ignore_numeric_type_changes: bool = False) -> Any:
|
||||
"""
|
||||
extract_index is zero for hash and 1 for count and None to get them both.
|
||||
To keep it backward compatible, we only get the hash by default so it is set to zero by default.
|
||||
"""
|
||||
|
||||
key = obj
|
||||
if obj is True:
|
||||
key = BoolObj.TRUE
|
||||
elif obj is False:
|
||||
key = BoolObj.FALSE
|
||||
elif use_enum_value and isinstance(obj, Enum):
|
||||
key = obj.value
|
||||
key = DeepHash._make_hash_key_for_lookup(key, ignore_numeric_type_changes=ignore_numeric_type_changes)
|
||||
|
||||
result_n_count: Tuple[Any, int] = (None, 0) # type: ignore
|
||||
|
||||
try:
|
||||
result_n_count = hashes[key]
|
||||
except (TypeError, KeyError):
|
||||
key = get_id(obj)
|
||||
try:
|
||||
result_n_count = hashes[key]
|
||||
except KeyError:
|
||||
raise KeyError(HASH_LOOKUP_ERR_MSG.format(obj)) from None
|
||||
|
||||
if obj is UNPROCESSED_KEY:
|
||||
extract_index = None
|
||||
|
||||
return result_n_count if extract_index is None else result_n_count[extract_index]
|
||||
|
||||
def __contains__(self, obj: Any) -> bool:
|
||||
key = self._make_hash_key(obj)
|
||||
result = False
|
||||
try:
|
||||
result = key in self.hashes
|
||||
except (TypeError, KeyError):
|
||||
result = False
|
||||
if not result:
|
||||
result = get_id(obj) in self.hashes
|
||||
return result
|
||||
|
||||
def get(self, key: Any, default: Any = None, extract_index: Optional[int] = 0) -> Any:
|
||||
"""
|
||||
Get method for the hashes dictionary.
|
||||
It can extract the hash for a given key that is already calculated when extract_index=0
|
||||
or the count of items that went to building the object when extract_index=1.
|
||||
"""
|
||||
return self.get_key(self.hashes, key, default=default, extract_index=extract_index,
|
||||
ignore_numeric_type_changes=self.ignore_numeric_type_changes)
|
||||
|
||||
@staticmethod
|
||||
def get_key(hashes: Dict[Any, Any], key: Any, default: Any = None, extract_index: Optional[int] = 0,
|
||||
use_enum_value: bool = False, ignore_numeric_type_changes: bool = False) -> Any:
|
||||
"""
|
||||
get_key method for the hashes dictionary.
|
||||
It can extract the hash for a given key that is already calculated when extract_index=0
|
||||
or the count of items that went to building the object when extract_index=1.
|
||||
"""
|
||||
try:
|
||||
result = DeepHash._getitem(hashes, key, extract_index=extract_index,
|
||||
use_enum_value=use_enum_value,
|
||||
ignore_numeric_type_changes=ignore_numeric_type_changes)
|
||||
except KeyError:
|
||||
result = default
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _unwrap_hash_key(key: Any) -> Any:
|
||||
"""Unwrap a (type, value) hash key back to the original value for public API."""
|
||||
if isinstance(key, tuple) and len(key) == 2 and isinstance(key[0], type) and isinstance(key[1], only_numbers):
|
||||
return key[1]
|
||||
return key
|
||||
|
||||
def _get_objects_to_hashes_dict(self, extract_index: Optional[int] = 0) -> Dict[Any, Any]:
|
||||
"""
|
||||
A dictionary containing only the objects to hashes,
|
||||
or a dictionary of objects to the count of items that went to build them.
|
||||
extract_index=0 for hashes and extract_index=1 for counts.
|
||||
"""
|
||||
result = dict_()
|
||||
for key, value in self.hashes.items():
|
||||
key = self._unwrap_hash_key(key)
|
||||
if key is UNPROCESSED_KEY:
|
||||
result[key] = value
|
||||
else:
|
||||
result[key] = value[extract_index]
|
||||
return result
|
||||
|
||||
def __eq__(self, other: Any) -> bool:
|
||||
if isinstance(other, DeepHash):
|
||||
return self.hashes == other.hashes
|
||||
else:
|
||||
# We only care about the hashes
|
||||
return self._get_objects_to_hashes_dict() == other
|
||||
|
||||
__req__ = __eq__
|
||||
|
||||
def __repr__(self) -> str:
|
||||
"""
|
||||
Hide the counts since it will be confusing to see them when they are hidden everywhere else.
|
||||
"""
|
||||
from deepdiff.summarize import summarize
|
||||
return summarize(self._get_objects_to_hashes_dict(extract_index=0), max_length=500)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return str(self._get_objects_to_hashes_dict(extract_index=0))
|
||||
|
||||
def __bool__(self) -> bool:
|
||||
return bool(self.hashes)
|
||||
|
||||
def keys(self) -> Any:
|
||||
return [self._unwrap_hash_key(k) for k in self.hashes.keys()]
|
||||
|
||||
def values(self) -> Generator[Any, None, None]:
|
||||
return (i[0] for i in self.hashes.values()) # Just grab the item and not its count
|
||||
|
||||
def items(self) -> Generator[Tuple[Any, Any], None, None]:
|
||||
return ((self._unwrap_hash_key(i), v[0]) for i, v in self.hashes.items())
|
||||
|
||||
def _prep_obj(self, obj: Any, parent: str, parents_ids: frozenset = EMPTY_FROZENSET, is_namedtuple: bool = False, is_pydantic_object: bool = False) -> HashTuple:
|
||||
"""prepping objects"""
|
||||
original_type = type(obj) if not isinstance(obj, type) else obj
|
||||
|
||||
obj_to_dict_strategies = []
|
||||
if is_namedtuple:
|
||||
obj_to_dict_strategies.append(lambda o: o._asdict())
|
||||
elif is_pydantic_object:
|
||||
obj_to_dict_strategies.append(lambda o: {k: v for (k, v) in o.__dict__.items() if v !="model_fields_set"})
|
||||
else:
|
||||
obj_to_dict_strategies.append(lambda o: o.__dict__)
|
||||
|
||||
if hasattr(obj, "__slots__"):
|
||||
obj_to_dict_strategies.append(lambda o: DeepHash._get_slots_dict(o))
|
||||
else:
|
||||
import inspect
|
||||
obj_to_dict_strategies.append(lambda o: dict(inspect.getmembers(o, lambda m: not inspect.isroutine(m))))
|
||||
|
||||
for get_dict in obj_to_dict_strategies:
|
||||
try:
|
||||
d = get_dict(obj)
|
||||
break
|
||||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
self.hashes[UNPROCESSED_KEY].append(obj) # type: ignore
|
||||
return (unprocessed, 0)
|
||||
obj = d
|
||||
|
||||
result, counts = self._prep_dict(obj, parent=parent, parents_ids=parents_ids,
|
||||
print_as_attribute=True, original_type=original_type)
|
||||
result = "nt{}".format(result) if is_namedtuple else "obj{}".format(result)
|
||||
return result, counts
|
||||
|
||||
def _skip_this(self, obj: Any, parent: str) -> bool:
|
||||
skip = False
|
||||
if self.exclude_paths and parent in self.exclude_paths:
|
||||
skip = True
|
||||
elif self.exclude_glob_paths and any(gp.match(parent) for gp in self.exclude_glob_paths):
|
||||
skip = True
|
||||
if (self.include_paths or self.include_glob_paths) and parent != 'root':
|
||||
skip = True
|
||||
if self.include_paths:
|
||||
if parent in self.include_paths:
|
||||
skip = False
|
||||
else:
|
||||
for prefix in self.include_paths:
|
||||
if parent.startswith(prefix):
|
||||
skip = False
|
||||
break
|
||||
if skip and self.include_glob_paths:
|
||||
for gp in self.include_glob_paths:
|
||||
if gp.match_or_is_ancestor(parent):
|
||||
skip = False
|
||||
break
|
||||
elif self.exclude_regex_paths and any(
|
||||
[exclude_regex_path.search(parent) for exclude_regex_path in self.exclude_regex_paths]): # type: ignore
|
||||
skip = True
|
||||
elif self.exclude_types_tuple and isinstance(obj, self.exclude_types_tuple):
|
||||
skip = True
|
||||
elif self.exclude_obj_callback and self.exclude_obj_callback(obj, parent):
|
||||
skip = True
|
||||
return skip
|
||||
|
||||
def _prep_dict(self, obj: Union[Dict[Any, Any], MutableMapping], parent: str, parents_ids: frozenset = EMPTY_FROZENSET, print_as_attribute: bool = False, original_type: Optional[type] = None) -> HashTuple:
|
||||
|
||||
result = []
|
||||
counts = 1
|
||||
|
||||
key_text = "%s{}".format(INDEX_VS_ATTRIBUTE[print_as_attribute])
|
||||
for key, item in obj.items():
|
||||
counts += 1
|
||||
# ignore private variables
|
||||
if self.ignore_private_variables and isinstance(key, str) and key.startswith('__'):
|
||||
continue
|
||||
key_formatted = "'%s'" % key if not print_as_attribute and isinstance(key, strings) else key
|
||||
key_in_report = key_text % (parent, key_formatted)
|
||||
|
||||
key_hash, _ = self._hash(key, parent=key_in_report, parents_ids=parents_ids)
|
||||
if not key_hash:
|
||||
continue
|
||||
item_id = get_id(item)
|
||||
if (parents_ids and item_id in parents_ids) or self._skip_this(item, parent=key_in_report):
|
||||
continue
|
||||
parents_ids_added = add_to_frozen_set(parents_ids, item_id)
|
||||
hashed, count = self._hash(item, parent=key_in_report, parents_ids=parents_ids_added)
|
||||
hashed = KEY_TO_VAL_STR.format(key_hash, hashed)
|
||||
result.append(hashed)
|
||||
counts += count
|
||||
|
||||
result.sort()
|
||||
result = ';'.join(result)
|
||||
if print_as_attribute:
|
||||
type_ = original_type or type(obj)
|
||||
type_str = type_.__name__
|
||||
for type_group in self.ignore_type_in_groups:
|
||||
if self.type_check_func(type_, type_group):
|
||||
type_str = ','.join(map(lambda x: x.__name__, type_group))
|
||||
break
|
||||
else:
|
||||
type_str = 'dict'
|
||||
return "{}:{{{}}}".format(type_str, result), counts
|
||||
|
||||
def _prep_iterable(self, obj: Iterable[Any], parent: str, parents_ids: frozenset = EMPTY_FROZENSET) -> HashTuple:
|
||||
|
||||
counts = 1
|
||||
result = defaultdict(int)
|
||||
|
||||
for i, item in enumerate(obj):
|
||||
new_parent = "{}[{}]".format(parent, i)
|
||||
if self._skip_this(item, parent=new_parent):
|
||||
continue
|
||||
|
||||
item_id = get_id(item)
|
||||
if parents_ids and item_id in parents_ids:
|
||||
continue
|
||||
|
||||
parents_ids_added = add_to_frozen_set(parents_ids, item_id)
|
||||
hashed, count = self._hash(item, parent=new_parent, parents_ids=parents_ids_added)
|
||||
# counting repetitions
|
||||
result[hashed] += 1
|
||||
counts += count
|
||||
|
||||
if self.ignore_repetition:
|
||||
result = list(result.keys())
|
||||
else:
|
||||
result = [
|
||||
'{}|{}'.format(i, v) for i, v in result.items()
|
||||
]
|
||||
|
||||
result = map(str, result) # making sure the result items are string so join command works.
|
||||
if self.ignore_iterable_order:
|
||||
result = sorted(result)
|
||||
result = ','.join(result)
|
||||
result = KEY_TO_VAL_STR.format(type(obj).__name__, result)
|
||||
|
||||
return result, counts
|
||||
|
||||
def _prep_bool(self, obj: bool) -> BoolObj:
|
||||
return BoolObj.TRUE if obj else BoolObj.FALSE
|
||||
|
||||
|
||||
def _prep_path(self, obj: Path) -> str:
|
||||
type_ = obj.__class__.__name__
|
||||
return KEY_TO_VAL_STR.format(type_, obj)
|
||||
|
||||
def _prep_number(self, obj: Union[int, float, complex]) -> str:
|
||||
type_ = "number" if self.ignore_numeric_type_changes else obj.__class__.__name__
|
||||
if self.significant_digits is not None:
|
||||
obj = self.number_to_string(obj, significant_digits=self.significant_digits,
|
||||
number_format_notation=self.number_format_notation) # type: ignore
|
||||
return KEY_TO_VAL_STR.format(type_, obj)
|
||||
|
||||
def _prep_ipranges(self, obj) -> str:
|
||||
type_ = 'iprange'
|
||||
obj = str(obj)
|
||||
return KEY_TO_VAL_STR.format(type_, obj)
|
||||
|
||||
def _prep_datetime(self, obj: datetime.datetime) -> str:
|
||||
type_ = 'datetime'
|
||||
obj = datetime_normalize(self.truncate_datetime, obj, default_timezone=self.default_timezone)
|
||||
return KEY_TO_VAL_STR.format(type_, obj)
|
||||
|
||||
def _prep_date(self, obj: datetime.date) -> str:
|
||||
type_ = 'datetime' # yes still datetime but it doesn't need normalization
|
||||
return KEY_TO_VAL_STR.format(type_, obj)
|
||||
|
||||
def _prep_tuple(self, obj: tuple, parent: str, parents_ids: frozenset) -> HashTuple:
|
||||
# Checking to see if it has _fields. Which probably means it is a named
|
||||
# tuple.
|
||||
try:
|
||||
obj._asdict # type: ignore
|
||||
# It must be a normal tuple
|
||||
except AttributeError:
|
||||
result, counts = self._prep_iterable(obj=obj, parent=parent, parents_ids=parents_ids)
|
||||
# We assume it is a namedtuple then
|
||||
else:
|
||||
result, counts = self._prep_obj(obj, parent, parents_ids=parents_ids, is_namedtuple=True)
|
||||
return result, counts
|
||||
|
||||
def _make_hash_key(self, obj: Any) -> Any:
|
||||
"""
|
||||
Create a key for the hashes dict that distinguishes numeric types.
|
||||
|
||||
In Python, 1 == 1.0 and hash(1) == hash(1.0), so int and float values
|
||||
collide as dict keys. When ignore_numeric_type_changes is False, we wrap
|
||||
numeric objects as (type, value) tuples so that each type gets its own
|
||||
cache entry and its own hash.
|
||||
"""
|
||||
if not self.ignore_numeric_type_changes and isinstance(obj, only_numbers):
|
||||
return (type(obj), obj)
|
||||
return obj
|
||||
|
||||
@staticmethod
|
||||
def _make_hash_key_for_lookup(obj: Any, ignore_numeric_type_changes: bool = False) -> Any:
|
||||
"""Static version of _make_hash_key for use in static accessor methods."""
|
||||
if not ignore_numeric_type_changes and isinstance(obj, only_numbers):
|
||||
return (type(obj), obj)
|
||||
return obj
|
||||
|
||||
def _hash(self, obj: Any, parent: str, parents_ids: frozenset = EMPTY_FROZENSET) -> HashTuple:
|
||||
"""The main hash method"""
|
||||
counts = 1
|
||||
if self.custom_operators is not None:
|
||||
for operator in self.custom_operators:
|
||||
func = getattr(operator, 'normalize_value_for_hashing', None)
|
||||
if func is None:
|
||||
raise NotImplementedError(f"{operator.__class__.__name__} needs to define a normalize_value_for_hashing method to be compatible with ignore_order=True or iterable_compare_func.".format(operator))
|
||||
else:
|
||||
obj = func(parent, obj)
|
||||
|
||||
if isinstance(obj, booleanTypes):
|
||||
obj = self._prep_bool(obj)
|
||||
result = None
|
||||
elif self.use_enum_value and isinstance(obj, Enum):
|
||||
obj = obj.value
|
||||
else:
|
||||
result = not_hashed
|
||||
hash_key = self._make_hash_key(obj)
|
||||
try:
|
||||
result, counts = self.hashes[hash_key]
|
||||
except (TypeError, KeyError):
|
||||
pass
|
||||
else:
|
||||
return result, counts
|
||||
|
||||
if self._skip_this(obj, parent):
|
||||
return None, 0
|
||||
|
||||
elif obj is None:
|
||||
result = 'NONE'
|
||||
|
||||
elif isinstance(obj, strings):
|
||||
result = prepare_string_for_hashing(
|
||||
obj,
|
||||
ignore_string_type_changes=self.ignore_string_type_changes,
|
||||
ignore_string_case=self.ignore_string_case,
|
||||
encodings=self.encodings,
|
||||
ignore_encoding_errors=self.ignore_encoding_errors,
|
||||
)
|
||||
|
||||
elif isinstance(obj, Path):
|
||||
result = self._prep_path(obj)
|
||||
|
||||
elif isinstance(obj, times):
|
||||
result = self._prep_datetime(obj) # type: ignore
|
||||
|
||||
elif isinstance(obj, datetime.date):
|
||||
result = self._prep_date(obj)
|
||||
|
||||
elif isinstance(obj, numbers): # type: ignore
|
||||
result = self._prep_number(obj)
|
||||
|
||||
elif isinstance(obj, ipranges):
|
||||
result = self._prep_ipranges(obj)
|
||||
|
||||
elif isinstance(obj, uuid.UUID):
|
||||
# Handle UUID objects (including uuid6.UUID) by using their integer value
|
||||
result = str(obj.int)
|
||||
|
||||
elif isinstance(obj, MutableMapping):
|
||||
result, counts = self._prep_dict(obj=obj, parent=parent, parents_ids=parents_ids)
|
||||
|
||||
elif isinstance(obj, tuple):
|
||||
result, counts = self._prep_tuple(obj=obj, parent=parent, parents_ids=parents_ids)
|
||||
|
||||
elif (pandas and isinstance(obj, pandas.DataFrame)): # type: ignore
|
||||
def gen(): # type: ignore
|
||||
yield ('dtype', obj.dtypes) # type: ignore
|
||||
yield ('index', obj.index) # type: ignore
|
||||
yield from obj.items() # type: ignore # which contains (column name, series tuples)
|
||||
result, counts = self._prep_iterable(obj=gen(), parent=parent, parents_ids=parents_ids)
|
||||
elif (polars and isinstance(obj, polars.DataFrame)): # type: ignore
|
||||
def gen():
|
||||
yield from obj.columns # type: ignore
|
||||
yield from list(obj.schema.items()) # type: ignore
|
||||
yield from obj.rows() # type: ignore
|
||||
result, counts = self._prep_iterable(obj=gen(), parent=parent, parents_ids=parents_ids)
|
||||
|
||||
elif isinstance(obj, Iterable):
|
||||
result, counts = self._prep_iterable(obj=obj, parent=parent, parents_ids=parents_ids)
|
||||
|
||||
elif obj == BoolObj.TRUE or obj == BoolObj.FALSE:
|
||||
result = 'bool:true' if obj is BoolObj.TRUE else 'bool:false'
|
||||
elif isinstance(obj, PydanticBaseModel):
|
||||
result, counts = self._prep_obj(obj=obj, parent=parent, parents_ids=parents_ids, is_pydantic_object=True)
|
||||
else:
|
||||
result, counts = self._prep_obj(obj=obj, parent=parent, parents_ids=parents_ids)
|
||||
|
||||
if result is not_hashed: # pragma: no cover
|
||||
self.hashes[UNPROCESSED_KEY].append(obj) # type: ignore
|
||||
|
||||
elif result is unprocessed:
|
||||
pass
|
||||
|
||||
elif self.apply_hash:
|
||||
if isinstance(obj, strings):
|
||||
result_cleaned = result
|
||||
else:
|
||||
result_cleaned = prepare_string_for_hashing(
|
||||
str(result), ignore_string_type_changes=self.ignore_string_type_changes,
|
||||
ignore_string_case=self.ignore_string_case)
|
||||
result = self.hasher(result_cleaned)
|
||||
|
||||
# It is important to keep the hash of all objects.
|
||||
# The hashes will be later used for comparing the objects.
|
||||
# Object to hash when possible otherwise ObjectID to hash
|
||||
try:
|
||||
self.hashes[hash_key] = (result, counts)
|
||||
except TypeError:
|
||||
obj_id = get_id(obj)
|
||||
self.hashes[obj_id] = (result, counts)
|
||||
|
||||
return result, counts
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
import doctest
|
||||
doctest.testmod()
|
||||
@@ -0,0 +1,428 @@
|
||||
import re
|
||||
import math
|
||||
import pytest
|
||||
from copy import deepcopy
|
||||
from typing import List, Any
|
||||
from deepdiff import DeepDiff
|
||||
from deepdiff.operator import BaseOperator, PrefixOrSuffixOperator, BaseOperatorPlus
|
||||
|
||||
|
||||
class TestOperators:
|
||||
def test_custom_operators_prevent_default(self):
|
||||
t1 = {
|
||||
"coordinates": [
|
||||
{"x": 5, "y": 5},
|
||||
{"x": 8, "y": 8}
|
||||
]
|
||||
}
|
||||
|
||||
t2 = {
|
||||
"coordinates": [
|
||||
{"x": 6, "y": 6},
|
||||
{"x": 88, "y": 88}
|
||||
]
|
||||
}
|
||||
|
||||
class L2DistanceDifferWithPreventDefault(BaseOperator):
|
||||
def __init__(self, regex_paths: List[str], distance_threshold: float):
|
||||
super().__init__(regex_paths)
|
||||
self.distance_threshold = distance_threshold
|
||||
|
||||
def _l2_distance(self, c1, c2):
|
||||
return math.sqrt(
|
||||
(c1["x"] - c2["x"]) ** 2 + (c1["y"] - c2["y"]) ** 2
|
||||
)
|
||||
|
||||
def give_up_diffing(self, level, diff_instance) -> bool:
|
||||
l2_distance = self._l2_distance(level.t1, level.t2)
|
||||
if l2_distance > self.distance_threshold:
|
||||
diff_instance.custom_report_result('distance_too_far', level, {
|
||||
"l2_distance": l2_distance
|
||||
})
|
||||
return True
|
||||
|
||||
ddiff = DeepDiff(t1, t2, custom_operators=[L2DistanceDifferWithPreventDefault(
|
||||
["^root\\['coordinates'\\]\\[\\d+\\]$"],
|
||||
1
|
||||
)])
|
||||
|
||||
expected = {
|
||||
'distance_too_far': {
|
||||
"root['coordinates'][0]": {'l2_distance': 1.4142135623730951},
|
||||
"root['coordinates'][1]": {'l2_distance': 113.13708498984761}
|
||||
}
|
||||
}
|
||||
assert expected == ddiff
|
||||
|
||||
def test_custom_operators_not_prevent_default(self):
|
||||
t1 = {
|
||||
"coordinates": [
|
||||
{"x": 5, "y": 5},
|
||||
{"x": 8, "y": 8}
|
||||
]
|
||||
}
|
||||
|
||||
t2 = {
|
||||
"coordinates": [
|
||||
{"x": 6, "y": 6},
|
||||
{"x": 88, "y": 88}
|
||||
]
|
||||
}
|
||||
|
||||
class L2DistanceDifferWithPreventDefault(BaseOperator):
|
||||
def __init__(self, regex_paths, distance_threshold):
|
||||
super().__init__(regex_paths)
|
||||
self.distance_threshold = distance_threshold
|
||||
|
||||
def _l2_distance(self, c1, c2):
|
||||
return math.sqrt(
|
||||
(c1["x"] - c2["x"]) ** 2 + (c1["y"] - c2["y"]) ** 2
|
||||
)
|
||||
|
||||
def give_up_diffing(self, level, diff_instance) -> bool:
|
||||
l2_distance = self._l2_distance(level.t1, level.t2)
|
||||
if l2_distance > self.distance_threshold:
|
||||
diff_instance.custom_report_result('distance_too_far', level, {
|
||||
"l2_distance": l2_distance
|
||||
})
|
||||
#
|
||||
return False
|
||||
|
||||
ddiff = DeepDiff(t1, t2, custom_operators=[L2DistanceDifferWithPreventDefault(
|
||||
["^root\\['coordinates'\\]\\[\\d+\\]$"],
|
||||
1
|
||||
)
|
||||
])
|
||||
expected = {
|
||||
'values_changed': {
|
||||
"root['coordinates'][0]['x']": {'new_value': 6, 'old_value': 5},
|
||||
"root['coordinates'][0]['y']": {'new_value': 6, 'old_value': 5},
|
||||
"root['coordinates'][1]['x']": {'new_value': 88, 'old_value': 8},
|
||||
"root['coordinates'][1]['y']": {'new_value': 88, 'old_value': 8}
|
||||
},
|
||||
'distance_too_far': {
|
||||
"root['coordinates'][0]": {'l2_distance': 1.4142135623730951},
|
||||
"root['coordinates'][1]": {'l2_distance': 113.13708498984761}
|
||||
}
|
||||
}
|
||||
assert expected == ddiff
|
||||
|
||||
def test_custom_operators_should_not_equal(self):
|
||||
t1 = {
|
||||
"id": 5,
|
||||
"expect_change_pos": 10,
|
||||
"expect_change_neg": 10,
|
||||
}
|
||||
|
||||
t2 = {
|
||||
"id": 5,
|
||||
"expect_change_pos": 100,
|
||||
"expect_change_neg": 10,
|
||||
}
|
||||
|
||||
class ExpectChangeOperator(BaseOperator):
|
||||
def __init__(self, regex_paths):
|
||||
super().__init__(regex_paths)
|
||||
|
||||
def give_up_diffing(self, level, diff_instance) -> bool:
|
||||
if level.t1 == level.t2:
|
||||
diff_instance.custom_report_result('unexpected:still', level, {
|
||||
"old": level.t1,
|
||||
"new": level.t2
|
||||
})
|
||||
|
||||
return True
|
||||
|
||||
ddiff = DeepDiff(t1, t2, custom_operators=[
|
||||
ExpectChangeOperator(regex_paths=["root\\['expect_change.*'\\]"])
|
||||
])
|
||||
|
||||
assert ddiff == {'unexpected:still': {"root['expect_change_neg']": {'old': 10, 'new': 10}}}
|
||||
|
||||
def test_custom_operator2(self):
|
||||
|
||||
class CustomClass:
|
||||
|
||||
def __init__(self, d: dict, l: list):
|
||||
self.dict = d
|
||||
self.dict['list'] = l
|
||||
|
||||
def __repr__(self):
|
||||
return "Class list is " + str(self.dict['list'])
|
||||
|
||||
custom1 = CustomClass(d=dict(a=1, b=2), l=[1, 2, 3])
|
||||
custom2 = CustomClass(d=dict(c=3, d=4), l=[1, 2, 3, 2])
|
||||
custom3 = CustomClass(d=dict(a=1, b=2), l=[1, 2, 3, 4])
|
||||
|
||||
class ListMatchOperator(BaseOperator):
|
||||
|
||||
def give_up_diffing(self, level, diff_instance) -> bool:
|
||||
if set(level.t1.dict['list']) == set(level.t2.dict['list']):
|
||||
return True
|
||||
return False
|
||||
|
||||
ddiff = DeepDiff(custom1, custom2, custom_operators=[
|
||||
ListMatchOperator(types=[CustomClass])
|
||||
])
|
||||
|
||||
assert {} == ddiff
|
||||
|
||||
ddiff2 = DeepDiff(custom2, custom3, threshold_to_diff_deeper=0, custom_operators=[
|
||||
ListMatchOperator(types=[CustomClass])
|
||||
])
|
||||
|
||||
expected = {
|
||||
'dictionary_item_added': ["root.dict['a']", "root.dict['b']"],
|
||||
'dictionary_item_removed': ["root.dict['c']", "root.dict['d']"],
|
||||
'values_changed': {"root.dict['list'][3]": {'new_value': 4, 'old_value': 2}}}
|
||||
|
||||
assert expected == ddiff2
|
||||
|
||||
def test_include_only_certain_path(self):
|
||||
|
||||
class MyOperator:
|
||||
|
||||
def __init__(self, include_paths):
|
||||
self.include_paths = include_paths
|
||||
|
||||
def match(self, level) -> bool:
|
||||
return True
|
||||
|
||||
def give_up_diffing(self, level, diff_instance) -> bool:
|
||||
return level.path() not in self.include_paths
|
||||
|
||||
t1 = {'a': [10, 11], 'b': [20, 21], 'c': [30, 31]}
|
||||
t2 = {'a': [10, 22], 'b': [20, 33], 'c': [30, 44]}
|
||||
|
||||
ddiff = DeepDiff(t1, t2, custom_operators=[
|
||||
MyOperator(include_paths="root['a'][1]")
|
||||
])
|
||||
|
||||
expected = {'values_changed': {"root['a'][1]": {'new_value': 22, 'old_value': 11}}}
|
||||
assert expected == ddiff
|
||||
|
||||
def test_give_up_diffing_on_first_diff(self):
|
||||
|
||||
class MyOperator:
|
||||
|
||||
def match(self, level) -> bool:
|
||||
return True
|
||||
|
||||
def give_up_diffing(self, level, diff_instance) -> bool:
|
||||
return any(diff_instance.tree.values())
|
||||
|
||||
t1 = [[1, 2], [3, 4], [5, 6]]
|
||||
t2 = [[1, 3], [3, 5], [5, 7]]
|
||||
|
||||
ddiff = DeepDiff(t1, t2, custom_operators=[
|
||||
MyOperator()
|
||||
])
|
||||
|
||||
expected = {'values_changed': {'root[0][1]': {'new_value': 3, 'old_value': 2}}}
|
||||
assert expected == ddiff
|
||||
|
||||
def test_prefix_or_suffix_diff(self):
|
||||
|
||||
t1 = {
|
||||
"key1": ["foo", "bar's food", "jack", "joe"]
|
||||
}
|
||||
t2 = {
|
||||
"key1": ["foo", "bar", "jill", "joe'car"]
|
||||
}
|
||||
|
||||
ddiff = DeepDiff(t1, t2, custom_operators=[
|
||||
PrefixOrSuffixOperator()
|
||||
])
|
||||
|
||||
expected = {'values_changed': {"root['key1'][2]": {'new_value': 'jill', 'old_value': 'jack'}}}
|
||||
assert expected == ddiff
|
||||
|
||||
with pytest.raises(NotImplementedError) as exp:
|
||||
DeepDiff(t1, t2, ignore_order=True, custom_operators=[
|
||||
PrefixOrSuffixOperator()
|
||||
])
|
||||
expected2 = 'PrefixOrSuffixOperator needs to define a normalize_value_for_hashing method to be compatible with ignore_order=True or iterable_compare_func.'
|
||||
assert expected2 == str(exp.value)
|
||||
|
||||
def test_custom_operator3_small_numbers(self):
|
||||
x = [2.0000000000000027, 2.500000000000005, 2.000000000000002, 3.000000000000001]
|
||||
y = [2.000000000000003, 2.500000000000005, 2.0000000000000027, 3.0000000000000027]
|
||||
result = DeepDiff(x, y)
|
||||
expected = {
|
||||
'values_changed': {
|
||||
'root[0]': {'new_value': 2.000000000000003, 'old_value': 2.0000000000000027},
|
||||
'root[2]': {'new_value': 2.0000000000000027, 'old_value': 2.000000000000002},
|
||||
'root[3]': {'new_value': 3.0000000000000027, 'old_value': 3.000000000000001}}}
|
||||
assert expected == result
|
||||
|
||||
class CustomCompare(BaseOperatorPlus):
|
||||
def __init__(self, tolerance, types):
|
||||
self.tolerance = tolerance
|
||||
self.types = types
|
||||
|
||||
def match(self, level) -> bool:
|
||||
if type(level.t1) in self.types:
|
||||
return True
|
||||
return False
|
||||
|
||||
def give_up_diffing(self, level, diff_instance) -> bool:
|
||||
relative = abs(abs(level.t1 - level.t2) / level.t1)
|
||||
if not max(relative, self.tolerance) == self.tolerance:
|
||||
custom_report = f'relative diff: {relative:.8e}'
|
||||
diff_instance.custom_report_result('diff', level, custom_report)
|
||||
return True
|
||||
|
||||
def normalize_value_for_hashing(self, parent: Any, obj: Any) -> Any:
|
||||
return obj
|
||||
|
||||
|
||||
def compare_func(x, y, level):
|
||||
return True
|
||||
|
||||
operators = [CustomCompare(types=[float], tolerance=5.5e-5)]
|
||||
result2 = DeepDiff(x, y, custom_operators=operators, iterable_compare_func=compare_func)
|
||||
assert {} == result2
|
||||
|
||||
result3 = DeepDiff(x, y, custom_operators=operators, zip_ordered_iterables=True)
|
||||
assert {} == result3, "We should get the same result as result2 when zip_ordered_iterables is True."
|
||||
|
||||
def test_custom_operator_and_ignore_order1_using_base_operator_plus(self):
|
||||
|
||||
d1 = {
|
||||
"Name": "SUB_OBJECT_FILES",
|
||||
"Values": {
|
||||
"Value": [
|
||||
"{f254498b-b752-4f35-bef5-6f1844b61eb7}",
|
||||
"{7fb2a550-1849-45c0-b273-9aa5e4eb9f2b}",
|
||||
"{3a614c62-4252-48eb-b279-1450ee8af182}",
|
||||
"{208f22c4-c256-4311-9a45-e6c37d343458}",
|
||||
"{1fcf5d37-ef19-43a7-a1ad-d17c7c1713c6}",
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
d2 = {
|
||||
"Name": "SUB_OBJECT_FILES",
|
||||
"Values": {
|
||||
"Value": [
|
||||
"{e5d18917-1a2c-4abe-b601-8ec002629953}",
|
||||
"{ea71ba1f-1339-4fae-bc28-a9ce9b8a8c67}",
|
||||
"{66bb6192-9cd2-4074-8be1-f2ac52877c70}",
|
||||
"{0c88b900-3755-4d10-93ef-b6a96dbcba90}",
|
||||
"{e39fdfc5-be6c-4f97-9345-9a8286381fe7}"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class RemoveGUIDsOperator(BaseOperatorPlus):
|
||||
_pattern = r"[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"
|
||||
_substitute = "guid"
|
||||
|
||||
def match(self, level) -> bool:
|
||||
return isinstance(level.t1, str) and isinstance(level.t2, str)
|
||||
|
||||
@classmethod
|
||||
def _remove_pattern(cls, t: str):
|
||||
return re.sub(cls._pattern, cls._substitute, t)
|
||||
|
||||
def give_up_diffing(self, level, diff_instance):
|
||||
t1 = self._remove_pattern(level.t1)
|
||||
t2 = self._remove_pattern(level.t2)
|
||||
return t1 == t2
|
||||
|
||||
def normalize_value_for_hashing(self, parent: Any, obj: Any) -> Any:
|
||||
"""
|
||||
Used for ignore_order=True
|
||||
"""
|
||||
if isinstance(obj, str):
|
||||
return self._remove_pattern(obj)
|
||||
return obj
|
||||
|
||||
|
||||
operator = RemoveGUIDsOperator()
|
||||
|
||||
diff1 = DeepDiff(d1, d2, custom_operators=[operator], log_stacktrace=True)
|
||||
assert not diff1
|
||||
|
||||
|
||||
diff2 = DeepDiff(d1, d2, ignore_order=True, custom_operators=[operator], log_stacktrace=True)
|
||||
assert not diff2
|
||||
|
||||
|
||||
def test_custom_operator_and_ignore_order2(self):
|
||||
d1 = {
|
||||
"Entity": {
|
||||
"Property": {
|
||||
"Name": "SUB_OBJECT_FILES",
|
||||
"Values": {
|
||||
"Value": [
|
||||
"{f254498b-b752-4f35-bef5-6f1844b61eb7}",
|
||||
"{7fb2a550-1849-45c0-b273-9aa5e4eb9f2b}",
|
||||
"{3a614c62-4252-48eb-b279-1450ee8af182}",
|
||||
"{208f22c4-c256-4311-9a45-e6c37d343458}",
|
||||
"{1fcf5d37-ef19-43a7-a1ad-d17c7c1713c6}",
|
||||
"{a9cbecc0-21dc-49ce-8b2c-d36352dae139}"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
d2 = {
|
||||
"Entity": {
|
||||
"Property": {
|
||||
"Name": "SUB_OBJECT_FILES",
|
||||
"Values": {
|
||||
"Value": [
|
||||
"{e5d18917-1a2c-4abe-b601-8ec002629953}",
|
||||
"{ea71ba1f-1339-4fae-bc28-a9ce9b8a8c67}",
|
||||
"{d7778018-a7b5-4246-8caa-f590138d99e5}",
|
||||
"{66bb6192-9cd2-4074-8be1-f2ac52877c70}",
|
||||
"{0c88b900-3755-4d10-93ef-b6a96dbcba90}",
|
||||
"{e39fdfc5-be6c-4f97-9345-9a8286381fe7}"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class RemovePatternOperator(BaseOperator):
|
||||
_pattern: str = ""
|
||||
_substitute: str = ""
|
||||
|
||||
@classmethod
|
||||
def _remove_pattern(cls, t: str):
|
||||
return re.sub(cls._pattern, cls._substitute, t)
|
||||
|
||||
def give_up_diffing(self, level, diff_instance):
|
||||
if isinstance(level.t1, str) and isinstance(level.t2, str):
|
||||
t1 = self._remove_pattern(level.t1)
|
||||
t2 = self._remove_pattern(level.t2)
|
||||
return t1 == t2
|
||||
return False
|
||||
|
||||
class RemoveGUIDsOperator(RemovePatternOperator):
|
||||
_pattern = r"[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"
|
||||
_substitute = "guid"
|
||||
|
||||
diff1 = DeepDiff(deepcopy(d1), deepcopy(d2), ignore_order=False, custom_operators=[RemoveGUIDsOperator(types=[str])])
|
||||
assert not diff1
|
||||
|
||||
with pytest.raises(NotImplementedError) as exp:
|
||||
DeepDiff(deepcopy(d1), deepcopy(d2), ignore_order=True, custom_operators=[RemoveGUIDsOperator(types=[str])])
|
||||
expected2 = 'RemoveGUIDsOperator needs to define a normalize_value_for_hashing method to be compatible with ignore_order=True or iterable_compare_func.'
|
||||
assert expected2 == str(exp.value)
|
||||
|
||||
|
||||
# --------- Let's implement the normalize_value_for_hashing to make it work with ignore_order=True ---------
|
||||
|
||||
class RemoveGUIDsOperatorIgnoreOrderReady(RemoveGUIDsOperator):
|
||||
def normalize_value_for_hashing(self, parent: Any, obj: Any) -> Any:
|
||||
if isinstance(obj, str):
|
||||
return self._remove_pattern(obj)
|
||||
return obj
|
||||
|
||||
diff3 = DeepDiff(deepcopy(d1), deepcopy(d2), ignore_order=True, custom_operators=[RemoveGUIDsOperatorIgnoreOrderReady(types=[str])])
|
||||
assert not diff3, "We shouldn't have a diff because we have normalized the string values to be all the same vlues."
|
||||
|
||||
Reference in new issue
Block a user