diff --git a/deepdiff/deephash.py b/deepdiff/deephash.py new file mode 100644 index 0000000..e0d60da --- /dev/null +++ b/deepdiff/deephash.py @@ -0,0 +1,754 @@ +#!/usr/bin/env python +import logging +import datetime +import uuid +from typing import Union, Optional, Any, List, TYPE_CHECKING, Dict, Tuple, Set, Callable, Generator +from collections.abc import Iterable, MutableMapping +from collections import defaultdict +from hashlib import sha1, sha256 +from pathlib import Path +from enum import Enum +import re +from deepdiff.helper import (strings, numbers, only_numbers, times, unprocessed, not_hashed, add_to_frozen_set, + convert_item_or_items_into_set_else_none, get_doc, ipranges, + convert_item_or_items_into_compiled_regexes_else_none, + get_id, type_is_subclass_of_type_group, type_in_type_group, + number_to_string, datetime_normalize, KEY_TO_VAL_STR, + get_truncate_datetime, dict_, add_root_to_paths, PydanticBaseModel, + separate_wildcard_and_exact_paths, + SetOrdered) + +from deepdiff.base import Base + +if TYPE_CHECKING: + from pytz.tzinfo import BaseTzInfo + import numpy as np + +# Type aliases for better readability +HashableType = Union[str, int, float, bytes, bool, tuple, frozenset, type(None)] +HashResult = Union[str, Any] # Can be string hash or unprocessed marker +HashTuple = Tuple[HashResult, int] # (hash_result, count) +HashesDict = Dict[Any, Union[HashTuple, List[Any]]] # Special case for UNPROCESSED_KEY +PathType = Union[str, List[str], Set[str]] +RegexType = Union[str, re.Pattern[str], List[Union[str, re.Pattern[str]]]] +NumberToStringFunc = Callable[..., str] # More flexible for different number_to_string implementations + + +try: + import pandas +except ImportError: + pandas = False # type: ignore + +try: + import polars +except ImportError: + polars = False # type: ignore +try: + import numpy as np + booleanTypes: Tuple[type, ...] = (bool, np.bool_) # type: ignore +except ImportError: + booleanTypes = (bool,) # type: ignore + +logger: logging.Logger = logging.getLogger(__name__) + +UNPROCESSED_KEY: object = object() + +EMPTY_FROZENSET: frozenset = frozenset() + +INDEX_VS_ATTRIBUTE: Tuple[str, str] = ('[%s]', '.%s') + + +HASH_LOOKUP_ERR_MSG: str = '{} is not one of the hashed items.' + + +def sha256hex(obj: Union[str, bytes]) -> str: + """Use Sha256 as a cryptographic hash.""" + if isinstance(obj, str): + obj = obj.encode('utf-8') + return sha256(obj).hexdigest() + + +def sha1hex(obj: Union[str, bytes]) -> str: + """Use Sha1 as a cryptographic hash.""" + if isinstance(obj, str): + obj = obj.encode('utf-8') + return sha1(obj).hexdigest() + + +default_hasher: Callable[[Union[str, bytes]], str] = sha256hex + + +def combine_hashes_lists(items: List[List[str]], prefix: Union[str, bytes]) -> str: + """ + Combines lists of hashes into one hash + This can be optimized in future. + It needs to work with both murmur3 hashes (int) and sha256 (str) + Although murmur3 is not used anymore. + """ + if isinstance(prefix, bytes): + prefix = prefix.decode('utf-8') + hashes_bytes = b'' + for item in items: + # In order to make sure the order of hashes in each item does not affect the hash + # we resort them. + hashes_bytes += (''.join(map(str, sorted(item))) + '--').encode('utf-8') + return prefix + str(default_hasher(hashes_bytes)) + + +class BoolObj(Enum): + TRUE = 1 + FALSE = 0 + + +def prepare_string_for_hashing( + obj: Union[str, bytes, memoryview], + ignore_string_type_changes: bool = False, + ignore_string_case: bool = False, + encodings: Optional[List[str]] = None, + ignore_encoding_errors: bool = False, +) -> str: + """ + Clean type conversions + """ + original_type = obj.__class__.__name__ + # https://docs.python.org/3/library/codecs.html#codecs.decode + errors_mode = 'ignore' if ignore_encoding_errors else 'strict' + if isinstance(obj, memoryview): + obj = obj.tobytes() + if isinstance(obj, bytes): + err = None + encodings = ['utf-8'] if encodings is None else encodings + encoded = False + for encoding in encodings: + try: + obj = obj.decode(encoding, errors=errors_mode) + encoded = True + break + except UnicodeDecodeError as er: + err = er + if not encoded and err is not None: + obj_decoded = obj.decode('utf-8', errors='ignore') # type: ignore + start = max(err.start - 20, 0) + start_prefix = '' + if start > 0: + start_prefix = '...' + end = err.end + 20 + end_suffix = '...' + if end >= len(obj): + end = len(obj) + end_suffix = '' + raise UnicodeDecodeError( + err.encoding, + err.object, + err.start, + err.end, + f"{err.reason} in '{start_prefix}{obj_decoded[start:end]}{end_suffix}'. Please either pass ignore_encoding_errors=True or pass the encoding via encodings=['utf-8', '...']." + ) from None + if not ignore_string_type_changes: + obj = KEY_TO_VAL_STR.format(original_type, obj) + if ignore_string_case: + obj = obj.lower() + return str(obj) + + +doc = get_doc('deephash_doc.rst') + + +class DeepHash(Base): + __doc__ = doc + + # Class attributes + hashes: Dict[Any, Any] + exclude_types_tuple: Tuple[type, ...] + ignore_repetition: bool + exclude_paths: Optional[SetOrdered] + include_paths: Optional[SetOrdered] + exclude_regex_paths: Optional[List[re.Pattern[str]]] + hasher: Callable[[Union[str, bytes]], str] + use_enum_value: bool + default_timezone: Union[datetime.timezone, "BaseTzInfo"] + significant_digits: Optional[int] + truncate_datetime: Optional[str] + number_format_notation: str + ignore_type_in_groups: Any + ignore_string_type_changes: bool + ignore_numeric_type_changes: bool + ignore_string_case: bool + exclude_obj_callback: Optional[Callable[[Any, str], bool]] + apply_hash: bool + type_check_func: Callable[[type, Any], bool] + number_to_string: Any + ignore_private_variables: bool + encodings: Optional[List[str]] + ignore_encoding_errors: bool + ignore_iterable_order: bool + custom_operators: Optional[List[Any]] + + def __init__(self, + obj: Any, + *, + apply_hash: bool = True, + custom_operators: Optional[List[Any]] = None, + default_timezone: Union[datetime.timezone, "BaseTzInfo"] = datetime.timezone.utc, + encodings: Optional[List[str]] = None, + exclude_glob_paths: Optional[List[Any]] = None, + exclude_obj_callback: Optional[Callable[[Any, str], bool]] = None, + exclude_paths: Optional[PathType] = None, + exclude_regex_paths: Optional[RegexType] = None, + exclude_types: Optional[Union[List[type], Set[type], Tuple[type, ...]]] = None, + hasher: Optional[Callable[[Union[str, bytes]], str]] = None, + hashes: Optional[Union[Dict[Any, Any], "DeepHash"]] = None, + ignore_encoding_errors: bool = False, + ignore_iterable_order: bool = True, + ignore_numeric_type_changes: bool = False, + ignore_private_variables: bool = True, + ignore_repetition: bool = True, + ignore_string_case: bool = False, + ignore_string_type_changes: bool = False, + ignore_type_in_groups: Any = None, + ignore_type_subclasses: bool = False, + ignore_uuid_types: bool = False, + include_glob_paths: Optional[List[Any]] = None, + include_paths: Optional[PathType] = None, + number_format_notation: str = "f", + number_to_string_func: Optional[NumberToStringFunc] = None, + parent: str = "root", + significant_digits: Optional[int] = None, + truncate_datetime: Optional[str] = None, + use_enum_value: bool = False, + **kwargs) -> None: + if kwargs: + raise ValueError( + ("The following parameter(s) are not valid: %s\n" + "The valid parameters are obj, hashes, exclude_types, significant_digits, truncate_datetime," + "exclude_paths, include_paths, exclude_regex_paths, hasher, ignore_repetition, " + "number_format_notation, apply_hash, ignore_type_in_groups, ignore_string_type_changes, " + "ignore_numeric_type_changes, ignore_type_subclasses, ignore_string_case, ignore_uuid_types, " + "number_to_string_func, ignore_private_variables, parent, use_enum_value, default_timezone " + "encodings, ignore_encoding_errors") % ', '.join(kwargs.keys())) + if isinstance(hashes, MutableMapping): + self.hashes = hashes + elif isinstance(hashes, DeepHash): + self.hashes = hashes.hashes + else: + self.hashes = dict_() + exclude_types = set() if exclude_types is None else set(exclude_types) + self.exclude_types_tuple = tuple(exclude_types) # we need tuple for checking isinstance + self.ignore_repetition = ignore_repetition + _exclude_set = convert_item_or_items_into_set_else_none(exclude_paths) + _exclude_exact, _exclude_globs = separate_wildcard_and_exact_paths(_exclude_set) + self.exclude_paths = add_root_to_paths(_exclude_exact) + self.exclude_glob_paths = exclude_glob_paths or _exclude_globs + _include_set = convert_item_or_items_into_set_else_none(include_paths) + _include_exact, _include_globs = separate_wildcard_and_exact_paths(_include_set) + self.include_paths = add_root_to_paths(_include_exact) + self.include_glob_paths = include_glob_paths or _include_globs + self.exclude_regex_paths = convert_item_or_items_into_compiled_regexes_else_none(exclude_regex_paths) + self.hasher = default_hasher if hasher is None else hasher + self.hashes[UNPROCESSED_KEY] = [] # type: ignore + self.use_enum_value = use_enum_value + self.default_timezone = default_timezone + self.significant_digits = self.get_significant_digits(significant_digits, ignore_numeric_type_changes) + self.truncate_datetime = get_truncate_datetime(truncate_datetime) + self.number_format_notation = number_format_notation + self.ignore_type_in_groups = self.get_ignore_types_in_groups( + ignore_type_in_groups=ignore_type_in_groups, + ignore_string_type_changes=ignore_string_type_changes, + ignore_numeric_type_changes=ignore_numeric_type_changes, + ignore_type_subclasses=ignore_type_subclasses, + ignore_uuid_types=ignore_uuid_types, + ) + self.ignore_string_type_changes = ignore_string_type_changes + self.ignore_numeric_type_changes = ignore_numeric_type_changes + self.ignore_string_case = ignore_string_case + self.exclude_obj_callback = exclude_obj_callback + # makes the hash return constant size result if true + # the only time it should be set to False is when + # testing the individual hash functions for different types of objects. + self.apply_hash = apply_hash + self.type_check_func = type_in_type_group if ignore_type_subclasses else type_is_subclass_of_type_group + # self.type_check_func = type_is_subclass_of_type_group if ignore_type_subclasses else type_in_type_group + self.number_to_string = number_to_string_func or number_to_string + self.ignore_private_variables = ignore_private_variables + self.encodings = encodings + self.ignore_encoding_errors = ignore_encoding_errors + self.ignore_iterable_order = ignore_iterable_order + self.custom_operators = custom_operators + + self._hash(obj, parent=parent, parents_ids=frozenset({get_id(obj)})) + + if self.hashes[UNPROCESSED_KEY]: + logger.warning("Can not hash the following items: {}.".format(self.hashes[UNPROCESSED_KEY])) + else: + del self.hashes[UNPROCESSED_KEY] + + sha256hex: Callable[[Union[str, bytes]], str] = sha256hex + sha1hex: Callable[[Union[str, bytes]], str] = sha1hex + + def __getitem__(self, obj: Any, extract_index: Optional[int] = 0) -> Any: + return self._getitem(self.hashes, obj, extract_index=extract_index, + use_enum_value=self.use_enum_value, + ignore_numeric_type_changes=self.ignore_numeric_type_changes) + + @staticmethod + def _get_slots_dict(obj: Any) -> Dict[str, Any]: + """Get a dict of initialized slot attributes. + + Uses object.__getattribute__ to check each slot directly, bypassing + __getattr__. For uninitialized slots on classes that define __getattr__, + falls back to getattr — letting it raise if the object is truly broken. + """ + result = {} + has_getattr = hasattr(type(obj), '__getattr__') + for slot in obj.__slots__: + try: + result[slot] = object.__getattribute__(obj, slot) + except AttributeError: + if has_getattr: + # The slot isn't initialized, but the class defines __getattr__. + # Try the normal getattr to let __getattr__ provide a value or + # raise — if it raises, we propagate to fail the strategy. + result[slot] = getattr(obj, slot) + return result + + @staticmethod + def _getitem(hashes: Dict[Any, Any], obj: Any, extract_index: Optional[int] = 0, + use_enum_value: bool = False, ignore_numeric_type_changes: bool = False) -> Any: + """ + extract_index is zero for hash and 1 for count and None to get them both. + To keep it backward compatible, we only get the hash by default so it is set to zero by default. + """ + + key = obj + if obj is True: + key = BoolObj.TRUE + elif obj is False: + key = BoolObj.FALSE + elif use_enum_value and isinstance(obj, Enum): + key = obj.value + key = DeepHash._make_hash_key_for_lookup(key, ignore_numeric_type_changes=ignore_numeric_type_changes) + + result_n_count: Tuple[Any, int] = (None, 0) # type: ignore + + try: + result_n_count = hashes[key] + except (TypeError, KeyError): + key = get_id(obj) + try: + result_n_count = hashes[key] + except KeyError: + raise KeyError(HASH_LOOKUP_ERR_MSG.format(obj)) from None + + if obj is UNPROCESSED_KEY: + extract_index = None + + return result_n_count if extract_index is None else result_n_count[extract_index] + + def __contains__(self, obj: Any) -> bool: + key = self._make_hash_key(obj) + result = False + try: + result = key in self.hashes + except (TypeError, KeyError): + result = False + if not result: + result = get_id(obj) in self.hashes + return result + + def get(self, key: Any, default: Any = None, extract_index: Optional[int] = 0) -> Any: + """ + Get method for the hashes dictionary. + It can extract the hash for a given key that is already calculated when extract_index=0 + or the count of items that went to building the object when extract_index=1. + """ + return self.get_key(self.hashes, key, default=default, extract_index=extract_index, + ignore_numeric_type_changes=self.ignore_numeric_type_changes) + + @staticmethod + def get_key(hashes: Dict[Any, Any], key: Any, default: Any = None, extract_index: Optional[int] = 0, + use_enum_value: bool = False, ignore_numeric_type_changes: bool = False) -> Any: + """ + get_key method for the hashes dictionary. + It can extract the hash for a given key that is already calculated when extract_index=0 + or the count of items that went to building the object when extract_index=1. + """ + try: + result = DeepHash._getitem(hashes, key, extract_index=extract_index, + use_enum_value=use_enum_value, + ignore_numeric_type_changes=ignore_numeric_type_changes) + except KeyError: + result = default + return result + + @staticmethod + def _unwrap_hash_key(key: Any) -> Any: + """Unwrap a (type, value) hash key back to the original value for public API.""" + if isinstance(key, tuple) and len(key) == 2 and isinstance(key[0], type) and isinstance(key[1], only_numbers): + return key[1] + return key + + def _get_objects_to_hashes_dict(self, extract_index: Optional[int] = 0) -> Dict[Any, Any]: + """ + A dictionary containing only the objects to hashes, + or a dictionary of objects to the count of items that went to build them. + extract_index=0 for hashes and extract_index=1 for counts. + """ + result = dict_() + for key, value in self.hashes.items(): + key = self._unwrap_hash_key(key) + if key is UNPROCESSED_KEY: + result[key] = value + else: + result[key] = value[extract_index] + return result + + def __eq__(self, other: Any) -> bool: + if isinstance(other, DeepHash): + return self.hashes == other.hashes + else: + # We only care about the hashes + return self._get_objects_to_hashes_dict() == other + + __req__ = __eq__ + + def __repr__(self) -> str: + """ + Hide the counts since it will be confusing to see them when they are hidden everywhere else. + """ + from deepdiff.summarize import summarize + return summarize(self._get_objects_to_hashes_dict(extract_index=0), max_length=500) + + def __str__(self) -> str: + return str(self._get_objects_to_hashes_dict(extract_index=0)) + + def __bool__(self) -> bool: + return bool(self.hashes) + + def keys(self) -> Any: + return [self._unwrap_hash_key(k) for k in self.hashes.keys()] + + def values(self) -> Generator[Any, None, None]: + return (i[0] for i in self.hashes.values()) # Just grab the item and not its count + + def items(self) -> Generator[Tuple[Any, Any], None, None]: + return ((self._unwrap_hash_key(i), v[0]) for i, v in self.hashes.items()) + + def _prep_obj(self, obj: Any, parent: str, parents_ids: frozenset = EMPTY_FROZENSET, is_namedtuple: bool = False, is_pydantic_object: bool = False) -> HashTuple: + """prepping objects""" + original_type = type(obj) if not isinstance(obj, type) else obj + + obj_to_dict_strategies = [] + if is_namedtuple: + obj_to_dict_strategies.append(lambda o: o._asdict()) + elif is_pydantic_object: + obj_to_dict_strategies.append(lambda o: {k: v for (k, v) in o.__dict__.items() if v !="model_fields_set"}) + else: + obj_to_dict_strategies.append(lambda o: o.__dict__) + + if hasattr(obj, "__slots__"): + obj_to_dict_strategies.append(lambda o: DeepHash._get_slots_dict(o)) + else: + import inspect + obj_to_dict_strategies.append(lambda o: dict(inspect.getmembers(o, lambda m: not inspect.isroutine(m)))) + + for get_dict in obj_to_dict_strategies: + try: + d = get_dict(obj) + break + except AttributeError: + pass + else: + self.hashes[UNPROCESSED_KEY].append(obj) # type: ignore + return (unprocessed, 0) + obj = d + + result, counts = self._prep_dict(obj, parent=parent, parents_ids=parents_ids, + print_as_attribute=True, original_type=original_type) + result = "nt{}".format(result) if is_namedtuple else "obj{}".format(result) + return result, counts + + def _skip_this(self, obj: Any, parent: str) -> bool: + skip = False + if self.exclude_paths and parent in self.exclude_paths: + skip = True + elif self.exclude_glob_paths and any(gp.match(parent) for gp in self.exclude_glob_paths): + skip = True + if (self.include_paths or self.include_glob_paths) and parent != 'root': + skip = True + if self.include_paths: + if parent in self.include_paths: + skip = False + else: + for prefix in self.include_paths: + if parent.startswith(prefix): + skip = False + break + if skip and self.include_glob_paths: + for gp in self.include_glob_paths: + if gp.match_or_is_ancestor(parent): + skip = False + break + elif self.exclude_regex_paths and any( + [exclude_regex_path.search(parent) for exclude_regex_path in self.exclude_regex_paths]): # type: ignore + skip = True + elif self.exclude_types_tuple and isinstance(obj, self.exclude_types_tuple): + skip = True + elif self.exclude_obj_callback and self.exclude_obj_callback(obj, parent): + skip = True + return skip + + def _prep_dict(self, obj: Union[Dict[Any, Any], MutableMapping], parent: str, parents_ids: frozenset = EMPTY_FROZENSET, print_as_attribute: bool = False, original_type: Optional[type] = None) -> HashTuple: + + result = [] + counts = 1 + + key_text = "%s{}".format(INDEX_VS_ATTRIBUTE[print_as_attribute]) + for key, item in obj.items(): + counts += 1 + # ignore private variables + if self.ignore_private_variables and isinstance(key, str) and key.startswith('__'): + continue + key_formatted = "'%s'" % key if not print_as_attribute and isinstance(key, strings) else key + key_in_report = key_text % (parent, key_formatted) + + key_hash, _ = self._hash(key, parent=key_in_report, parents_ids=parents_ids) + if not key_hash: + continue + item_id = get_id(item) + if (parents_ids and item_id in parents_ids) or self._skip_this(item, parent=key_in_report): + continue + parents_ids_added = add_to_frozen_set(parents_ids, item_id) + hashed, count = self._hash(item, parent=key_in_report, parents_ids=parents_ids_added) + hashed = KEY_TO_VAL_STR.format(key_hash, hashed) + result.append(hashed) + counts += count + + result.sort() + result = ';'.join(result) + if print_as_attribute: + type_ = original_type or type(obj) + type_str = type_.__name__ + for type_group in self.ignore_type_in_groups: + if self.type_check_func(type_, type_group): + type_str = ','.join(map(lambda x: x.__name__, type_group)) + break + else: + type_str = 'dict' + return "{}:{{{}}}".format(type_str, result), counts + + def _prep_iterable(self, obj: Iterable[Any], parent: str, parents_ids: frozenset = EMPTY_FROZENSET) -> HashTuple: + + counts = 1 + result = defaultdict(int) + + for i, item in enumerate(obj): + new_parent = "{}[{}]".format(parent, i) + if self._skip_this(item, parent=new_parent): + continue + + item_id = get_id(item) + if parents_ids and item_id in parents_ids: + continue + + parents_ids_added = add_to_frozen_set(parents_ids, item_id) + hashed, count = self._hash(item, parent=new_parent, parents_ids=parents_ids_added) + # counting repetitions + result[hashed] += 1 + counts += count + + if self.ignore_repetition: + result = list(result.keys()) + else: + result = [ + '{}|{}'.format(i, v) for i, v in result.items() + ] + + result = map(str, result) # making sure the result items are string so join command works. + if self.ignore_iterable_order: + result = sorted(result) + result = ','.join(result) + result = KEY_TO_VAL_STR.format(type(obj).__name__, result) + + return result, counts + + def _prep_bool(self, obj: bool) -> BoolObj: + return BoolObj.TRUE if obj else BoolObj.FALSE + + + def _prep_path(self, obj: Path) -> str: + type_ = obj.__class__.__name__ + return KEY_TO_VAL_STR.format(type_, obj) + + def _prep_number(self, obj: Union[int, float, complex]) -> str: + type_ = "number" if self.ignore_numeric_type_changes else obj.__class__.__name__ + if self.significant_digits is not None: + obj = self.number_to_string(obj, significant_digits=self.significant_digits, + number_format_notation=self.number_format_notation) # type: ignore + return KEY_TO_VAL_STR.format(type_, obj) + + def _prep_ipranges(self, obj) -> str: + type_ = 'iprange' + obj = str(obj) + return KEY_TO_VAL_STR.format(type_, obj) + + def _prep_datetime(self, obj: datetime.datetime) -> str: + type_ = 'datetime' + obj = datetime_normalize(self.truncate_datetime, obj, default_timezone=self.default_timezone) + return KEY_TO_VAL_STR.format(type_, obj) + + def _prep_date(self, obj: datetime.date) -> str: + type_ = 'datetime' # yes still datetime but it doesn't need normalization + return KEY_TO_VAL_STR.format(type_, obj) + + def _prep_tuple(self, obj: tuple, parent: str, parents_ids: frozenset) -> HashTuple: + # Checking to see if it has _fields. Which probably means it is a named + # tuple. + try: + obj._asdict # type: ignore + # It must be a normal tuple + except AttributeError: + result, counts = self._prep_iterable(obj=obj, parent=parent, parents_ids=parents_ids) + # We assume it is a namedtuple then + else: + result, counts = self._prep_obj(obj, parent, parents_ids=parents_ids, is_namedtuple=True) + return result, counts + + def _make_hash_key(self, obj: Any) -> Any: + """ + Create a key for the hashes dict that distinguishes numeric types. + + In Python, 1 == 1.0 and hash(1) == hash(1.0), so int and float values + collide as dict keys. When ignore_numeric_type_changes is False, we wrap + numeric objects as (type, value) tuples so that each type gets its own + cache entry and its own hash. + """ + if not self.ignore_numeric_type_changes and isinstance(obj, only_numbers): + return (type(obj), obj) + return obj + + @staticmethod + def _make_hash_key_for_lookup(obj: Any, ignore_numeric_type_changes: bool = False) -> Any: + """Static version of _make_hash_key for use in static accessor methods.""" + if not ignore_numeric_type_changes and isinstance(obj, only_numbers): + return (type(obj), obj) + return obj + + def _hash(self, obj: Any, parent: str, parents_ids: frozenset = EMPTY_FROZENSET) -> HashTuple: + """The main hash method""" + counts = 1 + if self.custom_operators is not None: + for operator in self.custom_operators: + func = getattr(operator, 'normalize_value_for_hashing', None) + if func is None: + raise NotImplementedError(f"{operator.__class__.__name__} needs to define a normalize_value_for_hashing method to be compatible with ignore_order=True or iterable_compare_func.".format(operator)) + else: + obj = func(parent, obj) + + if isinstance(obj, booleanTypes): + obj = self._prep_bool(obj) + result = None + elif self.use_enum_value and isinstance(obj, Enum): + obj = obj.value + else: + result = not_hashed + hash_key = self._make_hash_key(obj) + try: + result, counts = self.hashes[hash_key] + except (TypeError, KeyError): + pass + else: + return result, counts + + if self._skip_this(obj, parent): + return None, 0 + + elif obj is None: + result = 'NONE' + + elif isinstance(obj, strings): + result = prepare_string_for_hashing( + obj, + ignore_string_type_changes=self.ignore_string_type_changes, + ignore_string_case=self.ignore_string_case, + encodings=self.encodings, + ignore_encoding_errors=self.ignore_encoding_errors, + ) + + elif isinstance(obj, Path): + result = self._prep_path(obj) + + elif isinstance(obj, times): + result = self._prep_datetime(obj) # type: ignore + + elif isinstance(obj, datetime.date): + result = self._prep_date(obj) + + elif isinstance(obj, numbers): # type: ignore + result = self._prep_number(obj) + + elif isinstance(obj, ipranges): + result = self._prep_ipranges(obj) + + elif isinstance(obj, uuid.UUID): + # Handle UUID objects (including uuid6.UUID) by using their integer value + result = str(obj.int) + + elif isinstance(obj, MutableMapping): + result, counts = self._prep_dict(obj=obj, parent=parent, parents_ids=parents_ids) + + elif isinstance(obj, tuple): + result, counts = self._prep_tuple(obj=obj, parent=parent, parents_ids=parents_ids) + + elif (pandas and isinstance(obj, pandas.DataFrame)): # type: ignore + def gen(): # type: ignore + yield ('dtype', obj.dtypes) # type: ignore + yield ('index', obj.index) # type: ignore + yield from obj.items() # type: ignore # which contains (column name, series tuples) + result, counts = self._prep_iterable(obj=gen(), parent=parent, parents_ids=parents_ids) + elif (polars and isinstance(obj, polars.DataFrame)): # type: ignore + def gen(): + yield from obj.columns # type: ignore + yield from list(obj.schema.items()) # type: ignore + yield from obj.rows() # type: ignore + result, counts = self._prep_iterable(obj=gen(), parent=parent, parents_ids=parents_ids) + + elif isinstance(obj, Iterable): + result, counts = self._prep_iterable(obj=obj, parent=parent, parents_ids=parents_ids) + + elif obj == BoolObj.TRUE or obj == BoolObj.FALSE: + result = 'bool:true' if obj is BoolObj.TRUE else 'bool:false' + elif isinstance(obj, PydanticBaseModel): + result, counts = self._prep_obj(obj=obj, parent=parent, parents_ids=parents_ids, is_pydantic_object=True) + else: + result, counts = self._prep_obj(obj=obj, parent=parent, parents_ids=parents_ids) + + if result is not_hashed: # pragma: no cover + self.hashes[UNPROCESSED_KEY].append(obj) # type: ignore + + elif result is unprocessed: + pass + + elif self.apply_hash: + if isinstance(obj, strings): + result_cleaned = result + else: + result_cleaned = prepare_string_for_hashing( + str(result), ignore_string_type_changes=self.ignore_string_type_changes, + ignore_string_case=self.ignore_string_case) + result = self.hasher(result_cleaned) + + # It is important to keep the hash of all objects. + # The hashes will be later used for comparing the objects. + # Object to hash when possible otherwise ObjectID to hash + try: + self.hashes[hash_key] = (result, counts) + except TypeError: + obj_id = get_id(obj) + self.hashes[obj_id] = (result, counts) + + return result, counts + + +if __name__ == "__main__": # pragma: no cover + import doctest + doctest.testmod() diff --git a/tests/test_operators.py b/tests/test_operators.py new file mode 100644 index 0000000..9844468 --- /dev/null +++ b/tests/test_operators.py @@ -0,0 +1,428 @@ +import re +import math +import pytest +from copy import deepcopy +from typing import List, Any +from deepdiff import DeepDiff +from deepdiff.operator import BaseOperator, PrefixOrSuffixOperator, BaseOperatorPlus + + +class TestOperators: + def test_custom_operators_prevent_default(self): + t1 = { + "coordinates": [ + {"x": 5, "y": 5}, + {"x": 8, "y": 8} + ] + } + + t2 = { + "coordinates": [ + {"x": 6, "y": 6}, + {"x": 88, "y": 88} + ] + } + + class L2DistanceDifferWithPreventDefault(BaseOperator): + def __init__(self, regex_paths: List[str], distance_threshold: float): + super().__init__(regex_paths) + self.distance_threshold = distance_threshold + + def _l2_distance(self, c1, c2): + return math.sqrt( + (c1["x"] - c2["x"]) ** 2 + (c1["y"] - c2["y"]) ** 2 + ) + + def give_up_diffing(self, level, diff_instance) -> bool: + l2_distance = self._l2_distance(level.t1, level.t2) + if l2_distance > self.distance_threshold: + diff_instance.custom_report_result('distance_too_far', level, { + "l2_distance": l2_distance + }) + return True + + ddiff = DeepDiff(t1, t2, custom_operators=[L2DistanceDifferWithPreventDefault( + ["^root\\['coordinates'\\]\\[\\d+\\]$"], + 1 + )]) + + expected = { + 'distance_too_far': { + "root['coordinates'][0]": {'l2_distance': 1.4142135623730951}, + "root['coordinates'][1]": {'l2_distance': 113.13708498984761} + } + } + assert expected == ddiff + + def test_custom_operators_not_prevent_default(self): + t1 = { + "coordinates": [ + {"x": 5, "y": 5}, + {"x": 8, "y": 8} + ] + } + + t2 = { + "coordinates": [ + {"x": 6, "y": 6}, + {"x": 88, "y": 88} + ] + } + + class L2DistanceDifferWithPreventDefault(BaseOperator): + def __init__(self, regex_paths, distance_threshold): + super().__init__(regex_paths) + self.distance_threshold = distance_threshold + + def _l2_distance(self, c1, c2): + return math.sqrt( + (c1["x"] - c2["x"]) ** 2 + (c1["y"] - c2["y"]) ** 2 + ) + + def give_up_diffing(self, level, diff_instance) -> bool: + l2_distance = self._l2_distance(level.t1, level.t2) + if l2_distance > self.distance_threshold: + diff_instance.custom_report_result('distance_too_far', level, { + "l2_distance": l2_distance + }) + # + return False + + ddiff = DeepDiff(t1, t2, custom_operators=[L2DistanceDifferWithPreventDefault( + ["^root\\['coordinates'\\]\\[\\d+\\]$"], + 1 + ) + ]) + expected = { + 'values_changed': { + "root['coordinates'][0]['x']": {'new_value': 6, 'old_value': 5}, + "root['coordinates'][0]['y']": {'new_value': 6, 'old_value': 5}, + "root['coordinates'][1]['x']": {'new_value': 88, 'old_value': 8}, + "root['coordinates'][1]['y']": {'new_value': 88, 'old_value': 8} + }, + 'distance_too_far': { + "root['coordinates'][0]": {'l2_distance': 1.4142135623730951}, + "root['coordinates'][1]": {'l2_distance': 113.13708498984761} + } + } + assert expected == ddiff + + def test_custom_operators_should_not_equal(self): + t1 = { + "id": 5, + "expect_change_pos": 10, + "expect_change_neg": 10, + } + + t2 = { + "id": 5, + "expect_change_pos": 100, + "expect_change_neg": 10, + } + + class ExpectChangeOperator(BaseOperator): + def __init__(self, regex_paths): + super().__init__(regex_paths) + + def give_up_diffing(self, level, diff_instance) -> bool: + if level.t1 == level.t2: + diff_instance.custom_report_result('unexpected:still', level, { + "old": level.t1, + "new": level.t2 + }) + + return True + + ddiff = DeepDiff(t1, t2, custom_operators=[ + ExpectChangeOperator(regex_paths=["root\\['expect_change.*'\\]"]) + ]) + + assert ddiff == {'unexpected:still': {"root['expect_change_neg']": {'old': 10, 'new': 10}}} + + def test_custom_operator2(self): + + class CustomClass: + + def __init__(self, d: dict, l: list): + self.dict = d + self.dict['list'] = l + + def __repr__(self): + return "Class list is " + str(self.dict['list']) + + custom1 = CustomClass(d=dict(a=1, b=2), l=[1, 2, 3]) + custom2 = CustomClass(d=dict(c=3, d=4), l=[1, 2, 3, 2]) + custom3 = CustomClass(d=dict(a=1, b=2), l=[1, 2, 3, 4]) + + class ListMatchOperator(BaseOperator): + + def give_up_diffing(self, level, diff_instance) -> bool: + if set(level.t1.dict['list']) == set(level.t2.dict['list']): + return True + return False + + ddiff = DeepDiff(custom1, custom2, custom_operators=[ + ListMatchOperator(types=[CustomClass]) + ]) + + assert {} == ddiff + + ddiff2 = DeepDiff(custom2, custom3, threshold_to_diff_deeper=0, custom_operators=[ + ListMatchOperator(types=[CustomClass]) + ]) + + expected = { + 'dictionary_item_added': ["root.dict['a']", "root.dict['b']"], + 'dictionary_item_removed': ["root.dict['c']", "root.dict['d']"], + 'values_changed': {"root.dict['list'][3]": {'new_value': 4, 'old_value': 2}}} + + assert expected == ddiff2 + + def test_include_only_certain_path(self): + + class MyOperator: + + def __init__(self, include_paths): + self.include_paths = include_paths + + def match(self, level) -> bool: + return True + + def give_up_diffing(self, level, diff_instance) -> bool: + return level.path() not in self.include_paths + + t1 = {'a': [10, 11], 'b': [20, 21], 'c': [30, 31]} + t2 = {'a': [10, 22], 'b': [20, 33], 'c': [30, 44]} + + ddiff = DeepDiff(t1, t2, custom_operators=[ + MyOperator(include_paths="root['a'][1]") + ]) + + expected = {'values_changed': {"root['a'][1]": {'new_value': 22, 'old_value': 11}}} + assert expected == ddiff + + def test_give_up_diffing_on_first_diff(self): + + class MyOperator: + + def match(self, level) -> bool: + return True + + def give_up_diffing(self, level, diff_instance) -> bool: + return any(diff_instance.tree.values()) + + t1 = [[1, 2], [3, 4], [5, 6]] + t2 = [[1, 3], [3, 5], [5, 7]] + + ddiff = DeepDiff(t1, t2, custom_operators=[ + MyOperator() + ]) + + expected = {'values_changed': {'root[0][1]': {'new_value': 3, 'old_value': 2}}} + assert expected == ddiff + + def test_prefix_or_suffix_diff(self): + + t1 = { + "key1": ["foo", "bar's food", "jack", "joe"] + } + t2 = { + "key1": ["foo", "bar", "jill", "joe'car"] + } + + ddiff = DeepDiff(t1, t2, custom_operators=[ + PrefixOrSuffixOperator() + ]) + + expected = {'values_changed': {"root['key1'][2]": {'new_value': 'jill', 'old_value': 'jack'}}} + assert expected == ddiff + + with pytest.raises(NotImplementedError) as exp: + DeepDiff(t1, t2, ignore_order=True, custom_operators=[ + PrefixOrSuffixOperator() + ]) + expected2 = 'PrefixOrSuffixOperator needs to define a normalize_value_for_hashing method to be compatible with ignore_order=True or iterable_compare_func.' + assert expected2 == str(exp.value) + + def test_custom_operator3_small_numbers(self): + x = [2.0000000000000027, 2.500000000000005, 2.000000000000002, 3.000000000000001] + y = [2.000000000000003, 2.500000000000005, 2.0000000000000027, 3.0000000000000027] + result = DeepDiff(x, y) + expected = { + 'values_changed': { + 'root[0]': {'new_value': 2.000000000000003, 'old_value': 2.0000000000000027}, + 'root[2]': {'new_value': 2.0000000000000027, 'old_value': 2.000000000000002}, + 'root[3]': {'new_value': 3.0000000000000027, 'old_value': 3.000000000000001}}} + assert expected == result + + class CustomCompare(BaseOperatorPlus): + def __init__(self, tolerance, types): + self.tolerance = tolerance + self.types = types + + def match(self, level) -> bool: + if type(level.t1) in self.types: + return True + return False + + def give_up_diffing(self, level, diff_instance) -> bool: + relative = abs(abs(level.t1 - level.t2) / level.t1) + if not max(relative, self.tolerance) == self.tolerance: + custom_report = f'relative diff: {relative:.8e}' + diff_instance.custom_report_result('diff', level, custom_report) + return True + + def normalize_value_for_hashing(self, parent: Any, obj: Any) -> Any: + return obj + + + def compare_func(x, y, level): + return True + + operators = [CustomCompare(types=[float], tolerance=5.5e-5)] + result2 = DeepDiff(x, y, custom_operators=operators, iterable_compare_func=compare_func) + assert {} == result2 + + result3 = DeepDiff(x, y, custom_operators=operators, zip_ordered_iterables=True) + assert {} == result3, "We should get the same result as result2 when zip_ordered_iterables is True." + + def test_custom_operator_and_ignore_order1_using_base_operator_plus(self): + + d1 = { + "Name": "SUB_OBJECT_FILES", + "Values": { + "Value": [ + "{f254498b-b752-4f35-bef5-6f1844b61eb7}", + "{7fb2a550-1849-45c0-b273-9aa5e4eb9f2b}", + "{3a614c62-4252-48eb-b279-1450ee8af182}", + "{208f22c4-c256-4311-9a45-e6c37d343458}", + "{1fcf5d37-ef19-43a7-a1ad-d17c7c1713c6}", + ] + } + } + + d2 = { + "Name": "SUB_OBJECT_FILES", + "Values": { + "Value": [ + "{e5d18917-1a2c-4abe-b601-8ec002629953}", + "{ea71ba1f-1339-4fae-bc28-a9ce9b8a8c67}", + "{66bb6192-9cd2-4074-8be1-f2ac52877c70}", + "{0c88b900-3755-4d10-93ef-b6a96dbcba90}", + "{e39fdfc5-be6c-4f97-9345-9a8286381fe7}" + ] + } + } + + + class RemoveGUIDsOperator(BaseOperatorPlus): + _pattern = r"[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}" + _substitute = "guid" + + def match(self, level) -> bool: + return isinstance(level.t1, str) and isinstance(level.t2, str) + + @classmethod + def _remove_pattern(cls, t: str): + return re.sub(cls._pattern, cls._substitute, t) + + def give_up_diffing(self, level, diff_instance): + t1 = self._remove_pattern(level.t1) + t2 = self._remove_pattern(level.t2) + return t1 == t2 + + def normalize_value_for_hashing(self, parent: Any, obj: Any) -> Any: + """ + Used for ignore_order=True + """ + if isinstance(obj, str): + return self._remove_pattern(obj) + return obj + + + operator = RemoveGUIDsOperator() + + diff1 = DeepDiff(d1, d2, custom_operators=[operator], log_stacktrace=True) + assert not diff1 + + + diff2 = DeepDiff(d1, d2, ignore_order=True, custom_operators=[operator], log_stacktrace=True) + assert not diff2 + + + def test_custom_operator_and_ignore_order2(self): + d1 = { + "Entity": { + "Property": { + "Name": "SUB_OBJECT_FILES", + "Values": { + "Value": [ + "{f254498b-b752-4f35-bef5-6f1844b61eb7}", + "{7fb2a550-1849-45c0-b273-9aa5e4eb9f2b}", + "{3a614c62-4252-48eb-b279-1450ee8af182}", + "{208f22c4-c256-4311-9a45-e6c37d343458}", + "{1fcf5d37-ef19-43a7-a1ad-d17c7c1713c6}", + "{a9cbecc0-21dc-49ce-8b2c-d36352dae139}" + ] + } + } + } + } + + d2 = { + "Entity": { + "Property": { + "Name": "SUB_OBJECT_FILES", + "Values": { + "Value": [ + "{e5d18917-1a2c-4abe-b601-8ec002629953}", + "{ea71ba1f-1339-4fae-bc28-a9ce9b8a8c67}", + "{d7778018-a7b5-4246-8caa-f590138d99e5}", + "{66bb6192-9cd2-4074-8be1-f2ac52877c70}", + "{0c88b900-3755-4d10-93ef-b6a96dbcba90}", + "{e39fdfc5-be6c-4f97-9345-9a8286381fe7}" + ] + } + } + } + } + + class RemovePatternOperator(BaseOperator): + _pattern: str = "" + _substitute: str = "" + + @classmethod + def _remove_pattern(cls, t: str): + return re.sub(cls._pattern, cls._substitute, t) + + def give_up_diffing(self, level, diff_instance): + if isinstance(level.t1, str) and isinstance(level.t2, str): + t1 = self._remove_pattern(level.t1) + t2 = self._remove_pattern(level.t2) + return t1 == t2 + return False + + class RemoveGUIDsOperator(RemovePatternOperator): + _pattern = r"[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}" + _substitute = "guid" + + diff1 = DeepDiff(deepcopy(d1), deepcopy(d2), ignore_order=False, custom_operators=[RemoveGUIDsOperator(types=[str])]) + assert not diff1 + + with pytest.raises(NotImplementedError) as exp: + DeepDiff(deepcopy(d1), deepcopy(d2), ignore_order=True, custom_operators=[RemoveGUIDsOperator(types=[str])]) + expected2 = 'RemoveGUIDsOperator needs to define a normalize_value_for_hashing method to be compatible with ignore_order=True or iterable_compare_func.' + assert expected2 == str(exp.value) + + + # --------- Let's implement the normalize_value_for_hashing to make it work with ignore_order=True --------- + + class RemoveGUIDsOperatorIgnoreOrderReady(RemoveGUIDsOperator): + def normalize_value_for_hashing(self, parent: Any, obj: Any) -> Any: + if isinstance(obj, str): + return self._remove_pattern(obj) + return obj + + diff3 = DeepDiff(deepcopy(d1), deepcopy(d2), ignore_order=True, custom_operators=[RemoveGUIDsOperatorIgnoreOrderReady(types=[str])]) + assert not diff3, "We shouldn't have a diff because we have normalized the string values to be all the same vlues." +