Upstream seperman/deepdiff state of the touched files before a416378437

deepdiff/deephash.py

@@ -0,0 +1,754 @@
+#!/usr/bin/env python
+import logging
+import datetime
+import uuid
+from typing import Union, Optional, Any, List, TYPE_CHECKING, Dict, Tuple, Set, Callable, Generator
+from collections.abc import Iterable, MutableMapping
+from collections import defaultdict
+from hashlib import sha1, sha256
+from pathlib import Path
+from enum import Enum
+import re
+from deepdiff.helper import (strings, numbers, only_numbers, times, unprocessed, not_hashed, add_to_frozen_set,
+                             convert_item_or_items_into_set_else_none, get_doc, ipranges,
+                             convert_item_or_items_into_compiled_regexes_else_none,
+                             get_id, type_is_subclass_of_type_group, type_in_type_group,
+                             number_to_string, datetime_normalize, KEY_TO_VAL_STR,
+                             get_truncate_datetime, dict_, add_root_to_paths, PydanticBaseModel,
+                             separate_wildcard_and_exact_paths,
+                             SetOrdered)
+
+from deepdiff.base import Base
+
+if TYPE_CHECKING:
+    from pytz.tzinfo import BaseTzInfo
+    import numpy as np
+
+# Type aliases for better readability
+HashableType = Union[str, int, float, bytes, bool, tuple, frozenset, type(None)]
+HashResult = Union[str, Any]  # Can be string hash or unprocessed marker
+HashTuple = Tuple[HashResult, int]  # (hash_result, count)
+HashesDict = Dict[Any, Union[HashTuple, List[Any]]]  # Special case for UNPROCESSED_KEY
+PathType = Union[str, List[str], Set[str]]
+RegexType = Union[str, re.Pattern[str], List[Union[str, re.Pattern[str]]]]
+NumberToStringFunc = Callable[..., str]  # More flexible for different number_to_string implementations
+
+
+try:
+    import pandas
+except ImportError:
+    pandas = False  # type: ignore
+
+try:
+    import polars
+except ImportError:
+    polars = False  # type: ignore
+try:
+    import numpy as np
+    booleanTypes: Tuple[type, ...] = (bool, np.bool_)  # type: ignore
+except ImportError:
+    booleanTypes = (bool,)  # type: ignore
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+UNPROCESSED_KEY: object = object()
+
+EMPTY_FROZENSET: frozenset = frozenset()
+
+INDEX_VS_ATTRIBUTE: Tuple[str, str] = ('[%s]', '.%s')
+
+
+HASH_LOOKUP_ERR_MSG: str = '{} is not one of the hashed items.'
+
+
+def sha256hex(obj: Union[str, bytes]) -> str:
+    """Use Sha256 as a cryptographic hash."""
+    if isinstance(obj, str):
+        obj = obj.encode('utf-8')
+    return sha256(obj).hexdigest()
+
+
+def sha1hex(obj: Union[str, bytes]) -> str:
+    """Use Sha1 as a cryptographic hash."""
+    if isinstance(obj, str):
+        obj = obj.encode('utf-8')
+    return sha1(obj).hexdigest()
+
+
+default_hasher: Callable[[Union[str, bytes]], str] = sha256hex
+
+
+def combine_hashes_lists(items: List[List[str]], prefix: Union[str, bytes]) -> str:
+    """
+    Combines lists of hashes into one hash
+    This can be optimized in future.
+    It needs to work with both murmur3 hashes (int) and sha256 (str)
+    Although murmur3 is not used anymore.
+    """
+    if isinstance(prefix, bytes):
+        prefix = prefix.decode('utf-8')
+    hashes_bytes = b''
+    for item in items:
+        # In order to make sure the order of hashes in each item does not affect the hash
+        # we resort them.
+        hashes_bytes += (''.join(map(str, sorted(item))) + '--').encode('utf-8')
+    return prefix + str(default_hasher(hashes_bytes))
+
+
+class BoolObj(Enum):
+    TRUE = 1
+    FALSE = 0
+
+
+def prepare_string_for_hashing(
+        obj: Union[str, bytes, memoryview],
+        ignore_string_type_changes: bool = False,
+        ignore_string_case: bool = False,
+        encodings: Optional[List[str]] = None,
+        ignore_encoding_errors: bool = False,
+) -> str:
+    """
+    Clean type conversions
+    """
+    original_type = obj.__class__.__name__
+    # https://docs.python.org/3/library/codecs.html#codecs.decode
+    errors_mode = 'ignore' if ignore_encoding_errors else 'strict'
+    if isinstance(obj, memoryview):
+        obj = obj.tobytes()
+    if isinstance(obj, bytes):
+        err = None
+        encodings = ['utf-8'] if encodings is None else encodings
+        encoded = False
+        for encoding in encodings:
+            try:
+                obj = obj.decode(encoding, errors=errors_mode)
+                encoded = True
+                break
+            except UnicodeDecodeError as er:
+                err = er
+        if not encoded and err is not None:
+            obj_decoded = obj.decode('utf-8', errors='ignore')  # type: ignore
+            start = max(err.start - 20, 0)
+            start_prefix = ''
+            if start > 0:
+                start_prefix = '...'
+            end = err.end + 20
+            end_suffix = '...'
+            if end >= len(obj):
+                end = len(obj)
+                end_suffix = ''
+            raise UnicodeDecodeError(
+                err.encoding,
+                err.object,
+                err.start,
+                err.end,
+                f"{err.reason} in '{start_prefix}{obj_decoded[start:end]}{end_suffix}'. Please either pass ignore_encoding_errors=True or pass the encoding via encodings=['utf-8', '...']."
+            ) from None
+    if not ignore_string_type_changes:
+        obj = KEY_TO_VAL_STR.format(original_type, obj)
+    if ignore_string_case:
+        obj = obj.lower()
+    return str(obj)
+
+
+doc = get_doc('deephash_doc.rst')
+
+
+class DeepHash(Base):
+    __doc__ = doc
+    
+    # Class attributes
+    hashes: Dict[Any, Any]
+    exclude_types_tuple: Tuple[type, ...]
+    ignore_repetition: bool
+    exclude_paths: Optional[SetOrdered]
+    include_paths: Optional[SetOrdered]
+    exclude_regex_paths: Optional[List[re.Pattern[str]]]
+    hasher: Callable[[Union[str, bytes]], str]
+    use_enum_value: bool
+    default_timezone: Union[datetime.timezone, "BaseTzInfo"]
+    significant_digits: Optional[int]
+    truncate_datetime: Optional[str]
+    number_format_notation: str
+    ignore_type_in_groups: Any
+    ignore_string_type_changes: bool
+    ignore_numeric_type_changes: bool
+    ignore_string_case: bool
+    exclude_obj_callback: Optional[Callable[[Any, str], bool]]
+    apply_hash: bool
+    type_check_func: Callable[[type, Any], bool]
+    number_to_string: Any
+    ignore_private_variables: bool
+    encodings: Optional[List[str]]
+    ignore_encoding_errors: bool
+    ignore_iterable_order: bool
+    custom_operators: Optional[List[Any]]
+
+    def __init__(self,
+                 obj: Any,
+                 *,
+                 apply_hash: bool = True,
+                 custom_operators: Optional[List[Any]] = None,
+                 default_timezone: Union[datetime.timezone, "BaseTzInfo"] = datetime.timezone.utc,
+                 encodings: Optional[List[str]] = None,
+                 exclude_glob_paths: Optional[List[Any]] = None,
+                 exclude_obj_callback: Optional[Callable[[Any, str], bool]] = None,
+                 exclude_paths: Optional[PathType] = None,
+                 exclude_regex_paths: Optional[RegexType] = None,
+                 exclude_types: Optional[Union[List[type], Set[type], Tuple[type, ...]]] = None,
+                 hasher: Optional[Callable[[Union[str, bytes]], str]] = None,
+                 hashes: Optional[Union[Dict[Any, Any], "DeepHash"]] = None,
+                 ignore_encoding_errors: bool = False,
+                 ignore_iterable_order: bool = True,
+                 ignore_numeric_type_changes: bool = False,
+                 ignore_private_variables: bool = True,
+                 ignore_repetition: bool = True,
+                 ignore_string_case: bool = False,
+                 ignore_string_type_changes: bool = False,
+                 ignore_type_in_groups: Any = None,
+                 ignore_type_subclasses: bool = False,
+                 ignore_uuid_types: bool = False,
+                 include_glob_paths: Optional[List[Any]] = None,
+                 include_paths: Optional[PathType] = None,
+                 number_format_notation: str = "f",
+                 number_to_string_func: Optional[NumberToStringFunc] = None,
+                 parent: str = "root",
+                 significant_digits: Optional[int] = None,
+                 truncate_datetime: Optional[str] = None,
+                 use_enum_value: bool = False,
+                 **kwargs) -> None:
+        if kwargs:
+            raise ValueError(
+                ("The following parameter(s) are not valid: %s\n"
+                 "The valid parameters are obj, hashes, exclude_types, significant_digits, truncate_datetime,"
+                 "exclude_paths, include_paths, exclude_regex_paths, hasher, ignore_repetition, "
+                 "number_format_notation, apply_hash, ignore_type_in_groups, ignore_string_type_changes, "
+                 "ignore_numeric_type_changes, ignore_type_subclasses, ignore_string_case, ignore_uuid_types, "
+                 "number_to_string_func, ignore_private_variables, parent, use_enum_value, default_timezone "
+                 "encodings, ignore_encoding_errors") % ', '.join(kwargs.keys()))
+        if isinstance(hashes, MutableMapping):
+            self.hashes = hashes
+        elif isinstance(hashes, DeepHash):
+            self.hashes = hashes.hashes
+        else:
+            self.hashes = dict_()
+        exclude_types = set() if exclude_types is None else set(exclude_types)
+        self.exclude_types_tuple = tuple(exclude_types)  # we need tuple for checking isinstance
+        self.ignore_repetition = ignore_repetition
+        _exclude_set = convert_item_or_items_into_set_else_none(exclude_paths)
+        _exclude_exact, _exclude_globs = separate_wildcard_and_exact_paths(_exclude_set)
+        self.exclude_paths = add_root_to_paths(_exclude_exact)
+        self.exclude_glob_paths = exclude_glob_paths or _exclude_globs
+        _include_set = convert_item_or_items_into_set_else_none(include_paths)
+        _include_exact, _include_globs = separate_wildcard_and_exact_paths(_include_set)
+        self.include_paths = add_root_to_paths(_include_exact)
+        self.include_glob_paths = include_glob_paths or _include_globs
+        self.exclude_regex_paths = convert_item_or_items_into_compiled_regexes_else_none(exclude_regex_paths)
+        self.hasher = default_hasher if hasher is None else hasher
+        self.hashes[UNPROCESSED_KEY] = []  # type: ignore
+        self.use_enum_value = use_enum_value
+        self.default_timezone = default_timezone
+        self.significant_digits = self.get_significant_digits(significant_digits, ignore_numeric_type_changes)
+        self.truncate_datetime = get_truncate_datetime(truncate_datetime)
+        self.number_format_notation = number_format_notation
+        self.ignore_type_in_groups = self.get_ignore_types_in_groups(
+            ignore_type_in_groups=ignore_type_in_groups,
+            ignore_string_type_changes=ignore_string_type_changes,
+            ignore_numeric_type_changes=ignore_numeric_type_changes,
+            ignore_type_subclasses=ignore_type_subclasses,
+            ignore_uuid_types=ignore_uuid_types,
+        )
+        self.ignore_string_type_changes = ignore_string_type_changes
+        self.ignore_numeric_type_changes = ignore_numeric_type_changes
+        self.ignore_string_case = ignore_string_case
+        self.exclude_obj_callback = exclude_obj_callback
+        # makes the hash return constant size result if true
+        # the only time it should be set to False is when
+        # testing the individual hash functions for different types of objects.
+        self.apply_hash = apply_hash
+        self.type_check_func = type_in_type_group if ignore_type_subclasses else type_is_subclass_of_type_group
+        # self.type_check_func = type_is_subclass_of_type_group if ignore_type_subclasses else type_in_type_group
+        self.number_to_string = number_to_string_func or number_to_string
+        self.ignore_private_variables = ignore_private_variables
+        self.encodings = encodings
+        self.ignore_encoding_errors = ignore_encoding_errors
+        self.ignore_iterable_order = ignore_iterable_order
+        self.custom_operators = custom_operators
+
+        self._hash(obj, parent=parent, parents_ids=frozenset({get_id(obj)}))
+
+        if self.hashes[UNPROCESSED_KEY]:
+            logger.warning("Can not hash the following items: {}.".format(self.hashes[UNPROCESSED_KEY]))
+        else:
+            del self.hashes[UNPROCESSED_KEY]
+
+    sha256hex: Callable[[Union[str, bytes]], str] = sha256hex
+    sha1hex: Callable[[Union[str, bytes]], str] = sha1hex
+
+    def __getitem__(self, obj: Any, extract_index: Optional[int] = 0) -> Any:
+        return self._getitem(self.hashes, obj, extract_index=extract_index,
+                             use_enum_value=self.use_enum_value,
+                             ignore_numeric_type_changes=self.ignore_numeric_type_changes)
+
+    @staticmethod
+    def _get_slots_dict(obj: Any) -> Dict[str, Any]:
+        """Get a dict of initialized slot attributes.
+
+        Uses object.__getattribute__ to check each slot directly, bypassing
+        __getattr__. For uninitialized slots on classes that define __getattr__,
+        falls back to getattr — letting it raise if the object is truly broken.
+        """
+        result = {}
+        has_getattr = hasattr(type(obj), '__getattr__')
+        for slot in obj.__slots__:
+            try:
+                result[slot] = object.__getattribute__(obj, slot)
+            except AttributeError:
+                if has_getattr:
+                    # The slot isn't initialized, but the class defines __getattr__.
+                    # Try the normal getattr to let __getattr__ provide a value or
+                    # raise — if it raises, we propagate to fail the strategy.
+                    result[slot] = getattr(obj, slot)
+        return result
+
+    @staticmethod
+    def _getitem(hashes: Dict[Any, Any], obj: Any, extract_index: Optional[int] = 0,
+                 use_enum_value: bool = False, ignore_numeric_type_changes: bool = False) -> Any:
+        """
+        extract_index is zero for hash and 1 for count and None to get them both.
+        To keep it backward compatible, we only get the hash by default so it is set to zero by default.
+        """
+
+        key = obj
+        if obj is True:
+            key = BoolObj.TRUE
+        elif obj is False:
+            key = BoolObj.FALSE
+        elif use_enum_value and isinstance(obj, Enum):
+            key = obj.value
+        key = DeepHash._make_hash_key_for_lookup(key, ignore_numeric_type_changes=ignore_numeric_type_changes)
+
+        result_n_count: Tuple[Any, int] = (None, 0)  # type: ignore
+
+        try:
+            result_n_count = hashes[key]
+        except (TypeError, KeyError):
+            key = get_id(obj)
+            try:
+                result_n_count = hashes[key]
+            except KeyError:
+                raise KeyError(HASH_LOOKUP_ERR_MSG.format(obj)) from None
+
+        if obj is UNPROCESSED_KEY:
+            extract_index = None
+
+        return result_n_count if extract_index is None else result_n_count[extract_index]
+
+    def __contains__(self, obj: Any) -> bool:
+        key = self._make_hash_key(obj)
+        result = False
+        try:
+            result = key in self.hashes
+        except (TypeError, KeyError):
+            result = False
+        if not result:
+            result = get_id(obj) in self.hashes
+        return result
+
+    def get(self, key: Any, default: Any = None, extract_index: Optional[int] = 0) -> Any:
+        """
+        Get method for the hashes dictionary.
+        It can extract the hash for a given key that is already calculated when extract_index=0
+        or the count of items that went to building the object when extract_index=1.
+        """
+        return self.get_key(self.hashes, key, default=default, extract_index=extract_index,
+                            ignore_numeric_type_changes=self.ignore_numeric_type_changes)
+
+    @staticmethod
+    def get_key(hashes: Dict[Any, Any], key: Any, default: Any = None, extract_index: Optional[int] = 0,
+                use_enum_value: bool = False, ignore_numeric_type_changes: bool = False) -> Any:
+        """
+        get_key method for the hashes dictionary.
+        It can extract the hash for a given key that is already calculated when extract_index=0
+        or the count of items that went to building the object when extract_index=1.
+        """
+        try:
+            result = DeepHash._getitem(hashes, key, extract_index=extract_index,
+                                       use_enum_value=use_enum_value,
+                                       ignore_numeric_type_changes=ignore_numeric_type_changes)
+        except KeyError:
+            result = default
+        return result
+
+    @staticmethod
+    def _unwrap_hash_key(key: Any) -> Any:
+        """Unwrap a (type, value) hash key back to the original value for public API."""
+        if isinstance(key, tuple) and len(key) == 2 and isinstance(key[0], type) and isinstance(key[1], only_numbers):
+            return key[1]
+        return key
+
+    def _get_objects_to_hashes_dict(self, extract_index: Optional[int] = 0) -> Dict[Any, Any]:
+        """
+        A dictionary containing only the objects to hashes,
+        or a dictionary of objects to the count of items that went to build them.
+        extract_index=0 for hashes and extract_index=1 for counts.
+        """
+        result = dict_()
+        for key, value in self.hashes.items():
+            key = self._unwrap_hash_key(key)
+            if key is UNPROCESSED_KEY:
+                result[key] = value
+            else:
+                result[key] = value[extract_index]
+        return result
+
+    def __eq__(self, other: Any) -> bool:
+        if isinstance(other, DeepHash):
+            return self.hashes == other.hashes
+        else:
+            # We only care about the hashes
+            return self._get_objects_to_hashes_dict() == other
+
+    __req__ = __eq__
+
+    def __repr__(self) -> str:
+        """
+        Hide the counts since it will be confusing to see them when they are hidden everywhere else.
+        """
+        from deepdiff.summarize import summarize
+        return summarize(self._get_objects_to_hashes_dict(extract_index=0), max_length=500)
+
+    def __str__(self) -> str:
+        return str(self._get_objects_to_hashes_dict(extract_index=0))
+
+    def __bool__(self) -> bool:
+        return bool(self.hashes)
+
+    def keys(self) -> Any:
+        return [self._unwrap_hash_key(k) for k in self.hashes.keys()]
+
+    def values(self) -> Generator[Any, None, None]:
+        return (i[0] for i in self.hashes.values())  # Just grab the item and not its count
+
+    def items(self) -> Generator[Tuple[Any, Any], None, None]:
+        return ((self._unwrap_hash_key(i), v[0]) for i, v in self.hashes.items())
+
+    def _prep_obj(self, obj: Any, parent: str, parents_ids: frozenset = EMPTY_FROZENSET, is_namedtuple: bool = False, is_pydantic_object: bool = False) -> HashTuple:
+        """prepping objects"""
+        original_type = type(obj) if not isinstance(obj, type) else obj
+
+        obj_to_dict_strategies = []
+        if is_namedtuple:
+            obj_to_dict_strategies.append(lambda o: o._asdict())
+        elif is_pydantic_object:
+            obj_to_dict_strategies.append(lambda o: {k: v for (k, v) in o.__dict__.items() if v !="model_fields_set"})
+        else:
+            obj_to_dict_strategies.append(lambda o: o.__dict__)
+
+        if hasattr(obj, "__slots__"):
+            obj_to_dict_strategies.append(lambda o: DeepHash._get_slots_dict(o))
+        else:
+            import inspect
+            obj_to_dict_strategies.append(lambda o: dict(inspect.getmembers(o, lambda m: not inspect.isroutine(m))))
+
+        for get_dict in obj_to_dict_strategies:
+            try:
+                d = get_dict(obj)
+                break
+            except AttributeError:
+                pass
+        else:
+            self.hashes[UNPROCESSED_KEY].append(obj)  # type: ignore
+            return (unprocessed, 0)
+        obj = d
+
+        result, counts = self._prep_dict(obj, parent=parent, parents_ids=parents_ids,
+                                         print_as_attribute=True, original_type=original_type)
+        result = "nt{}".format(result) if is_namedtuple else "obj{}".format(result)
+        return result, counts
+
+    def _skip_this(self, obj: Any, parent: str) -> bool:
+        skip = False
+        if self.exclude_paths and parent in self.exclude_paths:
+            skip = True
+        elif self.exclude_glob_paths and any(gp.match(parent) for gp in self.exclude_glob_paths):
+            skip = True
+        if (self.include_paths or self.include_glob_paths) and parent != 'root':
+            skip = True
+            if self.include_paths:
+                if parent in self.include_paths:
+                    skip = False
+                else:
+                    for prefix in self.include_paths:
+                        if parent.startswith(prefix):
+                            skip = False
+                            break
+            if skip and self.include_glob_paths:
+                for gp in self.include_glob_paths:
+                    if gp.match_or_is_ancestor(parent):
+                        skip = False
+                        break
+        elif self.exclude_regex_paths and any(
+                [exclude_regex_path.search(parent) for exclude_regex_path in self.exclude_regex_paths]):  # type: ignore
+            skip = True
+        elif self.exclude_types_tuple and isinstance(obj, self.exclude_types_tuple):
+            skip = True
+        elif self.exclude_obj_callback and self.exclude_obj_callback(obj, parent):
+            skip = True
+        return skip
+
+    def _prep_dict(self, obj: Union[Dict[Any, Any], MutableMapping], parent: str, parents_ids: frozenset = EMPTY_FROZENSET, print_as_attribute: bool = False, original_type: Optional[type] = None) -> HashTuple:
+
+        result = []
+        counts = 1
+
+        key_text = "%s{}".format(INDEX_VS_ATTRIBUTE[print_as_attribute])
+        for key, item in obj.items():
+            counts += 1
+            # ignore private variables
+            if self.ignore_private_variables and isinstance(key, str) and key.startswith('__'):
+                continue
+            key_formatted = "'%s'" % key if not print_as_attribute and isinstance(key, strings) else key
+            key_in_report = key_text % (parent, key_formatted)
+
+            key_hash, _ = self._hash(key, parent=key_in_report, parents_ids=parents_ids)
+            if not key_hash:
+                continue
+            item_id = get_id(item)
+            if (parents_ids and item_id in parents_ids) or self._skip_this(item, parent=key_in_report):
+                continue
+            parents_ids_added = add_to_frozen_set(parents_ids, item_id)
+            hashed, count = self._hash(item, parent=key_in_report, parents_ids=parents_ids_added)
+            hashed = KEY_TO_VAL_STR.format(key_hash, hashed)
+            result.append(hashed)
+            counts += count
+
+        result.sort()
+        result = ';'.join(result)
+        if print_as_attribute:
+            type_ = original_type or type(obj)
+            type_str = type_.__name__
+            for type_group in self.ignore_type_in_groups:
+                if self.type_check_func(type_, type_group):
+                    type_str = ','.join(map(lambda x: x.__name__, type_group))
+                    break
+        else:
+            type_str = 'dict'
+        return "{}:{{{}}}".format(type_str, result), counts
+
+    def _prep_iterable(self, obj: Iterable[Any], parent: str, parents_ids: frozenset = EMPTY_FROZENSET) -> HashTuple:
+
+        counts = 1
+        result = defaultdict(int)
+
+        for i, item in enumerate(obj):
+            new_parent = "{}[{}]".format(parent, i)
+            if self._skip_this(item, parent=new_parent):
+                continue
+
+            item_id = get_id(item)
+            if parents_ids and item_id in parents_ids:
+                continue
+
+            parents_ids_added = add_to_frozen_set(parents_ids, item_id)
+            hashed, count = self._hash(item, parent=new_parent, parents_ids=parents_ids_added)
+            # counting repetitions
+            result[hashed] += 1
+            counts += count
+
+        if self.ignore_repetition:
+            result = list(result.keys())
+        else:
+            result = [
+                '{}|{}'.format(i, v) for i, v in result.items()
+            ]
+
+        result = map(str, result) # making sure the result items are string so join command works.
+        if self.ignore_iterable_order:
+            result = sorted(result)  
+        result = ','.join(result)
+        result = KEY_TO_VAL_STR.format(type(obj).__name__, result)
+
+        return result, counts
+
+    def _prep_bool(self, obj: bool) -> BoolObj:
+        return BoolObj.TRUE if obj else BoolObj.FALSE
+
+
+    def _prep_path(self, obj: Path) -> str:
+        type_ = obj.__class__.__name__
+        return KEY_TO_VAL_STR.format(type_, obj)
+
+    def _prep_number(self, obj: Union[int, float, complex]) -> str:
+        type_ = "number" if self.ignore_numeric_type_changes else obj.__class__.__name__
+        if self.significant_digits is not None:
+            obj = self.number_to_string(obj, significant_digits=self.significant_digits,
+                                        number_format_notation=self.number_format_notation)  # type: ignore
+        return KEY_TO_VAL_STR.format(type_, obj)
+
+    def _prep_ipranges(self, obj) -> str:
+        type_ = 'iprange'
+        obj = str(obj)
+        return KEY_TO_VAL_STR.format(type_, obj)
+
+    def _prep_datetime(self, obj: datetime.datetime) -> str:
+        type_ = 'datetime'
+        obj = datetime_normalize(self.truncate_datetime, obj, default_timezone=self.default_timezone)
+        return KEY_TO_VAL_STR.format(type_, obj)
+
+    def _prep_date(self, obj: datetime.date) -> str:
+        type_ = 'datetime'  # yes still datetime but it doesn't need normalization
+        return KEY_TO_VAL_STR.format(type_, obj)
+
+    def _prep_tuple(self, obj: tuple, parent: str, parents_ids: frozenset) -> HashTuple:
+        # Checking to see if it has _fields. Which probably means it is a named
+        # tuple.
+        try:
+            obj._asdict  # type: ignore
+        # It must be a normal tuple
+        except AttributeError:
+            result, counts = self._prep_iterable(obj=obj, parent=parent, parents_ids=parents_ids)
+        # We assume it is a namedtuple then
+        else:
+            result, counts = self._prep_obj(obj, parent, parents_ids=parents_ids, is_namedtuple=True)
+        return result, counts
+
+    def _make_hash_key(self, obj: Any) -> Any:
+        """
+        Create a key for the hashes dict that distinguishes numeric types.
+
+        In Python, 1 == 1.0 and hash(1) == hash(1.0), so int and float values
+        collide as dict keys. When ignore_numeric_type_changes is False, we wrap
+        numeric objects as (type, value) tuples so that each type gets its own
+        cache entry and its own hash.
+        """
+        if not self.ignore_numeric_type_changes and isinstance(obj, only_numbers):
+            return (type(obj), obj)
+        return obj
+
+    @staticmethod
+    def _make_hash_key_for_lookup(obj: Any, ignore_numeric_type_changes: bool = False) -> Any:
+        """Static version of _make_hash_key for use in static accessor methods."""
+        if not ignore_numeric_type_changes and isinstance(obj, only_numbers):
+            return (type(obj), obj)
+        return obj
+
+    def _hash(self, obj: Any, parent: str, parents_ids: frozenset = EMPTY_FROZENSET) -> HashTuple:
+        """The main hash method"""
+        counts = 1
+        if self.custom_operators is not None:
+            for operator in self.custom_operators:
+                func = getattr(operator, 'normalize_value_for_hashing', None)
+                if func is None:
+                    raise NotImplementedError(f"{operator.__class__.__name__} needs to define a normalize_value_for_hashing method to be compatible with ignore_order=True or iterable_compare_func.".format(operator))
+                else:
+                    obj = func(parent, obj)
+
+        if isinstance(obj, booleanTypes):
+            obj = self._prep_bool(obj)
+            result = None
+        elif self.use_enum_value and isinstance(obj, Enum):
+            obj = obj.value
+        else:
+            result = not_hashed
+        hash_key = self._make_hash_key(obj)
+        try:
+            result, counts = self.hashes[hash_key]
+        except (TypeError, KeyError):
+            pass
+        else:
+            return result, counts
+
+        if self._skip_this(obj, parent):
+            return None, 0
+
+        elif obj is None:
+            result = 'NONE'
+
+        elif isinstance(obj, strings):
+            result = prepare_string_for_hashing(
+                obj,
+                ignore_string_type_changes=self.ignore_string_type_changes,
+                ignore_string_case=self.ignore_string_case,
+                encodings=self.encodings,
+                ignore_encoding_errors=self.ignore_encoding_errors,
+            )
+
+        elif isinstance(obj, Path):
+            result = self._prep_path(obj)
+
+        elif isinstance(obj, times):
+            result = self._prep_datetime(obj)  # type: ignore
+
+        elif isinstance(obj, datetime.date):
+            result = self._prep_date(obj)
+
+        elif isinstance(obj, numbers):  # type: ignore
+            result = self._prep_number(obj)
+
+        elif isinstance(obj, ipranges):
+            result = self._prep_ipranges(obj)
+
+        elif isinstance(obj, uuid.UUID):
+            # Handle UUID objects (including uuid6.UUID) by using their integer value
+            result = str(obj.int)
+
+        elif isinstance(obj, MutableMapping):
+            result, counts = self._prep_dict(obj=obj, parent=parent, parents_ids=parents_ids)
+
+        elif isinstance(obj, tuple):
+            result, counts = self._prep_tuple(obj=obj, parent=parent, parents_ids=parents_ids)
+
+        elif (pandas and isinstance(obj, pandas.DataFrame)):  # type: ignore
+            def gen():  # type: ignore
+                yield ('dtype', obj.dtypes)  # type: ignore
+                yield ('index', obj.index)  # type: ignore
+                yield from obj.items()  # type: ignore  # which contains (column name, series tuples)
+            result, counts = self._prep_iterable(obj=gen(), parent=parent, parents_ids=parents_ids)
+        elif (polars and isinstance(obj, polars.DataFrame)):  # type: ignore
+            def gen():
+                yield from obj.columns  # type: ignore
+                yield from list(obj.schema.items())  # type: ignore
+                yield from obj.rows()  # type: ignore
+            result, counts = self._prep_iterable(obj=gen(), parent=parent, parents_ids=parents_ids)
+
+        elif isinstance(obj, Iterable):
+            result, counts = self._prep_iterable(obj=obj, parent=parent, parents_ids=parents_ids)
+
+        elif obj == BoolObj.TRUE or obj == BoolObj.FALSE:
+            result = 'bool:true' if obj is BoolObj.TRUE else 'bool:false'
+        elif isinstance(obj, PydanticBaseModel):
+            result, counts = self._prep_obj(obj=obj, parent=parent, parents_ids=parents_ids, is_pydantic_object=True)
+        else:
+            result, counts = self._prep_obj(obj=obj, parent=parent, parents_ids=parents_ids)
+
+        if result is not_hashed:  # pragma: no cover
+            self.hashes[UNPROCESSED_KEY].append(obj)  # type: ignore
+
+        elif result is unprocessed:
+            pass
+
+        elif self.apply_hash:
+            if isinstance(obj, strings):
+                result_cleaned = result
+            else:
+                result_cleaned = prepare_string_for_hashing(
+                    str(result), ignore_string_type_changes=self.ignore_string_type_changes,
+                    ignore_string_case=self.ignore_string_case)
+            result = self.hasher(result_cleaned)
+
+        # It is important to keep the hash of all objects.
+        # The hashes will be later used for comparing the objects.
+        # Object to hash when possible otherwise ObjectID to hash
+        try:
+            self.hashes[hash_key] = (result, counts)
+        except TypeError:
+            obj_id = get_id(obj)
+            self.hashes[obj_id] = (result, counts)
+
+        return result, counts
+
+
+if __name__ == "__main__":  # pragma: no cover
+    import doctest
+    doctest.testmod()

tests/test_operators.py

@@ -0,0 +1,428 @@
+import re
+import math
+import pytest
+from copy import deepcopy
+from typing import List, Any
+from deepdiff import DeepDiff
+from deepdiff.operator import BaseOperator, PrefixOrSuffixOperator, BaseOperatorPlus
+
+
+class TestOperators:
+    def test_custom_operators_prevent_default(self):
+        t1 = {
+            "coordinates": [
+                {"x": 5, "y": 5},
+                {"x": 8, "y": 8}
+            ]
+        }
+
+        t2 = {
+            "coordinates": [
+                {"x": 6, "y": 6},
+                {"x": 88, "y": 88}
+            ]
+        }
+
+        class L2DistanceDifferWithPreventDefault(BaseOperator):
+            def __init__(self, regex_paths: List[str], distance_threshold: float):
+                super().__init__(regex_paths)
+                self.distance_threshold = distance_threshold
+
+            def _l2_distance(self, c1, c2):
+                return math.sqrt(
+                    (c1["x"] - c2["x"]) ** 2 + (c1["y"] - c2["y"]) ** 2
+                )
+
+            def give_up_diffing(self, level, diff_instance) -> bool:
+                l2_distance = self._l2_distance(level.t1, level.t2)
+                if l2_distance > self.distance_threshold:
+                    diff_instance.custom_report_result('distance_too_far', level, {
+                        "l2_distance": l2_distance
+                    })
+                return True
+
+        ddiff = DeepDiff(t1, t2, custom_operators=[L2DistanceDifferWithPreventDefault(
+            ["^root\\['coordinates'\\]\\[\\d+\\]$"],
+            1
+        )])
+
+        expected = {
+            'distance_too_far': {
+                "root['coordinates'][0]": {'l2_distance': 1.4142135623730951},
+                "root['coordinates'][1]": {'l2_distance': 113.13708498984761}
+            }
+        }
+        assert expected == ddiff
+
+    def test_custom_operators_not_prevent_default(self):
+        t1 = {
+            "coordinates": [
+                {"x": 5, "y": 5},
+                {"x": 8, "y": 8}
+            ]
+        }
+
+        t2 = {
+            "coordinates": [
+                {"x": 6, "y": 6},
+                {"x": 88, "y": 88}
+            ]
+        }
+
+        class L2DistanceDifferWithPreventDefault(BaseOperator):
+            def __init__(self, regex_paths, distance_threshold):
+                super().__init__(regex_paths)
+                self.distance_threshold = distance_threshold
+
+            def _l2_distance(self, c1, c2):
+                return math.sqrt(
+                    (c1["x"] - c2["x"]) ** 2 + (c1["y"] - c2["y"]) ** 2
+                )
+
+            def give_up_diffing(self, level, diff_instance) -> bool:
+                l2_distance = self._l2_distance(level.t1, level.t2)
+                if l2_distance > self.distance_threshold:
+                    diff_instance.custom_report_result('distance_too_far', level, {
+                        "l2_distance": l2_distance
+                    })
+                #
+                return False
+
+        ddiff = DeepDiff(t1, t2, custom_operators=[L2DistanceDifferWithPreventDefault(
+            ["^root\\['coordinates'\\]\\[\\d+\\]$"],
+            1
+        )
+        ])
+        expected = {
+            'values_changed': {
+                "root['coordinates'][0]['x']": {'new_value': 6, 'old_value': 5},
+                "root['coordinates'][0]['y']": {'new_value': 6, 'old_value': 5},
+                "root['coordinates'][1]['x']": {'new_value': 88, 'old_value': 8},
+                "root['coordinates'][1]['y']": {'new_value': 88, 'old_value': 8}
+            },
+            'distance_too_far': {
+                "root['coordinates'][0]": {'l2_distance': 1.4142135623730951},
+                "root['coordinates'][1]": {'l2_distance': 113.13708498984761}
+            }
+        }
+        assert expected == ddiff
+
+    def test_custom_operators_should_not_equal(self):
+        t1 = {
+            "id": 5,
+            "expect_change_pos": 10,
+            "expect_change_neg": 10,
+        }
+
+        t2 = {
+            "id": 5,
+            "expect_change_pos": 100,
+            "expect_change_neg": 10,
+        }
+
+        class ExpectChangeOperator(BaseOperator):
+            def __init__(self, regex_paths):
+                super().__init__(regex_paths)
+
+            def give_up_diffing(self, level, diff_instance) -> bool:
+                if level.t1 == level.t2:
+                    diff_instance.custom_report_result('unexpected:still', level, {
+                        "old": level.t1,
+                        "new": level.t2
+                    })
+
+                return True
+
+        ddiff = DeepDiff(t1, t2, custom_operators=[
+            ExpectChangeOperator(regex_paths=["root\\['expect_change.*'\\]"])
+        ])
+
+        assert ddiff == {'unexpected:still': {"root['expect_change_neg']": {'old': 10, 'new': 10}}}
+
+    def test_custom_operator2(self):
+
+        class CustomClass:
+
+            def __init__(self, d: dict, l: list):
+                self.dict = d
+                self.dict['list'] = l
+
+            def __repr__(self):
+                return "Class list is " + str(self.dict['list'])
+
+        custom1 = CustomClass(d=dict(a=1, b=2), l=[1, 2, 3])
+        custom2 = CustomClass(d=dict(c=3, d=4), l=[1, 2, 3, 2])
+        custom3 = CustomClass(d=dict(a=1, b=2), l=[1, 2, 3, 4])
+
+        class ListMatchOperator(BaseOperator):
+
+            def give_up_diffing(self, level, diff_instance) -> bool:
+                if set(level.t1.dict['list']) == set(level.t2.dict['list']):
+                    return True
+                return False
+
+        ddiff = DeepDiff(custom1, custom2, custom_operators=[
+            ListMatchOperator(types=[CustomClass])
+        ])
+
+        assert {} == ddiff
+
+        ddiff2 = DeepDiff(custom2, custom3, threshold_to_diff_deeper=0, custom_operators=[
+            ListMatchOperator(types=[CustomClass])
+        ])
+
+        expected = {
+            'dictionary_item_added': ["root.dict['a']", "root.dict['b']"],
+            'dictionary_item_removed': ["root.dict['c']", "root.dict['d']"],
+            'values_changed': {"root.dict['list'][3]": {'new_value': 4, 'old_value': 2}}}
+
+        assert expected == ddiff2
+
+    def test_include_only_certain_path(self):
+
+        class MyOperator:
+
+            def __init__(self, include_paths):
+                self.include_paths = include_paths
+
+            def match(self, level) -> bool:
+                return True
+
+            def give_up_diffing(self, level, diff_instance) -> bool:
+                return level.path() not in self.include_paths
+
+        t1 = {'a': [10, 11], 'b': [20, 21], 'c': [30, 31]}
+        t2 = {'a': [10, 22], 'b': [20, 33], 'c': [30, 44]}
+
+        ddiff = DeepDiff(t1, t2, custom_operators=[
+            MyOperator(include_paths="root['a'][1]")
+        ])
+
+        expected = {'values_changed': {"root['a'][1]": {'new_value': 22, 'old_value': 11}}}
+        assert expected == ddiff
+
+    def test_give_up_diffing_on_first_diff(self):
+
+        class MyOperator:
+
+            def match(self, level) -> bool:
+                return True
+
+            def give_up_diffing(self, level, diff_instance) -> bool:
+                return any(diff_instance.tree.values())
+
+        t1 = [[1, 2], [3, 4], [5, 6]]
+        t2 = [[1, 3], [3, 5], [5, 7]]
+
+        ddiff = DeepDiff(t1, t2, custom_operators=[
+            MyOperator()
+        ])
+
+        expected = {'values_changed': {'root[0][1]': {'new_value': 3, 'old_value': 2}}}
+        assert expected == ddiff
+
+    def test_prefix_or_suffix_diff(self):
+
+        t1 = {
+            "key1": ["foo", "bar's food", "jack", "joe"]
+        }
+        t2 = {
+            "key1": ["foo", "bar", "jill", "joe'car"]
+        }
+
+        ddiff = DeepDiff(t1, t2, custom_operators=[
+            PrefixOrSuffixOperator()
+        ])
+
+        expected = {'values_changed': {"root['key1'][2]": {'new_value': 'jill', 'old_value': 'jack'}}}
+        assert expected == ddiff
+
+        with pytest.raises(NotImplementedError) as exp:
+            DeepDiff(t1, t2, ignore_order=True, custom_operators=[
+                PrefixOrSuffixOperator()
+            ])
+        expected2 = 'PrefixOrSuffixOperator needs to define a normalize_value_for_hashing method to be compatible with ignore_order=True or iterable_compare_func.'
+        assert expected2 == str(exp.value)
+
+    def test_custom_operator3_small_numbers(self):
+        x = [2.0000000000000027, 2.500000000000005, 2.000000000000002, 3.000000000000001]
+        y = [2.000000000000003, 2.500000000000005, 2.0000000000000027, 3.0000000000000027]
+        result = DeepDiff(x, y)
+        expected = {
+            'values_changed': {
+                'root[0]': {'new_value': 2.000000000000003, 'old_value': 2.0000000000000027},
+                'root[2]': {'new_value': 2.0000000000000027, 'old_value': 2.000000000000002},
+                'root[3]': {'new_value': 3.0000000000000027, 'old_value': 3.000000000000001}}}
+        assert expected == result
+
+        class CustomCompare(BaseOperatorPlus):
+            def __init__(self, tolerance, types):
+                self.tolerance = tolerance
+                self.types = types
+
+            def match(self, level) -> bool:
+                if type(level.t1) in self.types:
+                    return True
+                return False
+
+            def give_up_diffing(self, level, diff_instance) -> bool:
+                relative = abs(abs(level.t1 - level.t2) / level.t1)
+                if not max(relative, self.tolerance) == self.tolerance:
+                    custom_report = f'relative diff: {relative:.8e}'
+                    diff_instance.custom_report_result('diff', level, custom_report)
+                return True
+
+            def normalize_value_for_hashing(self, parent: Any, obj: Any) -> Any:
+                return obj
+
+
+        def compare_func(x, y, level):
+            return True
+
+        operators = [CustomCompare(types=[float], tolerance=5.5e-5)]
+        result2 = DeepDiff(x, y, custom_operators=operators, iterable_compare_func=compare_func)
+        assert {} == result2
+
+        result3 = DeepDiff(x, y, custom_operators=operators, zip_ordered_iterables=True)
+        assert {} == result3, "We should get the same result as result2 when zip_ordered_iterables is True."
+
+    def test_custom_operator_and_ignore_order1_using_base_operator_plus(self):
+
+        d1 = {
+            "Name": "SUB_OBJECT_FILES",
+            "Values": {
+                "Value": [
+                    "{f254498b-b752-4f35-bef5-6f1844b61eb7}",
+                    "{7fb2a550-1849-45c0-b273-9aa5e4eb9f2b}",
+                    "{3a614c62-4252-48eb-b279-1450ee8af182}",
+                    "{208f22c4-c256-4311-9a45-e6c37d343458}",
+                    "{1fcf5d37-ef19-43a7-a1ad-d17c7c1713c6}",
+                ]
+            }
+        }
+
+        d2 = {
+            "Name": "SUB_OBJECT_FILES",
+            "Values": {
+                "Value": [
+                    "{e5d18917-1a2c-4abe-b601-8ec002629953}",
+                    "{ea71ba1f-1339-4fae-bc28-a9ce9b8a8c67}",
+                    "{66bb6192-9cd2-4074-8be1-f2ac52877c70}",
+                    "{0c88b900-3755-4d10-93ef-b6a96dbcba90}",
+                    "{e39fdfc5-be6c-4f97-9345-9a8286381fe7}"
+                ]
+            }
+        }
+
+
+        class RemoveGUIDsOperator(BaseOperatorPlus):
+            _pattern = r"[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"
+            _substitute = "guid"
+
+            def match(self, level) -> bool:
+                return isinstance(level.t1, str) and isinstance(level.t2, str)
+
+            @classmethod
+            def _remove_pattern(cls, t: str):
+                return re.sub(cls._pattern, cls._substitute, t)
+
+            def give_up_diffing(self, level, diff_instance):
+                t1 = self._remove_pattern(level.t1)
+                t2 = self._remove_pattern(level.t2)
+                return t1 == t2
+
+            def normalize_value_for_hashing(self, parent: Any, obj: Any) -> Any:
+                """
+                Used for ignore_order=True
+                """
+                if isinstance(obj, str):
+                    return self._remove_pattern(obj)
+                return obj
+
+
+        operator = RemoveGUIDsOperator()
+
+        diff1 = DeepDiff(d1, d2, custom_operators=[operator], log_stacktrace=True)
+        assert not diff1
+
+
+        diff2 = DeepDiff(d1, d2, ignore_order=True, custom_operators=[operator], log_stacktrace=True)
+        assert not diff2
+
+
+    def test_custom_operator_and_ignore_order2(self):
+        d1 = {
+            "Entity": {
+                "Property": {
+                    "Name": "SUB_OBJECT_FILES",
+                    "Values": {
+                        "Value": [
+                            "{f254498b-b752-4f35-bef5-6f1844b61eb7}",
+                            "{7fb2a550-1849-45c0-b273-9aa5e4eb9f2b}",
+                            "{3a614c62-4252-48eb-b279-1450ee8af182}",
+                            "{208f22c4-c256-4311-9a45-e6c37d343458}",
+                            "{1fcf5d37-ef19-43a7-a1ad-d17c7c1713c6}",
+                            "{a9cbecc0-21dc-49ce-8b2c-d36352dae139}"
+                        ]
+                    }
+                }
+            }
+        }
+
+        d2 = {
+            "Entity": {
+                "Property": {
+                    "Name": "SUB_OBJECT_FILES",
+                    "Values": {
+                        "Value": [
+                            "{e5d18917-1a2c-4abe-b601-8ec002629953}",
+                            "{ea71ba1f-1339-4fae-bc28-a9ce9b8a8c67}",
+                            "{d7778018-a7b5-4246-8caa-f590138d99e5}",
+                            "{66bb6192-9cd2-4074-8be1-f2ac52877c70}",
+                            "{0c88b900-3755-4d10-93ef-b6a96dbcba90}",
+                            "{e39fdfc5-be6c-4f97-9345-9a8286381fe7}"
+                        ]
+                    }
+                }
+            }
+        }
+
+        class RemovePatternOperator(BaseOperator):
+            _pattern: str = ""
+            _substitute: str = ""
+
+            @classmethod
+            def _remove_pattern(cls, t: str):
+                return re.sub(cls._pattern, cls._substitute, t)
+
+            def give_up_diffing(self, level, diff_instance):
+                if isinstance(level.t1, str) and isinstance(level.t2, str):
+                    t1 = self._remove_pattern(level.t1)
+                    t2 = self._remove_pattern(level.t2)
+                    return t1 == t2
+                return False
+
+        class RemoveGUIDsOperator(RemovePatternOperator):
+            _pattern = r"[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"
+            _substitute = "guid"
+
+        diff1 = DeepDiff(deepcopy(d1), deepcopy(d2), ignore_order=False, custom_operators=[RemoveGUIDsOperator(types=[str])])
+        assert not diff1
+
+        with pytest.raises(NotImplementedError) as exp:
+            DeepDiff(deepcopy(d1), deepcopy(d2), ignore_order=True, custom_operators=[RemoveGUIDsOperator(types=[str])])
+        expected2 = 'RemoveGUIDsOperator needs to define a normalize_value_for_hashing method to be compatible with ignore_order=True or iterable_compare_func.'
+        assert expected2 == str(exp.value)
+
+
+        # --------- Let's implement the normalize_value_for_hashing to make it work with ignore_order=True ---------
+
+        class RemoveGUIDsOperatorIgnoreOrderReady(RemoveGUIDsOperator):
+            def normalize_value_for_hashing(self, parent: Any, obj: Any) -> Any:
+                if isinstance(obj, str):
+                    return self._remove_pattern(obj)
+                return obj
+
+        diff3 = DeepDiff(deepcopy(d1), deepcopy(d2), ignore_order=True, custom_operators=[RemoveGUIDsOperatorIgnoreOrderReady(types=[str])])
+        assert not diff3, "We shouldn't have a diff because we have normalized the string values to be all the same vlues."
+