commit aa2b15b1b32ab5abcb418166f7046e7012b46416 Author: upstream Date: Thu Sep 3 14:06:34 2026 +0200 Upstream seperman/deepdiff state of the touched files before f9aea3e22e diff --git a/deepdiff/diff.py b/deepdiff/diff.py new file mode 100644 index 0000000..e65fd4a --- /dev/null +++ b/deepdiff/diff.py @@ -0,0 +1,2412 @@ +#!/usr/bin/env python + +# In order to run the docstrings: +# python3 -m deepdiff.diff +# You might need to run it many times since dictionaries come in different orders +# every time you run the docstrings. +# However the docstring expects it in a specific order in order to pass! +import difflib +import logging +import types +import datetime +import uuid +from enum import Enum +from copy import deepcopy +from math import isclose as is_close +from typing import List, Dict, Callable, Union, Any, Pattern, Tuple, Optional, Set, FrozenSet, TYPE_CHECKING, Protocol, Literal, cast +from collections.abc import Mapping, Iterable, Sequence +from collections import defaultdict +from inspect import getmembers +from itertools import zip_longest +from functools import lru_cache +from deepdiff.helper import (strings, bytes_type, numbers, uuids, ListItemRemovedOrAdded, notpresent, not_found, + IndexedHash, unprocessed, add_to_frozen_set, basic_types, + convert_item_or_items_into_set_else_none, get_type, + convert_item_or_items_into_compiled_regexes_else_none, + type_is_subclass_of_type_group, type_in_type_group, get_doc, + number_to_string, datetime_normalize, KEY_TO_VAL_STR, booleans, + np_ndarray, np_floating, get_numpy_ndarray_rows, RepeatedTimer, + TEXT_VIEW, TREE_VIEW, DELTA_VIEW, COLORED_VIEW, COLORED_COMPACT_VIEW, + detailed__dict__, add_root_to_paths, + np, get_truncate_datetime, dict_, CannotCompare, ENUM_INCLUDE_KEYS, + PydanticBaseModel, Opcode, SetOrdered, ipranges, + separate_wildcard_and_exact_paths) +from deepdiff.serialization import SerializationMixin +from deepdiff.distance import DistanceMixin, logarithmic_similarity +from deepdiff.model import ( + RemapDict, ResultDict, TextResult, TreeResult, DiffLevel, + DictRelationship, AttributeRelationship, REPORT_KEYS, + SubscriptableIterableRelationship, NonSubscriptableIterableRelationship, + SetRelationship, NumpyArrayRelationship, CUSTOM_FIELD, + FORCE_DEFAULT, ChildRelationship, +) +from deepdiff.deephash import DeepHash, combine_hashes_lists +from deepdiff.base import Base +from deepdiff.lfucache import LFUCache, DummyLFU +from deepdiff.colored_view import ColoredView +from deepdiff._multiprocessing import ( + normalize_mp_config, compute_distances_parallel, + compute_hashes_parallel, compute_subtree_diffs_parallel, +) + +if TYPE_CHECKING: + from pytz.tzinfo import BaseTzInfo + + +logger = logging.getLogger(__name__) + +MAX_PASSES_REACHED_MSG = ( + 'DeepDiff has reached the max number of passes of {}. ' + 'You can possibly get more accurate results by increasing the max_passes parameter.') + +MAX_DIFFS_REACHED_MSG = ( + 'DeepDiff has reached the max number of diffs of {}. ' + 'You can possibly get more accurate results by increasing the max_diffs parameter.') + + +notpresent_indexed = IndexedHash(indexes=[0], item=notpresent) + +doc = get_doc('diff_doc.rst') + + +PROGRESS_MSG = "DeepDiff {} seconds in progress. Pass #{}, Diff #{}" + + +def _report_progress(_stats: Dict[str, Any], progress_logger: Callable[[str], None], duration: float) -> None: + """ + Report the progress every few seconds. + """ + progress_logger(PROGRESS_MSG.format(duration, _stats[PASSES_COUNT], _stats[DIFF_COUNT])) + + +DISTANCE_CACHE_HIT_COUNT = 'DISTANCE CACHE HIT COUNT' +DIFF_COUNT = 'DIFF COUNT' +PASSES_COUNT = 'PASSES COUNT' +MAX_PASS_LIMIT_REACHED = 'MAX PASS LIMIT REACHED' +MAX_DIFF_LIMIT_REACHED = 'MAX DIFF LIMIT REACHED' +DISTANCE_CACHE_ENABLED = 'DISTANCE CACHE ENABLED' +PREVIOUS_DIFF_COUNT = 'PREVIOUS DIFF COUNT' +PREVIOUS_DISTANCE_CACHE_HIT_COUNT = 'PREVIOUS DISTANCE CACHE HIT COUNT' +WORKER_DIFF_COUNT = 'WORKER DIFF COUNT' +WORKER_PASSES_COUNT = 'WORKER PASSES COUNT' +WORKER_DISTANCE_CACHE_HIT_COUNT = 'WORKER DISTANCE CACHE HIT COUNT' +WORKER_BATCH_COUNT = 'WORKER BATCH COUNT' +CANT_FIND_NUMPY_MSG = 'Unable to import numpy. This must be a bug in DeepDiff since a numpy array is detected.' +INVALID_VIEW_MSG = "view parameter must be one of 'text', 'tree', 'delta', 'colored' or 'colored_compact'. But {} was passed." +CUTOFF_RANGE_ERROR_MSG = 'cutoff_distance_for_pairs needs to be a positive float max 1.' +VERBOSE_LEVEL_RANGE_MSG = 'verbose_level should be 0, 1, or 2.' +PURGE_LEVEL_RANGE_MSG = 'cache_purge_level should be 0, 1, or 2.' +_ENABLE_CACHE_EVERY_X_DIFF = '_ENABLE_CACHE_EVERY_X_DIFF' + +model_fields_set = frozenset(["model_fields_set"]) + + +# What is the threshold to consider 2 items to be pairs. Only used when ignore_order = True. +CUTOFF_DISTANCE_FOR_PAIRS_DEFAULT = 0.3 + +# What is the threshold to calculate pairs of items between 2 iterables. +# For example 2 iterables that have nothing in common, do not need their pairs to be calculated. +CUTOFF_INTERSECTION_FOR_PAIRS_DEFAULT = 0.7 + +DEEPHASH_PARAM_KEYS = ( + 'exclude_types', + 'exclude_paths', + 'exclude_glob_paths', + 'include_paths', + 'include_glob_paths', + 'exclude_regex_paths', + 'hasher', + 'significant_digits', + 'number_format_notation', + 'ignore_string_type_changes', + 'ignore_numeric_type_changes', + 'ignore_uuid_types', + 'use_enum_value', + 'ignore_type_in_groups', + 'ignore_type_subclasses', + 'ignore_string_case', + 'exclude_obj_callback', + 'ignore_private_variables', + 'encodings', + 'ignore_encoding_errors', + 'default_timezone', + 'custom_operators', +) + + +class DeepDiffProtocol(Protocol): + t1: Any + t2: Any + cutoff_distance_for_pairs: float + use_log_scale: bool + log_scale_similarity_threshold: float + view: str + math_epsilon: Optional[float] + + + +class DeepDiff(ResultDict, SerializationMixin, DistanceMixin, DeepDiffProtocol, Base): + __doc__ = doc + + CACHE_AUTO_ADJUST_THRESHOLD = 0.25 + + def __init__(self, + t1: Any, + t2: Any, + _original_type: Optional[Any]=None, + cache_purge_level: int=1, + cache_size: int=0, + cache_tuning_sample_size: int=0, + custom_operators: Optional[List[Any]] =None, + cutoff_distance_for_pairs: float=CUTOFF_DISTANCE_FOR_PAIRS_DEFAULT, + cutoff_intersection_for_pairs: float=CUTOFF_INTERSECTION_FOR_PAIRS_DEFAULT, + default_timezone:Union[datetime.timezone, "BaseTzInfo"]=datetime.timezone.utc, + encodings: Optional[List[str]]=None, + exclude_obj_callback: Optional[Callable]=None, + exclude_obj_callback_strict: Optional[Callable]=None, + exclude_paths: Union[str, List[str], Set[str], FrozenSet[str], None]=None, + exclude_regex_paths: Union[str, List[str], Pattern[str], List[Pattern[str]], None]=None, + exclude_types: Optional[List[type]]=None, + get_deep_distance: bool=False, + group_by: Union[str, Tuple[str, str], Callable, None]=None, + group_by_sort_key: Union[str, Callable, None]=None, + hasher: Optional[Callable]=None, + hashes: Optional[Dict[Any, Any]]=None, + ignore_encoding_errors: bool=False, + ignore_nan_inequality: bool=False, + ignore_numeric_type_changes: bool=False, + ignore_order: bool=False, + ignore_order_func: Optional[Callable]=None, + ignore_private_variables: bool=True, + ignore_string_case: bool=False, + ignore_string_type_changes: bool=False, + ignore_type_in_groups: Optional[List[Tuple[Any, ...]]]=None, + ignore_type_subclasses: bool=False, + ignore_uuid_types: bool=False, + include_obj_callback: Optional[Callable]=None, + include_obj_callback_strict: Optional[Callable]=None, + include_paths: Union[str, List[str], None]=None, + iterable_compare_func: Optional[Callable]=None, + log_frequency_in_sec: int=0, + log_scale_similarity_threshold: float=0.1, + log_stacktrace: bool=False, + math_epsilon: Optional[float]=None, + max_diffs: Optional[int]=None, + max_passes: int=10000000, + multiprocessing: bool=False, + multiprocessing_workers: Optional[int]=None, + multiprocessing_threshold: Optional[int]=None, + number_format_notation: Literal["f", "e"]="f", + number_to_string_func: Optional[Callable]=None, + progress_logger: Callable[[str], None]=logger.info, + report_repetition: bool=False, + significant_digits: Optional[int]=None, + threshold_to_diff_deeper: float = 0.33, + truncate_datetime: Optional[str]=None, + use_enum_value: bool=False, + use_log_scale: bool=False, + verbose_level: int=1, + view: str=TEXT_VIEW, + zip_ordered_iterables: bool=False, + _parameters: Optional[Dict[str, Any]]=None, + _shared_parameters: Optional[Dict[str, Any]]=None, + **kwargs): + super().__init__() + # Defaults for glob path attributes — needed for non-root instances + # that may receive _parameters without these keys. + self.exclude_glob_paths = None + self.include_glob_paths = None + if kwargs: + raise ValueError(( + "The following parameter(s) are not valid: %s\n" + "The valid parameters are ignore_order, report_repetition, significant_digits, " + "number_format_notation, exclude_paths, include_paths, exclude_types, exclude_regex_paths, ignore_type_in_groups, " + "ignore_string_type_changes, ignore_numeric_type_changes, ignore_type_subclasses, ignore_uuid_types, truncate_datetime, " + "ignore_private_variables, ignore_nan_inequality, number_to_string_func, verbose_level, " + "view, hasher, hashes, max_passes, max_diffs, zip_ordered_iterables, " + "cutoff_distance_for_pairs, cutoff_intersection_for_pairs, log_frequency_in_sec, cache_size, " + "cache_tuning_sample_size, get_deep_distance, group_by, group_by_sort_key, cache_purge_level, log_stacktrace," + "math_epsilon, iterable_compare_func, use_enum_value, _original_type, threshold_to_diff_deeper, default_timezone " + "ignore_order_func, custom_operators, encodings, ignore_encoding_errors, use_log_scale, log_scale_similarity_threshold " + "multiprocessing, multiprocessing_workers, multiprocessing_threshold, " + "_parameters and _shared_parameters.") % ', '.join(kwargs.keys())) + + if _parameters: + self.__dict__.update(_parameters) + else: + self.custom_operators = custom_operators or [] + self.ignore_order = ignore_order + + self.ignore_order_func = ignore_order_func + + ignore_type_in_groups = ignore_type_in_groups or [] + if numbers == ignore_type_in_groups or numbers in ignore_type_in_groups: + ignore_numeric_type_changes = True + self.ignore_numeric_type_changes = ignore_numeric_type_changes + if strings == ignore_type_in_groups or strings in ignore_type_in_groups: + ignore_string_type_changes = True + # Handle ignore_uuid_types - check if uuid+str group is already in ignore_type_in_groups + uuid_str_group = (uuids[0], str) + if uuid_str_group == ignore_type_in_groups or uuid_str_group in ignore_type_in_groups: + ignore_uuid_types = True + self.ignore_uuid_types = ignore_uuid_types + self.use_enum_value = use_enum_value + self.log_scale_similarity_threshold = log_scale_similarity_threshold + self.use_log_scale = use_log_scale + self.default_timezone = default_timezone + self.log_stacktrace = log_stacktrace + self.threshold_to_diff_deeper = threshold_to_diff_deeper + self.ignore_string_type_changes = ignore_string_type_changes + self.ignore_type_in_groups = self.get_ignore_types_in_groups( + ignore_type_in_groups=ignore_type_in_groups, + ignore_string_type_changes=ignore_string_type_changes, + ignore_numeric_type_changes=ignore_numeric_type_changes, + ignore_type_subclasses=ignore_type_subclasses, + ignore_uuid_types=ignore_uuid_types) + self.report_repetition = report_repetition + _exclude_set = convert_item_or_items_into_set_else_none(exclude_paths) + _exclude_exact, self.exclude_glob_paths = separate_wildcard_and_exact_paths(_exclude_set) + self.exclude_paths = add_root_to_paths(_exclude_exact) + _include_set = convert_item_or_items_into_set_else_none(include_paths) + _include_exact, self.include_glob_paths = separate_wildcard_and_exact_paths(_include_set) + self.include_paths = add_root_to_paths(_include_exact) + self.exclude_regex_paths = convert_item_or_items_into_compiled_regexes_else_none(exclude_regex_paths) + self.exclude_types = set(exclude_types) if exclude_types else None + self.exclude_types_tuple = tuple(exclude_types) if exclude_types else None # we need tuple for checking isinstance + self.ignore_type_subclasses = ignore_type_subclasses + self.type_check_func = type_in_type_group if ignore_type_subclasses else type_is_subclass_of_type_group + self.ignore_string_case = ignore_string_case + self.exclude_obj_callback = exclude_obj_callback + self.exclude_obj_callback_strict = exclude_obj_callback_strict + self.include_obj_callback = include_obj_callback + self.include_obj_callback_strict = include_obj_callback_strict + self.number_to_string = number_to_string_func or number_to_string + self.iterable_compare_func = iterable_compare_func + self.zip_ordered_iterables = zip_ordered_iterables + self.ignore_private_variables = ignore_private_variables + self.ignore_nan_inequality = ignore_nan_inequality + self.hasher = hasher + self.cache_tuning_sample_size = cache_tuning_sample_size + self.group_by = group_by + if callable(group_by_sort_key): + self.group_by_sort_key = group_by_sort_key + elif group_by_sort_key: + def _group_by_sort_key(x): + return x[group_by_sort_key] + self.group_by_sort_key = _group_by_sort_key + else: + self.group_by_sort_key = None + self.encodings = encodings + self.ignore_encoding_errors = ignore_encoding_errors + + self.significant_digits = self.get_significant_digits(significant_digits, ignore_numeric_type_changes) + self.math_epsilon = math_epsilon + if self.math_epsilon is not None and self.ignore_order: + logger.warning("math_epsilon in conjunction with ignore_order=True is only used for flat object comparisons. Custom math_epsilon will not have an effect when comparing nested objects.") + self.truncate_datetime = get_truncate_datetime(truncate_datetime) + self.number_format_notation = number_format_notation + if verbose_level in {0, 1, 2}: + self.verbose_level = verbose_level + else: + raise ValueError(VERBOSE_LEVEL_RANGE_MSG) + if cache_purge_level not in {0, 1, 2}: + raise ValueError(PURGE_LEVEL_RANGE_MSG) + self.view = view + # Setting up the cache for dynamic programming. One dictionary per instance of root of DeepDiff running. + self.max_passes = max_passes + self.max_diffs = max_diffs + self.cutoff_distance_for_pairs = float(cutoff_distance_for_pairs) + self.cutoff_intersection_for_pairs = float(cutoff_intersection_for_pairs) + if self.cutoff_distance_for_pairs < 0 or self.cutoff_distance_for_pairs > 1: + raise ValueError(CUTOFF_RANGE_ERROR_MSG) + # _Parameters are the clean _parameters to initialize DeepDiff with so we avoid all the above + # cleaning functionalities when running DeepDiff recursively. + # However DeepHash has its own set of _parameters that are slightly different than DeepDIff. + # DeepDiff _parameters are transformed to DeepHash _parameters via _get_deephash_params method. + self.progress_logger = progress_logger + self.cache_size = cache_size + self._mp_config = normalize_mp_config( + multiprocessing, multiprocessing_workers, multiprocessing_threshold) + _parameters = self.__dict__.copy() + _parameters['group_by'] = None # overwriting since these parameters will be passed on to other passes. + if log_stacktrace: + self.log_err = logger.exception + else: + self.log_err = logger.error + + # Non-Root + if _shared_parameters: + self.is_root = False + self._shared_parameters = _shared_parameters + self.__dict__.update(_shared_parameters) + # We are in some pass other than root + progress_timer = None + # Root + else: + self.is_root = True + # Caching the DeepDiff results for dynamic programming + self._distance_cache = LFUCache(cache_size) if cache_size else DummyLFU() + self._stats = { + PASSES_COUNT: 0, + DIFF_COUNT: 0, + DISTANCE_CACHE_HIT_COUNT: 0, + PREVIOUS_DIFF_COUNT: 0, + PREVIOUS_DISTANCE_CACHE_HIT_COUNT: 0, + MAX_PASS_LIMIT_REACHED: False, + MAX_DIFF_LIMIT_REACHED: False, + DISTANCE_CACHE_ENABLED: bool(cache_size), + # Multiprocessing aggregates: each parallel batch sums per-worker + # _stats deltas into these keys. Parent-side counters above stay + # comparable to a serial run so existing tests are unaffected. + WORKER_DIFF_COUNT: 0, + WORKER_PASSES_COUNT: 0, + WORKER_DISTANCE_CACHE_HIT_COUNT: 0, + WORKER_BATCH_COUNT: 0, + } + self.hashes = dict_() if hashes is None else hashes + self._numpy_paths = dict_() # if _numpy_paths is None else _numpy_paths + self.group_by_keys = set() # Track keys that originated from group_by operations + self._shared_parameters = { + 'hashes': self.hashes, + '_stats': self._stats, + '_distance_cache': self._distance_cache, + 'group_by_keys': self.group_by_keys, + '_numpy_paths': self._numpy_paths, + _ENABLE_CACHE_EVERY_X_DIFF: self.cache_tuning_sample_size * 10, + } + if log_frequency_in_sec: + # Creating a progress log reporter that runs in a separate thread every log_frequency_in_sec seconds. + progress_timer = RepeatedTimer(log_frequency_in_sec, _report_progress, self._stats, progress_logger) + else: + progress_timer = None + + self._parameters = _parameters + self.deephash_parameters = self._get_deephash_params() + self.tree = TreeResult() + self._iterable_opcodes = {} + if group_by and self.is_root: + try: + original_t1 = t1 + t1 = self._group_iterable_to_dict(t1, group_by, item_name='t1') + except (KeyError, ValueError): + pass + else: + try: + t2 = self._group_iterable_to_dict(t2, group_by, item_name='t2') + except (KeyError, ValueError): + t1 = original_t1 + + self.t1 = t1 + self.t2 = t2 + + try: + root = DiffLevel(t1, t2, verbose_level=self.verbose_level) + # _original_type is only used to pass the original type of the data. Currently only used for numpy arrays. + # The reason is that we convert the numpy array to python list and then later for distance calculations + # we convert only the the last dimension of it into numpy arrays. + self._diff(root, parents_ids=frozenset({id(t1)}), _original_type=_original_type) + + if get_deep_distance and view in {TEXT_VIEW, TREE_VIEW}: + self.tree['deep_distance'] = self._get_rough_distance() + + self.tree.remove_empty_keys() + view_results = self._get_view_results(self.view) + if isinstance(view_results, ColoredView): + self.update(view_results.tree) + self._colored_view = view_results + else: + self.update(view_results) + finally: + if self.is_root: + if cache_purge_level: + del self._distance_cache + del self.hashes + del self._shared_parameters + del self._parameters + for key in (PREVIOUS_DIFF_COUNT, PREVIOUS_DISTANCE_CACHE_HIT_COUNT, + DISTANCE_CACHE_ENABLED): + del self._stats[key] + if progress_timer: + duration = progress_timer.stop() + self._stats['DURATION SEC'] = duration + logger.info('stats {}'.format(self.get_stats())) + if cache_purge_level == 2: + self.__dict__.clear() + + def _get_deephash_params(self): + result = {key: self._parameters.get(key) for key in DEEPHASH_PARAM_KEYS} + result['ignore_repetition'] = not self.report_repetition + result['number_to_string_func'] = self.number_to_string + return result + + def _report_result(self, report_type, change_level, local_tree=None): + """ + Add a detected change to the reference-style result dictionary. + report_type will be added to level. + (We'll create the text-style report from there later.) + :param report_type: A well defined string key describing the type of change. + Examples: "set_item_added", "values_changed" + :param change_level: A DiffLevel object describing the objects in question in their + before-change and after-change object structure. + + :local_tree: None + """ + + if not self._skip_this(change_level): + if self._skip_report_for_include_glob(change_level): + return + change_level.report_type = report_type + tree = self.tree if local_tree is None else local_tree + tree[report_type].add(change_level) + + def custom_report_result(self, report_type, level, extra_info=None): + """ + Add a detected change to the reference-style result dictionary. + report_type will be added to level. + (We'll create the text-style report from there later.) + + :param report_type: A well defined string key describing the type of change. + Examples: "set_item_added", "values_changed" + :param parent: A DiffLevel object describing the objects in question in their + before-change and after-change object structure. + :param extra_info: A dict that describe this result + :rtype: None + """ + + if not self._skip_this(level): + if self._skip_report_for_include_glob(level): + return + level.report_type = report_type + level.additional[CUSTOM_FIELD] = extra_info + self.tree[report_type].add(level) + + def _skip_report_for_include_glob(self, level): + """When include_glob_paths is set, _skip_this allows ancestors through for traversal. + This method does a stricter check at report time: only report if the path + actually matches a glob pattern or is a descendant of a matching path, + or if it already matches an exact include_path.""" + if not self.include_glob_paths: + return False + level_path = level.path() + # If exact include_paths already matched, don't skip + if self.include_paths: + if level_path in self.include_paths: + return False + for prefix in self.include_paths: + if prefix in level_path: + return False + # Check glob patterns: match or descendant + for gp in self.include_glob_paths: + if gp.match_or_is_descendant(level_path): + return False + return True + + @staticmethod + def _dict_from_slots(object: Any) -> Dict[str, Any]: + def unmangle(attribute: str) -> str: + if attribute.startswith('__') and attribute != '__weakref__': + return '_{type}{attribute}'.format( + type=type(object).__name__, + attribute=attribute + ) + return attribute + + all_slots = [] + + if isinstance(object, type): + mro = object.__mro__ # pragma: no cover. I have not been able to write a test for this case. But we still check for it. + else: + mro = object.__class__.__mro__ + + for type_in_mro in mro: + slots = getattr(type_in_mro, '__slots__', None) + if slots: + if isinstance(slots, strings): + all_slots.append(slots) + else: + all_slots.extend(slots) + + return {i: getattr(object, key) for i in all_slots if hasattr(object, key := unmangle(i))} + + def _diff_enum(self, level: Any, parents_ids: FrozenSet[int]=frozenset(), local_tree: Optional[Any]=None) -> None: + t1 = detailed__dict__(level.t1, include_keys=ENUM_INCLUDE_KEYS) + t2 = detailed__dict__(level.t2, include_keys=ENUM_INCLUDE_KEYS) + + self._diff_dict( + level, + parents_ids, + print_as_attribute=True, + override=True, + override_t1=t1, + override_t2=t2, + local_tree=local_tree, + ) + + def _diff_obj(self, level: Any, parents_ids: FrozenSet[int]=frozenset(), is_namedtuple: bool=False, local_tree: Optional[Any]=None, is_pydantic_object: bool=False) -> None: + """Difference of 2 objects""" + processing_error = False + t1: Optional[Dict[str, Any]] = None + t2: Optional[Dict[str, Any]] = None + try: + if is_namedtuple: + t1 = level.t1._asdict() + t2 = level.t2._asdict() + elif is_pydantic_object: + t1 = detailed__dict__(level.t1, ignore_private_variables=self.ignore_private_variables, ignore_keys=model_fields_set) + t2 = detailed__dict__(level.t2, ignore_private_variables=self.ignore_private_variables, ignore_keys=model_fields_set) + elif all('__dict__' in dir(t) for t in level): + t1 = detailed__dict__(level.t1, ignore_private_variables=self.ignore_private_variables) + t2 = detailed__dict__(level.t2, ignore_private_variables=self.ignore_private_variables) + elif all('__slots__' in dir(t) for t in level): + t1 = self._dict_from_slots(level.t1) + t2 = self._dict_from_slots(level.t2) + else: + t1 = {k: v for k, v in getmembers(level.t1) if not callable(v)} + t2 = {k: v for k, v in getmembers(level.t2) if not callable(v)} + except AttributeError: + processing_error = True + if processing_error is True or t1 is None or t2 is None: + self._report_result('unprocessed', level, local_tree=local_tree) + return + + self._diff_dict( + level, + parents_ids, + print_as_attribute=True, + override=True, + override_t1=t1, + override_t2=t2, + local_tree=local_tree, + ) + + def _skip_this(self, level: Any) -> bool: + """ + Check whether this comparison should be skipped because one of the objects to compare meets exclusion criteria. + :rtype: bool + """ + level_path = level.path() + skip = False + if self.exclude_paths and level_path in self.exclude_paths: + skip = True + elif self.exclude_glob_paths and any(gp.match(level_path) for gp in self.exclude_glob_paths): + skip = True + if not skip and (self.include_paths or self.include_glob_paths) and level_path != 'root': + skip = True + if self.include_paths: + if level_path in self.include_paths: + skip = False + else: + for prefix in self.include_paths: + if prefix in level_path or level_path in prefix: + skip = False + break + if skip and self.include_glob_paths: + for gp in self.include_glob_paths: + if gp.match_or_is_ancestor(level_path): + skip = False + break + elif self.exclude_regex_paths and any( + [exclude_regex_path.search(level_path) for exclude_regex_path in self.exclude_regex_paths]): + skip = True + elif self.exclude_types_tuple and \ + (isinstance(level.t1, self.exclude_types_tuple) or isinstance(level.t2, self.exclude_types_tuple)): + skip = True + elif self.exclude_obj_callback and \ + (self.exclude_obj_callback(level.t1, level_path) or self.exclude_obj_callback(level.t2, level_path)): + skip = True + elif self.exclude_obj_callback_strict and \ + (self.exclude_obj_callback_strict(level.t1, level_path) and + self.exclude_obj_callback_strict(level.t2, level_path)): + skip = True + elif self.include_obj_callback and level_path != 'root': + skip = True + if (self.include_obj_callback(level.t1, level_path) or self.include_obj_callback(level.t2, level_path)): + skip = False + elif self.include_obj_callback_strict and level_path != 'root': + skip = True + if (self.include_obj_callback_strict(level.t1, level_path) and + self.include_obj_callback_strict(level.t2, level_path)): + skip = False + + return skip + + def _skip_this_key(self, level: Any, key: Any) -> bool: + # if include_paths is not set, than treet every path as included + if self.include_paths is None and self.include_glob_paths is None: + return False + key_path = "{}['{}']".format(level.path(), key) + if self.include_paths: + if key_path in self.include_paths: + return False + if level.path() in self.include_paths: + # matches e.g. level+key root['foo']['bar']['veg'] include_paths ["root['foo']['bar']"] + return False + for prefix in self.include_paths: + if key_path in prefix: + # matches as long the prefix is longer than this object key + # eg.: level+key root['foo']['bar'] matches prefix root['foo']['bar'] from include paths + # level+key root['foo'] matches prefix root['foo']['bar'] from include_paths + # level+key root['foo']['bar'] DOES NOT match root['foo'] from include_paths This needs to be handled afterwards + return False + # check if a higher level is included as a whole (=without any sublevels specified) + # matches e.g. level+key root['foo']['bar']['veg'] include_paths ["root['foo']"] + # but does not match, if it is level+key root['foo']['bar']['veg'] include_paths ["root['foo']['bar']['fruits']"] + up = level.up + while up is not None: + if up.path() in self.include_paths: + return False + up = up.up + if self.include_glob_paths: + for gp in self.include_glob_paths: + if gp.match_or_is_ancestor(key_path): + return False + return True + + def _get_clean_to_keys_mapping(self, keys: Any, level: Any) -> Dict[Any, Any]: + """ + Get a dictionary of cleaned value of keys to the keys themselves. + This is mainly used to transform the keys when the type changes of keys should be ignored. + + TODO: needs also some key conversion for groups of types other than the built-in strings and numbers. + """ + result = dict_() + for key in keys: + if self.ignore_string_type_changes and isinstance(key, bytes): + clean_key = key.decode('utf-8') + elif self.ignore_string_type_changes and isinstance(key, memoryview): + clean_key = key.tobytes().decode('utf-8') + elif self.use_enum_value and isinstance(key, Enum): + clean_key = key.value + elif isinstance(key, numbers): + # Skip type prefixing for keys that originated from group_by operations + if hasattr(self, 'group_by_keys') and key in self.group_by_keys: + if self.significant_digits is None: + clean_key = key + else: + clean_key = self.number_to_string(key, significant_digits=self.significant_digits, + number_format_notation=self.number_format_notation) # type: ignore # type: ignore + else: + type_ = "number" if self.ignore_numeric_type_changes else key.__class__.__name__ + if self.significant_digits is None: + clean_key = key + else: + clean_key = self.number_to_string(key, significant_digits=self.significant_digits, + number_format_notation=self.number_format_notation) # type: ignore # type: ignore + clean_key = KEY_TO_VAL_STR.format(type_, clean_key) + else: + clean_key = key + if self.ignore_string_case and isinstance(clean_key, str): + clean_key = clean_key.lower() + if clean_key in result: + logger.warning(('{} and {} in {} become the same key when ignore_numeric_type_changes' + 'or ignore_numeric_type_changes are set to be true.').format( + key, result[clean_key], level.path())) + else: + result[clean_key] = key + return result + + def _diff_dict( + self, + level: Any, + parents_ids: FrozenSet[int]=frozenset([]), + print_as_attribute: bool=False, + override: bool=False, + override_t1: Any=None, + override_t2: Any=None, + local_tree: Optional[Any]=None, + ) -> None: + """Difference of 2 dictionaries""" + if override: + # for special stuff like custom objects and named tuples we receive preprocessed t1 and t2 + # but must not spoil the chain (=level) with it + t1 = override_t1 + t2 = override_t2 + else: + t1 = level.t1 + t2 = level.t2 + + if print_as_attribute: + item_added_key = "attribute_added" + item_removed_key = "attribute_removed" + rel_class = AttributeRelationship + else: + item_added_key = "dictionary_item_added" + item_removed_key = "dictionary_item_removed" + rel_class = DictRelationship + + if self.ignore_private_variables: + t1_keys = SetOrdered([key for key in t1 if not(isinstance(key, str) and key.startswith('__')) and not self._skip_this_key(level, key)]) + t2_keys = SetOrdered([key for key in t2 if not(isinstance(key, str) and key.startswith('__')) and not self._skip_this_key(level, key)]) + else: + t1_keys = SetOrdered([key for key in t1 if not self._skip_this_key(level, key)]) + t2_keys = SetOrdered([key for key in t2 if not self._skip_this_key(level, key)]) + if self.ignore_string_type_changes or self.ignore_numeric_type_changes or self.ignore_string_case: + t1_clean_to_keys = self._get_clean_to_keys_mapping(keys=t1_keys, level=level) + t2_clean_to_keys = self._get_clean_to_keys_mapping(keys=t2_keys, level=level) + t1_keys = SetOrdered(t1_clean_to_keys.keys()) + t2_keys = SetOrdered(t2_clean_to_keys.keys()) + else: + t1_clean_to_keys = t2_clean_to_keys = None + + t_keys_intersect = t2_keys & t1_keys + t_keys_added = t2_keys - t_keys_intersect + t_keys_removed = t1_keys - t_keys_intersect + + if self.threshold_to_diff_deeper: + if self.exclude_paths or self.exclude_glob_paths: + t_keys_union = {f"{level.path()}[{repr(key)}]" for key in (t2_keys | t1_keys)} + if self.exclude_paths: + t_keys_union -= self.exclude_paths + if self.exclude_glob_paths: + t_keys_union = {k for k in t_keys_union + if not any(gp.match(k) for gp in self.exclude_glob_paths)} + t_keys_union_len = len(t_keys_union) + else: + t_keys_union_len = len(t2_keys | t1_keys) + if t_keys_union_len > 1 and len(t_keys_intersect) / t_keys_union_len < self.threshold_to_diff_deeper: + self._report_result('values_changed', level, local_tree=local_tree) + return + + for key in t_keys_added: + if self._count_diff() is StopIteration: + return + + key = t2_clean_to_keys[key] if t2_clean_to_keys else key + change_level = level.branch_deeper( + notpresent, + t2[key], + child_relationship_class=rel_class, + child_relationship_param=key, + child_relationship_param2=key, + ) + self._report_result(item_added_key, change_level, local_tree=local_tree) + + for key in t_keys_removed: + if self._count_diff() is StopIteration: + return # pragma: no cover. This is already covered for addition. + + key = t1_clean_to_keys[key] if t1_clean_to_keys else key + change_level = level.branch_deeper( + t1[key], + notpresent, + child_relationship_class=rel_class, + child_relationship_param=key, + child_relationship_param2=key, + ) + self._report_result(item_removed_key, change_level, local_tree=local_tree) + + for key in t_keys_intersect: # key present in both dicts - need to compare values + if self._count_diff() is StopIteration: + return # pragma: no cover. This is already covered for addition. + + key1 = t1_clean_to_keys[key] if t1_clean_to_keys else key + key2 = t2_clean_to_keys[key] if t2_clean_to_keys else key + item_id = id(t1[key1]) + if parents_ids and item_id in parents_ids: + continue + parents_ids_added = add_to_frozen_set(parents_ids, item_id) + + # Go one level deeper + next_level = level.branch_deeper( + t1[key1], + t2[key2], + child_relationship_class=rel_class, + child_relationship_param=key, + child_relationship_param2=key, + ) + self._diff(next_level, parents_ids_added, local_tree=local_tree) + + def _diff_set(self, level: Any, local_tree: Optional[Any]=None) -> None: + """Difference of sets""" + t1_hashtable = self._create_hashtable(level, 't1') + t2_hashtable = self._create_hashtable(level, 't2') + + t1_hashes = set(t1_hashtable.keys()) + t2_hashes = set(t2_hashtable.keys()) + + hashes_added = t2_hashes - t1_hashes + hashes_removed = t1_hashes - t2_hashes + + items_added = [t2_hashtable[i].item for i in hashes_added] + items_removed = [t1_hashtable[i].item for i in hashes_removed] + + for item in items_added: + if self._count_diff() is StopIteration: + return # pragma: no cover. This is already covered for addition. + + change_level = level.branch_deeper( + notpresent, item, child_relationship_class=SetRelationship) + self._report_result('set_item_added', change_level, local_tree=local_tree) + + for item in items_removed: + if self._count_diff() is StopIteration: + return # pragma: no cover. This is already covered for addition. + + change_level = level.branch_deeper( + item, notpresent, child_relationship_class=SetRelationship) + self._report_result('set_item_removed', change_level, local_tree=local_tree) + + @staticmethod + def _iterables_subscriptable(t1: Any, t2: Any) -> bool: + try: + if getattr(t1, '__getitem__') and getattr(t2, '__getitem__'): + return True + else: # pragma: no cover + return False # should never happen + except AttributeError: + return False + + def _diff_iterable(self, level: Any, parents_ids: FrozenSet[int]=frozenset(), _original_type: Optional[type]=None, local_tree: Optional[Any]=None) -> None: + """Difference of iterables""" + if (self.ignore_order_func and self.ignore_order_func(level)) or self.ignore_order: + self._diff_iterable_with_deephash(level, parents_ids, _original_type=_original_type, local_tree=local_tree) + else: + self._diff_iterable_in_order(level, parents_ids, _original_type=_original_type, local_tree=local_tree) + + def _compare_in_order( + self, level, + t1_from_index: Optional[int]=None, t1_to_index: Optional[int]=None, + t2_from_index: Optional[int]=None, t2_to_index: Optional[int]=None + ) -> List[Tuple[Tuple[int, int], Tuple[Any, Any]]]: + """ + Default compare if `iterable_compare_func` is not provided. + This will compare in sequence order. + """ + if t1_from_index is None or t2_from_index is None: + return [((i, i), (x, y)) for i, (x, y) in enumerate( + zip_longest( + level.t1, level.t2, fillvalue=ListItemRemovedOrAdded))] + else: + t1_chunk = level.t1[t1_from_index:t1_to_index] + t2_chunk = level.t2[t2_from_index:t2_to_index] + return [((i + t1_from_index, i + t2_from_index), (x, y)) for i, (x, y) in enumerate( + zip_longest( + t1_chunk, t2_chunk, fillvalue=ListItemRemovedOrAdded))] + + def _get_matching_pairs( + self, level, + t1_from_index=None, t1_to_index=None, + t2_from_index=None, t2_to_index=None + ) -> List[Tuple[Tuple[int, int], Tuple[Any, Any]]]: + """ + Given a level get matching pairs. This returns list of two tuples in the form: + [ + (t1 index, t2 index), (t1 item, t2 item) + ] + + This will compare using the passed in `iterable_compare_func` if available. + Default it to compare in order + """ + + if self.iterable_compare_func is None: + # Match in order if there is no compare function provided + return self._compare_in_order( + level, + t1_from_index=t1_from_index, t1_to_index=t1_to_index, + t2_from_index=t2_from_index, t2_to_index=t2_to_index, + ) + try: + matches = [] + y_matched = set() + y_index_matched = set() + for i, x in enumerate(level.t1): + x_found = False + for j, y in enumerate(level.t2): + + if(j in y_index_matched): + # This ensures a one-to-one relationship of matches from t1 to t2. + # If y this index in t2 has already been matched to another x + # it cannot have another match, so just continue. + continue + + if(self.iterable_compare_func(x, y, level)): + deep_hash = DeepHash(y, + hashes=self.hashes, + apply_hash=True, + **self.deephash_parameters, + ) + y_index_matched.add(j) + y_matched.add(deep_hash[y]) + matches.append(((i, j), (x, y))) + x_found = True + break + + if(not x_found): + matches.append(((i, -1), (x, ListItemRemovedOrAdded))) + for j, y in enumerate(level.t2): + + deep_hash = DeepHash(y, + hashes=self.hashes, + apply_hash=True, + **self.deephash_parameters, + ) + if(deep_hash[y] not in y_matched): + matches.append(((-1, j), (ListItemRemovedOrAdded, y))) + return matches + except CannotCompare: + return self._compare_in_order( + level, + t1_from_index=t1_from_index, t1_to_index=t1_to_index, + t2_from_index=t2_from_index, t2_to_index=t2_to_index + ) + + def _diff_iterable_in_order(self, level, parents_ids=frozenset(), _original_type=None, local_tree=None): + # We're handling both subscriptable and non-subscriptable iterables. Which one is it? + subscriptable = self._iterables_subscriptable(level.t1, level.t2) + if subscriptable: + child_relationship_class = SubscriptableIterableRelationship + else: + child_relationship_class = NonSubscriptableIterableRelationship + + if ( + not self.zip_ordered_iterables + and isinstance(level.t1, Sequence) + and isinstance(level.t2, Sequence) + and self._all_values_basic_hashable(level.t1) + and self._all_values_basic_hashable(level.t2) + and self.iterable_compare_func is None + ): + local_tree_pass = TreeResult() + opcodes_with_values = self._diff_ordered_iterable_by_difflib( + level, + parents_ids=parents_ids, + _original_type=_original_type, + child_relationship_class=child_relationship_class, + local_tree=local_tree_pass, + ) + # Sometimes DeepDiff's old iterable diff does a better job than DeepDiff + if len(local_tree_pass) > 1: + local_tree_pass2 = TreeResult() + self._diff_by_forming_pairs_and_comparing_one_by_one( + level, + parents_ids=parents_ids, + _original_type=_original_type, + child_relationship_class=child_relationship_class, + local_tree=local_tree_pass2, + ) + if len(local_tree_pass) >= len(local_tree_pass2): + local_tree_pass = local_tree_pass2 + else: + self._iterable_opcodes[level.path(force=FORCE_DEFAULT)] = opcodes_with_values + for report_type, levels in local_tree_pass.items(): + if levels: + self.tree[report_type] |= levels + else: + self._diff_by_forming_pairs_and_comparing_one_by_one( + level, + parents_ids=parents_ids, + _original_type=_original_type, + child_relationship_class=child_relationship_class, + local_tree=local_tree, + ) + + def _all_values_basic_hashable(self, iterable: Iterable[Any]) -> bool: + """ + Are all items basic hashable types? + Or there are custom types too? + """ + + # We don't want to exhaust a generator + if isinstance(iterable, types.GeneratorType): + return False + for item in iterable: + if not isinstance(item, basic_types): + return False + return True + + def _diff_by_forming_pairs_and_comparing_one_by_one( + self, level, local_tree, parents_ids=frozenset(), + _original_type=None, child_relationship_class=None, + t1_from_index=None, t1_to_index=None, + t2_from_index=None, t2_to_index=None, + ): + for (i, j), (x, y) in self._get_matching_pairs( + level, + t1_from_index=t1_from_index, t1_to_index=t1_to_index, + t2_from_index=t2_from_index, t2_to_index=t2_to_index + ): + if self._count_diff() is StopIteration: + return # pragma: no cover. This is already covered for addition. + + reference_param1 = i + reference_param2 = j + if y is ListItemRemovedOrAdded: # item removed completely + change_level = level.branch_deeper( + x, + notpresent, + child_relationship_class=child_relationship_class, + child_relationship_param=reference_param1, + child_relationship_param2=reference_param2, + ) + self._report_result('iterable_item_removed', change_level, local_tree=local_tree) + + elif x is ListItemRemovedOrAdded: # new item added + change_level = level.branch_deeper( + notpresent, + y, + child_relationship_class=child_relationship_class, + child_relationship_param=reference_param1, + child_relationship_param2=reference_param2, + ) + self._report_result('iterable_item_added', change_level, local_tree=local_tree) + + else: # check if item value has changed + if (i != j and ((x == y) or self.iterable_compare_func)): + # Item moved + change_level = level.branch_deeper( + x, + y, + child_relationship_class=child_relationship_class, + child_relationship_param=reference_param1, + child_relationship_param2=reference_param2 + ) + self._report_result('iterable_item_moved', change_level, local_tree=local_tree) + + if self.iterable_compare_func: + # Mark additional context denoting that we have moved an item. + # This will allow for correctly setting paths relative to t2 when using an iterable_compare_func + level.additional["moved"] = True + + else: + continue + + item_id = id(x) + if parents_ids and item_id in parents_ids: + continue + parents_ids_added = add_to_frozen_set(parents_ids, item_id) + + # Go one level deeper + next_level = level.branch_deeper( + x, + y, + child_relationship_class=child_relationship_class, + child_relationship_param=reference_param1, + child_relationship_param2=reference_param2 + ) + self._diff(next_level, parents_ids_added, local_tree=local_tree) + + def _diff_ordered_iterable_by_difflib( + self, level, local_tree, parents_ids=frozenset(), _original_type=None, child_relationship_class=None, + ): + + seq = difflib.SequenceMatcher(isjunk=None, a=level.t1, b=level.t2, autojunk=False) + + opcodes = seq.get_opcodes() + opcodes_with_values = [] + + # TODO: this logic should be revisted so we detect reverse operations + # like when a replacement happens at index X and a reverse replacement happens at index Y + # in those cases we have a "iterable_item_moved" operation. + for tag, t1_from_index, t1_to_index, t2_from_index, t2_to_index in opcodes: + if tag == 'equal': + opcodes_with_values.append(Opcode( + tag, t1_from_index, t1_to_index, t2_from_index, t2_to_index, + )) + continue + # print('{:7} t1[{}:{}] --> t2[{}:{}] {!r:>8} --> {!r}'.format( + # tag, t1_from_index, t1_to_index, t2_from_index, t2_to_index, level.t1[t1_from_index:t1_to_index], level.t2[t2_from_index:t2_to_index])) + + opcodes_with_values.append(Opcode( + tag, t1_from_index, t1_to_index, t2_from_index, t2_to_index, + old_values = level.t1[t1_from_index: t1_to_index], + new_values = level.t2[t2_from_index: t2_to_index], + )) + + if tag == 'replace': + self._diff_by_forming_pairs_and_comparing_one_by_one( + level, local_tree=local_tree, parents_ids=parents_ids, + _original_type=_original_type, child_relationship_class=child_relationship_class, + t1_from_index=t1_from_index, t1_to_index=t1_to_index, + t2_from_index=t2_from_index, t2_to_index=t2_to_index, + ) + elif tag == 'delete': + for index, x in enumerate(level.t1[t1_from_index:t1_to_index]): + change_level = level.branch_deeper( + x, + notpresent, + child_relationship_class=child_relationship_class, + child_relationship_param=index + t1_from_index, + child_relationship_param2=index + t1_from_index, + ) + self._report_result('iterable_item_removed', change_level, local_tree=local_tree) + elif tag == 'insert': + for index, y in enumerate(level.t2[t2_from_index:t2_to_index]): + change_level = level.branch_deeper( + notpresent, + y, + child_relationship_class=child_relationship_class, + child_relationship_param=index + t2_from_index, + child_relationship_param2=index + t2_from_index, + ) + self._report_result('iterable_item_added', change_level, local_tree=local_tree) + return opcodes_with_values + + + def _diff_str(self, level, local_tree=None): + """Compare strings""" + if self.ignore_string_case: + level.t1 = level.t1.lower() + level.t2 = level.t2.lower() + + if type(level.t1) == type(level.t2) and level.t1 == level.t2: # NOQA + return + + # do we add a diff for convenience? + do_diff = True + t1_str = level.t1 + t2_str = level.t2 + + if isinstance(level.t1, memoryview): + try: + t1_str = level.t1.tobytes().decode('ascii') + except UnicodeDecodeError: + do_diff = False + elif isinstance(level.t1, bytes_type): + try: + t1_str = level.t1.decode('ascii') + except UnicodeDecodeError: + do_diff = False + + if isinstance(level.t2, memoryview): + try: + t2_str = level.t2.tobytes().decode('ascii') + except UnicodeDecodeError: + do_diff = False + elif isinstance(level.t2, bytes_type): + try: + t2_str = level.t2.decode('ascii') + except UnicodeDecodeError: + do_diff = False + + if isinstance(level.t1, Enum): + t1_str = level.t1.value + + if isinstance(level.t2, Enum): + t2_str = level.t2.value + + if t1_str == t2_str: + return + + if do_diff: + if '\n' in t1_str or isinstance(t2_str, str) and '\n' in t2_str: + diff = difflib.unified_diff( + t1_str.splitlines(), t2_str.splitlines(), lineterm='') + diff = list(diff) + if diff: + level.additional['diff'] = '\n'.join(diff) + + self._report_result('values_changed', level, local_tree=local_tree) + + def _diff_tuple(self, level, parents_ids, local_tree=None): + # Checking to see if it has _fields. Which probably means it is a named + # tuple. + try: + level.t1._asdict + # It must be a normal tuple + except AttributeError: + self._diff_iterable(level, parents_ids, local_tree=local_tree) + # We assume it is a namedtuple then + else: + self._diff_obj(level, parents_ids, is_namedtuple=True, local_tree=local_tree) + + def _add_hash(self, hashes, item_hash, item, i): + if item_hash in hashes: + hashes[item_hash].indexes.append(i) + else: + hashes[item_hash] = IndexedHash(indexes=[i], item=item) + + def _maybe_compute_hashes_parallel(self, level, obj): + """Optionally hash iterable items in worker processes. + + Returns a list of per-index ``item_hash`` values (or ``None`` for + items the worker could not process), aligned to ``enumerate(obj)`` + order. Returns ``None`` when the section ran serially (no + ``_mp_config``, below threshold, generator without ``__len__``, + unsafe inputs). + + Iteration order is captured here once via ``list(obj)`` so the + parent loop and the worker job list see the same items even for + order-sensitive iterables like sets. + """ + mp_config = getattr(self, '_mp_config', None) + if mp_config is None or not mp_config.enabled: + return None, None + try: + n = len(obj) + except TypeError: + # Generators / unsized iterables: serial fallback. Materializing + # would change semantics (single-pass consumption). + return None, None + if not mp_config.should_parallelize(n): + return None, None + + items = list(obj) + parent_base = level.path() + jobs = [ + (item, "{}[{}]".format(parent_base, i)) + for i, item in enumerate(items) + ] + hashes = compute_hashes_parallel( + jobs=jobs, + deephash_parameters=self.deephash_parameters, + config=mp_config, + ) + if hashes is None: + return None, None + return hashes, items + + def _create_hashtable(self, level, t): + """Create hashtable of {item_hash: (indexes, item)}""" + obj = getattr(level, t) + + # Optionally precompute item hashes in worker processes. Workers + # operate on serial-order job indices and the parent merges back + # in that same order, so output is independent of worker + # completion order. ``items`` is the materialized iterable when + # parallel ran (set/dict iteration is deterministic per run but we + # need a single pass we can re-walk here). + parallel_hashes, materialized_items = self._maybe_compute_hashes_parallel(level, obj) + iterator = enumerate(materialized_items) if materialized_items is not None else enumerate(obj) + + local_hashes = dict_() + for (i, item) in iterator: + try: + parent = "{}[{}]".format(level.path(), i) + if parallel_hashes is not None: + item_hash = parallel_hashes[i] + if item_hash is None: + # Worker could not process this item (KeyError or + # unprocessed marker). Mirror the serial pass: + # log once, skip. + self.log_err("Item %s was not processed while hashing " + "thus not counting this object." % + level.path()) + continue + self._add_hash(hashes=local_hashes, item_hash=item_hash, item=item, i=i) + continue + # Note: in the DeepDiff we only calculate the hash of items when we have to. + # So self.hashes does not include hashes of all objects in t1 and t2. + # It only includes the ones needed when comparing iterables. + # The self.hashes dictionary gets shared between different runs of DeepHash + # So that any object that is already calculated to have a hash is not re-calculated. + deep_hash = DeepHash( + item, + hashes=self.hashes, + parent=parent, + apply_hash=True, + **self.deephash_parameters, + ) + except UnicodeDecodeError as err: + err.reason = f"Can not produce a hash for {level.path()}: {err.reason}" + raise + except NotImplementedError: + raise + # except Exception as e: # pragma: no cover + # logger.error("Can not produce a hash for %s." + # "Not counting this object.\n %s" % + # (level.path(), e)) + else: + try: + item_hash = deep_hash[item] + except KeyError: + pass + else: + if item_hash is unprocessed: # pragma: no cover + self.log_err("Item %s was not processed while hashing " + "thus not counting this object." % + level.path()) + else: + self._add_hash(hashes=local_hashes, item_hash=item_hash, item=item, i=i) + + # Also we hash the iterables themselves too so that we can later create cache keys from those hashes. + # When the per-item loop ran in parallel, child hashes were not merged into ``self.hashes`` + # (cross-process id keys would not match). The iterable-level pass therefore re-hashes + # children serially; this is intentional — correctness over cache reuse for now. + DeepHash( + obj, + hashes=self.hashes, + parent=level.path(), + apply_hash=True, + **self.deephash_parameters, + ) + return local_hashes + + @staticmethod + @lru_cache(maxsize=2028) + def _get_distance_cache_key(added_hash, removed_hash): + key1, key2 = (added_hash, removed_hash) if added_hash > removed_hash else (removed_hash, added_hash) + if isinstance(key1, int): + # If the hash function produces integers we convert them to hex values. + # This was used when the default hash function was Murmur3 128bit which produces integers. + key1 = hex(key1).encode('utf-8') + key2 = hex(key2).encode('utf-8') + elif isinstance(key1, str): + key1 = key1.encode('utf-8') + key2 = key2.encode('utf-8') + return key1 + b'--' + key2 + b'dc' + + def _get_rough_distance_of_hashed_objs( + self, added_hash, removed_hash, added_hash_obj, removed_hash_obj, _original_type=None): + # We need the rough distance between the 2 objects to see if they qualify to be pairs or not + _distance = cache_key = None + if self._stats[DISTANCE_CACHE_ENABLED]: + cache_key = self._get_distance_cache_key(added_hash, removed_hash) + cached_distance = self._distance_cache.get(cache_key) + if cached_distance is not_found: + _distance = None + else: + self._stats[DISTANCE_CACHE_HIT_COUNT] += 1 + _distance = cast(float, cached_distance) + if _distance is None: + # We can only cache the rough distance and not the actual diff result for reuse. + # The reason is that we have modified the parameters explicitly so they are different and can't + # be used for diff reporting + diff = DeepDiff( + removed_hash_obj.item, added_hash_obj.item, + _parameters=self._parameters, + _shared_parameters=self._shared_parameters, + view=DELTA_VIEW, + _original_type=_original_type, + iterable_compare_func=self.iterable_compare_func, + ) + _distance = diff._get_rough_distance() + if cache_key and self._stats[DISTANCE_CACHE_ENABLED]: + self._distance_cache.set(cache_key, value=_distance) + return _distance + + def _maybe_compute_pair_distances_parallel( + self, hashes_added, hashes_removed, t1_hashtable, t2_hashtable, + parents_ids, _original_type, pre_calced_distances): + """Optionally run distance computation for non-cached pairs in workers. + + Returns a dict ``{(added_hash, removed_hash): distance}`` for pairs + whose distance was computed in parallel, or ``None`` if the section + ran serially (below threshold, unsafe inputs, no _mp_config, etc.). + + The job list is built in the exact order of the serial nested loop + so the parent merge order is identical regardless of how many workers + run or which one finishes first. + """ + mp_config = getattr(self, '_mp_config', None) + if mp_config is None or not mp_config.enabled: + return None + + # Build candidate job list in stable nested-loop order. We skip pairs + # that the serial loop also skips (loop detection, pre-calculated + # distance, distance cache hit) so workers only get real work. + jobs = [] + cache_enabled = self._stats[DISTANCE_CACHE_ENABLED] + for added_hash in hashes_added: + for removed_hash in hashes_removed: + added_hash_obj = t2_hashtable[added_hash] + removed_hash_obj = t1_hashtable[removed_hash] + if id(removed_hash_obj.item) in parents_ids: + continue + if pre_calced_distances and pre_calced_distances.get( + "{}--{}".format(added_hash, removed_hash)) is not None: + continue + if cache_enabled: + cache_key = self._get_distance_cache_key(added_hash, removed_hash) + if self._distance_cache.get(cache_key) is not not_found: + # Serial path will pull this from cache; no worker + # needed and we keep cache-hit accounting in the + # parent. + continue + jobs.append((added_hash, removed_hash, added_hash_obj.item, removed_hash_obj.item)) + + if not mp_config.should_parallelize(len(jobs)): + return None + + result = compute_distances_parallel( + jobs=jobs, + parameters=self._parameters, + original_type=_original_type, + iterable_compare_func=self.iterable_compare_func, + config=mp_config, + ) + if result is None: + return None + distances, worker_stats = result + self._merge_worker_stats(worker_stats) + return distances + + def _merge_worker_stats(self, worker_stats): + """Aggregate one parallel-batch's worker ``_stats`` delta into self._stats. + + Counters (DIFF / PASSES / DISTANCE CACHE HIT) sum into the matching + ``WORKER_*`` keys; limit flags OR-merge into the parent's existing + MAX_*_LIMIT_REACHED flags so any worker hitting a guard surfaces the + same warning state on the public ``get_stats()`` output. + """ + if not worker_stats: + return + self._stats[WORKER_DIFF_COUNT] += int(worker_stats.get('DIFF COUNT', 0) or 0) + self._stats[WORKER_PASSES_COUNT] += int(worker_stats.get('PASSES COUNT', 0) or 0) + self._stats[WORKER_DISTANCE_CACHE_HIT_COUNT] += int( + worker_stats.get('DISTANCE CACHE HIT COUNT', 0) or 0) + self._stats[WORKER_BATCH_COUNT] += 1 + if worker_stats.get(MAX_PASS_LIMIT_REACHED): + self._stats[MAX_PASS_LIMIT_REACHED] = True + if worker_stats.get(MAX_DIFF_LIMIT_REACHED): + self._stats[MAX_DIFF_LIMIT_REACHED] = True + + def _get_most_in_common_pairs_in_iterables( + self, hashes_added, hashes_removed, t1_hashtable, t2_hashtable, parents_ids, _original_type): + """ + Get the closest pairs between items that are removed and items that are added. + + returns a dictionary of hashes that are closest to each other. + The dictionary is going to be symmetrical so any key will be a value too and otherwise. + + Note that due to the current reporting structure in DeepDiff, we don't compare an item that + was added to an item that is in both t1 and t2. + + For example + + [{1, 2}, {4, 5, 6}] + [{1, 2}, {1, 2, 3}] + + is only compared between {4, 5, 6} and {1, 2, 3} even though technically {1, 2, 3} is + just one item different than {1, 2} + + Perhaps in future we can have a report key that is item duplicated and modified instead of just added. + """ + cache_key = None + if self._stats[DISTANCE_CACHE_ENABLED]: + cache_key = combine_hashes_lists(items=[hashes_added, hashes_removed], prefix='pairs_cache') + cached_pairs = self._distance_cache.get(cache_key) + if cached_pairs is not_found: + cached_pairs = None + else: + return cast(dict, cached_pairs).copy() + + # A dictionary of hashes to distances and each distance to an ordered set of hashes. + # It tells us about the distance of each object from other objects. + # And the objects with the same distances are grouped together in an ordered set. + # It also includes a "max" key that is just the value of the biggest current distance in the + # most_in_common_pairs dictionary. + def defaultdict_orderedset(): + return defaultdict(SetOrdered) + most_in_common_pairs = defaultdict(defaultdict_orderedset) + pairs = dict_() + + pre_calced_distances = None + if hashes_added and hashes_removed and np and len(hashes_added) > 1 and len(hashes_removed) > 1: + # pre-calculates distances ONLY for 1D arrays whether an _original_type + # was explicitly passed or a homogeneous array is detected. + # Numpy is needed for this optimization. + pre_calced_distances = self._precalculate_numpy_arrays_distance( + hashes_added, hashes_removed, t1_hashtable, t2_hashtable, _original_type) + + if hashes_added and hashes_removed \ + and self.iterable_compare_func \ + and len(hashes_added) > 0 and len(hashes_removed) > 0: + pre_calced_distances = self._precalculate_distance_by_custom_compare_func( + hashes_added, hashes_removed, t1_hashtable, t2_hashtable, _original_type) + + # Optionally precompute non-cached distances in worker processes. + # Returns a dict keyed by (added_hash, removed_hash). Pair selection + # below stays serial and walks the same nested loop order, so the + # public output is independent of worker completion order. + parallel_distances = self._maybe_compute_pair_distances_parallel( + hashes_added, hashes_removed, t1_hashtable, t2_hashtable, + parents_ids, _original_type, pre_calced_distances) + + for added_hash in hashes_added: + for removed_hash in hashes_removed: + added_hash_obj = t2_hashtable[added_hash] + removed_hash_obj = t1_hashtable[removed_hash] + + # Loop is detected + if id(removed_hash_obj.item) in parents_ids: + continue + + _distance = None + if pre_calced_distances: + _distance = pre_calced_distances.get("{}--{}".format(added_hash, removed_hash)) + if _distance is None and parallel_distances is not None: + _distance = parallel_distances.get((added_hash, removed_hash)) + if _distance is None: + _distance = self._get_rough_distance_of_hashed_objs( + added_hash, removed_hash, added_hash_obj, removed_hash_obj, _original_type) + _distance = cast(float, _distance) + # Left for future debugging + # print(f'{Fore.RED}distance of {added_hash_obj.item} and {removed_hash_obj.item}: {_distance}{Style.RESET_ALL}') + # Discard potential pairs that are too far. + if _distance >= self.cutoff_distance_for_pairs: + continue + pairs_of_item = most_in_common_pairs[added_hash] + pairs_of_item[_distance].add(removed_hash) + used_to_hashes = set() + + distances_to_from_hashes = defaultdict(SetOrdered) + for from_hash, distances_to_to_hashes in most_in_common_pairs.items(): + # del distances_to_to_hashes['max'] + for dist in distances_to_to_hashes: + distances_to_from_hashes[dist].add(from_hash) + + for dist in sorted(distances_to_from_hashes.keys()): + from_hashes = distances_to_from_hashes[dist] + while from_hashes: + from_hash = from_hashes.pop() + if from_hash not in used_to_hashes: + to_hashes = most_in_common_pairs[from_hash][dist] + while to_hashes: + to_hash = to_hashes.pop() + if to_hash not in used_to_hashes: + used_to_hashes.add(from_hash) + used_to_hashes.add(to_hash) + # Left for future debugging: + # print(f'{bcolors.FAIL}Adding {t2_hashtable[from_hash].item} as a pairs of {t1_hashtable[to_hash].item} with distance of {dist}{bcolors.ENDC}') + pairs[from_hash] = to_hash + + inverse_pairs = {v: k for k, v in pairs.items()} + pairs.update(inverse_pairs) + if cache_key and self._stats[DISTANCE_CACHE_ENABLED]: + self._distance_cache.set(cache_key, value=pairs) + return pairs.copy() + + def _subtree_parallel_safe(self): + """Return True if paired-subtree diffs in this run can be sent to workers. + + Excluded features are ones whose semantics depend on the *parent's* + absolute path or on parent-process state, neither of which is visible + in a worker: + + - ``custom_operators`` (per docs/multi_processing.md) can call + ``custom_report_result`` and mutate the parent diff instance. + - ``exclude_obj_callback`` / ``include_obj_callback`` (and their + ``_strict`` variants) receive the level path; in a worker that path + is rooted at the subtree, not the original tree, so they would fire + on the wrong paths. + - ``ignore_order_func`` is also called with the level and would see + worker-local paths. + + Path-only filters (``exclude_paths`` / ``include_paths`` / + ``exclude_regex_paths``) are handled by re-applying ``_skip_this`` + after rebasing rather than disabling parallelism. + """ + if self.custom_operators: + return False + if self.exclude_obj_callback or self.exclude_obj_callback_strict: + return False + if self.include_obj_callback or self.include_obj_callback_strict: + return False + if self.ignore_order_func: + return False + return True + + def _rebase_subtree_leaf(self, leaf, change_level): + """Splice a worker-built leaf chain onto the parent's ``change_level``. + + The worker constructed ``leaf`` inside a fresh ``DeepDiff`` whose root + DiffLevel holds the paired items themselves; that root is irrelevant + once we're back in the parent. We replace it with a *fresh copy* of + ``change_level`` (so each leaf gets its own up-chain — DiffLevel.up + is shared by reference, and reusing one chain across leaves would + scramble paths). + + Returns the rebased leaf. Path caches along the chain are cleared so + ``leaf.path()`` recomputes against the new up-chain. + """ + # Walk up to find the worker root (up=None). + worker_root = leaf + while worker_root.up is not None: + worker_root = worker_root.up + + new_cl = change_level.copy() # fresh, independent chain; new_cl is bottom + + if worker_root is leaf: + # The worker reported at the very root of its diff (e.g. the two + # paired items differ at the top level — type_changes, + # values_changed). Transfer the report payload onto our fresh + # change_level copy. + new_cl.report_type = leaf.report_type + new_cl.additional = leaf.additional + cur = new_cl + while cur is not None: + cur._path = dict_() + cur = cur.up + return new_cl + + first_under_root = worker_root.down + # Splice: new_cl takes worker_root's place. Setting .down auto-sets + # the opposite .up link (see DiffLevel.__setattr__). + new_cl.down = first_under_root + if worker_root.t1_child_rel is not None: + new_cl.t1_child_rel = ChildRelationship.create( + klass=worker_root.t1_child_rel.__class__, + parent=new_cl.t1, child=first_under_root.t1, + param=worker_root.t1_child_rel.param, + ) + if worker_root.t2_child_rel is not None: + new_cl.t2_child_rel = ChildRelationship.create( + klass=worker_root.t2_child_rel.__class__, + parent=new_cl.t2, child=first_under_root.t2, + param=worker_root.t2_child_rel.param, + ) + # Clear path cache on the entire chain so path() recomputes against + # the new up-chain. + cur = leaf + while cur is not None: + cur._path = dict_() + cur = cur.up + return leaf + + def _dispatch_subtree_jobs(self, pending_jobs, _original_type, local_tree): + """Run deferred paired-subtree diffs (parallel when allowed, else serial). + + ``pending_jobs`` is the list of ``(change_level, t1_item, t2_item, + parents_ids_added)`` tuples in the exact order the inline serial code + would have visited them. Parallel results are merged in that same + order regardless of worker completion order, so output is identical + to the equivalent serial run. + """ + if not pending_jobs: + return + + mp_config = getattr(self, '_mp_config', None) + parallel_results = None + if (mp_config is not None and mp_config.enabled + and mp_config.should_parallelize(len(pending_jobs))): + jobs_payload = [(t1_item, t2_item) for (_, t1_item, t2_item, _) in pending_jobs] + outcome = compute_subtree_diffs_parallel( + jobs=jobs_payload, + parameters=self._parameters, + original_type=_original_type, + config=mp_config, + ) + if outcome is not None: + parallel_results, worker_stats = outcome + self._merge_worker_stats(worker_stats) + + if parallel_results is None: + # Below threshold or unsafe inputs — run inline-equivalent serial. + # Walking pending_jobs in order matches how inline serial would + # have run them; the parent tree fills up the same way. + for change_level, _t1_item, _t2_item, parents_ids_added in pending_jobs: + self._diff(change_level, parents_ids_added, local_tree=local_tree) + return + + target_tree = self.tree if local_tree is None else local_tree + for (change_level, _t1_item, _t2_item, _parents_ids_added), entries in zip( + pending_jobs, parallel_results): + for report_type, leaf in entries: + rebased_leaf = self._rebase_subtree_leaf(leaf, change_level) + # Re-apply path-based filters in the parent — exclude_paths + # and friends were not applied correctly inside the worker + # because the worker's level paths are subtree-relative. + if self._skip_this(rebased_leaf): + continue + rebased_leaf.report_type = report_type + target_tree[report_type].add(rebased_leaf) + + def _diff_iterable_with_deephash(self, level, parents_ids, _original_type=None, local_tree=None): + """Diff of hashable or unhashable iterables. Only used when ignoring the order.""" + + full_t1_hashtable = self._create_hashtable(level, 't1') + full_t2_hashtable = self._create_hashtable(level, 't2') + t1_hashes = SetOrdered(full_t1_hashtable.keys()) + t2_hashes = SetOrdered(full_t2_hashtable.keys()) + hashes_added = t2_hashes - t1_hashes + hashes_removed = t1_hashes - t2_hashes + + # Deciding whether to calculate pairs or not. + if (len(hashes_added) + len(hashes_removed)) / (len(full_t1_hashtable) + len(full_t2_hashtable) + 1) > self.cutoff_intersection_for_pairs: + get_pairs = False + else: + get_pairs = True + + # reduce the size of hashtables + if self.report_repetition: + t1_hashtable = full_t1_hashtable + t2_hashtable = full_t2_hashtable + else: + t1_hashtable = {k: v for k, v in full_t1_hashtable.items() if k in hashes_removed} + t2_hashtable = {k: v for k, v in full_t2_hashtable.items() if k in hashes_added} + if self._stats[PASSES_COUNT] < self.max_passes and get_pairs: + self._stats[PASSES_COUNT] += 1 + pairs = self._get_most_in_common_pairs_in_iterables( + hashes_added, hashes_removed, t1_hashtable, t2_hashtable, parents_ids, _original_type) + elif get_pairs: + if not self._stats[MAX_PASS_LIMIT_REACHED]: + self._stats[MAX_PASS_LIMIT_REACHED] = True + logger.warning(MAX_PASSES_REACHED_MSG.format(self.max_passes)) + pairs = dict_() + else: + pairs = dict_() + + def get_other_pair(hash_value, in_t1=True): + """ + Gets the other paired indexed hash item to the hash_value in the pairs dictionary + in_t1: are we looking for the other pair in t1 or t2? + """ + if in_t1: + hashtable = t1_hashtable + the_other_hashes = hashes_removed + else: + hashtable = t2_hashtable + the_other_hashes = hashes_added + other = pairs.pop(hash_value, notpresent) + if other is notpresent: + other = notpresent_indexed + else: + # The pairs are symmetrical. + # removing the other direction of pair + # so it does not get used. + del pairs[other] + the_other_hashes.remove(other) + other = hashtable[other] + return other + + # Phase 3: paired-subtree diffs may be deferred so they can run in a + # worker pool. ``pending_subtree_jobs is None`` keeps the inline + # serial path (and the existing ordering of inline ``_diff`` calls + # vs. ``_report_result`` calls) — so any feature that disables + # subtree parallelism degrades cleanly to today's behavior. + mp_config = getattr(self, '_mp_config', None) + use_mp = ( + mp_config is not None and mp_config.enabled + and self._subtree_parallel_safe() + ) + pending_subtree_jobs = [] if use_mp else None + + if self.report_repetition: + for hash_value in hashes_added: + if self._count_diff() is StopIteration: + return # pragma: no cover. This is already covered for addition (when report_repetition=False). + other = get_other_pair(hash_value) + item_id = id(other.item) + indexes = t2_hashtable[hash_value].indexes if other.item is notpresent else other.indexes + # When we report repetitions, we want the child_relationship_param2 only if there is no repetition. + # Because when there is a repetition, we report it in a different way (iterable_items_added_at_indexes for example). + # When there is no repetition, we want child_relationship_param2 so that we report the "new_path" correctly. + if len(t2_hashtable[hash_value].indexes) == 1: + index2 = t2_hashtable[hash_value].indexes[0] + else: + index2 = None + for i in indexes: + change_level = level.branch_deeper( + other.item, + t2_hashtable[hash_value].item, + child_relationship_class=SubscriptableIterableRelationship, + child_relationship_param=i, + child_relationship_param2=index2, + ) + if other.item is notpresent: + self._report_result('iterable_item_added', change_level, local_tree=local_tree) + else: + parents_ids_added = add_to_frozen_set(parents_ids, item_id) + if pending_subtree_jobs is None: + self._diff(change_level, parents_ids_added, local_tree=local_tree) + else: + pending_subtree_jobs.append( + (change_level, other.item, t2_hashtable[hash_value].item, parents_ids_added)) + for hash_value in hashes_removed: + if self._count_diff() is StopIteration: + return # pragma: no cover. This is already covered for addition. + other = get_other_pair(hash_value, in_t1=False) + item_id = id(other.item) + # When we report repetitions, we want the child_relationship_param2 only if there is no repetition. + # Because when there is a repetition, we report it in a different way (iterable_items_added_at_indexes for example). + # When there is no repetition, we want child_relationship_param2 so that we report the "new_path" correctly. + if other.item is notpresent or len(other.indexes) > 1: + index2 = None + else: + index2 = other.indexes[0] + for i in t1_hashtable[hash_value].indexes: + change_level = level.branch_deeper( + t1_hashtable[hash_value].item, + other.item, + child_relationship_class=SubscriptableIterableRelationship, + child_relationship_param=i, + child_relationship_param2=index2, + ) + if other.item is notpresent: + self._report_result('iterable_item_removed', change_level, local_tree=local_tree) + else: + # I was not able to make a test case for the following 2 lines since the cases end up + # getting resolved above in the hashes_added calcs. However I am leaving these 2 lines + # in case things change in future. + parents_ids_added = add_to_frozen_set(parents_ids, item_id) # pragma: no cover. + if pending_subtree_jobs is None: # pragma: no cover. + self._diff(change_level, parents_ids_added, local_tree=local_tree) + else: # pragma: no cover. + pending_subtree_jobs.append( + (change_level, t1_hashtable[hash_value].item, other.item, parents_ids_added)) + + items_intersect = t2_hashes.intersection(t1_hashes) + + for hash_value in items_intersect: + t1_indexes = t1_hashtable[hash_value].indexes + t2_indexes = t2_hashtable[hash_value].indexes + t1_indexes_len = len(t1_indexes) + t2_indexes_len = len(t2_indexes) + if t1_indexes_len != t2_indexes_len: # this is a repetition change! + # create "change" entry, keep current level untouched to handle further changes + repetition_change_level = level.branch_deeper( + t1_hashtable[hash_value].item, + t2_hashtable[hash_value].item, # nb: those are equal! + child_relationship_class=SubscriptableIterableRelationship, + child_relationship_param=t1_hashtable[hash_value] + .indexes[0]) + repetition_change_level.additional['repetition'] = RemapDict( + old_repeat=t1_indexes_len, + new_repeat=t2_indexes_len, + old_indexes=t1_indexes, + new_indexes=t2_indexes) + self._report_result('repetition_change', + repetition_change_level, local_tree=local_tree) + + else: + for hash_value in hashes_added: + if self._count_diff() is StopIteration: + return + other = get_other_pair(hash_value) + item_id = id(other.item) + index = t2_hashtable[hash_value].indexes[0] if other.item is notpresent else other.indexes[0] + index2 = t2_hashtable[hash_value].indexes[0] + change_level = level.branch_deeper( + other.item, + t2_hashtable[hash_value].item, + child_relationship_class=SubscriptableIterableRelationship, + child_relationship_param=index, + child_relationship_param2=index2, + ) + if other.item is notpresent: + self._report_result('iterable_item_added', change_level, local_tree=local_tree) + else: + parents_ids_added = add_to_frozen_set(parents_ids, item_id) + if pending_subtree_jobs is None: + self._diff(change_level, parents_ids_added, local_tree=local_tree) + else: + pending_subtree_jobs.append( + (change_level, other.item, t2_hashtable[hash_value].item, parents_ids_added)) + + for hash_value in hashes_removed: + if self._count_diff() is StopIteration: + return # pragma: no cover. This is already covered for addition. + other = get_other_pair(hash_value, in_t1=False) + item_id = id(other.item) + index = t1_hashtable[hash_value].indexes[0] + index2 = t1_hashtable[hash_value].indexes[0] if other.item is notpresent else other.indexes[0] + change_level = level.branch_deeper( + t1_hashtable[hash_value].item, + other.item, + child_relationship_class=SubscriptableIterableRelationship, + child_relationship_param=index, + child_relationship_param2=index2, + ) + if other.item is notpresent: + self._report_result('iterable_item_removed', change_level, local_tree=local_tree) + else: + # Just like the case when report_repetition = True, these lines never run currently. + # However they will stay here in case things change in future. + parents_ids_added = add_to_frozen_set(parents_ids, item_id) # pragma: no cover. + if pending_subtree_jobs is None: # pragma: no cover. + self._diff(change_level, parents_ids_added, local_tree=local_tree) + else: # pragma: no cover. + pending_subtree_jobs.append( + (change_level, t1_hashtable[hash_value].item, other.item, parents_ids_added)) + + if pending_subtree_jobs: + self._dispatch_subtree_jobs(pending_subtree_jobs, _original_type, local_tree) + + def _diff_booleans(self, level, local_tree=None): + if level.t1 != level.t2: + self._report_result('values_changed', level, local_tree=local_tree) + + def _diff_numbers(self, level, local_tree=None, report_type_change=True): + """Diff Numbers""" + if report_type_change: + t1_type = "number" if self.ignore_numeric_type_changes else level.t1.__class__.__name__ + t2_type = "number" if self.ignore_numeric_type_changes else level.t2.__class__.__name__ + else: + t1_type = t2_type = '' + + if self.use_log_scale: + if not logarithmic_similarity(level.t1, level.t2, threshold=self.log_scale_similarity_threshold): + self._report_result('values_changed', level, local_tree=local_tree) + elif self.math_epsilon is not None: + if not is_close(level.t1, level.t2, abs_tol=self.math_epsilon): + self._report_result('values_changed', level, local_tree=local_tree) + elif self.significant_digits is None: + if level.t1 != level.t2: + self._report_result('values_changed', level, local_tree=local_tree) + else: + # Bernhard10: I use string formatting for comparison, to be consistent with usecases where + # data is read from files that were previously written from python and + # to be consistent with on-screen representation of numbers. + # Other options would be abs(t1-t2)<10**-self.significant_digits + # or math.is_close (python3.5+) + # Note that abs(3.25-3.251) = 0.0009999999999998899 < 0.001 + # Note also that "{:.3f}".format(1.1135) = 1.113, but "{:.3f}".format(1.11351) = 1.114 + # For Decimals, format seems to round 2.5 to 2 and 3.5 to 4 (to closest even number) + t1_s = self.number_to_string(level.t1, + significant_digits=self.significant_digits, + number_format_notation=self.number_format_notation) # type: ignore + t2_s = self.number_to_string(level.t2, + significant_digits=self.significant_digits, + number_format_notation=self.number_format_notation) # type: ignore + + t1_s = KEY_TO_VAL_STR.format(t1_type, t1_s) + t2_s = KEY_TO_VAL_STR.format(t2_type, t2_s) + if t1_s != t2_s: + self._report_result('values_changed', level, local_tree=local_tree) + + def _diff_ipranges(self, level, local_tree=None): + """Diff IP ranges""" + if str(level.t1) != str(level.t2): + self._report_result('values_changed', level, local_tree=local_tree) + + def _diff_datetime(self, level, local_tree=None): + """Diff DateTimes""" + level.t1 = datetime_normalize(self.truncate_datetime, level.t1, default_timezone=self.default_timezone) + level.t2 = datetime_normalize(self.truncate_datetime, level.t2, default_timezone=self.default_timezone) + + if level.t1 != level.t2: + self._report_result('values_changed', level, local_tree=local_tree) + + def _diff_time(self, level, local_tree=None): + """Diff DateTimes""" + if self.truncate_datetime: + level.t1 = datetime_normalize(self.truncate_datetime, level.t1, default_timezone=self.default_timezone) + level.t2 = datetime_normalize(self.truncate_datetime, level.t2, default_timezone=self.default_timezone) + + if level.t1 != level.t2: + self._report_result('values_changed', level, local_tree=local_tree) + + def _diff_uuids(self, level, local_tree=None): + """Diff UUIDs""" + if level.t1.int != level.t2.int: + self._report_result('values_changed', level, local_tree=local_tree) + + def _diff_numpy_array(self, level, parents_ids=frozenset(), local_tree=None): + """Diff numpy arrays""" + if level.path() not in self._numpy_paths: + self._numpy_paths[level.path()] = get_type(level.t2).__name__ + if np is None: + # This line should never be run. If it is ever called means the type check detected a numpy array + # which means numpy module needs to be available. So np can't be None. + raise ImportError(CANT_FIND_NUMPY_MSG) # pragma: no cover + + if (self.ignore_order_func and not self.ignore_order_func(level)) or not self.ignore_order: + # fast checks + if self.significant_digits is None: + if np.array_equal(level.t1, level.t2, equal_nan=self.ignore_nan_inequality): + return # all good + else: + try: + np.testing.assert_almost_equal(level.t1, level.t2, decimal=self.significant_digits) + except TypeError: + np.array_equal(level.t1, level.t2, equal_nan=self.ignore_nan_inequality) + except AssertionError: + pass # do detailed checking below + else: + return # all good + + # compare array meta-data + _original_type = level.t1.dtype + if level.t1.shape != level.t2.shape: + # arrays are converted to python lists so that certain features of DeepDiff can apply on them easier. + # They will be converted back to Numpy at their final dimension. + level.t1 = level.t1.tolist() + level.t2 = level.t2.tolist() + self._diff_iterable(level, parents_ids, _original_type=_original_type, local_tree=local_tree) + else: + # metadata same -- the difference is in the content + shape = level.t1.shape + dimensions = len(shape) + if dimensions == 1: + self._diff_iterable(level, parents_ids, _original_type=_original_type, local_tree=local_tree) + elif (self.ignore_order_func and self.ignore_order_func(level)) or self.ignore_order: + # arrays are converted to python lists so that certain features of DeepDiff can apply on them easier. + # They will be converted back to Numpy at their final dimension. + level.t1 = level.t1.tolist() + level.t2 = level.t2.tolist() + self._diff_iterable_with_deephash(level, parents_ids, _original_type=_original_type, local_tree=local_tree) + else: + for (t1_path, t1_row), (t2_path, t2_row) in zip( + get_numpy_ndarray_rows(level.t1, shape), + get_numpy_ndarray_rows(level.t2, shape)): + + new_level = level.branch_deeper( + t1_row, + t2_row, + child_relationship_class=NumpyArrayRelationship, + child_relationship_param=t1_path, + child_relationship_param2=t2_path, + ) + + self._diff_iterable_in_order(new_level, parents_ids, _original_type=_original_type, local_tree=local_tree) + + def _diff_types(self, level, local_tree=None): + """Diff types""" + level.report_type = 'type_changes' + self._report_result('type_changes', level, local_tree=local_tree) + + def _count_diff(self): + if (self.max_diffs is not None and self._stats[DIFF_COUNT] > self.max_diffs): + if not self._stats[MAX_DIFF_LIMIT_REACHED]: + self._stats[MAX_DIFF_LIMIT_REACHED] = True + logger.warning(MAX_DIFFS_REACHED_MSG.format(self.max_diffs)) + return StopIteration + self._stats[DIFF_COUNT] += 1 + if self.cache_size and self.cache_tuning_sample_size: + self._auto_tune_cache() + + def _auto_tune_cache(self): + take_sample = (self._stats[DIFF_COUNT] % self.cache_tuning_sample_size == 0) + if self.cache_tuning_sample_size: + if self._stats[DISTANCE_CACHE_ENABLED]: + if take_sample: + self._auto_off_cache() + # Turn on the cache once in a while + elif self._stats[DIFF_COUNT] % self._shared_parameters[_ENABLE_CACHE_EVERY_X_DIFF] == 0: + self.progress_logger('Re-enabling the distance and level caches.') + # decreasing the sampling frequency + self._shared_parameters[_ENABLE_CACHE_EVERY_X_DIFF] *= 10 + self._stats[DISTANCE_CACHE_ENABLED] = True + if take_sample: + for key in (PREVIOUS_DIFF_COUNT, PREVIOUS_DISTANCE_CACHE_HIT_COUNT): + self._stats[key] = self._stats[key[9:]] + + def _auto_off_cache(self): + """ + Auto adjust the cache based on the usage + """ + if self._stats[DISTANCE_CACHE_ENABLED]: + angle = (self._stats[DISTANCE_CACHE_HIT_COUNT] - self._stats['PREVIOUS {}'.format(DISTANCE_CACHE_HIT_COUNT)]) / (self._stats[DIFF_COUNT] - self._stats[PREVIOUS_DIFF_COUNT]) + if angle < self.CACHE_AUTO_ADJUST_THRESHOLD: + self._stats[DISTANCE_CACHE_ENABLED] = False + self.progress_logger('Due to minimal cache hits, {} is disabled.'.format('distance cache')) + + def _use_custom_operator(self, level): + """ + For each level we check all custom operators. + If any one of them was a match for the level, we run the diff of the operator. + If the operator returned True, the operator must have decided these objects should not + be compared anymore. It might have already reported their results. + In that case the report will appear in the final results of this diff. + Otherwise basically the 2 objects in the level are being omitted from the results. + """ + + for operator in self.custom_operators: + if operator.match(level): + prevent_default = operator.give_up_diffing(level=level, diff_instance=self) + if prevent_default: + return True + + return False + + def _diff(self, level, parents_ids=frozenset(), _original_type=None, local_tree=None): + """ + The main diff method + + **parameters** + + level: the tree level or tree node + parents_ids: the ids of all the parent objects in the tree from the current node. + _original_type: If the objects had an original type that was different than what currently exists in the level.t1 and t2 + """ + if self._count_diff() is StopIteration: + return + + if self._use_custom_operator(level): + return + + if level.t1 is level.t2: + return + + if self._skip_this(level): + return + + report_type_change = True + if get_type(level.t1) != get_type(level.t2): + for type_group in self.ignore_type_in_groups: + if self.type_check_func(level.t1, type_group) and self.type_check_func(level.t2, type_group): + report_type_change = False + break + if self.use_enum_value and isinstance(level.t1, Enum): + level.t1 = level.t1.value + report_type_change = False + if self.use_enum_value and isinstance(level.t2, Enum): + level.t2 = level.t2.value + report_type_change = False + if report_type_change: + self._diff_types(level, local_tree=local_tree) + return + # This is an edge case where t1=None or t2=None and None is in the ignore type group. + if level.t1 is None or level.t2 is None: + self._report_result('values_changed', level, local_tree=local_tree) + return + + if self.ignore_nan_inequality and isinstance(level.t1, (float, np_floating)) and str(level.t1) == str(level.t2) == 'nan': + return + + if isinstance(level.t1, booleans): + self._diff_booleans(level, local_tree=local_tree) + + elif isinstance(level.t1, strings): + # Special handling when comparing string with UUID and ignore_uuid_types is True + if self.ignore_uuid_types and isinstance(level.t2, uuids): + try: + # Convert string to UUID for comparison + t1_uuid = uuid.UUID(str(level.t1)) + if t1_uuid.int != level.t2.int: + self._report_result('values_changed', level, local_tree=local_tree) + except (ValueError, AttributeError): + # If string is not a valid UUID, report as changed + self._report_result('values_changed', level, local_tree=local_tree) + else: + self._diff_str(level, local_tree=local_tree) + + elif isinstance(level.t1, datetime.datetime): + self._diff_datetime(level, local_tree=local_tree) + + elif isinstance(level.t1, ipranges): + self._diff_ipranges(level, local_tree=local_tree) + + elif isinstance(level.t1, (datetime.date, datetime.timedelta, datetime.time)): + self._diff_time(level, local_tree=local_tree) + + elif isinstance(level.t1, uuids): + # Special handling when comparing UUID with string and ignore_uuid_types is True + if self.ignore_uuid_types and isinstance(level.t2, str): + try: + # Convert string to UUID for comparison + t2_uuid = uuid.UUID(level.t2) + if level.t1.int != t2_uuid.int: + self._report_result('values_changed', level, local_tree=local_tree) + except (ValueError, AttributeError): + # If string is not a valid UUID, report as changed + self._report_result('values_changed', level, local_tree=local_tree) + else: + self._diff_uuids(level, local_tree=local_tree) + + elif isinstance(level.t1, numbers): + self._diff_numbers(level, local_tree=local_tree, report_type_change=report_type_change) + + elif isinstance(level.t1, Mapping): + self._diff_dict(level, parents_ids, local_tree=local_tree) + + elif isinstance(level.t1, tuple): + self._diff_tuple(level, parents_ids, local_tree=local_tree) + + elif isinstance(level.t1, (set, frozenset, SetOrdered)): + self._diff_set(level, local_tree=local_tree) + + elif isinstance(level.t1, np_ndarray): + self._diff_numpy_array(level, parents_ids, local_tree=local_tree) + + elif isinstance(level.t1, PydanticBaseModel): + self._diff_obj(level, parents_ids, local_tree=local_tree, is_pydantic_object=True) + + elif isinstance(level.t1, Iterable): + self._diff_iterable(level, parents_ids, _original_type=_original_type, local_tree=local_tree) + + elif isinstance(level.t1, Enum): + self._diff_enum(level, parents_ids, local_tree=local_tree) + + else: + self._diff_obj(level, parents_ids) + + def _get_view_results(self, view, verbose_level=None): + """ + Get the results based on the view + """ + result = self.tree + if not self.report_repetition: # and self.is_root: + result.mutual_add_removes_to_become_value_changes() + if view == TREE_VIEW: + pass + elif view == TEXT_VIEW: + effective_verbose_level = verbose_level if verbose_level is not None else self.verbose_level + result = TextResult(tree_results=self.tree, verbose_level=effective_verbose_level) + result.remove_empty_keys() + elif view == DELTA_VIEW: + result = self._to_delta_dict(report_repetition_required=False) + elif view == COLORED_VIEW: + result = ColoredView(t2=self.t2, tree_result=self.tree, compact=False) + elif view == COLORED_COMPACT_VIEW: + result = ColoredView(t2=self.t2, tree_result=self.tree, compact=True) + else: + raise ValueError(INVALID_VIEW_MSG.format(view)) + return result + + @staticmethod + def _get_key_for_group_by(row, group_by, item_name): + """ + Get the key value to group a row by, using the specified group_by parameter. + + Example + >>> row = {'first': 'John', 'middle': 'Joe', 'last': 'Smith'} + >>> DeepDiff._get_key_for_group_by(row, 'first', 't1') + 'John' + >>> nested_row = {'id': 123, 'demographics': {'names': {'first': 'John', 'middle': 'Joe', 'last': 'Smith'}}} + >>> group_by = lambda x: x['demographics']['names']['first'] + >>> DeepDiff._get_key_for_group_by(nested_row, group_by, 't1') + 'John' + + Args: + row (dict): The dictionary (row) to extract the group by key from. + group_by (str or callable): The key name or function to call to get to the key value to group by. + item_name (str): The name of the item, used for error messages. + + Returns: + str: The key value to group by. + + Raises: + KeyError: If the specified key is not found in the row. + """ + try: + if callable(group_by): + return group_by(row) + return row.pop(group_by) + except KeyError: + logger.error("Unable to group {} by {}. The key is missing in {}".format(item_name, group_by, row)) + raise + + def _group_iterable_to_dict(self, item, group_by, item_name): + """ + Convert a list of dictionaries into a dictionary of dictionaries + where the key is the value of the group_by key in each dictionary. + """ + group_by_level2 = None + if isinstance(group_by, (list, tuple)): + group_by_level1 = group_by[0] + if len(group_by) > 1: + group_by_level2 = group_by[1] + else: + group_by_level1 = group_by + if isinstance(item, Iterable) and not isinstance(item, Mapping): + result = {} + item_copy = deepcopy(item) + for row in item_copy: + if isinstance(row, Mapping): + key1 = self._get_key_for_group_by(row, group_by_level1, item_name) + # Track keys created by group_by to avoid type prefixing later + if hasattr(self, 'group_by_keys'): + self.group_by_keys.add(key1) + if group_by_level2: + key2 = self._get_key_for_group_by(row, group_by_level2, item_name) + # Track level 2 keys as well + if hasattr(self, 'group_by_keys'): + self.group_by_keys.add(key2) + if key1 not in result: + result[key1] = {} + if self.group_by_sort_key: + if key2 not in result[key1]: + result[key1][key2] = [] + result_key1_key2 = result[key1][key2] + if row not in result_key1_key2: + result_key1_key2.append(row) + else: + result[key1][key2] = row + else: + if self.group_by_sort_key: + if key1 not in result: + result[key1] = [] + if row not in result[key1]: + result[key1].append(row) + else: + result[key1] = row + else: + msg = "Unable to group {} by {} since the item {} is not a dictionary.".format(item_name, group_by_level1, row) + logger.error(msg) + raise ValueError(msg) + if self.group_by_sort_key: + if group_by_level2: + for key1, row1 in result.items(): + for key2, row in row1.items(): + row.sort(key=self.group_by_sort_key) + else: + for key, row in result.items(): + row.sort(key=self.group_by_sort_key) + return result + msg = "Unable to group {} by {}".format(item_name, group_by) + logger.error(msg) + raise ValueError(msg) + + def get_stats(self): + """ + Get some stats on internals of the DeepDiff run. + """ + return self._stats + + @property + def affected_paths(self): + """ + Get the list of paths that were affected. + Whether a value was changed or they were added or removed. + + Example + >>> from pprint import pprint + >>> t1 = {1: 1, 2: 2, 3: [3], 4: 4} + >>> t2 = {1: 1, 2: 4, 3: [3, 4], 5: 5, 6: 6} + >>> ddiff = DeepDiff(t1, t2) + >>> pprint(ddiff, indent=4) + { 'dictionary_item_added': ['root[5]', 'root[6]'], + 'dictionary_item_removed': ['root[4]'], + 'iterable_item_added': {'root[3][1]': 4}, + 'values_changed': {'root[2]': {'new_value': 4, 'old_value': 2}}} + >>> sorted(ddiff.affected_paths) + ['root[2]', 'root[3][1]', 'root[4]', 'root[5]', 'root[6]'] + >>> sorted(ddiff.affected_root_keys) + [2, 3, 4, 5, 6] + + """ + result = SetOrdered() + for key in REPORT_KEYS: + value = self.get(key) + if value: + if isinstance(value, SetOrdered): + result |= value + else: + result |= SetOrdered(value.keys()) + return result + + @property + def affected_root_keys(self): + """ + Get the list of root keys that were affected. + Whether a value was changed or they were added or removed. + + Example + >>> from pprint import pprint + >>> t1 = {1: 1, 2: 2, 3: [3], 4: 4} + >>> t2 = {1: 1, 2: 4, 3: [3, 4], 5: 5, 6: 6} + >>> ddiff = DeepDiff(t1, t2) + >>> pprint(ddiff, indent=4) + { 'dictionary_item_added': ['root[5]', 'root[6]'], + 'dictionary_item_removed': ['root[4]'], + 'iterable_item_added': {'root[3][1]': 4}, + 'values_changed': {'root[2]': {'new_value': 4, 'old_value': 2}}} + >>> sorted(ddiff.affected_paths) + ['root[2]', 'root[3][1]', 'root[4]', 'root[5]', 'root[6]'] + >>> sorted(ddiff.affected_root_keys) + [2, 3, 4, 5, 6] + """ + result = SetOrdered() + for key in REPORT_KEYS: + value = self.tree.get(key) + if value: + if isinstance(value, SetOrdered): + values_list = value + else: + values_list = value.keys() + for item in values_list: + root_key = item.get_root_key() + if root_key is not notpresent: + result.add(root_key) + return result + + def __str__(self): + if hasattr(self, '_colored_view') and self.view in {COLORED_VIEW, COLORED_COMPACT_VIEW}: + return str(self._colored_view) + return super().__str__() + + +if __name__ == "__main__": # pragma: no cover + import doctest + doctest.testmod() diff --git a/tests/test_diff_text.py b/tests/test_diff_text.py new file mode 100644 index 0000000..fb0087b --- /dev/null +++ b/tests/test_diff_text.py @@ -0,0 +1,2274 @@ +#!/usr/bin/env python +import datetime +import pytest +import logging +import re +import uuid +from enum import Enum +from dataclasses import dataclass +from typing import List +from decimal import Decimal +from deepdiff import DeepDiff +from deepdiff.helper import pypy3, PydanticBaseModel, np_float64 +from tests import CustomClass + + +logging.disable(logging.CRITICAL) + + +class MyEnum1(Enum): + book = "book" + cake = "cake" + +class MyEnum2(str, Enum): + book = "book" + cake = "cake" + + +@dataclass(frozen=True) +class MyDataClass: + val: int + val2: int + + +class TestDeepDiffText: + """DeepDiff Tests.""" + + def test_same_objects(self): + t1 = {1: 1, 2: 2, 3: 3} + t2 = t1 + assert {} == DeepDiff(t1, t2) + + def test_item_type_change(self): + t1 = {1: 1, 2: 2, 3: 3} + t2 = {1: 1, 2: "2", 3: 3} + ddiff = DeepDiff(t1, t2) + assert { + 'type_changes': { + "root[2]": { + "old_value": 2, + "old_type": int, + "new_value": "2", + "new_type": str + } + } + } == ddiff + + def test_item_type_change_less_verbose(self): + t1 = {1: 1, 2: 2, 3: 3} + t2 = {1: 1, 2: "2", 3: 3} + assert {'type_changes': { + "root[2]": { + "old_type": int, + "new_type": str + } + }} == DeepDiff(t1, t2, verbose_level=0) + + def test_item_type_change_for_strings_ignored_by_default(self): + """ ignore_string_type_changes = True by default """ + + t1 = 'hello' + t2 = b'hello' + ddiff = DeepDiff(t1, t2, ignore_string_type_changes=True) + assert not ddiff + + def test_item_type_change_for_strings_override(self): + t1 = 'hello' + t2 = b'hello' + ddiff = DeepDiff(t1, t2, ignore_string_type_changes=False) + assert { + 'type_changes': { + 'root': { + 'old_type': str, + 'new_type': bytes, + 'old_value': 'hello', + 'new_value': b'hello' + } + } + } == ddiff + + def test_value_change(self): + t1 = {1: 1, 2: 2, 3: 3} + t2 = {1: 1, 2: 4, 3: 3} + result = { + 'values_changed': { + 'root[2]': { + "old_value": 2, + "new_value": 4 + } + } + } + assert result == DeepDiff(t1, t2) + + def test_item_added_and_removed(self): + t1 = {1: 1, 2: 2, 3: [3], 4: 4} + t2 = {1: 1, 2: 4, 3: [3, 4], 5: 5, 6: 6} + ddiff = DeepDiff(t1, t2, threshold_to_diff_deeper=0) + result = { + 'dictionary_item_added': ["root[5]", "root[6]"], + 'dictionary_item_removed': ["root[4]"], + 'values_changed': { + 'root[2]': { + 'new_value': 4, + 'old_value': 2 + } + }, + 'iterable_item_added': { + 'root[3][1]': 4 + } + } + assert result == ddiff + assert {'root[4]', 'root[5]', 'root[6]', 'root[3][1]', 'root[2]'} == ddiff.affected_paths + assert {4, 5, 6, 3, 2} == ddiff.affected_root_keys + + def test_item_added_and_removed_verbose(self): + t1 = {1: 1, 3: 3, 4: 4} + t2 = {1: 1, 3: 3, 5: 5, 6: 6} + ddiff = DeepDiff(t1, t2, verbose_level=2) + result = { + 'dictionary_item_removed': { + 'root[4]': 4 + }, + 'dictionary_item_added': { + 'root[6]': 6, + 'root[5]': 5 + } + } + assert result == ddiff + + def test_diffs_dates(self): + t1 = datetime.date(2016, 8, 8) + t2 = datetime.date(2016, 8, 7) + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root': { + 'new_value': t2, + 'old_value': t1 + } + } + } + assert result == ddiff + + def test_diffs_timedeltas(self): + t1 = datetime.timedelta(days=1, seconds=12) + t2 = datetime.timedelta(days=1, seconds=10) + t3 = datetime.timedelta(seconds=(60*60*24) + 12) + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root': { + 'new_value': t2, + 'old_value': t1 + } + } + } + assert result == ddiff + ddiff = DeepDiff(t1, t3) + result = {} + assert result == ddiff + + def test_diffs_times(self): + t1 = datetime.time(1, 1) + t2 = datetime.time(1, 2) + t3 = datetime.time(1, 1) + expected_result = { + 'values_changed': { + 'root': { + 'new_value': t2, + 'old_value': t1 + } + } + } + assert DeepDiff(t1, t2) == expected_result + assert DeepDiff(t1, t3) == {} + + def test_diffs_uuid1(self): + t1 = uuid.uuid1() + t2 = uuid.uuid1() + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root': { + 'new_value': t2, + 'old_value': t1 + } + } + } + assert result == ddiff + ddiff = DeepDiff(t1, t1) + result = {} + assert result == ddiff + + def test_diffs_uuid3(self): + t1 = uuid.uuid3(uuid.NAMESPACE_DNS, 'python.org') + t2 = uuid.uuid3(uuid.NAMESPACE_DNS, 'stackoverflow.com') + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root': { + 'new_value': t2, + 'old_value': t1 + } + } + } + assert result == ddiff + ddiff = DeepDiff(t1, t1) + result = {} + assert result == ddiff + + def test_diffs_uuid4(self): + t1 = uuid.uuid4() + t2 = uuid.uuid4() + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root': { + 'new_value': t2, + 'old_value': t1 + } + } + } + assert result == ddiff + ddiff = DeepDiff(t1, t1) + result = {} + assert result == ddiff + + def test_diffs_uuid5(self): + t1 = uuid.uuid5(uuid.NAMESPACE_DNS, 'python.org') + t2 = uuid.uuid5(uuid.NAMESPACE_DNS, 'stackoverflow.com') + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root': { + 'new_value': t2, + 'old_value': t1 + } + } + } + assert result == ddiff + ddiff = DeepDiff(t1, t1) + result = {} + assert result == ddiff + + def test_string_difference(self): + t1 = {1: 1, 2: 2, 3: 3, 4: {"a": "hello", "b": "world"}} + t2 = {1: 1, 2: 4, 3: 3, 4: {"a": "hello", "b": "world!"}} + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root[2]': { + 'old_value': 2, + 'new_value': 4 + }, + "root[4]['b']": { + 'old_value': 'world', + 'new_value': 'world!' + } + } + } + assert result == ddiff + + def test_diffs_equal_strings_when_not_identical(self): + t1 = 'hello' + t2 = 'hel' + t2 += 'lo' + assert t1 is not t2 + ddiff = DeepDiff(t1, t2) + assert {} == ddiff + + def test_string_difference2(self): + t1 = { + 1: 1, + 2: 2, + 3: 3, + 4: { + "a": "hello", + "b": "world!\nGoodbye!\n1\n2\nEnd" + } + } + t2 = {1: 1, 2: 2, 3: 3, 4: {"a": "hello", "b": "world\n1\n2\nEnd"}} + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + "root[4]['b']": { + 'diff': + '--- \n+++ \n@@ -1,5 +1,4 @@\n-world!\n-Goodbye!\n+world\n 1\n 2\n End', + 'new_value': 'world\n1\n2\nEnd', + 'old_value': 'world!\nGoodbye!\n1\n2\nEnd' + } + } + } + assert result == ddiff + + def test_string_difference_ignore_case(self): + t1 = "Hello" + t2 = "hello" + + ddiff = DeepDiff(t1, t2) + result = {'values_changed': {'root': {'new_value': 'hello', 'old_value': 'Hello'}}} + assert result == ddiff + + ddiff = DeepDiff(t1, t2, ignore_string_case=True) + result = {} + assert result == ddiff + + def test_string_dict_key_ignore_case(self): + t1 = {'User': {'AboutMe': 1, 'ALIAS': 1}} + t2 = {'User': {'Alias': 1, 'AboutMe': 1}} + ddiff = DeepDiff(t1, t2) + result = {'dictionary_item_added': ["root['User']['Alias']"], 'dictionary_item_removed': ["root['User']['ALIAS']"]} + assert result == ddiff + + ddiff = DeepDiff(t1, t2, ignore_string_case=True) + result = {} + assert result == ddiff + + def test_string_list_ignore_case(self): + t1 = ['AboutMe', 'ALIAS'] + t2 = ['aboutme', 'alias'] + ddiff = DeepDiff(t1, t2) + result = {'values_changed': {'root[0]': {'new_value': 'aboutme', 'old_value': 'AboutMe'}, 'root[1]': {'new_value': 'alias', 'old_value': 'ALIAS'}}} + assert result == ddiff + + ddiff = DeepDiff(t1, t2, ignore_string_case=True) + result = {} + assert result == ddiff + + def test_diff_quote_in_string(self): + t1 = { + "a']['b']['c": 1 + } + t2 = { + "a']['b']['c": 2 + } + diff = DeepDiff(t1, t2) + expected = {'values_changed': {'''root["a']['b']['c"]''': {'new_value': 2, 'old_value': 1}}} + assert expected == diff + + def test_diff_quote_and_double_quote_in_string(self): + t1 = {'''a'"a''': 1} + t2 = {'''a'"a''': 2} + diff = DeepDiff(t1, t2) + expected = {'values_changed': {'root["a\'"a"]': {'new_value': 2, 'old_value': 1}}} + assert expected == diff + + def test_bytes(self): + t1 = { + 1: 1, + 2: 2, + 3: 3, + 4: { + "a": b"hello", + "b": b"world!\nGoodbye!\n1\n2\nEnd", + "c": b"\x80", + } + } + t2 = { + 1: 1, + 2: 2, + 3: 3, + 4: { + "a": b"hello", + "b": b"world\n1\n2\nEnd", + "c": b'\x81', + } + } + + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + "root[4]['b']": { + 'diff': + '--- \n+++ \n@@ -1,5 +1,4 @@\n-world!\n-Goodbye!\n+world\n 1\n 2\n End', + 'new_value': b'world\n1\n2\nEnd', + 'old_value': b'world!\nGoodbye!\n1\n2\nEnd' + }, + "root[4]['c']": { + 'new_value': b'\x81', + 'old_value': b'\x80' + } + } + } + assert result == ddiff + + def test_unicode(self): + t1 = { + 1: 1, + 2: 2, + 3: 3, + 4: { + "a": "hello", + "b": "world!\nGoodbye!\n1\n2\nEnd" + } + } + t2 = {1: 1, 2: 2, 3: 3, 4: {"a": "hello", "b": "world\n1\n2\nEnd"}} + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + "root[4]['b']": { + 'diff': + '--- \n+++ \n@@ -1,5 +1,4 @@\n-world!\n-Goodbye!\n+world\n 1\n 2\n End', + 'new_value': 'world\n1\n2\nEnd', + 'old_value': 'world!\nGoodbye!\n1\n2\nEnd' + } + } + } + assert result == ddiff + + def test_type_change(self): + t1 = {1: 1, 2: 2, 3: 3, 4: {"a": "hello", "b": [1, 2, 3]}} + t2 = {1: 1, 2: 2, 3: 3, 4: {"a": "hello", "b": "world\n\n\nEnd"}} + ddiff = DeepDiff(t1, t2) + result = { + 'type_changes': { + "root[4]['b']": { + 'old_type': list, + 'new_value': 'world\n\n\nEnd', + 'old_value': [1, 2, 3], + 'new_type': str + } + } + } + assert result == ddiff + + def test_list_difference(self): + t1 = { + 1: 1, + 2: 2, + 3: 3, + 4: { + "a": "hello", + "b": [1, 2, 'to_be_removed', 'to_be_removed2'] + } + } + t2 = {1: 1, 2: 2, 3: 3, 4: {"a": "hello", "b": [1, 2]}} + ddiff = DeepDiff(t1, t2) + result = { + 'iterable_item_removed': { + "root[4]['b'][2]": "to_be_removed", + "root[4]['b'][3]": 'to_be_removed2' + } + } + assert result == ddiff + + def test_list_difference_add(self): + t1 = [1, 2] + t2 = [1, 2, 3, 5] + ddiff = DeepDiff(t1, t2) + result = {'iterable_item_added': {'root[2]': 3, 'root[3]': 5}} + assert result == ddiff + + def test_list_difference2(self): + t1 = {1: 1, 2: 2, 3: 3, 4: {"a": "hello", "b": [1, 2, 3, 10, 12]}} + t2 = {1: 1, 2: 2, 3: 3, 4: {"a": "hello", "b": [1, 3, 2]}} + result = { + 'values_changed': { + "root[4]['b'][2]": { + 'new_value': 2, + 'old_value': 3 + }, + "root[4]['b'][1]": { + 'new_value': 3, + 'old_value': 2 + } + }, + 'iterable_item_removed': { + "root[4]['b'][3]": 10, + "root[4]['b'][4]": 12 + } + } + ddiff = DeepDiff(t1, t2) + assert result == ddiff + + def test_list_difference3(self): + t1 = {1: 1, 2: 2, 3: 3, 4: {"a": "hello", "b": [1, 2, 5]}} + t2 = {1: 1, 2: 2, 3: 3, 4: {"a": "hello", "b": [1, 3, 2, 5]}} + ddiff = DeepDiff(t1, t2) + expected = {'iterable_item_added': {"root[4]['b'][1]": 3}} + assert expected == ddiff + + def test_list_difference4(self): + t1 = ["a", "b", "d", "e"] + t2 = ["a", "b", "c", "d", "e"] + ddiff = DeepDiff(t1, t2) + result = {'iterable_item_added': {'root[2]': 'c'}} + assert result == ddiff + + def test_list_difference5(self): + t1 = ["a", "b", "d", "e", "f", "g"] + t2 = ["a", "b", "c", "d", "e", "f"] + ddiff = DeepDiff(t1, t2) + result = {'iterable_item_added': {'root[2]': 'c'}, 'iterable_item_removed': {'root[5]': 'g'}} + assert result == ddiff + + def test_list_difference_with_tiny_variations(self): + t1 = ['a', 'b', 'c', 'd'] + t2 = ['f', 'b', 'a', 'g'] + + values = { + 'a': 2.0000000000000027, + 'b': 2.500000000000005, + 'c': 2.000000000000002, + 'd': 3.000000000000001, + 'f': 2.000000000000003, + 'g': 3.0000000000000027, + } + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root[0]': { + 'new_value': 'f', + 'old_value': 'a' + }, + 'root[2]': { + 'new_value': 'a', + 'old_value': 'c' + }, + 'root[3]': { + 'new_value': 'g', + 'old_value': 'd' + } + } + } + assert result == ddiff + + ddiff2 = DeepDiff(t1, t2, zip_ordered_iterables=True) + assert result == ddiff2 + # Now we change the characters with numbers with tiny variations + + t3 = [2.0000000000000027, 2.500000000000005, 2.000000000000002, 3.000000000000001] + t4 = [2.000000000000003, 2.500000000000005, 2.0000000000000027, 3.0000000000000027] + ddiff3 = DeepDiff(t3, t4) + + expected = {'values_changed': {}} + for path, report in result['values_changed'].items(): + expected['values_changed'][path] = { + 'new_value': values[report['new_value']], + 'old_value': values[report['old_value']], + } + assert expected == ddiff3 + + def test_list_of_booleans(self): + t1 = [False, False, True, True] + t2 = [False, False, False, True] + ddiff = DeepDiff(t1, t2) + assert {'values_changed': {'root[2]': {'new_value': False, 'old_value': True}}} == ddiff + + def test_set_of_none(self): + """ + https://github.com/seperman/deepdiff/issues/64 + """ + ddiff = DeepDiff(set(), {None}) + assert {'set_item_added': {'root[None]'}} == ddiff + + def test_list_that_contains_dictionary(self): + t1 = {1: 1, 2: 2, 3: 3, 4: {"a": "hello", "b": [1, 2, {1: 1, 2: 2}]}} + t2 = {1: 1, 2: 2, 3: 3, 4: {"a": "hello", "b": [1, 2, {1: 3}]}} + ddiff = DeepDiff(t1, t2) + result = { + 'dictionary_item_removed': {"root[4]['b'][2][2]"}, + 'values_changed': { + "root[4]['b'][2][1]": { + 'old_value': 1, + 'new_value': 3 + } + } + } + assert result == ddiff + + def test_set(self): + t1 = {1, 2, 8} + t2 = {1, 2, 3, 5} + ddiff = DeepDiff(t1, t2) + result = { + 'set_item_added': {'root[3]', 'root[5]'}, + 'set_item_removed': {'root[8]'} + } + assert result == ddiff + + def test_set_strings(self): + t1 = {"veggies", "tofu"} + t2 = {"veggies", "tofu", "seitan"} + ddiff = DeepDiff(t1, t2) + result = {'set_item_added': {"root['seitan']"}} + assert result == ddiff + + def test_frozenset(self): + t1 = frozenset([1, 2, 'B']) + t2 = frozenset([1, 2, 3, 5]) + ddiff = DeepDiff(t1, t2) + result = { + 'set_item_added': {'root[3]', 'root[5]'}, + 'set_item_removed': {"root['B']"} + } + assert result == ddiff + + def test_tuple(self): + t1 = (1, 2, 8) + t2 = (1, 2, 3, 5) + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root[2]': { + 'old_value': 8, + 'new_value': 3 + } + }, + 'iterable_item_added': { + 'root[3]': 5 + } + } + assert result == ddiff + + def test_named_tuples(self): + from collections import namedtuple + Point = namedtuple('Point', ['x', 'y']) + t1 = Point(x=11, y=22) + t2 = Point(x=11, y=23) + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root.y': { + 'old_value': 22, + 'new_value': 23 + } + } + } + assert result == ddiff + + def test_enums(self): + + class MyEnum(Enum): + A = 1 + B = 2 + + ddiff = DeepDiff(MyEnum.A, MyEnum(1)) + result = {} + assert ddiff == result + + ddiff = DeepDiff(MyEnum.A, MyEnum.B) + result = { + 'values_changed': { + 'root.name': { + 'old_value': 'A', + 'new_value': 'B' + }, + 'root.value': { + 'old_value': 1, + 'new_value': 2 + }, + } + } + assert ddiff == result + + def test_enum_ignore_type_change(self): + + diff = DeepDiff("book", MyEnum1.book) + expected = { + 'type_changes': {'root': {'old_type': str, 'new_type': MyEnum1, 'old_value': 'book', 'new_value': MyEnum1.book}}} + assert expected == diff + + diff2 = DeepDiff("book", MyEnum1.book, ignore_type_in_groups=[(Enum, str)]) + assert not diff2 + + diff3 = DeepDiff("book", MyEnum2.book, ignore_type_in_groups=[(Enum, str)]) + assert not diff3 + + def test_enum_use_enum_value1(self): + diff = DeepDiff("book", MyEnum2.book, use_enum_value=True) + assert not diff + + def test_enum_use_enum_value_in_dict_key(self): + diff = DeepDiff({"book": 2}, {MyEnum2.book: 2}, use_enum_value=True) + assert not diff + + def test_precompiled_regex(self): + + pattern_1 = re.compile('foo') + pattern_2 = re.compile('foo') + pattern_3 = re.compile('foo', flags=re.I) + pattern_4 = re.compile('(foo)') + pattern_5 = re.compile('bar') + + # same object + ddiff = DeepDiff(pattern_1, pattern_1) + result = {} + assert ddiff == result + + # same pattern, different object + ddiff = DeepDiff(pattern_1, pattern_2) + result = {} + assert ddiff == result + + # same pattern, different flags + ddiff = DeepDiff(pattern_1, pattern_3) + result = { + 'values_changed': { + 'root.flags': { + 'new_value': 34, + 'old_value': 32, + }, + } + } + assert ddiff == result + + # same pattern, different groups + ddiff = DeepDiff(pattern_1, pattern_4) + result = { + 'values_changed': { + 'root.pattern': { + 'new_value': '(foo)', + 'old_value': 'foo', + }, + 'root.groups': { + 'new_value': 1, + 'old_value': 0, + }, + } + } + assert ddiff == result + + # different pattern + ddiff = DeepDiff(pattern_1, pattern_5) + result = { + 'values_changed': { + 'root.pattern': { + 'new_value': 'bar', + 'old_value': 'foo', + }, + } + } + assert ddiff == result + + def test_custom_objects_change(self): + t1 = CustomClass(1) + t2 = CustomClass(2) + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root.a': { + 'old_value': 1, + 'new_value': 2 + } + } + } + assert result == ddiff + + def test_custom_objects_slot_change(self): + class ClassA: + __slots__ = ('x', 'y') + + def __init__(self, x, y): + self.x = x + self.y = y + + t1 = ClassA(1, 1) + t2 = ClassA(1, 2) + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root.y': { + 'old_value': 1, + 'new_value': 2 + } + } + } + assert result == ddiff + + def test_custom_class_changes_with_slot_changes(self): + class ClassA: + __slots__ = ['x', 'y'] + + def __init__(self, x, y): + self.x = x + self.y = y + + class ClassB: + __slots__ = ['x'] + + ddiff = DeepDiff(ClassA, ClassB) + result = {'type_changes': {'root': {'old_type': ClassA, 'new_type': ClassB}}} + assert result == ddiff + + def test_custom_class_changes_with_slot_change_when_ignore_type(self): + class ClassA: + __slots__ = ['x', 'y'] + + def __init__(self, x, y): + self.x = x + self.y = y + + class ClassB: + __slots__ = ['x'] + + ddiff = DeepDiff(ClassA, ClassB, ignore_type_in_groups=[(ClassA, ClassB)]) + result = {'attribute_removed': ['root.y']} + assert result == ddiff + + def test_custom_objects_slot_in_group_change(self): + class ClassA: + __slots__ = ('x', 'y') + + def __init__(self, x, y): + self.x = x + self.y = y + + class ClassB(ClassA): + pass + + t1 = ClassA(1, 1) + t2 = ClassB(1, 1) + ddiff = DeepDiff(t1, t2, ignore_type_in_groups=[(ClassA, ClassB)]) + result = {} + assert result == ddiff + + + def test_custom_class_changes_none_when_ignore_type(self): + ddiff1 = DeepDiff({'a': None}, {'a': 1}, ignore_type_subclasses=True, ignore_type_in_groups=[(int, float)]) + result = { + 'type_changes': { + "root['a']": { + 'old_type': type(None), + 'new_type': int, + 'old_value': None, + 'new_value': 1 + } + } + } + assert result == ddiff1 + + ddiff2 = DeepDiff({'a': None}, {'a': '1'}, ignore_type_subclasses=True, ignore_type_in_groups=[(str, int, float, None)]) + assert {'values_changed': {"root['a']": {'new_value': '1', 'old_value': None}}} == ddiff2 + + ddiff3 = DeepDiff({'a': '1'}, {'a': None}, ignore_type_subclasses=True, ignore_type_in_groups=[(str, int, float, None)]) + assert {'values_changed': {"root['a']": {'new_value': None, 'old_value': '1'}}} == ddiff3 + + ddiff4 = DeepDiff({'a': {'b': None}}, {'a': {'b': '1'}}, ignore_type_subclasses=True, ignore_type_in_groups=[(str, int, float, None)]) + assert {'values_changed': {"root['a']['b']": {'new_value': '1', 'old_value': None}}} == ddiff4 + + ddiff5 = DeepDiff({'a': {'b': '1'}}, {'a': {'b': None}}, ignore_type_subclasses=True, ignore_type_in_groups=[(str, int, float, None)]) + assert {'values_changed': {"root['a']['b']": {'new_value': None, 'old_value': '1'}}} == ddiff5 + + def test_custom_object_changes_when_ignore_type_in_groups(self): + class ClassA: + def __init__(self, x, y): + self.x = x + self.y = y + + class ClassB: + def __init__(self, x): + self.x = x + + class ClassC(ClassB): + def __repr__(self): + return "obj c" + + obj_a = ClassA(1, 2) + obj_c = ClassC(3) + ddiff = DeepDiff(obj_a, obj_c, ignore_type_in_groups=[(ClassA, ClassB)], ignore_type_subclasses=True) + result = {'type_changes': {'root': {'old_type': ClassA, 'new_type': ClassC, 'old_value': obj_a, 'new_value': obj_c}}} + assert result == ddiff + + ddiff = DeepDiff(obj_a, obj_c, ignore_type_in_groups=[(ClassA, ClassB)], ignore_type_subclasses=False) + result = {'values_changed': {'root.x': {'new_value': 3, 'old_value': 1}}, 'attribute_removed': ['root.y']} + assert result == ddiff + + def test_custom_objects_slot_in_parent_class_change(self): + class ClassA: + __slots__ = ['x'] + + class ClassB(ClassA): + __slots__ = ['y'] + + def __init__(self, x, y): + self.x = x + self.y = y + + t1 = ClassB(1, 1) + t2 = ClassB(2, 1) + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root.x': { + 'old_value': 1, + 'new_value': 2 + } + } + } + assert result == ddiff + + def test_custom_objects_with_single_protected_slot(self): + class ClassA: + __slots__ = '__a' + + def __init__(self): + self.__a = 1 + + def __str__(self): + return str(self.__a) + + t1 = ClassA() + t2 = ClassA() + ddiff = DeepDiff(t1, t2) + assert {} == ddiff + + def test_custom_objects_with_weakref_in_slots(self): + class ClassA: + __slots__ = ['a', '__weakref__'] + + def __init__(self, a): + self.a = a + + t1 = ClassA(1) + t2 = ClassA(2) + diff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root.a': { + 'new_value': 2, + 'old_value': 1 + } + }, + } + assert result == diff + + def get_custom_objects_add_and_remove(self): + class ClassA: + a = 1 + + def __init__(self, b): + self.b = b + self.d = 10 + + t1 = ClassA(1) + t2 = ClassA(2) + t2.c = "new attribute" + del t2.d + return t1, t2 + + def test_custom_objects_add_and_remove(self): + t1, t2 = self.get_custom_objects_add_and_remove() + ddiff = DeepDiff(t1, t2) + result = { + 'attribute_added': {'root.c'}, + 'values_changed': { + 'root.b': { + 'new_value': 2, + 'old_value': 1 + } + }, + 'attribute_removed': {'root.d'} + } + assert result == ddiff + + def test_custom_objects_add_and_remove_verbose(self): + t1, t2 = self.get_custom_objects_add_and_remove() + ddiff = DeepDiff(t1, t2, verbose_level=2) + result = { + 'attribute_added': { + 'root.c': 'new attribute' + }, + 'attribute_removed': { + 'root.d': 10 + }, + 'values_changed': { + 'root.b': { + 'new_value': 2, + 'old_value': 1 + } + } + } + assert result == ddiff + + def get_custom_object_with_added_removed_methods(self): + class ClassA: + VAL = 1 + VAL2 = 2 + + def method_a(self): + pass + + t1 = ClassA() + t2 = ClassA() + + def method_b(self): + pass + + def method_c(self): + return "hello" + + t2.method_b = method_b + t2.method_a = method_c + # Note that we are comparing ClassA instances. method_a originally was in ClassA + # But we also added another version of it to t2. So it comes up as + # added attribute. + return t1, t2 + + def test_custom_objects_add_and_remove_method(self): + t1, t2 = self.get_custom_object_with_added_removed_methods() + ddiff = DeepDiff(t1, t2) + + result = {'attribute_added': {'root.method_a', 'root.method_b'}} + assert result == ddiff + + def test_custom_objects_add_and_remove_method_verbose(self): + t1, t2 = self.get_custom_object_with_added_removed_methods() + ddiff = DeepDiff(t1, t2, verbose_level=2) + assert 'root.method_a' in ddiff['attribute_added'] + assert 'root.method_b' in ddiff['attribute_added'] + + def test_set_of_custom_objects(self): + member1 = CustomClass(13, 37) + member2 = CustomClass(13, 37) + t1 = {member1} + t2 = {member2} + ddiff = DeepDiff(t1, t2) + result = {} + assert result == ddiff + + def test_dictionary_of_custom_objects(self): + member1 = CustomClass(13, 37) + member2 = CustomClass(13, 37) + t1 = {1: member1} + t2 = {1: member2} + ddiff = DeepDiff(t1, t2) + result = {} + assert result == ddiff + + def test_dictionary_with_string_keys1(self): + t1 = {"veggie": "carrots"} + t2 = {"meat": "carrots"} + + diff = DeepDiff(t1, t2, threshold_to_diff_deeper=0) + assert {'dictionary_item_added': ["root['meat']"], + 'dictionary_item_removed': ["root['veggie']"]} == diff + + def test_dictionary_with_string_keys_threshold_to_diff_deeper(self): + t1 = {"veggie": "carrots"} + t2 = {"meat": "carrots"} + + diff = DeepDiff(t1, t2, threshold_to_diff_deeper=0.33) + assert {'values_changed': {'root': {'new_value': {'meat': 'carrots'}, 'old_value': {'veggie': 'carrots'}}}} == diff + + def test_dictionary_with_numeric_keys(self): + t1 = {Decimal('10.01'): "carrots"} + t2 = {10.01: "carrots"} + diff = DeepDiff(t1, t2, threshold_to_diff_deeper=0) + assert {'dictionary_item_added': ["root[10.01]"], 'dictionary_item_removed': ["root[Decimal('10.01')]"]} == diff + + diff2 = DeepDiff(t1, t2) + assert {'values_changed': {'root': {'new_value': {10.01: 'carrots'}, 'old_value': {Decimal('10.01'): 'carrots'}}}} == diff2 + + def test_loop(self): + class LoopTest: + def __init__(self, a): + self.loop = self + self.a = a + + t1 = LoopTest(1) + t2 = LoopTest(2) + + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root.a': { + 'old_value': 1, + 'new_value': 2 + } + } + } + assert result == ddiff + + def test_loop2(self): + class LoopTestA: + def __init__(self, a): + self.loop = LoopTestB + self.a = a + + class LoopTestB: + def __init__(self, a): + self.loop = LoopTestA + self.a = a + + t1 = LoopTestA(1) + t2 = LoopTestA(2) + + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root.a': { + 'old_value': 1, + 'new_value': 2 + } + } + } + assert result == ddiff + + def test_loop3(self): + class LoopTest: + def __init__(self, a): + self.loop = LoopTest + self.a = a + + t1 = LoopTest(1) + t2 = LoopTest(2) + + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root.a': { + 'old_value': 1, + 'new_value': 2 + } + } + } + assert result == ddiff + + def test_loop_in_lists(self): + t1 = [1, 2, 3] + t1.append(t1) + + t2 = [1, 2, 4] + t2.append(t2) + + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root[2]': { + 'new_value': 4, + 'old_value': 3 + } + } + } + assert result == ddiff + + def test_loop_in_lists2(self): + t1 = [1, 2, [3]] + t1[2].append(t1) + t2 = [1, 2, [4]] + t2[2].append(t2) + + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root[2][0]': { + 'old_value': 3, + 'new_value': 4 + } + } + } + assert result == ddiff + + def test_decimal(self): + t1 = {1: Decimal('10.1')} + t2 = {1: Decimal('2.2')} + ddiff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root[1]': { + 'new_value': Decimal('2.2'), + 'old_value': Decimal('10.1') + } + } + } + assert result == ddiff + + def test_unicode_string_type_changes(self): + """ + These tests were written when DeepDiff was in Python 2. + Writing b"你好" throws an exception in Python 3 so can't be used for testing. + These tests are currently useless till they are rewritten properly. + """ + unicode_string = {"hello": "你好"} + ascii_string = {"hello": "你好"} + ddiff = DeepDiff(unicode_string, ascii_string) + result = {} + assert result == ddiff + + def test_unicode_string_value_changes(self): + unicode_string = {"hello": "你好"} + ascii_string = {"hello": "你好hello"} + ddiff = DeepDiff(unicode_string, ascii_string) + result = { + 'values_changed': { + "root['hello']": { + 'old_value': '你好', + 'new_value': '你好hello' + } + } + } + assert result == ddiff + + def test_unicode_string_value_and_type_changes(self): + unicode_string = {"hello": "你好"} + ascii_string = {"hello": "你好hello"} + ddiff = DeepDiff(unicode_string, ascii_string) + # In python3, all string is unicode, so these 2 strings only diff + # in values + result = { + 'values_changed': { + "root['hello']": { + 'new_value': '你好hello', + 'old_value': '你好' + } + } + } + assert result == ddiff + + def test_int_to_unicode_string(self): + t1 = 1 + ascii_string = "你好" + ddiff = DeepDiff(t1, ascii_string) + # In python3, all string is unicode, so these 2 strings only diff + # in values + result = { + 'type_changes': { + 'root': { + 'old_type': int, + 'new_type': str, + 'old_value': 1, + 'new_value': '你好' + } + } + } + assert result == ddiff + + def test_int_to_unicode(self): + t1 = 1 + unicode_string = "你好" + ddiff = DeepDiff(t1, unicode_string) + # In python3, all string is unicode, so these 2 strings only diff + # in values + result = { + 'type_changes': { + 'root': { + 'old_type': int, + 'new_type': str, + 'old_value': 1, + 'new_value': '你好' + } + } + } + assert result == ddiff + + @pytest.mark.parametrize("test_num, t1, t2, ignore_numeric_type_changes, significant_digits, number_format_notation, result", [ + (1, 43.265798602382986, 43.71677762295505, False, 0, "f", {'values_changed': {'root': {'new_value': 43.71677762295505, 'old_value': 43.265798602382986}}}), # Note that it rounds the number so one becomes 43 and the other one is 44 + (2, Decimal('2.5'), Decimal('1.5'), False, 0, "f", {}), + (3, Decimal('2.5'), Decimal('1.5'), False, 1, "f", {'values_changed': {'root': {'new_value': Decimal('1.5'), 'old_value': Decimal('2.5')}}}), + (4, Decimal('2.5'), Decimal(2.5), False, 3, "f", {}), + (5, 1024, 1022, False, 2, "e", {}), + (6, {"key": [Decimal('2.0001'), Decimal('20000.0001')]}, {"key": [2.0002, 20000.0002]}, True, 4, "e", {'values_changed': {"root['key'][0]": {'new_value': 2.0002, 'old_value': Decimal('2.0001')}}}), + (7, [Decimal("999.99999999")], [Decimal("999.9999999999")], False, 6, "f", {}), + ]) + def test_significant_digits_and_notation(self, test_num, t1, t2, ignore_numeric_type_changes, significant_digits, number_format_notation, result): + ddiff = DeepDiff(t1, t2, significant_digits=significant_digits, number_format_notation=number_format_notation, + ignore_numeric_type_changes=ignore_numeric_type_changes) + assert result == ddiff, f"test_significant_digits_and_notation #{test_num} failed." + + def test_significant_digits_for_complex_imaginary_part(self): + t1 = 1.23 + 1.222254j + t2 = 1.23 + 1.222256j + ddiff = DeepDiff(t1, t2, significant_digits=4) + assert {} == ddiff + result = { + 'values_changed': { + 'root': { + 'new_value': (1.23 + 1.222256j), + 'old_value': (1.23 + 1.222254j) + } + } + } + ddiff = DeepDiff(t1, t2, significant_digits=5) + assert result == ddiff + + def test_significant_digits_for_complex_real_part(self): + t1 = 1.23446879 + 1.22225j + t2 = 1.23446764 + 1.22225j + ddiff = DeepDiff(t1, t2, significant_digits=5) + assert {} == ddiff + + def test_significant_digits_for_list_of_floats(self): + t1 = [1.2344, 5.67881, 6.778879] + t2 = [1.2343, 5.67882, 6.778878] + ddiff = DeepDiff(t1, t2, significant_digits=3) + assert {} == ddiff + ddiff = DeepDiff(t1, t2, significant_digits=4) + result = { + 'values_changed': { + 'root[0]': { + 'new_value': 1.2343, + 'old_value': 1.2344 + } + } + } + assert result == ddiff + ddiff = DeepDiff(t1, t2, significant_digits=5) + result = { + 'values_changed': { + 'root[0]': { + 'new_value': 1.2343, + 'old_value': 1.2344 + }, + 'root[1]': { + 'new_value': 5.67882, + 'old_value': 5.67881 + } + } + } + assert result == ddiff + ddiff = DeepDiff(t1, t2) + ddiff2 = DeepDiff(t1, t2, significant_digits=6) + assert ddiff2 == ddiff + + def test_negative_significant_digits(self): + with pytest.raises(ValueError): + DeepDiff(1, 1, significant_digits=-1) + + @pytest.mark.parametrize("t1, t2, significant_digits, expected_result", + [ + (10, 10.0, 5, {}), + (10, 10.2, 5, {'values_changed': {'root': {'new_value': 10.2, 'old_value': 10}}}), + (10, 10.2, 0, {}), + (Decimal(10), 10, 0, {}), + (Decimal(10), 10, 10, {}), + (Decimal(10), 10.0, 0, {}), + (Decimal(10), 10.0, 10, {}), + (Decimal('10.0'), 10.0, 5, {}), + (Decimal('10.01'), 10.01, 1, {}), + (Decimal('10.01'), 10.01, 2, {}), + (Decimal('10.01'), 10.01, 5, {}), + (Decimal('10.01'), 10.01, 8, {}), + (Decimal('10.010'), 10.01, 3, {}), + (Decimal('100000.1'), 100000.1, 0, {}), + (Decimal('100000.1'), 100000.1, 1, {}), + (Decimal('100000.1'), 100000.1, 5, {}), + (Decimal('100000'), 100000.1, 0, {}), + (Decimal('100000'), 100000.1, 1, {'values_changed': {'root': {'new_value': 100000.1, 'old_value': Decimal('100000')}}}), + (np_float64(123.93420232), 123.93420232, 0, {}), + ]) + def test_decimal_digits(self, t1, t2, significant_digits, expected_result): + ddiff = DeepDiff(t1, t2, ignore_numeric_type_changes=True, ignore_string_type_changes=True, significant_digits=significant_digits) + assert expected_result == ddiff + + @pytest.mark.parametrize('test_num, t1, t2, log_scale_similarity_threshold, expected', [ + ( + 1, + {'foo': 110, 'bar': 306}, # t1 + {'foo': 140, 'bar': 298}, # t2 + 0.01, # threshold + {'values_changed': {"root['foo']": {'new_value': 140, 'old_value': 110}, "root['bar']": {'new_value': 298, 'old_value': 306}}}, # expected + ), + ( + 2, + {'foo': 110, 'bar': 306}, # t1 + {'foo': 140, 'bar': 298}, # t2 + 0.1, # threshold + {'values_changed': {"root['foo']": {'new_value': 140, 'old_value': 110}}}, # expected + ), + ( + 2, + {'foo': 110, 'bar': 306}, # t1 + {'foo': 140, 'bar': 298}, # t2 + 0.3, # threshold + {}, # expected + ), + ]) + def test_log_scale(self, test_num, t1, t2, log_scale_similarity_threshold, expected): + diff = DeepDiff(t1, t2, use_log_scale=True, log_scale_similarity_threshold=log_scale_similarity_threshold) + assert expected == diff, f"test_log_scale #{test_num} failed." + + def test_ignore_type_in_groups(self): + t1 = [1, 2, 3] + t2 = [1.0, 2.0, 3.0] + ddiff = DeepDiff(t1, t2, ignore_type_in_groups=DeepDiff.numbers) + assert not ddiff + + def test_ignore_type_in_groups2(self): + t1 = [1, 2, 3] + t2 = [1.0, 2.0, 3.3] + ddiff = DeepDiff(t1, t2, ignore_type_in_groups=DeepDiff.numbers) + result = {'values_changed': {'root[2]': {'new_value': 3.3, 'old_value': 3}}} + assert result == ddiff + + def test_ignore_type_in_groups3(self): + t1 = {Decimal('10.01'): "carrots"} + t2 = {10.01: "carrots"} + + diff1 = DeepDiff(t1, t2, threshold_to_diff_deeper=0) + + diff2 = DeepDiff(t1, t2, ignore_numeric_type_changes=True) + + diff3 = DeepDiff(t1, t2, ignore_type_in_groups=DeepDiff.numbers) + assert {'dictionary_item_added': ["root[10.01]"], 'dictionary_item_removed': ["root[Decimal('10.01')]"]} == diff1 + assert {} == diff2 == diff3 + + def test_ignore_type_in_groups_just_numbers(self): + t1 = [1, 2, 3, 'a'] + t2 = [1.0, 2.0, 3.3, b'a'] + ddiff = DeepDiff(t1, t2, ignore_type_in_groups=[DeepDiff.numbers]) + result = {'values_changed': {'root[2]': {'new_value': 3.3, 'old_value': 3}}, + 'type_changes': {'root[3]': {'new_type': bytes, + 'new_value': b'a', + 'old_type': str, + 'old_value': 'a'}} + } + assert result == ddiff + + def test_ignore_type_in_groups_numbers_and_strings(self): + t1 = [1, 2, 3, 'a'] + t2 = [1.0, 2.0, 3.3, b'a'] + ddiff = DeepDiff(t1, t2, ignore_type_in_groups=[DeepDiff.numbers, DeepDiff.strings]) + result = {'values_changed': {'root[2]': {'new_value': 3.3, 'old_value': 3}}} + assert result == ddiff + + def test_ignore_type_in_groups_none_and_objects(self): + t1 = [1, 2, 3, 'a', None] + t2 = [1.0, 2.0, 3.3, b'a', 'hello'] + ddiff = DeepDiff(t1, t2, ignore_type_in_groups=[(1, 1.0), (None, str, bytes)]) + result = { + 'values_changed': { + 'root[2]': { + 'new_value': 3.3, + 'old_value': 3 + }, + 'root[4]': { + 'new_value': 'hello', + 'old_value': None + } + } + } + assert result == ddiff + + def test_ignore_type_in_groups_str_and_datetime(self): + now = datetime.datetime(2022, 4, 10, 0, 40, 41, 357857) + t1 = [1, 2, 3, 'a', now] + t2 = [1, 2, 3, 'a', 'now'] + ddiff = DeepDiff(t1, t2, ignore_type_in_groups=[(str, bytes, datetime.datetime)]) + now_utc = now.replace(tzinfo=datetime.timezone.utc) + result = {'values_changed': {'root[4]': {'new_value': 'now', 'old_value': now_utc}}} + assert result == ddiff + + def test_ignore_type_in_groups_float_vs_decimal(self): + diff = DeepDiff(float('0.1'), Decimal('0.1'), ignore_type_in_groups=[(float, Decimal)], significant_digits=2) + assert not diff + + @pytest.mark.parametrize("t1, t2, significant_digits, result", [ + ([0.1], [Decimal('0.10')], 55, + {'values_changed': {'root[0]': {'new_value': Decimal('0.10'), 'old_value': 0.1}}}), # Due to floating point arithmetics with high significant digits. + ([0.1], [Decimal('0.10')], 5, {}), # Same inputs as above but with significant digits that is low. + ([-0.1], [-Decimal('0.10')], 5, {}), + ([-Decimal('0.102')], [-Decimal('0.10')], 2, {}), + ([1], [Decimal('1.00000002')], 3, {}), + ]) + def test_ignore_numeric_type_changes_numbers_when_decimal(self, t1, t2, significant_digits, result): + ddiff = DeepDiff(t1, t2, ignore_numeric_type_changes=True, significant_digits=significant_digits) + assert result == ddiff + + @pytest.mark.skip(reason="REMAPPING DISABLED UNTIL KEY NAMES CHANGE AGAIN IN FUTURE") + def test_base_level_dictionary_remapping(self): + """ + Since subclassed dictionaries that override __getitem__ treat newdict.get(key) + differently than newdict['key'], we are unable to create a unittest with + self.assertIn() and have to resort to fetching the values of two keys and making + sure they are the same value. + """ + t1 = {1: 1, 2: 2} + t2 = {2: 2, 3: 3} + ddiff = DeepDiff(t1, t2) + assert ddiff['dic_item_added'] == ddiff['dictionary_item_added'] + assert ddiff['dic_item_removed'] == ddiff['dictionary_item_removed'] + + @pytest.mark.skip(reason="REMAPPING DISABLED UNTIL KEY NAMES CHANGE AGAIN IN FUTURE") + def test_index_and_repeat_dictionary_remapping(self): + t1 = [1, 3, 1, 4] + t2 = [4, 4, 1] + ddiff = DeepDiff(t1, t2, ignore_order=True, report_repetition=True) + assert ddiff['repetition_change']['root[0]']['newindexes'] == ddiff['repetition_change']['root[0]']['new_indexes'] + assert ddiff['repetition_change']['root[0]']['newrepeat'] == ddiff['repetition_change']['root[0]']['new_repeat'] + assert ddiff['repetition_change']['root[0]']['oldindexes'] == ddiff['repetition_change']['root[0]']['old_indexes'] + assert ddiff['repetition_change']['root[0]']['oldrepeat'] == ddiff['repetition_change']['root[0]']['old_repeat'] + + @pytest.mark.skip(reason="REMAPPING DISABLED UNTIL KEY NAMES CHANGE AGAIN IN FUTURE") + def test_value_and_type_dictionary_remapping(self): + t1 = {1: 1, 2: 2} + t2 = {1: 1, 2: '2'} + ddiff = DeepDiff(t1, t2) + assert ddiff['type_changes']['root[2]']['newtype'] == ddiff['type_changes']['root[2]']['new_type'] + assert ddiff['type_changes']['root[2]']['newvalue'] == ddiff['type_changes']['root[2]']['new_value'] + assert ddiff['type_changes']['root[2]']['oldtype'] == ddiff['type_changes']['root[2]']['old_type'] + assert ddiff['type_changes']['root[2]']['oldvalue'] == ddiff['type_changes']['root[2]']['old_value'] + + def test_skip_type(self): + l1 = logging.getLogger("test") + l2 = logging.getLogger("test2") + t1 = {"log": l1, 2: 1337} + t2 = {"log": l2, 2: 1337} + ddiff = DeepDiff(t1, t2, exclude_types={logging.Logger}) + assert {} == ddiff + + t1 = {"log": "book", 2: 1337} + t2 = {"log": l2, 2: 1337} + ddiff = DeepDiff(t1, t2, exclude_types={logging.Logger}) + assert {} == ddiff + + def test_skip_path1(self): + t1 = { + "for life": "vegan", + "ingredients": ["no meat", "no eggs", "no dairy"] + } + t2 = { + "for life": "vegan", + "ingredients": ["veggies", "tofu", "soy sauce"] + } + ddiff = DeepDiff(t1, t2, exclude_paths={"root['ingredients']"}) + assert {} == ddiff + + def test_skip_path2(self): + t1 = { + "for life": "vegan", + "ingredients": ["no meat", "no eggs", "no dairy"] + } + t2 = {"for life": "vegan"} + ddiff = DeepDiff(t1, t2, exclude_paths={"root['ingredients']"}) + assert {} == ddiff + + def test_skip_path2_key_names(self): + t1 = { + "for life": "vegan", + "ingredients": ["no meat", "no eggs", "no dairy"] + } + t2 = {"for life": "vegan"} + ddiff = DeepDiff(t1, t2, exclude_paths={"ingredients"}) + assert {} == ddiff + + def test_skip_path2_reverse(self): + t1 = { + "for life": "vegan", + "ingredients": ["no meat", "no eggs", "no dairy"] + } + t2 = {"for life": "vegan"} + ddiff = DeepDiff(t2, t1, exclude_paths={"root['ingredients']"}) + assert {} == ddiff + + def test_exclude_path_when_prefix_of_exclude_path_matches1(self): + diff = DeepDiff({}, {'foo': '', 'bar': ''}, exclude_paths=['foo', 'bar']) + assert not diff + + def test_include_path3(self): + t1 = { + "for life": "vegan", + "ingredients": ["no meat", "no eggs", "no dairy"] + } + t2 = {"for life": "vegan"} + ddiff = DeepDiff(t2, t1, include_paths={"root['for_life']"}) + assert {} == ddiff + + def test_include_path3_with_just_key_names(self): + t1 = { + "for life": "vegan", + "ingredients": ["no meat", "no eggs", "no dairy"] + } + t2 = {"for life": "vegan"} + ddiff = DeepDiff(t1, t2, include_paths={"for_life"}) + assert {} == ddiff + + def test_include_path4_nested(self): + t1 = { + "foo": {"bar": "potato"}, + "ingredients": ["no meat", "no eggs", "no dairy"] + } + t2 = { + "foo": {"bar": "banana"}, + "ingredients": ["bread", "cheese"] + } + ddiff = DeepDiff(t1, t2, include_paths="foo") + assert { + 'values_changed': { + "root['foo']['bar']": { + 'new_value': 'banana', + 'old_value': 'potato' + } + } + } == ddiff + + def test_include_path5(self): + diff = DeepDiff( + { + 'name': 'Testname', + 'code': 'bla', + 'noneCode': 'blu', + }, { + 'uid': '12345', + 'name': 'Testname2', + }, + ) + + diff2 = DeepDiff( + { + 'name': 'Testname', + 'code': 'bla', + 'noneCode': 'blu', + }, { + 'uid': '12345', + 'name': 'Testname2', + }, + include_paths = "root['name']" + ) + expected = {'values_changed': {'root': {'new_value': {'uid': '12345', 'name': 'Testname2'}, 'old_value': {'name': 'Testname', 'code': 'bla', 'noneCode': 'blu'}}}} + expected2 = {'values_changed': {"root['name']": {'new_value': 'Testname2', 'old_value': 'Testname'}}} + + assert expected == diff + assert expected2 == diff2 + + def test_include_path6(self): + t1 = [1, 2, 3, [4, 5, {6: 7}]] + t2 = [1, 2, 3, [4, 5, {6: 1000}]] + diff = DeepDiff( + t1, + t2, + ) + + diff2 = DeepDiff( + t1, + t2, + include_paths = "root[3]" + ) + + diff3 = DeepDiff( + t1, + t2, + include_paths = "root[4]" + ) + expected = {'values_changed': {'root[3][2][6]': {'new_value': 1000, 'old_value': 7}}} + assert expected == diff + assert diff == diff2 + assert not diff3 + + def test_skip_path4(self): + t1 = { + "for life": "vegan", + "ingredients": ["no meat", "no eggs", "no dairy"] + } + t2 = {"for life": "vegan", "zutaten": ["veggies", "tofu", "soy sauce"]} + ddiff = DeepDiff(t1, t2, exclude_paths={"root['ingredients']"}) + assert 'dictionary_item_added' in ddiff, {} + assert 'dictionary_item_removed' not in ddiff, {} + + def test_skip_path5(self): + t1 = [{'cn': 'tuser', 'first_name': 'Test', 'last_name': 'User', 'name': 'Test User', 'email': 'tuser@example.com'}] + t2 = [{'name': 'Test User', 'email': 'tuser@example.com'}] + + diff = DeepDiff( + t1, + t2, + ignore_order=True, + exclude_paths={ + "root[0]['cn']", + "root[0]['first_name']", + "root[0]['last_name']" + }) + + assert not diff + + def test_skip_custom_object_path(self): + t1 = CustomClass(1) + t2 = CustomClass(2) + ddiff = DeepDiff(t1, t2, exclude_paths=['root.a']) + result = {} + assert result == ddiff + + def test_skip_list_path(self): + t1 = ['a', 'b'] + t2 = ['a'] + ddiff = DeepDiff(t1, t2, exclude_paths=['root[1]']) + result = {} + assert result == ddiff + + def test_skip_dictionary_path(self): + t1 = {1: {2: "a"}} + t2 = {1: {}} + ddiff = DeepDiff(t1, t2, exclude_paths=['root[1][2]']) + result = {} + assert result == ddiff + + def test_skip_dictionary_path_with_custom_object(self): + obj1 = CustomClass(1) + obj2 = CustomClass(2) + + t1 = {1: {2: obj1}} + t2 = {1: {2: obj2}} + ddiff = DeepDiff(t1, t2, exclude_paths=['root[1][2].a']) + result = {} + assert result == ddiff + + # TODO: fix it for python 3.5, 3.6 and pypy3 + def test_skip_regexp(self): + t1 = [{'a': 1, 'b': 2}, {'c': 4, 'b': 5}] + t2 = [{'a': 1, 'b': 3}, {'c': 4, 'b': 5}] + ddiff = DeepDiff(t1, t2, exclude_regex_paths=[r"root\[\d+\]\['b'\]"]) + result = {} + assert result == ddiff + + def test_include_obj_callback(self): + def include_obj_callback(obj, path): + return True if "include" in path or isinstance(obj, int) else False + + t1 = {"x": 10, "y": "b", "z": "c", "include_me": "a"} + t2 = {"x": 10, "y": "c", "z": "b", "include_me": "b"} + ddiff = DeepDiff(t1, t2, include_obj_callback=include_obj_callback) + result = {'values_changed': {"root['include_me']": {'new_value': "b", 'old_value': "a"}}} + assert result == ddiff + assert {"root['include_me']"} == ddiff.affected_paths + assert {"include_me"} == ddiff.affected_root_keys + + def test_include_obj_callback_strict(self): + def include_obj_callback_strict(obj, path): + return True if isinstance(obj, int) and obj > 10 else False + + t1 = {"x": 11, "y": 10, "z": "c"} + t2 = {"x": 12, "y": 12, "z": "c"} + ddiff = DeepDiff(t1, t2, include_obj_callback_strict=include_obj_callback_strict) + result = {'values_changed': {"root['x']": {'new_value': 12, 'old_value': 11}}} + assert result == ddiff + assert {"root['x']"} == ddiff.affected_paths + assert {"x"} == ddiff.affected_root_keys + + def test_skip_exclude_obj_callback(self): + def exclude_obj_callback(obj, path): + return True if "skip" in path or isinstance(obj, int) else False + + t1 = {"x": 10, "y": "b", "z": "c", "skip_1": 0} + t2 = {"x": 12, "y": "b", "z": "c", "skip_2": 0} + ddiff = DeepDiff(t1, t2, exclude_obj_callback=exclude_obj_callback) + result = {} + assert result == ddiff + + def test_skip_exclude_obj_callback_strict(self): + def exclude_obj_callback_strict(obj, path): + return True if isinstance(obj, int) and obj > 10 else False + + t1 = {"x": 10, "y": "b", "z": "c"} + t2 = {"x": 12, "y": "b", "z": "c"} + ddiff = DeepDiff(t1, t2, exclude_obj_callback_strict=exclude_obj_callback_strict) + result = {'values_changed': {"root['x']": {'new_value': 12, 'old_value': 10}}} + assert result == ddiff + assert {"root['x']"} == ddiff.affected_paths + assert {"x"} == ddiff.affected_root_keys + + def test_skip_str_type_in_dictionary(self): + t1 = {1: {2: "a"}} + t2 = {1: {}} + ddiff = DeepDiff(t1, t2, exclude_types=[str]) + result = {} + assert result == ddiff + + def test_skip_str_type_in_dict_on_list(self): + t1 = [{1: "a"}] + t2 = [{}] + ddiff = DeepDiff(t1, t2, exclude_types=[str]) + result = {} + assert result == ddiff + + def test_unknown_parameters(self): + with pytest.raises(ValueError): + DeepDiff(1, 1, wrong_param=2) + + def test_bad_attribute(self): + class Bad: + __slots__ = ['x', 'y'] + + def __getattr__(self, key): + raise AttributeError("Bad item") + + def __str__(self): + return "Bad Object" + + t1 = Bad() + t2 = Bad() + + ddiff = DeepDiff(t1, t2) + result = {} + assert result == ddiff + + def test_dict_none_item_removed(self): + t1 = {1: None, 2: 2} + t2 = {2: 2} + ddiff = DeepDiff(t1, t2) + result = { + 'dictionary_item_removed': {'root[1]'} + } + assert result == ddiff + + def test_list_none_item_removed(self): + t1 = [1, 2, None] + t2 = [1, 2] + ddiff = DeepDiff(t1, t2) + result = { + 'iterable_item_removed': {'root[2]': None} + } + assert result == ddiff + assert {"root[2]"} == ddiff.affected_paths + + def test_list_item_removed_from_the_middle(self): + t1 = [0, 1, 2, 3, 'bye', 5, 6, 7, 8, 'a', 'b', 'c'] + t2 = [0, 1, 2, 3, 5, 6, 7, 8, 'a', 'b', 'c'] + diff = DeepDiff(t1, t2) + result = {'iterable_item_removed': {'root[4]': 'bye'}} + assert result == diff + assert {"root[4]"} == diff.affected_paths + assert {4} == diff.affected_root_keys + + # TODO: we need to support reporting that items have been swapped + # def test_item_moved(self): + # # currently all the items in the list need to be hashables + # t1 = [1, 2, 3, 4] + # t2 = [4, 2, 3, 1] + # diff = DeepDiff(t1, t2) + # result = {} # it should show that those items are swapped. + # assert result == diff + + def test_list_item_values_replace_in_the_middle(self): + t1 = [0, 1, 2, 3, 'bye', 5, 6, 7, 8, 'a', 'b', 'c'] + t2 = [0, 1, 2, 3, 'see', 'you', 'later', 5, 6, 7, 8, 'a', 'b', 'c'] + diff = DeepDiff(t1, t2) + result = { + 'values_changed': { + 'root[4]': { + 'old_value': 'bye', + 'new_value': 'see', + } + }, + 'iterable_item_added': { + 'root[5]': 'you', + 'root[6]': 'later' + } + } + assert result == diff + assert {'root[5]', 'root[6]', 'root[4]'} == diff.affected_paths + assert {4, 5, 6} == diff.affected_root_keys + + def test_non_subscriptable_iterable(self): + def gen1(): + yield 42 + yield 1337 + yield 31337 + + def gen2(): + yield 42 + yield 1337 + + t1 = gen1() + t2 = gen2() + ddiff = DeepDiff(t1, t2) + + result = {'iterable_item_removed': {'root[2]': 31337}} + # Note: In text-style results, we currently pretend this stuff is subscriptable for readability + + assert result == ddiff + assert {"root[2]"} == ddiff.affected_paths + + @pytest.mark.parametrize('t1, t2, params, expected_result', [ + (float('nan'), float('nan'), {}, ['values_changed']), + (float('nan'), float('nan'), {'ignore_nan_inequality': True}, []), + ([1, float('nan')], [1, float('nan')], {'ignore_nan_inequality': True}, []), + ]) + @pytest.mark.skipif(pypy3, reason="some versions of pypy3 have nan==nan") + def test_ignore_nan_inequality(self, t1, t2, params, expected_result): + assert expected_result == list(DeepDiff(t1, t2, **params).keys()) + + @pytest.mark.parametrize('ignore_order, ignore_private_variables, expected', [ + (True, True, {}), + (False, True, {}), + (True, False, {'values_changed': {"root[0]['schema']['items']['__ref']": {'new_value': 2, 'old_value': 1}}}), + (False, False, {'values_changed': {"root[0]['schema']['items']['__ref']": {'new_value': 2, 'old_value': 1}}}), + ]) + def test_private_variables(self, ignore_order, ignore_private_variables, expected): + t1 = [{'in': 'body', 'name': 'params', 'description': 'params', 'required': True, 'schema': {'type': 'array', 'items': {'__ref': 1}}}] + t2 = [{'in': 'body', 'name': 'params', 'description': 'params', 'required': True, 'schema': {'type': 'array', 'items': {'__ref': 2}}}] + diff = DeepDiff(t1, t2, ignore_order=ignore_order, ignore_private_variables=ignore_private_variables) + assert expected == diff + + def test_group_by1(self): + t1 = [ + {'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody'}, + {'id': 'BB', 'name': 'James', 'last_name': 'Blue'}, + {'id': 'CC', 'name': 'Mike', 'last_name': 'Apple'}, + ] + + t2 = [ + {'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody'}, + {'id': 'BB', 'name': 'James', 'last_name': 'Brown'}, + {'id': 'CC', 'name': 'Mike', 'last_name': 'Apple'}, + ] + + diff = DeepDiff(t1, t2) + expected = {'values_changed': {"root[1]['last_name']": { + 'new_value': 'Brown', + 'old_value': 'Blue'}}} + assert expected == diff + + diff = DeepDiff(t1, t2, group_by='id') + expected_grouped = {'values_changed': {"root['BB']['last_name']": { + 'new_value': 'Brown', + 'old_value': 'Blue'}}} + assert expected_grouped == diff + + def test_group_by2_when_repeats(self): + t1 = [ + {'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody', 'int_id': 2}, + {'id': 'BB', 'name': 'James', 'last_name': 'Blue', 'int_id': 20}, + {'id': 'BB', 'name': 'Jimmy', 'last_name': 'Red', 'int_id': 3}, + {'id': 'CC', 'name': 'Mike', 'last_name': 'Apple', 'int_id': 4}, + ] + + t2 = [ + {'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody', 'int_id': 2}, + {'id': 'BB', 'name': 'James', 'last_name': 'Brown', 'int_id': 20}, + {'id': 'CC', 'name': 'Mike', 'last_name': 'Apple', 'int_id': 4}, + ] + + diff = DeepDiff(t1, t2, group_by='id', group_by_sort_key='name') + expected_grouped = { + 'values_changed': { + "root['BB'][0]['last_name']": { + 'new_value': 'Brown', + 'old_value': 'Blue' + } + }, + 'iterable_item_removed': { + "root['BB'][1]": { + 'name': 'Jimmy', + 'last_name': 'Red', + 'int_id': 3 + } + } + } + assert expected_grouped == diff + + diff2 = DeepDiff(t1, t2, group_by='id', group_by_sort_key=lambda x: x['name']) + assert expected_grouped == diff2 + + def test_group_by3_when_repeats_and_group_by_list(self): + t1 = [ + {'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody', 'int_id': 2}, + {'id': 'BB', 'name': 'James', 'last_name': 'Blue', 'int_id': 20}, + {'id': 'BB', 'name': 'Jimmy', 'last_name': 'Red', 'int_id': 3}, + {'id': 'CC', 'name': 'Mike', 'last_name': 'Apple', 'int_id': 4}, + ] + + t2 = [ + {'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody', 'int_id': 2}, + {'id': 'BB', 'name': 'James', 'last_name': 'Brown', 'int_id': 20}, + {'id': 'CC', 'name': 'Mike', 'last_name': 'Apple', 'int_id': 4}, + ] + + diff1 = DeepDiff(t1, t2, group_by=['id', 'name']) + expected_grouped = { + 'dictionary_item_removed': ["root['BB']['Jimmy']"], + 'values_changed': { + "root['BB']['James']['last_name']": { + 'new_value': 'Brown', + 'old_value': 'Blue' + } + } + } + assert expected_grouped == diff1 + + diff2 = DeepDiff(t1, t2, group_by=['id', 'name'], group_by_sort_key='int_id') + expected_grouped = { + 'dictionary_item_removed': ["root['BB']['Jimmy']"], + 'values_changed': { + "root['BB']['James'][0]['last_name']": { + 'new_value': 'Brown', + 'old_value': 'Blue' + } + } + } + assert expected_grouped == diff2 + + def test_group_by_key_missing(self): + t1 = [ + {'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody'}, + {'id': 'BB', 'name': 'James', 'last_name': 'Blue'}, + {'name': 'Mike', 'last_name': 'Apple'}, + ] + + t2 = [ + {'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody'}, + {'id': 'BB', 'name': 'James', 'last_name': 'Blue'}, + {'id': 'CC', 'name': 'Mike', 'last_name': 'Apple'}, + ] + + diff = DeepDiff(t1, t2, group_by='id') + expected = {'dictionary_item_added': ["root[2]['id']"]} + assert expected == diff + + def test_group_by_not_dict1(self): + t1 = [ + {'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody'}, + None, + {'id': 'CC', 'name': 'Mike', 'last_name': 'Apple'}, + ] + + t2 = [ + {'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody'}, + {'id': 'BB'}, + {'id': 'CC', 'name': 'Mike', 'last_name': 'Apple'}, + ] + + diff = DeepDiff(t1, t2, group_by='id') + expected = { + 'type_changes': { + 'root[1]': { + 'old_type': None.__class__, + 'new_type': dict, + 'old_value': None, + 'new_value': { + 'id': 'BB' + } + } + }, + } + assert expected == diff + + def test_group_by_not_dict2(self): + t1 = [ + {'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody'}, + {'id': 'BB'}, + {'id': 'CC', 'name': 'Mike', 'last_name': 'Apple'}, + ] + + t2 = [ + {'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody'}, + None, + {'id': 'CC', 'name': 'Mike', 'last_name': 'Apple'}, + ] + + diff = DeepDiff(t1, t2, group_by='id') + expected = { + 'type_changes': { + 'root[1]': { + 'old_type': dict, + 'new_type': None.__class__, + 'new_value': None, + 'old_value': { + 'id': 'BB' + } + } + }, + } + assert expected == diff + + def test_group_by_not_list_of_dicts(self): + t1 = {1: 2} + + t2 = {1: 3} + + diff = DeepDiff(t1, t2, group_by='id') + expected = {'values_changed': {'root[1]': {'new_value': 3, 'old_value': 2}}} + assert expected == diff + assert {"root[1]"} == diff.affected_paths + + def test_datetime_in_key(self): + + now = datetime.datetime(2022, 4, 10, 0, 40, 41, 357857) + t1 = {now: 1, now + datetime.timedelta(1): 4} + t2 = {now: 2, now + datetime.timedelta(1): 4} + diff = DeepDiff(t1, t2) + expected = {'values_changed': {f'root[{repr(now)}]': {'new_value': 2, 'old_value': 1}}} + + assert expected == diff + + def test_property_values(self): + + class A: + _thing = 0 + + def __init__(self, a): + self.a = a + + @property + def thing(self): + A._thing += 1 + return A._thing + + @property + def __thing2(self): + A._thing += 1 + return A._thing + + diff = DeepDiff(A(1), A(1)) + expected = { + 'values_changed': { + 'root._thing': { + 'new_value': 1, + 'old_value': 0 + }, + 'root.thing': { + 'new_value': 2, + 'old_value': 1 + } + } + } + + assert expected == diff + + diff2 = DeepDiff(A(1), A(1), ignore_private_variables=False) + expected2 = { + 'values_changed': { + 'root._A__thing2': { + 'new_value': 5, + 'old_value': 3 + }, + 'root._thing': { + 'new_value': 5, + 'old_value': 3 + }, + 'root.thing': { + 'new_value': 6, + 'old_value': 4 + } + } + } + + assert expected2 == diff2 + + def test_diffs_rrules(self): + + from dateutil.rrule import MONTHLY, rrule + + d = DeepDiff( + rrule(freq=MONTHLY, count=5, dtstart=datetime.datetime(2014, 12, 31)), + rrule(freq=MONTHLY, count=4, dtstart=datetime.datetime(2011, 12, 31)), + ) + + assert d == { + "values_changed": { + "root[0]": { + "new_value": datetime.datetime(2011, 12, 31, 0, 0, tzinfo=datetime.timezone.utc), + "old_value": datetime.datetime(2014, 12, 31, 0, 0, tzinfo=datetime.timezone.utc), + }, + "root[1]": { + "new_value": datetime.datetime(2012, 1, 31, 0, 0, tzinfo=datetime.timezone.utc), + "old_value": datetime.datetime(2015, 1, 31, 0, 0, tzinfo=datetime.timezone.utc), + }, + "root[2]": { + "new_value": datetime.datetime(2012, 3, 31, 0, 0, tzinfo=datetime.timezone.utc), + "old_value": datetime.datetime(2015, 3, 31, 0, 0, tzinfo=datetime.timezone.utc), + }, + "root[3]": { + "new_value": datetime.datetime(2012, 5, 31, 0, 0, tzinfo=datetime.timezone.utc), + "old_value": datetime.datetime(2015, 5, 31, 0, 0, tzinfo=datetime.timezone.utc), + }, + }, + "iterable_item_removed": {"root[4]": datetime.datetime(2015, 7, 31, 0, 0)}, + } + + def test_pydantic1(self): + + class Foo(PydanticBaseModel): + thing: int = None + that: str + + t1 = Foo(thing=1, that='yes') + t2 = Foo(thing=2, that='yes') + + diff = DeepDiff(t1, t2) + expected = {'values_changed': {'root.thing': {'new_value': 2, 'old_value': 1}}} + assert expected == diff + + def test_pydantic2(self): + + class Foo(PydanticBaseModel): + thing: int = None + that: str + + class Bar(PydanticBaseModel): + stuff: List[Foo] + + t1 = Bar(stuff=[Foo(thing=1, that='yes')]) + t2 = Bar(stuff=[Foo(thing=2, that='yes')]) + + diff = DeepDiff(t1, t2) + expected = {'values_changed': {'root.stuff[0].thing': {'new_value': 2, 'old_value': 1}}} + assert expected == diff + + def test_dataclass1(self): + + + t1 = MyDataClass(1, 4) + t2 = MyDataClass(2, 4) + + diff = DeepDiff(t1, t2, exclude_regex_paths=["any"]) + assert {'values_changed': {'root.val': {'new_value': 2, 'old_value': 1}}} == diff + + def test_dataclass2(self): + + @dataclass(frozen=True) + class MyDataClass: + val: int + val2: int + + t1 = { + MyDataClass(1, 4): 10, + MyDataClass(2, 4): 20, + } + + t2 = { + MyDataClass(1, 4): 10, + MyDataClass(2, 4): 10, + } + + diff = DeepDiff(t1, t2, exclude_regex_paths=["any"]) + assert {'values_changed': {'root[MyDataClass(val=2,val2=4)]': {'new_value': 10, 'old_value': 20}}} == diff + + + def test_group_by_with_none_key_and_ignore_case(self): + """Test that group_by works with None keys when ignore_string_case is True""" + dict1 = [{'txt_field': 'FULL_NONE', 'group_id': None}, {'txt_field': 'FULL', 'group_id': 'a'}] + dict2 = [{'txt_field': 'PARTIAL_NONE', 'group_id': None}, {'txt_field': 'PARTIAL', 'group_id': 'a'}] + + diff = DeepDiff( + dict1, + dict2, + ignore_order=True, + group_by='group_id', + ignore_string_case=True + ) + + expected = {'values_changed': {"root[None]['txt_field']": + {'new_value': 'partial_none', 'old_value': 'full_none'}, + "root['a']['txt_field']": + {'new_value': 'partial', 'old_value': 'full'} + } + } + assert expected == diff + + def test_affected_root_keys_when_dict_empty(self): + diff = DeepDiff({}, {1:1, 2:2}, threshold_to_diff_deeper=0) + assert [1, 2] == diff.affected_root_keys + + diff2 = DeepDiff({}, {1:1, 2:2}) + assert [] == diff2.affected_root_keys + + def test_range1(self): + range1 = range(0, 10) + range2 = range(0, 8) + diff = DeepDiff(range1, range2) + assert {'iterable_item_removed': {'root[8]': 8, 'root[9]': 9}} == diff + + + def test_group_by_that_has_integers(self): + """Test that group_by with integer keys doesn't add type prefixes like 'int:33'""" + t1 = [{'row_num_in_file': 33, 'value': 'old'}] + t2 = [{'row_num_in_file': 33, 'value': 'new'}] + + diff = DeepDiff(t1, t2, group_by='row_num_in_file', ignore_string_type_changes=True) + + # Verify that the diff key contains the integer 33 without type prefix + changes = diff.get('values_changed', {}) + assert len(changes) == 1 + key = list(changes.keys())[0] + assert "int:" not in key + assert "[33]" in key or "['33']" in key