commit 1228fc287989654594ccbd87463c5dda391f4876 Author: upstream Date: Fri Sep 4 03:38:24 2026 +0200 Upstream pyparsing/pyparsing state of the touched files before 8e464a95fc diff --git a/pyparsing/core.py b/pyparsing/core.py new file mode 100644 index 0000000..7ed0af9 --- /dev/null +++ b/pyparsing/core.py @@ -0,0 +1,6992 @@ +# +# core.py +# +from __future__ import annotations + +import collections.abc +from collections import deque +import os +import typing +from typing import ( + Any, + Callable, + Generator, + NamedTuple, + Sequence, + TextIO, + Union, + cast, +) +from abc import ABC, abstractmethod +from enum import Enum +import string +import copy +import warnings +import re +import sys +from collections.abc import Iterable +import traceback +import types +from operator import itemgetter +from functools import wraps +from threading import RLock +from pathlib import Path + +from .warnings import PyparsingDeprecationWarning, PyparsingDiagnosticWarning +from .util import ( + _FifoCache, + _UnboundedCache, + __config_flags, + _collapse_string_to_ranges, + _convert_escaped_numerics_to_char, + _escape_regex_range_chars, + _flatten, + LRUMemo as _LRUMemo, + UnboundedMemo as _UnboundedMemo, + deprecate_argument, + replaced_by_pep8, +) +from .exceptions import * +from .actions import * +from .results import ParseResults, _ParseResultsWithOffset +from .unicode import pyparsing_unicode + +_MAX_INT = sys.maxsize +str_type: tuple[type, ...] = (str, bytes) + +# +# Copyright (c) 2003-2022 Paul T. McGuire +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +from functools import cached_property + + +class __compat__(__config_flags): + """ + A cross-version compatibility configuration for pyparsing features that will be + released in a future version. By setting values in this configuration to True, + those features can be enabled in prior versions for compatibility development + and testing. + + - ``collect_all_And_tokens`` - flag to enable fix for Issue #63 that fixes erroneous grouping + of results names when an :class:`And` expression is nested within an :class:`Or` or :class:`MatchFirst`; + maintained for compatibility, but setting to ``False`` no longer restores pre-2.3.1 + behavior + """ + + _type_desc = "compatibility" + + collect_all_And_tokens = True + + _all_names = [__ for __ in locals() if not __.startswith("_")] + _fixed_names = """ + collect_all_And_tokens + """.split() + + +class __diag__(__config_flags): + _type_desc = "diagnostic" + + warn_multiple_tokens_in_named_alternation = False + warn_ungrouped_named_tokens_in_collection = False + warn_name_set_on_empty_Forward = False + warn_on_parse_using_empty_Forward = False + warn_on_assignment_to_Forward = False + warn_on_multiple_string_args_to_oneof = False + warn_on_match_first_with_lshift_operator = False + enable_debug_on_named_expressions = False + + _all_names = [__ for __ in locals() if not __.startswith("_")] + _warning_names = [name for name in _all_names if name.startswith("warn")] + _debug_names = [name for name in _all_names if name.startswith("enable_debug")] + + @classmethod + def enable_all_warnings(cls) -> None: + for name in cls._warning_names: + cls.enable(name) + + +class Diagnostics(Enum): + """ + Diagnostic configuration (all default to disabled) + + - ``warn_multiple_tokens_in_named_alternation`` - flag to enable warnings when a results + name is defined on a :class:`MatchFirst` or :class:`Or` expression with one or more :class:`And` subexpressions + - ``warn_ungrouped_named_tokens_in_collection`` - flag to enable warnings when a results + name is defined on a containing expression with ungrouped subexpressions that also + have results names + - ``warn_name_set_on_empty_Forward`` - flag to enable warnings when a :class:`Forward` is defined + with a results name, but has no contents defined + - ``warn_on_parse_using_empty_Forward`` - flag to enable warnings when a :class:`Forward` is + defined in a grammar but has never had an expression attached to it + - ``warn_on_assignment_to_Forward`` - flag to enable warnings when a :class:`Forward` is defined + but is overwritten by assigning using ``'='`` instead of ``'<<='`` or ``'<<'`` + - ``warn_on_multiple_string_args_to_oneof`` - flag to enable warnings when :class:`one_of` is + incorrectly called with multiple str arguments + - ``enable_debug_on_named_expressions`` - flag to auto-enable debug on all subsequent + calls to :class:`ParserElement.set_name` + + Diagnostics are enabled/disabled by calling :class:`enable_diag` and :class:`disable_diag`. + All warnings can be enabled by calling :class:`enable_all_warnings`. + """ + + warn_multiple_tokens_in_named_alternation = 0 + warn_ungrouped_named_tokens_in_collection = 1 + warn_name_set_on_empty_Forward = 2 + warn_on_parse_using_empty_Forward = 3 + warn_on_assignment_to_Forward = 4 + warn_on_multiple_string_args_to_oneof = 5 + warn_on_match_first_with_lshift_operator = 6 + enable_debug_on_named_expressions = 7 + + +def enable_diag(diag_enum: Diagnostics) -> None: + """ + Enable a global pyparsing diagnostic flag (see :class:`Diagnostics`). + """ + __diag__.enable(diag_enum.name) + + +def disable_diag(diag_enum: Diagnostics) -> None: + """ + Disable a global pyparsing diagnostic flag (see :class:`Diagnostics`). + """ + __diag__.disable(diag_enum.name) + + +def enable_all_warnings() -> None: + """ + Enable all global pyparsing diagnostic warnings (see :class:`Diagnostics`). + """ + __diag__.enable_all_warnings() + + +# hide abstract class +del __config_flags + + +def _should_enable_warnings( + cmd_line_warn_options: typing.Iterable[str], warn_env_var: typing.Optional[str] +) -> bool: + enable = bool(warn_env_var) + for warn_opt in cmd_line_warn_options: + w_action, w_message, w_category, w_module, w_line = (warn_opt + "::::").split( + ":" + )[:5] + if not w_action.lower().startswith("i") and ( + not (w_message or w_category or w_module) or w_module == "pyparsing" + ): + enable = True + elif w_action.lower().startswith("i") and w_module in ("pyparsing", ""): + enable = False + return enable + + +if _should_enable_warnings( + sys.warnoptions, os.environ.get("PYPARSINGENABLEALLWARNINGS") +): + enable_all_warnings() + + +# build list of single arg builtins, that can be used as parse actions +# fmt: off +_single_arg_builtins = { + sum, len, sorted, reversed, list, tuple, set, any, all, min, max +} +# fmt: on + +_generatorType = types.GeneratorType +ParseImplReturnType = tuple[int, Any] +PostParseReturnType = Union[ParseResults, Sequence[ParseResults]] + +ParseCondition = Union[ + Callable[[], bool], + Callable[[ParseResults], bool], + Callable[[int, ParseResults], bool], + Callable[[str, int, ParseResults], bool], +] +ParseFailAction = Callable[[str, int, "ParserElement", Exception], None] +DebugStartAction = Callable[[str, int, "ParserElement", bool], None] +DebugSuccessAction = Callable[ + [str, int, int, "ParserElement", ParseResults, bool], None +] +DebugExceptionAction = Callable[[str, int, "ParserElement", Exception, bool], None] + + +alphas: str = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +identchars: str = pyparsing_unicode.Latin1.identchars +identbodychars: str = pyparsing_unicode.Latin1.identbodychars +nums: str = "0123456789" +hexnums: str = "0123456789ABCDEFabcdef" +alphanums: str = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" +printables: str = ( + '!"' + "#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" +) + + +class _ParseActionIndexError(Exception): + """ + Internal wrapper around IndexError so that IndexErrors raised inside + parse actions aren't misinterpreted as IndexErrors raised inside + ParserElement parseImpl methods. + """ + + def __init__(self, msg: str, exc: BaseException) -> None: + self.msg: str = msg + self.exc: BaseException = exc + + +_trim_arity_call_line: traceback.StackSummary = None # type: ignore[assignment] +pa_call_line_synth = () + + +def _trim_arity(func, max_limit=3): + """decorator to trim function calls to match the arity of the target""" + global _trim_arity_call_line, pa_call_line_synth + + if func in _single_arg_builtins: + return lambda s, l, t: func(t) + + limit = 0 + found_arity = False + + # synthesize what would be returned by traceback.extract_stack at the call to + # user's parse action 'func', so that we don't incur call penalty at parse time + + # fmt: off + LINE_DIFF = 9 + # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND + # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!! + _trim_arity_call_line = _trim_arity_call_line or traceback.extract_stack(limit=2)[-1] + pa_call_line_synth = pa_call_line_synth or (_trim_arity_call_line[0], _trim_arity_call_line[1] + LINE_DIFF) + + def wrapper(*args): + nonlocal found_arity, limit + if found_arity: + return func(*args[limit:]) + while 1: + try: + ret = func(*args[limit:]) + found_arity = True + return ret + except TypeError as te: + # re-raise TypeErrors if they did not come from our arity testing + if found_arity: + raise + else: + tb = te.__traceback__ + frames = traceback.extract_tb(tb, limit=2) + frame_summary = frames[-1] + trim_arity_type_error = ( + [frame_summary[:2]][-1][:2] == pa_call_line_synth + ) + del tb + + if trim_arity_type_error: + if limit < max_limit: + limit += 1 + continue + + raise + except IndexError as ie: + # wrap IndexErrors inside a _ParseActionIndexError + raise _ParseActionIndexError( + "IndexError raised in parse action", ie + ).with_traceback(None) + # fmt: on + + # copy func name to wrapper for sensible debug output + # (can't use functools.wraps, since that messes with function signature) + func_name = getattr(func, "__name__", getattr(func, "__class__").__name__) + wrapper.__name__ = func_name + wrapper.__doc__ = func.__doc__ + + return wrapper + + +def condition_as_parse_action( + fn: ParseCondition, message: typing.Optional[str] = None, fatal: bool = False +) -> ParseAction: + """ + Function to convert a simple predicate function that returns ``True`` or ``False`` + into a parse action. Can be used in places when a parse action is required + and :meth:`ParserElement.add_condition` cannot be used (such as when adding a condition + to an operator level in :class:`infix_notation`). + + Optional keyword arguments: + + :param message: define a custom message to be used in the raised exception + :param fatal: if ``True``, will raise :class:`ParseFatalException` + to stop parsing immediately; + otherwise will raise :class:`ParseException` + + """ + msg = message if message is not None else "failed user-defined condition" + exc_type = ParseFatalException if fatal else ParseException + fn = _trim_arity(fn) + + @wraps(fn) + def pa(s, l, t): + if not bool(fn(s, l, t)): + raise exc_type(s, l, msg) + + return pa + + +# control characters escaped so they can't corrupt the printed debug line +# (for example a stray '\r' would otherwise return the cursor to column 0) +_debug_control_char_map = {c: repr(chr(c))[1:-1] for c in (*range(0x20), 0x7F)} + + +def _default_start_debug_action( + instring: str, loc: int, expr: ParserElement, cache_hit: bool = False +): + cache_hit_str = "*" if cache_hit else "" + current_col = col(loc, instring) + current_line = line(loc, instring) + escaped_line = current_line.translate(_debug_control_char_map) + # keep the caret under the match location after escaping widens the line + caret_col = ( + len(current_line[: current_col - 1].translate(_debug_control_char_map)) + 1 + ) + print( + ( + f"{cache_hit_str}Match {expr} at loc {loc}({lineno(loc, instring)},{current_col})\n" + f" {escaped_line}\n" + f" {'^':>{caret_col}}" + ) + ) + + +def _default_success_debug_action( + instring: str, + startloc: int, + endloc: int, + expr: ParserElement, + toks: ParseResults, + cache_hit: bool = False, +): + cache_hit_str = "*" if cache_hit else "" + print(f"{cache_hit_str}Matched {expr} -> {toks.as_list()}") + + +def _default_exception_debug_action( + instring: str, + loc: int, + expr: ParserElement, + exc: Exception, + cache_hit: bool = False, +): + cache_hit_str = "*" if cache_hit else "" + print(f"{cache_hit_str}Match {expr} failed, {type(exc).__name__} raised: {exc}") + + +def null_debug_action(*args): + """'Do-nothing' debug action, to suppress debugging output during parsing.""" + + +class ParserElement(ABC): + """Abstract base level parser element class.""" + + DEFAULT_WHITE_CHARS: str = " \n\t\r" + verbose_stacktrace: bool = False + _literalStringClass: type = None # type: ignore[assignment] + + @staticmethod + def set_default_whitespace_chars(chars: str) -> None: + r""" + Overrides the default whitespace chars + + Example: + + .. doctest:: + + # default whitespace chars are space, and newline + >>> Word(alphas)[1, ...].parse_string("abc def\nghi jkl") + ParseResults(['abc', 'def', 'ghi', 'jkl'], {}) + + # change to just treat newline as significant + >>> ParserElement.set_default_whitespace_chars(" \t") + >>> Word(alphas)[1, ...].parse_string("abc def\nghi jkl") + ParseResults(['abc', 'def'], {}) + + # Reset to default + >>> ParserElement.set_default_whitespace_chars(" \n\t\r") + """ + ParserElement.DEFAULT_WHITE_CHARS = chars + + # update whitespace all parse expressions defined in this module + for expr in _builtin_exprs: + if expr.copyDefaultWhiteChars: + expr.whiteChars = set(chars) + + @staticmethod + def inline_literals_using(cls: type) -> None: + """ + Set class to be used for inclusion of string literals into a parser. + + Example: + + .. doctest:: + :options: +NORMALIZE_WHITESPACE + + # default literal class used is Literal + >>> integer = Word(nums) + >>> date_str = ( + ... integer("year") + '/' + ... + integer("month") + '/' + ... + integer("day") + ... ) + + >>> date_str.parse_string("1999/12/31") + ParseResults(['1999', '/', '12', '/', '31'], + {'year': '1999', 'month': '12', 'day': '31'}) + + # change to Suppress + >>> ParserElement.inline_literals_using(Suppress) + >>> date_str = ( + ... integer("year") + '/' + ... + integer("month") + '/' + ... + integer("day") + ... ) + + >>> date_str.parse_string("1999/12/31") + ParseResults(['1999', '12', '31'], + {'year': '1999', 'month': '12', 'day': '31'}) + + # Reset + >>> ParserElement.inline_literals_using(Literal) + """ + ParserElement._literalStringClass = cls + + @classmethod + def using_each(cls, seq, **class_kwargs): + """ + Yields a sequence of ``class(obj, **class_kwargs)`` for obj in seq. + + Example: + + .. testcode:: + + LPAR, RPAR, LBRACE, RBRACE, SEMI = Suppress.using_each("(){};") + + .. versionadded:: 3.1.0 + """ + yield from (cls(obj, **class_kwargs) for obj in seq) + + class DebugActions(NamedTuple): + debug_try: typing.Optional[DebugStartAction] + debug_match: typing.Optional[DebugSuccessAction] + debug_fail: typing.Optional[DebugExceptionAction] + + def __init__(self, savelist: bool = False) -> None: + self.parseAction: list[ParseAction] = list() + self.failAction: typing.Optional[ParseFailAction] = None + self.customName: str = None # type: ignore[assignment] + self._defaultName: typing.Optional[str] = None + self.resultsName: str = None # type: ignore[assignment] + self.saveAsList: bool = savelist + self.skipWhitespace: bool = True + self.whiteChars: set[str] = set(ParserElement.DEFAULT_WHITE_CHARS) + self.copyDefaultWhiteChars: bool = True + # used when checking for left-recursion + self._may_return_empty: bool = False + self.keepTabs: bool = False + self.ignoreExprs: list[ParserElement] = list() + self.debug: bool = False + self.streamlined: bool = False + # optimize exception handling for subclasses that don't advance parse index + self.mayIndexError: bool = True + self.errmsg: Union[str, None] = "" + # mark results names as modal (report only last) or cumulative (list all) + self.modalResults: bool = True + # custom debug actions + self.debugActions = self.DebugActions(None, None, None) + # avoid redundant calls to preParse + self.callPreparse: bool = True + self.callDuringTry: bool = False + self.suppress_warnings_: list[Diagnostics] = [] + self.show_in_diagram: bool = True + + @property + def mayReturnEmpty(self) -> bool: + """ + .. deprecated:: 3.3.0 + use _may_return_empty instead. + """ + return self._may_return_empty + + @mayReturnEmpty.setter + def mayReturnEmpty(self, value) -> None: + """ + .. deprecated:: 3.3.0 + use _may_return_empty instead. + """ + self._may_return_empty = value + + def suppress_warning(self, warning_type: Diagnostics) -> ParserElement: + """ + Suppress warnings emitted for a particular diagnostic on this expression. + + Example: + + .. doctest:: + + >>> label = pp.Word(pp.alphas) + + # Normally using an empty Forward in a grammar + # would print a warning, but we can suppress that + >>> base = pp.Forward().suppress_warning( + ... pp.Diagnostics.warn_on_parse_using_empty_Forward) + + >>> grammar = base | label + >>> print(grammar.parse_string("x")) + ['x'] + """ + self.suppress_warnings_.append(warning_type) + return self + + def visit_all(self): + """General-purpose method to yield all expressions and sub-expressions + in a grammar. Typically just for internal use. + """ + to_visit = deque([self]) + seen = set() + while to_visit: + cur = to_visit.popleft() + + # guard against looping forever through recursive grammars + if cur in seen: + continue + seen.add(cur) + + to_visit.extend(cur.recurse()) + yield cur + + def copy(self) -> ParserElement: + """ + Make a copy of this :class:`ParserElement`. Useful for defining + different parse actions for the same parsing pattern, using copies of + the original parse element. + + Example: + + .. testcode:: + + integer = Word(nums).set_parse_action( + lambda toks: int(toks[0])) + integerK = integer.copy().add_parse_action( + lambda toks: toks[0] * 1024) + Suppress("K") + integerM = integer.copy().add_parse_action( + lambda toks: toks[0] * 1024 * 1024) + Suppress("M") + + print( + (integerK | integerM | integer)[1, ...].parse_string( + "5K 100 640K 256M") + ) + + prints: + + .. testoutput:: + + [5120, 100, 655360, 268435456] + + Equivalent form of ``expr.copy()`` is just ``expr()``: + + .. testcode:: + + integerM = integer().add_parse_action( + lambda toks: toks[0] * 1024 * 1024) + Suppress("M") + """ + cpy = copy.copy(self) + cpy.parseAction = self.parseAction[:] + cpy.ignoreExprs = self.ignoreExprs[:] + if self.copyDefaultWhiteChars: + cpy.whiteChars = set(ParserElement.DEFAULT_WHITE_CHARS) + return cpy + + def set_results_name( + self, name: str, list_all_matches: bool = False, **kwargs + ) -> ParserElement: + """ + Define name for referencing matching tokens as a nested attribute + of the returned parse results. + + Normally, results names are assigned as you would assign keys in a dict: + any existing value is overwritten by later values. If it is necessary to + keep all values captured for a particular results name, call ``set_results_name`` + with ``list_all_matches`` = True. + + NOTE: ``set_results_name`` returns a *copy* of the original :class:`ParserElement` object; + this is so that the client can define a basic element, such as an + integer, and reference it in multiple places with different names. + + You can also set results names using the abbreviated syntax, + ``expr("name")`` in place of ``expr.set_results_name("name")`` + - see :meth:`__call__`. If ``list_all_matches`` is required, use + ``expr("name*")``. + + Example: + + .. testcode:: + + integer = Word(nums) + date_str = (integer.set_results_name("year") + '/' + + integer.set_results_name("month") + '/' + + integer.set_results_name("day")) + + # equivalent form: + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + """ + listAllMatches: bool = deprecate_argument(kwargs, "listAllMatches", False) + + list_all_matches = listAllMatches or list_all_matches + return self._setResultsName(name, list_all_matches) + + def _setResultsName(self, name, list_all_matches=False) -> ParserElement: + if name is None: + return self + newself = self.copy() + if name.endswith("*"): + name = name[:-1] + list_all_matches = True + newself.resultsName = name + newself.modalResults = not list_all_matches + return newself + + def set_break(self, break_flag: bool = True) -> ParserElement: + """ + Method to invoke the Python pdb debugger when this element is + about to be parsed. Set ``break_flag`` to ``True`` to enable, ``False`` to + disable. + """ + if break_flag: + _parseMethod = self._parse + + def breaker(instring, loc, do_actions=True, callPreParse=True): + # this call to breakpoint() is intentional, not a checkin error + breakpoint() + return _parseMethod(instring, loc, do_actions, callPreParse) + + breaker._originalParseMethod = _parseMethod # type: ignore [attr-defined] + self._parse = breaker # type: ignore [method-assign] + elif hasattr(self._parse, "_originalParseMethod"): + self._parse = self._parse._originalParseMethod # type: ignore [method-assign] + return self + + def set_parse_action( + self, *fns: ParseAction, call_during_try: bool = False, **kwargs: Any + ) -> ParserElement: + """ + Define one or more actions to perform when successfully matching parse element definition. + + Parse actions can be called to perform data conversions, do extra validation, + update external data structures, or enhance or replace the parsed tokens. + Each parse action ``fn`` is a callable method with 0-3 arguments, called as + ``fn(s, loc, toks)`` , ``fn(loc, toks)`` , ``fn(toks)`` , or just ``fn()`` , where: + + - ``s`` = the original string being parsed (see note below) + - ``loc`` = the location of the matching substring + - ``toks`` = a list of the matched tokens, packaged as a :class:`ParseResults` object + + The parsed tokens are passed to the parse action as ParseResults. They can be + modified in place using list-style append, extend, and pop operations to update + the parsed list elements; and with dictionary-style item set and del operations + to add, update, or remove any named results. If the tokens are modified in place, + it is not necessary to return them with a return statement. + + Parse actions can also completely replace the given tokens, with another ``ParseResults`` + object, or with some entirely different object (common for parse actions that perform data + conversions). A convenient way to build a new parse result is to define the values + using a dict, and then create the return value using :class:`ParseResults.from_dict`. + + If None is passed as the ``fn`` parse action, all previously added parse actions for this + expression are cleared. + + Optional keyword arguments: + + :param call_during_try: (default= ``False``) indicate if parse action + should be run during lookaheads and alternate + testing. For parse actions that have side + effects, it is important to only call the parse + action once it is determined that it is being + called as part of a successful parse. + For parse actions that perform additional + validation, then ``call_during_try`` should + be passed as True, so that the validation code + is included in the preliminary "try" parses. + + .. Note:: + The default parsing behavior is to expand tabs in the input string + before starting the parsing process. + See :meth:`parse_string` for more information on parsing strings + containing ```` s, and suggested methods to maintain a + consistent view of the parsed string, the parse location, and + line and column positions within the parsed string. + + Example: Parse dates in the form ``YYYY/MM/DD`` + ----------------------------------------------- + + Setup code: + + .. testcode:: + + def convert_to_int(toks): + '''a parse action to convert toks from str to int + at parse time''' + return int(toks[0]) + + def is_valid_date(instring, loc, toks): + '''a parse action to verify that the date is a valid date''' + from datetime import date + year, month, day = toks[::2] + try: + date(year, month, day) + except ValueError: + raise ParseException(instring, loc, "invalid date given") + + integer = Word(nums) + date_str = integer + '/' + integer + '/' + integer + + # add parse actions + integer.set_parse_action(convert_to_int) + date_str.set_parse_action(is_valid_date) + + Successful parse - note that integer fields are converted to ints: + + .. testcode:: + + print(date_str.parse_string("1999/12/31")) + + prints: + + .. testoutput:: + + [1999, '/', 12, '/', 31] + + Failure - invalid date: + + .. testcode:: + + date_str.parse_string("1999/13/31") + + prints: + + .. testoutput:: + + Traceback (most recent call last): + ParseException: invalid date given, found '1999' ... + """ + callDuringTry: bool = deprecate_argument(kwargs, "callDuringTry", False) + + if list(fns) == [None]: + self.parseAction.clear() + return self + + if not all(callable(fn) for fn in fns): + raise TypeError("parse actions must be callable") + self.parseAction[:] = [_trim_arity(fn) for fn in fns] + self.callDuringTry = self.callDuringTry or call_during_try or callDuringTry + + return self + + def add_parse_action( + self, *fns: ParseAction, call_during_try: bool = False, **kwargs: Any + ) -> ParserElement: + """ + Add one or more parse actions to expression's list of parse actions. See :class:`set_parse_action`. + + See examples in :class:`copy`. + """ + callDuringTry: bool = deprecate_argument(kwargs, "callDuringTry", False) + + self.parseAction += [_trim_arity(fn) for fn in fns] + self.callDuringTry = self.callDuringTry or callDuringTry or call_during_try + return self + + def add_condition( + self, *fns: ParseCondition, call_during_try: bool = False, **kwargs: Any + ) -> ParserElement: + """Add a boolean predicate function to expression's list of parse actions. See + :class:`set_parse_action` for function call signatures. Unlike ``set_parse_action``, + functions passed to ``add_condition`` need to return boolean success/fail of the condition. + + Optional keyword arguments: + + - ``message`` = define a custom message to be used in the raised exception + - ``fatal`` = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise + ParseException + - ``call_during_try`` = boolean to indicate if this method should be called during internal tryParse calls, + default=False + + Example: + + .. doctest:: + :options: +NORMALIZE_WHITESPACE + + >>> integer = Word(nums).set_parse_action(lambda toks: int(toks[0])) + >>> year_int = integer.copy().add_condition( + ... lambda toks: toks[0] >= 2000, + ... message="Only support years 2000 and later") + >>> date_str = year_int + '/' + integer + '/' + integer + + >>> result = date_str.parse_string("1999/12/31") + Traceback (most recent call last): + ParseException: Only support years 2000 and later... + """ + callDuringTry: bool = deprecate_argument(kwargs, "callDuringTry", False) + + for fn in fns: + self.parseAction.append( + condition_as_parse_action( + fn, + message=str(kwargs.get("message")), + fatal=bool(kwargs.get("fatal", False)), + ) + ) + + self.callDuringTry = self.callDuringTry or call_during_try or callDuringTry + return self + + def set_fail_action(self, fn: ParseFailAction) -> ParserElement: + """ + Define action to perform if parsing fails at this expression. + Fail acton fn is a callable function that takes the arguments + ``fn(s, loc, expr, err)`` where: + + - ``s`` = string being parsed + - ``loc`` = location where expression match was attempted and failed + - ``expr`` = the parse expression that failed + - ``err`` = the exception thrown + + The function returns no value. It may throw :class:`ParseFatalException` + if it is desired to stop parsing immediately.""" + self.failAction = fn + return self + + def _skipIgnorables(self, instring: str, loc: int) -> int: + if not self.ignoreExprs: + return loc + exprsFound = True + ignore_expr_fns = [e._parse for e in self.ignoreExprs] + last_loc = loc + while exprsFound: + exprsFound = False + for ignore_fn in ignore_expr_fns: + try: + while 1: + loc, dummy = ignore_fn(instring, loc) + exprsFound = True + except ParseException: + pass + # check if all ignore exprs matched but didn't actually advance the parse location + if loc == last_loc: + break + last_loc = loc + return loc + + def preParse(self, instring: str, loc: int) -> int: + if self.ignoreExprs: + loc = self._skipIgnorables(instring, loc) + + if self.skipWhitespace: + instrlen = len(instring) + white_chars = self.whiteChars + while loc < instrlen and instring[loc] in white_chars: + loc += 1 + + return loc + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + return loc, [] + + def postParse(self, instring, loc, tokenlist): + return tokenlist + + # @profile + def _parseNoCache( + self, instring, loc, do_actions=True, callPreParse=True + ) -> tuple[int, ParseResults]: + debugging = self.debug # and do_actions) + len_instring = len(instring) + + if debugging or self.failAction: + # print("Match {} at loc {}({}, {})".format(self, loc, lineno(loc, instring), col(loc, instring))) + try: + if callPreParse and self.callPreparse: + pre_loc = self.preParse(instring, loc) + else: + pre_loc = loc + tokens_start = pre_loc + if self.debugActions.debug_try: + self.debugActions.debug_try(instring, tokens_start, self, False) + if self.mayIndexError or pre_loc >= len_instring: + try: + loc, tokens = self.parseImpl(instring, pre_loc, do_actions) + except IndexError: + raise ParseException(instring, len_instring, self.errmsg, self) + else: + loc, tokens = self.parseImpl(instring, pre_loc, do_actions) + except Exception as err: + # print("Exception raised:", err) + if self.debugActions.debug_fail: + self.debugActions.debug_fail( + instring, tokens_start, self, err, False + ) + if self.failAction: + self.failAction(instring, tokens_start, self, err) + raise + else: + if callPreParse and self.callPreparse: + pre_loc = self.preParse(instring, loc) + else: + pre_loc = loc + tokens_start = pre_loc + if self.mayIndexError or pre_loc >= len_instring: + try: + loc, tokens = self.parseImpl(instring, pre_loc, do_actions) + except IndexError: + raise ParseException(instring, len_instring, self.errmsg, self) + else: + loc, tokens = self.parseImpl(instring, pre_loc, do_actions) + + tokens = self.postParse(instring, loc, tokens) + + ret_tokens = ParseResults( + tokens, self.resultsName, aslist=self.saveAsList, modal=self.modalResults + ) + if self.parseAction and (do_actions or self.callDuringTry): + if debugging: + try: + for fn in self.parseAction: + try: + tokens = fn(instring, tokens_start, ret_tokens) # type: ignore [call-arg, arg-type] + except IndexError as parse_action_exc: + exc = ParseException("exception raised in parse action") + raise exc from parse_action_exc + + if tokens is not None and tokens is not ret_tokens: + ret_tokens = ParseResults( + tokens, + self.resultsName, + aslist=self.saveAsList + and isinstance(tokens, (ParseResults, list)), + modal=self.modalResults, + ) + except Exception as err: + # print "Exception raised in user parse action:", err + if self.debugActions.debug_fail: + self.debugActions.debug_fail( + instring, tokens_start, self, err, False + ) + raise + else: + for fn in self.parseAction: + try: + tokens = fn(instring, tokens_start, ret_tokens) # type: ignore [call-arg, arg-type] + except IndexError as parse_action_exc: + exc = ParseException("exception raised in parse action") + raise exc from parse_action_exc + + if tokens is not None and tokens is not ret_tokens: + ret_tokens = ParseResults( + tokens, + self.resultsName, + aslist=self.saveAsList + and isinstance(tokens, (ParseResults, list)), + modal=self.modalResults, + ) + if debugging: + # print("Matched", self, "->", ret_tokens.as_list()) + if self.debugActions.debug_match: + self.debugActions.debug_match( + instring, tokens_start, loc, self, ret_tokens, False + ) + + return loc, ret_tokens + + def try_parse( + self, + instring: str, + loc: int, + *, + raise_fatal: bool = False, + do_actions: bool = False, + ) -> int: + try: + return self._parse(instring, loc, do_actions=do_actions)[0] + except ParseFatalException: + if raise_fatal: + raise + raise ParseException(instring, loc, self.errmsg, self) + + def can_parse_next(self, instring: str, loc: int, do_actions: bool = False) -> bool: + try: + self.try_parse(instring, loc, do_actions=do_actions) + except (ParseException, IndexError): + return False + else: + return True + + # cache for left-recursion in Forward references + recursion_lock = RLock() + recursion_memos: collections.abc.MutableMapping[ + tuple[int, Forward, bool], tuple[int, Union[ParseResults, Exception]] + ] = {} + + class _CacheType(typing.Protocol): + """ + Class to be used for packrat and left-recursion cacheing of results + and exceptions. + """ + + not_in_cache: bool + + def get(self, *args) -> typing.Any: ... + + def set(self, *args) -> None: ... + + def clear(self) -> None: ... + + class NullCache(dict): + """ + A null cache type for initialization of the packrat_cache class variable. + If/when enable_packrat() is called, this null cache will be replaced by a + proper _CacheType class instance. + """ + + not_in_cache: bool = True + + def get(self, *args) -> typing.Any: ... + + def set(self, *args) -> None: ... + + def clear(self) -> None: ... + + # class-level argument cache for optimizing repeated calls when backtracking + # through recursive expressions + packrat_cache: _CacheType = NullCache() + packrat_cache_lock = RLock() + packrat_cache_stats = [0, 0] + + # this method gets repeatedly called during backtracking with the same arguments - + # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression + def _parseCache( + self, instring, loc, do_actions=True, callPreParse=True + ) -> tuple[int, ParseResults]: + HIT, MISS = 0, 1 + lookup = (self, instring, loc, callPreParse, do_actions) + with ParserElement.packrat_cache_lock: + cache = ParserElement.packrat_cache + value = cache.get(lookup) + if value is cache.not_in_cache: + ParserElement.packrat_cache_stats[MISS] += 1 + try: + value = self._parseNoCache(instring, loc, do_actions, callPreParse) + except ParseBaseException as pe: + # cache a copy of the exception, without the traceback + cache.set(lookup, pe.__class__(*pe.args)) + raise + else: + cache.set(lookup, (value[0], value[1].copy(), loc)) + return value + else: + ParserElement.packrat_cache_stats[HIT] += 1 + if self.debug and self.debugActions.debug_try: + try: + self.debugActions.debug_try(instring, loc, self, cache_hit=True) # type: ignore [call-arg] + except TypeError: + pass + if isinstance(value, Exception): + if self.debug and self.debugActions.debug_fail: + try: + self.debugActions.debug_fail( + instring, loc, self, value, cache_hit=True # type: ignore [call-arg] + ) + except TypeError: + pass + raise value + + value = cast(tuple[int, ParseResults, int], value) + loc_, result, endloc = value[0], value[1].copy(), value[2] + if self.debug and self.debugActions.debug_match: + try: + self.debugActions.debug_match( + instring, loc_, endloc, self, result, cache_hit=True # type: ignore [call-arg] + ) + except TypeError: + pass + + return loc_, result + + _parse = _parseNoCache + + @staticmethod + def reset_cache() -> None: + """ + Clears caches used by packrat and left-recursion. + """ + with ParserElement.packrat_cache_lock: + ParserElement.packrat_cache.clear() + ParserElement.packrat_cache_stats[:] = [0] * len( + ParserElement.packrat_cache_stats + ) + ParserElement.recursion_memos.clear() + + # class attributes to keep caching status + _packratEnabled = False + _left_recursion_enabled = False + + @staticmethod + def disable_memoization() -> None: + """ + Disables active Packrat or Left Recursion parsing and their memoization + + This method also works if neither Packrat nor Left Recursion are enabled. + This makes it safe to call before activating Packrat nor Left Recursion + to clear any previous settings. + """ + with ParserElement.packrat_cache_lock: + ParserElement.reset_cache() + ParserElement._left_recursion_enabled = False + ParserElement._packratEnabled = False + ParserElement._parse = ParserElement._parseNoCache + + @staticmethod + def enable_left_recursion( + cache_size_limit: typing.Optional[int] = None, *, force=False + ) -> None: + """ + Enables "bounded recursion" parsing, which allows for both direct and indirect + left-recursion. During parsing, left-recursive :class:`Forward` elements are + repeatedly matched with a fixed recursion depth that is gradually increased + until finding the longest match. + + Example: + + .. testcode:: + + import pyparsing as pp + pp.ParserElement.enable_left_recursion() + + E = pp.Forward("E") + num = pp.Word(pp.nums) + + # match `num`, or `num '+' num`, or `num '+' num '+' num`, ... + E <<= E + '+' - num | num + + print(E.parse_string("1+2+3+4")) + + prints: + + .. testoutput:: + + ['1', '+', '2', '+', '3', '+', '4'] + + Recursion search naturally memoizes matches of ``Forward`` elements and may + thus skip reevaluation of parse actions during backtracking. This may break + programs with parse actions which rely on strict ordering of side-effects. + + Parameters: + + - ``cache_size_limit`` - (default=``None``) - memoize at most this many + ``Forward`` elements during matching; if ``None`` (the default), + memoize all ``Forward`` elements. + + Bounded Recursion parsing works similar but not identical to Packrat parsing, + thus the two cannot be used together. Use ``force=True`` to disable any + previous, conflicting settings. + """ + with ParserElement.packrat_cache_lock: + if force: + ParserElement.disable_memoization() + elif ParserElement._packratEnabled: + raise RuntimeError("Packrat and Bounded Recursion are not compatible") + if cache_size_limit is None: + ParserElement.recursion_memos = _UnboundedMemo() + elif cache_size_limit > 0: + ParserElement.recursion_memos = _LRUMemo(capacity=cache_size_limit) # type: ignore[assignment] + else: + raise NotImplementedError(f"Memo size of {cache_size_limit}") + ParserElement._left_recursion_enabled = True + + @staticmethod + def enable_packrat( + cache_size_limit: Union[int, None] = 128, *, force: bool = False + ) -> None: + """ + Enables "packrat" parsing, which adds memoizing to the parsing logic. + Repeated parse attempts at the same string location (which happens + often in many complex grammars) can immediately return a cached value, + instead of re-executing parsing/validating code. Memoizing is done of + both valid results and parsing exceptions. + + Parameters: + + - ``cache_size_limit`` - (default= ``128``) - if an integer value is provided + will limit the size of the packrat cache; if None is passed, then + the cache size will be unbounded; if 0 is passed, the cache will + be effectively disabled. + + This speedup may break existing programs that use parse actions that + have side-effects. For this reason, packrat parsing is disabled when + you first import pyparsing. To activate the packrat feature, your + program must call the class method :class:`ParserElement.enable_packrat`. + For best results, call ``enable_packrat()`` immediately after + importing pyparsing. + + .. Can't really be doctested, alas + + Example:: + + import pyparsing + pyparsing.ParserElement.enable_packrat() + + Packrat parsing works similar but not identical to Bounded Recursion parsing, + thus the two cannot be used together. Use ``force=True`` to disable any + previous, conflicting settings. + """ + with ParserElement.packrat_cache_lock: + if force: + ParserElement.disable_memoization() + elif ParserElement._left_recursion_enabled: + raise RuntimeError("Packrat and Bounded Recursion are not compatible") + + if ParserElement._packratEnabled: + return + + ParserElement._packratEnabled = True + if cache_size_limit is None: + ParserElement.packrat_cache = _UnboundedCache() + else: + ParserElement.packrat_cache = _FifoCache(cache_size_limit) + ParserElement._parse = ParserElement._parseCache + + def parse_string( + self, instring: str, parse_all: bool = False, **kwargs + ) -> ParseResults: + """ + Parse a string with respect to the parser definition. This function is intended as the primary interface to the + client code. + + :param instring: The input string to be parsed. + :param parse_all: If set, the entire input string must match the grammar. + :param parseAll: retained for pre-PEP8 compatibility, will be removed in a future release. + :raises ParseException: Raised if ``parse_all`` is set and the input string does not match the whole grammar. + :returns: the parsed data as a :class:`ParseResults` object, which may be accessed as a `list`, a `dict`, or + an object with attributes if the given parser includes results names. + + If the input string is required to match the entire grammar, ``parse_all`` flag must be set to ``True``. This + is also equivalent to ending the grammar with :class:`StringEnd`\\ (). + + To report proper column numbers, ``parse_string`` operates on a copy of the input string where all tabs are + converted to spaces (8 spaces per tab, as per the default in ``string.expandtabs``). If the input string + contains tabs and the grammar uses parse actions that use the ``loc`` argument to index into the string + being parsed, one can ensure a consistent view of the input string by doing one of the following: + + - calling ``parse_with_tabs`` on your grammar before calling ``parse_string`` (see :class:`parse_with_tabs`), + - define your parse action using the full ``(s,loc,toks)`` signature, and reference the input string using the + parse action's ``s`` argument, or + - explicitly expand the tabs in your input string before calling ``parse_string``. + + Examples: + + By default, partial matches are OK. + + .. doctest:: + + >>> res = Word('a').parse_string('aaaaabaaa') + >>> print(res) + ['aaaaa'] + + The parsing behavior varies by the inheriting class of this abstract class. Please refer to the children + directly to see more examples. + + It raises an exception if parse_all flag is set and instring does not match the whole grammar. + + .. doctest:: + + >>> res = Word('a').parse_string('aaaaabaaa', parse_all=True) + Traceback (most recent call last): + ParseException: Expected end of text, found 'b' ... + """ + parseAll: bool = deprecate_argument(kwargs, "parseAll", False) + + parse_all = parse_all or parseAll + + ParserElement.reset_cache() + if not self.streamlined: + self.streamline() + for e in self.ignoreExprs: + e.streamline() + if not self.keepTabs: + instring = instring.expandtabs() + try: + loc, tokens = self._parse(instring, 0) + if parse_all: + loc = self.preParse(instring, loc) + se = Empty() + StringEnd().set_debug(False) + se._parse(instring, loc) + except _ParseActionIndexError as pa_exc: + raise pa_exc.exc + except ParseBaseException as exc: + if ParserElement.verbose_stacktrace: + raise + + # catch and re-raise exception from here, clearing out pyparsing internal stack trace + raise exc.with_traceback(None) + else: + return tokens + + def scan_string( + self, + instring: str, + max_matches: int = _MAX_INT, + overlap: bool = False, + always_skip_whitespace=True, + *, + debug: bool = False, + **kwargs, + ) -> Generator[tuple[ParseResults, int, int], None, None]: + """ + Scan the input string for expression matches. Each match will return the + matching tokens, start location, and end location. May be called with optional + ``max_matches`` argument, to clip scanning after 'n' matches are found. If + ``overlap`` is specified, then overlapping matches will be reported. + + Note that the start and end locations are reported relative to the string + being parsed. See :class:`parse_string` for more information on parsing + strings with embedded tabs. + + Example: + + .. testcode:: + + source = "sldjf123lsdjjkf345sldkjf879lkjsfd987" + print(source) + for tokens, start, end in Word(alphas).scan_string(source): + print(' '*start + '^'*(end-start)) + print(' '*start + tokens[0]) + + prints: + + .. testoutput:: + + sldjf123lsdjjkf345sldkjf879lkjsfd987 + ^^^^^ + sldjf + ^^^^^^^ + lsdjjkf + ^^^^^^ + sldkjf + ^^^^^^ + lkjsfd + """ + maxMatches: int = deprecate_argument(kwargs, "maxMatches", _MAX_INT) + + max_matches = min(maxMatches, max_matches) + if not self.streamlined: + self.streamline() + for e in self.ignoreExprs: + e.streamline() + + if not self.keepTabs: + instring = str(instring).expandtabs() + instrlen = len(instring) + loc = 0 + if always_skip_whitespace: + preparser = Empty() + preparser.ignoreExprs = self.ignoreExprs + preparser.whiteChars = self.whiteChars + preparseFn = preparser.preParse + else: + preparseFn = self.preParse + parseFn = self._parse + ParserElement.reset_cache() + matches = 0 + try: + while loc <= instrlen and matches < max_matches: + try: + preloc: int = preparseFn(instring, loc) + nextLoc: int + tokens: ParseResults + nextLoc, tokens = parseFn(instring, preloc, callPreParse=False) + except ParseException: + loc = preloc + 1 + else: + if nextLoc > loc: + matches += 1 + if debug: + print( + { + "tokens": tokens.as_list(), + "start": preloc, + "end": nextLoc, + } + ) + yield tokens, preloc, nextLoc + if overlap: + nextloc = preparseFn(instring, loc) + if nextloc > loc: + loc = nextLoc + else: + loc += 1 + else: + loc = nextLoc + else: + loc = preloc + 1 + except ParseBaseException as exc: + if ParserElement.verbose_stacktrace: + raise + + # catch and re-raise exception from here, clears out pyparsing internal stack trace + raise exc.with_traceback(None) + + def transform_string(self, instring: str, *, debug: bool = False) -> str: + """ + Extension to :class:`scan_string`, to modify matching text with modified tokens that may + be returned from a parse action. To use ``transform_string``, define a grammar and + attach a parse action to it that modifies the returned token list. + Invoking ``transform_string()`` on a target string will then scan for matches, + and replace the matched text patterns according to the logic in the parse + action. ``transform_string()`` returns the resulting transformed string. + + Example: + + .. testcode:: + + quote = '''now is the winter of our discontent, + made glorious summer by this sun of york.''' + + wd = Word(alphas) + wd.set_parse_action(lambda toks: toks[0].title()) + + print(wd.transform_string(quote)) + + prints: + + .. testoutput:: + + Now Is The Winter Of Our Discontent, + Made Glorious Summer By This Sun Of York. + """ + out: list[str] = [] + lastE = 0 + # force preservation of s, to minimize unwanted transformation of string, and to + # keep string locs straight between transform_string and scan_string + self.keepTabs = True + try: + for t, s, e in self.scan_string(instring, debug=debug): + if s > lastE: + out.append(instring[lastE:s]) + lastE = e + + if not t: + continue + + if isinstance(t, ParseResults): + out += t.as_list() + elif isinstance(t, Iterable) and not isinstance(t, str_type): + out.extend(t) + else: + out.append(t) + + out.append(instring[lastE:]) + out = [o for o in out if o] + return "".join([str(s) for s in _flatten(out)]) + except ParseBaseException as exc: + if ParserElement.verbose_stacktrace: + raise + + # catch and re-raise exception from here, clears out pyparsing internal stack trace + raise exc.with_traceback(None) + + def search_string( + self, + instring: str, + max_matches: int = _MAX_INT, + *, + debug: bool = False, + **kwargs, + ) -> ParseResults: + """ + Another extension to :class:`scan_string`, simplifying the access to the tokens found + to match the given parse expression. May be called with optional + ``max_matches`` argument, to clip searching after 'n' matches are found. + + Example: + + .. testcode:: + + quote = '''More than Iron, more than Lead, + more than Gold I need Electricity''' + + # a capitalized word starts with an uppercase letter, + # followed by zero or more lowercase letters + cap_word = Word(alphas.upper(), alphas.lower()) + + print(cap_word.search_string(quote)) + + # the sum() builtin can be used to merge results + # into a single ParseResults object + print(sum(cap_word.search_string(quote))) + + prints: + + .. testoutput:: + + [['More'], ['Iron'], ['Lead'], ['Gold'], ['I'], ['Electricity']] + ['More', 'Iron', 'Lead', 'Gold', 'I', 'Electricity'] + """ + maxMatches: int = deprecate_argument(kwargs, "maxMatches", _MAX_INT) + + max_matches = min(maxMatches, max_matches) + try: + return ParseResults( + [ + t + for t, s, e in self.scan_string( + instring, + max_matches=max_matches, + always_skip_whitespace=False, + debug=debug, + ) + ] + ) + except ParseBaseException as exc: + if ParserElement.verbose_stacktrace: + raise + + # catch and re-raise exception from here, clears out pyparsing internal stack trace + raise exc.with_traceback(None) + + def split( + self, + instring: str, + maxsplit: int = _MAX_INT, + include_separators: bool = False, + **kwargs, + ) -> Generator[str, None, None]: + """ + Generator method to split a string using the given expression as a separator. + May be called with optional ``maxsplit`` argument, to limit the number of splits; + and the optional ``include_separators`` argument (default= ``False``), if the separating + matching text should be included in the split results. + + Example: + + .. testcode:: + + punc = one_of(list(".,;:/-!?")) + print(list(punc.split( + "This, this?, this sentence, is badly punctuated!"))) + + prints: + + .. testoutput:: + + ['This', ' this', '', ' this sentence', ' is badly punctuated', ''] + """ + includeSeparators: bool = deprecate_argument(kwargs, "includeSeparators", False) + + include_separators = includeSeparators or include_separators + last = 0 + for t, s, e in self.scan_string(instring, max_matches=maxsplit): + yield instring[last:s] + if include_separators: + yield t[0] + last = e + yield instring[last:] + + def __add__(self, other) -> ParserElement: + """ + Implementation of ``+`` operator - returns :class:`And`. Adding strings to a :class:`ParserElement` + converts them to :class:`Literal`\\ s by default. + + Example: + + .. testcode:: + + greet = Word(alphas) + "," + Word(alphas) + "!" + hello = "Hello, World!" + print(hello, "->", greet.parse_string(hello)) + + prints: + + .. testoutput:: + + Hello, World! -> ['Hello', ',', 'World', '!'] + + ``...`` may be used as a parse expression as a short form of :class:`SkipTo`: + + .. testcode:: + + Literal('start') + ... + Literal('end') + + is equivalent to: + + .. testcode:: + + Literal('start') + SkipTo('end')("_skipped*") + Literal('end') + + Note that the skipped text is returned with '_skipped' as a results name, + and to support having multiple skips in the same parser, the value returned is + a list of all skipped text. + """ + if other is Ellipsis: + return _PendingSkip(self) + + if isinstance(other, str_type): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented + return And([self, other]) + + def __radd__(self, other) -> ParserElement: + """ + Implementation of ``+`` operator when left operand is not a :class:`ParserElement` + """ + if other is Ellipsis: + return SkipTo(self)("_skipped*") + self + + if isinstance(other, str_type): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented + return other + self + + def __sub__(self, other) -> ParserElement: + """ + Implementation of ``-`` operator, returns :class:`And` with error stop + """ + if isinstance(other, str_type): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented + return self + And._ErrorStop() + other + + def __rsub__(self, other) -> ParserElement: + """ + Implementation of ``-`` operator when left operand is not a :class:`ParserElement` + """ + if isinstance(other, str_type): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented + return other - self + + def __mul__(self, other) -> ParserElement: + """ + Implementation of ``*`` operator, allows use of ``expr * 3`` in place of + ``expr + expr + expr``. Expressions may also be multiplied by a 2-integer + tuple, similar to ``{min, max}`` multipliers in regular expressions. Tuples + may also include ``None`` as in: + + - ``expr*(n, None)`` or ``expr*(n, )`` is equivalent + to ``expr*n + ZeroOrMore(expr)`` + (read as "at least n instances of ``expr``") + - ``expr*(None, n)`` is equivalent to ``expr*(0, n)`` + (read as "0 to n instances of ``expr``") + - ``expr*(None, None)`` is equivalent to ``ZeroOrMore(expr)`` + - ``expr*(1, None)`` is equivalent to ``OneOrMore(expr)`` + + Note that ``expr*(None, n)`` does not raise an exception if + more than n exprs exist in the input stream; that is, + ``expr*(None, n)`` does not enforce a maximum number of expr + occurrences. If this behavior is desired, then write + ``expr*(None, n) + ~expr`` + """ + if other is Ellipsis: + other = (0, None) + elif isinstance(other, tuple) and other[:1] == (Ellipsis,): + other = (0, *other[1:2], None)[:2] + + if not isinstance(other, (int, tuple)): + return NotImplemented + + if isinstance(other, int): + minElements, optElements = other, 0 + else: + other = tuple(o if o is not Ellipsis else None for o in other) + other = (*other, None, None)[:2] + if other[0] is None: + other = (0, other[1]) + if isinstance(other[0], int) and other[1] is None: + if other[0] == 0: + return ZeroOrMore(self) + if other[0] == 1: + return OneOrMore(self) + else: + return self * other[0] + ZeroOrMore(self) + elif isinstance(other[0], int) and isinstance(other[1], int): + minElements, optElements = other + optElements -= minElements + else: + return NotImplemented + + if minElements < 0: + raise ValueError("cannot multiply ParserElement by negative value") + if optElements < 0: + raise ValueError( + "second tuple value must be greater or equal to first tuple value" + ) + if minElements == optElements == 0: + return And([]) + + if optElements: + # Build the optional tail as a bounded ``ZeroOrMore`` instead of a + # deeply nested ``Opt(self + Opt(self + ...))`` chain. The nested + # form recursed ``optElements`` levels deep, which raised + # RecursionError for large upper bounds (e.g. ``expr[..., 1000]``) + # -- see issue #332. ``ZeroOrMore(..., max=optElements)`` is a flat + # loop that still *exits early* (it stops at the first non-match, + # just like the recursive form), so it preserves the original + # early-exit behavior while no longer scaling the call stack with + # the upper bound. + optionalTail = ZeroOrMore(self, max=optElements) + + if minElements: + if minElements == 1: + ret = self + optionalTail + else: + ret = And([self] * minElements) + optionalTail + else: + ret = optionalTail + else: + if minElements == 1: + ret = self + else: + ret = And([self] * minElements) + return ret + + def __rmul__(self, other) -> ParserElement: + return self.__mul__(other) + + def __or__(self, other) -> ParserElement: + """ + Implementation of ``|`` operator - returns :class:`MatchFirst` + + .. versionchanged:: 3.1.0 + Support ``expr | ""`` as a synonym for ``Optional(expr)``. + """ + if other is Ellipsis: + return _PendingSkip(self, must_skip=True) + + if isinstance(other, str_type): + # `expr | ""` is equivalent to `Opt(expr)` + if other == "": + return Opt(self) + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented + return MatchFirst([self, other]) + + def __ror__(self, other) -> ParserElement: + """ + Implementation of ``|`` operator when left operand is not a :class:`ParserElement` + """ + if isinstance(other, str_type): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented + return other | self + + def __xor__(self, other) -> ParserElement: + """ + Implementation of ``^`` operator - returns :class:`Or` + """ + if isinstance(other, str_type): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented + return Or([self, other]) + + def __rxor__(self, other) -> ParserElement: + """ + Implementation of ``^`` operator when left operand is not a :class:`ParserElement` + """ + if isinstance(other, str_type): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented + return other ^ self + + def __and__(self, other) -> ParserElement: + """ + Implementation of ``&`` operator - returns :class:`Each` + """ + if isinstance(other, str_type): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented + return Each([self, other]) + + def __rand__(self, other) -> ParserElement: + """ + Implementation of ``&`` operator when left operand is not a :class:`ParserElement` + """ + if isinstance(other, str_type): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented + return other & self + + def __invert__(self) -> ParserElement: + """ + Implementation of ``~`` operator - returns :class:`NotAny` + """ + return NotAny(self) + + # disable __iter__ to override legacy use of sequential access to __getitem__ to + # iterate over a sequence + __iter__ = None + + def __getitem__(self, key): + """ + use ``[]`` indexing notation as a short form for expression repetition: + + - ``expr[n]`` is equivalent to ``expr*n`` + - ``expr[m, n]`` is equivalent to ``expr*(m, n)`` + - ``expr[n, ...]`` or ``expr[n,]`` is equivalent + to ``expr*n + ZeroOrMore(expr)`` + (read as "at least n instances of ``expr``") + - ``expr[..., n]`` is equivalent to ``expr*(0, n)`` + (read as "0 to n instances of ``expr``") + - ``expr[...]`` and ``expr[0, ...]`` are equivalent to ``ZeroOrMore(expr)`` + - ``expr[1, ...]`` is equivalent to ``OneOrMore(expr)`` + + ``None`` may be used in place of ``...``. + + Note that ``expr[..., n]`` and ``expr[m, n]`` do not raise an exception + if more than ``n`` ``expr``\\ s exist in the input stream. If this behavior is + desired, then write ``expr[..., n] + ~expr``. + + For repetition with a stop_on expression, use slice notation: + + - ``expr[...: end_expr]`` and ``expr[0, ...: end_expr]`` are equivalent to ``ZeroOrMore(expr, stop_on=end_expr)`` + - ``expr[1, ...: end_expr]`` is equivalent to ``OneOrMore(expr, stop_on=end_expr)`` + + .. versionchanged:: 3.1.0 + Support for slice notation. + """ + + stop_on_defined = False + stop_on = NoMatch() + if isinstance(key, slice): + key, stop_on = key.start, key.stop + if key is None: + key = ... + stop_on_defined = True + elif isinstance(key, tuple) and isinstance(key[-1], slice): + key, stop_on = (key[0], key[1].start), key[1].stop + stop_on_defined = True + + # convert single arg keys to tuples + if isinstance(key, str_type): + key = (key,) + try: + iter(key) + except TypeError: + key = (key, key) + + if len(key) > 2: + raise TypeError( + f"only 1 or 2 index arguments supported ({key[:5]}{f'... [{len(key)}]' if len(key) > 5 else ''})" + ) + + # clip to 2 elements + ret = self * tuple(key[:2]) + ret = typing.cast(_MultipleMatch, ret) + + if stop_on_defined: + ret.stopOn(stop_on) + + return ret + + def __call__(self, name: typing.Optional[str] = None) -> ParserElement: + """ + Shortcut for :class:`set_results_name`, with ``list_all_matches=False``. + + If ``name`` is given with a trailing ``'*'`` character, then ``list_all_matches`` will be + passed as ``True``. + + If ``name`` is omitted, same as calling :class:`copy`. + + Example: + + .. testcode:: + + # these are equivalent + userdata = ( + Word(alphas).set_results_name("name") + + Word(nums + "-").set_results_name("socsecno") + ) + + userdata = Word(alphas)("name") + Word(nums + "-")("socsecno") + """ + if name is not None: + return self._setResultsName(name) + + return self.copy() + + def suppress(self) -> ParserElement: + """ + Suppresses the output of this :class:`ParserElement`; useful to keep punctuation from + cluttering up returned output. + """ + return Suppress(self) + + def ignore_whitespace(self, recursive: bool = True) -> ParserElement: + """ + Enables the skipping of whitespace before matching the characters in the + :class:`ParserElement`'s defined pattern. + + :param recursive: If ``True`` (the default), also enable whitespace skipping in child elements (if any) + """ + self.skipWhitespace = True + return self + + def leave_whitespace(self, recursive: bool = True) -> ParserElement: + """ + Disables the skipping of whitespace before matching the characters in the + :class:`ParserElement`'s defined pattern. This is normally only used internally by + the pyparsing module, but may be needed in some whitespace-sensitive grammars. + + :param recursive: If true (the default), also disable whitespace skipping in child elements (if any) + """ + self.skipWhitespace = False + return self + + def set_whitespace_chars( + self, chars: Union[set[str], str], copy_defaults: bool = False + ) -> ParserElement: + """ + Overrides the default whitespace chars + """ + self.skipWhitespace = True + self.whiteChars = set(chars) + self.copyDefaultWhiteChars = copy_defaults + return self + + def parse_with_tabs(self) -> ParserElement: + """ + Overrides default behavior to expand ```` s to spaces before parsing the input string. + Must be called before ``parse_string`` when the input grammar contains elements that + match ```` characters. + """ + self.keepTabs = True + return self + + def ignore(self, other: ParserElement) -> ParserElement: + """ + Define expression to be ignored (e.g., comments) while doing pattern + matching; may be called repeatedly, to define multiple comment or other + ignorable patterns. + + Example: + + .. doctest:: + + >>> patt = Word(alphas)[...] + >>> print(patt.parse_string('ablaj /* comment */ lskjd')) + ['ablaj'] + + >>> patt = Word(alphas)[...].ignore(c_style_comment) + >>> print(patt.parse_string('ablaj /* comment */ lskjd')) + ['ablaj', 'lskjd'] + """ + if isinstance(other, str_type): + other = Suppress(other) + + if isinstance(other, Suppress): + if other not in self.ignoreExprs: + self.ignoreExprs.append(other) + else: + self.ignoreExprs.append(Suppress(other.copy())) + return self + + def set_debug_actions( + self, + start_action: DebugStartAction, + success_action: DebugSuccessAction, + exception_action: DebugExceptionAction, + ) -> ParserElement: + """ + Customize display of debugging messages while doing pattern matching: + + :param start_action: method to be called when an expression is about to be parsed; + should have the signature:: + + fn(input_string: str, + location: int, + expression: ParserElement, + cache_hit: bool) + + :param success_action: method to be called when an expression has successfully parsed; + should have the signature:: + + fn(input_string: str, + start_location: int, + end_location: int, + expression: ParserELement, + parsed_tokens: ParseResults, + cache_hit: bool) + + :param exception_action: method to be called when expression fails to parse; + should have the signature:: + + fn(input_string: str, + location: int, + expression: ParserElement, + exception: Exception, + cache_hit: bool) + """ + self.debugActions = self.DebugActions( + start_action or _default_start_debug_action, # type: ignore[truthy-function] + success_action or _default_success_debug_action, # type: ignore[truthy-function] + exception_action or _default_exception_debug_action, # type: ignore[truthy-function] + ) + self.debug = any(self.debugActions) + return self + + def set_debug(self, flag: bool = True, recurse: bool = False) -> ParserElement: + """ + Enable display of debugging messages while doing pattern matching. + Set ``flag`` to ``True`` to enable, ``False`` to disable. + Set ``recurse`` to ``True`` to set the debug flag on this expression and all sub-expressions. + + Example: + + .. testcode:: + + wd = Word(alphas).set_name("alphaword") + integer = Word(nums).set_name("numword") + term = wd | integer + + # turn on debugging for wd + wd.set_debug() + + term[1, ...].parse_string("abc 123 xyz 890") + + prints: + + .. testoutput:: + :options: +NORMALIZE_WHITESPACE + + Match alphaword at loc 0(1,1) + abc 123 xyz 890 + ^ + Matched alphaword -> ['abc'] + Match alphaword at loc 4(1,5) + abc 123 xyz 890 + ^ + Match alphaword failed, ParseException raised: Expected alphaword, ... + Match alphaword at loc 8(1,9) + abc 123 xyz 890 + ^ + Matched alphaword -> ['xyz'] + Match alphaword at loc 12(1,13) + abc 123 xyz 890 + ^ + Match alphaword failed, ParseException raised: Expected alphaword, ... + abc 123 xyz 890 + ^ + Match alphaword failed, ParseException raised: Expected alphaword, found end of text ... + + The output shown is that produced by the default debug actions - custom debug actions can be + specified using :meth:`set_debug_actions`. Prior to attempting + to match the ``wd`` expression, the debugging message ``"Match at loc (,)"`` + is shown. Then if the parse succeeds, a ``"Matched"`` message is shown, or an ``"Exception raised"`` + message is shown. Also note the use of :meth:`set_name` to assign a human-readable name to the expression, + which makes debugging and exception messages easier to understand - for instance, the default + name created for the :class:`Word` expression without calling :meth:`set_name` is ``"W:(A-Za-z)"``. + + .. versionchanged:: 3.1.0 + ``recurse`` argument added. + """ + if recurse: + for expr in self.visit_all(): + expr.set_debug(flag, recurse=False) + return self + + if flag: + self.set_debug_actions( + _default_start_debug_action, + _default_success_debug_action, + _default_exception_debug_action, + ) + else: + self.debug = False + return self + + @property + def default_name(self) -> str: + if self._defaultName is None: + self._defaultName = self._generateDefaultName() + return self._defaultName + + @abstractmethod + def _generateDefaultName(self) -> str: + """ + Child classes must define this method, which defines how the ``default_name`` is set. + """ + + def set_name(self, name: typing.Optional[str]) -> ParserElement: + """ + Define name for this expression, makes debugging and exception messages clearer. If + `__diag__.enable_debug_on_named_expressions` is set to True, setting a name will also + enable debug for this expression. + + If `name` is None, clears any custom name for this expression, and clears the + debug flag is it was enabled via `__diag__.enable_debug_on_named_expressions`. + + Example: + + .. doctest:: + + >>> integer = Word(nums) + >>> integer.parse_string("ABC") + Traceback (most recent call last): + ParseException: Expected W:(0-9) (at char 0), (line:1, col:1) + + >>> integer.set_name("integer") + integer + >>> integer.parse_string("ABC") + Traceback (most recent call last): + ParseException: Expected integer (at char 0), (line:1, col:1) + + .. versionchanged:: 3.1.0 + Accept ``None`` as the ``name`` argument. + """ + self.customName = name # type: ignore[assignment] + self.errmsg = f"Expected {str(self)}" + + if __diag__.enable_debug_on_named_expressions: + self.set_debug(name is not None) + + return self + + @property + def name(self) -> str: + """ + Returns a user-defined name if available, but otherwise defaults back to the auto-generated name + """ + return self.customName if self.customName is not None else self.default_name + + @name.setter + def name(self, new_name) -> None: + self.set_name(new_name) + + def __str__(self) -> str: + return self.name + + def __repr__(self) -> str: + return str(self) + + def streamline(self) -> ParserElement: + self.streamlined = True + self._defaultName = None + return self + + def recurse(self) -> list[ParserElement]: + return [] + + def _checkRecursion(self, parseElementList): + subRecCheckList = parseElementList[:] + [self] + for e in self.recurse(): + e._checkRecursion(subRecCheckList) + + def validate(self, validateTrace=None) -> None: + """ + .. deprecated:: 3.0.0 + Do not use to check for left recursion. + + Check defined expressions for valid structure, check for infinite recursive definitions. + + """ + warnings.warn( + "ParserElement.validate() is deprecated, and should not be used to check for left recursion", + PyparsingDeprecationWarning, + stacklevel=2, + ) + self._checkRecursion([]) + + def parse_file( + self, + file_or_filename: Union[str, Path, TextIO], + encoding: str = "utf-8", + parse_all: bool = False, + **kwargs, + ) -> ParseResults: + """ + Execute the parse expression on the given file or filename. + If a filename is specified (instead of a file object), + the entire file is opened, read, and closed before parsing. + """ + parseAll: bool = deprecate_argument(kwargs, "parseAll", False) + + parse_all = parse_all or parseAll + try: + file_or_filename = typing.cast(TextIO, file_or_filename) + file_contents = file_or_filename.read() + except AttributeError: + file_or_filename = typing.cast(str, file_or_filename) + with open(file_or_filename, "r", encoding=encoding) as f: + file_contents = f.read() + try: + return self.parse_string(file_contents, parse_all) + except ParseBaseException as exc: + if ParserElement.verbose_stacktrace: + raise + + # catch and re-raise exception from here, clears out pyparsing internal stack trace + raise exc.with_traceback(None) + + def __eq__(self, other): + if self is other: + return True + elif isinstance(other, str_type): + return self.matches(other, parse_all=True) + elif isinstance(other, ParserElement): + return vars(self) == vars(other) + return False + + def __hash__(self): + return id(self) + + def matches(self, test_string: str, parse_all: bool = True, **kwargs) -> bool: + """ + Method for quick testing of a parser against a test string. Good for simple + inline microtests of sub expressions while building up larger parser. + + :param test_string: to test against this expression for a match + :param parse_all: flag to pass to :meth:`parse_string` when running tests + + Example: + + .. doctest:: + + >>> expr = Word(nums) + >>> expr.matches("100") + True + """ + parseAll: bool = deprecate_argument(kwargs, "parseAll", True) + + parse_all = parse_all and parseAll + try: + self.parse_string(str(test_string), parse_all=parse_all) + return True + except ParseBaseException: + return False + + def run_tests( + self, + tests: Union[str, list[str]], + parse_all: bool = True, + comment: typing.Optional[Union[ParserElement, str]] = "#", + full_dump: bool = True, + print_results: bool = True, + failure_tests: bool = False, + post_parse: typing.Optional[ + Callable[[str, ParseResults], typing.Optional[str]] + ] = None, + file: typing.Optional[TextIO] = None, + with_line_numbers: bool = False, + *, + parseAll: bool = True, + fullDump: bool = True, + printResults: bool = True, + failureTests: bool = False, + postParse: typing.Optional[ + Callable[[str, ParseResults], typing.Optional[str]] + ] = None, + ) -> tuple[bool, list[tuple[str, Union[ParseResults, Exception]]]]: + """ + Execute the parse expression on a series of test strings, showing each + test, the parsed results or where the parse failed. Quick and easy way to + run a parse expression against a list of sample strings. + + Parameters: + + - ``tests`` - a list of separate test strings, or a multiline string of test strings + - ``parse_all`` - (default= ``True``) - flag to pass to :class:`parse_string` when running tests + - ``comment`` - (default= ``'#'``) - expression for indicating embedded comments in the test + string; pass None to disable comment filtering + - ``full_dump`` - (default= ``True``) - dump results as list followed by results names in nested outline; + if False, only dump nested list + - ``print_results`` - (default= ``True``) prints test output to stdout + - ``failure_tests`` - (default= ``False``) indicates if these tests are expected to fail parsing + - ``post_parse`` - (default= ``None``) optional callback for successful parse results; called as + `fn(test_string, parse_results)` and returns a string to be added to the test output + - ``file`` - (default= ``None``) optional file-like object to which test output will be written; + if None, will default to ``sys.stdout`` + - ``with_line_numbers`` - default= ``False``) show test strings with line and column numbers + + Returns: a (success, results) tuple, where success indicates that all tests succeeded + (or failed if ``failure_tests`` is True), and the results contain a list of lines of each + test's output + + Passing example: + + .. testcode:: + + number_expr = pyparsing_common.number.copy() + + result = number_expr.run_tests(''' + # unsigned integer + 100 + # negative integer + -100 + # float with scientific notation + 6.02e23 + # integer with scientific notation + 1e-12 + # negative decimal number without leading digit + -.100 + ''') + print("Success" if result[0] else "Failed!") + + prints: + + .. testoutput:: + :options: +NORMALIZE_WHITESPACE + + + # unsigned integer + 100 + [100] + + # negative integer + -100 + [-100] + + # float with scientific notation + 6.02e23 + [6.02e+23] + + # integer with scientific notation + 1e-12 + [1e-12] + + # negative decimal number without leading digit + -.100 + [-0.1] + Success + + Failure-test example: + + .. testcode:: + + result = number_expr.run_tests(''' + # stray character + 100Z + # too many '.' + 3.14.159 + ''', failure_tests=True) + print("Success" if result[0] else "Failed!") + + prints: + + .. testoutput:: + :options: +NORMALIZE_WHITESPACE + + + # stray character + 100Z + 100Z + ^ + ParseException: Expected end of text, found 'Z' ... + + # too many '.' + 3.14.159 + 3.14.159 + ^ + ParseException: Expected end of text, found '.' ... + FAIL: Expected end of text, found '.' ... + Success + + Each test string must be on a single line. If you want to test a string that spans multiple + lines, create a test like this: + + .. testcode:: + + expr = Word(alphanums)[1,...] + expr.run_tests(r"this is a test\\n of strings that spans \\n 3 lines") + + .. testoutput:: + :options: +NORMALIZE_WHITESPACE + :hide: + + + this is a test\\n of strings that spans \\n 3 lines + ['this', 'is', 'a', 'test', 'of', 'strings', 'that', 'spans', '3', 'lines'] + + (Note that this is a raw string literal, you must include the leading ``'r'``.) + """ + from .testing import pyparsing_test + + parseAll = parseAll and parse_all + fullDump = fullDump and full_dump + printResults = printResults and print_results + failureTests = failureTests or failure_tests + postParse = postParse or post_parse + if isinstance(tests, str_type): + tests = typing.cast(str, tests) + line_strip = type(tests).strip + tests = [line_strip(test_line) for test_line in tests.rstrip().splitlines()] + comment_specified = comment is not None + if comment_specified: + if isinstance(comment, str_type): + comment = typing.cast(str, comment) + comment = Literal(comment) + comment = typing.cast(ParserElement, comment) + if file is None: + file = sys.stdout + print_ = file.write + + result: Union[ParseResults, Exception] + allResults: list[tuple[str, Union[ParseResults, Exception]]] = [] + comments: list[str] = [] + success = True + NL = Literal(r"\n").add_parse_action(replace_with("\n")).ignore(quoted_string) + BOM = "\ufeff" + nlstr = "\n" + for t in tests: + if comment_specified and comment.matches(t, False) or comments and not t: + comments.append( + pyparsing_test.with_line_numbers(t) if with_line_numbers else t + ) + continue + if not t: + continue + out = [ + f"{nlstr}{nlstr.join(comments) if comments else ''}", + pyparsing_test.with_line_numbers(t) if with_line_numbers else t, + ] + comments.clear() + try: + # convert newline marks to actual newlines, and strip leading BOM if present + t = NL.transform_string(t.lstrip(BOM)) + result = self.parse_string(t, parse_all=parse_all) + except ParseBaseException as pe: + fatal = "(FATAL) " if isinstance(pe, ParseFatalException) else "" + out.append(pe.explain()) + out.append(f"FAIL: {fatal}{pe}") + if ParserElement.verbose_stacktrace: + out.extend(traceback.format_tb(pe.__traceback__)) + success = success and failureTests + result = pe + except Exception as exc: + tag = "FAIL-EXCEPTION" + + # see if this exception was raised in a parse action + tb = exc.__traceback__ + it = iter(traceback.walk_tb(tb)) + for f, line in it: + if (f.f_code.co_filename, line) == pa_call_line_synth: + next_f = next(it)[0] + tag += f" (raised in parse action {next_f.f_code.co_name!r})" + break + + out.append(f"{tag}: {type(exc).__name__}: {exc}") + if ParserElement.verbose_stacktrace: + out.extend(traceback.format_tb(exc.__traceback__)) + success = success and failureTests + result = exc + else: + success = success and not failureTests + if postParse is not None: + try: + pp_value = postParse(t, result) + if pp_value is not None: + if isinstance(pp_value, ParseResults): + out.append(pp_value.dump()) + else: + out.append(str(pp_value)) + else: + out.append(result.dump()) + except Exception as e: + out.append(result.dump(full=fullDump)) + out.append( + f"{postParse.__name__} failed: {type(e).__name__}: {e}" + ) + else: + out.append(result.dump(full=fullDump)) + out.append("") + + if printResults: + print_("\n".join(out)) + + allResults.append((t, result)) + + return success, allResults + + def create_diagram( + self, + output_html: Union[TextIO, Path, str], + vertical: int = 3, + show_results_names: bool = False, + show_groups: bool = False, + embed: bool = False, + show_hidden: bool = False, + **kwargs, + ) -> None: + """ + Create a railroad diagram for the parser. + + Parameters: + + - ``output_html`` (str or file-like object) - output target for generated + diagram HTML + - ``vertical`` (int) - threshold for formatting multiple alternatives vertically + instead of horizontally (default=3) + - ``show_results_names`` - bool flag whether diagram should show annotations for + defined results names + - ``show_groups`` - bool flag whether groups should be highlighted with an unlabeled surrounding box + - ``show_hidden`` - bool flag to show diagram elements for internal elements that are usually hidden + - ``embed`` - bool flag whether generated HTML should omit , , and tags to embed + the resulting HTML in an enclosing HTML source + - ``head`` - str containing additional HTML to insert into the section of the generated code; + can be used to insert custom CSS styling + - ``body`` - str containing additional HTML to insert at the beginning of the section of the + generated code + + Additional diagram-formatting keyword arguments can also be included; + see railroad.Diagram class. + + .. versionchanged:: 3.1.0 + ``embed`` argument added. + """ + + try: + from .diagram import to_railroad, railroad_to_html + except ImportError as ie: + raise Exception( + "must ``pip install pyparsing[diagrams]`` to generate parser railroad diagrams" + ) from ie + + self.streamline() + + railroad = to_railroad( + self, + vertical=vertical, + show_results_names=show_results_names, + show_groups=show_groups, + show_hidden=show_hidden, + diagram_kwargs=kwargs, + ) + if not isinstance(output_html, (str, Path)): + # we were passed a file-like object, just write to it + output_html.write(railroad_to_html(railroad, embed=embed, **kwargs)) + return + + with open(output_html, "w", encoding="utf-8") as diag_file: + diag_file.write(railroad_to_html(railroad, embed=embed, **kwargs)) + + # Compatibility synonyms + # fmt: off + inlineLiteralsUsing = staticmethod(replaced_by_pep8("inlineLiteralsUsing", inline_literals_using)) + setDefaultWhitespaceChars = staticmethod(replaced_by_pep8( + "setDefaultWhitespaceChars", set_default_whitespace_chars + )) + disableMemoization = staticmethod(replaced_by_pep8("disableMemoization", disable_memoization)) + enableLeftRecursion = staticmethod(replaced_by_pep8("enableLeftRecursion", enable_left_recursion)) + enablePackrat = staticmethod(replaced_by_pep8("enablePackrat", enable_packrat)) + resetCache = staticmethod(replaced_by_pep8("resetCache", reset_cache)) + + setResultsName = replaced_by_pep8("setResultsName", set_results_name) + setBreak = replaced_by_pep8("setBreak", set_break) + setParseAction = replaced_by_pep8("setParseAction", set_parse_action) + addParseAction = replaced_by_pep8("addParseAction", add_parse_action) + addCondition = replaced_by_pep8("addCondition", add_condition) + setFailAction = replaced_by_pep8("setFailAction", set_fail_action) + tryParse = replaced_by_pep8("tryParse", try_parse) + parseString = replaced_by_pep8("parseString", parse_string) + scanString = replaced_by_pep8("scanString", scan_string) + transformString = replaced_by_pep8("transformString", transform_string) + searchString = replaced_by_pep8("searchString", search_string) + ignoreWhitespace = replaced_by_pep8("ignoreWhitespace", ignore_whitespace) + leaveWhitespace = replaced_by_pep8("leaveWhitespace", leave_whitespace) + setWhitespaceChars = replaced_by_pep8("setWhitespaceChars", set_whitespace_chars) + parseWithTabs = replaced_by_pep8("parseWithTabs", parse_with_tabs) + setDebugActions = replaced_by_pep8("setDebugActions", set_debug_actions) + setDebug = replaced_by_pep8("setDebug", set_debug) + setName = replaced_by_pep8("setName", set_name) + parseFile = replaced_by_pep8("parseFile", parse_file) + runTests = replaced_by_pep8("runTests", run_tests) + canParseNext = replaced_by_pep8("canParseNext", can_parse_next) + defaultName = default_name + # fmt: on + + +class _PendingSkip(ParserElement): + # internal placeholder class to hold a place were '...' is added to a parser element, + # once another ParserElement is added, this placeholder will be replaced with a SkipTo + def __init__(self, expr: ParserElement, must_skip: bool = False) -> None: + super().__init__() + self.anchor = expr + self.must_skip = must_skip + + def _generateDefaultName(self) -> str: + return str(self.anchor + Empty()).replace("Empty", "...") + + def __add__(self, other) -> ParserElement: + skipper = SkipTo(other).set_name("...")("_skipped*") + if self.must_skip: + + def must_skip(t): + if not t._skipped or t._skipped.as_list() == [""]: + del t[0] + t.pop("_skipped", None) + + def show_skip(t): + if t._skipped.as_list()[-1:] == [""]: + t.pop("_skipped") + t["_skipped"] = f"missing <{self.anchor!r}>" + + return ( + self.anchor + skipper().add_parse_action(must_skip) + | skipper().add_parse_action(show_skip) + ) + other + + return self.anchor + skipper + other + + def __repr__(self): + return self.defaultName + + def parseImpl(self, *args) -> ParseImplReturnType: + raise Exception( + "use of `...` expression without following SkipTo target expression" + ) + + +class Token(ParserElement): + """Abstract :class:`ParserElement` subclass, for defining atomic + matching patterns. + """ + + def __init__(self) -> None: + super().__init__(savelist=False) + + def _generateDefaultName(self) -> str: + return type(self).__name__ + + +class NoMatch(Token): + """ + A token that will never match. + """ + + def __init__(self) -> None: + super().__init__() + self._may_return_empty = True + self.mayIndexError = False + self.errmsg = "Unmatchable token" + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + raise ParseException(instring, loc, self.errmsg, self) + + +class Literal(Token): + """ + Token to exactly match a specified string. + + Example: + + .. doctest:: + + >>> Literal('abc').parse_string('abc') + ParseResults(['abc'], {}) + >>> Literal('abc').parse_string('abcdef') + ParseResults(['abc'], {}) + >>> Literal('abc').parse_string('ab') + Traceback (most recent call last): + ParseException: Expected 'abc', found 'ab' (at char 0), (line: 1, col: 1) + + For case-insensitive matching, use :class:`CaselessLiteral`. + + For keyword matching (force word break before and after the matched string), + use :class:`Keyword` or :class:`CaselessKeyword`. + """ + + def __new__(cls, match_string: str = "", **kwargs): + # Performance tuning: select a subclass with optimized parseImpl + if cls is Literal: + matchString: str = deprecate_argument(kwargs, "matchString", "") + + match_string = matchString or match_string + if not match_string: + return super().__new__(Empty) + if len(match_string) == 1: + return super().__new__(_SingleCharLiteral) + + # Default behavior + return super().__new__(cls) + + # Needed to make copy.copy() work correctly if we customize __new__ + def __getnewargs__(self): + return (self.match,) + + def __init__(self, match_string: str = "", **kwargs) -> None: + matchString: str = deprecate_argument(kwargs, "matchString", "") + + super().__init__() + match_string = matchString or match_string + self.match = match_string + self.matchLen = len(match_string) + self.firstMatchChar = match_string[:1] + self.errmsg = f"Expected {self.name}" + self._may_return_empty = False + self.mayIndexError = False + + def _generateDefaultName(self) -> str: + return repr(self.match) + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + if instring[loc] == self.firstMatchChar and instring.startswith( + self.match, loc + ): + return loc + self.matchLen, self.match + raise ParseException(instring, loc, self.errmsg, self) + + +class Empty(Literal): + """ + An empty token, will always match. + """ + + def __init__(self, match_string="", *, matchString="") -> None: + super().__init__("") + self._may_return_empty = True + self.mayIndexError = False + + def _generateDefaultName(self) -> str: + return "Empty" + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + return loc, [] + + +class _SingleCharLiteral(Literal): + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + if instring[loc] == self.firstMatchChar: + return loc + 1, self.match + raise ParseException(instring, loc, self.errmsg, self) + + +ParserElement._literalStringClass = Literal + + +class Keyword(Token): + """ + Token to exactly match a specified string as a keyword, that is, + it must be immediately preceded and followed by whitespace or + non-keyword characters. Compare with :class:`Literal`: + + - ``Literal("if")`` will match the leading ``'if'`` in + ``'ifAndOnlyIf'``. + - ``Keyword("if")`` will not; it will only match the leading + ``'if'`` in ``'if x=1'``, or ``'if(y==2)'`` + + Accepts two optional constructor arguments in addition to the + keyword string: + + - ``ident_chars`` is a string of characters that would be valid + identifier characters, defaulting to all alphanumerics + "_" and + "$" + - ``caseless`` allows case-insensitive matching, default is ``False``. + + Example: + + .. doctest:: + :options: +NORMALIZE_WHITESPACE + + >>> Keyword("start").parse_string("start") + ParseResults(['start'], {}) + >>> Keyword("start").parse_string("starting") + Traceback (most recent call last): + ParseException: Expected Keyword 'start', keyword was immediately + followed by keyword character, found 'ing' (at char 5), (line:1, col:6) + + .. doctest:: + :options: +NORMALIZE_WHITESPACE + + >>> Keyword("start").parse_string("starting").debug() + Traceback (most recent call last): + ParseException: Expected Keyword "start", keyword was immediately + followed by keyword character, found 'ing' ... + + For case-insensitive matching, use :class:`CaselessKeyword`. + """ + + DEFAULT_KEYWORD_CHARS = f"{alphanums}_$" + + def __init__( + self, + match_string: str = "", + ident_chars: typing.Optional[str] = None, + caseless: bool = False, + **kwargs, + ) -> None: + matchString = deprecate_argument(kwargs, "matchString", "") + identChars = deprecate_argument(kwargs, "identChars", None) + + super().__init__() + identChars = identChars or ident_chars + if identChars is None: + identChars = Keyword.DEFAULT_KEYWORD_CHARS + match_string = matchString or match_string + self.match = match_string + self.matchLen = len(match_string) + self.firstMatchChar = match_string[:1] + if not self.firstMatchChar: + raise ValueError("null string passed to Keyword; use Empty() instead") + self.errmsg = f"Expected {type(self).__name__} {self.name}" + self._may_return_empty = False + self.mayIndexError = False + self.caseless = caseless + if caseless: + self.caselessmatch = match_string.upper() + identChars = identChars.upper() + self.ident_chars = set(identChars) + + @property + def identChars(self) -> set[str]: + """ + .. deprecated:: 3.3.0 + use ident_chars instead. + + Property returning the characters being used as keyword characters for this expression. + """ + return self.ident_chars + + def _generateDefaultName(self) -> str: + return repr(self.match) + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + errmsg = self.errmsg or "" + errloc = loc + if self.caseless: + if instring[loc : loc + self.matchLen].upper() == self.caselessmatch: + if loc == 0 or instring[loc - 1].upper() not in self.identChars: + if ( + loc >= len(instring) - self.matchLen + or instring[loc + self.matchLen].upper() not in self.identChars + ): + return loc + self.matchLen, self.match + + # followed by keyword char + errmsg += ", was immediately followed by keyword character" + errloc = loc + self.matchLen + else: + # preceded by keyword char + errmsg += ", keyword was immediately preceded by keyword character" + errloc = loc - 1 + # else no match just raise plain exception + + elif ( + instring[loc] == self.firstMatchChar + and self.matchLen == 1 + or instring.startswith(self.match, loc) + ): + if loc == 0 or instring[loc - 1] not in self.identChars: + if ( + loc >= len(instring) - self.matchLen + or instring[loc + self.matchLen] not in self.identChars + ): + return loc + self.matchLen, self.match + + # followed by keyword char + errmsg += ", keyword was immediately followed by keyword character" + errloc = loc + self.matchLen + else: + # preceded by keyword char + errmsg += ", keyword was immediately preceded by keyword character" + errloc = loc - 1 + # else no match just raise plain exception + + raise ParseException(instring, errloc, errmsg, self) + + @staticmethod + def set_default_keyword_chars(chars) -> None: + """ + Overrides the default characters used by :class:`Keyword` expressions. + """ + Keyword.DEFAULT_KEYWORD_CHARS = chars + + # Compatibility synonyms + setDefaultKeywordChars = staticmethod( + replaced_by_pep8("setDefaultKeywordChars", set_default_keyword_chars) + ) + + +class CaselessLiteral(Literal): + """ + Token to match a specified string, ignoring case of letters. + Note: the matched results will always be in the case of the given + match string, NOT the case of the input text. + + Example: + + .. doctest:: + + >>> CaselessLiteral("CMD")[1, ...].parse_string("cmd CMD Cmd10") + ParseResults(['CMD', 'CMD', 'CMD'], {}) + + (Contrast with example for :class:`CaselessKeyword`.) + """ + + def __init__(self, match_string: str = "", **kwargs) -> None: + matchString: str = deprecate_argument(kwargs, "matchString", "") + + match_string = matchString or match_string + super().__init__(match_string.upper()) + # Preserve the defining literal. + self.returnString = match_string + self.errmsg = f"Expected {self.name}" + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + if instring[loc : loc + self.matchLen].upper() == self.match: + return loc + self.matchLen, self.returnString + raise ParseException(instring, loc, self.errmsg, self) + + +class CaselessKeyword(Keyword): + """ + Caseless version of :class:`Keyword`. + + Example: + + .. doctest:: + + >>> CaselessKeyword("CMD")[1, ...].parse_string("cmd CMD Cmd10") + ParseResults(['CMD', 'CMD'], {}) + + (Contrast with example for :class:`CaselessLiteral`.) + """ + + def __init__( + self, match_string: str = "", ident_chars: typing.Optional[str] = None, **kwargs + ) -> None: + matchString: str = deprecate_argument(kwargs, "matchString", "") + identChars: typing.Optional[str] = deprecate_argument( + kwargs, "identChars", None + ) + + identChars = identChars or ident_chars + match_string = matchString or match_string + super().__init__(match_string, identChars, caseless=True) + + +class CloseMatch(Token): + """A variation on :class:`Literal` which matches "close" matches, + that is, strings with at most 'n' mismatching characters. + :class:`CloseMatch` takes parameters: + + - ``match_string`` - string to be matched + - ``caseless`` - a boolean indicating whether to ignore casing when comparing characters + - ``max_mismatches`` - (``default=1``) maximum number of + mismatches allowed to count as a match + + The results from a successful parse will contain the matched text + from the input string and the following named results: + + - ``mismatches`` - a list of the positions within the + match_string where mismatches were found + - ``original`` - the original match_string used to compare + against the input string + + If ``mismatches`` is an empty list, then the match was an exact + match. + + Example: + + .. doctest:: + :options: +NORMALIZE_WHITESPACE + + >>> patt = CloseMatch("ATCATCGAATGGA") + >>> patt.parse_string("ATCATCGAAXGGA") + ParseResults(['ATCATCGAAXGGA'], + {'original': 'ATCATCGAATGGA', 'mismatches': [9]}) + + >>> patt.parse_string("ATCAXCGAAXGGA") + Traceback (most recent call last): + ParseException: Expected 'ATCATCGAATGGA' (with up to 1 mismatches), + found 'ATCAXCGAAXGGA' (at char 0), (line:1, col:1) + + # exact match + >>> patt.parse_string("ATCATCGAATGGA") + ParseResults(['ATCATCGAATGGA'], + {'original': 'ATCATCGAATGGA', 'mismatches': []}) + + # close match allowing up to 2 mismatches + >>> patt = CloseMatch("ATCATCGAATGGA", max_mismatches=2) + >>> patt.parse_string("ATCAXCGAAXGGA") + ParseResults(['ATCAXCGAAXGGA'], + {'original': 'ATCATCGAATGGA', 'mismatches': [4, 9]}) + """ + + def __init__( + self, + match_string: str, + max_mismatches: typing.Optional[int] = None, + *, + caseless=False, + **kwargs, + ) -> None: + maxMismatches: int = deprecate_argument(kwargs, "maxMismatches", 1) + + maxMismatches = max_mismatches if max_mismatches is not None else maxMismatches + super().__init__() + self.match_string = match_string + self.maxMismatches = maxMismatches + self.errmsg = f"Expected {self.match_string!r} (with up to {self.maxMismatches} mismatches)" + self.caseless = caseless + self.mayIndexError = False + self._may_return_empty = False + + def _generateDefaultName(self) -> str: + return f"{type(self).__name__}:{self.match_string!r}" + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + start = loc + instrlen = len(instring) + maxloc = start + len(self.match_string) + + if maxloc <= instrlen: + match_string = self.match_string + match_stringloc = 0 + mismatches = [] + maxMismatches = self.maxMismatches + + for match_stringloc, s_m in enumerate( + zip(instring[loc:maxloc], match_string) + ): + src, mat = s_m + if self.caseless: + src, mat = src.lower(), mat.lower() + + if src != mat: + mismatches.append(match_stringloc) + if len(mismatches) > maxMismatches: + break + else: + loc = start + match_stringloc + 1 + results = ParseResults([instring[start:loc]]) + results["original"] = match_string + results["mismatches"] = mismatches + return loc, results + + raise ParseException(instring, loc, self.errmsg, self) + + +class Word(Token): + """Token for matching words composed of allowed character sets. + + Parameters: + + - ``init_chars`` - string of all characters that should be used to + match as a word; "ABC" will match "AAA", "ABAB", "CBAC", etc.; + if ``body_chars`` is also specified, then this is the string of + initial characters + - ``body_chars`` - string of characters that + can be used for matching after a matched initial character as + given in ``init_chars``; if omitted, same as the initial characters + (default=``None``) + - ``min`` - minimum number of characters to match (default=1) + - ``max`` - maximum number of characters to match (default=0) + - ``exact`` - exact number of characters to match (default=0) + - ``as_keyword`` - match as a keyword (default=``False``) + - ``exclude_chars`` - characters that might be + found in the input ``body_chars`` string but which should not be + accepted for matching ;useful to define a word of all + printables except for one or two characters, for instance + (default=``None``) + + :class:`srange` is useful for defining custom character set strings + for defining :class:`Word` expressions, using range notation from + regular expression character sets. + + A common mistake is to use :class:`Word` to match a specific literal + string, as in ``Word("Address")``. Remember that :class:`Word` + uses the string argument to define *sets* of matchable characters. + This expression would match "Add", "AAA", "dAred", or any other word + made up of the characters 'A', 'd', 'r', 'e', and 's'. To match an + exact literal string, use :class:`Literal` or :class:`Keyword`. + + pyparsing includes helper strings for building Words: + + - :attr:`alphas` + - :attr:`nums` + - :attr:`alphanums` + - :attr:`hexnums` + - :attr:`alphas8bit` (alphabetic characters in ASCII range 128-255 + - accented, tilded, umlauted, etc.) + - :attr:`punc8bit` (non-alphabetic characters in ASCII range + 128-255 - currency, symbols, superscripts, diacriticals, etc.) + - :attr:`printables` (any non-whitespace character) + + ``alphas``, ``nums``, and ``printables`` are also defined in several + Unicode sets - see :class:`pyparsing_unicode`. + + Example: + + .. testcode:: + + # a word composed of digits + integer = Word(nums) + # Two equivalent alternate forms: + Word("0123456789") + Word(srange("[0-9]")) + + # a word with a leading capital, and zero or more lowercase + capitalized_word = Word(alphas.upper(), alphas.lower()) + + # hostnames are alphanumeric, with leading alpha, and '-' + hostname = Word(alphas, alphanums + '-') + + # roman numeral + # (not a strict parser, accepts invalid mix of characters) + roman = Word("IVXLCDM") + + # any string of non-whitespace characters, except for ',' + csv_value = Word(printables, exclude_chars=",") + + :raises ValueError: If ``min`` and ``max`` are both specified + and the test ``min <= max`` fails. + + .. versionchanged:: 3.1.0 + Raises :exc:`ValueError` if ``min`` > ``max``. + """ + + def __init__( + self, + init_chars: str = "", + body_chars: typing.Optional[str] = None, + min: int = 1, + max: int = 0, + exact: int = 0, + as_keyword: bool = False, + exclude_chars: typing.Optional[str] = None, + **kwargs, + ) -> None: + initChars: typing.Optional[str] = deprecate_argument(kwargs, "initChars", None) + bodyChars: typing.Optional[str] = deprecate_argument(kwargs, "bodyChars", None) + asKeyword: bool = deprecate_argument(kwargs, "asKeyword", False) + excludeChars: typing.Optional[str] = deprecate_argument( + kwargs, "excludeChars", None + ) + + initChars = initChars or init_chars + bodyChars = bodyChars or body_chars + asKeyword = asKeyword or as_keyword + excludeChars = excludeChars or exclude_chars + super().__init__() + if not initChars: + raise ValueError( + f"invalid {type(self).__name__}, initChars cannot be empty string" + ) + + initChars_set = set(initChars) + if excludeChars: + excludeChars_set = set(excludeChars) + initChars_set -= excludeChars_set + if bodyChars: + bodyChars = "".join(set(bodyChars) - excludeChars_set) + self.init_chars = initChars_set + self.initCharsOrig = "".join(sorted(initChars_set)) + + if bodyChars: + self.bodyChars = set(bodyChars) + self.bodyCharsOrig = "".join(sorted(bodyChars)) + else: + self.bodyChars = initChars_set + self.bodyCharsOrig = self.initCharsOrig + + self.maxSpecified = max > 0 + + if min < 1: + raise ValueError( + "cannot specify a minimum length < 1; use Opt(Word()) if zero-length word is permitted" + ) + + if self.maxSpecified and min > max: + raise ValueError( + f"invalid args, if min and max both specified min must be <= max (min={min}, max={max})" + ) + + self.minLen = min + + if max > 0: + self.maxLen = max + else: + self.maxLen = _MAX_INT + + if exact > 0: + min = max = exact + self.maxLen = exact + self.minLen = exact + + self.errmsg = f"Expected {self.name}" + self.mayIndexError = False + self.asKeyword = asKeyword + if self.asKeyword: + self.errmsg += " as a keyword" + + # see if we can make a regex for this Word + if " " not in (self.initChars | self.bodyChars): + if len(self.initChars) == 1: + re_leading_fragment = re.escape(self.initCharsOrig) + else: + re_leading_fragment = f"[{_collapse_string_to_ranges(self.initChars)}]" + + if self.bodyChars == self.initChars: + if max == 0 and self.minLen == 1: + repeat = "+" + elif max == 1: + repeat = "" + else: + if self.minLen != self.maxLen: + repeat = f"{{{self.minLen},{'' if self.maxLen == _MAX_INT else self.maxLen}}}" + else: + repeat = f"{{{self.minLen}}}" + self.reString = f"{re_leading_fragment}{repeat}" + else: + if max == 1: + re_body_fragment = "" + repeat = "" + else: + re_body_fragment = f"[{_collapse_string_to_ranges(self.bodyChars)}]" + if max == 0 and self.minLen == 1: + repeat = "*" + elif max == 2: + repeat = "?" if min <= 1 else "" + else: + if min != max: + repeat = f"{{{min - 1 if min > 0 else ''},{max - 1 if max > 0 else ''}}}" + else: + repeat = f"{{{min - 1 if min > 0 else ''}}}" + + self.reString = f"{re_leading_fragment}{re_body_fragment}{repeat}" + + if self.asKeyword: + self.reString = rf"\b{self.reString}\b" + + try: + self.re = re.compile(self.reString) + except re.error: + self.re = None # type: ignore[assignment] + else: + self.re_match = self.re.match + self.parseImpl = self.parseImpl_regex # type: ignore[method-assign] + + @property + def initChars(self) -> set[str]: + """ + .. deprecated:: 3.3.0 + use `init_chars` instead. + + Property returning the initial chars to be used when matching this + Word expression. If no body chars were specified, the initial characters + will also be the body characters. + """ + return set(self.init_chars) + + def copy(self) -> Word: + """ + Returns a copy of this expression. + + Generally only used internally by pyparsing. + """ + ret: Word = cast(Word, super().copy()) + if hasattr(self, "re_match"): + ret.re_match = self.re_match + ret.parseImpl = ret.parseImpl_regex # type: ignore[method-assign] + return ret + + def _generateDefaultName(self) -> str: + def charsAsStr(s): + max_repr_len = 16 + s = _collapse_string_to_ranges(s, re_escape=False) + + if len(s) > max_repr_len: + return f"{s[:max_repr_len - 3]}..." + + return s + + if self.initChars != self.bodyChars: + base = f"W:({charsAsStr(self.initChars)}, {charsAsStr(self.bodyChars)})" + else: + base = f"W:({charsAsStr(self.initChars)})" + + # add length specification + if self.minLen > 1 or self.maxLen != _MAX_INT: + if self.minLen == self.maxLen: + if self.minLen == 1: + return base[2:] + else: + return base + f"{{{self.minLen}}}" + elif self.maxLen == _MAX_INT: + return base + f"{{{self.minLen},...}}" + else: + return base + f"{{{self.minLen},{self.maxLen}}}" + return base + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + if instring[loc] not in self.initChars: + raise ParseException(instring, loc, self.errmsg, self) + + start = loc + loc += 1 + instrlen = len(instring) + body_chars: set[str] = self.bodyChars + maxloc = start + self.maxLen + maxloc = min(maxloc, instrlen) + while loc < maxloc and instring[loc] in body_chars: + loc += 1 + + throw_exception = False + if loc - start < self.minLen: + throw_exception = True + elif self.asKeyword and ( + (start > 0 and instring[start - 1] in body_chars) + or (loc < instrlen and instring[loc] in body_chars) + ): + throw_exception = True + + if throw_exception: + raise ParseException(instring, loc, self.errmsg, self) + + return loc, instring[start:loc] + + def parseImpl_regex(self, instring, loc, do_actions=True) -> ParseImplReturnType: + result = self.re_match(instring, loc) + if not result: + raise ParseException(instring, loc, self.errmsg, self) + + loc = result.end() + return loc, result[0] + + +class Char(Word): + """A short-cut class for defining :class:`Word` ``(characters, exact=1)``, + when defining a match of any single character in a string of + characters. + """ + + def __init__( + self, + charset: str, + as_keyword: bool = False, + exclude_chars: typing.Optional[str] = None, + **kwargs, + ) -> None: + asKeyword: bool = deprecate_argument(kwargs, "asKeyword", False) + excludeChars: typing.Optional[str] = deprecate_argument( + kwargs, "excludeChars", None + ) + + asKeyword = asKeyword or as_keyword + excludeChars = excludeChars or exclude_chars + super().__init__( + charset, exact=1, as_keyword=asKeyword, exclude_chars=excludeChars + ) + + +class Regex(Token): + r"""Token for matching strings that match a given regular + expression. Defined with string specifying the regular expression in + a form recognized by the stdlib Python `re module `_. + If the given regex contains named groups (defined using ``(?P...)``), + these will be preserved as named :class:`ParseResults`. + + If instead of the Python stdlib ``re`` module you wish to use a different RE module + (such as the ``regex`` module), you can do so by building your ``Regex`` object with + a compiled RE that was compiled using ``regex``. + + The parameters ``pattern`` and ``flags`` are passed + to the ``re.compile()`` function as-is. See the Python + `re module `_ module for an + explanation of the acceptable patterns and flags. + + Example: + + .. testcode:: + + realnum = Regex(r"[+-]?\d+\.\d*") + # ref: https://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression + roman = Regex(r"M{0,4}(CM|CD|D?{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})") + + # named fields in a regex will be returned as named results + date = Regex(r'(?P\d{4})-(?P\d\d?)-(?P\d\d?)') + + # the Regex class will accept regular expressions compiled using the + # re module + import re + parser = pp.Regex(re.compile(r'[0-9]')) + """ + + def __init__( + self, + pattern: Any, + flags: Union[re.RegexFlag, int] = 0, + as_group_list: bool = False, + as_match: bool = False, + **kwargs, + ) -> None: + super().__init__() + asGroupList: bool = deprecate_argument(kwargs, "asGroupList", False) + asMatch: bool = deprecate_argument(kwargs, "asMatch", False) + + asGroupList = asGroupList or as_group_list + asMatch = asMatch or as_match + + if isinstance(pattern, str_type): + if not pattern: + raise ValueError("null string passed to Regex; use Empty() instead") + + self._re = None + self._may_return_empty = None # type: ignore [assignment] + self.reString = self.pattern = pattern + + elif hasattr(pattern, "pattern") and hasattr(pattern, "match"): + self._re = pattern + self._may_return_empty = None # type: ignore [assignment] + self.pattern = self.reString = pattern.pattern + + elif callable(pattern): + # defer creating this pattern until we really need it + self.pattern = pattern + self._may_return_empty = None # type: ignore [assignment] + self._re = None + + else: + raise TypeError( + "Regex may only be constructed with a string or a compiled RE object," + " or a callable that takes no arguments and returns a string or a" + " compiled RE object" + ) + + self.flags = flags + self.errmsg = f"Expected {self.name}" + self.mayIndexError = False + self.asGroupList = asGroupList + self.asMatch = asMatch + if self.asGroupList: + self.parseImpl = self.parseImplAsGroupList # type: ignore [method-assign] + if self.asMatch: + self.parseImpl = self.parseImplAsMatch # type: ignore [method-assign] + + def copy(self) -> Regex: + """ + Returns a copy of this expression. + + Generally only used internally by pyparsing. + """ + ret: Regex = cast(Regex, super().copy()) + if self.asGroupList: + ret.parseImpl = ret.parseImplAsGroupList # type: ignore [method-assign] + if self.asMatch: + ret.parseImpl = ret.parseImplAsMatch # type: ignore [method-assign] + return ret + + @cached_property + def re(self) -> re.Pattern: + """ + Property returning the compiled regular expression for this Regex. + + Generally only used internally by pyparsing. + """ + if self._re: + return self._re + + if callable(self.pattern): + # replace self.pattern with the string returned by calling self.pattern() + self.pattern = cast(Callable[[], str], self.pattern)() + + # see if we got a compiled RE back instead of a str - if so, we're done + if hasattr(self.pattern, "pattern") and hasattr(self.pattern, "match"): + self._re = cast(re.Pattern[str], self.pattern) + self.pattern = self.reString = self._re.pattern + return self._re + + try: + self._re = re.compile(self.pattern, self.flags) + except re.error: + raise ValueError(f"invalid pattern ({self.pattern!r}) passed to Regex") + else: + self._may_return_empty = self.re.match("", pos=0) is not None + return self._re + + @cached_property + def re_match(self) -> Callable[[str, int], Any]: + return self.re.match + + @property + def mayReturnEmpty(self): + if self._may_return_empty is None: + # force compile of regex pattern, to set may_return_empty flag + self.re # noqa + return self._may_return_empty + + @mayReturnEmpty.setter + def mayReturnEmpty(self, value): + self._may_return_empty = value + + def _generateDefaultName(self) -> str: + unescaped = repr(self.pattern).replace("\\\\", "\\") + return f"Re:({unescaped})" + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + # explicit check for matching past the length of the string; + # this is done because the re module will not complain about + # a match with `pos > len(instring)`, it will just return "" + if loc > len(instring) and self.mayReturnEmpty: + raise ParseException(instring, loc, self.errmsg, self) + + result = self.re_match(instring, loc) + if not result: + raise ParseException(instring, loc, self.errmsg, self) + + loc = result.end() + ret = ParseResults(result[0]) + d = result.groupdict() + + for k, v in d.items(): + ret[k] = v + + return loc, ret + + def parseImplAsGroupList(self, instring, loc, do_actions=True): + if loc > len(instring) and self.mayReturnEmpty: + raise ParseException(instring, loc, self.errmsg, self) + + result = self.re_match(instring, loc) + if not result: + raise ParseException(instring, loc, self.errmsg, self) + + loc = result.end() + ret = result.groups() + return loc, ret + + def parseImplAsMatch(self, instring, loc, do_actions=True): + if loc > len(instring) and self.mayReturnEmpty: + raise ParseException(instring, loc, self.errmsg, self) + + result = self.re_match(instring, loc) + if not result: + raise ParseException(instring, loc, self.errmsg, self) + + loc = result.end() + ret = result + return loc, ret + + def sub(self, repl: str) -> ParserElement: + r""" + Return :class:`Regex` with an attached parse action to transform the parsed + result as if called using `re.sub(expr, repl, string) `_. + + Example: + + .. testcode:: + + make_html = Regex(r"(\w+):(.*?):").sub(r"<\1>\2") + print(make_html.transform_string("h1:main title:")) + + .. testoutput:: + +

main title

+ """ + if self.asGroupList: + raise TypeError("cannot use sub() with Regex(as_group_list=True)") + + if self.asMatch and callable(repl): + raise TypeError( + "cannot use sub() with a callable with Regex(as_match=True)" + ) + + if self.asMatch: + + def pa(tokens): + return tokens[0].expand(repl) + + else: + + def pa(tokens): + return self.re.sub(repl, tokens[0]) + + return self.add_parse_action(pa) + + +class QuotedString(Token): + r""" + Token for matching strings that are delimited by quoting characters. + + Defined with the following parameters: + + - ``quote_char`` - string of one or more characters defining the + quote delimiting string + - ``esc_char`` - character to re_escape quotes, typically backslash + (default= ``None``) + - ``esc_quote`` - special quote sequence to re_escape an embedded quote + string (such as SQL's ``""`` to re_escape an embedded ``"``) + (default= ``None``) + - ``multiline`` - boolean indicating whether quotes can span + multiple lines (default= ``False``) + - ``unquote_results`` - boolean indicating whether the matched text + should be unquoted (default= ``True``) + - ``end_quote_char`` - string of one or more characters defining the + end of the quote delimited string (default= ``None`` => same as + quote_char) + - ``convert_whitespace_escapes`` - convert escaped whitespace + (``'\t'``, ``'\n'``, etc.) to actual whitespace + (default= ``True``) + + .. caution:: ``convert_whitespace_escapes`` has no effect if + ``unquote_results`` is ``False``. + + Example: + + .. doctest:: + + >>> qs = QuotedString('"') + >>> print(qs.search_string('lsjdf "This is the quote" sldjf')) + [['This is the quote']] + >>> complex_qs = QuotedString('{{', end_quote_char='}}') + >>> print(complex_qs.search_string( + ... 'lsjdf {{This is the "quote"}} sldjf')) + [['This is the "quote"']] + >>> sql_qs = QuotedString('"', esc_quote='""') + >>> print(sql_qs.search_string( + ... 'lsjdf "This is the quote with ""embedded"" quotes" sldjf')) + [['This is the quote with "embedded" quotes']] + """ + + ws_map = dict(((r"\t", "\t"), (r"\n", "\n"), (r"\f", "\f"), (r"\r", "\r"))) + + def __init__( + self, + quote_char: str = "", + esc_char: typing.Optional[str] = None, + esc_quote: typing.Optional[str] = None, + multiline: bool = False, + unquote_results: bool = True, + end_quote_char: typing.Optional[str] = None, + convert_whitespace_escapes: bool = True, + **kwargs, + ) -> None: + super().__init__() + quoteChar: str = deprecate_argument(kwargs, "quoteChar", "") + escChar: str = deprecate_argument(kwargs, "escChar", None) + escQuote: str = deprecate_argument(kwargs, "escQuote", None) + unquoteResults: bool = deprecate_argument(kwargs, "unquoteResults", True) + endQuoteChar: typing.Optional[str] = deprecate_argument( + kwargs, "endQuoteChar", None + ) + convertWhitespaceEscapes: bool = deprecate_argument( + kwargs, "convertWhitespaceEscapes", True + ) + + esc_char = escChar or esc_char + esc_quote = escQuote or esc_quote + unquote_results = unquoteResults and unquote_results + end_quote_char = endQuoteChar or end_quote_char + convert_whitespace_escapes = ( + convertWhitespaceEscapes and convert_whitespace_escapes + ) + quote_char = quoteChar or quote_char + + # reject empty or whitespace-only quote chars, but preserve any + # whitespace that is part of a valid quote delimiter (e.g. a leading + # newline in a multiline quote such as "\n;") + if not quote_char.strip(): + raise ValueError("quote_char cannot be the empty string") + + if end_quote_char is None: + end_quote_char = quote_char + elif not end_quote_char.strip(): + raise ValueError("end_quote_char cannot be the empty string") + + self.quote_char: str = quote_char + self.quote_char_len: int = len(quote_char) + self.first_quote_char: str = quote_char[0] + self.end_quote_char: str = end_quote_char + self.end_quote_char_len: int = len(end_quote_char) + self.esc_char: str = esc_char or "" + self.has_esc_char: bool = esc_char is not None + self.esc_quote: str = esc_quote or "" + self.unquote_results: bool = unquote_results + self.convert_whitespace_escapes: bool = convert_whitespace_escapes + self.multiline = multiline + self.re_flags = re.RegexFlag(0) + + # fmt: off + # build up re pattern for the content between the quote delimiters + inner_pattern: list[str] = [] + + if esc_quote: + inner_pattern.append(rf"(?:{re.escape(esc_quote)})") + + if esc_char: + inner_pattern.append(rf"(?:{re.escape(esc_char)}.)") + + if len(self.end_quote_char) > 1: + inner_pattern.append( + "(?:" + + "|".join( + f"(?:{re.escape(self.end_quote_char[:i])}(?!{re.escape(self.end_quote_char[i:])}))" + for i in range(len(self.end_quote_char) - 1, 0, -1) + ) + + ")" + ) + + if self.multiline: + self.re_flags |= re.MULTILINE | re.DOTALL + inner_pattern.append( + rf"(?:[^{_escape_regex_range_chars(self.end_quote_char[0])}" + rf"{(_escape_regex_range_chars(self.esc_char) if self.has_esc_char else '')}])" + ) + else: + inner_pattern.append( + rf"(?:[^{_escape_regex_range_chars(self.end_quote_char[0])}\n\r" + rf"{(_escape_regex_range_chars(self.esc_char) if self.has_esc_char else '')}])" + ) + + self.pattern = "".join( + [ + re.escape(self.quote_char), + "(?:", + '|'.join(inner_pattern), + ")*", + re.escape(self.end_quote_char), + ] + ) + + if self.unquote_results: + if self.convert_whitespace_escapes: + self.unquote_scan_re = re.compile( + rf"({'|'.join(re.escape(k) for k in self.ws_map)})" + rf"|(\\[0-7]{3}|\\0|\\x[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4})" + rf"|({re.escape(self.esc_char)}.)" + rf"|(\n|.)", + flags=self.re_flags, + ) + else: + self.unquote_scan_re = re.compile( + rf"({re.escape(self.esc_char)}.)" + rf"|(\n|.)", + flags=self.re_flags + ) + # fmt: on + + try: + self.re = re.compile(self.pattern, self.re_flags) + self.reString = self.pattern + self.re_match = self.re.match + except re.error: + raise ValueError(f"invalid pattern {self.pattern!r} passed to Regex") + + self.errmsg = f"Expected {self.name}" + self.mayIndexError = False + self._may_return_empty = True + + def _generateDefaultName(self) -> str: + if self.quote_char == self.end_quote_char and isinstance( + self.quote_char, str_type + ): + return f"string enclosed in {self.quote_char!r}" + + return f"quoted string, starting with {self.quote_char} ending with {self.end_quote_char}" + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + # check first character of opening quote to see if that is a match + # before doing the more complicated regex match + result = ( + instring[loc] == self.first_quote_char + and self.re_match(instring, loc) + or None + ) + if not result: + raise ParseException(instring, loc, self.errmsg, self) + + # get ending loc and matched string from regex matching result + loc = result.end() + ret = result[0] + + if self.unquote_results: + # strip off quotes + ret = ret[self.quote_char_len : -self.end_quote_char_len] + + if isinstance(ret, str_type): + # fmt: off + if self.convert_whitespace_escapes: + # as we iterate over matches in the input string, + # collect from whichever match group of the unquote_scan_re + # regex matches (only 1 group will match at any given time) + ret = "".join( + # match group 1 matches \t, \n, etc. + self.ws_map[g] if (g := match[1]) + # match group 2 matches escaped octal, null, hex, and Unicode + # sequences + else _convert_escaped_numerics_to_char(g[1:]) if (g := match[2]) + # match group 3 matches escaped characters + else g[-1] if (g := match[3]) + # match group 4 matches any character + else match[4] + for match in self.unquote_scan_re.finditer(ret) + ) + else: + ret = "".join( + # match group 1 matches escaped characters + g[-1] if (g := match[1]) + # match group 2 matches any character + else match[2] + for match in self.unquote_scan_re.finditer(ret) + ) + # fmt: on + + # replace escaped quotes + if self.esc_quote: + ret = ret.replace(self.esc_quote, self.end_quote_char) + + return loc, ret + + +class CharsNotIn(Token): + """Token for matching words composed of characters *not* in a given + set (will include whitespace in matched characters if not listed in + the provided exclusion set - see example). Defined with string + containing all disallowed characters, and an optional minimum, + maximum, and/or exact length. The default value for ``min`` is + 1 (a minimum value < 1 is not valid); the default values for + ``max`` and ``exact`` are 0, meaning no maximum or exact + length restriction. + + Example: + + .. testcode:: + + # define a comma-separated-value as anything that is not a ',' + csv_value = CharsNotIn(',') + print( + DelimitedList(csv_value).parse_string( + "dkls,lsdkjf,s12 34,@!#,213" + ) + ) + + prints: + + .. testoutput:: + + ['dkls', 'lsdkjf', 's12 34', '@!#', '213'] + """ + + def __init__( + self, not_chars: str = "", min: int = 1, max: int = 0, exact: int = 0, **kwargs + ) -> None: + super().__init__() + notChars: str = deprecate_argument(kwargs, "notChars", "") + + self.skipWhitespace = False + self.notChars = not_chars or notChars + self.notCharsSet = set(self.notChars) + + if min < 1: + raise ValueError( + "cannot specify a minimum length < 1; use" + " Opt(CharsNotIn()) if zero-length char group is permitted" + ) + + self.minLen = min + + if max > 0: + self.maxLen = max + else: + self.maxLen = _MAX_INT + + if exact > 0: + self.maxLen = exact + self.minLen = exact + + self.errmsg = f"Expected {self.name}" + self._may_return_empty = self.minLen == 0 + self.mayIndexError = False + + def _generateDefaultName(self) -> str: + not_chars_str = _collapse_string_to_ranges(self.notChars) + if len(not_chars_str) > 16: + return f"!W:({self.notChars[: 16 - 3]}...)" + else: + return f"!W:({self.notChars})" + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + notchars = self.notCharsSet + if instring[loc] in notchars: + raise ParseException(instring, loc, self.errmsg, self) + + start = loc + loc += 1 + maxlen = min(start + self.maxLen, len(instring)) + while loc < maxlen and instring[loc] not in notchars: + loc += 1 + + if loc - start < self.minLen: + raise ParseException(instring, loc, self.errmsg, self) + + return loc, instring[start:loc] + + +class White(Token): + """Special matching class for matching whitespace. Normally, + whitespace is ignored by pyparsing grammars. This class is included + when some whitespace structures are significant. Define with + a string containing the whitespace characters to be matched; default + is ``" \\t\\r\\n"``. Also takes optional ``min``, + ``max``, and ``exact`` arguments, as defined for the + :class:`Word` class. + """ + + whiteStrs = { + " ": "", + "\t": "", + "\n": "", + "\r": "", + "\f": "", + "\u00a0": "", + "\u1680": "", + "\u180e": "", + "\u2000": "", + "\u2001": "", + "\u2002": "", + "\u2003": "", + "\u2004": "", + "\u2005": "", + "\u2006": "", + "\u2007": "", + "\u2008": "", + "\u2009": "", + "\u200a": "", + "\u200b": "", + "\u202f": "", + "\u205f": "", + "\u3000": "", + } + + def __init__( + self, ws: str = " \t\r\n", min: int = 1, max: int = 0, exact: int = 0 + ) -> None: + super().__init__() + self.matchWhite = ws + self.set_whitespace_chars( + "".join(c for c in self.whiteStrs if c not in self.matchWhite), + copy_defaults=True, + ) + # self.leave_whitespace() + self._may_return_empty = True + self.errmsg = f"Expected {self.name}" + + self.minLen = min + + if max > 0: + self.maxLen = max + else: + self.maxLen = _MAX_INT + + if exact > 0: + self.maxLen = exact + self.minLen = exact + + def _generateDefaultName(self) -> str: + return "".join(White.whiteStrs[c] for c in self.matchWhite) + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + if instring[loc] not in self.matchWhite: + raise ParseException(instring, loc, self.errmsg, self) + start = loc + loc += 1 + maxloc = start + self.maxLen + maxloc = min(maxloc, len(instring)) + while loc < maxloc and instring[loc] in self.matchWhite: + loc += 1 + + if loc - start < self.minLen: + raise ParseException(instring, loc, self.errmsg, self) + + return loc, instring[start:loc] + + +class PositionToken(Token): + def __init__(self) -> None: + super().__init__() + self._may_return_empty = True + self.mayIndexError = False + + +class GoToColumn(PositionToken): + """Token to advance to a specific column of input text; useful for + tabular report scraping. + """ + + def __init__(self, colno: int) -> None: + super().__init__() + self.col = colno + + def preParse(self, instring: str, loc: int) -> int: + if col(loc, instring) == self.col: + return loc + + instrlen = len(instring) + if self.ignoreExprs: + loc = self._skipIgnorables(instring, loc) + while ( + loc < instrlen + and instring[loc].isspace() + and col(loc, instring) != self.col + ): + loc += 1 + + return loc + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + thiscol = col(loc, instring) + if thiscol > self.col: + raise ParseException(instring, loc, "Text not in expected column", self) + newloc = loc + self.col - thiscol + ret = instring[loc:newloc] + return newloc, ret + + +class LineStart(PositionToken): + r"""Matches if current position is at the logical beginning of a line (after skipping whitespace) + within the parse string + + Example: + + .. testcode:: + + test = '''\ + AAA this line + AAA and this line + AAA and even this line + B AAA but definitely not this line + ''' + + for t in (LineStart() + 'AAA' + rest_of_line).search_string(test): + print(t) + + prints: + + .. testoutput:: + + ['AAA', ' this line'] + ['AAA', ' and this line'] + ['AAA', ' and even this line'] + + """ + + def __init__(self) -> None: + super().__init__() + self.leave_whitespace() + self.orig_whiteChars = set() | self.whiteChars + self.whiteChars.discard("\n") + self.skipper = Empty().set_whitespace_chars(self.whiteChars) + self.set_name("start of line") + + def preParse(self, instring: str, loc: int) -> int: + if loc == 0: + return loc + + ret = self.skipper.preParse(instring, loc) + + if "\n" in self.orig_whiteChars: + while instring[ret : ret + 1] == "\n": + ret = self.skipper.preParse(instring, ret + 1) + + return ret + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + if col(loc, instring) == 1: + return loc, [] + raise ParseException(instring, loc, self.errmsg, self) + + +class LineEnd(PositionToken): + """Matches if current position is at the end of a line within the + parse string + """ + + def __init__(self) -> None: + super().__init__() + self.whiteChars.discard("\n") + self.set_whitespace_chars(self.whiteChars, copy_defaults=False) + self.set_name("end of line") + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + if loc < len(instring): + if instring[loc] == "\n": + return loc + 1, "\n" + else: + raise ParseException(instring, loc, self.errmsg, self) + elif loc == len(instring): + return loc + 1, [] + else: + raise ParseException(instring, loc, self.errmsg, self) + + +class StringStart(PositionToken): + """Matches if current position is at the beginning of the parse + string + """ + + def __init__(self) -> None: + super().__init__() + self.set_name("start of text") + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + # see if entire string up to here is just whitespace and ignoreables + if loc != 0 and loc != self.preParse(instring, 0): + raise ParseException(instring, loc, self.errmsg, self) + + return loc, [] + + +class StringEnd(PositionToken): + """ + Matches if current position is at the end of the parse string + """ + + def __init__(self) -> None: + super().__init__() + self.set_name("end of text") + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + if loc < len(instring): + raise ParseException(instring, loc, self.errmsg, self) + if loc == len(instring): + return loc + 1, [] + if loc > len(instring): + return loc, [] + + raise ParseException(instring, loc, self.errmsg, self) + + +class WordStart(PositionToken): + """Matches if the current position is at the beginning of a + :class:`Word`, and is not preceded by any character in a given + set of ``word_chars`` (default= ``printables``). To emulate the + ``\b`` behavior of regular expressions, use + ``WordStart(alphanums)``. ``WordStart`` will also match at + the beginning of the string being parsed, or at the beginning of + a line. + """ + + def __init__(self, word_chars: str = printables, **kwargs) -> None: + wordChars: str = deprecate_argument(kwargs, "wordChars", printables) + + wordChars = word_chars if wordChars == printables else wordChars + super().__init__() + self.wordChars = set(wordChars) + self.set_name("start of a word") + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + if loc != 0: + if ( + instring[loc - 1] in self.wordChars + or instring[loc] not in self.wordChars + ): + raise ParseException(instring, loc, self.errmsg, self) + return loc, [] + + +class WordEnd(PositionToken): + """Matches if the current position is at the end of a :class:`Word`, + and is not followed by any character in a given set of ``word_chars`` + (default= ``printables``). To emulate the ``\b`` behavior of + regular expressions, use ``WordEnd(alphanums)``. ``WordEnd`` + will also match at the end of the string being parsed, or at the end + of a line. + """ + + def __init__(self, word_chars: str = printables, **kwargs) -> None: + wordChars: str = deprecate_argument(kwargs, "wordChars", printables) + + wordChars = word_chars if wordChars == printables else wordChars + super().__init__() + self.wordChars = set(wordChars) + self.skipWhitespace = False + self.set_name("end of a word") + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + instrlen = len(instring) + if instrlen > 0 and loc < instrlen: + if ( + instring[loc] in self.wordChars + or instring[loc - 1] not in self.wordChars + ): + raise ParseException(instring, loc, self.errmsg, self) + return loc, [] + + +class Tag(Token): + """ + A meta-element for inserting a named result into the parsed + tokens that may be checked later in a parse action or while + processing the parsed results. Accepts an optional tag value, + defaulting to `True`. + + Example: + + .. doctest:: + + >>> end_punc = "." | ("!" + Tag("enthusiastic")) + >>> greeting = "Hello," + Word(alphas) + end_punc + + >>> result = greeting.parse_string("Hello, World.") + >>> print(result.dump()) + ['Hello,', 'World', '.'] + + >>> result = greeting.parse_string("Hello, World!") + >>> print(result.dump()) + ['Hello,', 'World', '!'] + - enthusiastic: True + + .. versionadded:: 3.1.0 + """ + + def __init__(self, tag_name: str, value: Any = True) -> None: + super().__init__() + self._may_return_empty = True + self.mayIndexError = False + self.leave_whitespace() + self.tag_name = tag_name + self.tag_value = value + self.add_parse_action(self._add_tag) + self.show_in_diagram = False + + def _add_tag(self, tokens: ParseResults): + tokens[self.tag_name] = self.tag_value + + def _generateDefaultName(self) -> str: + return f"{type(self).__name__}:{self.tag_name}={self.tag_value!r}" + + +class ParseExpression(ParserElement): + """Abstract subclass of ParserElement, for combining and + post-processing parsed tokens. + """ + + def __init__( + self, exprs: typing.Iterable[ParserElement], savelist: bool = False + ) -> None: + super().__init__(savelist) + self.exprs: list[ParserElement] + if isinstance(exprs, _generatorType): + exprs = list(exprs) + + if isinstance(exprs, str_type): + self.exprs = [self._literalStringClass(exprs)] + elif isinstance(exprs, ParserElement): + self.exprs = [exprs] + elif isinstance(exprs, Iterable): + exprs = list(exprs) + # if sequence of strings provided, wrap with Literal + if any(isinstance(expr, str_type) for expr in exprs): + exprs = ( + self._literalStringClass(e) if isinstance(e, str_type) else e + for e in exprs + ) + self.exprs = list(exprs) + else: + try: + self.exprs = list(exprs) + except TypeError: + self.exprs = [exprs] + self.callPreparse = False + + def recurse(self) -> list[ParserElement]: + return self.exprs[:] + + def append(self, other) -> ParserElement: + """ + Add an expression to the list of expressions related to this ParseExpression instance. + """ + self.exprs.append(other) + self._defaultName = None + return self + + def leave_whitespace(self, recursive: bool = True) -> ParserElement: + """ + Extends ``leave_whitespace`` defined in base class, and also invokes ``leave_whitespace`` on + all contained expressions. + """ + super().leave_whitespace(recursive) + + if recursive: + self.exprs = [e.copy() for e in self.exprs] + for e in self.exprs: + e.leave_whitespace(recursive) + return self + + def ignore_whitespace(self, recursive: bool = True) -> ParserElement: + """ + Extends ``ignore_whitespace`` defined in base class, and also invokes ``ignore_whitespace`` on + all contained expressions. + """ + super().ignore_whitespace(recursive) + if recursive: + self.exprs = [e.copy() for e in self.exprs] + for e in self.exprs: + e.ignore_whitespace(recursive) + return self + + def ignore(self, other) -> ParserElement: + """ + Define expression to be ignored (e.g., comments) while doing pattern + matching; may be called repeatedly, to define multiple comment or other + ignorable patterns. + """ + if isinstance(other, Suppress): + if other not in self.ignoreExprs: + super().ignore(other) + for e in self.exprs: + e.ignore(self.ignoreExprs[-1]) + else: + super().ignore(other) + for e in self.exprs: + e.ignore(self.ignoreExprs[-1]) + return self + + def _generateDefaultName(self) -> str: + return f"{type(self).__name__}:({self.exprs})" + + def streamline(self) -> ParserElement: + if self.streamlined: + return self + + super().streamline() + + for e in self.exprs: + e.streamline() + + # collapse nested :class:`And`'s of the form ``And(And(And(a, b), c), d)`` to ``And(a, b, c, d)`` + # but only if there are no parse actions or resultsNames on the nested And's + # (likewise for :class:`Or`'s and :class:`MatchFirst`'s) + if len(self.exprs) == 2: + first, second = self.exprs + if ( + isinstance(first, self.__class__) + and not first.parseAction + and first.resultsName is None + and not first.debug + ): + self.exprs[:] = (*first.exprs, second) + self._defaultName = None + self._may_return_empty |= first.mayReturnEmpty + self.mayIndexError |= first.mayIndexError + + last = self.exprs[-1] + if ( + isinstance(last, self.__class__) + and not last.parseAction + and last.resultsName is None + and not last.debug + ): + self.exprs[-1:] = last.exprs + self._defaultName = None + self._may_return_empty |= last.mayReturnEmpty + self.mayIndexError |= last.mayIndexError + + self.errmsg = f"Expected {self}" + + return self + + def validate(self, validateTrace=None) -> None: + warnings.warn( + "ParserElement.validate() is deprecated, and should not be used to check for left recursion", + PyparsingDeprecationWarning, + stacklevel=2, + ) + tmp = (validateTrace if validateTrace is not None else [])[:] + [self] + for e in self.exprs: + e.validate(tmp) + self._checkRecursion([]) + + def copy(self) -> ParserElement: + """ + Returns a copy of this expression. + + Generally only used internally by pyparsing. + """ + ret = super().copy() + ret = typing.cast(ParseExpression, ret) + ret.exprs = [e.copy() for e in self.exprs] + return ret + + def _setResultsName(self, name, list_all_matches=False) -> ParserElement: + if not ( + __diag__.warn_ungrouped_named_tokens_in_collection + and Diagnostics.warn_ungrouped_named_tokens_in_collection + not in self.suppress_warnings_ + ): + return super()._setResultsName(name, list_all_matches) + + for e in self.exprs: + if ( + isinstance(e, ParserElement) + and e.resultsName + and ( + Diagnostics.warn_ungrouped_named_tokens_in_collection + not in e.suppress_warnings_ + ) + ): + warning = ( + "warn_ungrouped_named_tokens_in_collection:" + f" setting results name {name!r} on {type(self).__name__} expression" + f" collides with {e.resultsName!r} on contained expression" + ) + warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3) + break + + return super()._setResultsName(name, list_all_matches) + + # Compatibility synonyms + # fmt: off + leaveWhitespace = replaced_by_pep8("leaveWhitespace", leave_whitespace) + ignoreWhitespace = replaced_by_pep8("ignoreWhitespace", ignore_whitespace) + # fmt: on + + +class And(ParseExpression): + """ + Requires all given :class:`ParserElement` s to be found in the given order. + Expressions may be separated by whitespace. + May be constructed using the ``'+'`` operator. + May also be constructed using the ``'-'`` operator, which will + suppress backtracking. + + Example: + + .. testcode:: + + integer = Word(nums) + name_expr = Word(alphas)[1, ...] + + expr = And([integer("id"), name_expr("name"), integer("age")]) + # more easily written as: + expr = integer("id") + name_expr("name") + integer("age") + """ + + class _ErrorStop(Empty): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.leave_whitespace() + + def _generateDefaultName(self) -> str: + return "-" + + def __init__( + self, + exprs_arg: typing.Iterable[Union[ParserElement, str]], + savelist: bool = True, + ) -> None: + # instantiate exprs as a list, converting strs to ParserElements + exprs: list[ParserElement] = [ + self._literalStringClass(e) if isinstance(e, str) else e for e in exprs_arg + ] + + # convert any Ellipsis elements to SkipTo + if Ellipsis in exprs: + + # Ellipsis cannot be the last element + if exprs[-1] is Ellipsis: + raise Exception("cannot construct And with sequence ending in ...") + + tmp: list[ParserElement] = [] + for cur_expr, next_expr in zip(exprs, exprs[1:]): + if cur_expr is Ellipsis: + tmp.append(SkipTo(next_expr)("_skipped*")) + else: + tmp.append(cur_expr) + + exprs[:-1] = tmp + + super().__init__(exprs, savelist) + if self.exprs: + self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs) + if not isinstance(self.exprs[0], White): + self.set_whitespace_chars( + self.exprs[0].whiteChars, + copy_defaults=self.exprs[0].copyDefaultWhiteChars, + ) + self.skipWhitespace = self.exprs[0].skipWhitespace + else: + self.skipWhitespace = False + else: + self._may_return_empty = True + self.callPreparse = True + + def streamline(self) -> ParserElement: + """ + Collapse `And` expressions like `And(And(And(A, B), C), D)` + to `And(A, B, C, D)`. + + .. doctest:: + + >>> expr = Word("A") + Word("B") + Word("C") + Word("D") + >>> # Using '+' operator creates nested And expression + >>> expr + {{{W:(A) W:(B)} W:(C)} W:(D)} + >>> # streamline simplifies to a single And with multiple expressions + >>> expr.streamline() + {W:(A) W:(B) W:(C) W:(D)} + + Guards against collapsing out expressions that have special features, + such as results names or parse actions. + + Resolves pending Skip commands defined using `...` terms. + """ + # collapse any _PendingSkip's + if self.exprs and any( + isinstance(e, ParseExpression) + and e.exprs + and isinstance(e.exprs[-1], _PendingSkip) + for e in self.exprs[:-1] + ): + deleted_expr_marker = NoMatch() + for i, e in enumerate(self.exprs[:-1]): + if e is deleted_expr_marker: + continue + if ( + isinstance(e, ParseExpression) + and e.exprs + and isinstance(e.exprs[-1], _PendingSkip) + ): + e.exprs[-1] = e.exprs[-1] + self.exprs[i + 1] + self.exprs[i + 1] = deleted_expr_marker + self.exprs = [e for e in self.exprs if e is not deleted_expr_marker] + + super().streamline() + + # link any IndentedBlocks to the prior expression + prev: ParserElement + cur: ParserElement + for prev, cur in zip(self.exprs, self.exprs[1:]): + # traverse cur or any first embedded expr of cur looking for an IndentedBlock + # (but watch out for recursive grammar) + seen = set() + while True: + if id(cur) in seen: + break + seen.add(id(cur)) + if isinstance(cur, IndentedBlock): + prev.add_parse_action( + lambda s, l, t, cur_=cur: setattr( + cur_, "parent_anchor", col(l, s) + ) + ) + break + subs = cur.recurse() + next_first = next(iter(subs), None) + if next_first is None: + break + cur = typing.cast(ParserElement, next_first) + + self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs) + return self + + def parseImpl(self, instring, loc, do_actions=True): + + # if no exprs defined, assume we contain a single Empty + # (consistent with behavior of `all([])` returning True) + exprs = iter(self.exprs or (Empty(),)) + + # pass False as callPreParse arg to _parse for first element, since we already + # pre-parsed the string as part of our And pre-parsing + loc, resultlist = next(exprs)._parse( + instring, loc, do_actions, callPreParse=False + ) + + # iterate over remaining expressions + raise_syntax_error_immediately = False + for e in exprs: + # if isinstance(e, And._ErrorStop): + if type(e) is And._ErrorStop: + raise_syntax_error_immediately = True + continue + + if raise_syntax_error_immediately: + try: + loc, exprtokens = e._parse(instring, loc, do_actions) + except ParseSyntaxException: + raise + except ParseBaseException as pe: + pe.__traceback__ = None + raise ParseSyntaxException._from_exception(pe) + except IndexError: + raise ParseSyntaxException( + instring, len(instring), self.errmsg, self + ) + else: + loc, exprtokens = e._parse(instring, loc, do_actions) + resultlist += exprtokens + return loc, resultlist + + def __iadd__(self, other): + if isinstance(other, str_type): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented + return self.append(other) # And([self, other]) + + def _checkRecursion(self, parseElementList): + subRecCheckList = parseElementList[:] + [self] + for e in self.exprs: + e._checkRecursion(subRecCheckList) + if not e.mayReturnEmpty: + break + + def _generateDefaultName(self) -> str: + inner = " ".join(str(e) for e in self.exprs) + # strip off redundant inner {}'s + while len(inner) > 1 and inner[0 :: len(inner) - 1] == "{}": + inner = inner[1:-1] + return f"{{{inner}}}" + + +class Or(ParseExpression): + """Requires that at least one :class:`ParserElement` is found. If + two expressions match, the expression that matches the longest + string will be used. May be constructed using the ``'^'`` + operator. + + Example: + + .. testcode:: + + # construct Or using '^' operator + + number = Word(nums) ^ Combine(Word(nums) + '.' + Word(nums)) + print(number.search_string("123 3.1416 789")) + + prints: + + .. testoutput:: + + [['123'], ['3.1416'], ['789']] + """ + + def __init__( + self, exprs: typing.Iterable[ParserElement], savelist: bool = False + ) -> None: + super().__init__(exprs, savelist) + if self.exprs: + self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs) + self.skipWhitespace = all(e.skipWhitespace for e in self.exprs) + else: + self._may_return_empty = True + + def streamline(self) -> ParserElement: + super().streamline() + if self.exprs: + self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs) + self.saveAsList = any(e.saveAsList for e in self.exprs) + self.skipWhitespace = all( + e.skipWhitespace and not isinstance(e, White) for e in self.exprs + ) + else: + self.saveAsList = False + return self + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + maxExcLoc = -1 + maxException = None + matches: list[tuple[int, ParserElement]] = [] + fatals: list[ParseFatalException] = [] + if all(e.callPreparse for e in self.exprs): + loc = self.preParse(instring, loc) + for e in self.exprs: + try: + loc2 = e.try_parse(instring, loc, raise_fatal=True) + except ParseFatalException as pfe: + pfe.__traceback__ = None + pfe.parser_element = e + fatals.append(pfe) + maxException = None + maxExcLoc = -1 + except ParseException as err: + if not fatals: + err.__traceback__ = None + if err.loc > maxExcLoc: + maxException = err + maxExcLoc = err.loc + except IndexError: + if len(instring) > maxExcLoc: + maxException = ParseException( + instring, len(instring), e.errmsg, self + ) + maxExcLoc = len(instring) + else: + # save match among all matches, to retry longest to shortest + matches.append((loc2, e)) + + if matches: + # re-evaluate all matches in descending order of length of match, in case attached actions + # might change whether or how much they match of the input. + matches.sort(key=itemgetter(0), reverse=True) + + if not do_actions: + # no further conditions or parse actions to change the selection of + # alternative, so the first match will be the best match + best_expr = matches[0][1] + return best_expr._parse(instring, loc, do_actions) + + longest: tuple[int, typing.Optional[ParseResults]] = -1, None + for loc1, expr1 in matches: + if loc1 <= longest[0]: + # already have a longer match than this one will deliver, we are done + return longest + + try: + loc2, toks = expr1._parse(instring, loc, do_actions) + except ParseException as err: + err.__traceback__ = None + if err.loc > maxExcLoc: + maxException = err + maxExcLoc = err.loc + else: + if loc2 >= loc1: + return loc2, toks + # didn't match as much as before + elif loc2 > longest[0]: + longest = loc2, toks + + if longest != (-1, None): + return longest + + if fatals: + if len(fatals) > 1: + fatals.sort(key=lambda e: -e.loc) + if fatals[0].loc == fatals[1].loc: + fatals.sort(key=lambda e: (-e.loc, -len(str(e.parser_element)))) + max_fatal = fatals[0] + raise max_fatal + + if maxException is not None: + # infer from this check that all alternatives failed at the current position + # so emit this collective error message instead of any single error message + parse_start_loc = self.preParse(instring, loc) + if maxExcLoc == parse_start_loc: + maxException.msg = self.errmsg or "" + raise maxException + + raise ParseException(instring, loc, "no defined alternatives to match", self) + + def __ixor__(self, other): + if isinstance(other, str_type): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented + return self.append(other) # Or([self, other]) + + def _generateDefaultName(self) -> str: + return f"{{{' ^ '.join(str(e) for e in self.exprs)}}}" + + def _setResultsName(self, name, list_all_matches=False) -> ParserElement: + if ( + __diag__.warn_multiple_tokens_in_named_alternation + and Diagnostics.warn_multiple_tokens_in_named_alternation + not in self.suppress_warnings_ + ): + if any( + isinstance(e, And) + and Diagnostics.warn_multiple_tokens_in_named_alternation + not in e.suppress_warnings_ + for e in self.exprs + ): + warning = ( + "warn_multiple_tokens_in_named_alternation:" + f" setting results name {name!r} on {type(self).__name__} expression" + " will return a list of all parsed tokens in an And alternative," + " in prior versions only the first token was returned; enclose" + " contained argument in Group" + ) + warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3) + + return super()._setResultsName(name, list_all_matches) + + +class MatchFirst(ParseExpression): + """Requires that at least one :class:`ParserElement` is found. If + more than one expression matches, the first one listed is the one that will + match. May be constructed using the ``'|'`` operator. + + Example: Construct MatchFirst using '|' operator + + .. doctest:: + + # watch the order of expressions to match + >>> number = Word(nums) | Combine(Word(nums) + '.' + Word(nums)) + >>> print(number.search_string("123 3.1416 789")) # Fail! + [['123'], ['3'], ['1416'], ['789']] + + # put more selective expression first + >>> number = Combine(Word(nums) + '.' + Word(nums)) | Word(nums) + >>> print(number.search_string("123 3.1416 789")) # Better + [['123'], ['3.1416'], ['789']] + """ + + def __init__( + self, exprs: typing.Iterable[ParserElement], savelist: bool = False + ) -> None: + super().__init__(exprs, savelist) + if self.exprs: + self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs) + self.skipWhitespace = all(e.skipWhitespace for e in self.exprs) + else: + self._may_return_empty = True + + def streamline(self) -> ParserElement: + if self.streamlined: + return self + + super().streamline() + if self.exprs: + self.saveAsList = any(e.saveAsList for e in self.exprs) + self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs) + self.skipWhitespace = all( + e.skipWhitespace and not isinstance(e, White) for e in self.exprs + ) + else: + self.saveAsList = False + self._may_return_empty = True + return self + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + maxExcLoc = -1 + maxException = None + + for e in self.exprs: + try: + return e._parse(instring, loc, do_actions) + except ParseFatalException as pfe: + pfe.__traceback__ = None + pfe.parser_element = e + raise + except ParseException as err: + if err.loc > maxExcLoc: + maxException = err + maxExcLoc = err.loc + except IndexError: + if len(instring) > maxExcLoc: + maxException = ParseException( + instring, len(instring), e.errmsg, self + ) + maxExcLoc = len(instring) + + if maxException is not None: + # infer from this check that all alternatives failed at the current position + # so emit this collective error message instead of any individual error message + parse_start_loc = self.preParse(instring, loc) + if maxExcLoc == parse_start_loc: + maxException.msg = self.errmsg or "" + raise maxException + + raise ParseException(instring, loc, "no defined alternatives to match", self) + + def __ior__(self, other): + if isinstance(other, str_type): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented + return self.append(other) # MatchFirst([self, other]) + + def _generateDefaultName(self) -> str: + return f"{{{' | '.join(str(e) for e in self.exprs)}}}" + + def _setResultsName(self, name, list_all_matches=False) -> ParserElement: + if ( + __diag__.warn_multiple_tokens_in_named_alternation + and Diagnostics.warn_multiple_tokens_in_named_alternation + not in self.suppress_warnings_ + ): + if any( + isinstance(e, And) + and Diagnostics.warn_multiple_tokens_in_named_alternation + not in e.suppress_warnings_ + for e in self.exprs + ): + warning = ( + "warn_multiple_tokens_in_named_alternation:" + f" setting results name {name!r} on {type(self).__name__} expression" + " will return a list of all parsed tokens in an And alternative," + " in prior versions only the first token was returned; enclose" + " contained argument in Group" + ) + warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3) + + return super()._setResultsName(name, list_all_matches) + + +class Each(ParseExpression): + """Requires all given :class:`ParserElement` s to be found, but in + any order. Expressions may be separated by whitespace. + + May be constructed using the ``'&'`` operator. + + Example: + + .. testcode:: + + color = one_of("RED ORANGE YELLOW GREEN BLUE PURPLE BLACK WHITE BROWN") + shape_type = one_of("SQUARE CIRCLE TRIANGLE STAR HEXAGON OCTAGON") + integer = Word(nums) + shape_attr = "shape:" + shape_type("shape") + posn_attr = "posn:" + Group(integer("x") + ',' + integer("y"))("posn") + color_attr = "color:" + color("color") + size_attr = "size:" + integer("size") + + # use Each (using operator '&') to accept attributes in any order + # (shape and posn are required, color and size are optional) + shape_spec = shape_attr & posn_attr & Opt(color_attr) & Opt(size_attr) + + shape_spec.run_tests(''' + shape: SQUARE color: BLACK posn: 100, 120 + shape: CIRCLE size: 50 color: BLUE posn: 50,80 + color:GREEN size:20 shape:TRIANGLE posn:20,40 + ''' + ) + + prints: + + .. testoutput:: + :options: +NORMALIZE_WHITESPACE + + + shape: SQUARE color: BLACK posn: 100, 120 + ['shape:', 'SQUARE', 'color:', 'BLACK', 'posn:', ['100', ',', '120']] + - color: 'BLACK' + - posn: ['100', ',', '120'] + - x: '100' + - y: '120' + - shape: 'SQUARE' + ... + + shape: CIRCLE size: 50 color: BLUE posn: 50,80 + ['shape:', 'CIRCLE', 'size:', '50', 'color:', 'BLUE', + 'posn:', ['50', ',', '80']] + - color: 'BLUE' + - posn: ['50', ',', '80'] + - x: '50' + - y: '80' + - shape: 'CIRCLE' + - size: '50' + ... + + color:GREEN size:20 shape:TRIANGLE posn:20,40 + ['color:', 'GREEN', 'size:', '20', 'shape:', 'TRIANGLE', + 'posn:', ['20', ',', '40']] + - color: 'GREEN' + - posn: ['20', ',', '40'] + - x: '20' + - y: '40' + - shape: 'TRIANGLE' + - size: '20' + ... + """ + + def __init__( + self, exprs: typing.Iterable[ParserElement], savelist: bool = True + ) -> None: + super().__init__(exprs, savelist) + if self.exprs: + self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs) + else: + self._may_return_empty = True + self.skipWhitespace = True + self.initExprGroups = True + self.saveAsList = True + + def __iand__(self, other): + if isinstance(other, str_type): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + return NotImplemented + return self.append(other) # Each([self, other]) + + def streamline(self) -> ParserElement: + super().streamline() + if self.exprs: + self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs) + else: + self._may_return_empty = True + return self + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + if self.initExprGroups: + self.opt1map = dict( + (id(e.expr), e) for e in self.exprs if isinstance(e, Opt) + ) + opt1 = [e.expr for e in self.exprs if isinstance(e, Opt)] + opt2 = [ + e + for e in self.exprs + if e.mayReturnEmpty and not isinstance(e, (Opt, Regex, ZeroOrMore)) + ] + self.optionals = opt1 + opt2 + self.multioptionals = [ + e.expr.set_results_name(e.resultsName, list_all_matches=True) + for e in self.exprs + if isinstance(e, _MultipleMatch) + ] + self.multirequired = [ + e.expr.set_results_name(e.resultsName, list_all_matches=True) + for e in self.exprs + if isinstance(e, OneOrMore) + ] + self.required = [ + e for e in self.exprs if not isinstance(e, (Opt, ZeroOrMore, OneOrMore)) + ] + self.required += self.multirequired + self.initExprGroups = False + + tmpLoc = loc + tmpReqd = self.required[:] + tmpOpt = self.optionals[:] + multis = self.multioptionals[:] + matchOrder: list[ParserElement] = [] + + keepMatching = True + failed: list[ParserElement] = [] + fatals: list[ParseFatalException] = [] + while keepMatching: + tmpExprs = tmpReqd + tmpOpt + multis + failed.clear() + fatals.clear() + for e in tmpExprs: + try: + tmpLoc = e.try_parse(instring, tmpLoc, raise_fatal=True) + except ParseFatalException as pfe: + pfe.__traceback__ = None + pfe.parser_element = e + fatals.append(pfe) + failed.append(e) + except ParseException: + failed.append(e) + else: + matchOrder.append(self.opt1map.get(id(e), e)) + if e in tmpReqd: + tmpReqd.remove(e) + elif e in tmpOpt: + tmpOpt.remove(e) + if len(failed) == len(tmpExprs): + keepMatching = False + + # look for any ParseFatalExceptions + if fatals: + if len(fatals) > 1: + fatals.sort(key=lambda e: -e.loc) + if fatals[0].loc == fatals[1].loc: + fatals.sort(key=lambda e: (-e.loc, -len(str(e.parser_element)))) + max_fatal = fatals[0] + raise max_fatal + + if tmpReqd: + missing = ", ".join([str(e) for e in tmpReqd]) + raise ParseException( + instring, + loc, + f"Missing one or more required elements ({missing})", + ) + + # add any unmatched Opts, in case they have default values defined + matchOrder += [e for e in self.exprs if isinstance(e, Opt) and e.expr in tmpOpt] + + total_results = ParseResults([]) + for e in matchOrder: + loc, results = e._parse(instring, loc, do_actions) + total_results += results + + return loc, total_results + + def _generateDefaultName(self) -> str: + return f"{{{' & '.join(str(e) for e in self.exprs)}}}" + + +class ParseElementEnhance(ParserElement): + """Abstract subclass of :class:`ParserElement`, for combining and + post-processing parsed tokens. + """ + + def __init__(self, expr: Union[ParserElement, str], savelist: bool = False) -> None: + super().__init__(savelist) + if isinstance(expr, str_type): + expr_str = typing.cast(str, expr) + if issubclass(self._literalStringClass, Token): + expr = self._literalStringClass(expr_str) # type: ignore[call-arg] + elif issubclass(type(self), self._literalStringClass): + expr = Literal(expr_str) + else: + expr = self._literalStringClass(Literal(expr_str)) # type: ignore[assignment, call-arg] + expr = typing.cast(ParserElement, expr) + self.expr = expr + if expr is not None: + self.mayIndexError = expr.mayIndexError + self._may_return_empty = expr.mayReturnEmpty + self.set_whitespace_chars( + expr.whiteChars, copy_defaults=expr.copyDefaultWhiteChars + ) + self.skipWhitespace = expr.skipWhitespace + self.saveAsList = expr.saveAsList + self.callPreparse = expr.callPreparse + self.ignoreExprs.extend(expr.ignoreExprs) + + def recurse(self) -> list[ParserElement]: + return [self.expr] if self.expr is not None else [] + + def parseImpl(self, instring, loc, do_actions=True): + if self.expr is None: + raise ParseException(instring, loc, "No expression defined", self) + + try: + return self.expr._parse(instring, loc, do_actions, callPreParse=False) + except ParseSyntaxException: + raise + except ParseBaseException as pbe: + pbe.pstr = pbe.pstr or instring + pbe.loc = pbe.loc or loc + pbe.parser_element = pbe.parser_element or self + if not isinstance(self, Forward) and self.customName is not None: + if self.errmsg: + pbe.msg = self.errmsg + raise + + def leave_whitespace(self, recursive: bool = True) -> ParserElement: + """ + Extends ``leave_whitespace`` defined in base class, and also invokes ``leave_whitespace`` on + the contained expression. + """ + super().leave_whitespace(recursive) + + if recursive: + if self.expr is not None: + self.expr = self.expr.copy() + self.expr.leave_whitespace(recursive) + return self + + def ignore_whitespace(self, recursive: bool = True) -> ParserElement: + """ + Extends ``ignore_whitespace`` defined in base class, and also invokes ``ignore_whitespace`` on + the contained expression. + """ + super().ignore_whitespace(recursive) + + if recursive: + if self.expr is not None: + self.expr = self.expr.copy() + self.expr.ignore_whitespace(recursive) + return self + + def ignore(self, other) -> ParserElement: + """ + Define expression to be ignored (e.g., comments) while doing pattern + matching; may be called repeatedly, to define multiple comment or other + ignorable patterns. + """ + if not isinstance(other, Suppress) or other not in self.ignoreExprs: + super().ignore(other) + if self.expr is not None: + self.expr.ignore(self.ignoreExprs[-1]) + + return self + + def streamline(self) -> ParserElement: + super().streamline() + if self.expr is not None: + self.expr.streamline() + return self + + def _checkRecursion(self, parseElementList): + if self in parseElementList: + raise RecursiveGrammarException(parseElementList + [self]) + subRecCheckList = parseElementList[:] + [self] + if self.expr is not None: + self.expr._checkRecursion(subRecCheckList) + + def validate(self, validateTrace=None) -> None: + warnings.warn( + "ParserElement.validate() is deprecated, and should not be used to check for left recursion", + PyparsingDeprecationWarning, + stacklevel=2, + ) + if validateTrace is None: + validateTrace = [] + tmp = validateTrace[:] + [self] + if self.expr is not None: + self.expr.validate(tmp) + self._checkRecursion([]) + + def _generateDefaultName(self) -> str: + return f"{type(self).__name__}:({self.expr})" + + # Compatibility synonyms + # fmt: off + leaveWhitespace = replaced_by_pep8("leaveWhitespace", leave_whitespace) + ignoreWhitespace = replaced_by_pep8("ignoreWhitespace", ignore_whitespace) + # fmt: on + + +class IndentedBlock(ParseElementEnhance): + """ + Expression to match one or more expressions at a given indentation level. + Useful for parsing text where structure is implied by indentation (like Python source code). + + Example: + + .. testcode:: + + ''' + BNF: + statement ::= assignment_stmt | if_stmt + assignment_stmt ::= identifier '=' rvalue + rvalue ::= identifier | integer + if_stmt ::= 'if' bool_condition block + block ::= ([indent] statement)... + identifier ::= [A..Za..z] + integer ::= [0..9]... + bool_condition ::= 'TRUE' | 'FALSE' + ''' + + IF, TRUE, FALSE = Keyword.using_each("IF TRUE FALSE".split()) + + statement = Forward() + identifier = Char(alphas) + integer = Word(nums).add_parse_action(lambda t: int(t[0])) + rvalue = identifier | integer + assignment_stmt = identifier + "=" + rvalue + + if_stmt = IF + (TRUE | FALSE) + IndentedBlock(statement) + + statement <<= Group(assignment_stmt | if_stmt) + + result = if_stmt.parse_string(''' + IF TRUE + a = 1000 + b = 2000 + IF FALSE + z = 100 + ''') + print(result.dump()) + + .. testoutput:: + + ['IF', 'TRUE', [['a', '=', 1000], ['b', '=', 2000], ['IF', 'FALSE', [['z', '=', 100]]]]] + [0]: + IF + [1]: + TRUE + [2]: + [['a', '=', 1000], ['b', '=', 2000], ['IF', 'FALSE', [['z', '=', 100]]]] + [0]: + ['a', '=', 1000] + [1]: + ['b', '=', 2000] + [2]: + ['IF', 'FALSE', [['z', '=', 100]]] + [0]: + IF + [1]: + FALSE + [2]: + [['z', '=', 100]] + [0]: + ['z', '=', 100] + """ + + class _Indent(Empty): + def __init__(self, ref_col: int) -> None: + super().__init__() + self.errmsg = f"expected indent at column {ref_col}" + self.add_condition(lambda s, l, t: col(l, s) == ref_col) + + class _IndentGreater(Empty): + def __init__(self, ref_col: int) -> None: + super().__init__() + self.errmsg = f"expected indent at column greater than {ref_col}" + self.add_condition(lambda s, l, t: col(l, s) > ref_col) + + def __init__( + self, expr: ParserElement, *, recursive: bool = False, grouped: bool = True + ) -> None: + super().__init__(expr, savelist=True) + # if recursive: + # raise NotImplementedError("IndentedBlock with recursive is not implemented") + self._recursive = recursive + self._grouped = grouped + self.parent_anchor = 1 + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + # advance parse position to non-whitespace by using an Empty() + # this should be the column to be used for all subsequent indented lines + anchor_loc = Empty().preParse(instring, loc) + + # see if self.expr matches at the current location - if not it will raise an exception + # and no further work is necessary + self.expr.try_parse(instring, anchor_loc, do_actions=do_actions) + + indent_col = col(anchor_loc, instring) + peer_detect_expr = self._Indent(indent_col) + + inner_expr = Empty() + peer_detect_expr + self.expr + if self._recursive: + sub_indent = self._IndentGreater(indent_col) + nested_block = IndentedBlock( + self.expr, recursive=self._recursive, grouped=self._grouped + ) + nested_block.set_debug(self.debug) + nested_block.parent_anchor = indent_col + inner_expr += Opt(sub_indent + nested_block) + + inner_expr.set_name(f"inner {hex(id(inner_expr))[-4:].upper()}@{indent_col}") + block = OneOrMore(inner_expr) + + trailing_undent = self._Indent(self.parent_anchor) | StringEnd() + + if self._grouped: + wrapper = Group + else: + wrapper = lambda expr: expr # type: ignore[misc, assignment] + return (wrapper(block) + Optional(trailing_undent)).parseImpl( + instring, anchor_loc, do_actions + ) + + +class AtStringStart(ParseElementEnhance): + """Matches if expression matches at the beginning of the parse + string:: + + AtStringStart(Word(nums)).parse_string("123") + # prints ["123"] + + AtStringStart(Word(nums)).parse_string(" 123") + # raises ParseException + """ + + def __init__(self, expr: Union[ParserElement, str]) -> None: + super().__init__(expr) + self.callPreparse = False + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + if loc != 0: + raise ParseException(instring, loc, "not found at string start") + return super().parseImpl(instring, loc, do_actions) + + +class AtLineStart(ParseElementEnhance): + r"""Matches if an expression matches at the beginning of a line within + the parse string + + Example: + + .. testcode:: + + test = '''\ + BBB this line + BBB and this line + BBB but not this one + A BBB and definitely not this one + ''' + + for t in (AtLineStart('BBB') + rest_of_line).search_string(test): + print(t) + + prints: + + .. testoutput:: + + ['BBB', ' this line'] + ['BBB', ' and this line'] + """ + + def __init__(self, expr: Union[ParserElement, str]) -> None: + super().__init__(expr) + self.callPreparse = False + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + if col(loc, instring) != 1: + raise ParseException(instring, loc, "not found at line start") + return super().parseImpl(instring, loc, do_actions) + + +class FollowedBy(ParseElementEnhance): + """Lookahead matching of the given parse expression. + ``FollowedBy`` does *not* advance the parsing position within + the input string, it only verifies that the specified parse + expression matches at the current position. ``FollowedBy`` + always returns a null token list. If any results names are defined + in the lookahead expression, those *will* be returned for access by + name. + + Example: + + .. testcode:: + + # use FollowedBy to match a label only if it is followed by a ':' + data_word = Word(alphas) + label = data_word + FollowedBy(':') + attr_expr = Group( + label + Suppress(':') + + OneOrMore(data_word, stop_on=label).set_parse_action(' '.join) + ) + + attr_expr[1, ...].parse_string( + "shape: SQUARE color: BLACK posn: upper left").pprint() + + prints: + + .. testoutput:: + + [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']] + """ + + def __init__(self, expr: Union[ParserElement, str]) -> None: + super().__init__(expr) + self._may_return_empty = True + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + # by using self._expr.parse and deleting the contents of the returned ParseResults list + # we keep any named results that were defined in the FollowedBy expression + _, ret = self.expr._parse(instring, loc, do_actions=do_actions) + del ret[:] + + return loc, ret + + +class PrecededBy(ParseElementEnhance): + """Lookbehind matching of the given parse expression. + ``PrecededBy`` does not advance the parsing position within the + input string, it only verifies that the specified parse expression + matches prior to the current position. ``PrecededBy`` always + returns a null token list, but if a results name is defined on the + given expression, it is returned. + + Parameters: + + - ``expr`` - expression that must match prior to the current parse + location + - ``retreat`` - (default= ``None``) - (int) maximum number of characters + to lookbehind prior to the current parse location + + If the lookbehind expression is a string, :class:`Literal`, + :class:`Keyword`, or a :class:`Word` or :class:`CharsNotIn` + with a specified exact or maximum length, then the retreat + parameter is not required. Otherwise, retreat must be specified to + give a maximum number of characters to look back from + the current parse position for a lookbehind match. + + Example: + + .. testcode:: + + # VB-style variable names with type prefixes + int_var = PrecededBy("#") + pyparsing_common.identifier + str_var = PrecededBy("$") + pyparsing_common.identifier + """ + + def __init__(self, expr: Union[ParserElement, str], retreat: int = 0) -> None: + super().__init__(expr) + self.expr = self.expr().leave_whitespace() + self._may_return_empty = True + self.mayIndexError = False + self.exact = False + if isinstance(expr, str_type): + expr = typing.cast(str, expr) + retreat = len(expr) + self.exact = True + elif isinstance(expr, (Literal, Keyword)): + retreat = expr.matchLen + self.exact = True + elif isinstance(expr, (Word, CharsNotIn)) and expr.maxLen != _MAX_INT: + retreat = expr.maxLen + self.exact = True + elif isinstance(expr, PositionToken): + retreat = 0 + self.exact = True + self.retreat = retreat + self.errmsg = f"not preceded by {expr}" + self.skipWhitespace = False + self.parseAction.append(lambda s, l, t: t.__delitem__(slice(None, None))) + + def parseImpl(self, instring, loc=0, do_actions=True) -> ParseImplReturnType: + if self.exact: + if loc < self.retreat: + raise ParseException(instring, loc, self.errmsg, self) + start = loc - self.retreat + _, ret = self.expr._parse(instring, start) + return loc, ret + + # retreat specified a maximum lookbehind window, iterate + test_expr = self.expr + StringEnd() + instring_slice = instring[max(0, loc - self.retreat) : loc] + last_expr: ParseBaseException = ParseException(instring, loc, self.errmsg, self) + + for offset in range(1, min(loc, self.retreat + 1) + 1): + try: + # print('trying', offset, instring_slice, repr(instring_slice[loc - offset:])) + _, ret = test_expr._parse(instring_slice, len(instring_slice) - offset) + except ParseBaseException as pbe: + last_expr = pbe + else: + break + else: + raise last_expr + + return loc, ret + + +class Located(ParseElementEnhance): + """ + Decorates a returned token with its starting and ending + locations in the input string. + + This helper adds the following results names: + + - ``locn_start`` - location where matched expression begins + - ``locn_end`` - location where matched expression ends + - ``value`` - the actual parsed results + + Be careful if the input text contains ```` characters, you + may want to call :class:`ParserElement.parse_with_tabs` + + Example: + + .. testcode:: + + wd = Word(alphas) + for match in Located(wd).search_string("ljsdf123lksdjjf123lkkjj1222"): + print(match) + + prints: + + .. testoutput:: + + [0, ['ljsdf'], 5] + [8, ['lksdjjf'], 15] + [18, ['lkkjj'], 23] + """ + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + # skip leading whitespace before capturing the start location, so + # locn_start marks the start of the match and not preceding whitespace, + # even when the wrapped expression delegates whitespace skipping to its + # sub-expressions (e.g. And/MatchFirst). Issue #621. + start = self.expr.preParse(instring, loc) + loc, tokens = self.expr._parse(instring, start, do_actions, callPreParse=False) + ret_tokens = ParseResults([start, tokens, loc]) + ret_tokens["locn_start"] = start + ret_tokens["value"] = tokens + ret_tokens["locn_end"] = loc + if self.resultsName: + # must return as a list, so that the name will be attached to the complete group + return loc, [ret_tokens] + else: + return loc, ret_tokens + + +class NotAny(ParseElementEnhance): + """ + Lookahead to disallow matching with the given parse expression. + ``NotAny`` does *not* advance the parsing position within the + input string, it only verifies that the specified parse expression + does *not* match at the current position. Also, ``NotAny`` does + *not* skip over leading whitespace. ``NotAny`` always returns + a null token list. May be constructed using the ``'~'`` operator. + + Example: + + .. testcode:: + + AND, OR, NOT = map(CaselessKeyword, "AND OR NOT".split()) + + # take care not to mistake keywords for identifiers + ident = ~(AND | OR | NOT) + Word(alphas) + boolean_term = Opt(NOT) + ident + + # very crude boolean expression - to support parenthesis groups and + # operation hierarchy, use infix_notation + boolean_expr = boolean_term + ((AND | OR) + boolean_term)[...] + + # integers that are followed by "." are actually floats + integer = Word(nums) + ~Char(".") + """ + + def __init__(self, expr: Union[ParserElement, str]) -> None: + super().__init__(expr) + # do NOT use self.leave_whitespace(), don't want to propagate to exprs + # self.leave_whitespace() + self.skipWhitespace = False + + self._may_return_empty = True + self.errmsg = f"Found unwanted token, {self.expr}" + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + if self.expr.can_parse_next(instring, loc, do_actions=do_actions): + raise ParseException(instring, loc, self.errmsg, self) + return loc, [] + + def _generateDefaultName(self) -> str: + return f"~{{{self.expr}}}" + + +class _MultipleMatch(ParseElementEnhance): + def __init__( + self, + expr: Union[str, ParserElement], + stop_on: typing.Optional[Union[ParserElement, str]] = None, + max: typing.Optional[int] = None, + **kwargs, + ) -> None: + stopOn: typing.Optional[Union[ParserElement, str]] = deprecate_argument( + kwargs, "stopOn", None + ) + + super().__init__(expr) + if max is not None and max <= 0: + raise ValueError("max must be greater than 0") + stopOn = stopOn or stop_on + self.saveAsList = True + self.max_count = max + ender = stopOn + if isinstance(ender, str_type): + ender = self._literalStringClass(ender) + self.stopOn(ender) + + def stop_on(self, ender) -> ParserElement: + if isinstance(ender, str_type): + ender = self._literalStringClass(ender) + self.not_ender = ~ender if ender is not None else None + return self + + stopOn = stop_on + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + self_expr_parse = self.expr._parse + self_skip_ignorables = self._skipIgnorables + check_ender = False + if self.not_ender is not None: + try_not_ender = self.not_ender.try_parse + check_ender = True + + # must be at least one (but first see if we are the stopOn sentinel; + # if so, fail) + if check_ender: + try_not_ender(instring, loc) + loc, tokens = self_expr_parse(instring, loc, do_actions) + match_count = 1 + try: + hasIgnoreExprs = not not self.ignoreExprs + while self.max_count is None or match_count < self.max_count: + if check_ender: + try_not_ender(instring, loc) + if hasIgnoreExprs: + preloc = self_skip_ignorables(instring, loc) + else: + preloc = loc + loc, tmptokens = self_expr_parse(instring, preloc, do_actions) + tokens += tmptokens + match_count += 1 + except (ParseException, IndexError): + pass + + return loc, tokens + + def _setResultsName(self, name, list_all_matches=False) -> ParserElement: + if ( + __diag__.warn_ungrouped_named_tokens_in_collection + and Diagnostics.warn_ungrouped_named_tokens_in_collection + not in self.suppress_warnings_ + ): + for e in [self.expr] + self.expr.recurse(): + if ( + isinstance(e, ParserElement) + and e.resultsName + and ( + Diagnostics.warn_ungrouped_named_tokens_in_collection + not in e.suppress_warnings_ + ) + ): + warning = ( + "warn_ungrouped_named_tokens_in_collection:" + f" setting results name {name!r} on {type(self).__name__} expression" + f" collides with {e.resultsName!r} on contained expression" + ) + warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3) + break + + return super()._setResultsName(name, list_all_matches) + + +class OneOrMore(_MultipleMatch): + """ + Repetition of one or more of the given expression. + + Parameters: + + - ``expr`` - expression that must match one or more times + - ``stop_on`` - (default= ``None``) - expression for a terminating sentinel + (only required if the sentinel would ordinarily match the repetition + expression) + + Example: + + .. doctest:: + + >>> data_word = Word(alphas) + >>> label = data_word + FollowedBy(':') + >>> attr_expr = Group( + ... label + Suppress(':') + ... + OneOrMore(data_word).set_parse_action(' '.join)) + + >>> text = "shape: SQUARE posn: upper left color: BLACK" + + # Fail! read 'posn' as data instead of next label + >>> attr_expr[1, ...].parse_string(text).pprint() + [['shape', 'SQUARE posn']] + + # use stop_on attribute for OneOrMore + # to avoid reading label string as part of the data + >>> attr_expr = Group( + ... label + Suppress(':') + ... + OneOrMore( + ... data_word, stop_on=label).set_parse_action(' '.join)) + >>> OneOrMore(attr_expr).parse_string(text).pprint() # Better + [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']] + + # could also be written as + >>> (attr_expr * (1,)).parse_string(text).pprint() + [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']] + """ + + def _generateDefaultName(self) -> str: + return f"{{{self.expr}}}..." + + +class ZeroOrMore(_MultipleMatch): + """ + Optional repetition of zero or more of the given expression. + + Parameters: + + - ``expr`` - expression that must match zero or more times + - ``stop_on`` - expression for a terminating sentinel + (only required if the sentinel would ordinarily match the repetition + expression) - (default= ``None``) + + Example: similar to :class:`OneOrMore` + """ + + def __init__( + self, + expr: Union[str, ParserElement], + stop_on: typing.Optional[Union[ParserElement, str]] = None, + max: typing.Optional[int] = None, + **kwargs, + ) -> None: + stopOn: Union[ParserElement, str] = deprecate_argument(kwargs, "stopOn", None) + + super().__init__(expr, stop_on=stopOn or stop_on, max=max) + self._may_return_empty = True + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + try: + return super().parseImpl(instring, loc, do_actions) + except (ParseException, IndexError): + return loc, ParseResults([], name=self.resultsName) + + def _generateDefaultName(self) -> str: + return f"[{self.expr}]..." + + +class DelimitedList(ParseElementEnhance): + """Helper to define a delimited list of expressions - the delimiter + defaults to ','. By default, the list elements and delimiters can + have intervening whitespace, and comments, but this can be + overridden by passing ``combine=True`` in the constructor. If + ``combine`` is set to ``True``, the matching tokens are + returned as a single token string, with the delimiters included; + otherwise, the matching tokens are returned as a list of tokens, + with the delimiters suppressed. + + If ``allow_trailing_delim`` is set to True, then the list may end with + a delimiter. + + Example: + + .. doctest:: + + >>> DelimitedList(Word(alphas)).parse_string("aa,bb,cc") + ParseResults(['aa', 'bb', 'cc'], {}) + >>> DelimitedList(Word(hexnums), delim=':', combine=True + ... ).parse_string("AA:BB:CC:DD:EE") + ParseResults(['AA:BB:CC:DD:EE'], {}) + + .. versionadded:: 3.1.0 + """ + + def __init__( + self, + expr: Union[str, ParserElement], + delim: Union[str, ParserElement] = ",", + combine: bool = False, + min: typing.Optional[int] = None, + max: typing.Optional[int] = None, + *, + allow_trailing_delim: bool = False, + ) -> None: + if isinstance(expr, str_type): + expr = ParserElement._literalStringClass(expr) + expr = typing.cast(ParserElement, expr) + + if min is not None and min < 1: + raise ValueError("min must be greater than 0") + + if max is not None and min is not None and max < min: + raise ValueError("max must be greater than, or equal to min") + + self.content = expr + self.raw_delim = str(delim) + self.delim = delim + self.combine = combine + if not combine: + self.delim = Suppress(delim) if not isinstance(delim, Suppress) else delim + self.min = min or 1 + self.max = max + self.allow_trailing_delim = allow_trailing_delim + + delim_list_expr = self.content + (self.delim + self.content) * ( + self.min - 1, + None if self.max is None else self.max - 1, + ) + if self.allow_trailing_delim: + delim_list_expr += Opt(self.delim) + + if self.combine: + delim_list_expr = Combine(delim_list_expr) + + super().__init__(delim_list_expr, savelist=True) + + def _generateDefaultName(self) -> str: + content_expr = self.content.streamline() + return f"{content_expr} [{self.raw_delim} {content_expr}]..." + + +class _NullToken: + def __bool__(self): + return False + + def __str__(self): + return "" + + +class Opt(ParseElementEnhance): + """ + Optional matching of the given expression. + + :param expr: expression that must match zero or more times + :param default: (optional) - value to be returned + if the optional expression is not found. + + Example: + + .. testcode:: + + # US postal code can be a 5-digit zip, plus optional 4-digit qualifier + zip = Combine(Word(nums, exact=5) + Opt('-' + Word(nums, exact=4))) + zip.run_tests(''' + # traditional ZIP code + 12345 + + # ZIP+4 form + 12101-0001 + + # invalid ZIP + 98765- + ''') + + prints: + + .. testoutput:: + :options: +NORMALIZE_WHITESPACE + + + # traditional ZIP code + 12345 + ['12345'] + + # ZIP+4 form + 12101-0001 + ['12101-0001'] + + # invalid ZIP + 98765- + 98765- + ^ + ParseException: Expected end of text, found '-' (at char 5), (line:1, col:6) + FAIL: Expected end of text, found '-' (at char 5), (line:1, col:6) + """ + + __optionalNotMatched = _NullToken() + + def __init__( + self, expr: Union[ParserElement, str], default: Any = __optionalNotMatched + ) -> None: + super().__init__(expr, savelist=False) + self.saveAsList = self.expr.saveAsList + self.defaultValue = default + self._may_return_empty = True + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + self_expr = self.expr + try: + loc, tokens = self_expr._parse( + instring, loc, do_actions, callPreParse=False + ) + except (ParseException, IndexError): + default_value = self.defaultValue + if default_value is not self.__optionalNotMatched: + if self_expr.resultsName: + tokens = ParseResults([default_value]) + tokens[self_expr.resultsName] = default_value + else: + tokens = [default_value] # type: ignore[assignment] + else: + tokens = [] # type: ignore[assignment] + return loc, tokens + + def _generateDefaultName(self) -> str: + inner = str(self.expr) + # strip off redundant inner {}'s + while len(inner) > 1 and inner[0 :: len(inner) - 1] == "{}": + inner = inner[1:-1] + return f"[{inner}]" + + +Optional = Opt + + +class SkipTo(ParseElementEnhance): + """ + Token for skipping over all undefined text until the matched + expression is found. + + :param expr: target expression marking the end of the data to be skipped + :param include: if ``True``, the target expression is also parsed + (the skipped text and target expression are returned + as a 2-element list) (default= ``False``). + + :param ignore: (default= ``None``) used to define grammars + (typically quoted strings and comments) + that might contain false matches to the target expression + + :param fail_on: (default= ``None``) define expressions that + are not allowed to be included in the skipped test; + if found before the target expression is found, + the :class:`SkipTo` is not a match + + Example: + + .. testcode:: + + report = ''' + Outstanding Issues Report - 1 Jan 2000 + + # | Severity | Description | Days Open + -----+----------+-------------------------------------------+----------- + 101 | Critical | Intermittent system crash | 6 + 94 | Cosmetic | Spelling error on Login ('log|n') | 14 + 79 | Minor | System slow when running too many reports | 47 + ''' + integer = Word(nums) + SEP = Suppress('|') + # use SkipTo to simply match everything up until the next SEP + # - ignore quoted strings, so that a '|' character inside a quoted string does not match + # - parse action will call token.strip() for each matched token, i.e., the description body + string_data = SkipTo(SEP, ignore=quoted_string) + string_data.set_parse_action(token_map(str.strip)) + ticket_expr = (integer("issue_num") + SEP + + string_data("sev") + SEP + + string_data("desc") + SEP + + integer("days_open")) + + for tkt in ticket_expr.search_string(report): + print(tkt.dump()) + + prints: + + .. testoutput:: + + ['101', 'Critical', 'Intermittent system crash', '6'] + - days_open: '6' + - desc: 'Intermittent system crash' + - issue_num: '101' + - sev: 'Critical' + ['94', 'Cosmetic', "Spelling error on Login ('log|n')", '14'] + - days_open: '14' + - desc: "Spelling error on Login ('log|n')" + - issue_num: '94' + - sev: 'Cosmetic' + ['79', 'Minor', 'System slow when running too many reports', '47'] + - days_open: '47' + - desc: 'System slow when running too many reports' + - issue_num: '79' + - sev: 'Minor' + """ + + def __init__( + self, + other: Union[ParserElement, str], + include: bool = False, + ignore: typing.Optional[Union[ParserElement, str]] = None, + fail_on: typing.Optional[Union[ParserElement, str]] = None, + **kwargs, + ) -> None: + failOn: typing.Optional[Union[ParserElement, str]] = deprecate_argument( + kwargs, "failOn", None + ) + + super().__init__(other) + failOn = failOn or fail_on + self.ignoreExpr = ignore + self._may_return_empty = True + self.mayIndexError = False + self.includeMatch = include + self.saveAsList = False + if isinstance(failOn, str_type): + self.failOn = self._literalStringClass(failOn) + else: + self.failOn = failOn + self.errmsg = f"No match found for {self.expr}" + self.ignorer = Empty().leave_whitespace() + self._update_ignorer() + + def _update_ignorer(self): + # rebuild internal ignore expr from current ignore exprs and assigned ignoreExpr + self.ignorer.ignoreExprs.clear() + for e in self.expr.ignoreExprs: + self.ignorer.ignore(e) + if self.ignoreExpr: + self.ignorer.ignore(self.ignoreExpr) + + def ignore(self, expr): + """ + Define expression to be ignored (e.g., comments) while doing pattern + matching; may be called repeatedly, to define multiple comment or other + ignorable patterns. + """ + super().ignore(expr) + self._update_ignorer() + + def parseImpl(self, instring, loc, do_actions=True): + startloc = loc + instrlen = len(instring) + self_expr_parse = self.expr._parse + self_failOn_canParseNext = ( + self.failOn.can_parse_next if self.failOn is not None else None + ) + ignorer_try_parse = self.ignorer.try_parse if self.ignorer.ignoreExprs else None + + tmploc = loc + while tmploc <= instrlen: + if self_failOn_canParseNext is not None: + # break if failOn expression matches + if self_failOn_canParseNext(instring, tmploc): + break + + if ignorer_try_parse is not None: + # advance past ignore expressions + prev_tmploc = tmploc + while 1: + try: + tmploc = ignorer_try_parse(instring, tmploc) + except ParseBaseException: + break + # see if all ignorers matched, but didn't actually ignore anything + if tmploc == prev_tmploc: + break + prev_tmploc = tmploc + + try: + self_expr_parse(instring, tmploc, do_actions=False, callPreParse=False) + except (ParseException, IndexError): + # no match, advance loc in string + tmploc += 1 + else: + # matched skipto expr, done + break + + else: + # ran off the end of the input string without matching skipto expr, fail + raise ParseException(instring, loc, self.errmsg, self) + + # build up return values + loc = tmploc + skiptext = instring[startloc:loc] + skipresult = ParseResults(skiptext) + + if self.includeMatch: + loc, mat = self_expr_parse(instring, loc, do_actions, callPreParse=False) + skipresult += mat + + return loc, skipresult + + +class Forward(ParseElementEnhance): + """ + Forward declaration of an expression to be defined later - + used for recursive grammars, such as algebraic infix notation. + When the expression is known, it is assigned to the ``Forward`` + instance using the ``'<<'`` operator. + + .. Note:: + + Take care when assigning to ``Forward`` not to overlook + precedence of operators. + + Specifically, ``'|'`` has a lower precedence than ``'<<'``, so that:: + + fwd_expr << a | b | c + + will actually be evaluated as:: + + (fwd_expr << a) | b | c + + thereby leaving b and c out as parseable alternatives. + It is recommended that you explicitly group the values + inserted into the :class:`Forward`:: + + fwd_expr << (a | b | c) + + Converting to use the ``'<<='`` operator instead will avoid this problem. + + See :meth:`ParseResults.pprint` for an example of a recursive + parser created using :class:`Forward`. + """ + + def __init__( + self, other: typing.Optional[Union[ParserElement, str]] = None + ) -> None: + self.caller_frame = traceback.extract_stack(limit=2)[0] + super().__init__(other, savelist=False) # type: ignore[arg-type] + self.lshift_line = None + + def __lshift__(self, other) -> Forward: + if hasattr(self, "caller_frame"): + del self.caller_frame + if isinstance(other, str_type): + other = self._literalStringClass(other) + + if not isinstance(other, ParserElement): + return NotImplemented + + self.expr = other + self.streamlined = other.streamlined + self.mayIndexError = self.expr.mayIndexError + self._may_return_empty = self.expr.mayReturnEmpty + self.set_whitespace_chars( + self.expr.whiteChars, copy_defaults=self.expr.copyDefaultWhiteChars + ) + self.skipWhitespace = self.expr.skipWhitespace + self.saveAsList = self.expr.saveAsList + self.ignoreExprs.extend(self.expr.ignoreExprs) + self.lshift_line = traceback.extract_stack(limit=2)[-2] # type: ignore[assignment] + return self + + def __ilshift__(self, other) -> Forward: + if not isinstance(other, ParserElement): + return NotImplemented + + return self << other + + def __or__(self, other) -> ParserElement: + caller_line = traceback.extract_stack(limit=2)[-2] + if ( + __diag__.warn_on_match_first_with_lshift_operator + and caller_line == self.lshift_line + and Diagnostics.warn_on_match_first_with_lshift_operator + not in self.suppress_warnings_ + ): + warnings.warn( + "warn_on_match_first_with_lshift_operator:" + " using '<<' operator with '|' is probably an error, use '<<='", + PyparsingDiagnosticWarning, + stacklevel=2, + ) + ret = super().__or__(other) + return ret + + def __del__(self): + # see if we are getting dropped because of '=' reassignment of var instead of '<<=' or '<<' + if ( + self.expr is None + and __diag__.warn_on_assignment_to_Forward + and Diagnostics.warn_on_assignment_to_Forward not in self.suppress_warnings_ + ): + warnings.warn_explicit( + "warn_on_assignment_to_Forward:" + " Forward defined here but no expression attached later using '<<=' or '<<'", + UserWarning, + filename=self.caller_frame.filename, + lineno=self.caller_frame.lineno, + ) + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + if ( + self.expr is None + and __diag__.warn_on_parse_using_empty_Forward + and Diagnostics.warn_on_parse_using_empty_Forward + not in self.suppress_warnings_ + ): + # walk stack until parse_string, scan_string, search_string, or transform_string is found + parse_fns = ( + "parse_string", + "scan_string", + "search_string", + "transform_string", + ) + tb = traceback.extract_stack(limit=200) + for i, frm in enumerate(reversed(tb), start=1): + if frm.name in parse_fns: + stacklevel = i + 1 + break + else: + stacklevel = 2 + warnings.warn( + "warn_on_parse_using_empty_Forward:" + " Forward expression was never assigned a value, will not parse any input", + PyparsingDiagnosticWarning, + stacklevel=stacklevel, + ) + if not ParserElement._left_recursion_enabled: + return super().parseImpl(instring, loc, do_actions) + # ## Bounded Recursion algorithm ## + # Recursion only needs to be processed at ``Forward`` elements, since they are + # the only ones that can actually refer to themselves. The general idea is + # to handle recursion stepwise: We start at no recursion, then recurse once, + # recurse twice, ..., until more recursion offers no benefit (we hit the bound). + # + # The "trick" here is that each ``Forward`` gets evaluated in two contexts + # - to *match* a specific recursion level, and + # - to *search* the bounded recursion level + # and the two run concurrently. The *search* must *match* each recursion level + # to find the best possible match. This is handled by a memo table, which + # provides the previous match to the next level match attempt. + # + # See also "Left Recursion in Parsing Expression Grammars", Medeiros et al. + # + # There is a complication since we not only *parse* but also *transform* via + # actions: We do not want to run the actions too often while expanding. Thus, + # we expand using `do_actions=False` and only run `do_actions=True` if the next + # recursion level is acceptable. + with ParserElement.recursion_lock: + memo = ParserElement.recursion_memos + try: + # we are parsing at a specific recursion expansion - use it as-is + prev_loc, prev_result = memo[loc, self, do_actions] + if isinstance(prev_result, Exception): + raise prev_result + return prev_loc, prev_result.copy() + except KeyError: + act_key = (loc, self, True) + peek_key = (loc, self, False) + # we are searching for the best recursion expansion - keep on improving + # both `do_actions` cases must be tracked separately here! + prev_loc, prev_peek = memo[peek_key] = ( + loc - 1, + ParseException( + instring, loc, "Forward recursion without base case", self + ), + ) + if do_actions: + memo[act_key] = memo[peek_key] + while True: + try: + new_loc, new_peek = super().parseImpl(instring, loc, False) + except ParseException: + # we failed before getting any match - do not hide the error + if isinstance(prev_peek, Exception): + raise + new_loc, new_peek = prev_loc, prev_peek + # the match did not get better: we are done + if new_loc <= prev_loc: + if do_actions: + # replace the match for do_actions=False as well, + # in case the action did backtrack + prev_loc, prev_result = memo[peek_key] = memo[act_key] + del memo[peek_key], memo[act_key] + return prev_loc, copy.copy(prev_result) + del memo[peek_key] + return prev_loc, copy.copy(prev_peek) + # the match did get better: see if we can improve further + if do_actions: + try: + memo[act_key] = super().parseImpl(instring, loc, True) + except ParseException as e: + memo[peek_key] = memo[act_key] = (new_loc, e) + raise + prev_loc, prev_peek = memo[peek_key] = new_loc, new_peek + + def leave_whitespace(self, recursive: bool = True) -> ParserElement: + """ + Extends ``leave_whitespace`` defined in base class. + """ + self.skipWhitespace = False + return self + + def ignore_whitespace(self, recursive: bool = True) -> ParserElement: + """ + Extends ``ignore_whitespace`` defined in base class. + """ + self.skipWhitespace = True + return self + + def streamline(self) -> ParserElement: + if not self.streamlined: + self.streamlined = True + if self.expr is not None: + self.expr.streamline() + return self + + def validate(self, validateTrace=None) -> None: + warnings.warn( + "ParserElement.validate() is deprecated, and should not be used to check for left recursion", + PyparsingDeprecationWarning, + stacklevel=2, + ) + if validateTrace is None: + validateTrace = [] + + if self not in validateTrace: + tmp = validateTrace[:] + [self] + if self.expr is not None: + self.expr.validate(tmp) + self._checkRecursion([]) + + def _generateDefaultName(self) -> str: + # Avoid infinite recursion by setting a temporary _defaultName + save_default_name = self._defaultName + self._defaultName = ": ..." + + # Use the string representation of main expression. + try: + if self.expr is not None: + ret_string = str(self.expr)[:1000] + else: + ret_string = "None" + except Exception: + ret_string = "..." + + self._defaultName = save_default_name + return f"{type(self).__name__}: {ret_string}" + + def copy(self) -> ParserElement: + """ + Returns a copy of this expression. + + Generally only used internally by pyparsing. + """ + if self.expr is not None: + return super().copy() + else: + ret = Forward() + ret <<= self + return ret + + def _setResultsName(self, name, list_all_matches=False) -> ParserElement: + # fmt: off + if ( + __diag__.warn_name_set_on_empty_Forward + and Diagnostics.warn_name_set_on_empty_Forward not in self.suppress_warnings_ + and self.expr is None + ): + warning = ( + "warn_name_set_on_empty_Forward:" + f" setting results name {name!r} on {type(self).__name__} expression" + " that has no contained expression" + ) + warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3) + # fmt: on + + return super()._setResultsName(name, list_all_matches) + + # Compatibility synonyms + # fmt: off + leaveWhitespace = replaced_by_pep8("leaveWhitespace", leave_whitespace) + ignoreWhitespace = replaced_by_pep8("ignoreWhitespace", ignore_whitespace) + # fmt: on + + +class TokenConverter(ParseElementEnhance): + """ + Abstract subclass of :class:`ParseElementEnhance`, for converting parsed results. + """ + + def __init__(self, expr: Union[ParserElement, str], savelist=False) -> None: + super().__init__(expr) # , savelist) + self.saveAsList = False + + +class Combine(TokenConverter): + """Converter to concatenate all matching tokens to a single string. + By default, the matching patterns must also be contiguous in the + input string; this can be disabled by specifying + ``'adjacent=False'`` in the constructor. + + Example: + + .. doctest:: + + >>> real = Word(nums) + '.' + Word(nums) + >>> print(real.parse_string('3.1416')) + ['3', '.', '1416'] + + >>> # will also erroneously match the following + >>> print(real.parse_string('3. 1416')) + ['3', '.', '1416'] + + >>> real = Combine(Word(nums) + '.' + Word(nums)) + >>> print(real.parse_string('3.1416')) + ['3.1416'] + + >>> # no match when there are internal spaces + >>> print(real.parse_string('3. 1416')) + Traceback (most recent call last): + ParseException: Expected W:(0123...) + """ + + def __init__( + self, + expr: ParserElement, + join_string: str = "", + adjacent: bool = True, + *, + joinString: typing.Optional[str] = None, + ) -> None: + super().__init__(expr) + joinString = joinString if joinString is not None else join_string + # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself + if adjacent: + self.leave_whitespace() + self.adjacent = adjacent + self.skipWhitespace = True + self.joinString = joinString + self.callPreparse = True + + def ignore(self, other) -> ParserElement: + """ + Define expression to be ignored (e.g., comments) while doing pattern + matching; may be called repeatedly, to define multiple comment or other + ignorable patterns. + """ + if self.adjacent: + ParserElement.ignore(self, other) + else: + super().ignore(other) + return self + + def postParse(self, instring, loc, tokenlist): + retToks = tokenlist.copy() + del retToks[:] + retToks += ParseResults( + ["".join(tokenlist._asStringList(self.joinString))], modal=self.modalResults + ) + + if self.resultsName and retToks.haskeys(): + return [retToks] + else: + return retToks + + +class Group(TokenConverter): + """Converter to return the matched tokens as a list - useful for + returning tokens of :class:`ZeroOrMore` and :class:`OneOrMore` expressions. + + The optional ``aslist`` argument when set to True will return the + parsed tokens as a Python list instead of a pyparsing ParseResults. + + Example: + + .. doctest:: + + >>> ident = Word(alphas) + >>> num = Word(nums) + >>> term = ident | num + >>> func = ident + Opt(DelimitedList(term)) + >>> print(func.parse_string("fn a, b, 100")) + ['fn', 'a', 'b', '100'] + + >>> func = ident + Group(Opt(DelimitedList(term))) + >>> print(func.parse_string("fn a, b, 100")) + ['fn', ['a', 'b', '100']] + """ + + def __init__(self, expr: ParserElement, aslist: bool = False) -> None: + super().__init__(expr) + self.saveAsList = True + self._asPythonList = aslist + + def postParse(self, instring, loc, tokenlist): + if self._asPythonList: + return ParseResults.List( + tokenlist.as_list() + if isinstance(tokenlist, ParseResults) + else list(tokenlist) + ) + + return [tokenlist] + + +class Dict(TokenConverter): + """Converter to return a repetitive expression as a list, but also + as a dictionary. Each element can also be referenced using the first + token in the expression as its key. Useful for tabular report + scraping when the first column can be used as a item key. + + The optional ``asdict`` argument when set to True will return the + parsed tokens as a Python dict instead of a pyparsing ParseResults. + + Example: + + .. doctest:: + + >>> data_word = Word(alphas) + >>> label = data_word + FollowedBy(':') + + >>> attr_expr = ( + ... label + Suppress(':') + ... + OneOrMore(data_word, stop_on=label) + ... .set_parse_action(' '.join) + ... ) + + >>> text = "shape: SQUARE posn: upper left color: light blue texture: burlap" + + >>> # print attributes as plain groups + >>> print(attr_expr[1, ...].parse_string(text).dump()) + ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap'] + + # instead of OneOrMore(expr), parse using Dict(Group(expr)[1, ...]) + # Dict will auto-assign names. + >>> result = Dict(Group(attr_expr)[1, ...]).parse_string(text) + >>> print(result.dump()) + [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']] + - color: 'light blue' + - posn: 'upper left' + - shape: 'SQUARE' + - texture: 'burlap' + [0]: + ['shape', 'SQUARE'] + [1]: + ['posn', 'upper left'] + [2]: + ['color', 'light blue'] + [3]: + ['texture', 'burlap'] + + # access named fields as dict entries, or output as dict + >>> print(result['shape']) + SQUARE + >>> print(result.as_dict()) + {'shape': 'SQUARE', 'posn': 'upper left', 'color': 'light blue', 'texture': 'burlap'} + + See more examples at :class:`ParseResults` of accessing fields by results name. + """ + + def __init__(self, expr: ParserElement, asdict: bool = False) -> None: + super().__init__(expr) + self.saveAsList = True + self._asPythonDict = asdict + + def postParse(self, instring, loc, tokenlist): + for i, tok in enumerate(tokenlist): + if len(tok) == 0: + continue + + ikey = tok[0] + if isinstance(ikey, int): + ikey = str(ikey).strip() + + # ParseResults are not hashable, cannot be used as a dict key + if isinstance(ikey, ParseResults): + continue + + if len(tok) == 1: + tokenlist[ikey] = _ParseResultsWithOffset("", i) + + elif len(tok) == 2 and not isinstance(tok[1], ParseResults): + tokenlist[ikey] = _ParseResultsWithOffset(tok[1], i) + + else: + try: + dictvalue = tok.copy() # ParseResults(i) + except Exception: + exc = TypeError( + "could not extract dict values from parsed results" + " - Dict expression must contain Grouped expressions" + ) + raise exc from None + + del dictvalue[0] + + if len(dictvalue) != 1 or ( + isinstance(dictvalue, ParseResults) and dictvalue.haskeys() + ): + tokenlist[ikey] = _ParseResultsWithOffset(dictvalue, i) + else: + tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0], i) + + # Mark the tokenlist so that as_dict() knows to serialize an empty + # result as {} rather than [], preserving dict semantics even when + # there are no matched entries. + tokenlist._is_dict_context = True + + if self._asPythonDict: + return [tokenlist.as_dict()] if self.resultsName else tokenlist.as_dict() + + return [tokenlist] if self.resultsName else tokenlist + + +class Suppress(TokenConverter): + """Converter for ignoring the results of a parsed expression. + + Example: + + .. doctest:: + + >>> source = "a, b, c,d" + >>> wd = Word(alphas) + >>> wd_list1 = wd + (',' + wd)[...] + >>> print(wd_list1.parse_string(source)) + ['a', ',', 'b', ',', 'c', ',', 'd'] + + # often, delimiters that are useful during parsing are just in the + # way afterward - use Suppress to keep them out of the parsed output + >>> wd_list2 = wd + (Suppress(',') + wd)[...] + >>> print(wd_list2.parse_string(source)) + ['a', 'b', 'c', 'd'] + + # Skipped text (using '...') can be suppressed as well + >>> source = "lead in START relevant text END trailing text" + >>> start_marker = Keyword("START") + >>> end_marker = Keyword("END") + >>> find_body = Suppress(...) + start_marker + ... + end_marker + >>> print(find_body.parse_string(source)) + ['START', 'relevant text ', 'END'] + + (See also :class:`DelimitedList`.) + """ + + def __init__(self, expr: Union[ParserElement, str], savelist: bool = False) -> None: + if expr is ...: + expr = _PendingSkip(NoMatch()) + super().__init__(expr) + + def __add__(self, other) -> ParserElement: + if isinstance(self.expr, _PendingSkip): + return Suppress(SkipTo(other)) + other + + return super().__add__(other) + + def __sub__(self, other) -> ParserElement: + if isinstance(self.expr, _PendingSkip): + return Suppress(SkipTo(other)) - other + + return super().__sub__(other) + + def postParse(self, instring, loc, tokenlist): + return [] + + def suppress(self) -> ParserElement: + return self + + +# XXX: Example needs to be re-done for updated output +def trace_parse_action(f: ParseAction) -> ParseAction: + """Decorator for debugging parse actions. + + When the parse action is called, this decorator will print + ``">> entering method-name(line:, , )"``. + When the parse action completes, the decorator will print + ``"<<"`` followed by the returned value, or any exception that the parse action raised. + + Example: + + .. testsetup:: stderr + + import sys + sys.stderr = sys.stdout + + .. testcleanup:: stderr + + sys.stderr = sys.__stderr__ + + .. testcode:: stderr + + wd = Word(alphas) + + @trace_parse_action + def remove_duplicate_chars(tokens): + return ''.join(sorted(set(''.join(tokens)))) + + wds = wd[1, ...].set_parse_action(remove_duplicate_chars) + print(wds.parse_string("slkdjs sld sldd sdlf sdljf")) + + prints: + + .. testoutput:: stderr + :options: +NORMALIZE_WHITESPACE + + >>entering remove_duplicate_chars(line: 'slkdjs sld sldd sdlf sdljf', + 0, ParseResults(['slkdjs', 'sld', 'sldd', 'sdlf', 'sdljf'], {})) + < 3: + thisFunc = f"{type(paArgs[0]).__name__}.{thisFunc}" + sys.stderr.write(f">>entering {thisFunc}(line: {line(l, s)!r}, {l}, {t!r})\n") + try: + ret = f(*paArgs) + except Exception as exc: + sys.stderr.write( + f"< str: + r"""Helper to easily define string ranges for use in :class:`Word` + construction. Borrows syntax from regexp ``'[]'`` string range + definitions:: + + srange("[0-9]") -> "0123456789" + srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz" + srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_" + + The input string must be enclosed in []'s, and the returned string + is the expanded character set joined into a single string. The + values enclosed in the []'s may be: + + - a single character + - an escaped character with a leading backslash (such as ``\-`` + or ``\]``) + - an escaped hex character with a leading ``'\x'`` + (``\x21``, which is a ``'!'`` character) (``\0x##`` + is also supported for backwards compatibility) + - an escaped octal character with a leading ``'\0'`` + (``\041``, which is a ``'!'`` character) + - a range of any of the above, separated by a dash (``'a-z'``, + etc.) + - any combination of the above (``'aeiouy'``, + ``'a-zA-Z0-9_$'``, etc.) + """ + + def _expanded(p): + if isinstance(p, ParseResults): + yield from (chr(c) for c in range(ord(p[0]), ord(p[1]) + 1)) + else: + yield p + + try: + return "".join( + [c for part in _reBracketExpr.parse_string(s).body for c in _expanded(part)] + ) + except Exception as e: + return "" + + +def token_map(func, *args) -> ParseAction: + """Helper to define a parse action by mapping a function to all + elements of a :class:`ParseResults` list. If any additional args are passed, + they are forwarded to the given function as additional arguments + after the token, as in + ``hex_integer = Word(hexnums).set_parse_action(token_map(int, 16))``, + which will convert the parsed data to an integer using base 16. + + Example (compare the last to example in :class:`ParserElement.transform_string`:: + + hex_ints = Word(hexnums)[1, ...].set_parse_action(token_map(int, 16)) + hex_ints.run_tests(''' + 00 11 22 aa FF 0a 0d 1a + ''') + + upperword = Word(alphas).set_parse_action(token_map(str.upper)) + upperword[1, ...].run_tests(''' + my kingdom for a horse + ''') + + wd = Word(alphas).set_parse_action(token_map(str.title)) + wd[1, ...].set_parse_action(' '.join).run_tests(''' + now is the winter of our discontent made glorious summer by this sun of york + ''') + + prints:: + + 00 11 22 aa FF 0a 0d 1a + [0, 17, 34, 170, 255, 10, 13, 26] + + my kingdom for a horse + ['MY', 'KINGDOM', 'FOR', 'A', 'HORSE'] + + now is the winter of our discontent made glorious summer by this sun of york + ['Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York'] + """ + + def pa(s, l, t): + return [func(tokn, *args) for tokn in t] + + func_name = getattr(func, "__name__", getattr(func, "__class__").__name__) + pa.__name__ = func_name + + return pa + + +def autoname_elements() -> None: + """ + Utility to simplify mass-naming of parser elements, for + generating railroad diagram with named subdiagrams. + """ + + # guard against _getframe not being implemented in the current Python + getframe_fn = getattr(sys, "_getframe", lambda _: None) + calling_frame = getframe_fn(1) + if calling_frame is None: + return + + # find all locals in the calling frame that are ParserElements + calling_frame = typing.cast(types.FrameType, calling_frame) + for name, var in calling_frame.f_locals.items(): + # if no custom name defined, set the name to the var name + if isinstance(var, ParserElement) and not var.customName: + var.set_name(name) + + +dbl_quoted_string = ( + Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*"') +).set_name("string enclosed in double quotes") + +sgl_quoted_string = Regex( + r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*'" +).set_name("string enclosed in single quotes") + +quoted_string = Combine( + Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*"').set_name( + "double quoted string" + ) + | Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*'").set_name( + "single quoted string" + ) +).set_name("quoted string using single or double quotes") + +# XXX: Is there some way to make this show up in API docs? +# .. versionadded:: 3.1.0 +python_quoted_string = Combine( + Regex(r'"""(?:[^"\\]|""(?!")|"(?!"")|\\.)*"""', flags=re.MULTILINE).set_name( + "multiline double quoted string" + ) + ^ Regex(r"'''(?:[^'\\]|''(?!')|'(?!'')|\\.)*'''", flags=re.MULTILINE).set_name( + "multiline single quoted string" + ) + ^ Regex(r'"(?:[^"\n\r\\]|(?:\\")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*"').set_name( + "double quoted string" + ) + ^ Regex(r"'(?:[^'\n\r\\]|(?:\\')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*'").set_name( + "single quoted string" + ) +).set_name("Python quoted string") + +unicode_string = Combine("u" + quoted_string.copy()).set_name("unicode string literal") + + +alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]") +punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]") + +# build list of built-in expressions, for future reference if a global default value +# gets updated +_builtin_exprs: list[ParserElement] = [ + v for v in vars().values() if isinstance(v, ParserElement) +] + +# Compatibility synonyms +# fmt: off +sglQuotedString = sgl_quoted_string +dblQuotedString = dbl_quoted_string +quotedString = quoted_string +unicodeString = unicode_string +lineStart = line_start +lineEnd = line_end +stringStart = string_start +stringEnd = string_end +nullDebugAction = replaced_by_pep8("nullDebugAction", null_debug_action) +traceParseAction = replaced_by_pep8("traceParseAction", trace_parse_action) +conditionAsParseAction = replaced_by_pep8("conditionAsParseAction", condition_as_parse_action) +tokenMap = replaced_by_pep8("tokenMap", token_map) +# fmt: on diff --git a/tests/test_unit.py b/tests/test_unit.py new file mode 100644 index 0000000..4bd5e70 --- /dev/null +++ b/tests/test_unit.py @@ -0,0 +1,12092 @@ +# +# test_unit.py +# +# Unit tests for pyparsing module +# +# Copyright 2002-2021, Paul McGuire +# +# +import collections +import contextlib +import datetime +import random +import re +import shlex +import sys +import sysconfig +import warnings +from types import SimpleNamespace +from io import StringIO +from textwrap import dedent +from typing import Any +import unittest +from unittest.mock import patch, mock_open +import subprocess + +import pyparsing as pp +from examples.jsonParser import jsonObject +from pyparsing import ( + ParserElement, + ParseException, + ParseFatalException, + PyparsingDeprecationWarning, +) +from tests.json_parser_tests import test1, test2, test3, test4, test5 +import platform + +python_full_version = sys.version_info +python_version = python_full_version[:2] + +ppc = pp.pyparsing_common +ppt = pp.pyparsing_test + +# see which Python implementation we are running +python_impl = platform.python_implementation() +CPYTHON_ENV = python_impl == "CPython" +IRON_PYTHON_ENV = python_impl == "IronPython" +JYTHON_ENV = python_impl == "Jython" +PYPY_ENV = python_impl == "PyPy" + +# global flags for Python config settings +_config_vars = sysconfig.get_config_vars() +_config_args = set(shlex.split(_config_vars.get("CONFIG_ARGS", ""))) +PYTHON_JIT_ENABLED = "--enable-experimental-jit" in _config_args +PYTHON_FREE_THREADED = _config_vars.get("Py_GIL_DISABLED", 0) == 1 + +# get full stack traces during testing +pp.ParserElement.verbose_stacktrace = True + + +# simple utility for flattening nested lists +def flatten(nested_list): + if not isinstance(nested_list, list): + return [nested_list] + if not nested_list: + return nested_list + return flatten(nested_list[0]) + flatten(nested_list[1:]) + + +class resetting: + def __init__(self, ob, attrname: str, *attrnames): + self.ob = ob + self.unset_attr = object() + self.save_attrs = [attrname, *attrnames] + self.save_values = [ + getattr(ob, name, self.unset_attr) for name in self.save_attrs + ] + + def __enter__(self): + pass + + def __exit__(self, *args): + for attr, value in zip(self.save_attrs, self.save_values): + if value is not self.unset_attr: + setattr(self.ob, attr, value) + else: + delattr(self.ob, attr) + + +def current_method_name(level=2): + import traceback + + stack = traceback.extract_stack(limit=level) + return stack[0].name + + +def __(): + return f"{current_method_name(3)}: " + + +class TestCase(unittest.TestCase): + @contextlib.contextmanager + def assertRaises(self, expected_exception_type: Any, msg: Any = None): + """ + Simple wrapper to print out the exceptions raised after assertRaises + """ + with super().assertRaises(expected_exception_type, msg=msg) as ar: + yield + + if getattr(ar, "exception", None) is not None: + print( + f"Raised expected exception: {type(ar.exception).__name__}: {ar.exception}" + ) + else: + print(f"Expected {expected_exception_type.__name__} exception not raised") + return ar + + @contextlib.contextmanager + def assertWarns(self, expected_warning_type: Any, msg: Any = None): + """ + Simple wrapper to print out the warnings raised after assertWarns + """ + with super().assertWarns(expected_warning_type, msg=msg) as ar: + yield + + if getattr(ar, "warning", None) is not None: + print(f"Raised expected warning: {type(ar.warning).__name__}: {ar.warning}") + else: + print(f"Expected {expected_warning_type.__name__} warning not raised") + return ar + + @contextlib.contextmanager + def assertDoesNotWarn(self, warning_type: type = UserWarning, msg: str = None): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("error") + try: + yield + except Exception as e: + if msg is None: + msg = f"unexpected warning {e} raised" + if isinstance(e, warning_type): + self.fail(f"{msg}: {e}") + else: + raise + finally: + warnings.simplefilter("default") + + @contextlib.contextmanager + def resetting_recursion_limit(self): + """Reset the recursion limit to its original value after + the context manager exits.""" + orig_recursion_limit = sys.getrecursionlimit() + try: + yield + finally: + sys.setrecursionlimit(orig_recursion_limit) + +class Test01_PyparsingTestInit(TestCase): + def runTest(self): + print( + "Beginning test of pyparsing, version", + pp.__version__, + pp.__version_time__, + ) + config_options = [] + if PYTHON_JIT_ENABLED: + config_options.append("JIT enabled") + if PYTHON_FREE_THREADED: + config_options.append("free_threaded") + config_options_str = f" ({','.join(config_options)})" + print( + f"Python version {sys.version}" + f"{config_options_str if config_options else ''}" + ) + print(f"__version_info__ : {pp.__version_info__}") + print(f"__version_info__ repr: {repr(pp.__version_info__)}") + + +class Test01a_PyparsingEnvironmentTests(TestCase): + def runTest(self): + # test warnings enable detection + # fmt: off + tests = [ + (([], "",), False), + ((["d", ], "",), True), + ((["d", "i:::pyparsing", ], "",), False), + ((["d:::pyparsing", ], "",), True), + ((["d:::pyparsing", "i", ], "",), False), + ((["d:::blah", ], "",), False), + ((["i", ], "",), False), + (([], "1",), True), + ((["d", ], "1",), True), + ((["d", "i:::pyparsing", ], "1",), False), + ((["d:::pyparsing", ], "1",), True), + ((["d:::pyparsing", "i", ], "1",), False), + ((["d:::blah", ], "1",), True), + ((["i", ], "1",), False), + ] + # fmt: on + + all_success = True + for args, expected in tests: + message = f"{args} should be {expected}" + print(message, end=" -> ") + actual = pp.core._should_enable_warnings(*args) + print("PASS" if actual == expected else "FAIL") + if actual != expected: + all_success = False + self.assertTrue(all_success, "failed warnings enable test") + + +class Test01b_PyparsingUnitTestUtilitiesTests(TestCase): + def runTest(self): + with ppt.reset_pyparsing_context(): + pp.enable_diag(pp.Diagnostics.warn_on_parse_using_empty_Forward) + + # test assertDoesNotWarn raises an AssertionError + with self.assertRaises(AssertionError): + with self.assertDoesNotWarn( + msg="warned when parsing with an empty Forward expression warning was suppressed", + ): + base = pp.Forward() + try: + print(base.parse_string("x")) + except ParseException as pe: + pass + + +class Test02_WithoutPackrat(ppt.TestParseResultsAsserts, TestCase): + suite_context = None + save_suite_context = None + + def setUp(self): + self.suite_context.restore() + + def test000_assert_packrat_status(self): + print("Packrat enabled:", ParserElement._packratEnabled) + self.assertFalse(ParserElement._packratEnabled, "packrat enabled") + + def testScanStringWithOverlap(self): + parser = pp.Word(pp.alphas, exact=3) + without_overlaps = sum( + t for t, s, e in parser.scan_string("ABCDEFGHI") + ).as_list() + self.assertEqual( + ["ABC", "DEF", "GHI"], + without_overlaps, + msg="scan_string without overlaps failed", + ) + with_overlaps = sum( + t for t, s, e in parser.scan_string("ABCDEFGHI", overlap=True) + ).as_list() + self.assertEqual( + ["ABC", "BCD", "CDE", "DEF", "EFG", "FGH", "GHI"], + with_overlaps, + msg="scan_string with overlaps failed", + ) + + def testCombineWithResultsNames(self): + # test case reproducing Issue #350 + from pyparsing import White, alphas, Word + + parser = White(" \t").set_results_name("indent") + Word( + alphas + ).set_results_name("word") + result = parser.parse_string(" test") + print(result.dump()) + self.assertParseResultsEquals( + result, [" ", "test"], {"indent": " ", "word": "test"} + ) + + parser = White(" \t") + Word(alphas).set_results_name("word") + result = parser.parse_string(" test") + print(result.dump()) + self.assertParseResultsEquals(result, [" ", "test"], {"word": "test"}) + + def testTransformString(self): + make_int_with_commas = ppc.integer().add_parse_action(lambda t: f"{t[0]:,}") + lower_case_words = pp.Word(pp.alphas.lower(), as_keyword=True) + pp.Optional( + pp.White() + ) + nested_list = pp.nested_expr().add_parse_action(pp.ParseResults.as_list) + transformer = make_int_with_commas | nested_list | lower_case_words.suppress() + + in_string = ( + "I wish to buy 12345 shares of Acme Industries (as a gift to my (ex)wife)" + ) + print(in_string) + out_string = transformer.transform_string(in_string) + print(out_string) + self.assertEqual( + "I 12,345 Acme Industries asagifttomyexwife", + out_string, + msg="failure in transform_string", + ) + + def testTransformStringWithLeadingWhitespace(self): + sample = "\n\ncheck" + sample = " check" + keywords = pp.one_of("aaa bbb", as_keyword=True) + ident = ~keywords + pp.Word(pp.alphas) + ident = pp.Combine(~keywords + pp.Word(pp.alphas)) + # ident.add_parse_action(lambda t: t[0].upper()) + ident.add_parse_action(ppc.upcase_tokens) + transformed = ident.transform_string(sample) + + print(ppt.with_line_numbers(sample)) + print(ppt.with_line_numbers(transformed)) + self.assertEqual(sample.replace("check", "CHECK"), transformed) + + def testTransformStringWithLeadingNotAny(self): + sample = "print a100" + keywords = set("print read".split()) + ident = pp.Word(pp.alphas, pp.alphanums).add_condition( + lambda t: t[0] not in keywords + ) + print(ident.search_string(sample)) + + def testTransformStringWithExpectedLeadingWhitespace(self): + sample1 = "\n\ncheck aaa" + sample2 = " check aaa" + keywords = pp.one_of("aaa bbb", as_keyword=True) + # This construct only works with parse_string, not with scan_string or its siblings + # ident = ~keywords + pp.Word(pp.alphas) + ident = pp.Word(pp.alphas) + ident.add_parse_action(ppc.upcase_tokens) + + for sample in sample1, sample2: + transformed = (keywords | ident).transform_string(sample) + print(ppt.with_line_numbers(sample)) + print(ppt.with_line_numbers(transformed)) + self.assertEqual(sample.replace("check", "CHECK"), transformed) + print() + + def testTransformStringWithLeadingWhitespaceFromTranslateProject(self): + from pyparsing import Keyword, Word, alphas, alphanums, Combine + + block_start = (Keyword("{") | Keyword("BEGIN")).set_name("block_start") + block_end = (Keyword("}") | Keyword("END")).set_name("block_end") + reserved_words = block_start | block_end + + # this is the first critical part of this test, an And with a leading NotAny + # This construct only works with parse_string, not with scan_string or its siblings + # name_id = ~reserved_words + Word(alphas, alphanums + "_").set_name("name_id") + name_id = Word(alphas, alphanums + "_").set_name("name_id") + + dialog = name_id("block_id") + (Keyword("DIALOGEX") | Keyword("DIALOG"))( + "block_type" + ) + string_table = Keyword("STRINGTABLE")("block_type") + + test_string = ( + """\r\nSTRINGTABLE\r\nBEGIN\r\n// Comment\r\nIDS_1 "Copied"\r\nEND\r\n""" + ) + print("Original:") + print(repr(test_string)) + print("Should match:") + # this is the second critical part of this test, an Or or MatchFirst including dialog + for parser in (dialog ^ string_table, dialog | string_table): + result = (reserved_words | parser).transform_string(test_string) + print(repr(result)) + self.assertEqual( + test_string, + result, + "Failed whitespace skipping with NotAny and MatchFirst/Or", + ) + + def testCuneiformTransformString(self): + + class Cuneiform(pp.unicode_set): + """Unicode set for Cuneiform Character Range""" + + _ranges: list[tuple[int, ...]] = [ + (0x10380, 0x103D5), + (0x12000, 0x123FF), + (0x12400, 0x1247F), + ] + + # define a MINIMAL Python parser + LPAR, RPAR, COLON, EQ = map(pp.Suppress, "():=") + def_ = pp.Keyword("𒁴𒈫", ident_chars=Cuneiform.identbodychars).set_name("def") + any_keyword = def_ + ident = (~any_keyword) + pp.Word( + Cuneiform.identchars, Cuneiform.identbodychars, as_keyword=True + ) + str_expr = pp.infix_notation( + pp.QuotedString('"') | pp.common.integer, + [ + ("*", 2, pp.OpAssoc.LEFT), + ("+", 2, pp.OpAssoc.LEFT), + ], + ) + + rvalue = pp.Forward() + fn_call = (ident + pp.Group(LPAR + pp.Optional(rvalue) + RPAR)).set_name( + "fn_call" + ) + + rvalue <<= fn_call | ident | str_expr | pp.common.number + assignment_stmt = ident + EQ + rvalue + + stmt = pp.Group(fn_call | assignment_stmt).set_name("stmt") + + fn_def = pp.Group( + def_ + ident + pp.Group(LPAR + pp.Optional(rvalue) + RPAR) + COLON + ).set_name("fn_def") + fn_body = pp.IndentedBlock(stmt).set_name("fn_body") + fn_expr = pp.Group(fn_def + pp.Group(fn_body)) + + script = fn_expr[...] + stmt[...] + + # parse some Python written in Cuneiform + cuneiform_hello_world = dedent( + r""" + 𒁴𒈫 𒀄𒂖𒆷𒁎(): + 𒀁 = "𒀄𒂖𒆷𒁎, 𒍟𒁎𒉿𒆷𒀳!\n" * 3 + 𒄑𒉿𒅔𒋫(𒀁) + + 𒀄𒂖𒆷𒁎() + """ + ) + + # use transform_string to convert keywords and builtins to runnable Python + names_map = { + "𒄑𒉿𒅔𒋫": "print", + } + ident.add_parse_action(lambda t: names_map.get(t[0], t[0])) + def_.add_parse_action(lambda: "def") + + print("\nconvert Cuneiform Python to executable Python") + transformed = ( + # always put ident last + (def_ | ident) + .ignore(pp.quoted_string) + .transform_string(cuneiform_hello_world) + ) + + expected = dedent( + r""" + def 𒀄𒂖𒆷𒁎(): + 𒀁 = "𒀄𒂖𒆷𒁎, 𒍟𒁎𒉿𒆷𒀳!\n" * 3 + print(𒀁) + + 𒀄𒂖𒆷𒁎() + """ + ) + + print( + "=================\n" + + cuneiform_hello_world # .strip() + + "\n=================\n" + + transformed + + "\n=================\n" + ) + + self.assertEqual(expected, transformed) + + def testUpdateDefaultWhitespace(self): + prev_default_whitespace_chars = pp.ParserElement.DEFAULT_WHITE_CHARS + try: + pp.dbl_quoted_string.copyDefaultWhiteChars = False + pp.ParserElement.set_default_whitespace_chars(" \t") + self.assertEqual( + set(" \t"), + set(pp.sgl_quoted_string.whiteChars), + "set_default_whitespace_chars did not update sgl_quoted_string", + ) + self.assertEqual( + set(prev_default_whitespace_chars), + set(pp.dbl_quoted_string.whiteChars), + "set_default_whitespace_chars updated dbl_quoted_string but should not", + ) + finally: + pp.dbl_quoted_string.copyDefaultWhiteChars = True + pp.ParserElement.set_default_whitespace_chars(prev_default_whitespace_chars) + + self.assertEqual( + set(prev_default_whitespace_chars), + set(pp.dbl_quoted_string.whiteChars), + "set_default_whitespace_chars updated dbl_quoted_string", + ) + + with ppt.reset_pyparsing_context(): + pp.ParserElement.set_default_whitespace_chars(" \t") + self.assertNotEqual( + set(prev_default_whitespace_chars), + set(pp.dbl_quoted_string.whiteChars), + "set_default_whitespace_chars updated dbl_quoted_string but should not", + ) + + EOL = pp.LineEnd().suppress().set_name("EOL") + + # Identifiers is a string + optional $ + identifier = pp.Combine(pp.Word(pp.alphas) + pp.Optional("$")) + + # Literals (number or double quoted string) + literal = ppc.number | pp.dbl_quoted_string + expression = literal | identifier + # expression.set_name("expression").set_debug() + # ppc.number.set_debug() + # ppc.integer.set_debug() + + line_number = ppc.integer + + # Keywords + PRINT = pp.CaselessKeyword("print") + print_stmt = PRINT - pp.ZeroOrMore(expression | ";") + statement = print_stmt + code_line = pp.Group(line_number + statement + EOL) + program = pp.ZeroOrMore(code_line) + + test = """\ + 10 print 123; + 20 print 234; 567; + 30 print 890 + """ + + parsed_program = program.parse_string(test, parse_all=True) + print(parsed_program.dump()) + self.assertEqual( + 3, + len(parsed_program), + "failed to apply new whitespace chars to existing builtins", + ) + + def testUpdateDefaultWhitespace2(self): + with ppt.reset_pyparsing_context(): + expr_tests = [ + (pp.dbl_quoted_string, '"abc"'), + (pp.sgl_quoted_string, "'def'"), + (ppc.integer, "123"), + (ppc.number, "4.56"), + (ppc.identifier, "a_bc"), + ] + NL = pp.LineEnd() + + for expr, test_str in expr_tests: + parser = pp.Group(expr[1, ...] + pp.Optional(NL))[1, ...] + test_string = "\n".join([test_str] * 3) + result = parser.parse_string(test_string, parse_all=True) + print(result.dump()) + self.assertEqual(1, len(result), f"failed {test_string!r}") + + pp.ParserElement.set_default_whitespace_chars(" \t") + + for expr, test_str in expr_tests: + parser = pp.Group(expr[1, ...] + pp.Optional(NL))[1, ...] + test_string = "\n".join([test_str] * 3) + result = parser.parse_string(test_string, parse_all=True) + print(result.dump()) + self.assertEqual(3, len(result), f"failed {test_string!r}") + + pp.ParserElement.set_default_whitespace_chars(" \n\t") + + for expr, test_str in expr_tests: + parser = pp.Group(expr[1, ...] + pp.Optional(NL))[1, ...] + test_string = "\n".join([test_str] * 3) + result = parser.parse_string(test_string, parse_all=True) + print(result.dump()) + self.assertEqual(1, len(result), f"failed {test_string!r}") + + def testParseFourFn(self): + import examples.fourFn as fourFn + import math + + def test(s, ans): + fourFn.exprStack[:] = [] + results = fourFn.BNF().parse_string(s, parse_all=True) + try: + resultValue = fourFn.evaluate_stack(fourFn.exprStack) + except Exception: + self.assertIsNone(ans, f"exception raised for expression {s!r}") + else: + self.assertEqual( + ans, + resultValue, + f"failed to evaluate {s}, got {resultValue:f}", + ) + print(s, "->", resultValue) + + test("9", 9) + test("-9", -9) + test("--9", 9) + test("-E", -math.e) + test("9 + 3 + 5", 9 + 3 + 5) + test("9 + 3 / 11", 9 + 3.0 / 11) + test("(9 + 3)", (9 + 3)) + test("(9+3) / 11", (9 + 3.0) / 11) + test("9 - 12 - 6", 9 - 12 - 6) + test("9 - (12 - 6)", 9 - (12 - 6)) + test("2*3.14159", 2 * 3.14159) + test("3.1415926535*3.1415926535 / 10", 3.1415926535 * 3.1415926535 / 10) + test("PI * PI / 10", math.pi * math.pi / 10) + test("PI*PI/10", math.pi * math.pi / 10) + test("PI^2", math.pi**2) + test("round(PI^2)", round(math.pi**2)) + test("6.02E23 * 8.048", 6.02e23 * 8.048) + test("e / 3", math.e / 3) + test("sin(PI/2)", math.sin(math.pi / 2)) + test("10+sin(PI/4)^2", 10 + math.sin(math.pi / 4) ** 2) + test("trunc(E)", int(math.e)) + test("trunc(-E)", int(-math.e)) + test("round(E)", round(math.e)) + test("round(-E)", round(-math.e)) + test("E^PI", math.e**math.pi) + test("exp(0)", 1) + test("exp(1)", math.e) + test("2^3^2", 2**3**2) + test("(2^3)^2", (2**3) ** 2) + test("2^3+2", 2**3 + 2) + test("2^3+5", 2**3 + 5) + test("2^9", 2**9) + test("sgn(-2)", -1) + test("sgn(0)", 0) + test("sgn(0.1)", 1) + test("foo(0.1)", None) + test("round(E, 3)", round(math.e, 3)) + test("round(PI^2, 3)", round(math.pi**2, 3)) + test("sgn(cos(PI/4))", 1) + test("sgn(cos(PI/2))", 0) + test("sgn(cos(PI*3/4))", -1) + test("+(sgn(cos(PI/4)))", 1) + test("-(sgn(cos(PI/4)))", -1) + + def testParseSQL(self): + # SQL parser uses packrat parsing, not compatible with LR + if ParserElement._left_recursion_enabled: + return + + import examples.simpleSQL as simpleSQL + + def test(s, num_expected_toks, expected_errloc=-1): + try: + sqlToks = flatten( + simpleSQL.simpleSQL.parse_string(s, parse_all=True).as_list() + ) + print(s, sqlToks, len(sqlToks)) + self.assertEqual( + num_expected_toks, + len(sqlToks), + f"invalid parsed tokens, expected {num_expected_toks}, found {len(sqlToks)} ({sqlToks})", + ) + except ParseException as e: + if expected_errloc >= 0: + self.assertEqual( + expected_errloc, + e.loc, + f"expected error at {expected_errloc}, found at {e.loc}", + ) + + test("SELECT * from XYZZY, ABC", 6) + test("select * from SYS.XYZZY", 5) + test("Select A from Sys.dual", 5) + test("Select A,B,C from Sys.dual", 7) + test("Select A, B, C from Sys.dual", 7) + test("Select A, B, C from Sys.dual, Table2 ", 8) + test("Xelect A, B, C from Sys.dual", 0, 0) + test("Select A, B, C frox Sys.dual", 0, 15) + test("Select", 0, 6) + test("Select &&& frox Sys.dual", 0, 7) + test("Select A from Sys.dual where a in ('RED','GREEN','BLUE')", 12) + test( + "Select A from Sys.dual where a in ('RED','GREEN','BLUE') and b in (10,20,30)", + 20, + ) + test( + "Select A,b from table1,table2 where table1.id eq table2.id -- test out comparison operators", + 10, + ) + + def testParseConfigFile(self): + from examples import configParse + + def test(fnam, num_expected_toks, resCheckList): + print("Parsing", fnam, "...", end=" ") + with open(fnam) as infile: + iniFileLines = "\n".join(infile.read().splitlines()) + iniData = configParse.inifile_BNF().parse_string( + iniFileLines, parse_all=True + ) + print(len(flatten(iniData.as_list()))) + print(list(iniData.keys())) + self.assertEqual( + num_expected_toks, + len(flatten(iniData.as_list())), + f"file {fnam} not parsed correctly", + ) + for chkkey, chkexpect in resCheckList: + var = iniData + for attr in chkkey.split("."): + var = getattr(var, attr) + print(chkkey, var, chkexpect) + self.assertEqual( + chkexpect, + var, + f"ParseConfigFileTest: failed to parse ini {chkkey!r} as expected {chkexpect!r}, found {var}", + ) + print("OK") + + test( + "tests/karthik.ini", + 23, + [("users.K", "8"), ("users.mod_scheme", "'QPSK'"), ("users.Na", "K+2")], + ) + test( + "examples/Setup.ini", + 125, + [ + ("Startup.audioinf", "M3i"), + ("Languages.key1", "0x0003"), + ("test.foo", "bar"), + ], + ) + + def testParseJSONData(self): + expected = [ + { + "glossary": { + "GlossDiv": { + "GlossList": [ + { + "Abbrev": "ISO 8879:1986", + "Acronym": "SGML", + "AvogadroNumber": 6.02e23, + "EmptyDict": {}, + "EmptyList": [], + "EvenPrimesGreaterThan2": [], + "FermatTheoremInMargin": False, + "GlossDef": "A meta-markup language, " + "used to create markup " + "languages such as " + "DocBook.", + "GlossSeeAlso": ["GML", "XML", "markup"], + "GlossTerm": "Standard Generalized " "Markup Language", + "ID": "SGML", + "LargestPrimeLessThan100": 97, + "MapRequiringFiveColors": None, + "PrimesLessThan10": [2, 3, 5, 7], + "SortAs": "SGML", + } + ], + "title": "S", + }, + "title": "example glossary", + } + }, + { + "menu": { + "id": "file", + "popup": { + "menuitem": [ + {"onclick": "CreateNewDoc()", "value": "New"}, + {"onclick": "OpenDoc()", "value": "Open"}, + {"onclick": "CloseDoc()", "value": "Close"}, + ] + }, + "value": "File:", + } + }, + { + "widget": { + "debug": "on", + "image": { + "alignment": "center", + "hOffset": 250, + "name": "sun1", + "src": "Images/Sun.png", + "vOffset": 250, + }, + "text": { + "alignment": "center", + "data": "Click Here", + "hOffset": 250, + "name": "text1", + "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;", + "size": 36, + "style": "bold", + "vOffset": 100, + }, + "window": { + "height": 500, + "name": "main_window", + "title": "Sample Konfabulator Widget", + "width": 500, + }, + } + }, + { + "web-app": { + "servlet": [ + { + "init-param": { + "cachePackageTagsRefresh": 60, + "cachePackageTagsStore": 200, + "cachePackageTagsTrack": 200, + "cachePagesDirtyRead": 10, + "cachePagesRefresh": 10, + "cachePagesStore": 100, + "cachePagesTrack": 200, + "cacheTemplatesRefresh": 15, + "cacheTemplatesStore": 50, + "cacheTemplatesTrack": 100, + "configGlossary:adminEmail": "ksm@pobox.com", + "configGlossary:installationAt": "Philadelphia, " "PA", + "configGlossary:poweredBy": "Cofax", + "configGlossary:poweredByIcon": "/images/cofax.gif", + "configGlossary:staticPath": "/content/static", + "dataStoreClass": "org.cofax.SqlDataStore", + "dataStoreConnUsageLimit": 100, + "dataStoreDriver": "com.microsoft.jdbc.sqlserver.SQLServerDriver", + "dataStoreInitConns": 10, + "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log", + "dataStoreLogLevel": "debug", + "dataStoreMaxConns": 100, + "dataStoreName": "cofax", + "dataStorePassword": "dataStoreTestQuery", + "dataStoreTestQuery": "SET NOCOUNT " + "ON;select " + "test='test';", + "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon", + "dataStoreUser": "sa", + "defaultFileTemplate": "articleTemplate.htm", + "defaultListTemplate": "listTemplate.htm", + "jspFileTemplate": "articleTemplate.jsp", + "jspListTemplate": "listTemplate.jsp", + "maxUrlLength": 500, + "redirectionClass": "org.cofax.SqlRedirection", + "searchEngineFileTemplate": "forSearchEngines.htm", + "searchEngineListTemplate": "forSearchEnginesList.htm", + "searchEngineRobotsDb": "WEB-INF/robots.db", + "templateLoaderClass": "org.cofax.FilesTemplateLoader", + "templateOverridePath": "", + "templatePath": "templates", + "templateProcessorClass": "org.cofax.WysiwygTemplate", + "useDataStore": True, + "useJSP": False, + }, + "servlet-class": "org.cofax.cds.CDSServlet", + "servlet-name": "cofaxCDS", + }, + { + "init-param": { + "mailHost": "mail1", + "mailHostOverride": "mail2", + }, + "servlet-class": "org.cofax.cds.EmailServlet", + "servlet-name": "cofaxEmail", + }, + { + "servlet-class": "org.cofax.cds.AdminServlet", + "servlet-name": "cofaxAdmin", + }, + { + "servlet-class": "org.cofax.cds.FileServlet", + "servlet-name": "fileServlet", + }, + { + "init-param": { + "adminGroupID": 4, + "betaServer": True, + "dataLog": 1, + "dataLogLocation": "/usr/local/tomcat/logs/dataLog.log", + "dataLogMaxSize": "", + "fileTransferFolder": "/usr/local/tomcat/webapps/content/fileTransferFolder", + "log": 1, + "logLocation": "/usr/local/tomcat/logs/CofaxTools.log", + "logMaxSize": "", + "lookInContext": 1, + "removePageCache": "/content/admin/remove?cache=pages&id=", + "removeTemplateCache": "/content/admin/remove?cache=templates&id=", + "templatePath": "toolstemplates/", + }, + "servlet-class": "org.cofax.cms.CofaxToolsServlet", + "servlet-name": "cofaxTools", + }, + ], + "servlet-mapping": { + "cofaxAdmin": "/admin/*", + "cofaxCDS": "/", + "cofaxEmail": "/cofaxutil/aemail/*", + "cofaxTools": "/tools/*", + "fileServlet": "/static/*", + }, + "taglib": { + "taglib-location": "/WEB-INF/tlds/cofax.tld", + "taglib-uri": "cofax.tld", + }, + } + }, + { + "menu": { + "header": "SVG Viewer", + "items": [ + {"id": "Open"}, + {"id": "OpenNew", "label": "Open New"}, + None, + {"id": "ZoomIn", "label": "Zoom In"}, + {"id": "ZoomOut", "label": "Zoom Out"}, + {"id": "OriginalView", "label": "Original View"}, + None, + {"id": "Quality"}, + {"id": "Pause"}, + {"id": "Mute"}, + None, + {"id": "Find", "label": "Find..."}, + {"id": "FindAgain", "label": "Find Again"}, + {"id": "Copy"}, + {"id": "CopyAgain", "label": "Copy Again"}, + {"id": "CopySVG", "label": "Copy SVG"}, + {"id": "ViewSVG", "label": "View SVG"}, + {"id": "ViewSource", "label": "View Source"}, + {"id": "SaveAs", "label": "Save As"}, + None, + {"id": "Help"}, + {"id": "About", "label": "About Adobe CVG Viewer..."}, + ], + } + }, + ] + + for t, exp_result in zip((test1, test2, test3, test4, test5), expected): + result = jsonObject.parse_string(t, parse_all=True) + self.assertEqual(exp_result, result[0]) + + def testParseCommaSeparatedValues(self): + testData = [ + "a,b,c,100.2,,3", + "d, e, j k , m ", + "'Hello, World', f, g , , 5.1,x", + "John Doe, 123 Main St., Cleveland, Ohio", + "Jane Doe, 456 St. James St., Los Angeles , California ", + "", + ] + testVals = [ + [(3, "100.2"), (4, ""), (5, "3")], + [(2, "j k"), (3, "m")], + [(0, "'Hello, World'"), (2, "g"), (3, "")], + [(0, "John Doe"), (1, "123 Main St."), (2, "Cleveland"), (3, "Ohio")], + [ + (0, "Jane Doe"), + (1, "456 St. James St."), + (2, "Los Angeles"), + (3, "California"), + ], + ] + for line, tests in zip(testData, testVals): + print(f"Parsing: {line!r} ->", end=" ") + results = ppc.comma_separated_list.parse_string(line, parse_all=True) + print(results) + for t in tests: + if not (len(results) > t[0] and results[t[0]] == t[1]): + print("$$$", results.dump()) + print("$$$", results[0]) + self.assertTrue( + len(results) > t[0] and results[t[0]] == t[1], + f"failed on {line}, item {t[0]:d} s/b '{t[1]}', got '{results.as_list()}'", + ) + + def testParseEBNF(self): + from examples import ebnf + + print("Constructing EBNF parser with pyparsing...") + + grammar = """ + syntax = (syntax_rule), {(syntax_rule)}; + syntax_rule = meta_identifier, '=', definitions_list, ';'; + definitions_list = single_definition, {'|', single_definition}; + single_definition = syntactic_term, {',', syntactic_term}; + syntactic_term = syntactic_factor,['-', syntactic_factor]; + syntactic_factor = [integer, '*'], syntactic_primary; + syntactic_primary = optional_sequence | repeated_sequence | + grouped_sequence | meta_identifier | terminal_string; + optional_sequence = '[', definitions_list, ']'; + repeated_sequence = '{', definitions_list, '}'; + grouped_sequence = '(', definitions_list, ')'; + (* + terminal_string = "'", character - "'", {character - "'"}, "'" | + '"', character - '"', {character - '"'}, '"'; + meta_identifier = letter, {letter | digit}; + integer = digit, {digit}; + *) + """ + + table = {} + table["terminal_string"] = pp.quoted_string + table["meta_identifier"] = pp.Word(pp.alphas + "_", pp.alphas + "_" + pp.nums) + table["integer"] = pp.Word(pp.nums) + + print("Parsing EBNF grammar with EBNF parser...") + parsers = ebnf.parse(grammar, table) + ebnf_parser = parsers["syntax"] + ebnf_comment = pp.Literal("(*") + ... + "*)" + ebnf_parser.ignore(ebnf_comment) + print("-", "\n- ".join(parsers.keys())) + self.assertEqual( + 13, len(list(parsers.keys())), "failed to construct syntax grammar" + ) + + print("Parsing EBNF grammar with generated EBNF parser...") + parsed_chars = ebnf_parser.parse_string(grammar, parse_all=True) + parsed_char_len = len(parsed_chars) + + print("],\n".join(str(parsed_chars.as_list()).split("],"))) + self.assertEqual( + 98, + len(flatten(parsed_chars.as_list())), + "failed to tokenize grammar correctly", + ) + + def testParseEBNFmissingDefinitions(self): + """ + Test detection of missing definitions in EBNF + """ + from examples import ebnf + + grammar = """ + (* + EBNF for number_words.py + *) + number = [thousands, [and]], [hundreds, [and]], [one_to_99]; + """ + + with self.assertRaisesRegex( + AssertionError, + r"Missing definitions for \['thousands', 'and', 'hundreds', 'one_to_99']", + ): + ebnf.parse(grammar) + + def testParseIDL(self): + from examples import idlParse + + def test(strng, numToks, expectedErrloc=0): + print(strng) + try: + bnf = idlParse.CORBA_IDL_BNF() + tokens = bnf.parse_string(strng, parse_all=True) + print("tokens = ") + tokens.pprint() + tokens = flatten(tokens.as_list()) + print(len(tokens)) + self.assertEqual( + numToks, + len(tokens), + f"error matching IDL string, {strng} -> {tokens}", + ) + except ParseException as err: + print(err.line) + print(f"{' ' * (err.column - 1)}^") + print(err) + self.assertEqual( + 0, + numToks, + f"unexpected ParseException while parsing {strng}, {err}", + ) + self.assertEqual( + expectedErrloc, + err.loc, + f"expected ParseException at {expectedErrloc}, found exception at {err.loc}", + ) + + test( + """ + /* + * a block comment * + */ + typedef string[10] tenStrings; + typedef sequence stringSeq; + typedef sequence< sequence > stringSeqSeq; + + interface QoSAdmin { + stringSeq method1(in string arg1, inout long arg2); + stringSeqSeq method2(in string arg1, inout long arg2, inout long arg3); + string method3(); + }; + """, + 59, + ) + test( + """ + /* + * a block comment * + */ + typedef string[10] tenStrings; + typedef + /** ** *** **** * + * a block comment * + */ + sequence /*comment inside an And */ stringSeq; + /* */ /**/ /***/ /****/ + typedef sequence< sequence > stringSeqSeq; + + interface QoSAdmin { + stringSeq method1(in string arg1, inout long arg2); + stringSeqSeq method2(in string arg1, inout long arg2, inout long arg3); + string method3(); + }; + """, + 59, + ) + test( + r""" + const string test="Test String\n"; + const long a = 0; + const long b = -100; + const float c = 3.14159; + const long d = 0x007f7f7f; + exception TestException + { + string msg; + sequence dataStrings; + }; + + interface TestInterface + { + void method1(in string arg1, inout long arg2); + }; + """, + 60, + ) + test( + """ + module Test1 + { + exception TestException + { + string msg; + ]; + + interface TestInterface + { + void method1(in string arg1, inout long arg2) + raises (TestException); + }; + }; + """, + 0, + 56, + ) + test( + """ + module Test1 + { + exception TestException + { + string msg; + }; + + }; + """, + 13, + ) + + def testParseVerilog(self): + pass + + def testScanString(self): + testdata = """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameIP AddressLocation
time-a.nist.gov129.6.15.28NIST, Gaithersburg, Maryland
time-b.nist.gov129.6.15.29NIST, Gaithersburg, Maryland
time-a.timefreq.bldrdoc.gov132.163.4.101NIST, Boulder, Colorado
time-b.timefreq.bldrdoc.gov132.163.4.102NIST, Boulder, Colorado
time-c.timefreq.bldrdoc.gov132.163.4.103NIST, Boulder, Colorado
+ """ + integer = pp.Word(pp.nums) + ipAddress = pp.Combine(integer + "." + integer + "." + integer + "." + integer) + tdStart = pp.Suppress("") + tdEnd = pp.Suppress("") + timeServerPattern = ( + tdStart + + ipAddress("ipAddr") + + tdEnd + + tdStart + + pp.CharsNotIn("<")("loc") + + tdEnd + ) + servers = [ + srvr.ipAddr + for srvr, startloc, endloc in timeServerPattern.scan_string(testdata) + ] + + print(servers) + self.assertEqual( + [ + "129.6.15.28", + "129.6.15.29", + "132.163.4.101", + "132.163.4.102", + "132.163.4.103", + ], + servers, + "failed scan_string()", + ) + + servers = [ + srvr.ipAddr + for srvr, startloc, endloc in timeServerPattern.scan_string( + testdata, max_matches=3 + ) + ] + + self.assertEqual( + [ + "129.6.15.28", + "129.6.15.29", + "132.163.4.101", + ], + servers, + "failed scan_string() with max_matches=3", + ) + + # test for string_end detection in scan_string + foundStringEnds = [r for r in pp.StringEnd().scan_string("xyzzy")] + print(foundStringEnds) + self.assertTrue(foundStringEnds, "Failed to find StringEnd in scan_string") + + def testQuotedStrings(self): + testData = """ + 'a valid single quoted string' + 'an invalid single quoted string + because it spans lines' + "a valid double quoted string" + "an invalid double quoted string + because it spans lines" + """ + print(testData) + + with self.subTest(): + sglStrings = [ + (t[0], b, e) for (t, b, e) in pp.sgl_quoted_string.scan_string(testData) + ] + print(sglStrings) + self.assertTrue( + len(sglStrings) == 1 + and (sglStrings[0][1] == 17 and sglStrings[0][2] == 47), + "single quoted string failure", + ) + + with self.subTest(): + dblStrings = [ + (t[0], b, e) for (t, b, e) in pp.dbl_quoted_string.scan_string(testData) + ] + print(dblStrings) + self.assertTrue( + len(dblStrings) == 1 + and (dblStrings[0][1] == 154 and dblStrings[0][2] == 184), + "double quoted string failure", + ) + + with self.subTest(): + allStrings = [ + (t[0], b, e) for (t, b, e) in pp.quoted_string.scan_string(testData) + ] + print(allStrings) + self.assertTrue( + len(allStrings) == 2 + and (allStrings[0][1] == 17 and allStrings[0][2] == 47) + and (allStrings[1][1] == 154 and allStrings[1][2] == 184), + "quoted string failure", + ) + + escapedQuoteTest = r""" + 'This string has an escaped (\') quote character' + "This string has an escaped (\") quote character" + """ + + with self.subTest(): + sglStrings = [ + (t[0], b, e) + for (t, b, e) in pp.sgl_quoted_string.scan_string(escapedQuoteTest) + ] + print(sglStrings) + self.assertTrue( + len(sglStrings) == 1 + and (sglStrings[0][1] == 17 and sglStrings[0][2] == 66), + f"single quoted string escaped quote failure ({sglStrings[0]})", + ) + + with self.subTest(): + dblStrings = [ + (t[0], b, e) + for (t, b, e) in pp.dbl_quoted_string.scan_string(escapedQuoteTest) + ] + print(dblStrings) + self.assertTrue( + len(dblStrings) == 1 + and (dblStrings[0][1] == 83 and dblStrings[0][2] == 132), + f"double quoted string escaped quote failure ({dblStrings[0]})", + ) + + with self.subTest(): + allStrings = [ + (t[0], b, e) + for (t, b, e) in pp.quoted_string.scan_string(escapedQuoteTest) + ] + print(allStrings) + self.assertTrue( + len(allStrings) == 2 + and ( + allStrings[0][1] == 17 + and allStrings[0][2] == 66 + and allStrings[1][1] == 83 + and allStrings[1][2] == 132 + ), + f"quoted string escaped quote failure ({[str(s[0]) for s in allStrings]})", + ) + + dblQuoteTest = r""" + 'This string has an doubled ('') quote character' + "This string has an doubled ("") quote character" + """ + with self.subTest(): + sglStrings = [ + (t[0], b, e) + for (t, b, e) in pp.sgl_quoted_string.scan_string(dblQuoteTest) + ] + print(sglStrings) + self.assertTrue( + len(sglStrings) == 1 + and (sglStrings[0][1] == 17 and sglStrings[0][2] == 66), + f"single quoted string escaped quote failure ({sglStrings[0]})", + ) + + with self.subTest(): + dblStrings = [ + (t[0], b, e) + for (t, b, e) in pp.dbl_quoted_string.scan_string(dblQuoteTest) + ] + print(dblStrings) + self.assertTrue( + len(dblStrings) == 1 + and (dblStrings[0][1] == 83 and dblStrings[0][2] == 132), + f"double quoted string escaped quote failure ({dblStrings[0]})", + ) + + with self.subTest(): + allStrings = [ + (t[0], b, e) for (t, b, e) in pp.quoted_string.scan_string(dblQuoteTest) + ] + print(allStrings) + self.assertTrue( + len(allStrings) == 2 + and ( + allStrings[0][1] == 17 + and allStrings[0][2] == 66 + and allStrings[1][1] == 83 + and allStrings[1][2] == 132 + ), + f"quoted string escaped quote failure ({[str(s[0]) for s in allStrings]})", + ) + + # test invalid end_quote_char + with self.subTest(): + with self.assertRaises( + ValueError, msg="issue raising error for invalid end_quote_char" + ): + expr = pp.QuotedString('"', end_quote_char=" ") + + with self.subTest(): + source = """ + ''' + multiline quote with comment # this is a comment + ''' + \"\"\" + multiline quote with comment # this is a comment + \"\"\" + "single line quote with comment # this is a comment" + 'single line quote with comment # this is a comment' + """ + stripped = ( + pp.python_style_comment.ignore(pp.python_quoted_string) + .suppress() + .transform_string(source) + ) + self.assertEqual(source, stripped) + + def testQuotedStringUnquotesAndConvertWhitespaceEscapes(self): + # test for Issue #474 + # fmt: off + backslash = chr(92) # a single backslash + tab = "\t" + newline = "\n" + test_string_0 = f'"{backslash}{backslash}n"' # r"\\n" + test_string_1 = f'"{backslash}t{backslash}{backslash}n"' # r"\t\\n" + test_string_2 = f'"a{backslash}tb"' # r"a\tb" + test_string_3 = f'"{backslash}{backslash}{backslash}n"' # r"\\\n" + T, F = True, False # these make the test cases format nicely + for test_parameters in ( + # Parameters are the arguments to creating a QuotedString + # and the expected parsed list of characters): + # - unquote_results + # - convert_whitespace_escapes + # - test string + # - expected parsed characters (broken out as separate + # list items (all those doubled backslashes make it + # difficult to interpret the output) + (T, T, test_string_0, [backslash, "n"]), + (T, F, test_string_0, [backslash, "n"]), + (F, F, test_string_0, ['"', backslash, backslash, "n", '"']), + (T, T, test_string_1, [tab, backslash, "n"]), + (T, F, test_string_1, ["t", backslash, "n"]), + (F, F, test_string_1, ['"', backslash, "t", backslash, backslash, "n", '"']), + (T, T, test_string_2, ["a", tab, "b"]), + (T, F, test_string_2, ["a", "t", "b"]), + (F, F, test_string_2, ['"', "a", backslash, "t", "b", '"']), + (T, T, test_string_3, [backslash, newline]), + (T, F, test_string_3, [backslash, "n"]), + (F, F, test_string_3, ['"', backslash, backslash, backslash, "n", '"']), + ): + unquote_results, convert_ws_escapes, test_string, expected_list = test_parameters + test_description = f"Testing with parameters {test_parameters}" + with self.subTest(msg=test_description): + print(test_description) + print(f"unquote_results: {unquote_results}" + f"\nconvert_whitespace_escapes: {convert_ws_escapes}") + qs_expr = pp.QuotedString( + quote_char='"', + esc_char='\\', + unquote_results=unquote_results, + convert_whitespace_escapes=convert_ws_escapes + ) + result = qs_expr.parse_string(test_string) + + # do this instead of assertParserAndCheckList to explicitly + # check and display the separate items in the list + print("Results:") + control_chars = {newline: "", backslash: "", tab: ""} + print(f"[{', '.join(control_chars.get(c, repr(c)) for c in result[0])}]") + self.assertEqual(expected_list, list(result[0])) + + print() + # fmt: on + + def testPythonQuotedStrings(self): + # fmt: off + success1, _ = pp.python_quoted_string.run_tests([ + '"""xyz"""', + '''"""xyz + """''', + '"""xyz "" """', + '''"""xyz "" + """''', + '"""xyz " """', + '''"""xyz " + """''', + r'''"""xyz \""" + + """''', + "'''xyz'''", + """'''xyz + '''""", + "'''xyz '' '''", + """'''xyz '' + '''""", + "'''xyz ' '''", + """'''xyz ' + '''""", + r"""'''xyz \''' + '''""", + ]) + + print("\n\nFailure tests") + success2, _ = pp.python_quoted_string.run_tests([ + '"xyz"""', + ], failure_tests=True) + + self.assertTrue(success1 and success2, "Python quoted string matching failure") + # fmt: on + + def testCaselessOneOf(self): + caseless1 = pp.one_of("d a b c aA B A C", caseless=True) + caseless1str = str(caseless1) + print(caseless1str) + caseless2 = pp.one_of("d a b c Aa B A C", caseless=True) + caseless2str = str(caseless2) + print(caseless2str) + self.assertEqual( + caseless1str.upper(), + caseless2str.upper(), + "one_of not handling caseless option properly", + ) + self.assertNotEqual( + caseless1str, caseless2str, "Caseless option properly sorted" + ) + + res = caseless1[...].parse_string("AAaaAaaA", parse_all=True) + print(res) + self.assertEqual(4, len(res), "caseless1 one_of failed") + self.assertEqual( + "aA" * 4, "".join(res), "caseless1 CaselessLiteral return failed" + ) + + res = caseless2[...].parse_string("AAaaAaaA", parse_all=True) + print(res) + self.assertEqual(4, len(res), "caseless2 one_of failed") + self.assertEqual( + "Aa" * 4, "".join(res), "caseless1 CaselessLiteral return failed" + ) + + def testCStyleCommentParser(self): + print("verify processing of C-style /* */ comments") + testdata = f""" + /* */ + /** **/ + /**/ + /*{'*' * 1_000_000}*/ + /****/ + /* /*/ + /** /*/ + /*** /*/ + /* + ablsjdflj + */ + """ + for test_expr in ( + pp.c_style_comment, + pp.cpp_style_comment, + pp.java_style_comment, + ): + with self.subTest("parse test - /* */ comments", test_expr=test_expr): + found_matches = [ + len(t[0]) for t, s, e in test_expr.scan_string(testdata) + ] + self.assertEqual( + [5, 7, 4, 1000004, 6, 6, 7, 8, 33], + found_matches, + f"only found {test_expr} lengths {found_matches}", + ) + + found_lines = [ + pp.lineno(s, testdata) + for t, s, e in test_expr.scan_string(testdata) + ] + self.assertEqual( + [2, 3, 4, 5, 6, 7, 8, 9, 10], + found_lines, + f"only found {test_expr} on lines {found_lines}", + ) + + def testHtmlCommentParser(self): + print("verify processing of HTML comments") + + test_expr = pp.html_comment + testdata = """ + + + + + + + + + + """ + found_matches = [len(t[0]) for t, s, e in test_expr.scan_string(testdata)] + self.assertEqual( + [8, 10, 7, 8, 9, 9, 10, 11, 79], + found_matches, + f"only found {test_expr} lengths {found_matches}", + ) + + found_lines = [ + pp.lineno(s, testdata) for t, s, e in pp.html_comment.scan_string(testdata) + ] + self.assertEqual( + [2, 3, 4, 5, 6, 7, 8, 9, 10], + found_lines, + f"only found HTML comments on lines {found_lines}", + ) + + def testDoubleSlashCommentParser(self): + print("verify processing of C++ and Java comments - // comments") + + # test C++ single line comments that have line terminated with '\' (should continue comment to following line) + testdata = r""" + // comment1 + // comment2 \ + still comment 2 + // comment 3 + """ + for test_expr in ( + pp.dbl_slash_comment, + pp.cpp_style_comment, + pp.java_style_comment, + ): + with self.subTest("parse test - // comments", test_expr=test_expr): + found_matches = [ + len(t[0]) for t, s, e in test_expr.scan_string(testdata) + ] + self.assertEqual( + [11, 41, 12], + found_matches, + f"only found {test_expr} lengths {found_matches}", + ) + + found_lines = [ + pp.lineno(s, testdata) + for t, s, e in test_expr.scan_string(testdata) + ] + self.assertEqual( + [2, 3, 5], + found_lines, + f"only found {test_expr} on lines {found_lines}", + ) + + def testReCatastrophicBacktrackingInQuotedStringParsers(self): + # reported by webpentest - 2016-04-28 + print( + "testing catastrophic RE backtracking in implementation of quoted string parsers" + ) + repeat = 5000 + for expr, test_string in [ + (pp.dblQuotedString, '"' + "\xff" * repeat), + (pp.sglQuotedString, "'" + "\xff" * repeat), + (pp.quotedString, '"' + "\xff" * repeat), + (pp.quotedString, "'" + "\xff" * repeat), + (pp.QuotedString('"'), '"' + "\xff" * repeat), + (pp.QuotedString("'"), "'" + "\xff" * repeat), + ]: + test_string_label = f"{test_string[:2]}..." + with self.subTest(expr=expr, test_string=repr(test_string_label)): + # parse a valid quoted string + expr.parse_string(test_string + test_string[0], parse_all=True) + + # try to parse a quoted string with no trailing quote + with self.assertRaisesParseException(): + expr.parse_string(test_string, parse_all=True) + + def testReCatastrophicBacktrackingInCommentParsers(self): + print( + "testing catastrophic RE backtracking in implementation of comment parsers" + ) + for expr, test_string in [ + (pp.c_style_comment, f"/*{'*' * 500}"), + (pp.cpp_style_comment, f"/*{'*' * 500}"), + (pp.java_style_comment, f"/*{'*' * 500}"), + (pp.html_comment, f"<-- {'-' * 500}"), + ]: + with self.subTest("Test catastrophic RE backtracking", expr=expr): + try: + expr.parse_string(test_string) + except pp.ParseException: + continue + + def testParseExpressionResults(self): + a = pp.Word("a", pp.alphas).set_name("A") + b = pp.Word("b", pp.alphas).set_name("B") + c = pp.Word("c", pp.alphas).set_name("C") + ab = (a + b).set_name("AB") + abc = (ab + c).set_name("ABC") + word = pp.Word(pp.alphas).set_name("word") + + words = pp.Group(pp.OneOrMore(~a + word)).set_name("words") + + phrase = ( + words("Head") + + pp.Group(a + pp.Optional(b + pp.Optional(c)))("ABC") + + words("Tail") + ) + + results = phrase.parse_string( + "xavier yeti alpha beta charlie will beaver", parse_all=True + ) + print(results, results.Head, results.ABC, results.Tail) + for key, ln in [("Head", 2), ("ABC", 3), ("Tail", 2)]: + self.assertEqual( + ln, + len(results[key]), + f"expected {ln:d} elements in {key}, found {results[key]}", + ) + + def testParseKeyword(self): + kw = pp.Keyword("if") + lit = pp.Literal("if") + + def test(s, litShouldPass, kwShouldPass): + print("Test", s) + print("Match Literal", end=" ") + try: + print(lit.parse_string(s, parse_all=False)) + except Exception: + print("failed") + if litShouldPass: + self.fail(f"Literal failed to match {s}, should have") + else: + if not litShouldPass: + self.fail(f"Literal matched {s}, should not have") + + print("Match Keyword", end=" ") + try: + print(kw.parse_string(s, parse_all=False)) + except Exception: + print("failed") + if kwShouldPass: + self.fail(f"Keyword failed to match {s}, should have") + else: + if not kwShouldPass: + self.fail(f"Keyword matched {s}, should not have") + + test("ifOnlyIfOnly", True, False) + test("if(OnlyIfOnly)", True, True) + test("if (OnlyIf Only)", True, True) + + kw = pp.Keyword("if", caseless=True) + + test("IFOnlyIfOnly", False, False) + test("If(OnlyIfOnly)", False, True) + test("iF (OnlyIf Only)", False, True) + + with self.assertRaises( + ValueError, msg="failed to warn empty string passed to Keyword" + ): + kw = pp.Keyword("") + + def testParseExpressionResultsAccumulate(self): + num = pp.Word(pp.nums).set_name("num")("base10*") + hexnum = pp.Combine("0x" + pp.Word(pp.nums)).set_name("hexnum")("hex*") + name = pp.Word(pp.alphas).set_name("word")("word*") + list_of_num = pp.DelimitedList(hexnum | num | name, ",") + + tokens = list_of_num.parse_string("1, 0x2, 3, 0x4, aaa", parse_all=True) + print(tokens.dump()) + self.assertParseResultsEquals( + tokens, + expected_list=["1", "0x2", "3", "0x4", "aaa"], + expected_dict={ + "base10": ["1", "3"], + "hex": ["0x2", "0x4"], + "word": ["aaa"], + }, + ) + + lbrack = pp.Literal("(").suppress() + rbrack = pp.Literal(")").suppress() + integer = pp.Word(pp.nums).set_name("int") + variable = pp.Word(pp.alphas, max=1).set_name("variable") + relation_body_item = ( + variable | integer | pp.quoted_string().set_parse_action(pp.remove_quotes) + ) + relation_name = pp.Word(pp.alphas + "_", pp.alphanums + "_") + relation_body = lbrack + pp.Group(pp.DelimitedList(relation_body_item)) + rbrack + Goal = pp.Dict(pp.Group(relation_name + relation_body)) + Comparison_Predicate = pp.Group(variable + pp.one_of("< >") + integer)("pred*") + Query = Goal("head") + ":-" + pp.DelimitedList(Goal | Comparison_Predicate) + + test = """Q(x,y,z):-Bloo(x,"Mitsis",y),Foo(y,z,1243),y>28,x<12,x>3""" + + queryRes = Query.parse_string(test, parse_all=True) + print(queryRes.dump()) + self.assertParseResultsEquals( + queryRes.pred, + expected_list=[["y", ">", "28"], ["x", "<", "12"], ["x", ">", "3"]], + msg=f"Incorrect list for attribute pred, {queryRes.pred.as_list()}", + ) + + def testReStringRange(self): + testCases = ( + r"[A-Z]", + r"[A-A]", + r"[A-Za-z]", + r"[A-z]", + r"[\ -\~]", + r"[\0x20-0]", + r"[\0x21-\0x7E]", + r"[\0xa1-\0xfe]", + r"[\040-0]", + r"[A-Za-z0-9]", + r"[A-Za-z0-9_]", + r"[A-Za-z0-9_$]", + r"[A-Za-z0-9_$\-]", + r"[^0-9\\]", + r"[a-zA-Z]", + r"[/\^~]", + r"[=\+\-!]", + r"[A-]", + r"[-A]", + r"[\x21]", + r"[а-яА-ЯёЁA-Z$_\041α-ω]", + r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]", + r"[\0xa1-\0xbf\0xd7\0xf7]", + r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]", + r"[\0xa1-\0xbf\0xd7\0xf7]", + r"[\\[\]\/\-\*\.\$\+\^\?()~ ]", + ) + expectedResults = ( + "ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "A", + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", + "ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz", + " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~", + " !\"#$%&'()*+,-./0", + "!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~", + "¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþ", + " !\"#$%&'()*+,-./0", + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_", + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_$", + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_$-", + "0123456789\\", + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ", + "/^~", + "=+-!", + "A-", + "-A", + "!", + "абвгдежзийклмнопрстуфхцчшщъыьэюяАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯёЁABCDEFGHIJKLMNOPQRSTUVWXYZ$_!αβγδεζηθικλμνξοπρςστυφχψω", + "ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ", + "¡¢£¤¥¦§¨©ª«¬\xad®¯°±²³´µ¶·¸¹º»¼½¾¿×÷", + pp.alphas8bit, + pp.punc8bit, + r"\[]/-*.$+^?()~ ", + ) + for test in zip(testCases, expectedResults): + t, exp = test + res = pp.srange(t) + # print(t, "->", res) + self.assertEqual( + exp, + res, + f"srange error, srange({t!r})->'{res!r}', expected '{exp!r}'", + ) + + def testSkipToParserTests(self): + thingToFind = pp.Literal("working") + testExpr = ( + pp.SkipTo(pp.Literal(";"), include=True, ignore=pp.c_style_comment) + + thingToFind + ) + + def test_parse(someText): + print(testExpr.parse_string(someText, parse_all=True)) + + # This first test works, as the SkipTo expression is immediately following the ignore expression (c_style_comment) + test_parse("some text /* comment with ; in */; working") + # This second test previously failed, as there is text following the ignore expression, and before the SkipTo expression. + test_parse("some text /* comment with ; in */some other stuff; working") + + # tests for optional fail_on argument + testExpr = ( + pp.SkipTo( + pp.Literal(";"), + include=True, + ignore=pp.c_style_comment, + fail_on="other", + ) + + thingToFind + ) + test_parse("some text /* comment with ; in */; working") + + with self.assertRaisesParseException(): + test_parse("some text /* comment with ; in */some other stuff; working") + + # test that we correctly create named results + text = "prefixDATAsuffix" + data = pp.Literal("DATA") + suffix = pp.Literal("suffix") + expr = pp.SkipTo(data + suffix)("prefix") + data + suffix + result = expr.parse_string(text, parse_all=True) + self.assertTrue( + isinstance(result.prefix, str), + "SkipTo created with wrong saveAsList attribute", + ) + + alpha_word = ( + ~pp.Literal("end") + pp.Word(pp.alphas, as_keyword=True) + ).set_name("alpha") + num_word = pp.Word(pp.nums, as_keyword=True).set_name("int") + + def test(expr, test_string, expected_list, expected_dict): + if (expected_list, expected_dict) == (None, None): + with self.assertRaises( + Exception, msg=f"{expr} failed to parse {test_string!r}" + ): + expr.parse_string(test_string, parse_all=True) + else: + result = expr.parse_string(test_string, parse_all=True) + self.assertParseResultsEquals( + result, expected_list=expected_list, expected_dict=expected_dict + ) + + # ellipses for SkipTo + e = ... + pp.Literal("end") + test(e, "start 123 end", ["start 123 ", "end"], {"_skipped": ["start 123 "]}) + + e = pp.Suppress(...) + pp.Literal("end") + test(e, "start 123 end", ["end"], {}) + + e = pp.Literal("start") + ... + pp.Literal("end") + test(e, "start 123 end", ["start", "123 ", "end"], {"_skipped": ["123 "]}) + + e = ... + pp.Literal("middle") + ... + pp.Literal("end") + test( + e, + "start 123 middle 456 end", + ["start 123 ", "middle", "456 ", "end"], + {"_skipped": ["start 123 ", "456 "]}, + ) + + e = pp.Suppress(...) + pp.Literal("middle") + ... + pp.Literal("end") + test( + e, + "start 123 middle 456 end", + ["middle", "456 ", "end"], + {"_skipped": ["456 "]}, + ) + + e = pp.Literal("start") + ... + test(e, "start 123 end", None, None) + + e = pp.And(["start", ..., "end"]) + test(e, "start 123 end", ["start", "123 ", "end"], {"_skipped": ["123 "]}) + + e = pp.And([..., "end"]) + test(e, "start 123 end", ["start 123 ", "end"], {"_skipped": ["start 123 "]}) + + e = "start" + (num_word | ...) + "end" + test(e, "start 456 end", ["start", "456", "end"], {}) + test( + e, + "start 123 456 end", + ["start", "123", "456 ", "end"], + {"_skipped": ["456 "]}, + ) + test(e, "start end", ["start", "", "end"], {"_skipped": ["missing "]}) + + # e = define_expr('"start" + (num_word | ...)("inner") + "end"') + # test(e, "start 456 end", ['start', '456', 'end'], {'inner': '456'}) + + e = "start" + (alpha_word[...] & num_word[...] | ...) + "end" + test(e, "start 456 red end", ["start", "456", "red", "end"], {}) + test(e, "start red 456 end", ["start", "red", "456", "end"], {}) + test( + e, + "start 456 red + end", + ["start", "456", "red", "+ ", "end"], + {"_skipped": ["+ "]}, + ) + test(e, "start red end", ["start", "red", "end"], {}) + test(e, "start 456 end", ["start", "456", "end"], {}) + test(e, "start end", ["start", "end"], {}) + test(e, "start 456 + end", ["start", "456", "+ ", "end"], {"_skipped": ["+ "]}) + + e = "start" + (alpha_word[1, ...] & num_word[1, ...] | ...) + "end" + test(e, "start 456 red end", ["start", "456", "red", "end"], {}) + test(e, "start red 456 end", ["start", "red", "456", "end"], {}) + test( + e, + "start 456 red + end", + ["start", "456", "red", "+ ", "end"], + {"_skipped": ["+ "]}, + ) + test(e, "start red end", ["start", "red ", "end"], {"_skipped": ["red "]}) + test(e, "start 456 end", ["start", "456 ", "end"], {"_skipped": ["456 "]}) + test( + e, + "start end", + ["start", "", "end"], + {"_skipped": ["missing <{{alpha}... & {int}...}>"]}, + ) + test(e, "start 456 + end", ["start", "456 + ", "end"], {"_skipped": ["456 + "]}) + + e = "start" + (alpha_word | ...) + (num_word | ...) + "end" + test(e, "start red 456 end", ["start", "red", "456", "end"], {}) + test( + e, + "start red end", + ["start", "red", "", "end"], + {"_skipped": ["missing "]}, + ) + test( + e, + "start 456 end", + ["start", "", "456", "end"], + {"_skipped": ["missing "]}, + ) + test( + e, + "start end", + ["start", "", "", "end"], + {"_skipped": ["missing ", "missing "]}, + ) + + e = pp.Literal("start") + ... + "+" + ... + "end" + test( + e, + "start red + 456 end", + ["start", "red ", "+", "456 ", "end"], + {"_skipped": ["red ", "456 "]}, + ) + + def testSkipToPreParseIgnoreExprs(self): + # added to verify fix to Issue #475 + from pyparsing import Word, alphanums, python_style_comment + + some_grammar = Word(alphanums) + ":=" + ... + ";" + some_grammar.ignore(python_style_comment) + try: + result = some_grammar.parse_string( + """\ + var1 := 2 # 3; <== this semi-colon will match! + + 1; + """, + parse_all=True, + ) + except ParseException as pe: + print(pe.explain()) + raise + else: + print(result.dump()) + + def testSkipToIgnoreExpr2(self): + a, star = pp.Literal.using_each("a*") + wrapper = a + ... + a + expr = star + pp.SkipTo(star, ignore=wrapper) + star + + # pyparsing 3.0.9 -> ['*', 'a_*_a', '*'] + # pyparsing 3.1.0 -> ['*', '', '*'] + self.assertParseAndCheckList(expr, "*a_*_a*", ["*", "a_*_a", "*"]) + + def testEllipsisRepetition(self): + word = pp.Word(pp.alphas).set_name("word") + num = pp.Word(pp.nums).set_name("num") + + exprs = [ + word[...] + num, + word * ... + num, + word[0, ...] + num, + word[1, ...] + num, + word[2, ...] + num, + word[..., 3] + num, + word[2] + num, + ] + + expected_res = [ + r"([abcd]+ )*\d+", + r"([abcd]+ )*\d+", + r"([abcd]+ )*\d+", + r"([abcd]+ )+\d+", + r"([abcd]+ ){2,}\d+", + r"([abcd]+ ){0,3}\d+", + r"([abcd]+ ){2}\d+", + ] + + tests = ["aa bb cc dd 123", "bb cc dd 123", "cc dd 123", "dd 123", "123"] + + all_success = True + for expr, expected_re in zip(exprs, expected_res): + successful_tests = [t for t in tests if re.match(expected_re, t)] + failure_tests = [t for t in tests if not re.match(expected_re, t)] + success1, _ = expr.run_tests(successful_tests) + success2, _ = expr.run_tests(failure_tests, failure_tests=True) + all_success = all_success and success1 and success2 + if not all_success: + print("Failed expression:", expr) + break + + self.assertTrue(all_success, "failed getItem_ellipsis test") + + def testEllipsisRepetitionWithResultsNames(self): + label = pp.Word(pp.alphas) + val = ppc.integer() + parser = label("label") + pp.ZeroOrMore(val)("values") + + _, results = parser.run_tests( + """ + a 1 + b 1 2 3 + c + """ + ) + expected = [ + (["a", 1], {"label": "a", "values": [1]}), + (["b", 1, 2, 3], {"label": "b", "values": [1, 2, 3]}), + (["c"], {"label": "c", "values": []}), + ] + for obs, exp in zip(results, expected): + test, result = obs + exp_list, exp_dict = exp + self.assertParseResultsEquals( + result, expected_list=exp_list, expected_dict=exp_dict + ) + + parser = label("label") + val[...]("values") + + _, results = parser.run_tests( + """ + a 1 + b 1 2 3 + c + """ + ) + expected = [ + (["a", 1], {"label": "a", "values": [1]}), + (["b", 1, 2, 3], {"label": "b", "values": [1, 2, 3]}), + (["c"], {"label": "c", "values": []}), + ] + for obs, exp in zip(results, expected): + test, result = obs + exp_list, exp_dict = exp + self.assertParseResultsEquals( + result, expected_list=exp_list, expected_dict=exp_dict + ) + + pt = pp.Group(val("x") + pp.Suppress(",") + val("y")) + parser = label("label") + pt[...]("points") + _, results = parser.run_tests( + """ + a 1,1 + b 1,1 2,2 3,3 + c + """ + ) + expected = [ + (["a", [1, 1]], {"label": "a", "points": [{"x": 1, "y": 1}]}), + ( + ["b", [1, 1], [2, 2], [3, 3]], + { + "label": "b", + "points": [{"x": 1, "y": 1}, {"x": 2, "y": 2}, {"x": 3, "y": 3}], + }, + ), + (["c"], {"label": "c", "points": []}), + ] + for obs, exp in zip(results, expected): + test, result = obs + exp_list, exp_dict = exp + self.assertParseResultsEquals( + result, expected_list=exp_list, expected_dict=exp_dict + ) + + def testCustomQuotes(self): + testString = r""" + sdlfjs :sdf\:jls::djf: sl:kfsjf + sdlfjs -sdf\:jls::--djf: sl-kfsjf + sdlfjs -sdf\:::jls::--djf: sl:::-kfsjf + sdlfjs ^sdf\:jls^^--djf^ sl-kfsjf + sdlfjs ^^^==sdf\:j=lz::--djf: sl=^^=kfsjf + sdlfjs ==sdf\:j=ls::--djf: sl==kfsjf^^^ + """ + print(testString) + + colonQuotes = pp.QuotedString(":", "\\", "::") + dashQuotes = pp.QuotedString("-", "\\", "--") + hatQuotes = pp.QuotedString("^", "\\") + hatQuotes1 = pp.QuotedString("^", "\\", "^^") + dblEqQuotes = pp.QuotedString("==", "\\") + + def test(label, quoteExpr, expected): + print(label) + print(quoteExpr.pattern) + print(quoteExpr.search_string(testString)) + print(quoteExpr.search_string(testString)[0][0]) + print(f"{expected}") + self.assertEqual( + expected, + quoteExpr.search_string(testString)[0][0], + f"failed to match {quoteExpr}, expected '{expected}', got '{quoteExpr.search_string(testString)[0]}'", + ) + print() + + test("colonQuotes", colonQuotes, r"sdf:jls:djf") + test("dashQuotes", dashQuotes, r"sdf:jls::-djf: sl") + test("hatQuotes", hatQuotes, r"sdf:jls") + test("hatQuotes1", hatQuotes1, r"sdf:jls^--djf") + test("dblEqQuotes", dblEqQuotes, r"sdf:j=ls::--djf: sl") + test("::: quotes", pp.QuotedString(":::"), "jls::--djf: sl") + test("==-- quotes", pp.QuotedString("==", end_quote_char="--"), r"sdf\:j=lz::") + test( + "^^^ multiline quotes", + pp.QuotedString("^^^", multiline=True), + r"""==sdf\:j=lz::--djf: sl=^^=kfsjf + sdlfjs ==sdf\:j=ls::--djf: sl==kfsjf""", + ) + with self.assertRaises(ValueError): + pp.QuotedString("", "\\") + + def testCustomQuotes2(self): + qs = pp.QuotedString(quote_char=".[", end_quote_char="].") + print(qs.reString) + self.assertParseAndCheckList(qs, ".[...].", ["..."]) + self.assertParseAndCheckList(qs, ".[].", [""]) + self.assertParseAndCheckList(qs, ".[]].", ["]"]) + self.assertParseAndCheckList(qs, ".[]]].", ["]]"]) + + qs = pp.QuotedString(quote_char="+*", end_quote_char="*+") + print(qs.reString) + self.assertParseAndCheckList(qs, "+*...*+", ["..."]) + self.assertParseAndCheckList(qs, "+**+", [""]) + self.assertParseAndCheckList(qs, "+***+", ["*"]) + self.assertParseAndCheckList(qs, "+****+", ["**"]) + + qs = pp.QuotedString(quote_char="*/", end_quote_char="/*") + print(qs.reString) + self.assertParseAndCheckList(qs, "*/.../*", ["..."]) + self.assertParseAndCheckList(qs, "*//*", [""]) + self.assertParseAndCheckList(qs, "*///*", ["/"]) + self.assertParseAndCheckList(qs, "*////*", ["//"]) + + def testQuotedStringWithWhitespaceInQuoteChars(self): + # Issue #492 - a leading/trailing whitespace character that is part of a + # multi-character quote delimiter (such as a newline in "\n;") must not + # be stripped away, which previously collapsed the delimiter to ";". + with ppt.reset_pyparsing_context(): + pp.ParserElement.set_default_whitespace_chars("") + newline_quote = pp.QuotedString("\n;", multiline=True) + self.assertParseAndCheckList( + newline_quote, "\n;Hi \n mum!\n;", ["Hi \n mum!"] + ) + self.assertEqual( + [["Hi \n mum!"]], + newline_quote.search_string("lsjdf \n;Hi \n mum!\n; sldjf").as_list(), + ) + self.assertEqual( + [["Hi \n m;um!"]], + newline_quote.search_string("lsjdf \n;Hi \n m;um!\n; sldjf").as_list(), + ) + + # a fully whitespace (or empty) quote_char is still rejected + for bad_quote in ("", " ", "\n"): + with self.assertRaises(ValueError): + pp.QuotedString(bad_quote) + with self.assertRaises(ValueError): + pp.QuotedString('"', end_quote_char=bad_quote) + + def testRepeater(self): + if ParserElement._packratEnabled or ParserElement._left_recursion_enabled: + print("skipping this test, not compatible with memoization") + return + + first = pp.Word("abcdef").set_name("word1") + bridge = pp.Word(pp.nums).set_name("number") + second = pp.match_previous_literal(first).set_name("repeat(word1Literal)") + + seq = first + bridge + second + + tests = [ + ("abc12abc", True), + ("abc12aabc", False), + ("abc12cba", True), + ("abc12bca", True), + ] + + for tst, expected in tests: + found = False + for tokens, start, end in seq.scan_string(tst): + f, b, s = tokens + print(f, b, s) + found = True + if not found: + print("No literal match in", tst) + self.assertEqual( + expected, + found, + f"Failed repeater for test: {tst}, matching {seq}", + ) + print() + + # retest using match_previous_expr instead of match_previous_literal + second = pp.match_previous_expr(first).set_name("repeat(word1expr)") + seq = first + bridge + second + + tests = [("abc12abc", True), ("abc12cba", False), ("abc12abcdef", False)] + + for tst, expected in tests: + found = False + for tokens, start, end in seq.scan_string(tst): + print(tokens) + found = True + if not found: + print("No expression match in", tst) + self.assertEqual( + expected, + found, + f"Failed repeater for test: {tst}, matching {seq}", + ) + + print() + + first = pp.Word("abcdef").set_name("word1") + bridge = pp.Word(pp.nums).set_name("number") + second = pp.match_previous_expr(first).set_name("repeat(word1)") + seq = first + bridge + second + csFirst = seq.set_name("word-num-word") + csSecond = pp.match_previous_expr(csFirst) + compoundSeq = csFirst + ":" + csSecond + compoundSeq.streamline() + print(compoundSeq) + + tests = [ + ("abc12abc:abc12abc", True), + ("abc12cba:abc12abc", False), + ("abc12abc:abc12abcdef", False), + ] + + for tst, expected in tests: + found = False + for tokens, start, end in compoundSeq.scan_string(tst): + print("match:", tokens) + found = True + break + if not found: + print("No expression match in", tst) + self.assertEqual( + expected, + found, + f"Failed repeater for test: {tst}, matching {seq}", + ) + + print() + eFirst = pp.Word(pp.nums) + eSecond = pp.match_previous_expr(eFirst) + eSeq = eFirst + ":" + eSecond + + tests = [("1:1A", True), ("1:10", False)] + + for tst, expected in tests: + found = False + for tokens, start, end in eSeq.scan_string(tst): + print(tokens) + found = True + if not found: + print("No match in", tst) + self.assertEqual( + expected, + found, + f"Failed repeater for test: {tst}, matching {seq}", + ) + + def testRepeater2(self): + """test match_previous_literal with empty repeater""" + + if ParserElement._packratEnabled or ParserElement._left_recursion_enabled: + print("skipping this test, not compatible with memoization") + return + + first = pp.Optional(pp.Word("abcdef").set_name("words1")) + bridge = pp.Word(pp.nums).set_name("number") + second = pp.match_previous_literal(first).set_name("repeat(word1Literal)") + + seq = first + bridge + second + + tst = "12" + expected = ["12"] + result = seq.parse_string(tst, parse_all=True) + print(result.dump()) + + self.assertParseResultsEquals(result, expected_list=expected) + + def testRepeater3(self): + """test match_previous_literal with multiple repeater tokens""" + + if ParserElement._packratEnabled or ParserElement._left_recursion_enabled: + print("skipping this test, not compatible with memoization") + return + + first = pp.Word("a") + pp.Word("d") + bridge = pp.Word(pp.nums).set_name("number") + second = pp.match_previous_literal(first) # ("second") + + seq = first + bridge + second + + tst = "aaaddd12aaaddd" + expected = ["aaa", "ddd", "12", "aaa", "ddd"] + result = seq.parse_string(tst, parse_all=True) + print(result.dump()) + + self.assertParseResultsEquals(result, expected_list=expected) + + def testRepeater4(self): + """test match_previous_expr with multiple repeater tokens""" + + if ParserElement._packratEnabled or ParserElement._left_recursion_enabled: + print("skipping this test, not compatible with memoization") + return + + first = pp.Group(pp.Word(pp.alphas) + pp.Word(pp.alphas)) + bridge = pp.Word(pp.nums) + + # no matching is used - this is just here for a sanity check + # second = pp.Group(pp.Word(pp.alphas) + pp.Word(pp.alphas))("second") + # second = pp.Group(pp.Word(pp.alphas) + pp.Word(pp.alphas)).set_results_name("second") + + # ISSUE: when match_previous_expr returns multiple tokens the matching tokens are nested an extra level deep. + # This behavior is not seen with a single return token (see testRepeater5 directly below.) + second = pp.match_previous_expr(first) + + expr = first + bridge.suppress() + second + + tst = "aaa ddd 12 aaa ddd" + expected = [["aaa", "ddd"], ["aaa", "ddd"]] + result = expr.parse_string(tst, parse_all=True) + print(result.dump()) + + self.assertParseResultsEquals(result, expected_list=expected) + + def testRepeater5(self): + """a simplified testRepeater4 to examine match_previous_expr with a single repeater token""" + + if ParserElement._packratEnabled or ParserElement._left_recursion_enabled: + print("skipping this test, not compatible with memoization") + return + + first = pp.Word(pp.alphas) + bridge = pp.Word(pp.nums) + second = pp.match_previous_expr(first) + + expr = first + bridge.suppress() + second + + tst = "aaa 12 aaa" + expected = tst.replace("12", "").split() + result = expr.parse_string(tst, parse_all=True) + print(result.dump()) + + self.assertParseResultsEquals(result, expected_list=expected) + + def testRecursiveCombine(self): + testInput = "myc(114)r(11)dd" + stream = pp.Forward() + stream <<= pp.Optional(pp.Word(pp.alphas)) + pp.Optional( + "(" + pp.Word(pp.nums) + ")" + stream + ) + expected = ["".join(stream.parse_string(testInput, parse_all=True))] + print(expected) + + stream = pp.Forward() + stream << pp.Combine( + pp.Optional(pp.Word(pp.alphas)) + + pp.Optional("(" + pp.Word(pp.nums) + ")" + stream) + ) + testVal = stream.parse_string(testInput, parse_all=True) + print(testVal) + + self.assertParseResultsEquals(testVal, expected_list=expected) + + def testRepeaterRecursiveWhitespace(self): + """test match_previous_expr with recursive whitespace""" + + if ParserElement._packratEnabled or ParserElement._left_recursion_enabled: + print("skipping this test, not compatible with memoization") + return + + COLON = pp.Suppress(":").set_name(":") + first = pp.Char(pp.nums) + second = pp.match_previous_expr(first) + expr = first + COLON + second + expr.leave_whitespace(recursive=True) + + tests = [ + ("1:1", True), + ("1: 1", False), + ("1:2", False), + ("1:a", False), + ] + + for tst, expected in tests: + if expected: + expected_int_list = [t.strip() for t in tst.split(":")] + self.assertParseAndCheckList( + expr, tst, expected_int_list, + msg=f"Failed recursive whitespace repeater test, expected pass: {expr=} {tst=}" + ) + else: + with self.assertRaisesParseException( + msg=f"Failed recursive whitespace repeater test, expected fail: {expr=} {tst=}" + ): + expr.parse_string(tst) + + + def testRepeaterRecursiveFalse(self): + """test match_previous_expr with recursive=False""" + + if ParserElement._packratEnabled or ParserElement._left_recursion_enabled: + print("skipping this test, not compatible with memoization") + return + + COLON = pp.Suppress(":").set_name(":") + first = pp.Char(pp.nums) + second = pp.match_previous_expr(first) + expr = first + COLON + second + expr.leave_whitespace(recursive=False) + + tests = [ + ("1:1", True), + ("1: 1", True), + ("1:2", False), + ] + + for tst, expected in tests: + if expected: + expected_int_list = [t.strip() for t in tst.split(":")] + self.assertParseAndCheckList( + expr, tst, expected_int_list, + msg=f"Failed recursive=False repeater test, expected pass: {expr=} {tst=}" + ) + else: + with self.assertRaisesParseException( + msg=f"Failed recursive=False repeater test, expected fail: {expr=} {tst=}" + ): + expr.parse_string(tst) + + def testRepeaterPreservesParseAction(self): + """test match_previous_expr preserves existing parse actions""" + + if ParserElement._packratEnabled or ParserElement._left_recursion_enabled: + print("skipping this test, not compatible with memoization") + return + + COLON = pp.Suppress(":").set_name(":") + first = ppc.integer + second = pp.match_previous_expr(first) + expr = first + COLON + second + expr.leave_whitespace(recursive=True) + + tests = [ + ("1:1", True), + ("1: 1", False), + ("1:2", False), + ] + + for tst, expected in tests: + if expected: + expected_int_list = [int(t) for t in tst.split(":")] + self.assertParseAndCheckList( + expr, tst, expected_int_list, + msg=f"Failed parse action preservation repeater test, expected pass: {expr=} {tst=}" + ) + else: + with self.assertRaisesParseException( + msg=f"Failed parse action preservation repeater test, expected fail: {expr=} {tst=}" + ): + expr.parse_string(tst) + + def testRepeaterNested(self): + """test match_previous_expr with nested and repeated matches""" + + if ParserElement._packratEnabled or ParserElement._left_recursion_enabled: + print("skipping this test, not compatible with memoization") + return + + LBRACE, RBRACE, SLASH = map(pp.Suppress, "{}/") + tag_name = pp.Word(pp.alphanums + "_") + open_tag = LBRACE + tag_name + RBRACE + close_tag = LBRACE + SLASH + pp.match_previous_expr(tag_name) + RBRACE + + content = pp.Forward() + content <<= (open_tag + pp.Group(content) + close_tag) | pp.Word(pp.alphas) + + tests = [ + ("{a}xx{/a}", True), + ("{a}{b}xx{/b}{/a}", True), + ("{a}{b}{c}xx{/c}{/b}{/a}", True), + ("{a}{b}xx{/a}{/b}", False), + ("{a}{b}xx{/b}{/c}", False), + ("{a}xx{/b}", False), + ] + + for tst, expected in tests: + if expected: + content.parse_string(tst, parse_all=True) + else: + with self.assertRaisesParseException( + msg=f"Failed nested repeater test, expected fail: {tst=}" + ): + content.parse_string(tst, parse_all=True) + + def testRepeaterReusedParser(self): + """test match_previous_expr matches repeatedly, and after a failed parse""" + + if ParserElement._packratEnabled or ParserElement._left_recursion_enabled: + print("skipping this test, not compatible with memoization") + return + + first = pp.Word(pp.alphas) + second = pp.match_previous_expr(first) + expr = first + pp.Suppress(":") + second + + # repeated matches within a single parse + self.assertParseAndCheckList( + pp.OneOrMore(expr), "a:a b:b c:c", ["a", "a", "b", "b", "c", "c"] + ) + + # a mismatch must not poison later parses of the same expression + tests = [("d:d", True), ("e:f", False), ("g:g", True)] + for tst, expected in tests: + if expected: + self.assertParseAndCheckList( + expr, + tst, + tst.split(":"), + msg=f"Failed reused repeater test, expected pass: {tst=}", + ) + else: + with self.assertRaisesParseException( + msg=f"Failed reused repeater test, expected fail: {tst=}" + ): + expr.parse_string(tst, parse_all=True) + + def testSetNameToStrAndNone(self): + wd = pp.Word(pp.alphas) + with self.subTest(): + self.assertEqual("W:(A-Za-z)", wd.name) + + with self.subTest(): + wd.set_name("test_word") + self.assertEqual("test_word", wd.name) + + with self.subTest(): + wd.set_name(None) + self.assertEqual("W:(A-Za-z)", wd.name) + + # same tests but using name property setter + with self.subTest(): + wd.name = "test_word" + self.assertEqual("test_word", wd.name) + + with self.subTest(): + wd.name = None + self.assertEqual("W:(A-Za-z)", wd.name) + + def testCombineSetName(self): + ab = pp.Combine( + pp.Literal("a").set_name("AAA") | pp.Literal("b").set_name("BBB") + ).set_name("AB") + self.assertEqual("AB", ab.name) + self.assertEqual("AB", str(ab)) + with self.assertRaisesParseException(expected_msg="Expected AB"): + ab.parse_string("C") + + def testHTMLEntities(self): + html_source = dedent( + """\ + This & that + 2 > 1 + 0 < 1 + Don't get excited! + I said "Don't get excited!" + Copyright © 2021 + Dot ⟶ ˙ + """ + ) + transformer = pp.common_html_entity().add_parse_action(pp.replace_html_entity) + transformed = transformer.transform_string(html_source) + print(transformed) + + expected = dedent( + """\ + This & that + 2 > 1 + 0 < 1 + Don't get excited! + I said "Don't get excited!" + Copyright © 2021 + Dot ⟶ ˙ + """ + ) + self.assertEqual(expected, transformed) + + def testInfixNotationBasicArithEval(self): + import ast + + integer = pp.Word(pp.nums).set_parse_action(lambda t: int(t[0])) + variable = pp.Word(pp.alphas, exact=1) + operand = integer | variable + + expop = pp.Literal("^") + signop = pp.one_of("+ -") + multop = pp.one_of("* /") + plusop = pp.one_of("+ -") + factop = pp.Literal("!") + + # fmt: off + expr = pp.infix_notation( + operand, + [ + (factop, 1, pp.OpAssoc.LEFT), + (expop, 2, pp.OpAssoc.RIGHT), + (signop, 1, pp.OpAssoc.RIGHT), + (multop, 2, pp.OpAssoc.LEFT), + (plusop, 2, pp.OpAssoc.LEFT), + ], + ) + # fmt: on + + test = [ + "9 + 2 + 3", + "9 + 2 * 3", + "(9 + 2) * 3", + "(9 + -2) * 3", + "(9 + --2) * 3", + "(9 + -2) * 3^2^2", + "(9! + -2) * 3^2^2", + "M*X + B", + "M*(X + B)", + "1+2*-3^4*5+-+-6", + "3!!", + ] + expected = """[[9, '+', 2, '+', 3]] + [[9, '+', [2, '*', 3]]] + [[[9, '+', 2], '*', 3]] + [[[9, '+', ['-', 2]], '*', 3]] + [[[9, '+', ['-', ['-', 2]]], '*', 3]] + [[[9, '+', ['-', 2]], '*', [3, '^', [2, '^', 2]]]] + [[[[9, '!'], '+', ['-', 2]], '*', [3, '^', [2, '^', 2]]]] + [[['M', '*', 'X'], '+', 'B']] + [['M', '*', ['X', '+', 'B']]] + [[1, '+', [2, '*', ['-', [3, '^', 4]], '*', 5], '+', ['-', ['+', ['-', 6]]]]] + [[3, '!', '!']]""".split( + "\n" + ) + expected = [ast.literal_eval(x.strip()) for x in expected] + for test_str, exp_list in zip(test, expected): + self.assertParseAndCheckList(expr, test_str, exp_list, verbose=True) + + def testInfixNotationEvalBoolExprUsingAstClasses(self): + boolVars = {"True": True, "False": False} + + class BoolOperand: + reprsymbol = "" + + def __init__(self, t): + self.args = t[0][0::2] + + def __str__(self): + sep = f" {self.reprsymbol} " + return f"({sep.join(map(str, self.args))})" + + class BoolAnd(BoolOperand): + reprsymbol = "&" + + def __bool__(self): + for a in self.args: + if isinstance(a, str): + v = boolVars[a] + else: + v = bool(a) + if not v: + return False + return True + + class BoolOr(BoolOperand): + reprsymbol = "|" + + def __bool__(self): + for a in self.args: + if isinstance(a, str): + v = boolVars[a] + else: + v = bool(a) + if v: + return True + return False + + class BoolNot: + def __init__(self, t): + self.arg = t[0][1] + + def __str__(self): + return f"~{self.arg}" + + def __bool__(self): + if isinstance(self.arg, str): + v = boolVars[self.arg] + else: + v = bool(self.arg) + return not v + + boolOperand = pp.Word(pp.alphas, max=1, as_keyword=True) | pp.one_of( + "True False" + ) + # fmt: off + boolExpr = pp.infix_notation( + boolOperand, + [ + ("not", 1, pp.OpAssoc.RIGHT, BoolNot), + ("and", 2, pp.OpAssoc.LEFT, BoolAnd), + ("or", 2, pp.OpAssoc.LEFT, BoolOr), + ], + ) + # fmt: on + test = [ + "p and not q", + "not not p", + "not(p and q)", + "q or not p and r", + "q or not p or not r", + "q or not (p and r)", + "p or q or r", + "p or q or r and False", + "(p or q or r) and False", + ] + + boolVars["p"] = True + boolVars["q"] = False + boolVars["r"] = True + print("p =", boolVars["p"]) + print("q =", boolVars["q"]) + print("r =", boolVars["r"]) + print() + for t in test: + res = boolExpr.parse_string(t, parse_all=True) + print(t, "\n", res[0], "=", bool(res[0]), "\n") + expected = eval(t, {}, boolVars) + self.assertEqual(expected, bool(res[0]), f"failed boolean eval test {t}") + + def testInfixNotationMinimalParseActionCalls(self): + count = 0 + + def evaluate_int(t): + nonlocal count + value = int(t[0]) + print("evaluate_int", value) + count += 1 + return value + + integer = pp.Word(pp.nums).set_parse_action(evaluate_int) + variable = pp.Word(pp.alphas, exact=1) + operand = integer | variable + + expop = pp.Literal("^") + signop = pp.one_of("+ -") + multop = pp.one_of("* /") + plusop = pp.one_of("+ -") + factop = pp.Literal("!") + + # fmt: off + expr = pp.infix_notation( + operand, + [ + (factop, 1, pp.OpAssoc.LEFT), + (expop, 2, pp.OpAssoc.LEFT), + (signop, 1, pp.OpAssoc.RIGHT), + (multop, 2, pp.OpAssoc.LEFT), + (plusop, 2, pp.OpAssoc.LEFT), + ], + ) + # fmt: on + + test = ["9"] + for t in test: + count = 0 + print(f"{t!r} => {expr.parse_string(t, parse_all=True)} (count={count})") + self.assertEqual(1, count, "count evaluated too many times!") + + def testInfixNotationWithParseActions(self): + word = pp.Word(pp.alphas) + + def supLiteral(s): + """Returns the suppressed literal s""" + return pp.Literal(s).suppress() + + def booleanExpr(atom): + ops = [ + (supLiteral("!"), 1, pp.OpAssoc.RIGHT, lambda s, l, t: ["!", t[0][0]]), + (pp.one_of("= !="), 2, pp.OpAssoc.LEFT), + (supLiteral("&"), 2, pp.OpAssoc.LEFT, lambda s, l, t: ["&", t[0]]), + (supLiteral("|"), 2, pp.OpAssoc.LEFT, lambda s, l, t: ["|", t[0]]), + ] + return pp.infix_notation(atom, ops) + + f = booleanExpr(word) + pp.StringEnd() + + tests = [ + ("bar = foo", [["bar", "=", "foo"]]), + ( + "bar = foo & baz = fee", + ["&", [["bar", "=", "foo"], ["baz", "=", "fee"]]], + ), + ] + for test, expected in tests: + print(test) + results = f.parse_string(test, parse_all=True) + print(results) + self.assertParseResultsEquals(results, expected_list=expected) + print() + + def testInfixNotationGrammarTest5(self): + expop = pp.Literal("**") + signop = pp.one_of("+ -") + multop = pp.one_of("* /") + plusop = pp.one_of("+ -") + + class ExprNode: + def __init__(self, tokens): + self.tokens = tokens[0] + + def eval(self): + return None + + class NumberNode(ExprNode): + def eval(self): + return self.tokens + + class SignOp(ExprNode): + def eval(self): + mult = {"+": 1, "-": -1}[self.tokens[0]] + return mult * self.tokens[1].eval() + + class BinOp(ExprNode): + opn_map = {} + + def eval(self): + ret = self.tokens[0].eval() + for op, operand in zip(self.tokens[1::2], self.tokens[2::2]): + ret = self.opn_map[op](ret, operand.eval()) + return ret + + class ExpOp(BinOp): + opn_map = {"**": lambda a, b: b**a} + + class MultOp(BinOp): + import operator + + opn_map = {"*": operator.mul, "/": operator.truediv} + + class AddOp(BinOp): + import operator + + opn_map = {"+": operator.add, "-": operator.sub} + + operand = ppc.number().set_parse_action(NumberNode) + # fmt: off + expr = pp.infix_notation( + operand, + [ + (expop, 2, pp.OpAssoc.LEFT, (lambda pr: [pr[0][::-1]], ExpOp)), + (signop, 1, pp.OpAssoc.RIGHT, SignOp), + (multop, 2, pp.OpAssoc.LEFT, MultOp), + (plusop, 2, pp.OpAssoc.LEFT, AddOp), + ], + ) + # fmt: on + + tests = """\ + 2+7 + 2**3 + 2**3**2 + 3**9 + 3**3**2 + """ + + for t in tests.splitlines(): + t = t.strip() + if not t: + continue + + parsed = expr.parse_string(t, parse_all=True) + eval_value = parsed[0].eval() + self.assertEqual( + eval(t), + eval_value, + f"Error evaluating {t!r}, expected {eval(t)!r}, got {eval_value!r}", + ) + + def testInfixNotationExceptions(self): + num = pp.Word(pp.nums) + + # fmt: off + + # arity 3 with None opExpr - should raise ValueError + with self.assertRaises(ValueError): + expr = pp.infix_notation( + num, + [ + (None, 3, pp.OpAssoc.LEFT), + ] + ) + + # arity 3 with invalid tuple - should raise ValueError + with self.assertRaises(ValueError): + expr = pp.infix_notation( + num, + [ + (("+", "-", "*"), 3, pp.OpAssoc.LEFT), + ] + ) + + # left arity > 3 - should raise ValueError + with self.assertRaises(ValueError): + expr = pp.infix_notation( + num, + [ + ("*", 4, pp.OpAssoc.LEFT), + ] + ) + + # right arity > 3 - should raise ValueError + with self.assertRaises(ValueError): + expr = pp.infix_notation( + num, + [ + ("*", 4, pp.OpAssoc.RIGHT), + ] + ) + + # assoc not from OpAssoc - should raise ValueError + with self.assertRaises(ValueError): + expr = pp.infix_notation( + num, + [ + ("*", 2, "LEFT"), + ] + ) + # fmt: on + + def testInfixNotationWithNonOperators(self): + # left arity 2 with None expr + # right arity 2 with None expr + num = pp.Word(pp.nums).add_parse_action(pp.token_map(int)) + ident = ppc.identifier() + + # fmt: off + for assoc in (pp.OpAssoc.LEFT, pp.OpAssoc.RIGHT): + expr = pp.infix_notation( + num | ident, + [ + (None, 2, assoc), + ("+", 2, pp.OpAssoc.LEFT), + ] + ) + self.assertParseAndCheckList(expr, "3x+2", [[[3, "x"], "+", 2]]) + # fmt: on + + def testInfixNotationTernaryOperator(self): + # left arity 3 + # right arity 3 + num = pp.Word(pp.nums).add_parse_action(pp.token_map(int)) + + # fmt: off + for assoc in (pp.OpAssoc.LEFT, pp.OpAssoc.RIGHT): + expr = pp.infix_notation( + num, + [ + ("+", 2, pp.OpAssoc.LEFT), + (("?", ":"), 3, assoc), + ] + ) + self.assertParseAndCheckList( + expr, "3 + 2? 12: 13", [[[3, "+", 2], "?", 12, ":", 13]] + ) + # fmt: on + + def testInfixNotationWithAlternateParenSymbols(self): + num = pp.Word(pp.nums).add_parse_action(pp.token_map(int)) + + # fmt: off + expr = pp.infix_notation( + num, + [ + ("+", 2, pp.OpAssoc.LEFT), + ], + lpar="(", + rpar=")", + ) + self.assertParseAndCheckList( + expr, "3 + (2 + 11)", [[3, '+', [2, '+', 11]]] + ) + + expr = pp.infix_notation( + num, + [ + ("+", 2, pp.OpAssoc.LEFT), + ], + lpar="<", + rpar=">", + ) + self.assertParseAndCheckList( + expr, "3 + <2 + 11>", [[3, '+', [2, '+', 11]]] + ) + + expr = pp.infix_notation( + num, + [ + ("+", 2, pp.OpAssoc.LEFT), + ], + lpar=pp.Literal("<"), + rpar=pp.Literal(">"), + ) + self.assertParseAndCheckList( + expr, "3 + <2 + 11>", [[3, '+', ['<', [2, '+', 11], '>']]] + ) + + expr = pp.infix_notation( + num, + [ + ("+", 2, pp.OpAssoc.LEFT), + ], + lpar=pp.Literal("<<"), + rpar=pp.Literal(">>"), + ) + self.assertParseAndCheckList( + expr, "3 + <<2 + 11>>", [[3, '+', ['<<', [2, '+', 11], '>>']]] + ) + + # fmt: on + + def testParseResultsPickle(self): + import pickle + + # test 1 + body = pp.make_html_tags("BODY")[0] + result = body.parse_string( + "", parse_all=True + ) + print(result.dump()) + + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + print("Test pickle dump protocol", protocol) + try: + pickleString = pickle.dumps(result, protocol) + except Exception as e: + print("dumps exception:", e) + newresult = pp.ParseResults() + else: + newresult = pickle.loads(pickleString) + print(newresult.dump()) + + self.assertEqual( + result.dump(), + newresult.dump(), + f"Error pickling ParseResults object (protocol={protocol})", + ) + + # check all internal attributes + for attr in pp.ParseResults.__slots__: + # can't pickle/unpickle parent reference + if attr == "_parent": + continue + self.assertEqual( + getattr(result, attr), + getattr(newresult, attr), + f"Mismatch in ParseResults.{attr}" + ) + + + def testParseResultsPickle2(self): + import pickle + + word = pp.Word(pp.alphas + "'.") + salutation = pp.OneOrMore(word) + comma = pp.Literal(",") + greetee = pp.OneOrMore(word) + endpunc = pp.one_of("! ?") + greeting = ( + salutation("greeting") + + pp.Suppress(comma) + + greetee("greetee") + + endpunc("punc*")[1, ...] + ) + + string = "Good morning, Miss Crabtree!" + + result = greeting.parse_string(string, parse_all=True) + self.assertParseResultsEquals( + result, + ["Good", "morning", "Miss", "Crabtree", "!"], + { + "greeting": ["Good", "morning"], + "greetee": ["Miss", "Crabtree"], + "punc": ["!"], + }, + ) + print(result.dump()) + + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + print("Test pickle dump protocol", protocol) + try: + pickleString = pickle.dumps(result, protocol) + except Exception as e: + print("dumps exception:", e) + newresult = pp.ParseResults() + else: + newresult = pickle.loads(pickleString) + print(newresult.dump()) + self.assertEqual( + newresult.dump(), + result.dump(), + f"failed to pickle/unpickle ParseResults: expected {result!r}, got {newresult!r}", + ) + + def testParseResultsPickle3(self): + import pickle + + # result with aslist=False + res_not_as_list = pp.Word("ABC").parse_string("BABBAB", parse_all=True) + + # result with aslist=True + res_as_list = pp.Group(pp.Word("ABC")).parse_string("BABBAB", parse_all=True) + + # result with modal=True + res_modal = pp.Word("ABC")("name").parse_string("BABBAB", parse_all=True) + # self.assertTrue(res_modal._modal) + + # result with modal=False + res_not_modal = pp.Word("ABC")("name*").parse_string("BABBAB", parse_all=True) + # self.assertFalse(res_not_modal._modal) + + for result in (res_as_list, res_not_as_list, res_modal, res_not_modal): + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + print("Test pickle dump protocol", protocol) + try: + pickleString = pickle.dumps(result, protocol) + except Exception as e: + print("dumps exception:", e) + newresult = pp.ParseResults() + else: + newresult = pickle.loads(pickleString) + print(newresult.dump()) + self.assertEqual( + newresult.dump(), + result.dump(), + f"failed to pickle/unpickle ParseResults: expected {result!r}, got {newresult!r}", + ) + + def testParseResultsInsertWithResultsNames(self): + test_string = "1 2 3 dice rolled first try" + + wd = pp.Word(pp.alphas) + num = ppc.number + + expr = ( + pp.Group(num[1, ...])("nums") + + wd("label") + + pp.Group(wd[...])("additional") + ) + + result = expr.parse_string(test_string, parse_all=True) + print("Pre-insert") + print(result.dump()) + + result.insert(1, sum(result.nums)) + + print("\nPost-insert") + print(result.dump()) + + self.assertParseResultsEquals( + result, + expected_list=[[1, 2, 3], 6, "dice", ["rolled", "first", "try"]], + expected_dict={ + "additional": ["rolled", "first", "try"], + "label": "dice", + "nums": [1, 2, 3], + }, + ) + + def testParseResultsStringListUsingCombine(self): + test_string = "1 2 3 dice rolled first try" + + wd = pp.Word(pp.alphas) + num = ppc.number + + expr = pp.Combine( + pp.Group(num[1, ...])("nums") + + wd("label") + + pp.Group(wd[...])("additional"), + join_string="/", + adjacent=False, + ) + self.assertEqual( + "123/dice/rolledfirsttry", expr.parse_string(test_string, parse_all=True)[0] + ) + + def testParseResultsAcceptingACollectionTypeValue(self): + # from Issue #276 - ParseResults parameterizes generic types if passed as the value of toklist parameter + # https://github.com/pyparsing/pyparsing/issues/276?notification_referrer_id=MDE4Ok5vdGlmaWNhdGlvblRocmVhZDE4MzU4NDYwNzI6MzgzODc1 + # + # behavior of ParseResults code changed with Python 3.9 + + results_with_int = pp.ParseResults(toklist=int, name="type_", aslist=False) + self.assertEqual(int, results_with_int["type_"]) + + results_with_tuple = pp.ParseResults(toklist=tuple, name="type_", aslist=False) + self.assertEqual(tuple, results_with_tuple["type_"]) + + def testParseResultsReturningDunderAttribute(self): + # from Issue #208 + parser = pp.Word(pp.alphas)("A") + result = parser.parse_string("abc", parse_all=True) + print(result.dump()) + self.assertEqual("abc", result.A) + self.assertEqual("", result.B) + with self.assertRaises(AttributeError): + result.__xyz__ + + def testParseResultsNamedResultWithEmptyString(self): + # from Issue #470 + + # Check which values can be returned from a parse action + for test_value, expected_in_result_by_name in [ + ("x", True), + ("", True), + (True, True), + (False, True), + (1, True), + (0, True), + (None, True), + (b"", True), + (b"a", True), + ([], False), + ((), False), + ]: + msg = ( + f"value = {test_value!r}," + f" expected X {'not ' if not expected_in_result_by_name else ''}in result" + ) + with self.subTest(msg): + print(msg) + grammar = ( + (pp.Suppress("a") + pp.ZeroOrMore("x")) + .add_parse_action(lambda p: test_value) + .set_results_name("X") + ) + result = grammar.parse_string("a") + print(result.dump()) + if expected_in_result_by_name: + self.assertIn( + "X", + result, + f"Expected X not found for parse action value {test_value!r}", + ) + print(repr(result["X"])) + else: + self.assertNotIn( + "X", + result, + f"Unexpected X found for parse action value {test_value!r}", + ) + with self.assertRaises(KeyError): + print(repr(result["X"])) + print() + + # Do not add a parse result. + msg = "value = , expected X in result" + with self.subTest(msg): + print(msg) + grammar = (pp.Suppress("a") + pp.ZeroOrMore("x")).set_results_name("X") + result = grammar.parse_string("a") + print(result.dump()) + self.assertIn("X", result, f"Expected X not found with no parse action") + print() + + # Test by directly creating a ParseResults + print("Create empty string value directly") + result = pp.ParseResults("", name="X") + print(result.dump()) + self.assertIn( + "X", + result, + "failed to construct ParseResults with named value using empty string", + ) + print(repr(result["X"])) + print() + + print("Create empty string value from a dict") + result = pp.ParseResults.from_dict({"X": ""}) + print(result.dump()) + self.assertIn( + "X", + result, + "failed to construct ParseResults with named value using from_dict", + ) + print(repr(result["X"])) + + def testMatchOnlyAtCol(self): + """successfully use match_only_at_col helper function""" + + expr = pp.Word(pp.nums) + expr.set_parse_action(pp.match_only_at_col(5)) + largerExpr = pp.ZeroOrMore(pp.Word("A")) + expr + pp.ZeroOrMore(pp.Word("A")) + + res = largerExpr.parse_string("A A 3 A", parse_all=True) + print(res.dump()) + + def testMatchOnlyAtColErr(self): + """raise a ParseException in match_only_at_col with incorrect col""" + + expr = pp.Word(pp.nums) + expr.set_parse_action(pp.match_only_at_col(1)) + largerExpr = pp.ZeroOrMore(pp.Word("A")) + expr + pp.ZeroOrMore(pp.Word("A")) + + with self.assertRaisesParseException(): + largerExpr.parse_string("A A 3 A", parse_all=True) + + def testParseResultsWithNamedTuple(self): + expr = pp.Literal("A")("Achar") + expr.set_parse_action(pp.replace_with(tuple(["A", "Z"]))) + + res = expr.parse_string("A", parse_all=True) + print(repr(res)) + print(res.Achar) + self.assertParseResultsEquals( + res, + expected_dict={"Achar": ("A", "Z")}, + msg=f"Failed accessing named results containing a tuple, got {res.Achar!r}", + ) + + def testParserElementAddOperatorWithOtherTypes(self): + """test the overridden "+" operator with other data types""" + + # ParserElement + str + with self.subTest(): + expr = pp.Word(pp.alphas)("first") + pp.Word(pp.alphas)("second") + "suf" + result = expr.parse_string("spam eggs suf", parse_all=True) + print(result) + + expected_l = ["spam", "eggs", "suf"] + self.assertParseResultsEquals( + result, expected_l, msg="issue with ParserElement + str" + ) + + # str + ParserElement + with self.subTest(): + expr = "pre" + pp.Word(pp.alphas)("first") + pp.Word(pp.alphas)("second") + result = expr.parse_string("pre spam eggs", parse_all=True) + print(result) + + expected_l = ["pre", "spam", "eggs"] + self.assertParseResultsEquals( + result, expected_l, msg="issue with str + ParserElement" + ) + + # ParserElement + int + with self.subTest(): + expr = None + with self.assertRaises(TypeError, msg="failed to warn ParserElement + int"): + expr = pp.Word(pp.alphas)("first") + pp.Word(pp.alphas)("second") + 12 + self.assertEqual(expr, None) + + # int + ParserElement + with self.subTest(): + expr = None + with self.assertRaises(TypeError, msg="failed to warn int + ParserElement"): + expr = 12 + pp.Word(pp.alphas)("first") + pp.Word(pp.alphas)("second") + self.assertEqual(expr, None) + + def testParserElementSubOperatorWithOtherTypes(self): + """test the overridden "-" operator with other data types""" + + # ParserElement - str + with self.subTest(): + expr = pp.Word(pp.alphas)("first") + pp.Word(pp.alphas)("second") - "suf" + result = expr.parse_string("spam eggs suf", parse_all=True) + print(result) + expected = ["spam", "eggs", "suf"] + self.assertParseResultsEquals( + result, expected, msg="issue with ParserElement - str" + ) + + # str - ParserElement + with self.subTest(): + expr = "pre" - pp.Word(pp.alphas)("first") + pp.Word(pp.alphas)("second") + result = expr.parse_string("pre spam eggs", parse_all=True) + print(result) + expected = ["pre", "spam", "eggs"] + self.assertParseResultsEquals( + result, expected, msg="issue with str - ParserElement" + ) + + # ParserElement - int + with self.subTest(): + expr = None + with self.assertRaises(TypeError, msg="failed to warn ParserElement - int"): + expr = pp.Word(pp.alphas)("first") + pp.Word(pp.alphas)("second") - 12 + self.assertEqual(expr, None) + + # int - ParserElement + with self.subTest(): + expr = None + with self.assertRaises(TypeError, msg="failed to warn int - ParserElement"): + expr = 12 - pp.Word(pp.alphas)("first") + pp.Word(pp.alphas)("second") + self.assertEqual(expr, None) + + def testParserElementMulOperatorWithTuples(self): + """test ParserElement "*" with various tuples""" + + # ParserElement * (None, n) + expr = pp.Word(pp.alphas)("first") + pp.Word(pp.nums)("second*") * (None, 3) + + with self.subTest(): + results1 = expr.parse_string("spam", parse_all=True) + print(results1.dump()) + expected = ["spam"] + self.assertParseResultsEquals( + results1, expected, msg="issue with ParserElement * w/ optional matches" + ) + + with self.subTest(): + results2 = expr.parse_string("spam 12 23 34", parse_all=True) + print(results2.dump()) + expected = ["spam", "12", "23", "34"] + self.assertParseResultsEquals( + results2, expected, msg="issue with ParserElement * w/ optional matches" + ) + + # ParserElement * (1, 1) + with self.subTest(): + expr = pp.Word(pp.alphas)("first") + pp.Word(pp.nums)("second*") * (1, 1) + results = expr.parse_string("spam 45", parse_all=True) + print(results.dump()) + + expected = ["spam", "45"] + self.assertParseResultsEquals( + results, expected, msg="issue with ParserElement * (1, 1)" + ) + + # ParserElement * (1, 1+n) + with self.subTest(): + expr = pp.Word(pp.alphas)("first") + pp.Word(pp.nums)("second*") * (1, 3) + + results1 = expr.parse_string("spam 100", parse_all=True) + print(results1.dump()) + expected = ["spam", "100"] + self.assertParseResultsEquals( + results1, expected, msg="issue with ParserElement * (1, 1+n)" + ) + + with self.subTest(): + results2 = expr.parse_string("spam 100 200 300", parse_all=True) + print(results2.dump()) + expected = ["spam", "100", "200", "300"] + self.assertParseResultsEquals( + results2, expected, msg="issue with ParserElement * (1, 1+n)" + ) + + # ParserElement * (lesser, greater) + with self.subTest(): + expr = pp.Word(pp.alphas)("first") + pp.Word(pp.nums)("second*") * (2, 3) + + results1 = expr.parse_string("spam 1 2", parse_all=True) + print(results1.dump()) + expected = ["spam", "1", "2"] + self.assertParseResultsEquals( + results1, expected, msg="issue with ParserElement * (lesser, greater)" + ) + + with self.subTest(): + results2 = expr.parse_string("spam 1 2 3", parse_all=True) + print(results2.dump()) + expected = ["spam", "1", "2", "3"] + self.assertParseResultsEquals( + results2, expected, msg="issue with ParserElement * (lesser, greater)" + ) + + # ParserElement * (greater, lesser) + with self.subTest(): + with self.assertRaises( + ValueError, msg="ParserElement * (greater, lesser) should raise error" + ): + expr = pp.Word(pp.alphas)("first") + pp.Word(pp.nums)("second") * (3, 2) + + # ParserElement * (str, str) + with self.subTest(): + with self.assertRaises( + TypeError, msg="ParserElement * (str, str) should raise error" + ): + expr = pp.Word(pp.alphas)("first") + pp.Word(pp.nums)("second") * ( + "2", + "3", + ) + + def testBoundedRepetitionLargeUpperBound(self): + """issue #332 - expr[..., upper_bound] (and expr * (0, upper_bound)) + with a large upper_bound must not raise RecursionError during + construction or parsing. + """ + # Lower the recursion limit so a deeply nested implementation would + # fail deterministically, instead of depending on the host's default. + with self.resetting_recursion_limit(): + sys.setrecursionlimit(300) + # both the [...] shorthand and the explicit *(0, n) form + expr_shorthand = pp.Literal("A")[..., 1000] + expr_tuple = pp.Word(pp.nums) * (0, 1000) + + # check for recursion during parsing + expr_shorthand.parse_string("A " * 1000) + + # construction must succeed + self.assertIsNotNone(expr_shorthand) + self.assertIsNotNone(expr_tuple) + + # parsing a small input must succeed and yield the matched tokens + test_string = "A A A A A" + expected_list = test_string.split() + self.assertParseAndCheckList( + expr_shorthand, test_string, expected_list + ) + + # the upper bound is still enforced (only 3 words allowed before num) + with self.assertRaisesParseException(): + (pp.Word(pp.alphas)[..., 3] + pp.Word(pp.nums)).parse_string("a b c d 1") + + def testBoundedRepetitionLargeUpperBoundEarlyExit(self): + """issue #332 - the optional tail of ``expr[..., upper_bound]`` must + still *exit early* (like the original recursive implementation): once + the repeated expression can no longer match, parsing must stop instead + of attempting it for every remaining slot in the upper bound. + + This guards against a regression where the tail was flattened into a + list of independent ``Optional`` expressions, which always tried to + match the repeated expression for all ``upper_bound`` slots. + """ + def count_fails(*args): + count_fails.fail_counter += 1 + count_fails.fail_counter = 0 + + A_expr = pp.Literal("A").set_fail_action(count_fails) + expr = A_expr[..., 1000] + + # input has a single match, so a correct early-exit implementation + # must fail the repeated expression only once -- not + # once per slot in the (large) upper bound. + expr.parse_string("A") + self.assertEqual( + count_fails.fail_counter, + 1, + "bounded repetition did not exit early; it attempted the" + f" repeated expression {count_fails.fail_counter} times for a single match", + ) + + def testParserElementMulByZero(self): + alpwd = pp.Word(pp.alphas) + numwd = pp.Word(pp.nums) + + test_string = "abd def ghi jkl" + + with self.subTest(): + parser = alpwd * 2 + numwd * 0 + alpwd * 2 + self.assertParseAndCheckList( + parser, test_string, expected_list=test_string.split() + ) + + with self.subTest(): + parser = alpwd * 2 + numwd * (0, 0) + alpwd * 2 + self.assertParseAndCheckList( + parser, test_string, expected_list=test_string.split() + ) + + def testParserElementMulOperatorWithOtherTypes(self): + """test the overridden "*" operator with other data types""" + + # ParserElement * str + with self.subTest(): + with self.assertRaises( + TypeError, msg="ParserElement * str should raise error" + ): + expr = pp.Word(pp.alphas)("first") + pp.Word(pp.nums)("second") * "3" + + # str * ParserElement + with self.subTest(): + with self.assertRaises( + TypeError, msg="str * ParserElement should raise error" + ): + expr = pp.Word(pp.alphas)("first") + "3" * pp.Word(pp.nums)("second") + + # ParserElement * int + with self.subTest(): + expr = pp.Word(pp.alphas)("first") + pp.Word(pp.nums)("second*") * 2 + results = expr.parse_string("spam 11 22", parse_all=True) + + print(results.dump()) + expected = ["spam", "11", "22"] + self.assertParseResultsEquals( + results, expected, msg="issue with ParserElement * int" + ) + + # int * ParserElement + with self.subTest(): + expr = pp.Word(pp.alphas)("first") + 2 * pp.Word(pp.nums)("second*") + results = expr.parse_string("spam 111 222", parse_all=True) + + print(results.dump()) + expected = ["spam", "111", "222"] + self.assertParseResultsEquals( + results, expected, msg="issue with int * ParserElement" + ) + + def testParserElementMatchFirstOperatorWithOtherTypes(self): + """test the overridden "|" operator with other data types""" + + # ParserElement | int + with self.subTest(): + expr = None + with self.assertRaises(TypeError, msg="failed to warn ParserElement | int"): + expr = pp.Word(pp.alphas)("first") + (pp.Word(pp.alphas)("second") | 12) + self.assertEqual(expr, None) + + # int | ParserElement + with self.subTest(): + expr = None + with self.assertRaises(TypeError, msg="failed to warn int | ParserElement"): + expr = pp.Word(pp.alphas)("first") + (12 | pp.Word(pp.alphas)("second")) + self.assertEqual(expr, None) + + def testParserElementMatchLongestWithOtherTypes(self): + """test the overridden "^" operator with other data types""" + + # ParserElement ^ str + with self.subTest(): + expr = pp.Word(pp.alphas)("first") + (pp.Word(pp.nums)("second") ^ "eggs") + result = expr.parse_string("spam eggs", parse_all=True) + print(result) + + expected = ["spam", "eggs"] + self.assertParseResultsEquals( + result, expected, msg="issue with ParserElement ^ str" + ) + + # str ^ ParserElement + with self.subTest(): + expr = ("pre" ^ pp.Word("pr")("first")) + pp.Word(pp.alphas)("second") + result = expr.parse_string("pre eggs", parse_all=True) + print(result) + + expected = ["pre", "eggs"] + self.assertParseResultsEquals( + result, expected, msg="issue with str ^ ParserElement" + ) + + # ParserElement ^ int + with self.subTest(): + expr = None + with self.assertRaises(TypeError, msg="failed to warn ParserElement ^ int"): + expr = pp.Word(pp.alphas)("first") + (pp.Word(pp.alphas)("second") ^ 54) + self.assertEqual(expr, None) + + # int ^ ParserElement + with self.subTest(): + expr = None + with self.assertRaises(TypeError, msg="failed to warn int ^ ParserElement"): + expr = pp.Word(pp.alphas)("first") + (65 ^ pp.Word(pp.alphas)("second")) + self.assertEqual(expr, None) + + def testParserElementEachOperatorWithOtherTypes(self): + """test the overridden "&" operator with other data types""" + + # ParserElement & str + with self.subTest(): + expr = pp.Word(pp.alphas)("first") + (pp.Word(pp.alphas)("second") & "and") + with self.assertRaisesParseException(msg="issue with ParserElement & str"): + result = expr.parse_string("spam and eggs", parse_all=True) + + # str & ParserElement + with self.subTest(): + expr = pp.Word(pp.alphas)("first") + ("and" & pp.Word(pp.alphas)("second")) + result = expr.parse_string("spam and eggs", parse_all=True) + + print(result.dump()) + expected_l = ["spam", "and", "eggs"] + expected_d = {"first": "spam", "second": "eggs"} + self.assertParseResultsEquals( + result, + expected_list=expected_l, + expected_dict=expected_d, + msg="issue with str & ParserElement", + ) + + # ParserElement & int + with self.subTest(): + expr = None + with self.assertRaises(TypeError, msg="failed to warn ParserElement & int"): + expr = pp.Word(pp.alphas)("first") + (pp.Word(pp.alphas) & 78) + self.assertEqual(expr, None) + + # int & ParserElement + with self.subTest(): + expr = None + with self.assertRaises(TypeError, msg="failed to warn int & ParserElement"): + expr = pp.Word(pp.alphas)("first") + (89 & pp.Word(pp.alphas)) + self.assertEqual(expr, None) + + def testLshiftOperatorWithOtherTypes(self): + # Forward << ParserElement + with self.subTest(): + f = pp.Forward() + f << pp.Word(pp.alphas)[...] + test_string = "sljdf sldkjf Ljs" + result = f.parse_string(test_string) + print(result) + self.assertEqual(test_string.split(), result.as_list()) + + # Forward << str + with self.subTest(): + f = pp.Forward() + f << "AAA" + test_string = "AAA" + result = f.parse_string(test_string) + print(result) + self.assertEqual(test_string.split(), result.as_list()) + + # Forward << int + with self.subTest(): + f = pp.Forward() + with self.assertRaises(TypeError, msg="failed to warn int & ParserElement"): + f << 12 + + def testParserElementPassedThreeArgsToMultiplierShorthand(self): + """test the ParserElement form expr[m,n,o]""" + + with self.assertRaises( + TypeError, msg="failed to warn three index arguments to expr[m, n, o]" + ): + expr = pp.Word(pp.alphas)[2, 3, 4] + + def testParserElementPassedStrToMultiplierShorthand(self): + """test the ParserElement form expr[str]""" + + with self.assertRaises( + TypeError, msg="failed to raise expected error using string multiplier" + ): + expr2 = pp.Word(pp.alphas)["2"] + + def testParseResultsNewEdgeCases(self): + """test less common paths of ParseResults.__new__()""" + + parser = pp.Word(pp.alphas)[...] + result = parser.parse_string("sldkjf sldkjf", parse_all=True) + + # hasattr uses __getattr__, which for ParseResults will return "" if the + # results name is not defined. So hasattr() won't work with ParseResults. + # Have to use __contains__ instead to test for existence. + # self.assertFalse(hasattr(result, "A")) + self.assertFalse("A" in result) + + # create new ParseResults w/ None + result1 = pp.ParseResults(None) + print(result1.dump()) + self.assertParseResultsEquals( + result1, [], msg="ParseResults(None) should return empty ParseResults" + ) + + # create new ParseResults w/ integer name + result2 = pp.ParseResults(name=12) + print(result2.dump()) + self.assertEqual( + "12", + result2.get_name(), + "ParseResults int name should be accepted and converted to str", + ) + + # create new ParseResults w/ generator type + gen = (a for a in range(1, 6)) + result3 = pp.ParseResults(gen) + print(result3.dump()) + expected3 = [1, 2, 3, 4, 5] + self.assertParseResultsEquals( + result3, expected3, msg="issue initializing ParseResults w/ gen type" + ) + + def testParseResultsReversed(self): + """test simple case of reversed(ParseResults)""" + + tst = "1 2 3 4 5" + expr = pp.OneOrMore(pp.Word(pp.nums)) + result = expr.parse_string(tst, parse_all=True) + + reversed_list = [ii for ii in reversed(result)] + print(reversed_list) + expected = ["5", "4", "3", "2", "1"] + self.assertEqual( + expected, reversed_list, msg="issue calling reversed(ParseResults)" + ) + + def testParseResultsValues(self): + """test simple case of ParseResults.values()""" + + expr = pp.Word(pp.alphas)("first") + pp.Word(pp.alphas)("second") + result = expr.parse_string("spam eggs", parse_all=True) + + values_set = set(result.values()) + print(values_set) + expected = {"spam", "eggs"} + self.assertEqual( + expected, values_set, msg="issue calling ParseResults.values()" + ) + + def testParseResultsAppend(self): + """test simple case of ParseResults.append()""" + + # use a parse action to compute the sum of the parsed integers, and add it to the end + def append_sum(tokens): + tokens.append(sum(map(int, tokens))) + + expr = pp.OneOrMore(pp.Word(pp.nums)).add_parse_action(append_sum) + result = expr.parse_string("0 123 321", parse_all=True) + + expected = ["0", "123", "321", 444] + print(result.dump()) + self.assertParseResultsEquals( + result, expected, msg="issue with ParseResults.append()" + ) + + def testParseResultsArithmeticContract(self): + """test ParseResults.__add__ and __radd__ arithmetic contract""" + + import operator + + pr = pp.ParseResults(["a", "b"]) + + # ParseResults + ParseResults (normal case) + pr_sum = pr + pp.ParseResults(["c"]) + self.assertEqual(["a", "b", "c"], list(pr_sum)) + + # ParseResults + non-ParseResults -> TypeError (was AttributeError) + with self.assertRaises( + TypeError, + msg="ParseResults + int should raise TypeError, not AttributeError", + ): + pr + 1 + + with self.assertRaises( + TypeError, + msg="ParseResults + str should raise TypeError", + ): + pr + "extra" + + # non-zero int + ParseResults -> TypeError (was RecursionError) + with self.assertRaises( + TypeError, + msg="non-zero int + ParseResults should raise TypeError, not RecursionError", + ): + 1 + pr + + with self.assertRaises( + TypeError, + msg="float + ParseResults should raise TypeError", + ): + 1.5 + pr + + with self.assertRaises( + TypeError, + msg="str + ParseResults should raise TypeError", + ): + "x" + pr + + # 0 + ParseResults works (supports sum() builtin) + zero_sum = 0 + pr + self.assertEqual(["a", "b"], list(zero_sum)) + + # sum() of ParseResults objects works + total = sum( + [ + pp.ParseResults(["x"]), + pp.ParseResults(["y"]), + pp.ParseResults(["z"]), + ] + ) + self.assertEqual(["x", "y", "z"], list(total)) + + # reduce(operator.add, ...) also works + import functools + + reduced = functools.reduce( + operator.add, + [pp.ParseResults(["m"]), pp.ParseResults(["n"])], + ) + self.assertEqual(["m", "n"], list(reduced)) + + # ParseResults += non-ParseResults -> TypeError (was AttributeError), + # mirroring the __add__ contract above. + pr_iadd = pp.ParseResults(["a", "b"]) + with self.assertRaises( + TypeError, + msg="ParseResults += int should raise TypeError, not AttributeError", + ): + pr_iadd += 1 + with self.assertRaises( + TypeError, + msg="ParseResults += list should raise TypeError, not AttributeError", + ): + pr_iadd += ["extra"] + + # ParseResults += ParseResults still works + pr_iadd += pp.ParseResults(["c"]) + self.assertEqual(["a", "b", "c"], list(pr_iadd)) + + def testParseResultsClear(self): + """test simple case of ParseResults.clear()""" + + tst = "spam eggs" + expr = pp.Word(pp.alphas)("first") + pp.Word(pp.alphas)("second") + result = expr.parse_string(tst, parse_all=True) + + print(result.dump()) + self.assertParseResultsEquals( + result, ["spam", "eggs"], msg="issue with ParseResults before clear()" + ) + + result.clear() + + print(result.dump()) + self.assertParseResultsEquals( + result, + expected_list=[], + expected_dict={}, + msg="issue with ParseResults.clear()", + ) + + def testParseResultsExtendWithString(self): + """test ParseResults.extend() with input of type str""" + + # use a parse action to append the reverse of the matched strings to make a palindrome + def make_palindrome(tokens): + tokens.extend(reversed([t[::-1] for t in tokens])) + + tst = "abc def ghi" + expr = pp.OneOrMore(pp.Word(pp.alphas)) + result = expr.add_parse_action(make_palindrome).parse_string( + tst, parse_all=True + ) + print(result.dump()) + + expected = ["abc", "def", "ghi", "ihg", "fed", "cba"] + self.assertParseResultsEquals( + result, expected, msg="issue with ParseResults.extend(str)" + ) + + def testParseResultsExtendWithParseResults(self): + """test ParseResults.extend() with input of type ParseResults""" + + expr = pp.OneOrMore(pp.Word(pp.alphas)) + result1 = expr.parse_string("spam eggs", parse_all=True) + result2 = expr.parse_string("foo bar", parse_all=True) + + result1.extend(result2) + print(result1.dump()) + expected = ["spam", "eggs", "foo", "bar"] + self.assertParseResultsEquals( + result1, expected, msg="issue with ParseResults.extend(ParseResults)" + ) + + def testQuotedStringLoc(self): + expr = pp.QuotedString("'") + expr.add_parse_action(lambda t: t[0].upper()) + + test_string = "Using 'quotes' for 'sarcasm' or 'emphasis' is not good 'style'." + transformed = expr.transform_string(test_string) + print(test_string) + print(transformed) + expected = re.sub(r"'([^']+)'", lambda match: match[1].upper(), test_string) + self.assertEqual(expected, transformed) + + def testParseResultsWithNestedNames(self): + from pyparsing import ( + Dict, + Literal, + Group, + Optional, + Regex, + QuotedString, + one_of, + Or, + CaselessKeyword, + ZeroOrMore, + ) + + RELATION_SYMBOLS = "= > < >= <= <> ==" + + def _set_info(string, location, tokens): + for t in tokens: + try: + t["_info_"] = (string, location) + except TypeError: + pass + tokens["_info_"] = (string, location) + + def keywords(name): + words = "any all within encloses adj".split() + return Or(map(CaselessKeyword, words)) + + charString1 = Group(Regex(r'[^()=<>"/\s]+'))("identifier") + charString1.add_parse_action(_set_info) + charString2 = Group(QuotedString('"', "\\"))("quoted") + charString2.add_parse_action(_set_info) + + term = Group(charString1 | charString2) + modifier_key = charString1 + + # relations + comparitor_symbol = one_of(RELATION_SYMBOLS) + named_comparitors = keywords("comparitors") + comparitor = Group(comparitor_symbol | named_comparitors)("comparitor") + comparitor.add_parse_action(_set_info) + + def modifier_list1(key): + modifier = Dict( + Literal("/") + + Group(modifier_key(key))("name") + + Optional(comparitor_symbol("symbol") + term("value")) + )("modifier") + modifier.add_parse_action(_set_info) + return ZeroOrMore(modifier)("modifier_list") + + def modifier_list2(key): + modifier = Dict( + Literal("/") + + Group(modifier_key(key))("name") + + Optional(comparitor_symbol("symbol") + term("value")), + asdict=True, + )("modifier") + modifier.add_parse_action(_set_info) + return ZeroOrMore(modifier)("modifier_list") + + def modifier_list3(key): + modifier = Group( # this line is different from the others, must group to get results names + Dict( + Literal("/") + + Group(modifier_key(key))("name") + + Optional(comparitor_symbol("symbol") + term("value")) + ) + ) + modifier.add_parse_action(_set_info) + return ZeroOrMore(modifier)("modifier_list") + + def modifier_list4(key): + modifier = Dict( + Literal("/") + + Group(modifier_key(key))("name") + + Optional(comparitor_symbol("symbol") + term("value")), + asdict=True, + ) + modifier.add_parse_action(_set_info) + return ZeroOrMore(modifier)("modifier_list") + + for modifier_list_fn in ( + modifier_list1, + modifier_list2, + modifier_list3, + modifier_list4, + ): + modifier_parser = modifier_list_fn("default") + + result = modifier_parser.parse_string( + "/respectaccents/ignoreaccents", parse_all=True + ) + for r in result: + print(r) + print(r.get("_info_")) + self.assertEqual([0, 15], [r["_info_"][1] for r in result]) + + def testParseResultsFromDict(self): + """test helper classmethod ParseResults.from_dict()""" + + dict = { + "first": "123", + "second": 456, + "third": {"threeStr": "789", "threeInt": 789}, + } + name = "trios" + result = pp.ParseResults.from_dict(dict, name=name) + + print(result.dump()) + expected = {name: dict} + self.assertParseResultsEquals( + result, + expected_dict=expected, + msg="issue creating ParseResults.from _dict()", + ) + + def testParseResultsDir(self): + """test dir(ParseResults)""" + + dict = {"first": "123", "second": "456", "third": "789"} + name = "trios" + result = pp.ParseResults.from_dict(dict, name=name) + dir_result = dir(result) + + print(dir_result) + self.assertIn( + name, dir_result, msg="name value wasn't returned by dir(ParseResults)" + ) + self.assertIn( + "as_list", dir_result, msg="as_list was not returned by dir(ParseResults)" + ) + + def testParseResultsInsert(self): + """test ParseResults.insert() with named tokens""" + + from random import randint + + result = pp.Word(pp.alphas)[...].parse_string( + "A B C D E F G H I J", parse_all=True + ) + compare_list = result.as_list() + + print(result) + print(compare_list) + + for s in "abcdefghij": + index = randint(-5, 5) + result.insert(index, s) + compare_list.insert(index, s) + + print(result) + print(compare_list) + + self.assertParseResultsEquals( + result, compare_list, msg="issue with ParseResults.insert()" + ) + + def testParseResultsAddingSuppressedTokenWithResultsName(self): + parser = "aaa" + (pp.NoMatch() | pp.Suppress("-"))("B") + try: + dd = parser.parse_string("aaa -").as_dict() + except RecursionError: + self.fail("fail getting named result when empty") + + def testParseResultsBool(self): + result = pp.Word(pp.alphas)[...].parse_string("AAA", parse_all=True) + self.assertTrue(result, "non-empty ParseResults evaluated as False") + + result = pp.Word(pp.alphas)[...].parse_string("", parse_all=True) + self.assertFalse(result, "empty ParseResults evaluated as True") + + result["A"] = 0 + self.assertTrue( + result, + "ParseResults with empty list but containing a results name evaluated as False", + ) + + def testParseResultsWithAsListWithAndWithoutFlattening(self): + ppc = pp.common + + # define a recursive grammar so we can easily build nested ParseResults + LPAR, RPAR = pp.Suppress.using_each("()") + fn_call = pp.Forward() + fn_arg = fn_call | ppc.identifier | ppc.number + fn_call <<= ppc.identifier + pp.Group( + LPAR + pp.Optional(pp.DelimitedList(fn_arg)) + RPAR + ) + + tests = [ + ("random()", ["random", []]), + ("sin(theta)", ["sin", ["theta"]]), + ("sin(rad(30))", ["sin", ["rad", [30]]]), + ("sin(rad(30), rad(60, 180))", ["sin", ["rad", [30], "rad", [60, 180]]]), + ( + "sin(rad(30), rad(60, 180), alpha)", + ["sin", ["rad", [30], "rad", [60, 180], "alpha"]], + ), + ] + for test_string, expected in tests: + with self.subTest(): + print(test_string) + observed = fn_call.parse_string(test_string, parse_all=True) + print(observed.as_list()) + self.assertEqual(expected, observed.as_list()) + print(observed.as_list(flatten=True)) + self.assertEqual(flatten(expected), observed.as_list(flatten=True)) + print() + + def testParseResultsCopy(self): + expr = ( + pp.Word(pp.nums) + + pp.Group(pp.Word(pp.alphas)("key") + "=" + pp.Word(pp.nums)("value"))[...] + ) + result = expr.parse_string("1 a=100 b=200 c=300") + print(result.dump()) + + r2 = result.copy() + print(r2.dump()) + + # check copy is different, but contained results is the same as in original + self.assertFalse(r2 is result, "copy failed") + self.assertTrue(r2[1] is result[1], "shallow copy failed") + + # update result sub-element in place + result[1][0] = "z" + self.assertParseResultsEquals( + result, + expected_list=[ + "1", + ["z", "=", "100"], + ["b", "=", "200"], + ["c", "=", "300"], + ], + ) + + # update contained results, verify list and dict contents are updated as expected + result[1][0] = result[1]["key"] = "q" + result[1]["xyz"] = 1000 + print(result.dump()) + self.assertParseResultsEquals( + result, + expected_list=[ + "1", + ["q", "=", "100"], + ["b", "=", "200"], + ["c", "=", "300"], + ], + ) + self.assertParseResultsEquals( + result[1], expected_dict={"key": "q", "value": "100", "xyz": 1000} + ) + + # verify that list and dict contents are the same in copy + self.assertParseResultsEquals( + r2, + expected_list=[ + "1", + ["q", "=", "100"], + ["b", "=", "200"], + ["c", "=", "300"], + ], + ) + self.assertParseResultsEquals( + r2[1], expected_dict={"key": "q", "value": "100", "xyz": 1000} + ) + + def testParseResultsCopyResultsNamesAreIndependent(self): + # copy() shares the contained results, but renumbering the offsets of + # one must not renumber the other's: insert/pop/del shift the recorded + # positions, and a shared list would shift them for both. + expr = pp.Word(pp.nums) + pp.Word(pp.nums)("b") + + # mutating the copy must not disturb the original's names + result = expr.parse_string("1 2") + r2 = result.copy() + r2.insert(0, "X") + del result[0] + self.assertEqual("b", result.get_name()) + + # ...and the other way round + result = expr.parse_string("1 2") + r2 = result.copy() + result.insert(0, "X") + del r2[0] + self.assertEqual("b", r2.get_name()) + + def testParseResultsDeepcopy(self): + expr = ( + pp.Word(pp.nums) + + pp.Group( + pp.Word(pp.alphas)("key") + + "=" + + pp.Group( + pp.Word(pp.nums) + )("value") + )[...] + ) + result = expr.parse_string("1 a=100 b=200 c=300") + orig_elements = result._toklist[:] + + r2 = result.deepcopy() + print(r2.dump()) + + # check copy and contained results are different from original + self.assertFalse(r2 is result, "copy failed") + self.assertFalse(r2[1] is result[1], "deep copy failed") + + # check that the copy's list and dict entries are the same object + for i in (1, 2, 3): + self.assertTrue(r2[i][2] is r2[i].value) + + # check copy and original are equal + self.assertEqual(result.as_dict(), r2.as_dict()) + self.assertEqual(result.as_list(), r2.as_list()) + + # check original is unchanged + self.assertTrue( + all( + orig_element is result_element + for orig_element, result_element in zip(orig_elements, result._toklist) + ) + ) + + # update contained results + result[1][0] = result[1]["key"] = "q" + result[1]["xyz"] = 1000 + print(result.dump()) + + # verify that list and dict contents are unchanged in the copy + self.assertParseResultsEquals( + r2, + expected_list=[ + "1", + ["a", "=", ["100"]], + ["b", "=", ["200"]], + ["c", "=", ["300"]], + ], + ) + self.assertParseResultsEquals(r2[1], expected_dict={"key": "a", "value": ["100"]}) + + def testParseResultsDeepcopy2(self): + expr = ( + pp.Word(pp.nums) + + pp.Group( + pp.Word(pp.alphas)("key") + "=" + pp.Word(pp.nums)("value"), aslist=True + )[...] + ) + result = expr.parse_string("1 a=100 b=200 c=300") + + r2 = result.deepcopy() + print(r2.dump()) + + # check copy and contained results are different from original + self.assertFalse(r2 is result, "copy failed") + self.assertFalse(r2[1] is result[1], "deep copy failed") + + # update contained results + result[1][0] = "q" + print(result.dump()) + + # verify that list and dict contents are unchanged in the copy + self.assertParseResultsEquals( + r2, + expected_list=[ + "1", + ["a", "=", "100"], + ["b", "=", "200"], + ["c", "=", "300"], + ], + ) + + def testParseResultsDeepcopy3(self): + expr = ( + pp.Word(pp.nums) + + pp.Group( + ( + pp.Word(pp.alphas)("key") + "=" + pp.Word(pp.nums)("value") + ).add_parse_action(lambda t: tuple(t)) + )[...] + ) + result = expr.parse_string("1 a=100 b=200 c=300") + + r2 = result.deepcopy() + print(r2.dump()) + + # check copy and contained results are different from original + self.assertFalse(r2 is result, "copy failed") + self.assertFalse(r2[1] is result[1], "deep copy failed") + + # update contained results + result[1][0] = "q" + print(result.dump()) + + # verify that list and dict contents are unchanged in the copy + self.assertParseResultsEquals( + r2, + expected_list=[ + "1", + [("a", "=", "100")], + [("b", "=", "200")], + [("c", "=", "300")], + ], + ) + + def testParseResultsDeepcopy4(self): + # a top-level results name pointing at a mutable sub-ParseResults + # (the named token is also the list item) must be deep-copied, so the + # copy stays decoupled from the original whether reached by index or + # by name + expr = pp.Group(pp.Word(pp.nums)("a") + pp.Word(pp.nums)("b"))("grp") + pp.Word( + pp.alphas + )("w") + result = expr.parse_string("1 2 xyz") + + r2 = result.deepcopy() + + # named result and list item must be the same object within the copy, + # and both must differ from the original + self.assertTrue(r2["grp"] is r2[0], "named result diverged from list item") + self.assertFalse( + r2["grp"] is result["grp"], "deep copy failed for named result" + ) + + # mutate the original's named sub-result; the deep copy must not change + result["grp"][0] = result["grp"]["a"] = "999" + + self.assertParseResultsEquals( + r2, + expected_list=[["1", "2"], "xyz"], + expected_dict={"grp": {"a": "1", "b": "2"}, "w": "xyz"}, + ) + self.assertEqual({"a": "1", "b": "2"}, r2["grp"].as_dict()) + + def testIgnoreString(self): + """test ParserElement.ignore() passed a string arg""" + + tst = "I like totally like love pickles" + expr = pp.Word(pp.alphas)[...].ignore("like") + result = expr.parse_string(tst, parse_all=True) + + print(result) + expected = ["I", "totally", "love", "pickles"] + self.assertParseResultsEquals(result, expected, msg="issue with ignore(string)") + + def testParseHTMLTags(self): + test = """ + + + + + + + """ + results = [ + ("startBody", False, "", ""), + ("startBody", False, "#00FFCC", ""), + ("startBody", True, "#00FFAA", ""), + ("startBody", False, "#00FFBB", "black"), + ("startBody", True, "", ""), + ("endBody", False, "", ""), + ] + + bodyStart, bodyEnd = pp.make_html_tags("BODY") + resIter = iter(results) + for t, s, e in (bodyStart | bodyEnd).scan_string(test): + print(test[s:e], "->", t) + (expectedType, expectedEmpty, expectedBG, expectedFG) = next(resIter) + + print(t.dump()) + if "startBody" in t: + self.assertEqual( + expectedEmpty, + bool(t.empty), + f"expected {expectedEmpty and 'empty' or 'not empty'} token," + f" got {t.empty and 'empty' or 'not empty'}", + ) + self.assertEqual( + expectedBG, + t.bgcolor, + f"failed to match BGCOLOR, expected {expectedBG}, got {t.bgcolor}", + ) + self.assertEqual( + expectedFG, + t.fgcolor, + f"failed to match FGCOLOR, expected {expectedFG}, got {t.bgcolor}", + ) + elif "endBody" in t: + print("end tag") + pass + else: + print("BAD!!!") + + def testSetParseActionUncallableErr(self): + """raise a TypeError in set_parse_action() by adding uncallable arg""" + + expr = pp.Literal("A")("Achar") + uncallable = 12 + + with self.assertRaises(TypeError): + expr.set_parse_action(uncallable) + + res = expr.parse_string("A", parse_all=True) + print(res.dump()) + + def testMulWithNegativeNumber(self): + """raise a ValueError in __mul__ by multiplying a negative number""" + + with self.assertRaises(ValueError): + pp.Literal("A")("Achar") * (-1) + + def testMulWithEllipsis(self): + """multiply an expression with Ellipsis as ``expr * ...`` to match ZeroOrMore""" + + expr = pp.Literal("A")("Achar") * ... + res = expr.parse_string("A", parse_all=True) + self.assertEqual( + ["A"], res.as_list(), "expected expr * ... to match ZeroOrMore" + ) + print(res.dump()) + + def testUpcaseDowncaseUnicode(self): + import sys + + ppu = pp.pyparsing_unicode + + a = "\u00bfC\u00f3mo esta usted?" + if not JYTHON_ENV: + ualphas = ppu.alphas + else: + ualphas = "".join( + chr(i) + for i in list(range(0xD800)) + list(range(0xE000, sys.maxunicode)) + if chr(i).isalpha() + ) + uword = pp.Word(ualphas).set_parse_action(ppc.upcase_tokens) + + print = lambda *args: None + print(uword.search_string(a)) + + uword = pp.Word(ualphas).set_parse_action(ppc.downcase_tokens) + + print(uword.search_string(a)) + + kw = pp.Keyword("mykey", caseless=True).set_parse_action(ppc.upcase_tokens)( + "rname" + ) + ret = kw.parse_string("mykey", parse_all=True) + print(ret.rname) + self.assertEqual( + "MYKEY", ret.rname, "failed to upcase with named result (pyparsing_common)" + ) + + kw = pp.Keyword("MYKEY", caseless=True).set_parse_action(ppc.downcase_tokens)( + "rname" + ) + ret = kw.parse_string("mykey", parse_all=True) + print(ret.rname) + self.assertEqual("mykey", ret.rname, "failed to upcase with named result") + + if not IRON_PYTHON_ENV: + # test html data + html = " \ + Производитель, модель \ + BenQ-Siemens CF61 \ + " # .decode('utf-8') + + # 'Manufacturer, model + text_manuf = "Производитель, модель" + manufacturer = pp.Literal(text_manuf) + + td_start, td_end = pp.make_html_tags("td") + manuf_body = ( + td_start.suppress() + + manufacturer + + pp.SkipTo(td_end)("cells*") + + td_end.suppress() + ) + + def testRegexDeferredCompile(self): + """test deferred compilation of Regex patterns""" + re_expr = pp.Regex(r"[A-Z]*") + self.assertIsNone( + re_expr._may_return_empty, + "failed to initialize _may_return_empty flag to None", + ) + self.assertEqual(re_expr._re, None) + + compiled = re_expr.re + self.assertTrue( + re_expr._may_return_empty, "failed to set _may_return_empty flag to True" + ) + self.assertEqual(re_expr._re, compiled) + + non_empty_re_expr = pp.Regex(r"[A-Z]+") + self.assertIsNone( + non_empty_re_expr._may_return_empty, + "failed to initialize _may_return_empty flag to None", + ) + self.assertEqual(non_empty_re_expr._re, None) + + compiled = non_empty_re_expr.re + self.assertFalse( + non_empty_re_expr._may_return_empty, + "failed to set _may_return_empty flag to False", + ) + self.assertEqual(non_empty_re_expr._re, compiled) + + def testRegexDeferredCompileCommonHtmlEntity(self): + # this is the most important expression to defer, because it takes a long time to compile + perf_test_common_html_entity = pp.common_html_entity() + + # force internal var to None, to simulate a fresh instance + perf_test_common_html_entity._re = None + + # just how long does this take anyway? + from time import perf_counter + + start = perf_counter() + perf_test_common_html_entity.re # noqa + elapsed = perf_counter() - start + print(f"elapsed time to compile common_html_entity: {elapsed:.4f} sec") + + def testParseUsingRegex(self): + signedInt = pp.Regex(r"[-+][0-9]+") + unsignedInt = pp.Regex(r"[0-9]+") + simpleString = pp.Regex(r'("[^\"]*")|(\'[^\']*\')') + namedGrouping = pp.Regex(r'("(?P[^\"]*)")') + compiledRE = pp.Regex(re.compile(r"[A-Z]+")) + + def testMatch(expression, instring, shouldPass, expectedString=None): + if shouldPass: + try: + result = expression.parse_string(instring, parse_all=False) + print(f"{repr(expression)} correctly matched {repr(instring)}") + if expectedString != result[0]: + print("\tbut failed to match the pattern as expected:") + print( + f"\tproduced {repr(result[0])} instead of {repr(expectedString)}" + ) + return False + return True + except pp.ParseException: + print(f"{expression!r} incorrectly failed to match {instring!r}") + else: + try: + result = expression.parse_string(instring, parse_all=False) + print(f"{expression!r} incorrectly matched {instring!r}") + print(f"\tproduced {result[0]!r} as a result") + except pp.ParseException: + print(f"{expression!r} correctly failed to match {instring!r}") + return True + return False + + # These should fail + for i, (test_expr, test_string) in enumerate( + [ + (signedInt, "1234 foo"), + (signedInt, " +foo"), + (unsignedInt, "abc"), + (unsignedInt, "+123 foo"), + (simpleString, "foo"), + (simpleString, "\"foo bar'"), + (simpleString, "'foo bar\""), + (compiledRE, "blah"), + ], + start=1, + ): + with self.subTest(test_expr=test_expr, test_string=test_string): + self.assertTrue( + testMatch( + test_expr, + test_string, + False, + ), + f"Re: ({i}) passed, expected fail", + ) + + # These should pass + for i, (test_expr, test_string, expected_match) in enumerate( + [ + (signedInt, " +123", "+123"), + (signedInt, "+123", "+123"), + (signedInt, "+123 foo", "+123"), + (signedInt, "-0 foo", "-0"), + (unsignedInt, "123 foo", "123"), + (unsignedInt, "0 foo", "0"), + (simpleString, '"foo"', '"foo"'), + (simpleString, "'foo bar' baz", "'foo bar'"), + (compiledRE, "BLAH", "BLAH"), + (namedGrouping, '"foo bar" baz', '"foo bar"'), + ], + start=i + 1, + ): + with self.subTest(test_expr=test_expr, test_string=test_string): + self.assertTrue( + testMatch( + test_expr, + test_string, + True, + expected_match, + ), + f"Re: ({i}) failed, expected pass", + ) + + ret = namedGrouping.parse_string('"zork" blah', parse_all=False) + print(ret) + print(list(ret.items())) + print(ret.content) + self.assertEqual("zork", ret.content, "named group lookup failed") + + self.assertEqual( + simpleString.parse_string('"zork" blah', parse_all=False)[0], + ret[0], + "Regex not properly returning ParseResults for named vs. unnamed groups", + ) + + try: + print("lets try an invalid RE") + invRe = pp.Regex("(\"[^\"]*\")|('[^']*'").re + except ValueError as e: + print("successfully rejected an invalid RE:", end=" ") + print(e) + else: + self.fail("failed to reject invalid RE") + + with self.assertRaises( + ValueError, msg="failed to warn empty string passed to Regex" + ): + pp.Regex("").re # noqa + + def testRegexAsType(self): + test_str = "sldkjfj 123 456 lsdfkj" + + print("return as list of match groups") + expr = pp.Regex(r"\w+ (\d+) (\d+) (\w+)", as_group_list=True) + expected_group_list = [tuple(test_str.split()[1:])] + result = expr.parse_string(test_str, parse_all=True) + print(result.dump()) + print(expected_group_list) + self.assertParseResultsEquals( + result, + expected_list=expected_group_list, + msg="incorrect group list returned by Regex)", + ) + + print("return as re.match instance") + expr = pp.Regex( + r"\w+ (?P\d+) (?P\d+) (?P\w+)", as_match=True + ) + result = expr.parse_string(test_str, parse_all=True) + print(result.dump()) + print(result[0].groups()) + print(expected_group_list) + self.assertEqual( + {"num1": "123", "num2": "456", "last_word": "lsdfkj"}, + result[0].groupdict(), + "invalid group dict from Regex(as_match=True)", + ) + self.assertEqual( + expected_group_list[0], + result[0].groups(), + "incorrect group list returned by Regex(as_match)", + ) + + def testRegexSub(self): + print("test sub with string") + expr = pp.Regex(r"").sub("'Richard III'") + result = expr.transform_string("This is the title: <title>") + print(result) + self.assertEqual( + "This is the title: 'Richard III'", + result, + "incorrect Regex.sub result with simple string", + ) + + print("test sub with re string") + expr = pp.Regex(r"([Hh]\d):\s*(.*)").sub(r"<\1>\2</\1>") + result = expr.transform_string( + "h1: This is the main heading\nh2: This is the sub-heading" + ) + print(result) + self.assertEqual( + "<h1>This is the main heading</h1>\n<h2>This is the sub-heading</h2>", + result, + "incorrect Regex.sub result with re string", + ) + + print("test sub with re string (Regex returns re.match)") + expr = pp.Regex(r"([Hh]\d):\s*(.*)", as_match=True).sub(r"<\1>\2</\1>") + result = expr.transform_string( + "h1: This is the main heading\nh2: This is the sub-heading" + ) + print(result) + self.assertEqual( + "<h1>This is the main heading</h1>\n<h2>This is the sub-heading</h2>", + result, + "incorrect Regex.sub result with re string", + ) + + print("test sub with callable that return str") + expr = pp.Regex(r"<(.*?)>").sub(lambda m: m.group(1).upper()) + result = expr.transform_string("I want this in upcase: <what? what?>") + print(result) + self.assertEqual( + "I want this in upcase: WHAT? WHAT?", + result, + "incorrect Regex.sub result with callable", + ) + + with self.assertRaises(TypeError): + pp.Regex(r"<(.*?)>", as_match=True).sub(lambda m: m.group(1).upper()) + + with self.assertRaises(TypeError): + pp.Regex(r"<(.*?)>", as_group_list=True).sub(lambda m: m.group(1).upper()) + + with self.assertRaises(TypeError): + pp.Regex(r"<(.*?)>", as_group_list=True).sub("") + + def testRegexInvalidType(self): + """test Regex of an invalid type""" + + with self.assertRaises(TypeError, msg="issue with Regex of type int"): + expr = pp.Regex(12) + + def testRegexLoopPastEndOfString(self): + """test Regex matching after end of string""" + NL = pp.LineEnd().suppress() + empty_line = pp.rest_of_line() + NL + result = empty_line[1, 10].parse_string("\n\n") + self.assertEqual(3, len(result)) + + def testPrecededBy(self): + num = pp.Word(pp.nums).set_parse_action(lambda t: int(t[0])) + interesting_num = pp.PrecededBy(pp.Char("abc")("prefix*")) + num + semi_interesting_num = pp.PrecededBy("_") + num + crazy_num = pp.PrecededBy(pp.Word("^", "$%^")("prefix*"), 10) + num + boring_num = ~pp.PrecededBy(pp.Char("abc_$%^" + pp.nums)) + num + very_boring_num = pp.PrecededBy(pp.WordStart()) + num + finicky_num = pp.PrecededBy(pp.Word("^", "$%^"), retreat=3) + num + + s = "c384 b8324 _9293874 _293 404 $%^$^%$2939" + print(s) + for expr, expected_list, expected_dict in [ + (interesting_num, [384, 8324], {"prefix": ["c", "b"]}), + (semi_interesting_num, [9293874, 293], {}), + (boring_num, [404], {}), + (crazy_num, [2939], {"prefix": ["^%$"]}), + (finicky_num, [2939], {}), + (very_boring_num, [404], {}), + ]: + # print(expr.search_string(s)) + result = sum(expr.search_string(s)) + print(result.dump()) + self.assertParseResultsEquals(result, expected_list, expected_dict) + + # infinite loop test - from Issue #127 + string_test = "notworking" + # negs = pp.Or(['not', 'un'])('negs') + negs_pb = pp.PrecededBy("not", retreat=100)("negs_lb") + # negs_pb = pp.PrecededBy(negs, retreat=100)('negs_lb') + pattern = (negs_pb + pp.Literal("working"))("main") + + results = pattern.search_string(string_test) + try: + print(results.dump()) + except RecursionError: + self.fail("got maximum excursion limit exception") + else: + print("got maximum excursion limit exception") + + def testCountedArray(self): + testString = "2 5 7 6 0 1 2 3 4 5 0 3 5 4 3" + + integer = pp.Word(pp.nums).set_parse_action(lambda t: int(t[0])) + countedField = pp.counted_array(integer) + + r = pp.OneOrMore(pp.Group(countedField)).parse_string( + testString, parse_all=True + ) + print(testString) + print(r) + + self.assertParseResultsEquals( + r, expected_list=[[5, 7], [0, 1, 2, 3, 4, 5], [], [5, 4, 3]] + ) + + # addresses bug raised by Ralf Vosseler + def testCountedArrayTest2(self): + testString = "2 5 7 6 0 1 2 3 4 5 0 3 5 4 3" + + integer = pp.Word(pp.nums).set_parse_action(lambda t: int(t[0])) + countedField = pp.counted_array(integer) + + dummy = pp.Word("A") + r = pp.OneOrMore(pp.Group(dummy ^ countedField)).parse_string( + testString, parse_all=True + ) + print(testString) + print(r) + + self.assertParseResultsEquals( + r, expected_list=[[5, 7], [0, 1, 2, 3, 4, 5], [], [5, 4, 3]] + ) + + def testCountedArrayTest3(self): + int_chars = "_" + pp.alphas + array_counter = pp.Word(int_chars).set_parse_action( + lambda t: int_chars.index(t[0]) + ) + + # 123456789012345678901234567890 + testString = "B 5 7 F 0 1 2 3 4 5 _ C 5 4 3" + + integer = pp.Word(pp.nums).set_parse_action(lambda t: int(t[0])) + countedField = pp.counted_array(integer, int_expr=array_counter) + + r = pp.OneOrMore(pp.Group(countedField)).parse_string( + testString, parse_all=True + ) + print(testString) + print(r) + + self.assertParseResultsEquals( + r, expected_list=[[5, 7], [0, 1, 2, 3, 4, 5], [], [5, 4, 3]] + ) + + def testCountedArrayTest4(self): + ppc = pp.pyparsing_common + + # array counter contains several fields - first field *must* be the number of + # items in the array + # - number of elements + # - type of elements + # - source of elements + counter_with_metadata = ( + ppc.integer("count") + ppc.identifier("type") + ppc.identifier("source") + ) + + countedField = pp.counted_array( + pp.Word(pp.alphanums), int_expr=counter_with_metadata + ) + + testString = ( + "5 string input item1 item2 item3 item4 item5 0 int user 2 int file 3 8" + ) + r = pp.Group(countedField("items"))[...].parse_string( + testString, parse_all=True + ) + + print(testString) + print(r.dump()) + print(f"type = {r.type!r}") + print(f"source = {r.source!r}") + + self.assertParseResultsEquals( + r, + expected_list=[ + ["item1", "item2", "item3", "item4", "item5"], + [], + ["3", "8"], + ], + ) + + self.assertParseResultsEquals( + r[0], + expected_dict={ + "count": 5, + "source": "input", + "type": "string", + "items": ["item1", "item2", "item3", "item4", "item5"], + }, + ) + + # parse with additional fields between the count and the actual list items + count_with_metadata = ppc.integer + pp.Word(pp.alphas)("type") + typed_array = pp.counted_array( + pp.Word(pp.alphanums), int_expr=count_with_metadata + )("items") + result = typed_array.parse_string("3 bool True True False", parse_all=True) + print(result.dump()) + + self.assertParseResultsEquals( + result, + expected_list=["True", "True", "False"], + expected_dict={"type": "bool", "items": ["True", "True", "False"]}, + ) + + def testLineStart(self): + pass_tests = [ + """\ + AAA + BBB + """, + """\ + AAA... + BBB + """, + ] + fail_tests = [ + """\ + AAA... + ...BBB + """, + """\ + AAA BBB + """, + ] + + # cleanup test strings + pass_tests = [ + "\n".join(s.lstrip() for s in t.splitlines()).replace(".", " ") + for t in pass_tests + ] + fail_tests = [ + "\n".join(s.lstrip() for s in t.splitlines()).replace(".", " ") + for t in fail_tests + ] + + test_patt = pp.Word("A") - pp.LineStart() + pp.Word("B") + print(test_patt.streamline()) + success, _ = test_patt.run_tests(pass_tests) + self.assertTrue(success, "failed LineStart passing tests (1)") + + success, _ = test_patt.run_tests(fail_tests, failure_tests=True) + self.assertTrue(success, "failed LineStart failure mode tests (1)") + + with ppt.reset_pyparsing_context(): + print(r"no \n in default whitespace chars") + pp.ParserElement.set_default_whitespace_chars(" ") + + test_patt = pp.Word("A") - pp.LineStart() + pp.Word("B") + print(test_patt.streamline()) + # should fail the pass tests too, since \n is no longer valid whitespace and we aren't parsing for it + success, _ = test_patt.run_tests(pass_tests, failure_tests=True) + self.assertTrue(success, "failed LineStart passing tests (2)") + + success, _ = test_patt.run_tests(fail_tests, failure_tests=True) + self.assertTrue(success, "failed LineStart failure mode tests (2)") + + test_patt = ( + pp.Word("A") + - pp.LineEnd().suppress() + + pp.LineStart() + + pp.Word("B") + + pp.LineEnd().suppress() + ) + print(test_patt.streamline()) + success, _ = test_patt.run_tests(pass_tests) + self.assertTrue(success, "failed LineStart passing tests (3)") + + success, _ = test_patt.run_tests(fail_tests, failure_tests=True) + self.assertTrue(success, "failed LineStart failure mode tests (3)") + + def testLineStart2(self): + test = """\ + AAA 1 + AAA 2 + + AAA + + B AAA + + """ + + test = dedent(test) + print(pp.testing.with_line_numbers(test)) + + print("normal parsing") + for t, s, e in (pp.LineStart() + "AAA").scan_string(test): + print(s, e, pp.lineno(s, test), pp.line(s, test), repr(t)) + print() + self.assertEqual( + "A", t[0][0], "failed LineStart with insignificant newlines" + ) + + print(r"parsing without \n in whitespace chars") + with ppt.reset_pyparsing_context(): + pp.ParserElement.set_default_whitespace_chars(" ") + for t, s, e in (pp.LineStart() + "AAA").scan_string(test): + print(s, e, pp.lineno(s, test), pp.line(s, test), repr(test[s])) + print() + self.assertEqual( + "A", t[0][0], "failed LineStart with insignificant newlines" + ) + + def testLineStartWithLeadingSpaces(self): + # testing issue #272 + # reverted in 3.0.2 - LineStart() + expr will match expr even if there + # are leading spaces. To force "only at column 1" matching, use + # AtLineStart(expr). + instring = dedent( + """ + a + b + c + d + e + f + g + """ + ) + print(pp.testing.with_line_numbers(instring)) + + alpha_line = ( + pp.LineStart().leave_whitespace() + + pp.Word(pp.alphas) + + pp.LineEnd().suppress() + ) + + tests = [ + alpha_line, + pp.Group(alpha_line), + alpha_line | pp.Word("_"), + alpha_line | alpha_line, + pp.MatchFirst([alpha_line, alpha_line]), + alpha_line ^ pp.Word("_"), + alpha_line ^ alpha_line, + pp.Or([alpha_line, pp.Word("_")]), + pp.LineStart() + pp.Word(pp.alphas) + pp.LineEnd().suppress(), + pp.And([pp.LineStart(), pp.Word(pp.alphas), pp.LineEnd().suppress()]), + ] + fails = [] + for test in tests: + print(test.search_string(instring)) + if ["a", "b", "c", "d", "e", "f", "g"] != flatten( + sum(test.search_string(instring)).as_list() + ): + fails.append(test) + if fails: + self.fail( + "failed LineStart tests:\n{}".format( + "\n".join(str(expr) for expr in fails) + ) + ) + + def testAtLineStart(self): + test = dedent( + """\ + AAA this line + AAA and this line + AAA but not this one + B AAA and definitely not this one + """ + ) + + expr = pp.AtLineStart("AAA") + pp.rest_of_line + for t in expr.search_string(test): + print(t) + + self.assertEqual( + ["AAA", " this line", "AAA", " and this line"], + sum(expr.search_string(test)).as_list(), + ) + + def testStringStart(self): + self.assertParseAndCheckList( + pp.StringStart() + pp.Word(pp.nums), "123", ["123"] + ) + self.assertParseAndCheckList( + pp.StringStart() + pp.Word(pp.nums), " 123", ["123"] + ) + self.assertParseAndCheckList(pp.StringStart() + "123", "123", ["123"]) + self.assertParseAndCheckList(pp.StringStart() + "123", " 123", ["123"]) + self.assertParseAndCheckList(pp.AtStringStart(pp.Word(pp.nums)), "123", ["123"]) + + self.assertParseAndCheckList(pp.AtStringStart("123"), "123", ["123"]) + + with self.assertRaisesParseException(): + pp.AtStringStart(pp.Word(pp.nums)).parse_string(" 123") + + with self.assertRaisesParseException(): + pp.AtStringStart("123").parse_string(" 123") + + def testStringStartAndLineStartInsideAnd(self): + # fmt: off + P_MTARG = ( + pp.StringStart() + + pp.Word("abcde") + + pp.StringEnd() + ) + + P_MTARG2 = ( + pp.LineStart() + + pp.Word("abcde") + + pp.StringEnd() + ) + + P_MTARG3 = ( + pp.AtLineStart(pp.Word("abcde")) + + pp.StringEnd() + ) + # fmt: on + + def test(expr, string): + expr.streamline() + print(expr, repr(string), end=" ") + print(expr.parse_string(string)) + + test(P_MTARG, "aaa") + test(P_MTARG2, "aaa") + test(P_MTARG2, "\naaa") + test(P_MTARG2, " aaa") + test(P_MTARG2, "\n aaa") + + with self.assertRaisesParseException(): + test(P_MTARG3, " aaa") + with self.assertRaisesParseException(): + test(P_MTARG3, "\n aaa") + + def testLineAndStringEnd(self): + NLs = pp.OneOrMore(pp.line_end) + bnf1 = pp.DelimitedList(pp.Word(pp.alphanums).leave_whitespace(), NLs) + bnf2 = pp.Word(pp.alphanums) + pp.string_end + bnf3 = pp.Word(pp.alphanums) + pp.SkipTo(pp.string_end) + tests = [ + ("testA\ntestB\ntestC\n", ["testA", "testB", "testC"]), + ("testD\ntestE\ntestF", ["testD", "testE", "testF"]), + ("a", ["a"]), + ] + + for test, expected in tests: + res1 = bnf1.parse_string(test, parse_all=True) + print(res1, "=?", expected) + self.assertParseResultsEquals( + res1, + expected_list=expected, + msg=f"Failed line_end/string_end test (1): {test!r} -> {res1}", + ) + + res2 = bnf2.search_string(test)[0] + print(res2, "=?", expected[-1:]) + self.assertParseResultsEquals( + res2, + expected_list=expected[-1:], + msg=f"Failed line_end/string_end test (2): {test!r} -> {res2}", + ) + + res3 = bnf3.parse_string(test, parse_all=True) + first = res3[0] + rest = res3[1] + # ~ print res3.dump() + print(repr(rest), "=?", repr(test[len(first) + 1 :])) + self.assertEqual( + rest, + test[len(first) + 1 :], + msg=f"Failed line_end/string_end test (3): {test!r} -> {res3.as_list()}", + ) + print() + + k = pp.Regex(r"a+", flags=re.S + re.M) + k = k.parse_with_tabs() + k = k.leave_whitespace() + + tests = [ + (r"aaa", ["aaa"]), + (r"\naaa", None), + (r"a\naa", None), + (r"aaa\n", None), + ] + for i, (src, expected) in enumerate(tests): + with self.subTest("", src=src, expected=expected): + print(i, repr(src).replace("\\\\", "\\"), end=" ") + if expected is None: + with self.assertRaisesParseException(): + k.parse_string(src, parse_all=True) + else: + res = k.parse_string(src, parse_all=True) + self.assertParseResultsEquals( + res, expected, msg=f"Failed on parse_all=True test {i}" + ) + + def testVariableParseActionArgs(self): + pa3 = lambda s, l, t: t + pa2 = lambda l, t: t + pa1 = lambda t: t + pa0 = lambda: None + + class Callable3: + def __call__(self, s, l, t): + return t + + class Callable2: + def __call__(self, l, t): + return t + + class Callable1: + def __call__(self, t): + return t + + class Callable0: + def __call__(self): + return + + class CallableS3: + @staticmethod + def __call__(s, l, t): + return t + + class CallableS2: + @staticmethod + def __call__(l, t): + return t + + class CallableS1: + @staticmethod + def __call__(t): + return t + + class CallableS0: + @staticmethod + def __call__(): + return + + class CallableC3: + @classmethod + def __call__(cls, s, l, t): + return t + + class CallableC2: + @classmethod + def __call__(cls, l, t): + return t + + class CallableC1: + @classmethod + def __call__(cls, t): + return t + + class CallableC0: + @classmethod + def __call__(cls): + return + + class parseActionHolder: + @staticmethod + def pa3(s, l, t): + return t + + @staticmethod + def pa2(l, t): + return t + + @staticmethod + def pa1(t): + return t + + @staticmethod + def pa0(): + return + + def paArgs(*args): + print(args) + return args[2] + + class ClassAsPA0: + def __init__(self): + pass + + def __str__(self): + return "A" + + class ClassAsPA1: + def __init__(self, t): + print("making a ClassAsPA1") + self.t = t + + def __str__(self): + return self.t[0] + + class ClassAsPA2: + def __init__(self, l, t): + self.t = t + + def __str__(self): + return self.t[0] + + class ClassAsPA3: + def __init__(self, s, l, t): + self.t = t + + def __str__(self): + return self.t[0] + + class ClassAsPAStarNew(tuple): + def __new__(cls, *args): + print("make a ClassAsPAStarNew", args) + return tuple.__new__(cls, *args[2].as_list()) + + def __str__(self): + return "".join(self) + + A = pp.Literal("A").set_parse_action(pa0) + B = pp.Literal("B").set_parse_action(pa1) + C = pp.Literal("C").set_parse_action(pa2) + D = pp.Literal("D").set_parse_action(pa3) + E = pp.Literal("E").set_parse_action(Callable0()) + F = pp.Literal("F").set_parse_action(Callable1()) + G = pp.Literal("G").set_parse_action(Callable2()) + H = pp.Literal("H").set_parse_action(Callable3()) + I = pp.Literal("I").set_parse_action(CallableS0()) + J = pp.Literal("J").set_parse_action(CallableS1()) + K = pp.Literal("K").set_parse_action(CallableS2()) + L = pp.Literal("L").set_parse_action(CallableS3()) + M = pp.Literal("M").set_parse_action(CallableC0()) + N = pp.Literal("N").set_parse_action(CallableC1()) + O = pp.Literal("O").set_parse_action(CallableC2()) + P = pp.Literal("P").set_parse_action(CallableC3()) + Q = pp.Literal("Q").set_parse_action(paArgs) + R = pp.Literal("R").set_parse_action(parseActionHolder.pa3) + S = pp.Literal("S").set_parse_action(parseActionHolder.pa2) + T = pp.Literal("T").set_parse_action(parseActionHolder.pa1) + U = pp.Literal("U").set_parse_action(parseActionHolder.pa0) + V = pp.Literal("V") + + # fmt: off + gg = pp.OneOrMore( + A | B | C | D | E | F | G | H | I | J | K | L | M | N | O | P | Q | R | S | U | V | B | T + ) + # fmt: on + testString = "VUTSRQPONMLKJIHGFEDCBA" + res = gg.parse_string(testString, parse_all=True) + print(res) + self.assertParseResultsEquals( + res, + expected_list=list(testString), + msg="Failed to parse using variable length parse actions", + ) + + A = pp.Literal("A").set_parse_action(ClassAsPA0) + B = pp.Literal("B").set_parse_action(ClassAsPA1) + C = pp.Literal("C").set_parse_action(ClassAsPA2) + D = pp.Literal("D").set_parse_action(ClassAsPA3) + E = pp.Literal("E").set_parse_action(ClassAsPAStarNew) + + # fmt: off + gg = pp.OneOrMore( + A | B | C | D | E | F | G | H | I | J | K | L | M | N | O | P | Q | R | S | T | U | V + ) + # fmt: on + testString = "VUTSRQPONMLKJIHGFEDCBA" + res = gg.parse_string(testString, parse_all=True) + print(list(map(str, res))) + self.assertEqual( + list(testString), + list(map(str, res)), + "Failed to parse using variable length parse actions " + "using class constructors as parse actions", + ) + + def testSingleArgException(self): + testMessage = "just one arg" + try: + raise pp.ParseFatalException(testMessage) + except pp.ParseBaseException as pbe: + print("Received expected exception:", pbe) + raisedMsg = pbe.msg + self.assertEqual( + testMessage, raisedMsg, "Failed to get correct exception message" + ) + + def testOriginalTextFor(self): + def rfn(t): + return f"{t.src}:{len(''.join(t))}" + + makeHTMLStartTag = lambda tag: pp.original_text_for( + pp.make_html_tags(tag)[0], as_string=False + ) + + # use the lambda, Luke + start = makeHTMLStartTag("IMG") + + # don't replace our fancy parse action with rfn, + # append rfn to the list of parse actions + start.add_parse_action(rfn) + + text = """_<img src="images/cal.png" + alt="cal image" width="16" height="15">_""" + s = start.transform_string(text) + print(s) + self.assertTrue( + s.startswith("_images/cal.png:"), "failed to preserve input s properly" + ) + self.assertTrue( + s.endswith("77_"), "failed to return full original text properly" + ) + + tag_fields = makeHTMLStartTag("IMG").search_string(text)[0] + print(sorted(tag_fields.keys())) + self.assertEqual( + ["alt", "empty", "height", "src", "startImg", "tag", "width"], + sorted(tag_fields.keys()), + "failed to preserve results names in original_text_for", + ) + + def testPackratParsingCacheCopy(self): + integer = pp.Word(pp.nums).set_name("integer") + id = pp.Word(pp.alphas + "_", pp.alphanums + "_") + simpleType = pp.Literal("int") + arrayType = simpleType + ("[" + pp.DelimitedList(integer) + "]")[...] + varType = arrayType | simpleType + varDec = varType + pp.DelimitedList(id + pp.Optional("=" + integer)) + ";" + + codeBlock = pp.Literal("{}") + + funcDef = ( + pp.Optional(varType | "void") + + id + + "(" + + (pp.DelimitedList(varType + id) | "void" | pp.empty) + + ")" + + codeBlock + ) + + program = varDec | funcDef + input = "int f(){}" + self.assertParseAndCheckList( + program, + input, + ["int", "f", "(", ")", "{}"], + msg="Error in packrat parsing", + verbose=True, + ) + + def testPackratParsingCacheCopyTest2(self): + DO, AA = list(map(pp.Keyword, "DO AA".split())) + LPAR, RPAR = list(map(pp.Suppress, "()")) + identifier = ~AA + pp.Word("Z") + + function_name = identifier.copy() + # ~ function_name = ~AA + Word("Z") #identifier.copy() + expr = pp.Forward().set_name("expr") + expr <<= pp.Group( + function_name + LPAR + pp.Optional(pp.DelimitedList(expr)) + RPAR + ).set_name("functionCall") | identifier.set_name( + "ident" + ) # .set_debug()#.set_break() + + stmt = DO + pp.Group(pp.DelimitedList(identifier + ".*" | expr)) + result = stmt.parse_string("DO Z", parse_all=True) + print(result.as_list()) + self.assertEqual( + 1, len(result[1]), "packrat parsing is duplicating And term exprs" + ) + + def testParseResultsDel(self): + grammar = pp.OneOrMore(pp.Word(pp.nums))("ints") + pp.OneOrMore( + pp.Word(pp.alphas) + )("words") + res = grammar.parse_string("123 456 ABC DEF", parse_all=True) + print(res.dump()) + origInts = res.ints.as_list() + origWords = res.words.as_list() + del res[1] + del res["words"] + print(res.dump()) + self.assertEqual("ABC", res[1], "failed to delete 0'th element correctly") + self.assertEqual( + origInts, + res.ints.as_list(), + "updated named attributes, should have updated list only", + ) + self.assertEqual("", res.words, "failed to update named attribute correctly") + self.assertEqual( + "DEF", res[-1], "updated list, should have updated named attributes only" + ) + + # delete a slice (modifies list, leaves dict intact) + del res[:-1] + self.assertEqual(res.as_list(), ["DEF"]) + self.assertEqual(res.as_dict(), {"ints": ["123", "456"]}) + + def test_delitem_slices(self): + results = pp.Char(pp.alphas)("char*")[...].parse_string(pp.alphas[:10]) + chars_list = list(pp.alphas[:10]) + self.assertEqual(results.as_list(), chars_list) + print(results.char) + self.assertEqual(results.char.as_list(), chars_list) + + del results[::2] + print(results.as_list()) + self.assertEqual(results.as_list(), chars_list[1::2]) + print(results.char) + self.assertEqual(results.char.as_list(), chars_list) + + del results[:] + print(results.as_list()) + self.assertEqual(results.as_list(), []) + self.assertEqual(results.char.as_list(), chars_list) + + def test_delitem_slices_copy(self): + results = pp.Char(pp.alphas)("char*")[...].parse_string(pp.alphas[:10]) + chars_list = list(pp.alphas[:10]) + self.assertEqual(results.as_list(), chars_list) + print(results.char) + self.assertEqual(results.char.as_list(), chars_list) + + res2 = results.copy() + + del results[::2] + print(results.as_list()) + self.assertEqual(results.as_list(), chars_list[1::2]) + self.assertEqual(res2.as_list(), chars_list) + print(results.char) + self.assertEqual(results.char.as_list(), chars_list) + + del results[:] + print(results.as_list()) + self.assertEqual(results.as_list(), []) + self.assertEqual(res2.as_list(), chars_list) + self.assertEqual(results.char.as_list(), chars_list) + + def testWithAttributeParseAction(self): + """ + This unit test checks with_attribute in these ways: + + * Argument forms as keywords and tuples + * Selecting matching tags by attribute + * Case-insensitive attribute matching + * Correctly matching tags having the attribute, and rejecting tags not having the attribute + + (Unit test written by voigts as part of the Google Highly Open Participation Contest) + """ + + data = """ + <a>1</a> + <a b="x">2</a> + <a B="x">3</a> + <a b="X">4</a> + <a b="y">5</a> + <a class="boo">8</ a> + """ + tagStart, tagEnd = pp.make_html_tags("a") + + expr = tagStart + pp.Word(pp.nums)("value") + tagEnd + + expected = ( + [ + ["a", ["b", "x"], False, "2", "</a>"], + ["a", ["b", "x"], False, "3", "</a>"], + ], + [ + ["a", ["b", "x"], False, "2", "</a>"], + ["a", ["b", "x"], False, "3", "</a>"], + ], + [["a", ["class", "boo"], False, "8", "</a>"]], + ) + + for attrib, exp in zip( + [ + pp.with_attribute(b="x"), + # with_attribute(B="x"), + pp.with_attribute(("b", "x")), + # with_attribute(("B", "x")), + pp.with_class("boo"), + ], + expected, + ): + tagStart.set_parse_action(attrib) + result = expr.search_string(data) + + print(result.dump()) + self.assertParseResultsEquals( + result, + expected_list=exp, + msg=f"Failed test, expected {expected}, got {result.as_list()}", + ) + + def testNestedExpressions(self): + """ + This unit test checks nested_expr in these ways: + - use of default arguments + - use of non-default arguments (such as a pyparsing-defined comment + expression in place of quoted_string) + - use of a custom content expression + - use of a pyparsing expression for opener and closer is *OPTIONAL* + - use of input data containing nesting delimiters + - correct grouping of parsed tokens according to nesting of opening + and closing delimiters in the input string + + (Unit test written by christoph... as part of the Google Highly Open Participation Contest) + """ + + # All defaults. Straight out of the example script. Also, qualifies for + # the bonus: note the fact that (Z | (E^F) & D) is not parsed :-). + # Tests for bug fixed in 1.4.10 + print("Test defaults:") + teststring = "((ax + by)*C) (Z | (E^F) & D)" + + expr = pp.nested_expr() + + expected = [[["ax", "+", "by"], "*C"]] + result = expr.parse_string(teststring, parse_all=False) + print(result.dump()) + self.assertParseResultsEquals( + result, + expected_list=expected, + msg=f"Defaults didn't work. That's a bad sign. Expected: {expected}, got: {result}", + ) + + # Going through non-defaults, one by one; trying to think of anything + # odd that might not be properly handled. + + # Change opener + print("\nNon-default opener") + teststring = "[[ ax + by)*C)" + expected = [[["ax", "+", "by"], "*C"]] + expr = pp.nested_expr("[") + self.assertParseAndCheckList( + expr, + teststring, + expected, + f"Non-default opener didn't work. Expected: {expected}, got: {result}", + verbose=True, + ) + + # Change closer + print("\nNon-default closer") + + teststring = "((ax + by]*C]" + expected = [[["ax", "+", "by"], "*C"]] + expr = pp.nested_expr(closer="]") + self.assertParseAndCheckList( + expr, + teststring, + expected, + f"Non-default closer didn't work. Expected: {expected}, got: {result}", + verbose=True, + ) + + # #Multicharacter opener, closer + # opener = "bar" + # closer = "baz" + print("\nLiteral expressions for opener and closer") + + opener, closer = map(pp.Literal, "bar baz".split()) + expr = pp.nested_expr( + opener, closer, content=pp.Regex(r"([^b ]|b(?!a)|ba(?![rz]))+") + ) + + teststring = "barbar ax + bybaz*Cbaz" + expected = [[["ax", "+", "by"], "*C"]] + self.assertParseAndCheckList( + expr, + teststring, + expected, + f"Multicharacter opener and closer didn't work. Expected: {expected}, got: {result}", + verbose=True, + ) + + # Lisp-ish comments + print("\nUse ignore expression (1)") + comment = pp.Regex(r";;.*") + teststring = """ + (let ((greeting "Hello, world!")) ;;(foo bar + (display greeting)) + """ + + expected = [ + [ + "let", + [["greeting", '"Hello,', 'world!"']], + ";;(foo bar", + ["display", "greeting"], + ] + ] + expr = pp.nested_expr(ignore_expr=comment) + self.assertParseAndCheckList( + expr, + teststring, + expected, + f'Lisp-ish comments (";; <...> $") didn\'t work. Expected: {expected}, got: {result}', + verbose=True, + ) + + # Lisp-ish comments, using a standard bit of pyparsing, and an Or. + print("\nUse ignore expression (2)") + comment = ";;" + pp.rest_of_line + + teststring = """ + (let ((greeting "Hello, )world!")) ;;(foo bar + (display greeting)) + """ + + expected = [ + [ + "let", + [["greeting", '"Hello, )world!"']], + ";;", + "(foo bar", + ["display", "greeting"], + ] + ] + expr = pp.nested_expr(ignore_expr=(comment ^ pp.quoted_string)) + self.assertParseAndCheckList( + expr, + teststring, + expected, + f'Lisp-ish comments (";; <...> $") and quoted strings didn\'t work. Expected: {expected}, got: {result}', + verbose=True, + ) + + def testNestedExpressions2(self): + """test nested_expr with conditions that explore other paths + + identical opener and closer + opener and/or closer of type other than string or iterable + multi-character opener and/or closer + single character opener and closer with ignore_expr=None + multi-character opener and/or closer with ignore_expr=None + """ + + name = pp.Word(pp.alphanums + "_") + + # identical opener and closer + with self.assertRaises( + ValueError, msg="matching opener and closer should raise error" + ): + expr = name + pp.nested_expr(opener="{", closer="{") + + # opener and/or closer of type other than string or iterable + with self.assertRaises( + ValueError, msg="opener and closer as ints should raise error" + ): + expr = name + pp.nested_expr(opener=12, closer=18) + + # multi-character opener and/or closer + tstMulti = "aName {{ outer {{ 'inner with opener {{ and closer }} in quoted string' }} }}" + expr = name + pp.nested_expr(opener="{{", closer="}}") + result = expr.parse_string(tstMulti, parse_all=True) + expected = [ + "aName", + ["outer", ["'inner with opener {{ and closer }} in quoted string'"]], + ] + print(result.dump()) + self.assertParseResultsEquals( + result, expected, msg="issue with multi-character opener and closer" + ) + + # single character opener and closer with ignore_expr=None + tst = "aName { outer { 'inner with opener { and closer } in quoted string' }}" + expr = name + pp.nested_expr(opener="{", closer="}", ignore_expr=None) + singleCharResult = expr.parse_string(tst, parse_all=True) + print(singleCharResult.dump()) + + # multi-character opener and/or closer with ignore_expr=None + expr = name + pp.nested_expr(opener="{{", closer="}}", ignore_expr=None) + multiCharResult = expr.parse_string(tstMulti, parse_all=True) + print(multiCharResult.dump()) + + self.assertParseResultsEquals( + singleCharResult, + multiCharResult.as_list(), + msg="using different openers and closers shouldn't affect resulting ParseResults", + ) + + def testNestedExpressions3(self): + + prior_ws_chars = pp.ParserElement.DEFAULT_WHITE_CHARS + with ppt.reset_pyparsing_context(): + pp.ParserElement.set_default_whitespace_chars("") + + input_str = dedent( + """\ + selector + { + a:b; + c:d; + selector + { + a:b; + c:d; + } + y:z; + }""" + ) + + print(ppt.with_line_numbers(input_str, 1, 100)) + + nested_result = ( + pp.nested_expr("{", "}").parse_string("{" + input_str + "}").as_list() + ) + expected_result = [ + [ + "selector\n", + [ + "\n a:b;\n c:d;\n selector\n ", + ["\n a:b;\n c:d;\n "], + "\n y:z;\n", + ], + ] + ] + self.assertEqual(nested_result, expected_result) + + # make sure things have been put back properly + self.assertEqual(pp.ParserElement.DEFAULT_WHITE_CHARS, prior_ws_chars) + + def testNestedExpressions4(self): + allowed = pp.alphas + plot_options_short = pp.nested_expr( + "[", "]", content=pp.OneOrMore(pp.Word(allowed) ^ pp.quoted_string) + ).set_results_name("plot_options") + + self.assertParseAndCheckList( + plot_options_short, + "[slkjdfl sldjf [lsdf'lsdf']]", + [["slkjdfl", "sldjf", ["lsdf", "'lsdf'"]]], + ) + + def testNestedExpressionDoesNotOverwriteParseActions(self): + content = pp.Word(pp.nums + " ") + + content.add_parse_action(lambda t: None) + orig_pa = content.parseAction[0] + + expr = pp.nested_expr(content=content) + assert content.parseAction[0] is orig_pa + + def testNestedExpressionRandom(self): + import random + + word_chars = pp.alphanums + + def get_random_character(_charset=word_chars + " "): + return random.choice(_charset) + + def create_random_quoted_string(): + quote_char = random.choice(('"', "'")) + yield quote_char + yield from (get_random_character() for _ in range(random.randint(0, 12))) + yield quote_char + + def create_random_nested_expression(): + yield "[" + + if random.random() < 0.25: + yield from create_random_quoted_string() + + for _ in range(random.randint(0, 16)): + rnd = random.random() + if rnd < 0.25: + yield from create_random_quoted_string() + elif rnd < 0.3: + yield from create_random_nested_expression() + else: + yield from ( + get_random_character() for _ in range(random.randint(1, 4)) + ) + + if random.random() < 0.25: + yield from create_random_quoted_string() + + yield "]" + + num_reps = 150 + + # simulate nested_expr + LBRACK, RBRACK = pp.Suppress.using_each("[]") + wd = pp.Word(word_chars) + qs = pp.quoted_string() + ls = pp.Forward() + ls <<= pp.Group(LBRACK + (qs | ls | wd)[...] + RBRACK) + + def crack_nested_string(s) -> list: + return ls.parse_string(s, parse_all=True).as_list() + + expr = pp.nested_expr("[", "]") + for _ in range(num_reps): + nested_str = "".join(create_random_nested_expression()) + # print(nested_str) + cracked_result = crack_nested_string(nested_str) + self.assertParseAndCheckList( + expr, + nested_str, + cracked_result, + f"Failed: {nested_str}, expected {cracked_result}", + verbose=False, + ) + + # test multi-character nesting delimiters + expr = pp.nested_expr("<<", ">>") + for _ in range(num_reps): + nested_str = "".join(create_random_nested_expression()) + # print(nested_str) + cracked_result = crack_nested_string(nested_str) + nested_str = nested_str.replace("[", "<<").replace("]", ">>") + self.assertParseAndCheckList( + expr, + nested_str, + cracked_result, + f"Failed: {nested_str}, expected {cracked_result}", + verbose=False, + ) + + # test with no ignore_expr (no quoted string handling) + expr = pp.nested_expr("[", "]", ignore_expr=None) + for _ in range(num_reps): + nested_str = "".join(create_random_nested_expression()) + nested_str = nested_str.replace('"', "").replace("'", "") + # print(nested_str) + cracked_result = crack_nested_string(nested_str) + self.assertParseAndCheckList( + expr, + nested_str, + cracked_result, + f"Failed: {nested_str}, expected {cracked_result}", + verbose=False, + ) + + # test multi-character nesting delimiters, with no ignore_expr + expr = pp.nested_expr("<<", ">>", ignore_expr=None) + for _ in range(num_reps): + nested_str = "".join(create_random_nested_expression()) + nested_str = nested_str.replace('"', "").replace("'", "") + # print(nested_str) + cracked_result = crack_nested_string(nested_str) + nested_str = nested_str.replace("[", "<<").replace("]", ">>") + self.assertParseAndCheckList( + expr, + nested_str, + cracked_result, + f"Failed: {nested_str}, expected {cracked_result}", + verbose=False, + ) + + def testWordMinMaxArgs(self): + parsers = [ + "A" + pp.Word(pp.nums), + "A" + pp.Word(pp.nums, min=1), + "A" + pp.Word(pp.nums, max=6), + "A" + pp.Word(pp.nums, min=1, max=6), + "A" + pp.Word(pp.nums, min=1), + "A" + pp.Word(pp.nums, min=2), + "A" + pp.Word(pp.nums, min=2, max=6), + pp.Word("A", pp.nums), + pp.Word("A", pp.nums, min=1), + pp.Word("A", pp.nums, max=6), + pp.Word("A", pp.nums, min=1, max=6), + pp.Word("A", pp.nums, min=1), + pp.Word("A", pp.nums, min=2), + pp.Word("A", pp.nums, min=2, max=6), + pp.Word(pp.alphas, pp.nums), + pp.Word(pp.alphas, pp.nums, min=1), + pp.Word(pp.alphas, pp.nums, max=6), + pp.Word(pp.alphas, pp.nums, min=1, max=6), + pp.Word(pp.alphas, pp.nums, min=1), + pp.Word(pp.alphas, pp.nums, min=2), + pp.Word(pp.alphas, pp.nums, min=2, max=6), + ] + + fails = [] + for p in parsers: + print(p, getattr(p, "reString", "..."), end=" ", flush=True) + try: + p.parse_string("A123", parse_all=True) + except Exception as e: + print(" <<< FAIL") + fails.append(p) + else: + print() + if fails: + self.fail(f"{','.join(str(f) for f in fails)} failed to match") + + def testWordMinMaxExactArgs(self): + for minarg in range(1, 9): + for maxarg in range(minarg, 10): + with self.subTest(minarg=minarg, maxarg=maxarg): + expr = pp.Word("AB", pp.nums, min=minarg, max=maxarg) + print(minarg, maxarg, expr.reString, end=" ") + trailing = expr.reString.rpartition("]")[-1] + expected_special = { + (1, 1): "", + (1, 2): "?", + (2, 2): "", + } + expected_default = ( + f"{{{minarg - 1}}}" + if minarg == maxarg + else f"{{{minarg - 1},{maxarg - 1}}}" + ) + expected = expected_special.get((minarg, maxarg), expected_default) + + print(trailing == expected) + + self.assertEqual(trailing, expected) + + self.assertParseAndCheckList( + expr + pp.rest_of_line.suppress(), + "A1234567890", + ["A1234567890"[:maxarg]], + ) + + for exarg in range(1, 9): + with self.subTest(exarg=exarg): + expr = pp.Word("AB", pp.nums, exact=exarg) + print(exarg, expr.reString, end=" ") + trailing = expr.reString.rpartition("]")[-1] + if exarg < 3: + expected = "" + else: + expected = f"{{{exarg - 1}}}" + print(trailing == expected) + + self.assertEqual(trailing, expected) + + self.assertParseAndCheckList( + expr + pp.rest_of_line.suppress(), + "A1234567890", + ["A1234567890"[:exarg]], + ) + + def testWordMaxIndependentOfWhitespaceInCharset(self): + # A Word constructed with an explicit `max` uses an internal regex + # fast-path only when its character set contains no space; otherwise + # it falls back to the char-by-char parseImpl. Both paths must agree: + # `max` means "match at most `max` characters, leaving the remainder" + # (see testWordMinMaxExactArgs and the "W:(0-9){1,3}" repr contract). + for extra, path in [("", "regex fast-path"), (" ", "char-by-char path")]: + with self.subTest(charset_extra=repr(extra), path=path): + expr = pp.Word(pp.nums + extra, max=3)("field") + pp.Word(pp.nums)( + "rest" + ) + self.assertParseAndCheckDict( + expr, + "0123456", + {"field": "012", "rest": "3456"}, + "Failed to parse regex and non-regex parse_impl in Word consistently" + ) + + def testWordMin(self): + # failing tests + for min_val in range(3, 5): + with self.subTest(min_val=min_val): + wd = pp.Word("a", "1", min=min_val) + print(min_val, wd.reString) + with self.assertRaisesParseException(): + wd.parse_string("a1") + + for min_val in range(2, 5): + with self.subTest(min_val=min_val): + wd = pp.Word("a", min=min_val) + print(min_val, wd.reString) + with self.assertRaisesParseException(): + wd.parse_string("a") + + for min_val in range(3, 5): + with self.subTest(min_val=min_val): + wd = pp.Word("a", "1", min=min_val) + print(min_val, wd.reString) + with self.assertRaisesParseException(): + wd.parse_string("a1") + + # passing tests + for min_val in range(2, 5): + with self.subTest(min_val=min_val): + wd = pp.Word("a", min=min_val) + test_string = "a" * min_val + self.assertParseAndCheckList( + wd, + test_string, + [test_string], + msg=f"Word(min={min_val}) failed", + verbose=True, + ) + + for min_val in range(2, 5): + with self.subTest(min_val=min_val): + wd = pp.Word("a", "1", min=min_val) + test_string = "a" + "1" * (min_val - 1) + self.assertParseAndCheckList( + wd, + test_string, + [test_string], + msg=f"Word(min={min_val}) failed", + verbose=True, + ) + + def testWordExact(self): + # failing tests + for exact_val in range(2, 5): + with self.subTest(exact_val=exact_val): + wd = pp.Word("a", exact=exact_val) + print(exact_val, wd.reString) + with self.assertRaisesParseException(): + wd.parse_string("a") + + # passing tests + for exact_val in range(2, 5): + with self.subTest(exact_val=exact_val): + wd = pp.Word("a", exact=exact_val) + test_string = "a" * exact_val + self.assertParseAndCheckList( + wd, + test_string, + [test_string], + msg=f"Word(exact={exact_val}) failed", + verbose=True, + ) + + def testInvalidMinMaxArgs(self): + with self.assertRaises(ValueError): + wd = pp.Word(min=2, max=1) + + def testWordExclude(self): + allButPunc = pp.Word(pp.printables, exclude_chars=".,:;-_!?") + + test = "Hello, Mr. Ed, it's Wilbur!" + result = allButPunc.search_string(test).as_list() + print(result) + self.assertEqual( + [["Hello"], ["Mr"], ["Ed"], ["it's"], ["Wilbur"]], + result, + "failed WordExcludeTest", + ) + + def testWordExclude2(self): + punc_chars = ".,:;-_!?" + + all_but_punc = pp.Word(pp.printables, exclude_chars=punc_chars) + all_and_punc = pp.Word(pp.printables) + + assert set(punc_chars) & set(all_but_punc.init_chars) == set() + + expr = all_but_punc("no_punc*") | all_and_punc("with_punc*") + + self.assertParseAndCheckDict( + expr[...], + "Mr. Ed,", + {"no_punc": ["Mr", "Ed"], "with_punc": [".", ","]}, + "failed matching with exclude_chars (1)", + ) + + self.assertParseAndCheckDict( + expr[...], + ":Mr. Ed,", + {"no_punc": ["Ed"], "with_punc": [":Mr.", ","]}, + "failed matching with exclude_chars (2)", + ) + + def testWordMinOfZero(self): + """test a Word with min=0""" + + with self.assertRaises(ValueError, msg="expected min 0 to error"): + expr = pp.Word(pp.nums, min=0, max=10) + + @staticmethod + def setup_testWordMaxGreaterThanZeroAndAsKeyword(): + # fmt: off + bool_operand = ( + pp.Word(pp.alphas, max=1, as_keyword=True) + | pp.one_of("True False") + ) + test_string = "p q r False" + return SimpleNamespace(**locals()) + # fmt: on + + def testWordMaxGreaterThanZeroAndAsKeyword1(self): + """test a Word with max>0 and as_keyword=True""" + setup = self.setup_testWordMaxGreaterThanZeroAndAsKeyword() + + result = setup.bool_operand[...].parse_string(setup.test_string, parse_all=True) + self.assertParseAndCheckList( + setup.bool_operand[...], + setup.test_string, + setup.test_string.split(), + msg=f"{__()}Failed to parse Word(max=1, as_keyword=True)", + verbose=True, + ) + + def testWordMaxGreaterThanZeroAndAsKeyword2(self): + """test a Word with max>0 and as_keyword=True""" + setup = self.setup_testWordMaxGreaterThanZeroAndAsKeyword() + + with self.assertRaisesParseException( + msg=f"{__()}Failed to detect Word with max > 0 and as_keyword=True" + ): + setup.bool_operand.parse_string("abc", parse_all=True) + + def testCharAsKeyword(self): + """test a Char with as_keyword=True""" + + grade = pp.OneOrMore(pp.Char("ABCDF", as_keyword=True)) + + # all single char words + result = grade.parse_string("B B C A D", parse_all=True) + + print(result) + expected = ["B", "B", "C", "A", "D"] + self.assertParseResultsEquals( + result, expected, msg="issue with Char as_keyword=True" + ) + + # NOT all single char words + test2 = "B BB C A D" + result2 = grade.parse_string(test2, parse_all=False) + + print(result2) + expected2 = ["B"] + self.assertParseResultsEquals( + result2, expected2, msg="issue with Char as_keyword=True parsing 2 chars" + ) + + def testCharRe(self): + expr = pp.Char("ABCDEFG") + self.assertEqual("[A-G]", expr.reString) + + def testCharsNotIn(self): + """test CharsNotIn initialized with various arguments""" + + vowels = "AEIOU" + tst = "bcdfghjklmnpqrstvwxyz" + + # default args + consonants = pp.CharsNotIn(vowels) + result = consonants.parse_string(tst, parse_all=True) + print(result) + self.assertParseResultsEquals( + result, [tst], msg="issue with CharsNotIn w/ default args" + ) + + # min = 0 + with self.assertRaises(ValueError, msg="issue with CharsNotIn w/ min=0"): + consonants = pp.CharsNotIn(vowels, min=0) + + # max > 0 + consonants = pp.CharsNotIn(vowels, max=5) + result = consonants.parse_string(tst, parse_all=False) + print(result) + self.assertParseResultsEquals( + result, [tst[:5]], msg="issue with CharsNotIn w max > 0" + ) + + # exact > 0 + consonants = pp.CharsNotIn(vowels, exact=10) + result = consonants.parse_string(tst[:10], parse_all=True) + print(result) + self.assertParseResultsEquals( + result, [tst[:10]], msg="issue with CharsNotIn w/ exact > 0" + ) + + # min > length + consonants = pp.CharsNotIn(vowels, min=25) + with self.assertRaisesParseException(msg="issue with CharsNotIn min > tokens"): + result = consonants.parse_string(tst, parse_all=True) + + def testParseAll(self): + testExpr = pp.Word("A") + + tests = [ + ("AAAAA", False, True), + ("AAAAA", True, True), + ("AAABB", False, True), + ("AAABB", True, False), + ] + for s, parse_allFlag, shouldSucceed in tests: + try: + print( + f"'{s}' parse_all={parse_allFlag} (shouldSucceed={shouldSucceed})" + ) + testExpr.parse_string(s, parse_all=parse_allFlag) + self.assertTrue( + shouldSucceed, "successfully parsed when should have failed" + ) + except ParseException as pe: + print(pe.explain()) + self.assertFalse( + shouldSucceed, "failed to parse when should have succeeded" + ) + + # add test for trailing comments + testExpr.ignore(pp.cpp_style_comment) + + tests = [ + ("AAAAA //blah", False, True), + ("AAAAA //blah", True, True), + ("AAABB //blah", False, True), + ("AAABB //blah", True, False), + ] + for s, parse_allFlag, shouldSucceed in tests: + try: + print( + f"'{s}' parse_all={parse_allFlag} (shouldSucceed={shouldSucceed})" + ) + testExpr.parse_string(s, parse_all=parse_allFlag) + self.assertTrue( + shouldSucceed, "successfully parsed when should have failed" + ) + except ParseException as pe: + print(pe.explain()) + self.assertFalse( + shouldSucceed, "failed to parse when should have succeeded" + ) + + # add test with very long expression string + # testExpr = pp.MatchFirst([pp.Literal(c) for c in pp.printables if c != 'B'])[1, ...] + anything_but_an_f = pp.OneOrMore( + pp.MatchFirst([pp.Literal(c) for c in pp.printables if c != "f"]) + ) + testExpr = pp.Word("012") + anything_but_an_f + + tests = [ + ("00aab", False, True), + ("00aab", True, True), + ("00aaf", False, True), + ("00aaf", True, False), + ] + for s, parse_allFlag, shouldSucceed in tests: + try: + print( + f"'{s}' parse_all={parse_allFlag} (shouldSucceed={shouldSucceed})" + ) + testExpr.parse_string(s, parse_all=parse_allFlag) + self.assertTrue( + shouldSucceed, "successfully parsed when should have failed" + ) + except ParseException as pe: + print(pe.explain()) + self.assertFalse( + shouldSucceed, "failed to parse when should have succeeded" + ) + + def testGreedyQuotedStrings(self): + src = """\ + "string1", "strin""g2" + 'string1', 'string2' + ^string1^, ^string2^ + <string1>, <string2>""" + + testExprs = ( + pp.sgl_quoted_string, + pp.dbl_quoted_string, + pp.quoted_string, + pp.QuotedString('"', esc_quote='""'), + pp.QuotedString("'", esc_quote="''"), + pp.QuotedString("^"), + pp.QuotedString("<", end_quote_char=">"), + ) + for expr in testExprs: + strs = pp.DelimitedList(expr).search_string(src) + print(strs) + self.assertTrue( + bool(strs), f"no matches found for test expression '{expr}'" + ) + for lst in strs: + self.assertEqual( + 2, len(lst), f"invalid match found for test expression '{expr}'" + ) + + src = """'ms1',1,0,'2009-12-22','2009-12-22 10:41:22') ON DUPLICATE KEY UPDATE sent_count = sent_count + 1, mtime = '2009-12-22 10:41:22';""" + tok_sql_quoted_value = pp.QuotedString( + "'", "\\", "''", True, False + ) ^ pp.QuotedString('"', "\\", '""', True, False) + tok_sql_computed_value = pp.Word(pp.nums) + tok_sql_identifier = pp.Word(pp.alphas) + + val = tok_sql_quoted_value | tok_sql_computed_value | tok_sql_identifier + vals = pp.DelimitedList(val) + print(vals.parse_string(src, parse_all=False)) + self.assertEqual( + 5, + len(vals.parse_string(src, parse_all=False)), + "error in greedy quote escaping", + ) + + def testQuotedStringEscapedQuotes(self): + quoted = pp.QuotedString('"', esc_quote='""') + res = quoted.parse_string('"like ""SQL"""', parse_all=True) + print(res.as_list()) + self.assertEqual(['like "SQL"'], res.as_list()) + + # Issue #263 - handle case when the esc_quote is not a repeated character + quoted = pp.QuotedString("y", esc_char=None, esc_quote="xy") + res = quoted.parse_string("yaaay", parse_all=True) + self.assertEqual(["aaa"], res.as_list()) + res = quoted.parse_string("yaaaxyaaay", parse_all=True) + print(res.as_list()) + self.assertEqual(["aaayaaa"], res.as_list()) + + def testQuotedStringEscapedExtendedChars(self): + quoted = pp.QuotedString("'") + self.assertParseAndCheckList( + quoted, + "'null: \0 octal: \267 hex: \xb7 unicode: \u00b7'", + ["null: \x00 octal: · hex: · unicode: ·"], + "failed to parse embedded numeric escapes", + ) + + def testWordBoundaryExpressions(self): + ws = pp.WordStart() + we = pp.WordEnd() + vowel = pp.one_of(list("AEIOUY")) + consonant = pp.one_of(list("BCDFGHJKLMNPQRSTVWXZ")) + + leadingVowel = ws + vowel + trailingVowel = vowel + we + leadingConsonant = ws + consonant + trailingConsonant = consonant + we + internalVowel = ~ws + vowel + ~we + + bnf = leadingVowel | trailingVowel + + tests = """\ + ABC DEF GHI + JKL MNO PQR + STU VWX YZ """.splitlines() + tests.append("\n".join(tests)) + + expectedResult = [ + [["D", "G"], ["A"], ["C", "F"], ["I"], ["E"], ["A", "I"]], + [["J", "M", "P"], [], ["L", "R"], ["O"], [], ["O"]], + [["S", "V"], ["Y"], ["X", "Z"], ["U"], [], ["U", "Y"]], + [ + ["D", "G", "J", "M", "P", "S", "V"], + ["A", "Y"], + ["C", "F", "L", "R", "X", "Z"], + ["I", "O", "U"], + ["E"], + ["A", "I", "O", "U", "Y"], + ], + ] + + for t, expected in zip(tests, expectedResult): + print(t) + results = [ + flatten(e.search_string(t).as_list()) + for e in [ + leadingConsonant, + leadingVowel, + trailingConsonant, + trailingVowel, + internalVowel, + bnf, + ] + ] + print(results) + print() + self.assertEqual( + expected, + results, + f"Failed WordBoundaryTest, expected {expected}, got {results}", + ) + + def testWordBoundaryExpressions2(self): + from itertools import product + + ws1 = pp.WordStart(pp.alphas) + ws2 = pp.WordStart(word_chars=pp.alphas) + ws3 = pp.WordStart(word_chars=pp.alphas) + we1 = pp.WordEnd(pp.alphas) + we2 = pp.WordEnd(word_chars=pp.alphas) + we3 = pp.WordEnd(word_chars=pp.alphas) + + for i, (ws, we) in enumerate(product((ws1, ws2, ws3), (we1, we2, we3))): + try: + expr = "(" + ws + pp.Word(pp.alphas) + we + ")" + expr.parse_string("(abc)", parse_all=True) + except pp.ParseException as pe: + self.fail(f"Test {i} failed: {pe}") + else: + pass + + def testRequiredEach(self): + parser = pp.Keyword("bam") & pp.Keyword("boo") + try: + res1 = parser.parse_string("bam boo", parse_all=True) + print(res1.as_list()) + res2 = parser.parse_string("boo bam", parse_all=True) + print(res2.as_list()) + except ParseException: + failed = True + else: + failed = False + self.assertFalse(failed, "invalid logic in Each") + + self.assertEqual( + set(res1), + set(res2), + f"Failed RequiredEachTest, expected {res1.as_list()}" + f" and {res2.as_list} to contain the same words in any order", + ) + + def testOptionalEachTest1(self): + for the_input in [ + "Tal Weiss Major", + "Tal Major", + "Weiss Major", + "Major", + "Major Tal", + "Major Weiss", + "Major Tal Weiss", + ]: + print(the_input) + parser1 = (pp.Optional("Tal") + pp.Optional("Weiss")) & pp.Keyword("Major") + parser2 = pp.Optional( + pp.Optional("Tal") + pp.Optional("Weiss") + ) & pp.Keyword("Major") + parser3 = (pp.Keyword("Tal") | pp.Keyword("Weiss"))[...] & pp.Keyword( + "Major" + ) + + p1res = parser1.parse_string(the_input, parse_all=True) + + p2res = parser2.parse_string(the_input, parse_all=True) + self.assertEqual( + p1res.as_list(), + p2res.as_list(), + f"Each failed to match with nested Optionals, {p1res.as_list()} should match {p2res.as_list()}", + ) + + p3res = parser3.parse_string(the_input, parse_all=True) + self.assertEqual( + p1res.as_list(), + p3res.as_list(), + f"Each failed to match with repeated Optionals, {p1res.as_list()} should match {p3res.as_list()}", + ) + + def testOptionalEachTest2(self): + word = pp.Word(pp.alphanums + "_").set_name("word") + with_stmt = "with" + pp.OneOrMore(pp.Group(word("key") + "=" + word("value")))( + "overrides" + ) + using_stmt = "using" + pp.Regex("id-[0-9a-f]{8}")("id") + modifiers = pp.Optional(with_stmt("with_stmt")) & pp.Optional( + using_stmt("using_stmt") + ) + + self.assertEqual("with foo=bar bing=baz using id-deadbeef", modifiers) + self.assertNotEqual( + "with foo=bar bing=baz using id-deadbeef using id-feedfeed", modifiers + ) + + def testOptionalEachTest3(self): + foo = pp.Literal("foo") + bar = pp.Literal("bar") + + openBrace = pp.Suppress(pp.Literal("{")) + closeBrace = pp.Suppress(pp.Literal("}")) + + exp = openBrace + (foo[1, ...]("foo") & bar[...]("bar")) + closeBrace + + tests = """\ + {foo} + {bar foo bar foo bar foo} + """.splitlines() + for test in tests: + test = test.strip() + if not test: + continue + self.assertParseAndCheckList( + exp, + test, + test.strip("{}").split(), + f"failed to parse Each expression {test!r}", + verbose=True, + ) + + with self.assertRaisesParseException(): + exp.parse_string("{bar}", parse_all=True) + + def testOptionalEachTest4(self): + expr = (~ppc.iso8601_date + ppc.integer("id")) & ( + pp.Group(ppc.iso8601_date)("date*")[...] + ) + + success, _ = expr.run_tests( + """ + 1999-12-31 100 2001-01-01 + 42 + """ + ) + self.assertTrue(success) + + def testEachWithParseFatalException(self): + option_expr = pp.Keyword("options") - "(" + ppc.integer + ")" + step_expr1 = pp.Keyword("step") - "(" + ppc.integer + ")" + step_expr2 = pp.Keyword("step") - "(" + ppc.integer + "Z" + ")" + step_expr = step_expr1 ^ step_expr2 + + parser = option_expr & step_expr[...] + tests = [ + ( + "options(100) step(A)", + "Expected integer, found 'A' (at char 18), (line:1, col:19)", + ), + ( + "step(A) options(100)", + "Expected integer, found 'A' (at char 5), (line:1, col:6)", + ), + ( + "options(100) step(100A)", + """Expected 'Z', found 'A' (at char 21), (line:1, col:22)""", + ), + ( + "options(100) step(22) step(100ZA)", + """Expected ')', found 'A' (at char 31), (line:1, col:32)""", + ), + ] + test_lookup = dict(tests) + + success, output = parser.run_tests((t[0] for t in tests), failure_tests=True) + for test_str, result in output: + self.assertEqual( + test_lookup[test_str], + str(result), + f"incorrect exception raised for test string {test_str!r}", + ) + + def testEachWithMultipleMatch(self): + size = "size" + pp.one_of("S M L XL") + color = pp.Group( + "color" + pp.one_of("red orange yellow green blue purple white black brown") + ) + size.set_name("size_spec") + color.set_name("color_spec") + + spec0 = size("size") & color[...]("colors") + spec1 = size("size") & color[1, ...]("colors") + + for spec in (spec0, spec1): + for test, expected_dict in [ + ( + "size M color red color yellow", + { + "colors": [["color", "red"], ["color", "yellow"]], + "size": ["size", "M"], + }, + ), + ( + "color green size M color red color yellow", + { + "colors": [ + ["color", "green"], + ["color", "red"], + ["color", "yellow"], + ], + "size": ["size", "M"], + }, + ), + ]: + result = spec.parse_string(test, parse_all=True) + self.assertParseResultsEquals(result, expected_dict=expected_dict) + + def testSumParseResults(self): + samplestr1 = "garbage;DOB 10-10-2010;more garbage\nID PARI12345678;more garbage" + samplestr2 = "garbage;ID PARI12345678;more garbage\nDOB 10-10-2010;more garbage" + samplestr3 = "garbage;DOB 10-10-2010" + samplestr4 = "garbage;ID PARI12345678;more garbage- I am cool" + + res1 = "ID:PARI12345678 DOB:10-10-2010 INFO:" + res2 = "ID:PARI12345678 DOB:10-10-2010 INFO:" + res3 = "ID: DOB:10-10-2010 INFO:" + res4 = "ID:PARI12345678 DOB: INFO: I am cool" + + dob_ref = "DOB" + pp.Regex(r"\d{2}-\d{2}-\d{4}")("dob") + id_ref = "ID" + pp.Word(pp.alphanums, exact=12)("id") + info_ref = "-" + pp.rest_of_line("info") + + person_data = dob_ref | id_ref | info_ref + + tests = (samplestr1, samplestr2, samplestr3, samplestr4) + results = (res1, res2, res3, res4) + for test, expected in zip(tests, results): + person = sum(person_data.search_string(test)) + result = f"ID:{person.id} DOB:{person.dob} INFO:{person.info}" + print(test) + print(expected) + print(result) + for pd in person_data.search_string(test): + print(pd.dump()) + print() + self.assertEqual( + expected, + result, + f"Failed to parse '{test}' correctly, \nexpected '{expected}', got '{result}'", + ) + + def testMarkInputLine(self): + samplestr1 = "DOB 100-10-2010;more garbage\nID PARI12345678;more garbage" + + dob_ref = "DOB" + pp.Regex(r"\d{2}-\d{2}-\d{4}")("dob") + + try: + res = dob_ref.parse_string(samplestr1, parse_all=True) + except ParseException as pe: + outstr = pe.mark_input_line() + print(outstr) + self.assertEqual( + "DOB >!<100-10-2010;more garbage", + outstr, + "did not properly create marked input line", + ) + else: + self.fail("test construction failed - should have raised an exception") + + def testDeprecatedNonPep8Names(self): + word = pp.Word(pp.alphas) + with self.assertWarns(PyparsingDeprecationWarning): + word[...].parseString("ABC DEF") + with self.assertWarns(DeprecationWarning): + word[...].parseString("ABC DEF") + + def testLocatedExpr(self): + # 012345678901234567890123456789012345678901234567890 + samplestr1 = "DOB 10-10-2010;more garbage;ID PARI12345678 ;more garbage" + + with self.assertWarns(PyparsingDeprecationWarning): + id_ref = pp.locatedExpr("ID" + pp.Word(pp.alphanums, exact=12)("id")) + + res = id_ref.search_string(samplestr1)[0][0] + print(res.dump()) + self.assertEqual( + "ID PARI12345678", + samplestr1[res.locn_start : res.locn_end], + "incorrect location calculation", + ) + + def testLocatedExprUsingLocated(self): + # 012345678901234567890123456789012345678901234567890 + samplestr1 = "DOB 10-10-2010;more garbage;ID PARI12345678 ;more garbage" + + id_ref = pp.Located("ID" + pp.Word(pp.alphanums, exact=12)("id")) + + res = id_ref.search_string(samplestr1)[0] + print(res.dump()) + self.assertEqual( + "ID PARI12345678", + samplestr1[res.locn_start : res.locn_end], + "incorrect location calculation", + ) + self.assertParseResultsEquals( + res, + [28, ["ID", "PARI12345678"], 43], + {"locn_end": 43, "locn_start": 28, "value": {"id": "PARI12345678"}}, + ) + self.assertEqual("PARI12345678", res.value.id) + + # if Located has a results name, handle appropriately + id_ref = pp.Located("ID" + pp.Word(pp.alphanums, exact=12)("id"))("loc") + + res = id_ref.search_string(samplestr1)[0] + print(res.dump()) + self.assertEqual( + "ID PARI12345678", + samplestr1[res.loc.locn_start : res.loc.locn_end], + "incorrect location calculation", + ) + self.assertParseResultsEquals( + res.loc, + [28, ["ID", "PARI12345678"], 43], + {"locn_end": 43, "locn_start": 28, "value": {"id": "PARI12345678"}}, + ) + self.assertEqual("PARI12345678", res.loc.value.id) + + wd = pp.Word(pp.alphas) + test_string = "ljsdf123lksdjjf123lkkjj1222" + pp_matches = pp.Located(wd).search_string(test_string) + re_matches = re.finditer("[a-z]+", test_string) + for pp_match, re_match in zip(pp_matches, re_matches): + self.assertParseResultsEquals( + pp_match, [re_match.start(), [re_match.group(0)], re_match.end()] + ) + print(pp_match) + print(re_match) + print(pp_match.value) + + def testLocatedExprLeadingWhitespace(self): + # Located should mark the start of the match, not leading whitespace the + # wrapped expression skips - including when whitespace skipping is + # delegated to sub-expressions (MatchFirst, And). Issue #621. + abc = pp.Keyword("abc") + + # a single token already reported the correct location + single = pp.Located(abc).parse_string(" abc") + self.assertParseResultsEquals(single, [3, ["abc"], 6]) + + # a MatchFirst used to report locn_start=0, a " abc" span + match_first = pp.Located(abc | abc).parse_string(" abc") + self.assertParseResultsEquals(match_first, [3, ["abc"], 6]) + self.assertEqual( + "abc", + " abc"[match_first.locn_start : match_first.locn_end], + "Located(MatchFirst) included leading whitespace in its location", + ) + + # an And likewise delegates whitespace skipping + sample = " ID PARI12345678" + seq = pp.Located(pp.Literal("ID") + pp.Word(pp.alphanums)).parse_string(sample) + self.assertEqual( + "ID PARI12345678", + sample[seq.locn_start : seq.locn_end], + "Located(And) included leading whitespace in its location", + ) + + # leave_whitespace() must keep the leading whitespace + leave_ws = pp.Located(pp.Word(" abc").leave_whitespace()).parse_string(" abc") + self.assertEqual( + 0, + leave_ws.locn_start, + "Located should not skip whitespace for a leave_whitespace expression", + ) + + def testPop(self): + source = "AAA 123 456 789 234" + patt = pp.Word(pp.alphas)("name") + pp.Word(pp.nums) * (1,) + + result = patt.parse_string(source, parse_all=True) + tests = [ + (0, "AAA", ["123", "456", "789", "234"]), + (None, "234", ["123", "456", "789"]), + ("name", "AAA", ["123", "456", "789"]), + (-1, "789", ["123", "456"]), + ] + for test in tests: + idx, val, remaining = test + if idx is not None: + ret = result.pop(idx) + else: + ret = result.pop() + print("EXP:", val, remaining) + print("GOT:", ret, result.as_list()) + print(ret, result.as_list()) + self.assertEqual( + val, + ret, + f"wrong value returned, got {ret!r}, expected {val!r}", + ) + self.assertEqual( + remaining, + result.as_list(), + f"list is in wrong state after pop, got {result.as_list()!r}, expected {remaining!r}", + ) + print() + + prevlist = result.as_list() + ret = result.pop("name", default="noname") + print(ret) + print(result.as_list()) + self.assertEqual( + "noname", + ret, + f"default value not successfully returned, got {ret!r}, expected {'noname'!r}", + ) + self.assertEqual( + prevlist, + result.as_list(), + f"list is in wrong state after pop, got {result.as_list()!r}, expected {remaining!r}", + ) + + def testPopKwargsErr(self): + """raise a TypeError in pop by adding invalid named args""" + + source = "AAA 123 456 789 234" + patt = pp.Word(pp.alphas)("name") + pp.Word(pp.nums) * (1,) + result = patt.parse_string(source, parse_all=True) + print(result.dump()) + + with self.assertRaises(TypeError): + result.pop(notDefault="foo") + + def testAddCondition(self): + numParser = pp.Word(pp.nums) + numParser.add_parse_action(lambda s, l, t: int(t[0])) + numParser.add_condition(lambda s, l, t: t[0] % 2) + numParser.add_condition(lambda s, l, t: t[0] >= 7) + + result = numParser.search_string("1 2 3 4 5 6 7 8 9 10") + print(result.as_list()) + self.assertEqual( + [[7], [9]], result.as_list(), "failed to properly process conditions" + ) + + numParser = pp.Word(pp.nums) + numParser.add_parse_action(lambda s, l, t: int(t[0])) + rangeParser = numParser("from_") + pp.Suppress("-") + numParser("to") + + result = rangeParser.search_string("1-4 2-4 4-3 5 6 7 8 9 10") + print(result.as_list()) + self.assertEqual( + [[1, 4], [2, 4], [4, 3]], + result.as_list(), + "failed to properly process conditions", + ) + + rangeParser.add_condition( + lambda t: t.to > t.from_, message="from must be <= to", fatal=False + ) + result = rangeParser.search_string("1-4 2-4 4-3 5 6 7 8 9 10") + print(result.as_list()) + self.assertEqual( + [[1, 4], [2, 4]], result.as_list(), "failed to properly process conditions" + ) + + rangeParser = numParser("from_") + pp.Suppress("-") + numParser("to") + rangeParser.add_condition( + lambda t: t.to > t.from_, message="from must be <= to", fatal=True + ) + try: + result = rangeParser.search_string("1-4 2-4 4-3 5 6 7 8 9 10") + self.fail("failed to interrupt parsing on fatal condition failure") + except ParseFatalException: + print("detected fatal condition") + + def testPatientOr(self): + # Two expressions and a input string which could - syntactically - be matched against + # both expressions. The "Literal" expression is considered invalid though, so this PE + # should always detect the "Word" expression. + def validate(token): + if token[0] == "def": + raise pp.ParseException("signalling invalid token") + return token + + a = pp.Word("de").set_name("Word") # .set_debug() + b = ( + pp.Literal("def").set_name("Literal").set_parse_action(validate) + ) # .set_debug() + c = pp.Literal("d").set_name("d") # .set_debug() + + # The "Literal" expressions's ParseAction is not executed directly after syntactically + # detecting the "Literal" Expression but only after the Or-decision has been made + # (which is too late)... + try: + result = (a ^ b ^ c).parse_string("def", parse_all=False) + print(result) + self.assertEqual( + ["de"], + result.as_list(), + f"failed to select longest match, chose {result}", + ) + except ParseException: + failed = True + else: + failed = False + + if failed: + self.fail( + "invalid logic in Or, fails on longest match with exception in parse action" + ) + + # from issue #93 + word = pp.Word(pp.alphas).set_name("word") + word_1 = ( + pp.Word(pp.alphas) + .set_name("word_1") + .add_condition(lambda t: len(t[0]) == 1) + ) + + a = word + (word_1 + word ^ word) + b = word * 3 + c = a ^ b + c.streamline() + print(c) + test_string = "foo bar temp" + result = c.parse_string(test_string, parse_all=True) + print(test_string, "->", result.as_list()) + + self.assertEqual( + test_string.split(), result.as_list(), "failed to match longest choice" + ) + + def testEachWithOptionalWithResultsName(self): + result = (pp.Optional("foo")("one") & pp.Optional("bar")("two")).parse_string( + "bar foo", parse_all=True + ) + print(result.dump()) + self.assertEqual(sorted(["one", "two"]), sorted(result.keys())) + + def testUnicodeExpression(self): + z = "a" | pp.Literal("\u1111") + z.streamline() + try: + z.parse_string("b", parse_all=True) + except ParseException as pe: + self.assertEqual( + r"""Expected {'a' | 'ᄑ'}""", + pe.msg, + f"Invalid error message raised, got {pe.msg!r}", + ) + + def testSetName(self): + a = pp.one_of("a b c") + b = pp.one_of("d e f") + # fmt: off + arith_expr = pp.infix_notation( + pp.Word(pp.nums), + [ + (pp.one_of("* /").set_name("* | /"), 2, pp.OpAssoc.LEFT), + (pp.one_of("+ -").set_name("+ | -"), 2, pp.OpAssoc.LEFT), + ], + ) + arith_expr2 = pp.infix_notation( + pp.Word(pp.nums), + [ + (("?", ":"), 3, pp.OpAssoc.LEFT), + ] + ) + # fmt: on + recursive = pp.Forward() + recursive <<= a + (b + recursive)[...] + + tests = [ + a, + b, + (a | b), + arith_expr, + arith_expr.expr, + arith_expr2, + arith_expr2.expr, + recursive, + pp.DelimitedList(pp.Word(pp.nums).set_name("int")), + pp.counted_array(pp.Word(pp.nums).set_name("int")), + pp.nested_expr(), + pp.make_html_tags("Z"), + (pp.any_open_tag, pp.any_close_tag), + pp.common_html_entity, + pp.common_html_entity.set_parse_action( + pp.replace_html_entity + ).transform_string("lsdjkf <lsdjkf>&'"&xyzzy;"), + ] + + expected = map( + str.strip, + """\ + 'a' | 'b' | 'c' + 'd' | 'e' | 'f' + {'a' | 'b' | 'c' | 'd' | 'e' | 'f'} + W:(0-9)_expression + + | - operations + W:(0-9)_expression + ?: operations + Forward: {'a' | 'b' | 'c' [{'d' | 'e' | 'f' : ...}]...} + int [, int]... + (len) int... + nested () expression + (<Z>, </Z>) + (<any tag>, </any tag>) + common HTML entity + lsdjkf <lsdjkf>&'"&xyzzy;""".splitlines(), + ) + + for t, e in zip(tests, expected): + with self.subTest("set_name", t=t, e=e): + tname = str(t) + print(tname) + self.assertEqual( + e, + tname, + f"expression name mismatch, expected {e} got {tname}", + ) + + def testTrimArityExceptionMasking(self): + invalid_message = "<lambda>() missing 1 required positional argument: 't'" + try: + pp.Word("a").set_parse_action(lambda t: t[0] + 1).parse_string( + "aaa", parse_all=True + ) + except Exception as e: + exc_msg = str(e) + self.assertNotEqual( + exc_msg, + invalid_message, + "failed to catch TypeError thrown in _trim_arity", + ) + + def testTrimArityExceptionMaskingTest2(self): + # construct deep call tree + def A(): + import traceback + + traceback.print_stack(limit=2) + + invalid_message = "<lambda>() missing 1 required positional argument: 't'" + try: + pp.Word("a").set_parse_action(lambda t: t[0] + 1).parse_string( + "aaa", parse_all=True + ) + except Exception as e: + exc_msg = str(e) + self.assertNotEqual( + exc_msg, + invalid_message, + "failed to catch TypeError thrown in _trim_arity", + ) + + def B(): + A() + + def C(): + B() + + def D(): + C() + + def E(): + D() + + def F(): + E() + + def G(): + F() + + def H(): + G() + + def J(): + H() + + def K(): + J() + + K() + + def testClearParseActions(self): + realnum = ppc.real() + self.assertEqual( + 3.14159, + realnum.parse_string("3.14159", parse_all=True)[0], + "failed basic real number parsing", + ) + + # clear parse action that converts to float + realnum.set_parse_action(None) + self.assertEqual( + "3.14159", + realnum.parse_string("3.14159", parse_all=True)[0], + "failed clearing parse action", + ) + + # add a new parse action that tests if a '.' is prsent + realnum.add_parse_action(lambda t: "." in t[0]) + self.assertEqual( + True, + realnum.parse_string("3.14159", parse_all=True)[0], + "failed setting new parse action after clearing parse action", + ) + + def testOneOrMoreStop(self): + test = "BEGIN aaa bbb ccc END" + BEGIN, END = map(pp.Keyword, "BEGIN,END".split(",")) + body_word = pp.Word(pp.alphas).set_name("word") + for ender in (END, "END", pp.CaselessKeyword("END")): + expr = BEGIN + pp.OneOrMore(body_word, stop_on=ender) + END + self.assertEqual( + expr, test, f"Did not successfully stop on ending expression {ender!r}" + ) + + expr = BEGIN + body_word[1, ...].stopOn(ender) + END + self.assertParseAndCheckList( + expr, + test, + test.split(), + f"Did not successfully stop on ending expression {ender!r}", + ) + + expr = BEGIN + body_word[1, ...:ender] + END + self.assertParseAndCheckList( + expr, + test, + test.split(), + f"Did not successfully stop on ending expression {ender!r}", + ) + + expr = BEGIN + body_word[(1, ...):ender] + END + self.assertParseAndCheckList( + expr, + test, + test.split(), + f"Did not successfully stop on ending expression {ender!r}", + ) + + number = pp.Word(pp.nums + ",.()").set_name("number with optional commas") + parser = pp.OneOrMore(pp.Word(pp.alphanums + "-/."), stop_on=number)( + "id" + ).set_parse_action(" ".join) + number("data") + self.assertParseAndCheckList( + parser, + " XXX Y/123 1,234.567890", + ["XXX Y/123", "1,234.567890"], + f"Did not successfully stop on ending expression {number!r}", + verbose=True, + ) + + def testZeroOrMoreStop(self): + test = "BEGIN END" + BEGIN, END = map(pp.Keyword, "BEGIN,END".split(",")) + body_word = pp.Word(pp.alphas).set_name("word") + for ender in (END, "END", pp.CaselessKeyword("END")): + expr = BEGIN + pp.ZeroOrMore(body_word, stop_on=ender) + END + self.assertParseAndCheckList( + expr, + test, + test.split(), + f"Did not successfully stop on ending expression {ender!r}", + ) + + expr = BEGIN + body_word[...].stopOn(ender) + END + self.assertParseAndCheckList( + expr, + test, + test.split(), + f"Did not successfully stop on ending expression {ender!r}", + ) + + expr = BEGIN + body_word[...:ender] + END + self.assertParseAndCheckList( + expr, + test, + test.split(), + f"Did not successfully stop on ending expression {ender!r}", + ) + + expr = BEGIN + body_word[:ender] + END + self.assertParseAndCheckList( + expr, + test, + test.split(), + f"Did not successfully stop on ending expression {ender!r}", + ) + + def testOneOrMoreMax(self): + # Test OneOrMore with max + expr = pp.OneOrMore(pp.Word(pp.nums), max=3) + + # Should match 1, 2, or 3 + self.assertEqual(len(expr.parse_string("1")), 1) + self.assertEqual(len(expr.parse_string("1 2")), 2) + self.assertEqual(len(expr.parse_string("1 2 3")), 3) + + # Should match ONLY 3 and leave the rest + res = expr.parse_string("1 2 3 4 5") + self.assertEqual(len(res), 3) + self.assertEqual(res.as_list(), ["1", "2", "3"]) + + # Test max=1 + expr = pp.OneOrMore(pp.Word(pp.nums), max=1) + res = expr.parse_string("1 2 3") + self.assertEqual(len(res), 1) + + # Test max=0 (should raise ValueError) + with self.assertRaises(ValueError): + pp.OneOrMore(pp.Word(pp.nums), max=0) + + def testZeroOrMoreMax(self): + # Test ZeroOrMore with max + expr = pp.ZeroOrMore(pp.Word(pp.nums), max=3) + + # Should match 0, 1, 2, or 3 + self.assertEqual(len(expr.parse_string("")), 0) + self.assertEqual(len(expr.parse_string("1")), 1) + self.assertEqual(len(expr.parse_string("1 2")), 2) + self.assertEqual(len(expr.parse_string("1 2 3")), 3) + + # Should match ONLY 3 and leave the rest + res = expr.parse_string("1 2 3 4 5") + self.assertEqual(len(res), 3) + self.assertEqual(res.as_list(), ["1", "2", "3"]) + + # Test max=1 + expr = pp.ZeroOrMore(pp.Word(pp.nums), max=1) + res = expr.parse_string("1 2 3") + self.assertEqual(len(res), 1) + + # Test max=0 (should raise ValueError) + with self.assertRaises(ValueError): + pp.ZeroOrMore(pp.Word(pp.nums), max=0) + + def testOneOrMoreMaxNegative(self): + # If max is negative, OneOrMore should raise ValueError + with self.assertRaises(ValueError): + pp.OneOrMore(pp.Word(pp.nums), max=-1) + + def testZeroOrMoreMaxNegative(self): + # If max is negative, ZeroOrMore should raise ValueError + with self.assertRaises(ValueError): + pp.ZeroOrMore(pp.Word(pp.nums), max=-1) + + def testNestedAsDict(self): + equals = pp.Literal("=").suppress() + lbracket = pp.Literal("[").suppress() + rbracket = pp.Literal("]").suppress() + lbrace = pp.Literal("{").suppress() + rbrace = pp.Literal("}").suppress() + + value_dict = pp.Forward() + value_list = pp.Forward() + value_string = pp.Word(pp.alphanums + "@. ") + + value = value_list ^ value_dict ^ value_string + values = pp.Group(pp.DelimitedList(value, ",")) + # ~ values = DelimitedList(value, ",").set_parse_action(lambda toks: [toks.as_list()]) + + value_list <<= lbracket + values + rbracket + + identifier = pp.Word(pp.alphanums + "_.") + + assignment = pp.Group(identifier + equals + pp.Optional(value)) + assignments = pp.Dict(pp.DelimitedList(assignment, ";")) + value_dict <<= lbrace + assignments + rbrace + + response = assignments + + rsp = ( + "username=goat; errors={username=[already taken, too short]}; empty_field=" + ) + result_dict = response.parse_string(rsp, parse_all=True).as_dict() + print(result_dict) + self.assertEqual( + "goat", + result_dict["username"], + "failed to process string in ParseResults correctly", + ) + self.assertEqual( + ["already taken", "too short"], + result_dict["errors"]["username"], + "failed to process nested ParseResults correctly", + ) + + def testTraceParseActionDecorator(self): + @pp.trace_parse_action + def convert_to_int(t): + return int(t[0]) + + class Z: + def __call__(self, other): + return other[0] * 1000 + + integer = pp.Word(pp.nums).add_parse_action(convert_to_int) + integer.add_parse_action(pp.trace_parse_action(lambda t: t[0] * 10)) + integer.add_parse_action(pp.trace_parse_action(Z())) + integer.parse_string("132", parse_all=True) + + def testTraceParseActionDecorator_with_exception(self): + @pp.trace_parse_action + def convert_to_int_raising_type_error(t): + return int(t[0]) + ".000" + + @pp.trace_parse_action + def convert_to_int_raising_index_error(t): + return int(t[1]) + + @pp.trace_parse_action + def convert_to_int_raising_value_error(t): + a, b = t[0] + return int(t[1]) + + @pp.trace_parse_action + def convert_to_int_raising_parse_exception(t): + pp.Word(pp.alphas).parse_string("123") + + for pa, expected_message in ( + (convert_to_int_raising_type_error, "TypeError:"), + (convert_to_int_raising_index_error, "IndexError:"), + (convert_to_int_raising_value_error, "ValueError:"), + (convert_to_int_raising_parse_exception, "ParseException:"), + ): + print(f"Using parse action {pa.__name__!r}") + integer = pp.Word(pp.nums).set_parse_action(pa) + stderr_capture = StringIO() + try: + with contextlib.redirect_stderr(stderr_capture): + integer.parse_string("132", parse_all=True) + except Exception as exc: + print(f"Exception raised: {type(exc).__name__}: {exc}") + else: + print("No exception raised") + stderr_text = stderr_capture.getvalue() + print(stderr_text) + self.assertTrue( + expected_message in stderr_text, + f"Expected exception type {expected_message!r} not found in trace_parse_action output", + ) + + def testRunTests(self): + integer = pp.Word(pp.nums).set_parse_action(lambda t: int(t[0])) + intrange = integer("start") + "-" + integer("end") + intrange.add_condition( + lambda t: t.end > t.start, + message="invalid range, start must be <= end", + fatal=True, + ) + intrange.add_parse_action(lambda t: list(range(t.start, t.end + 1))) + + indices = pp.DelimitedList(intrange | integer) + indices.add_parse_action(lambda t: sorted(set(t))) + + tests = """\ + # normal data + 1-3,2-4,6,8-10,16 + + # lone integer + 11""" + results = indices.run_tests(tests, print_results=False)[1] + + expectedResults = [[1, 2, 3, 4, 6, 8, 9, 10, 16], [11]] + for res, expected in zip(results, expectedResults): + print(res[1].as_list()) + print(expected) + self.assertEqual( + expected, res[1].as_list(), "failed test: " + str(expected) + ) + + tests = """\ + # invalid range + 1-2, 3-1, 4-6, 7, 12 + """ + success, _ = indices.run_tests(tests, print_results=False, failure_tests=True) + self.assertTrue(success, "failed to raise exception on improper range test") + + def testRunTestsPostParse(self): + integer = ppc.integer + fraction = integer("numerator") + "/" + integer("denominator") + + accum = [] + + def eval_fraction(test, result): + accum.append((test, result.as_list())) + return f"eval: {result.numerator / result.denominator}" + + success, _ = fraction.run_tests( + """\ + 1/2 + 1/0 + """, + post_parse=eval_fraction, + ) + self.assertTrue(success, "failed to parse fractions in RunTestsPostParse") + + expected_accum = [("1/2", [1, "/", 2]), ("1/0", [1, "/", 0])] + self.assertEqual( + expected_accum, accum, "failed to call post_parse method during run_tests" + ) + + def testConvertToDateErr(self): + """raise a ParseException in convert_to_date with incompatible date str""" + + expr = pp.Word(pp.alphanums + "-") + expr.add_parse_action(ppc.convert_to_date()) + + with self.assertRaisesParseException(): + expr.parse_string("1997-07-error", parse_all=True) + + def testConvertToDatetimeErr(self): + """raise a ParseException in convert_to_datetime with incompatible datetime str""" + + expr = pp.Word(pp.alphanums + "-") + expr.add_parse_action(ppc.convert_to_datetime()) + + with self.assertRaisesParseException(): + expr.parse_string("1997-07-error", parse_all=True) + + def testCommonExpressions(self): + import ast + + with self.subTest("MAC address success run_tests"): + success, _ = ppc.mac_address.run_tests( + """ + AA:BB:CC:DD:EE:FF + AA.BB.CC.DD.EE.FF + AA-BB-CC-DD-EE-FF + """ + ) + self.assertTrue(success, "error in parsing valid MAC address") + + with self.subTest("MAC address expected failure run_tests"): + success, _ = ppc.mac_address.run_tests( + """ + # mixed delimiters + AA.BB:CC:DD:EE:FF + """, + failure_tests=True, + ) + self.assertTrue(success, "error in detecting invalid mac address") + + with self.subTest("IPv4 address success run_tests"): + success, _ = ppc.ipv4_address.run_tests( + """ + 0.0.0.0 + 1.1.1.1 + 127.0.0.1 + 1.10.100.199 + 255.255.255.255 + """ + ) + self.assertTrue(success, "error in parsing valid IPv4 address") + + with self.subTest("IPv4 address expected failure run_tests"): + success, _ = ppc.ipv4_address.run_tests( + """ + # out of range value + 256.255.255.255 + """, + failure_tests=True, + ) + self.assertTrue(success, "error in detecting invalid IPv4 address") + + with self.subTest("IPv6 address success run_tests"): + success, _ = ppc.ipv6_address.run_tests( + """ + 2001:0db8:85a3:0000:0000:8a2e:0370:7334 + 2134::1234:4567:2468:1236:2444:2106 + 0:0:0:0:0:0:A00:1 + 1080::8:800:200C:417A + ::A00:1 + + # loopback address + ::1 + + # the null address + :: + + # ipv4 compatibility form + ::ffff:192.168.0.1 + """ + ) + self.assertTrue(success, "error in parsing valid IPv6 address") + + with self.subTest("IPv6 address expected failure run_tests"): + success, _ = ppc.ipv6_address.run_tests( + """ + # too few values + 1080:0:0:0:8:800:200C + + # too many ::'s, only 1 allowed + 2134::1234:4567::2444:2106 + """, + failure_tests=True, + ) + self.assertTrue(success, "error in detecting invalid IPv6 address") + + with self.subTest("ppc.number success run_tests"): + success, _ = ppc.number.run_tests( + """ + 100 + -100 + +100 + 3.14159 + 6.02e23 + 1e-12 + """ + ) + self.assertTrue(success, "error in parsing valid numerics") + + with self.subTest("ppc.sci_real success run_tests"): + success, _ = ppc.sci_real.run_tests( + """ + 1e12 + -1e12 + 3.14159 + 6.02e23 + """ + ) + self.assertTrue(success, "error in parsing valid scientific notation reals") + + # any int or real number, returned as float + with self.subTest("ppc.fnumber success run_tests"): + success, _ = ppc.fnumber.run_tests( + """ + 100 + -100 + +100 + 3.14159 + 6.02e23 + 1e-12 + """ + ) + self.assertTrue(success, "error in parsing valid numerics") + + with self.subTest("ppc.ieee_float success run_tests"): + success, _ = ppc.ieee_float.run_tests( + """ + 100 + 3.14159 + 6.02e23 + 1E-12 + 0 + -0 + NaN + -nan + inf + -Infinity + """ + ) + self.assertTrue(success, "error in parsing valid floating-point literals") + + with self.subTest("ppc.iso8601_date success run_tests"): + success, results = ppc.iso8601_date.run_tests( + """ + 1997 + 1997-07 + 1997-07-16 + """ + ) + self.assertTrue(success, "error in parsing valid iso8601_date") + expected = [ + ("1997", None, None), + ("1997", "07", None), + ("1997", "07", "16"), + ] + for r, exp in zip(results, expected): + self.assertEqual( + exp, + (r[1].year, r[1].month, r[1].day), + "failed to parse date into fields", + ) + + with self.subTest("ppc.iso8601_date conversion success run_tests"): + success, results = ( + ppc.iso8601_date() + .add_parse_action(ppc.convert_to_date()) + .run_tests( + """ + 1997-07-16 + """ + ) + ) + self.assertTrue( + success, "error in parsing valid iso8601_date with parse action" + ) + self.assertEqual( + datetime.date(1997, 7, 16), + results[0][1][0], + "error in parsing valid iso8601_date with parse action - incorrect value", + ) + + with self.subTest("ppc.iso8601_datetime success run_tests"): + success, results = ppc.iso8601_datetime.run_tests( + """ + 1997-07-16T19:20+01:00 + 1997-07-16T19:20:30+01:00 + 1997-07-16T19:20:30.45Z + 1997-07-16 19:20:30.45 + """ + ) + self.assertTrue(success, "error in parsing valid iso8601_datetime") + + with self.subTest("ppc.iso8601_datetime conversion success run_tests"): + success, results = ( + ppc.iso8601_datetime() + .add_parse_action(ppc.convert_to_datetime()) + .run_tests( + """ + 1997-07-16T19:20:30.45 + """ + ) + ) + + self.assertTrue(success, "error in parsing valid iso8601_datetime") + self.assertEqual( + datetime.datetime(1997, 7, 16, 19, 20, 30, 450000), + results[0][1][0], + "error in parsing valid iso8601_datetime - incorrect value", + ) + + with self.subTest("ppc.as_datetime fractional seconds run_tests"): + success, results = ( + ppc.iso8601_datetime() + .add_parse_action(ppc.as_datetime) + .run_tests( + """ + 1997-07-16T19:20:30.45 + """ + ) + ) + + self.assertTrue(success, "error in parsing valid iso8601_datetime") + # as_datetime must scale fractional seconds to microseconds, matching + # convert_to_datetime above (0.45 s -> 450000 us, not 449 us). + self.assertEqual( + datetime.datetime(1997, 7, 16, 19, 20, 30, 450000), + results[0][1][0], + "error in as_datetime fractional seconds - incorrect microseconds", + ) + + with self.subTest( + "ppc.as_datetime fractional seconds rounding up carry into next second" + ): + # A fractional-seconds value that rounds up to a full second (e.g. the + # trailing digits of a nanosecond-precision timestamp) must carry into + # the next second, not overflow datetime's 0..999999 microsecond arg. + parse_dt = ppc.iso8601_datetime().add_parse_action(ppc.as_datetime) + self.assertEqual( + datetime.datetime(2021, 1, 1, 0, 0, 1), + parse_dt.parse_string("2021-01-01T00:00:00.999999999")[0], + "as_datetime must carry a rounded-up fractional second, not raise", + ) + self.assertEqual( + datetime.datetime(2021, 6, 15, 12, 31, 0), + parse_dt.parse_string("2021-06-15T12:30:59.9999995")[0], + "as_datetime must carry a rounded-up fractional second across minutes", + ) + + with self.subTest( + "ppc.as_datetime ParseException receives input string, not tokens" + ): + # as_datetime should raise ParseException with the input string + # as pstr, not the ParseResults tokens, so that the exception + # message can be formatted without a secondary TypeError. + result = ppc.iso8601_datetime.parse_string("1997-13-01T00:00:00") + msg = "" + try: + ppc.as_datetime("", 0, result) + except ParseException as pe: + msg = str(pe) + except Exception: + pass + self.assertNotIn( + "TypeError", + msg, + "ParseException string must not contain TypeError " + "(pstr should be the input string, not tokens).", + ) + + with self.subTest("ppc.uuid success run_tests"): + success, _ = ppc.uuid.run_tests( + """ + 123e4567-e89b-12d3-a456-426655440000 + """ + ) + self.assertTrue(success, "failed to parse valid uuid") + + with self.subTest("ppc.fraction success run_tests"): + success, _ = ppc.fraction.run_tests( + """ + 1/2 + -15/16 + -3/-4 + """ + ) + self.assertTrue(success, "failed to parse valid fraction") + + with self.subTest("ppc.mixed_integer success run_tests"): + success, _ = ppc.mixed_integer.run_tests( + """ + 1/2 + -15/16 + -3/-4 + 1 1/2 + 2 -15/16 + 0 -3/-4 + 12 + """ + ) + self.assertTrue(success, "failed to parse valid mixed integer") + + with self.subTest("ppc.number success run_tests"): + success, results = ppc.number.run_tests( + """ + 100 + -3 + 1.732 + -3.14159 + 6.02e23""" + ) + self.assertTrue(success, "failed to parse numerics") + + for test, result in results: + expected = ast.literal_eval(test) + self.assertEqual( + expected, + result[0], + f"numeric parse failed (wrong value) ({result[0]} should be {expected})", + ) + self.assertEqual( + type(expected), + type(result[0]), + f"numeric parse failed (wrong type) ({type(result[0])} should be {type(expected)})", + ) + + def testDateTimeValidation(self): + if sys.version_info[:2] < (3, 10): + return + + ppc = pp.pyparsing_common + date_expr = ppc.iso8601_date_validated + datetime_expr = ppc.iso8601_datetime_validated + + valid_dates = [ + "2023-01-01", + "2024-02-29", # Leap year + "2000-02-29", # Century leap year + "1900-01-01", + "9999-12-31", + "2023-12", # Optional day + "2023", # Optional month and day + ] + for d in valid_dates: + with self.subTest(date=d): + date_expr.parse_string(d, parse_all=True) + + invalid_dates = [ + "2023-02-29", # Not a leap year + "2023-04-31", # April has 30 days + "2023-13-01", # Invalid month + "2023-00-01", # Invalid month + "2023-01-32", # Invalid day + "2023-01-00", # Invalid day + "1900-02-29", # 1900 was not a leap year + ] + for d in invalid_dates: + with self.subTest(date=d): + with self.assertRaises(pp.ParseException): + date_expr.parse_string(d, parse_all=True) + + valid_datetimes = [ + "2023-01-01T12:00:00", + "2023-01-01 12:00:00", + "2023-01-01T12:00:00Z", + "2023-01-01T12:00:00+05:00", + "2023-01-01T12:00:00-05:00", + "2023-01-01T12:00:00.123", + "2023-01-01T12:00:00.123Z", + "2024-02-29T23:59:59", + ] + for dt in valid_datetimes: + with self.subTest(datetime=dt): + datetime_expr.parse_string(dt, parse_all=True) + + invalid_datetimes = [ + "2023-02-29T12:00:00", # Invalid date + "2023-01-01T24:00:00", # Invalid hour + "2023-01-01T12:60:00", # Invalid minute + "2023-01-01T12:00:60", # Invalid second + # "2023-01-01T12:00:00+24:00", # Invalid timezone (Regex might catch it or not, valid_date_time doesn't check it) + ] + for dt in invalid_datetimes: + with self.subTest(datetime=dt): + with self.assertRaises(pp.ParseException): + datetime_expr.parse_string(dt, parse_all=True) + + def testCommonUrl(self): + url_good_tests = """\ + http://foo.com/blah_blah + http://foo.com/blah_blah/ + http://foo.com/blah_blah_(wikipedia) + http://foo.com/blah_blah_(wikipedia)_(again) + http://www.example.com/wpstyle/?p=364 + https://www.example.com/foo/?bar=baz&inga=42&quux + http://✪df.ws/123 + http://userid:password@example.com:8080 + http://userid:password@example.com:8080/ + http://userid@example.com + http://userid@example.com/ + http://userid@example.com:8080 + http://userid@example.com:8080/ + http://userid:password@example.com + http://userid:password@example.com/ + http://142.42.1.1/ + http://142.42.1.1:8080/ + http://➡.ws/䨹 + http://⌘.ws + http://⌘.ws/ + http://foo.com/blah_(wikipedia)#cite-1 + http://foo.com/blah_(wikipedia)_blah#cite-1 + http://foo.com/unicode_(✪)_in_parens + http://foo.com/(something)?after=parens + http://☺.damowmow.com/ + http://code.google.com/events/#&product=browser + http://j.mp + ftp://foo.bar/baz + http://foo.bar/?q=Test%20URL-encoded%20stuff + http://مثال.إختبار + """ + success, report = ppc.url.run_tests(url_good_tests) + self.assertTrue(success) + + url_bad_tests = """\ + http:// + http://. + http://.. + http://../ + http://? + http://?? + http://??/ + http://# + http://## + http://##/ + # skip: http://foo.bar?q=Spaces should be encoded + // + //a + ///a + /// + http:///a + foo.com + rdar://1234 + h://test + http:// shouldfail.com + + :// should fail + http://foo.bar/foo(bar)baz quux + ftps://foo.bar/ + http://-error-.invalid/ + # skip: http://a.b--c.de/ + http://-a.b.co + http://a.b-.co + http://0.0.0.0 + http://10.1.1.0 + http://10.1.1.255 + http://224.1.1.1 + http://1.1.1.1.1 + http://123.123.123 + http://3628126748 + http://.www.foo.bar/ + # skip: http://www.foo.bar./ + http://.www.foo.bar./ + http://10.1.1.1 + """ + success, report = ppc.url.run_tests(url_bad_tests, failure_tests=True) + self.assertTrue(success) + + def testCommonUrlParts(self): + from urllib.parse import urlparse + + sample_url = "https://bob:secret@www.example.com:8080/path/to/resource?filter=int#book-mark" + + parts = urlparse(sample_url) + expected = { + "scheme": parts.scheme, + "auth": f"{parts.username}:{parts.password}", + "host": parts.hostname, + "port": str(parts.port), + "path": parts.path, + "query": parts.query, + "fragment": parts.fragment, + "url": sample_url, + } + + self.assertParseAndCheckDict(ppc.url, sample_url, expected, verbose=True) + + def testCommonUrlExprs(self): + def extract_parts(s, split=" "): + return [[_.strip(split)] for _ in s.strip(split).split(split)] + + test_string = "http://example.com https://blah.org " + self.assertParseAndCheckList( + pp.Group(ppc.url)[...], test_string, extract_parts(test_string) + ) + + test_string = test_string.replace(" ", " , ") + self.assertParseAndCheckList( + pp.DelimitedList(pp.Group(ppc.url), allow_trailing_delim=True), + test_string, + extract_parts(test_string, " , "), + ) + + def testNumericExpressions(self): + # disable parse actions that do type conversion so we don't accidentally trigger + # conversion exceptions when what we want to check is the parsing expression + real = ppc.real().set_parse_action(None) + sci_real = ppc.sci_real().set_parse_action(None) + signed_integer = ppc.signed_integer().set_parse_action(None) + + from itertools import product + + def make_tests(): + leading_sign = ["+", "-", ""] + leading_digit = ["0", ""] + dot = [".", ""] + decimal_digit = ["1", ""] + e = ["e", "E", ""] + e_sign = ["+", "-", ""] + e_int = ["22", ""] + stray = ["9", ".", ""] + + seen = set() + seen.add("") + for parts in product( + leading_sign, + stray, + leading_digit, + dot, + decimal_digit, + stray, + e, + e_sign, + e_int, + stray, + ): + parts_str = "".join(parts).strip() + if parts_str in seen: + continue + seen.add(parts_str) + yield parts_str + + print(len(seen) - 1, "tests produced") + + # collect tests into valid/invalid sets, depending on whether they evaluate to valid Python floats or ints + valid_ints = set() + valid_reals = set() + valid_sci_reals = set() + invalid_ints = set() + invalid_reals = set() + invalid_sci_reals = set() + + # check which strings parse as valid floats or ints, and store in related valid or invalid test sets + for test_str in make_tests(): + if "." in test_str or "e" in test_str.lower(): + try: + float(test_str) + except ValueError: + invalid_sci_reals.add(test_str) + if "e" not in test_str.lower(): + invalid_reals.add(test_str) + else: + valid_sci_reals.add(test_str) + if "e" not in test_str.lower(): + valid_reals.add(test_str) + + try: + int(test_str) + except ValueError: + invalid_ints.add(test_str) + else: + valid_ints.add(test_str) + + # now try all the test sets against their respective expressions + all_pass = True + suppress_results = {"print_results": False} + for expr, tests, is_fail, fn in zip( + [real, sci_real, signed_integer] * 2, + [ + valid_reals, + valid_sci_reals, + valid_ints, + invalid_reals, + invalid_sci_reals, + invalid_ints, + ], + [False, False, False, True, True, True], + [float, float, int] * 2, + ): + # + # success, test_results = expr.run_tests(sorted(tests, key=len), failure_tests=is_fail, **suppress_results) + # filter_result_fn = (lambda r: isinstance(r, Exception), + # lambda r: not isinstance(r, Exception))[is_fail] + # print(expr, ('FAIL', 'PASS')[success], "{}valid tests ({})".format(len(tests), + # 'in' if is_fail else '')) + # if not success: + # all_pass = False + # for test_string, result in test_results: + # if filter_result_fn(result): + # try: + # test_value = fn(test_string) + # except ValueError as ve: + # test_value = str(ve) + # print("{!r}: {} {} {}".format(test_string, result, + # expr.matches(test_string, parse_all=True), test_value)) + + success = True + for t in tests: + if expr.matches(t, parse_all=True): + if is_fail: + print(t, "should fail but did not") + success = False + else: + if not is_fail: + print(t, "should not fail but did") + success = False + print( + expr, + ("FAIL", "PASS")[success], + f"{'in' if is_fail else ''}valid tests ({len(tests)})", + ) + all_pass = all_pass and success + + self.assertTrue(all_pass, "failed one or more numeric tests") + + def testTokenMap(self): + parser = pp.OneOrMore(pp.Word(pp.hexnums)).set_parse_action( + pp.token_map(int, 16) + ) + success, report = parser.run_tests( + """ + 00 11 22 aa FF 0a 0d 1a + """ + ) + + self.assertRunTestResults( + (success, report), + [([0, 17, 34, 170, 255, 10, 13, 26], "token_map parse action failed")], + msg="failed to parse hex integers", + ) + + def testParseFile(self): + s = """ + 123 456 789 + """ + from pathlib import Path + + integer = ppc.integer + test_parser = integer[1, ...] + + input_file_as_stringio = StringIO(s) + input_file_as_str = "tests/parsefiletest_input_file.txt" + input_file_as_path = Path(input_file_as_str) + + expected_list = [int(i) for i in s.split()] + + for input_file in ( + input_file_as_stringio, + input_file_as_str, + input_file_as_path, + ): + with self.subTest(input_file=input_file): + print(f"parse_file() called with {type(input_file).__name__}") + results = test_parser.parse_file(input_file) + print(results) + self.assertEqual(expected_list, results.as_list()) + + def testHTMLStripper(self): + sample = """ + <html> + Here is some sample <i>HTML</i> text. + </html> + """ + read_everything = pp.original_text_for(pp.OneOrMore(pp.Word(pp.printables))) + read_everything.add_parse_action(ppc.strip_html_tags) + + result = read_everything.parse_string(sample, parse_all=True) + self.assertEqual("Here is some sample HTML text.", result[0].strip()) + + def testExprSplitter(self): + expr = pp.Literal(";") + pp.Empty() + expr.ignore(pp.quoted_string) + expr.ignore(pp.python_style_comment) + + sample = """ + def main(): + this_semi_does_nothing(); + neither_does_this_but_there_are_spaces_afterward(); + a = "a;b"; return a # this is a comment; it has a semicolon! + + def b(): + if False: + z=1000;b("; in quotes"); c=200;return z + return ';' + + class Foo(object): + def bar(self): + '''a docstring; with a semicolon''' + a = 10; b = 11; c = 12 + + # this comment; has several; semicolons + if self.spam: + x = 12; return x # so; does; this; one + x = 15;;; y += x; return y + + def baz(self): + return self.bar + """ + expected = [ + [" this_semi_does_nothing()", ""], + [" neither_does_this_but_there_are_spaces_afterward()", ""], + [ + ' a = "a;b"', + "return a # this is a comment; it has a semicolon!", + ], + [" z=1000", 'b("; in quotes")', "c=200", "return z"], + [" return ';'"], + [" '''a docstring; with a semicolon'''"], + [" a = 10", "b = 11", "c = 12"], + [" # this comment; has several; semicolons"], + [" x = 12", "return x # so; does; this; one"], + [" x = 15", "", "", "y += x", "return y"], + ] + + exp_iter = iter(expected) + for line in filter(lambda ll: ";" in ll, sample.splitlines()): + print(str(list(expr.split(line))) + ",") + self.assertEqual( + next(exp_iter), list(expr.split(line)), "invalid split on expression" + ) + + print() + + expected = [ + [" this_semi_does_nothing()", ";", ""], + [" neither_does_this_but_there_are_spaces_afterward()", ";", ""], + [ + ' a = "a;b"', + ";", + "return a # this is a comment; it has a semicolon!", + ], + [ + " z=1000", + ";", + 'b("; in quotes")', + ";", + "c=200", + ";", + "return z", + ], + [" return ';'"], + [" '''a docstring; with a semicolon'''"], + [" a = 10", ";", "b = 11", ";", "c = 12"], + [" # this comment; has several; semicolons"], + [" x = 12", ";", "return x # so; does; this; one"], + [ + " x = 15", + ";", + "", + ";", + "", + ";", + "y += x", + ";", + "return y", + ], + ] + exp_iter = iter(expected) + for line in filter(lambda ll: ";" in ll, sample.splitlines()): + print(str(list(expr.split(line, include_separators=True))) + ",") + self.assertEqual( + next(exp_iter), + list(expr.split(line, include_separators=True)), + "invalid split on expression", + ) + + print() + + expected = [ + [" this_semi_does_nothing()", ""], + [" neither_does_this_but_there_are_spaces_afterward()", ""], + [ + ' a = "a;b"', + "return a # this is a comment; it has a semicolon!", + ], + [" z=1000", 'b("; in quotes"); c=200;return z'], + [" a = 10", "b = 11; c = 12"], + [" x = 12", "return x # so; does; this; one"], + [" x = 15", ";; y += x; return y"], + ] + exp_iter = iter(expected) + for line in sample.splitlines(): + pieces = list(expr.split(line, maxsplit=1)) + print(str(pieces) + ",") + if len(pieces) == 2: + exp = next(exp_iter) + self.assertEqual( + exp, pieces, "invalid split on expression with maxSplits=1" + ) + elif len(pieces) == 1: + self.assertEqual( + 0, + len(expr.search_string(line)), + "invalid split with maxSplits=1 when expr not present", + ) + else: + print("\n>>> " + line) + self.fail("invalid split on expression with maxSplits=1, corner case") + + def testParseFatalException(self): + with self.assertRaisesParseException( + exc_type=ParseFatalException, msg="failed to raise ErrorStop exception" + ): + expr = "ZZZ" - pp.Word(pp.nums) + expr.parse_string("ZZZ bad", parse_all=True) + + def testParseFatalException2(self): + # Fatal exception raised in MatchFirst should not be superseded later non-fatal exceptions + # addresses Issue #251 + + def raise_exception(tokens): + raise pp.ParseSyntaxException("should raise here") + + test = pp.MatchFirst( + ( + pp.pyparsing_common.integer + pp.pyparsing_common.identifier + ).set_parse_action(raise_exception) + | pp.pyparsing_common.number + ) + + with self.assertRaisesParseException(pp.ParseFatalException): + test.parse_string("1s", parse_all=True) + + def testParseFatalException3(self): + # Fatal exception raised in MatchFirst should not be superseded later non-fatal exceptions + # addresses Issue #251 + + test = pp.MatchFirst( + (pp.pyparsing_common.integer - pp.pyparsing_common.identifier) + | pp.pyparsing_common.integer + ) + + with self.assertRaisesParseException(pp.ParseFatalException): + test.parse_string("1", parse_all=True) + + def testInlineLiteralsUsing(self): + wd = pp.Word(pp.alphas) + + pp.ParserElement.inline_literals_using(pp.Suppress) + result = (wd + "," + wd + pp.one_of("! . ?")).parse_string( + "Hello, World!", parse_all=True + ) + self.assertEqual(3, len(result), "inline_literals_using(Suppress) failed!") + + pp.ParserElement.inline_literals_using(pp.Literal) + result = (wd + "," + wd + pp.one_of("! . ?")).parse_string( + "Hello, World!", parse_all=True + ) + self.assertEqual(4, len(result), "inline_literals_using(Literal) failed!") + + pp.ParserElement.inline_literals_using(pp.CaselessKeyword) + self.assertParseAndCheckList( + "SELECT" + wd + "FROM" + wd, + "select color from colors", + expected_list=["SELECT", "color", "FROM", "colors"], + msg="inline_literals_using(CaselessKeyword) failed!", + verbose=True, + ) + + pp.ParserElement.inline_literals_using(pp.CaselessLiteral) + self.assertParseAndCheckList( + "SELECT" + wd + "FROM" + wd, + "select color from colors", + expected_list=["SELECT", "color", "FROM", "colors"], + msg="inline_literals_using(CaselessLiteral) failed!", + verbose=True, + ) + + integer = pp.Word(pp.nums) + pp.ParserElement.inline_literals_using(pp.Literal) + date_str = integer("year") + "/" + integer("month") + "/" + integer("day") + self.assertParseAndCheckList( + date_str, + "1999/12/31", + expected_list=["1999", "/", "12", "/", "31"], + msg="inline_literals_using(example 1) failed!", + verbose=True, + ) + + # change to Suppress + pp.ParserElement.inline_literals_using(pp.Suppress) + date_str = integer("year") + "/" + integer("month") + "/" + integer("day") + + self.assertParseAndCheckList( + date_str, + "1999/12/31", + expected_list=["1999", "12", "31"], + msg="inline_literals_using(example 2) failed!", + verbose=True, + ) + + def testCloseMatch(self): + searchseq = pp.CloseMatch("ATCATCGAATGGA", 2) + + _, results = searchseq.run_tests( + """ + ATCATCGAATGGA + XTCATCGAATGGX + ATCATCGAAXGGA + ATCAXXGAATGGA + ATCAXXGAATGXA + ATCAXXGAATGG + """ + ) + expected = ([], [0, 12], [9], [4, 5], None, None) + + for r, exp in zip(results, expected): + if exp is not None: + self.assertEqual( + exp, + r[1].mismatches, + f"fail CloseMatch between {searchseq.match_string!r} and {r[0]!r}", + ) + print( + r[0], + ( + f"exc: {r[1]}" + if exp is None and isinstance(r[1], Exception) + else ("no match", "match")[r[1].mismatches == exp] + ), + ) + + def testCloseMatchCaseless(self): + searchseq = pp.CloseMatch("ATCATCGAATGGA", 2, caseless=True) + + _, results = searchseq.run_tests( + """ + atcatcgaatgga + xtcatcgaatggx + atcatcgaaxgga + atcaxxgaatgga + atcaxxgaatgxa + atcaxxgaatgg + """ + ) + expected = ([], [0, 12], [9], [4, 5], None, None) + + for r, exp in zip(results, expected): + if exp is not None: + self.assertEqual( + exp, + r[1].mismatches, + f"fail CaselessCloseMatch between {searchseq.match_string!r} and {r[0]!r}", + ) + print( + r[0], + ( + f"exc: {r[1]}" + if exp is None and isinstance(r[1], Exception) + else ("no match", "match")[r[1].mismatches == exp] + ), + ) + + def testDefaultKeywordChars(self): + with self.assertRaisesParseException( + msg="failed to fail matching keyword using updated keyword chars" + ): + pp.Keyword("start").parse_string("start1000", parse_all=True) + + try: + pp.Keyword("start", ident_chars=pp.alphas).parse_string( + "start1000", parse_all=False + ) + except pp.ParseException: + self.fail("failed to match keyword using updated keyword chars") + + with ppt.reset_pyparsing_context(): + pp.Keyword.set_default_keyword_chars(pp.alphas) + try: + pp.Keyword("start").parse_string("start1000", parse_all=False) + except pp.ParseException: + self.fail("failed to match keyword using updated keyword chars") + + with self.assertRaisesParseException( + msg="failed to fail matching keyword using updated keyword chars" + ): + pp.CaselessKeyword("START").parse_string("start1000", parse_all=False) + + try: + pp.CaselessKeyword("START", ident_chars=pp.alphas).parse_string( + "start1000", parse_all=False + ) + except pp.ParseException: + self.fail("failed to match keyword using updated keyword chars") + + with ppt.reset_pyparsing_context(): + pp.Keyword.set_default_keyword_chars(pp.alphas) + try: + pp.CaselessKeyword("START").parse_string("start1000", parse_all=False) + except pp.ParseException: + self.assertTrue( + False, "failed to match keyword using updated keyword chars" + ) + + def testKeywordCopyIdentChars(self): + a_keyword = pp.Keyword("start", ident_chars="_") + b_keyword = a_keyword.copy() + self.assertEqual(a_keyword.ident_chars, b_keyword.ident_chars) + + def testCopyLiteralAttrs(self): + lit = pp.Literal("foo").leave_whitespace() + lit2 = lit.copy() + self.assertFalse(lit2.skipWhitespace) + lit3 = lit2.ignore_whitespace().copy() + self.assertTrue(lit3.skipWhitespace) + + def testWordParseImpl(self): + ppc = pp.common + ppu = pp.unicode + ident_exprs = [ + ppc.identifier, + pp.Word(pp.alphas), + pp.Literal("identifier"), + pp.Regex(r"[a-zA-Z_][a-zA-Z0-9_]*"), + pp.Regex(r"[a-zA-Z_][a-zA-Z0-9_]*", as_match=True), + pp.Regex(r"[a-zA-Z_][a-zA-Z0-9_]*", as_group_list=True), + ppu.BMP.identifier, + pp.Char(pp.alphas), + pp.Keyword("identifier"), + ] + for ident in ident_exprs: + with self.subTest(msg=f"Using expression type {type(ident).__name__}"): + custom_id = ( + ident() + .set_name("standard identifier") + .copy() + .set_name("custom identifier") + ) + with self.assertRaisesParseException( + expected_msg="Expected custom identifier" + ): + custom_id.parse_string("1", parse_all=True) + + with self.subTest(msg="Using integer"): + custom_int = ( + ppc.integer() + .set_name("standard integer") + .copy() + .set_name("custom integer") + ) + with self.assertRaisesParseException( + expected_msg="Expected custom integer" + ): + custom_int.parse_string("z", parse_all=True) + + def testWordCopyWhenWordCharsIncludeSpace(self): + # Issue #618 - with copy() + word_with_space = pp.Word(pp.alphas + " ") + word_with_space.parse_string("ABC") + # no longer raises exception + word_with_space.copy().parse_string("ABC") + + def testWordCopyWhenWordCharsIncludeSpace2(self): + # Issue #618 - with DelimitedList() + element = pp.QuotedString('"') | pp.Combine( + pp.Word(" abcdefghijklmnopqrstuvwxyz") + ) + element.parse_string("abc") + element_list = pp.DelimitedList(element) + # no longer raises exception + element_list.parse_string("abc") + + def testWordCopyWhenWordCharsIncludeSpace3(self): + # Issue #618 - with results name + word_with_space = pp.Word(pp.alphas + " ") + word_with_space.parse_string("ABC") + # no longer raises exception + word_with_space("trouble").parse_string("ABC") + + def testLiteralVsKeyword(self): + integer = ppc.integer + literal_expr = integer + pp.Literal("start") + integer + keyword_expr = integer + pp.Keyword("start") + integer + caseless_keyword_expr = integer + pp.CaselessKeyword("START") + integer + word_keyword_expr = ( + integer + pp.Word(pp.alphas, as_keyword=True).set_name("word") + integer + ) + + print() + test_string = "1 start 2" + print(test_string) + print(literal_expr, literal_expr.parse_string(test_string, parse_all=True)) + print(keyword_expr, keyword_expr.parse_string(test_string, parse_all=True)) + print( + caseless_keyword_expr, + caseless_keyword_expr.parse_string(test_string, parse_all=True), + ) + print( + word_keyword_expr, + word_keyword_expr.parse_string(test_string, parse_all=True), + ) + print() + + test_string = "3 start4" + print(test_string) + print(literal_expr, literal_expr.parse_string(test_string, parse_all=True)) + with self.assertRaisesParseException( + msg="failed to fail matching keyword using updated keyword chars" + ): + print(keyword_expr.parse_string(test_string, parse_all=True)) + + with self.assertRaisesParseException( + msg="failed to fail matching keyword using updated keyword chars" + ): + print(caseless_keyword_expr.parse_string(test_string, parse_all=True)) + + with self.assertRaisesParseException( + msg="failed to fail matching keyword using updated keyword chars" + ): + print(word_keyword_expr.parse_string(test_string, parse_all=True)) + print() + + test_string = "5start 6" + print(test_string) + print(literal_expr.parse_string(test_string, parse_all=True)) + with self.assertRaisesParseException( + msg="failed to fail matching keyword using updated keyword chars" + ): + print(keyword_expr.parse_string(test_string, parse_all=True)) + + with self.assertRaisesParseException( + msg="failed to fail matching keyword using updated keyword chars" + ): + print(caseless_keyword_expr.parse_string(test_string, parse_all=True)) + + with self.assertRaisesParseException( + msg="failed to fail matching keyword using updated keyword chars" + ): + print(word_keyword_expr.parse_string(test_string, parse_all=True)) + + def testCol(self): + test = "*\n* \n* ALF\n*\n" + initials = [c for i, c in enumerate(test) if pp.col(i, test) == 1] + print(initials) + self.assertTrue( + len(initials) == 4 and all(c == "*" for c in initials), "fail col test" + ) + + def testLiteralException(self): + for cls in ( + pp.Literal, + pp.CaselessLiteral, + pp.Keyword, + pp.CaselessKeyword, + pp.Word, + pp.Regex, + ): + expr = cls("xyz") # .set_name('{}_expr'.format(cls.__name__.lower())) + + try: + expr.parse_string(" ", parse_all=True) + except Exception as e: + print(cls.__name__, str(e)) + self.assertTrue( + isinstance(e, pp.ParseBaseException), + f"class {cls.__name__} raised wrong exception type {type(e).__name__}", + ) + + def testParseActionIndexErrorException(self): + """ + Tests raising an IndexError in a parse action + """ + import traceback + + number = pp.Word(pp.nums) + + def number_action(): + raise IndexError # this is the important line! + + number.add_parse_action(number_action) + symbol = pp.Word("abcd", max=1) + expr = pp.Group(number) ^ symbol + + try: + expr.parse_string("1 + 2", parse_all=True) + except IndexError as ie: + pass + except Exception as e: + traceback.print_exc() + self.fail(f"Expected IndexError not raised, raised {type(e).__name__}: {e}") + else: + self.fail("Expected IndexError not raised") + + # tests Issue #22 + def testParseActionNesting(self): + vals = pp.OneOrMore(ppc.integer)("int_values") + + def add_total(tokens): + tokens["total"] = sum(tokens) + return tokens + + vals.add_parse_action(add_total) + results = vals.parse_string("244 23 13 2343", parse_all=True) + print(results.dump()) + self.assertParseResultsEquals( + results, + expected_dict={"int_values": [244, 23, 13, 2343], "total": 2623}, + msg="noop parse action changed ParseResults structure", + ) + + name = pp.Word(pp.alphas)("name") + score = pp.Word(pp.nums + ".")("score") + nameScore = pp.Group(name + score) + line1 = nameScore("Rider") + + result1 = line1.parse_string("Mauney 46.5", parse_all=True) + + print("### before parse action is added ###") + print("result1.dump():\n" + result1.dump() + "\n") + before_pa_dict = result1.as_dict() + + line1.set_parse_action(lambda t: t) + + result1 = line1.parse_string("Mauney 46.5", parse_all=True) + after_pa_dict = result1.as_dict() + + print("### after parse action was added ###") + print("result1.dump():\n" + result1.dump() + "\n") + self.assertEqual( + before_pa_dict, + after_pa_dict, + "noop parse action changed ParseResults structure", + ) + + def testParseActionWithDelimitedList(self): + class AnnotatedToken: + def __init__(self, kind, elements): + self.kind = kind + self.elements = elements + + def __str__(self): + return f"AnnotatedToken({self.kind!r}, {self.elements!r})" + + def __eq__(self, other): + return ( + type(self) == type(other) + and self.kind == other.kind + and self.elements == other.elements + ) + + __repr__ = __str__ + + def annotate(name): + def _(t): + return AnnotatedToken(name, t.as_list()) + + return _ + + identifier = pp.Word(pp.srange("[a-z0-9]")) + numeral = pp.Word(pp.nums) + + named_number_value = pp.Suppress("(") + numeral + pp.Suppress(")") + named_number = identifier + named_number_value + + named_number_list = ( + pp.Suppress("{") + + pp.Group(pp.Optional(pp.DelimitedList(named_number))) + + pp.Suppress("}") + ) + + # repro but in #345 - DelimitedList silently changes contents of named_number + named_number_value.set_parse_action(annotate("val")) + + test_string = "{ x1(1), x2(2) }" + expected = [ + ["x1", AnnotatedToken("val", ["1"]), "x2", AnnotatedToken("val", ["2"])] + ] + + self.assertParseAndCheckList(named_number_list, test_string, expected) + + def testParseActionRunsInNotAny(self): + # see Issue #482 + data = """ [gog1] [G1] [gog2] [gog3] [gog4] [G2] [gog5] [G3] [gog6] """ + + poi_type = pp.Word(pp.alphas).set_results_name("type") + poi = pp.Suppress("[") + poi_type + pp.Char(pp.nums) + pp.Suppress("]") + + def cnd_is_type(val): + return lambda toks: toks.type == val + + poi_gog = poi("gog").add_condition(cnd_is_type("gog")) + poi_g = poi("g").add_condition(cnd_is_type("G")) + + pattern = poi_gog + ~poi_g + + matches = pattern.search_string(data).as_list() + self.assertEqual( + [["gog", "2"], ["gog", "3"], ["gog", "6"]], + matches, + "failed testing parse actions being run inside a NotAny", + ) + + def testParseResultsNameBelowUngroupedName(self): + rule_num = pp.Regex("[0-9]+")("LIT_NUM*") + list_num = pp.Group( + pp.Literal("[")("START_LIST") + + pp.DelimitedList(rule_num)("LIST_VALUES") + + pp.Literal("]")("END_LIST") + )("LIST") + + test_string = "[ 1,2,3,4,5,6 ]" + success, _ = list_num.run_tests(test_string) + self.assertTrue(success) + + U = list_num.parse_string(test_string, parse_all=True) + self.assertTrue( + "LIT_NUM" not in U.LIST.LIST_VALUES, + "results name retained as sub in ungrouped named result", + ) + + def testParseResultsNamesInGroupWithDict(self): + key = ppc.identifier() + value = ppc.integer() + lat = ppc.real() + long = ppc.real() + EQ = pp.Suppress("=") + + data = ( + lat("lat") + + long("long") + + pp.Dict(pp.OneOrMore(pp.Group(key + EQ + value))) + ) + site = pp.QuotedString('"')("name") + pp.Group(data)("data") + + test_string = '"Golden Gate Bridge" 37.819722 -122.478611 height=746 span=4200' + success, _ = site.run_tests(test_string) + self.assertTrue(success) + + a, aEnd = pp.make_html_tags("a") + attrs = a.parse_string("<a href='blah'>", parse_all=True) + print(attrs.dump()) + self.assertParseResultsEquals( + attrs, + expected_dict={ + "startA": {"href": "blah", "tag": "a", "empty": False}, + "href": "blah", + "tag": "a", + "empty": False, + }, + ) + + def testMakeXMLTags(self): + """test helper function make_xml_tags in simple use case""" + + body, bodyEnd = pp.make_xml_tags("body") + tst = "<body>Hello</body>" + expr = body + pp.Word(pp.alphas)("contents") + bodyEnd + result = expr.parse_string(tst, parse_all=True) + print(result.dump()) + self.assertParseResultsEquals( + result, ["body", False, "Hello", "</body>"], msg="issue using make_xml_tags" + ) + + def testFollowedBy(self): + expr = pp.Word(pp.alphas)("item") + pp.FollowedBy(ppc.integer("qty")) + result = expr.parse_string("balloon 99", parse_all=False) + print(result.dump()) + self.assertTrue("qty" in result, "failed to capture results name in FollowedBy") + self.assertEqual( + {"item": "balloon", "qty": 99}, + result.as_dict(), + "invalid results name structure from FollowedBy", + ) + + def testSetBreak(self): + """ + Test behavior of ParserElement.set_break(), to invoke the debugger before parsing that element is attempted. + + Temporarily monkeypatches sys.breakpointhook(). + """ + was_called = False + + def mock_set_trace(*args, **kwargs): + nonlocal was_called + was_called = True + + wd = pp.Word(pp.alphas) + wd.set_break() + + print("Before parsing with set_break:", was_called) + + with ppt.reset_pyparsing_context(): + sys.breakpointhook = mock_set_trace + wd.parse_string("ABC", parse_all=True) + + print("After parsing with set_break:", was_called) + sys.breakpointhook = sys.__breakpointhook__ + self.assertTrue(was_called, "set_trace wasn't called by set_break") + + def testUnicodeTests(self): + import unicodedata + + ppu = pp.pyparsing_unicode + + unicode_version = unicodedata.unidata_version + print(f"Unicode version {unicode_version}") + + # verify ranges are converted to sets properly + for unicode_property, expected_length in [ + ("alphas", 48965), + ("alphanums", 49430), + ("identchars", 49013), + ("identbodychars", 50729), + ("printables", 65484), + ]: + charset = getattr(ppu.BMP, unicode_property) + charset_len = len(charset) + + # this subtest is sensitive to the Unicode version used in the current + # python version + if unicode_version == "14.0.0": + with self.subTest(unicode_property=unicode_property, msg="verify len"): + print(f"ppu.BMP.{unicode_property:14}: {charset_len:6d}") + self.assertEqual( + charset_len, + expected_length, + f"incorrect number of ppu.BMP.{unicode_property}," + f" found {charset_len} expected {expected_length}", + ) + + with self.subTest(unicode_property=unicode_property, msg="verify unique"): + char_counts = collections.Counter(charset) + self.assertTrue( + all(count == 1 for count in char_counts.values()), + f"duplicate items found in ppu.BMP.{unicode_property}:" + f" {[(ord(c), c) for c, count in char_counts.items() if count > 1]}", + ) + + # verify proper merging of ranges by addition + kanji_printables = ppu.Japanese.Kanji.printables + katakana_printables = ppu.Japanese.Katakana.printables + hiragana_printables = ppu.Japanese.Hiragana.printables + japanese_printables = ppu.Japanese.printables + with self.subTest(msg="verify constructing ranges by merging types"): + self.assertEqual( + set(kanji_printables + katakana_printables + hiragana_printables), + set(japanese_printables), + "failed to construct ranges by merging Japanese types", + ) + + # verify proper merging of ranges using multiple inheritance + cjk_printables = ppu.CJK.printables + chinese_printables = ppu.Chinese.printables + korean_printables = ppu.Korean.printables + with self.subTest( + msg="verify merging ranges by using multiple inheritance generates unique list of characters" + ): + char_counts = collections.Counter(cjk_printables) + self.assertTrue( + all(count == 1 for count in char_counts.values()), + "duplicate items found in ppu.CJK.printables:" + f" {[(ord(c), c) for c, count in char_counts.items() if count > 1]}", + ) + + with self.subTest( + msg="verify merging ranges by using multiple inheritance generates sorted list of characters" + ): + self.assertEqual( + list(cjk_printables), + sorted(cjk_printables), + "CJK printables are not sorted", + ) + + with self.subTest( + msg="verify summing chars is equivalent to merging ranges by using multiple inheritance (CJK)" + ): + print( + len(set(chinese_printables + korean_printables + japanese_printables)), + len(cjk_printables), + ) + + self.assertEqual( + set(chinese_printables + korean_printables + japanese_printables), + set(cjk_printables), + "failed to construct ranges by merging Chinese, Japanese and Korean", + ) + + def testUnicodeTests2(self): + ppu = pp.unicode + + alphas = ppu.Greek.alphas + greet = pp.Word(alphas) + "," + pp.Word(alphas) + "!" + + # input string + hello = "Καλημέρα, κόσμε!" + result = greet.parse_string(hello, parse_all=True) + print(result) + self.assertParseResultsEquals( + result, + expected_list=["Καλημέρα", ",", "κόσμε", "!"], + msg="Failed to parse Greek 'Hello, World!' using " + "pyparsing_unicode.Greek.alphas", + ) + + # define a custom unicode range using multiple inheritance + class Turkish_set(ppu.Latin1, ppu.LatinA): + pass + + for attrname in "printables alphas nums identchars identbodychars".split(): + with self.subTest( + "verify unicode_set composed using MI", attrname=attrname + ): + latin1_value = getattr(ppu.Latin1, attrname) + latinA_value = getattr(ppu.LatinA, attrname) + turkish_value = getattr(Turkish_set, attrname) + self.assertEqual( + set(latin1_value + latinA_value), + set(turkish_value), + f"failed to construct ranges by merging Latin1 and LatinA ({attrname})", + ) + + with self.subTest("Test using new Turkish_set for parsing"): + key = pp.Word(Turkish_set.alphas) + value = ppc.integer | pp.Word(Turkish_set.alphas, Turkish_set.alphanums) + EQ = pp.Suppress("=") + key_value = key + EQ + value + + sample = """\ + şehir=İzmir + ülke=Türkiye + nüfus=4279677""" + result = pp.Dict(pp.OneOrMore(pp.Group(key_value))).parse_string( + sample, parse_all=True + ) + + print(result.dump()) + self.assertParseResultsEquals( + result, + expected_dict={"şehir": "İzmir", "ülke": "Türkiye", "nüfus": 4279677}, + msg="Failed to parse Turkish key-value pairs", + ) + + # Basic Multilingual Plane only contains chars up to 65535 + def filter_16_bit(s): + return "".join(c for c in s if ord(c) < 2**16) + + with self.subTest(): + bmp_printables = ppu.BMP.printables + sample = ( + "".join( + random.choice(filter_16_bit(unicode_set.printables)) + for unicode_set in ( + ppu.Japanese, + Turkish_set, + ppu.Greek, + ppu.Hebrew, + ppu.Devanagari, + ppu.Hangul, + ppu.Latin1, + ppu.Chinese, + ppu.Cyrillic, + ppu.Arabic, + ppu.Thai, + ) + for _ in range(8) + ) + + "\N{REPLACEMENT CHARACTER}" + ) + print(sample) + self.assertParseAndCheckList(pp.Word(bmp_printables), sample, [sample]) + + def testUnicodeSetNameEquivalence(self): + ppu = pp.unicode + + for ascii_name, unicode_name in [ + ("Arabic", "العربية"), + ("Chinese", "中文"), + ("Cyrillic", "кириллица"), + ("Greek", "Ελληνικά"), + ("Hebrew", "עִברִית"), + ("Japanese", "日本語"), + ("Korean", "한국어"), + ("Thai", "ไทย"), + ("Devanagari", "देवनागरी"), + ]: + with self.subTest(ascii_name=ascii_name, unicode_name=unicode_name): + self.assertTrue( + eval(f"ppu.{ascii_name} is ppu.{unicode_name}", {}, locals()) + ) + + def testIndentedBlock(self): + # parse pseudo-yaml indented text + + EQ = pp.Suppress("=") + stack = [1] + key = ppc.identifier + value = pp.Forward() + key_value = pp.Group(key + EQ + value) + compound_value = pp.Dict(pp.ungroup(pp.IndentedBlock(key_value, grouped=True))) + value <<= ppc.integer | pp.QuotedString("'") | compound_value + parser = pp.Dict(pp.OneOrMore(key_value)) + + text = """ + a = 100 + b = 101 + c = + c1 = 200 + c2 = + c21 = 999 + c3 = 'A horse, a horse, my kingdom for a horse' + d = 505 + """ + text = dedent(text) + print(text) + + result = parser.parse_string(text, parse_all=True) + print(result.dump()) + self.assertEqual(100, result.a, "invalid indented block result") + self.assertEqual(200, result.c.c1, "invalid indented block result") + self.assertEqual(999, result.c.c2.c21, "invalid indented block result") + + # exercise indented_block with example posted in issue #87 + def testIndentedBlockTest2(self): + indent_stack = [1] + + key = pp.Word(pp.alphas, pp.alphanums) + pp.Suppress(":") + stmt = pp.Forward() + + suite = pp.IndentedBlock(stmt, grouped=True) + body = key + suite + + pattern = ( + pp.Word(pp.alphas) + + pp.Suppress("(") + + pp.Word(pp.alphas) + + pp.Suppress(")") + ) + stmt <<= pattern + + def key_parse_action(toks): + print(f"Parsing '{toks[0]}'...") + + key.set_parse_action(key_parse_action) + header = pp.Suppress("[") + pp.Literal("test") + pp.Suppress("]") + content = header - pp.OneOrMore(pp.IndentedBlock(body)) + + contents = pp.Forward() + suites = pp.IndentedBlock(content) + + extra = pp.Literal("extra") + pp.Suppress(":") - suites + contents <<= content | extra + + parser = pp.OneOrMore(contents) + + sample = dedent( + """ + extra: + [test] + one0: + two (three) + four0: + five (seven) + extra: + [test] + one1: + two (three) + four1: + five (seven) + """ + ) + + success, _ = parser.run_tests([sample]) + self.assertTrue(success, "Failed IndentedBlock test for issue #87") + + sample2 = dedent( + """ + extra: + [test] + one: + two (three) + four: + five (seven) + extra: + [test] + one: + two (three) + four: + five (seven) + + [test] + one: + two (three) + four: + five (seven) + + [test] + eight: + nine (ten) + eleven: + twelve (thirteen) + + fourteen: + fifteen (sixteen) + seventeen: + eighteen (nineteen) + """ + ) + + del indent_stack[1:] + success, _ = parser.run_tests([sample2]) + self.assertTrue(success, "Failed indented_block multi-block test for issue #87") + + def testIndentedBlockScan(self): + def get_parser(): + """ + A valid statement is the word "block:", followed by an indent, followed by the letter A only, or another block + """ + stack = [1] + block = pp.Forward() + body = pp.IndentedBlock(pp.Literal("A") ^ block) + block <<= pp.Literal("block:") + body + return block + + # This input string is a perfect match for the parser, so a single match is found + p1 = get_parser() + r1 = list( + p1.scan_string( + dedent( + """\ + block: + A + """ + ) + ) + ) + self.assertEqual(1, len(r1)) + + # This input string is a perfect match for the parser, except for the letter B instead of A, so this will fail (and should) + p2 = get_parser() + r2 = list( + p2.scan_string( + dedent( + """\ + block: + B + """ + ) + ) + ) + self.assertEqual(0, len(r2)) + + # This input string contains both string A and string B, and it finds one match (as it should) + p3 = get_parser() + r3 = list( + p3.scan_string( + dedent( + """\ + block: + A + block: + B + """ + ) + ) + ) + self.assertEqual(1, len(r3)) + + # This input string contains both string A and string B, but in a different order. + p4 = get_parser() + r4 = list( + p4.scan_string( + dedent( + """\ + block: + B + block: + A + """ + ) + ) + ) + self.assertEqual(1, len(r4)) + + # This is the same as case 3, but with nesting + p5 = get_parser() + r5 = list( + p5.scan_string( + dedent( + """\ + block: + block: + A + block: + block: + B + """ + ) + ) + ) + self.assertEqual(1, len(r5)) + + # This is the same as case 4, but with nesting + p6 = get_parser() + r6 = list( + p6.scan_string( + dedent( + """\ + block: + block: + B + block: + block: + A + """ + ) + ) + ) + self.assertEqual(1, len(r6)) + + def testIndentedBlockClass(self): + data = """\ + A + 100 + 101 + + 102 + B + 200 + 201 + + C + 300 + + """ + + integer = ppc.integer + group = pp.Group(pp.Char(pp.alphas) + pp.IndentedBlock(integer)) + + group[...].parse_string(data, parse_all=True).pprint() + + self.assertParseAndCheckList( + group[...], data, [["A", [100, 101, 102]], ["B", [200, 201]], ["C", [300]]] + ) + + def testIndentedBlockClass2(self): + datas = [ + """\ + A + 100 + B + 200 + 201 + """, + """\ + A + 100 + B + 200 + 201 + """, + """\ + A + 100 + B + 200 + 201 + """, + ] + integer = ppc.integer + group = pp.Group( + pp.Char(pp.alphas) + pp.IndentedBlock(integer, recursive=False) + ) + + for data in datas: + print() + print(ppt.with_line_numbers(data)) + + print(group[...].parse_string(data).as_list()) + self.assertParseAndCheckList( + group[...] + integer.suppress(), + data, + [["A", [100]], ["B", [200]]], + verbose=False, + ) + + def testIndentedBlockClassWithRecursion(self): + data = """\ + + A + 100 + 101 + + 102 + B + b + 200 + 201 + + C + 300 + + """ + + integer = ppc.integer + group = pp.Forward() + group <<= pp.Group(pp.Char(pp.alphas) + pp.IndentedBlock(integer | group)) + + print("using search_string") + print(group.search_string(data)) + # print(sum(group.search_string(data)).dump()) + + self.assertParseAndCheckList( + group[...], + data, + [["A", [100, 101, 102]], ["B", [["b", [200, 201]]]], ["C", [300]]], + ) + + print("using parse_string") + print(group[...].parse_string(data, parse_all=True).dump()) + + dotted_int = pp.DelimitedList( + pp.Word(pp.nums), ".", allow_trailing_delim=True, combine=True + ) + indented_expr = pp.IndentedBlock(dotted_int, recursive=True, grouped=True) + # indented_expr = pp.Forward() + # indented_expr <<= pp.IndentedBlock(dotted_int + indented_expr)) + good_data = """\ + 1. + 1.1 + 1.1.1 + 1.1.2 + 2.""" + bad_data1 = """\ + 1. + 1.1 + 1.1.1 + 1.2 + 2.""" + bad_data2 = """\ + 1. + 1.1 + 1.1.1 + 1.2 + 2.""" + print("test good indentation") + print(pp.pyparsing_test.with_line_numbers(good_data)) + print(indented_expr.parse_string(good_data, parse_all=True).as_list()) + print() + + print("test bad indentation") + print(pp.pyparsing_test.with_line_numbers(bad_data1)) + with self.assertRaisesParseException( + msg="Failed to raise exception with bad indentation 1" + ): + indented_expr.parse_string(bad_data1, parse_all=True) + + print(pp.pyparsing_test.with_line_numbers(bad_data2)) + with self.assertRaisesParseException( + msg="Failed to raise exception with bad indentation 2" + ): + indented_expr.parse_string(bad_data2, parse_all=True) + + def testInvalidDiagSetting(self): + with self.assertRaises( + ValueError, + msg="failed to raise exception when setting non-existent __diag__", + ): + pp.__diag__.enable("xyzzy") + + with self.assertWarns( + UserWarning, msg="failed to warn disabling 'collect_all_And_tokens" + ): + pp.__compat__.disable("collect_all_And_tokens") + + def testParseResultsWithNameMatchFirst(self): + expr_a = pp.Literal("not") + pp.Literal("the") + pp.Literal("bird") + expr_b = pp.Literal("the") + pp.Literal("bird") + expr = (expr_a | expr_b)("rexp") + + success, report = expr.run_tests( + """\ + not the bird + the bird + """ + ) + results = [rpt[1] for rpt in report] + self.assertParseResultsEquals( + results[0], ["not", "the", "bird"], {"rexp": ["not", "the", "bird"]} + ) + self.assertParseResultsEquals( + results[1], ["the", "bird"], {"rexp": ["the", "bird"]} + ) + + # test compatibility mode, no longer restoring pre-2.3.1 behavior + with ppt.reset_pyparsing_context(): + pp.__compat__.collect_all_And_tokens = False + pp.enable_diag(pp.Diagnostics.warn_multiple_tokens_in_named_alternation) + expr_a = pp.Literal("not") + pp.Literal("the") + pp.Literal("bird") + expr_b = pp.Literal("the") + pp.Literal("bird") + with self.assertWarns( + UserWarning, msg="failed to warn of And within alternation" + ): + expr = (expr_a | expr_b)("rexp") + + with self.assertDoesNotWarn( + UserWarning, + msg="warned when And within alternation warning was suppressed", + ): + expr = (expr_a | expr_b).suppress_warning( + pp.Diagnostics.warn_multiple_tokens_in_named_alternation + )("rexp") + + success, report = expr.run_tests( + """ + not the bird + the bird + """ + ) + results = [rpt[1] for rpt in report] + self.assertParseResultsEquals( + results[0], ["not", "the", "bird"], {"rexp": ["not", "the", "bird"]} + ) + self.assertParseResultsEquals( + results[1], ["the", "bird"], {"rexp": ["the", "bird"]} + ) + + def testParseResultsWithNameOr(self): + expr_a = pp.Literal("not") + pp.Literal("the") + pp.Literal("bird") + expr_b = pp.Literal("the") + pp.Literal("bird") + expr = (expr_a ^ expr_b)("rexp") + success, _ = expr.run_tests( + """\ + not the bird + the bird + """ + ) + self.assertTrue(success) + + result = expr.parse_string("not the bird", parse_all=True) + self.assertParseResultsEquals( + result, ["not", "the", "bird"], {"rexp": ["not", "the", "bird"]} + ) + result = expr.parse_string("the bird", parse_all=True) + self.assertParseResultsEquals( + result, ["the", "bird"], {"rexp": ["the", "bird"]} + ) + + expr = (expr_a | expr_b)("rexp") + success, _ = expr.run_tests( + """\ + not the bird + the bird + """ + ) + self.assertTrue(success) + + result = expr.parse_string("not the bird", parse_all=True) + self.assertParseResultsEquals( + result, ["not", "the", "bird"], {"rexp": ["not", "the", "bird"]} + ) + result = expr.parse_string("the bird", parse_all=True) + self.assertParseResultsEquals( + result, ["the", "bird"], {"rexp": ["the", "bird"]} + ) + + # test compatibility mode, no longer restoring pre-2.3.1 behavior + with ppt.reset_pyparsing_context(): + pp.__compat__.collect_all_And_tokens = False + pp.enable_diag(pp.Diagnostics.warn_multiple_tokens_in_named_alternation) + expr_a = pp.Literal("not") + pp.Literal("the") + pp.Literal("bird") + expr_b = pp.Literal("the") + pp.Literal("bird") + + with self.assertWarns( + UserWarning, msg="failed to warn of And within alternation" + ): + expr = (expr_a ^ expr_b)("rexp") + + with self.assertDoesNotWarn( + UserWarning, + msg="warned when And within alternation warning was suppressed", + ): + expr = (expr_a ^ expr_b).suppress_warning( + pp.Diagnostics.warn_multiple_tokens_in_named_alternation + )("rexp") + + success, _ = expr.run_tests( + """\ + not the bird + the bird + """ + ) + self.assertTrue(success) + self.assertEqual( + "not the bird".split(), + list(expr.parse_string("not the bird", parse_all=True)["rexp"]), + ) + self.assertEqual( + "the bird".split(), + list(expr.parse_string("the bird", parse_all=True)["rexp"]), + ) + + def testEmptyDictDoesNotRaiseException(self): + key = pp.Word(pp.alphas) + value = pp.Word(pp.nums) + EQ = pp.Suppress("=") + key_value_dict = pp.dict_of(key, EQ + value) + + print( + key_value_dict.parse_string( + """\ + a = 10 + b = 20 + """, + parse_all=True, + ).dump() + ) + + try: + print(key_value_dict.parse_string("", parse_all=True).dump()) + except pp.ParseException as pe: + print(pp.ParseException.explain(pe)) + else: + self.fail("failed to raise exception when matching empty string") + + def testEmptyDictReturnsDictWhenCallingAsDict(self): + parser1 = pp.Dict(pp.ZeroOrMore(pp.Group(pp.Word(pp.alphas) + pp.Word(pp.alphas)))) + parser2 = parser1.copy()('fubar') + + p1_result = parser1.parse_string('foo bar').as_dict() # {'foo': 'bar'} ✓ + p2_result = parser2.parse_string('foo bar').as_dict() # {'fubar': {'foo': 'bar'}} ✓ + self.assertEqual({"foo": "bar"}, p1_result) + self.assertEqual({"fubar": {"foo": "bar"}}, p2_result) + + p1_result = parser1.parse_string('').as_dict() # {} ✓ + p2_result = parser2.parse_string('').as_dict() # {'fubar': []} ✗ should be {'fubar': {}} + self.assertEqual({}, p1_result) + self.assertEqual({"fubar": {}}, p2_result) + + def testExplainException(self): + expr = pp.Word(pp.nums).set_name("int") + pp.Word(pp.alphas).set_name("word") + try: + expr.parse_string("123 355", parse_all=True) + except pp.ParseException as pe: + print(pe.explain(depth=0)) + + expr = pp.Word(pp.nums).set_name("int") - pp.Word(pp.alphas).set_name("word") + try: + expr.parse_string("123 355 (test using ErrorStop)", parse_all=True) + except pp.ParseSyntaxException as pe: + print(pe.explain()) + + integer = pp.Word(pp.nums).set_name("int").add_parse_action(lambda t: int(t[0])) + expr = integer + integer + + def divide_args(t): + integer.parse_string("A", parse_all=True) + return t[0] / t[1] + + expr.add_parse_action(divide_args) + try: + expr.parse_string("123 0", parse_all=True) + except pp.ParseException as pe: + print(pe.explain()) + except Exception as exc: + print(pp.ParseBaseException.explain_exception(exc)) + raise + + def testExplainExceptionWithMemoizationCheck(self): + if pp.ParserElement._left_recursion_enabled or pp.ParserElement._packratEnabled: + print("test does local memoization enable/disable during test") + return + + pp.ParserElement.disable_memoization() + + integer = pp.Word(pp.nums).set_name("int").add_parse_action(lambda t: int(t[0])) + expr = integer + integer + + def divide_args(t): + integer.parse_string("A", parse_all=True) + return t[0] / t[1] + + expr.add_parse_action(divide_args) + for memo_kind, enable_memo in [ + ("Packrat", pp.ParserElement.enable_packrat), + ("Left Recursion", pp.ParserElement.enable_left_recursion), + ]: + enable_memo(force=True) + print("Explain for", memo_kind) + + try: + expr.parse_string("123 0", parse_all=True) + except pp.ParseException as pe: + print(pe.explain()) + except Exception as exc: + print(pp.ParseBaseException.explain_exception(exc)) + raise + + # make sure we leave the state compatible with everything + pp.ParserElement.disable_memoization() + + def testCaselessKeywordVsKeywordCaseless(self): + frule = pp.Keyword("t", caseless=True) + pp.Keyword("yes", caseless=True) + crule = pp.CaselessKeyword("t") + pp.CaselessKeyword("yes") + + flist = frule.search_string("not yes").as_list() + print(flist) + clist = crule.search_string("not yes").as_list() + print(clist) + self.assertEqual( + flist, + clist, + "CaselessKeyword not working the same as Keyword(caseless=True)", + ) + + def testOneOf(self): + expr = pp.one_of("a b abb") + assert expr.pattern == "abb|a|b" + + expr = pp.one_of("a abb b abb") + assert expr.pattern == "abb|a|b" + + expr = pp.one_of("a abb abbb b abb") + assert expr.pattern == "abbb|abb|a|b" + + expr = pp.one_of("a abbb abb b abb") + assert expr.pattern == "abbb|abb|a|b" + + # make sure regex-unsafe characters are properly escaped + expr = pp.one_of("a+ b* c? () +a *b ?c") + assert expr.pattern == r"a\+|b\*|c\?|\(\)|\+a|\*b|\?c" + + def testOneOfKeywords(self): + literal_expr = pp.one_of("a b c") + success, _ = literal_expr[...].run_tests( + """ + # literal one_of tests + a b c + a a a + abc + """ + ) + self.assertTrue(success, "failed literal one_of matching") + + keyword_expr = pp.one_of("a b c", as_keyword=True) + success, _ = keyword_expr[...].run_tests( + """ + # keyword one_of tests + a b c + a a a + """ + ) + self.assertTrue(success, "failed keyword one_of matching") + + success, _ = keyword_expr[...].run_tests( + """ + # keyword one_of failure tests + abc + """, + failure_tests=True, + ) + self.assertTrue(success, "failed keyword one_of failure tests") + + def testWarnUngroupedNamedTokens(self): + """ + - warn_ungrouped_named_tokens_in_collection - flag to enable warnings when a results + name is defined on a containing expression with ungrouped subexpressions that also + have results names (default=True) + """ + with self.assertDoesNotWarn( + msg=f"raised {pp.Diagnostics.warn_ungrouped_named_tokens_in_collection} warning when not enabled" + ): + COMMA = pp.Suppress(",").set_name("comma") + coord = ppc.integer("x") + COMMA + ppc.integer("y") + path = coord[...].set_results_name("path") + + with ppt.reset_pyparsing_context(): + pp.enable_diag(pp.Diagnostics.warn_ungrouped_named_tokens_in_collection) + + COMMA = pp.Suppress(",").set_name("comma") + coord = ppc.integer("x") + COMMA + ppc.integer("y") + + # this should emit a warning + with self.assertWarns( + UserWarning, + msg="failed to warn with named repetition of" + " ungrouped named expressions", + ): + path = coord[...].set_results_name("path") + + with self.assertDoesNotWarn( + UserWarning, + msg="warned when named repetition of" + " ungrouped named expressions warning was suppressed", + ): + path = ( + coord[...] + .suppress_warning( + pp.Diagnostics.warn_ungrouped_named_tokens_in_collection + ) + .set_results_name("path") + ) + + def testDontWarnUngroupedNamedTokensIfWarningSuppressed(self): + with ppt.reset_pyparsing_context(): + pp.enable_diag(pp.Diagnostics.warn_ungrouped_named_tokens_in_collection) + + with self.assertDoesNotWarn( + msg=f"raised {pp.Diagnostics.warn_ungrouped_named_tokens_in_collection}" + f" warning when warn on ungrouped named tokens was suppressed (original_text_for)" + ): + pp.original_text_for(pp.Word("ABC")[...])("words") + + def testWarnNameSetOnEmptyForward(self): + """ + - warn_name_set_on_empty_Forward - flag to enable warnings when a Forward is defined + with a results name, but has no contents defined (default=False) + """ + + with self.assertDoesNotWarn( + msg=f"raised {pp.Diagnostics.warn_name_set_on_empty_Forward} warning when not enabled" + ): + base = pp.Forward()("z") + + with ppt.reset_pyparsing_context(): + pp.enable_diag(pp.Diagnostics.warn_name_set_on_empty_Forward) + + base = pp.Forward() + + with self.assertWarns( + UserWarning, + msg="failed to warn when naming an empty Forward expression", + ): + base("x") + + with self.assertDoesNotWarn( + UserWarning, + msg="warned when naming an empty Forward expression warning was suppressed", + ): + base.suppress_warning(pp.Diagnostics.warn_name_set_on_empty_Forward)( + "x" + ) + + def testWarnParsingEmptyForward(self): + """ + - warn_on_parse_using_empty_Forward - flag to enable warnings when a Forward + has no contents defined (default=False) + """ + + with self.assertDoesNotWarn( + msg=f"raised {pp.Diagnostics.warn_on_parse_using_empty_Forward} warning when not enabled" + ): + base = pp.Forward() + try: + print(base.parse_string("x", parse_all=True)) + except ParseException as pe: + pass + + with ppt.reset_pyparsing_context(): + pp.enable_diag(pp.Diagnostics.warn_on_parse_using_empty_Forward) + + base = pp.Forward() + + with self.assertWarns( + UserWarning, + msg="failed to warn when parsing using an empty Forward expression", + ): + try: + print(base.parse_string("x", parse_all=True)) + except ParseException as pe: + pass + + with self.assertDoesNotWarn( + UserWarning, + msg="warned when parsing using an empty Forward expression warning was suppressed", + ): + base.suppress_warning(pp.Diagnostics.warn_on_parse_using_empty_Forward) + try: + print(base.parse_string("x", parse_all=True)) + except ParseException as pe: + pass + + def testWarnIncorrectAssignmentToForward(self): + """ + - warn_on_parse_using_empty_Forward - flag to enable warnings when a Forward + has no contents defined (default=False) + """ + if PYPY_ENV: + print("warn_on_assignment_to_Forward not supported on PyPy") + return + + def a_method(): + base = pp.Forward() + base = pp.Word(pp.alphas)[...] | "(" + base + ")" + + with self.assertDoesNotWarn( + msg=f"raised {pp.Diagnostics.warn_on_assignment_to_Forward} warning when not enabled" + ): + a_method() + + with ppt.reset_pyparsing_context(): + pp.enable_diag(pp.Diagnostics.warn_on_assignment_to_Forward) + + with self.assertWarns( + UserWarning, + msg="failed to warn when using '=' to assign expression to a Forward", + ): + a_method() + + def a_method(): + base = pp.Forward().suppress_warning( + pp.Diagnostics.warn_on_assignment_to_Forward + ) + base = pp.Word(pp.alphas)[...] | "(" + base + ")" + + with self.assertDoesNotWarn( + UserWarning, + msg="warned when using '=' to assign expression to a Forward warning was suppressed", + ): + a_method() + + def testWarnOnMultipleStringArgsToOneOf(self): + """ + - warn_on_multiple_string_args_to_oneof - flag to enable warnings when one_of is + incorrectly called with multiple str arguments (default=True) + """ + with self.assertDoesNotWarn( + msg=f"raised {pp.Diagnostics.warn_on_multiple_string_args_to_oneof} warning when not enabled" + ): + a = pp.one_of("A", "B") + + with ppt.reset_pyparsing_context(): + pp.enable_diag(pp.Diagnostics.warn_on_multiple_string_args_to_oneof) + + with self.assertWarns( + UserWarning, + msg="failed to warn when incorrectly calling one_of(string, string)", + ): + a = pp.one_of("A", "B") + + def testAutonameElements(self): + with ppt.reset_pyparsing_context(): + pp.enable_diag(pp.Diagnostics.enable_debug_on_named_expressions) + + a = pp.Literal("a") + b = pp.Literal("b").set_name("bbb") + z = pp.Literal("z") + leading_a = a + pp.FollowedBy(z | a | b) + + grammar = (z | leading_a | b)[...] + "a" + + self.assertFalse(a.debug) + self.assertFalse(a.customName) + pp.autoname_elements() + self.assertTrue(a.debug) + self.assertEqual("a", a.name) + self.assertEqual("bbb", b.name) + + def testDelimitedListName(self): + bool_constant = pp.Literal("True") | "true" | "False" | "false" + bool_list = pp.DelimitedList(bool_constant) + print(bool_list) + self.assertEqual( + "{'True' | 'true' | 'False' | 'false'} [, {'True' | 'true' | 'False' | 'false'}]...", + str(bool_list), + ) + + bool_constant.set_name("bool") + print(bool_constant) + print(bool_constant.streamline()) + bool_list2 = pp.DelimitedList(bool_constant) + print(bool_constant) + print(bool_constant.streamline()) + print(bool_list2) + with self.subTest(): + self.assertEqual("bool [, bool]...", str(bool_list2)) + + with self.subTest(): + street_address = pp.common.integer.set_name("integer") + pp.Word(pp.alphas)[ + 1, ... + ].set_name("street_name") + self.assertEqual( + "{integer street_name} [, {integer street_name}]...", + str(pp.DelimitedList(street_address)), + ) + + with self.subTest(): + operand = pp.Char(pp.alphas).set_name("var") + math = pp.infix_notation( + operand, + [ + (pp.one_of("+ -"), 2, pp.OpAssoc.LEFT), + ], + ) + self.assertEqual( + "var_expression [, var_expression]...", + str(pp.DelimitedList(math)), + ) + + def testDelimitedListOfStrLiterals(self): + expr = pp.DelimitedList("ABC") + print(str(expr)) + source = "ABC, ABC,ABC" + self.assertParseAndCheckList( + expr, source, [s.strip() for s in source.split(",")] + ) + + def testDelimitedListMinMax(self): + source = "ABC, ABC,ABC" + with self.assertRaises(ValueError, msg="min must be greater than 0"): + pp.DelimitedList("ABC", min=0) + with self.assertRaises( + ValueError, msg="max must be greater than, or equal to min" + ): + pp.DelimitedList("ABC", min=1, max=0) + with self.assertRaises(pp.ParseException): + pp.DelimitedList("ABC", min=4).parse_string(source) + + source_expr_pairs = [ + ("ABC, ABC", pp.DelimitedList("ABC", max=2)), + (source, pp.DelimitedList("ABC", min=2, max=4)), + ] + for source, expr in source_expr_pairs: + print(str(expr)) + self.assertParseAndCheckList( + expr, source, [s.strip() for s in source.split(",")] + ) + + def testDelimitedListParseActions1(self): + # from issue #408 + keyword = pp.Keyword("foobar") + untyped_identifier = ~keyword + pp.Word(pp.alphas) + dotted_vars = pp.DelimitedList(untyped_identifier, delim=".") + lvalue = pp.Opt(dotted_vars) + + # uncomment this line to see the problem + stmt = pp.DelimitedList(pp.Opt(dotted_vars)) + # stmt = DelimitedList(dotted_vars) + # stmt = pp.Opt(dotted_vars) + + def parse_identifier(toks): + print("YAY!", toks) + + untyped_identifier.set_parse_action(parse_identifier) + + save_stdout = StringIO() + with contextlib.redirect_stdout(save_stdout): + dotted_vars.parse_string("B.C") + + self.assertEqual( + dedent( + """\ + YAY! ['B'] + YAY! ['C'] + """ + ), + save_stdout.getvalue(), + ) + + def testDelimitedListParseActions2(self): + # from issue #408 + keyword = pp.Keyword("foobar") + untyped_identifier = ~keyword + pp.Word(pp.alphas) + dotted_vars = pp.DelimitedList(untyped_identifier, delim=".") + lvalue = pp.Opt(dotted_vars) + + # uncomment this line to see the problem + # stmt = DelimitedList(Opt(dotted_vars)) + stmt = pp.DelimitedList(dotted_vars) + # stmt = pp.Opt(dotted_vars) + + def parse_identifier(toks): + print("YAY!", toks) + + untyped_identifier.set_parse_action(parse_identifier) + + save_stdout = StringIO() + with contextlib.redirect_stdout(save_stdout): + dotted_vars.parse_string("B.C") + + self.assertEqual( + dedent( + """\ + YAY! ['B'] + YAY! ['C'] + """ + ), + save_stdout.getvalue(), + ) + + def testDelimitedListParseActions3(self): + # from issue #408 + keyword = pp.Keyword("foobar") + untyped_identifier = ~keyword + pp.Word(pp.alphas) + dotted_vars = pp.DelimitedList(untyped_identifier, delim=".") + lvalue = pp.Opt(dotted_vars) + + # uncomment this line to see the problem + # stmt = DelimitedList(Opt(dotted_vars)) + # stmt = DelimitedList(dotted_vars) + stmt = pp.Opt(dotted_vars) + + def parse_identifier(toks): + print("YAY!", toks) + + untyped_identifier.set_parse_action(parse_identifier) + + save_stdout = StringIO() + with contextlib.redirect_stdout(save_stdout): + dotted_vars.parse_string("B.C") + + self.assertEqual( + dedent( + """\ + YAY! ['B'] + YAY! ['C'] + """ + ), + save_stdout.getvalue(), + ) + + def testTagElements(self): + end_punc = ( + ("." + pp.Tag("mood", "normal")) + | ("!" + pp.Tag("mood", "excited")) + | ("?" + pp.Tag("mood", "curious")) + ) + greeting = "Hello" + pp.Word(pp.alphas) + end_punc[1, ...] + + for ending, expected_mood in [ + (".", "normal"), + ("!", "excited"), + ("?", "curious"), + ("!!", "excited"), + ("!?", "curious"), + ]: + self.assertParseAndCheckDict( + greeting, f"Hello World{ending}", {"mood": expected_mood} + ) + + def testEnableDebugOnNamedExpressions(self): + """ + - enable_debug_on_named_expressions - flag to auto-enable debug on all subsequent + calls to ParserElement.set_name() (default=False) + """ + with ppt.reset_pyparsing_context(): + test_stdout = StringIO() + + with resetting(sys, "stdout", "stderr"): + sys.stdout = test_stdout + sys.stderr = test_stdout + + pp.enable_diag(pp.Diagnostics.enable_debug_on_named_expressions) + integer = pp.Word(pp.nums).set_name("integer") + + integer[...].parse_string("1 2 3", parse_all=True) + + expected_debug_output = dedent( + """\ + Match integer at loc 0(1,1) + 1 2 3 + ^ + Matched integer -> ['1'] + Match integer at loc 2(1,3) + 1 2 3 + ^ + Matched integer -> ['2'] + Match integer at loc 4(1,5) + 1 2 3 + ^ + Matched integer -> ['3'] + Match integer at loc 5(1,6) + 1 2 3 + ^ + Match integer failed, ParseException raised: Expected integer, found end of text (at char 5), (line:1, col:6) + """ + ) + output = test_stdout.getvalue() + print(output) + self.assertEqual( + expected_debug_output, + output, + "failed to auto-enable debug on named expressions " + "using enable_debug_on_named_expressions", + ) + + def testEnableDebugOnExpressionWithParseAction(self): + test_stdout = StringIO() + with resetting(sys, "stdout", "stderr"): + sys.stdout = test_stdout + sys.stderr = test_stdout + + parser = (ppc.integer().set_debug() | pp.Word(pp.alphanums).set_debug())[ + ... + ] + parser.set_debug() + parser.parse_string("123 A100", parse_all=True) + + # now turn off debug - should only get output for components, not overall parser + print() + parser.set_debug(False) + parser.parse_string("123 A100", parse_all=True) + + expected_debug_output = dedent( + """\ + Match [{integer | W:(0-9A-Za-z)}]... at loc 0(1,1) + 123 A100 + ^ + Match integer at loc 0(1,1) + 123 A100 + ^ + Matched integer -> [123] + Match integer at loc 4(1,5) + 123 A100 + ^ + Match integer failed, ParseException raised: Expected integer, found 'A100' (at char 4), (line:1, col:5) + Match W:(0-9A-Za-z) at loc 4(1,5) + 123 A100 + ^ + Matched W:(0-9A-Za-z) -> ['A100'] + Match integer at loc 8(1,9) + 123 A100 + ^ + Match integer failed, ParseException raised: Expected integer, found end of text (at char 8), (line:1, col:9) + Match W:(0-9A-Za-z) at loc 8(1,9) + 123 A100 + ^ + Match W:(0-9A-Za-z) failed, ParseException raised: Expected W:(0-9A-Za-z), found end of text (at char 8), (line:1, col:9) + Matched [{integer | W:(0-9A-Za-z)}]... -> [123, 'A100'] + + Match integer at loc 0(1,1) + 123 A100 + ^ + Matched integer -> [123] + Match integer at loc 4(1,5) + 123 A100 + ^ + Match integer failed, ParseException raised: Expected integer, found 'A100' (at char 4), (line:1, col:5) + Match W:(0-9A-Za-z) at loc 4(1,5) + 123 A100 + ^ + Matched W:(0-9A-Za-z) -> ['A100'] + Match integer at loc 8(1,9) + 123 A100 + ^ + Match integer failed, ParseException raised: Expected integer, found end of text (at char 8), (line:1, col:9) + Match W:(0-9A-Za-z) at loc 8(1,9) + 123 A100 + ^ + Match W:(0-9A-Za-z) failed, ParseException raised: Expected W:(0-9A-Za-z), found end of text (at char 8), (line:1, col:9) + """ + ) + output = test_stdout.getvalue() + print(output) + self.assertEqual( + expected_debug_output, + output, + "invalid debug output when using parse action", + ) + + def testSetDebugWithControlCharInLine(self): + # issue #496 - a '\r' embedded in the parsed line used to be printed + # verbatim, sending the terminal cursor back to column 0 and corrupting + # the debug output; control characters should be shown escaped instead + with ppt.reset_pyparsing_context(): + test_stdout = StringIO() + + with resetting(sys, "stdout", "stderr"): + sys.stdout = test_stdout + sys.stderr = test_stdout + + word = pp.Word(pp.alphas).set_name("word").set_debug() + word[...].parse_string("abc\rdef", parse_all=True) + + expected_debug_output = dedent( + """\ + Match word at loc 0(1,1) + abc\\rdef + ^ + Matched word -> ['abc'] + Match word at loc 4(1,5) + abc\\rdef + ^ + Matched word -> ['def'] + Match word at loc 7(1,8) + abc\\rdef + ^ + Match word failed, ParseException raised: Expected word, found end of text (at char 7), (line:1, col:8) + """ + ) + self.assertEqual( + expected_debug_output, + test_stdout.getvalue(), + "control characters in debug output were not escaped", + ) + + def testEnableDebugWithCachedExpressionsMarkedWithAsterisk(self): + a = pp.Literal("a").set_name("A").set_debug() + b = pp.Literal("b").set_name("B").set_debug() + z = pp.Literal("z").set_name("Z").set_debug() + leading_a = a + pp.FollowedBy(z | a | b) + leading_a.set_name("leading_a").set_debug() + + grammar = (z | leading_a | b)[...] + "a" + + # parse test string and capture debug output + test_stdout = StringIO() + with resetting(sys, "stdout", "stderr"): + sys.stdout = test_stdout + sys.stderr = test_stdout + grammar.parse_string("aba", parse_all=True) + + expected_debug_output = dedent( + """\ + Match Z at loc 0(1,1) + aba + ^ + Match Z failed, ParseException raised: Expected Z, found 'aba' (at char 0), (line:1, col:1) + Match leading_a at loc 0(1,1) + aba + ^ + Match A at loc 0(1,1) + aba + ^ + Matched A -> ['a'] + Match Z at loc 1(1,2) + aba + ^ + Match Z failed, ParseException raised: Expected Z, found 'ba' (at char 1), (line:1, col:2) + Match A at loc 1(1,2) + aba + ^ + Match A failed, ParseException raised: Expected A, found 'ba' (at char 1), (line:1, col:2) + Match B at loc 1(1,2) + aba + ^ + Matched B -> ['b'] + Matched leading_a -> ['a'] + *Match Z at loc 1(1,2) + aba + ^ + *Match Z failed, ParseException raised: Expected Z, found 'ba' (at char 1), (line:1, col:2) + Match leading_a at loc 1(1,2) + aba + ^ + Match A at loc 1(1,2) + aba + ^ + Match A failed, ParseException raised: Expected A, found 'ba' (at char 1), (line:1, col:2) + Match leading_a failed, ParseException raised: Expected A, found 'ba' (at char 1), (line:1, col:2) + *Match B at loc 1(1,2) + aba + ^ + *Matched B -> ['b'] + Match Z at loc 2(1,3) + aba + ^ + Match Z failed, ParseException raised: Expected Z, found 'a' (at char 2), (line:1, col:3) + Match leading_a at loc 2(1,3) + aba + ^ + Match A at loc 2(1,3) + aba + ^ + Matched A -> ['a'] + Match Z at loc 3(1,4) + aba + ^ + Match Z failed, ParseException raised: Expected Z, found end of text (at char 3), (line:1, col:4) + Match A at loc 3(1,4) + aba + ^ + Match A failed, ParseException raised: Expected A, found end of text (at char 3), (line:1, col:4) + Match B at loc 3(1,4) + aba + ^ + Match B failed, ParseException raised: Expected B, found end of text (at char 3), (line:1, col:4) + Match leading_a failed, ParseException raised: Expected {Z | A | B}, found end of text (at char 3), (line:1, col:4) + Match B at loc 2(1,3) + aba + ^ + Match B failed, ParseException raised: Expected B, found 'a' (at char 2), (line:1, col:3) + """ + ) + if pp.ParserElement._packratEnabled: + packrat_status = "enabled" + else: + # remove '*' cache markers from expected output + expected_debug_output = expected_debug_output.replace("*", "") + packrat_status = "disabled" + print("Packrat status:", packrat_status) + + output = test_stdout.getvalue() + print(output) + self.assertEqual( + expected_debug_output, + output, + ( + f"invalid debug output showing cached results marked with '*'," + f" and packrat parsing {packrat_status}" + ), + ) + + def testSetDebugRecursively(self): + expr = pp.Word(pp.alphas) + contained = expr + pp.Empty().set_name("innermost") + depth = 4 + for _ in range(depth): + contained = pp.Group(contained + pp.Empty()) + contained.set_debug(recurse=True) + self.assertTrue(expr.debug) + # contained.parse_string("ABC") + test_stdout = StringIO() + with resetting(sys, "stdout", "stderr"): + sys.stdout = test_stdout + sys.stderr = test_stdout + contained.parse_string("aba", parse_all=True) + + output = test_stdout.getvalue() + print(output) + self.assertEqual(depth, output.count("Matched Empty -> []")) + self.assertEqual(1, output.count("Matched innermost -> []")) + + def testSetDebugRecursivelyWithForward(self): + expr = pp.Word(pp.alphas).set_name("innermost") + contained = pp.infix_notation( + expr, + [ + ("NOT", 1, pp.OpAssoc.RIGHT), + ("AND", 2, pp.OpAssoc.LEFT), + ("OR", 2, pp.OpAssoc.LEFT), + ], + ) + + contained.set_debug(recurse=True) + self.assertTrue(expr.debug) + + # contained.parse_string("ABC") + test_stdout = StringIO() + with resetting(sys, "stdout", "stderr"): + sys.stdout = test_stdout + sys.stderr = test_stdout + contained.parse_string("aba", parse_all=True) + + output = test_stdout.getvalue() + print(output) + # count of matches varies with packrat state, can't match exact count, but at least test if contains + # self.assertEqual(4, output.count("Matched innermost -> ['aba']")) + self.assertTrue("Matched innermost -> ['aba']" in output) + + def testUndesirableButCommonPractices(self): + # While these are valid constructs, and they are not encouraged + # there is apparently a lot of code out there using these + # coding styles. + # + # Even though they are not encouraged, we shouldn't break them. + + # Create an And using a list of expressions instead of using '+' operator + expr = pp.And([pp.Word("abc"), pp.Word("123")]) + success, _ = expr.run_tests( + """ + aaa 333 + b 1 + ababab 32123 + """ + ) + self.assertTrue(success) + + success, _ = expr.run_tests( + """\ + aad 111 + """, + failure_tests=True, + ) + self.assertTrue(success) + + # Passing a single expression to a ParseExpression, when it really wants a sequence + expr = pp.Or(pp.Or(ppc.integer)) + success, _ = expr.run_tests( + """\ + 123 + 456 + """ + ) + self.assertTrue(success) + + success, _ = expr.run_tests( + """\ + abc + """, + failure_tests=True, + ) + self.assertTrue(success) + + def testEnableWarnDiags(self): + import pprint + + def filtered_vars(var_dict): + dunders = [nm for nm in var_dict if nm.startswith("__")] + return { + k: v + for k, v in var_dict.items() + if isinstance(v, bool) and k not in dunders + } + + pprint.pprint(filtered_vars(vars(pp.__diag__)), width=30) + + warn_names = pp.__diag__._warning_names + other_names = pp.__diag__._debug_names + + # make sure they are off by default + for diag_name in warn_names: + self.assertFalse( + getattr(pp.__diag__, diag_name), + f"__diag__.{diag_name} not set to True", + ) + + with ppt.reset_pyparsing_context(): + # enable all warn_* diag_names + pp.enable_all_warnings() + pprint.pprint(filtered_vars(vars(pp.__diag__)), width=30) + + # make sure they are on after being enabled + for diag_name in warn_names: + self.assertTrue( + getattr(pp.__diag__, diag_name), + f"__diag__.{diag_name} not set to True", + ) + + # non-warn diag_names must be enabled individually + for diag_name in other_names: + self.assertFalse( + getattr(pp.__diag__, diag_name), + f"__diag__.{diag_name} not set to True", + ) + + # make sure they are off after AutoReset + for diag_name in warn_names: + self.assertFalse( + getattr(pp.__diag__, diag_name), + f"__diag__.{diag_name} not set to True", + ) + + def testWordInternalReRangeWithConsecutiveChars(self): + self.assertParseAndCheckList( + pp.Word("ABCDEMNXYZ"), + "ABCDEMNXYZABCDEMNXYZABCDEMNXYZ", + ["ABCDEMNXYZABCDEMNXYZABCDEMNXYZ"], + ) + + def testWordInternalReRangesKnownSet(self): + tests = [ + ("ABCDEMNXYZ", "[A-EMNX-Z]+"), + (pp.printables, "[!-~]+"), + (pp.alphas, "[A-Za-z]+"), + (pp.alphanums, "[0-9A-Za-z]+"), + (pp.pyparsing_unicode.Latin1.printables, "[!-~¡-ÿ]+"), + (pp.pyparsing_unicode.Latin1.alphas, "[A-Za-zªµºÀ-ÖØ-öø-ÿ]+"), + (pp.pyparsing_unicode.Latin1.alphanums, "[0-9A-Za-zª²³µ¹ºÀ-ÖØ-öø-ÿ]+"), + (pp.alphas8bit, "[À-ÖØ-öø-ÿ]+"), + ] + failed = [] + for word_string, expected_re in tests: + try: + msg = f"failed to generate correct internal re for {word_string!r}" + resultant_re = pp.Word(word_string).reString + self.assertEqual( + expected_re, + resultant_re, + msg + f"; expected {expected_re!r} got {resultant_re!r}", + ) + except AssertionError: + failed.append(msg) + + if failed: + print("Errors:\n{}".format("\n".join(failed))) + self.fail("failed to generate correct internal re's") + + def testWordInternalReRanges(self): + import random + + esc_chars = r"\^-][" + esc_chars2 = r"*+.?" + + def esc_re_set_char(c): + return "\\" + c if c in esc_chars else c + + def esc_re_set2_char(c): + return "\\" + c if c in esc_chars + esc_chars2 else c + + for esc_char in esc_chars + esc_chars2: + # test escape char as first character in range + next_char = chr(ord(esc_char) + 1) + prev_char = chr(ord(esc_char) - 1) + esc_word = pp.Word(esc_char + next_char) + expected = rf"[{esc_re_set_char(esc_char)}{esc_re_set_char(next_char)}]+" + print( + f"Testing escape char: {esc_char} -> {esc_word} re: '{esc_word.reString}')" + ) + self.assertEqual( + expected, esc_word.reString, "failed to generate correct internal re" + ) + test_string = "".join( + random.choice([esc_char, next_char]) for __ in range(16) + ) + print( + f"Match '{test_string}' -> {test_string == esc_word.parse_string(test_string, parse_all=True)[0]}" + ) + self.assertEqual( + test_string, + esc_word.parse_string(test_string, parse_all=True)[0], + "Word using escaped range char failed to parse", + ) + + # test escape char as last character in range + esc_word = pp.Word(prev_char + esc_char) + expected = rf"[{esc_re_set_char(prev_char)}{esc_re_set_char(esc_char)}]+" + print( + f"Testing escape char: {esc_char} -> {esc_word} re: '{esc_word.reString}')" + ) + self.assertEqual( + expected, esc_word.reString, "failed to generate correct internal re" + ) + test_string = "".join( + random.choice([esc_char, prev_char]) for __ in range(16) + ) + print( + f"Match '{test_string}' -> {test_string == esc_word.parse_string(test_string, parse_all=True)[0]}" + ) + self.assertEqual( + test_string, + esc_word.parse_string(test_string, parse_all=True)[0], + "Word using escaped range char failed to parse", + ) + + # test escape char as first character in range + next_char = chr(ord(esc_char) + 1) + prev_char = chr(ord(esc_char) - 1) + esc_word = pp.Word(esc_char + next_char) + expected = rf"[{esc_re_set_char(esc_char)}{esc_re_set_char(next_char)}]+" + print( + f"Testing escape char: {esc_char} -> {esc_word} re: '{esc_word.reString}')" + ) + self.assertEqual( + expected, esc_word.reString, "failed to generate correct internal re" + ) + test_string = "".join( + random.choice([esc_char, next_char]) for __ in range(16) + ) + print( + f"Match '{test_string}' -> {test_string == esc_word.parse_string(test_string, parse_all=True)[0]}" + ) + self.assertEqual( + test_string, + esc_word.parse_string(test_string, parse_all=True)[0], + "Word using escaped range char failed to parse", + ) + + # test escape char as only character in range + esc_word = pp.Word(esc_char, pp.alphas.upper()) + expected = rf"{esc_re_set2_char(esc_char)}[A-Z]*" + print( + f"Testing escape char: {esc_char} -> {esc_word} re: '{esc_word.reString}')" + ) + self.assertEqual( + expected, esc_word.reString, "failed to generate correct internal re" + ) + test_string = esc_char + "".join( + random.choice(pp.alphas.upper()) for __ in range(16) + ) + print( + f"Match '{test_string}' -> {test_string == esc_word.parse_string(test_string, parse_all=True)[0]}" + ) + self.assertEqual( + test_string, + esc_word.parse_string(test_string, parse_all=True)[0], + "Word using escaped range char failed to parse", + ) + + # test escape char as only character + esc_word = pp.Word(esc_char, pp.alphas.upper()) + expected = rf"{re.escape(esc_char)}[A-Z]*" + print( + f"Testing escape char: {esc_char} -> {esc_word} re: '{esc_word.reString}')" + ) + self.assertEqual( + expected, esc_word.reString, "failed to generate correct internal re" + ) + test_string = esc_char + "".join( + random.choice(pp.alphas.upper()) for __ in range(16) + ) + print( + f"Match '{test_string}' -> {test_string == esc_word.parse_string(test_string, parse_all=True)[0]}" + ) + self.assertEqual( + test_string, + esc_word.parse_string(test_string, parse_all=True)[0], + "Word using escaped range char failed to parse", + ) + print() + + def testWordWithIdentChars(self): + ppu = pp.pyparsing_unicode + + latin_identifier = pp.Word(pp.identchars, pp.identbodychars)("latin*") + japanese_identifier = ppu.Japanese.identifier("japanese*") + cjk_identifier = ppu.CJK.identifier("cjk*") + greek_identifier = ppu.Greek.identifier("greek*") + cyrillic_identifier = ppu.Cyrillic.identifier("cyrillic*") + thai_identifier = ppu.Thai.identifier("thai*") + idents = ( + latin_identifier + | japanese_identifier + | cjk_identifier # must follow japanese_identifier, since CJK is superset + | thai_identifier + | greek_identifier + | cyrillic_identifier + ) + + result = idents[...].parse_string( + "abc_100 кириллицаx_10 日本語f_300 ไทยg_600 def_200 漢字y_300 한국어_中文c_400 Ελληνικάb_500", + parse_all=True, + ) + self.assertParseResultsEquals( + result, + [ + "abc_100", + "кириллицаx_10", + "日本語f_300", + "ไทยg_600", + "def_200", + "漢字y_300", + "한국어_中文c_400", + "Ελληνικάb_500", + ], + { + "cjk": ["한국어_中文c_400"], + "cyrillic": ["кириллицаx_10"], + "greek": ["Ελληνικάb_500"], + "japanese": ["日本語f_300", "漢字y_300"], + "latin": ["abc_100", "def_200"], + "thai": ["ไทยg_600"], + }, + ) + + def testChainedTernaryOperator(self): + # fmt: off + TERNARY_INFIX = pp.infix_notation( + ppc.integer, + [ + (("?", ":"), 3, pp.OpAssoc.LEFT), + ] + ) + self.assertParseAndCheckList( + TERNARY_INFIX, "1?1:0?1:0", [[1, "?", 1, ":", 0, "?", 1, ":", 0]] + ) + + TERNARY_INFIX = pp.infix_notation( + ppc.integer, + [ + (("?", ":"), 3, pp.OpAssoc.RIGHT), + ] + ) + self.assertParseAndCheckList( + TERNARY_INFIX, "1?1:0?1:0", [[1, "?", 1, ":", [0, "?", 1, ":", 0]]] + ) + # fmt: on + + def testOneOfWithDuplicateSymbols(self): + # test making one_of with duplicate symbols + print("verify one_of handles duplicate symbols") + try: + test1 = pp.one_of("a b c d a") + except RuntimeError: + self.fail( + "still have infinite loop in one_of with duplicate symbols (string input)" + ) + + print("verify one_of handles duplicate symbols") + try: + test1 = pp.one_of("a a a b c d a") + except RuntimeError: + self.fail( + "still have infinite loop in one_of with duplicate symbols (string input)" + ) + + assert test1.pattern == "[abcd]" + + print("verify one_of handles generator input") + try: + test1 = pp.one_of(c for c in "a b c d a d d d" if not c.isspace()) + except RuntimeError: + self.fail( + "still have infinite loop in one_of with duplicate symbols (generator input)" + ) + + assert test1.pattern == "[abcd]" + + print("verify one_of handles list input") + try: + test1 = pp.one_of("a b c d a".split()) + except RuntimeError: + self.fail( + "still have infinite loop in one_of with duplicate symbols (list input)" + ) + + assert test1.pattern == "[abcd]" + + print("verify one_of handles set input") + try: + test1 = pp.one_of(set("a b c d a".split())) + except RuntimeError: + self.fail( + "still have infinite loop in one_of with duplicate symbols (set input)" + ) + + # set will generate scrambled letters, get pattern but resort to test + pattern_letters = test1.pattern[1:-1] + assert sorted(pattern_letters) == sorted("abcd") + + def testOneOfWithEmptyList(self): + """test one_of helper function with an empty list as input""" + + tst = [] + result = pp.one_of(tst) + + expected = True + found = isinstance(result, pp.NoMatch) + self.assertEqual(expected, found) + + def testOneOfWithUnexpectedInput(self): + """test one_of with an input that isn't a string or iterable""" + + with self.assertRaises( + TypeError, msg="failed to warn use of integer for one_of" + ): + expr = pp.one_of(6) + + def testMatchFirstIteratesOverAllChoices(self): + # test MatchFirst bugfix + print("verify MatchFirst iterates properly") + results = pp.quoted_string.parse_string( + "'this is a single quoted string'", parse_all=True + ) + self.assertTrue( + len(results) > 0, "MatchFirst error - not iterating over all choices" + ) + + def testStreamlineOfExpressionsAfterSetName(self): + bool_constant = pp.Literal("True") | "true" | "False" | "false" + self.assertEqual( + "{'True' | 'true' | 'False' | 'false'}", str(bool_constant.streamline()) + ) + bool_constant.set_name("bool") + self.assertEqual("bool", str(bool_constant.streamline())) + + def testStreamlineOfSubexpressions(self): + # verify streamline of subexpressions + print("verify proper streamline logic") + compound = pp.Literal("A") + "B" + "C" + "D" + self.assertEqual(2, len(compound.exprs), "bad test setup") + print(compound) + compound.streamline() + print(compound) + self.assertEqual(4, len(compound.exprs), "streamline not working") + + def testOptionalWithResultsNameAndNoMatch(self): + # test for Optional with results name and no match + print("verify Optional's do not cause match failure if have results name") + testGrammar = pp.Literal("A") + pp.Optional("B")("gotB") + pp.Literal("C") + try: + testGrammar.parse_string("ABC", parse_all=True) + testGrammar.parse_string("AC", parse_all=True) + except pp.ParseException as pe: + print(pe.pstr, "->", pe) + self.fail(f"error in Optional matching of string {pe.pstr}") + + def testReturnOfFurthestException(self): + # test return of furthest exception + testGrammar = ( + pp.Literal("A") | (pp.Literal("B") + pp.Literal("C")) | pp.Literal("E") + ) + try: + testGrammar.parse_string("BC", parse_all=True) + testGrammar.parse_string("BD", parse_all=True) + except pp.ParseException as pe: + print(pe.pstr, "->", pe) + self.assertEqual("BD", pe.pstr, "wrong test string failed to parse") + self.assertEqual( + 1, pe.loc, "error in Optional matching, pe.loc=" + str(pe.loc) + ) + self.assertTrue( + "found 'D'" in str(pe), "wrong alternative raised exception" + ) + + def testValidateCorrectlyDetectsInvalidLeftRecursion(self): + # test validate + print("verify behavior of validate()") + if IRON_PYTHON_ENV: + print("disable this test under IronPython") + return + + def testValidation(grmr, gnam, isValid): + try: + grmr.streamline() + with self.assertWarns( + PyparsingDeprecationWarning, + msg="failed to warn validate() is deprecated", + ): + grmr.validate() + self.assertTrue(isValid, "validate() accepted invalid grammar " + gnam) + except pp.RecursiveGrammarException as rge: + print(grmr) + print(rge) + self.assertFalse(isValid, "validate() rejected valid grammar " + gnam) + + fwd = pp.Forward() + g1 = pp.OneOrMore((pp.Literal("A") + "B" + "C") | fwd) + g2 = ("C" + g1)[...] + fwd <<= pp.Group(g2) + testValidation(fwd, "fwd", isValid=True) + + fwd2 = pp.Forward() + fwd2 <<= pp.Group("A" | fwd2) + testValidation(fwd2, "fwd2", isValid=False) + + fwd3 = pp.Forward() + fwd3 <<= pp.Optional("A") + fwd3 + testValidation(fwd3, "fwd3", isValid=False) + + def testGetNameBehavior(self): + # test get_name + print("verify behavior of get_name()") + aaa = pp.Group(pp.Word("a")("A")) + bbb = pp.Group(pp.Word("b")("B")) + ccc = pp.Group(":" + pp.Word("c")("C")) + g1 = "XXX" + (aaa | bbb | ccc)[...] + teststring = "XXX b bb a bbb bbbb aa bbbbb :c bbbbbb aaa" + names = [] + print(g1.parse_string(teststring, parse_all=True).dump()) + for t in g1.parse_string(teststring, parse_all=True): + print(t, repr(t)) + try: + names.append(t[0].get_name()) + except Exception: + try: + names.append(t.get_name()) + except Exception: + names.append(None) + print(teststring) + print(names) + self.assertEqual( + [None, "B", "B", "A", "B", "B", "A", "B", None, "B", "A"], + names, + "failure in getting names for tokens", + ) + + IF, AND, BUT = map(pp.Keyword, "if and but".split()) + ident = ~(IF | AND | BUT) + pp.Word(pp.alphas)("non-key") + scanner = pp.OneOrMore(IF | AND | BUT | ident) + + def getNameTester(s, l, t): + print(t, t.get_name()) + + ident.add_parse_action(getNameTester) + scanner.parse_string("lsjd sldkjf IF Saslkj AND lsdjf", parse_all=True) + + # test ParseResults.get() method + print("verify behavior of ParseResults.get()") + # use sum() to merge separate groups into single ParseResults + res = sum(g1.parse_string(teststring, parse_all=True)[1:]) + print(res.dump()) + print(res.get("A", "A not found")) + print(res.get("D", "!D")) + self.assertEqual( + "aaa", res.get("A", "A not found"), "get on existing key failed" + ) + self.assertEqual("!D", res.get("D", "!D"), "get on missing key failed") + + def testOptionalBeyondEndOfString(self): + print("verify handling of Optional's beyond the end of string") + testGrammar = "A" + pp.Optional("B") + pp.Optional("C") + pp.Optional("D") + testGrammar.parse_string("A", parse_all=True) + testGrammar.parse_string("AB", parse_all=True) + + def testCreateLiteralWithEmptyString(self): + # test creating Literal with empty string + print('verify that Literal("") is optimized to Empty()') + e = pp.Literal("") + self.assertIsInstance(e, pp.Empty) + + def testLineMethodSpecialCaseAtStart(self): + # test line() behavior when starting at 0 and the opening line is an \n + print("verify correct line() behavior when first line is empty string") + self.assertEqual( + "", + pp.line(0, "\nabc\ndef\n"), + "Error in line() with empty first line in text", + ) + txt = "\nabc\ndef\n" + results = [pp.line(i, txt) for i in range(len(txt))] + self.assertEqual( + ["", "abc", "abc", "abc", "abc", "def", "def", "def", "def"], + results, + "Error in line() with empty first line in text", + ) + txt = "abc\ndef\n" + results = [pp.line(i, txt) for i in range(len(txt))] + self.assertEqual( + ["abc", "abc", "abc", "abc", "def", "def", "def", "def"], + results, + "Error in line() with non-empty first line in text", + ) + + def testRepeatedTokensWhenPackratting(self): + # test bugfix with repeated tokens when packrat parsing enabled + print("verify behavior with repeated tokens when packrat parsing is enabled") + a = pp.Literal("a") + b = pp.Literal("b") + c = pp.Literal("c") + + abb = a + b + b + abc = a + b + c + aba = a + b + a + grammar = abb | abc | aba + + self.assertEqual( + "aba", + "".join(grammar.parse_string("aba", parse_all=True)), + "Packrat ABA failure!", + ) + + def testSetResultsNameWithOneOrMoreAndZeroOrMore(self): + print("verify behavior of set_results_name with OneOrMore and ZeroOrMore") + stmt = pp.Keyword("test") + print(stmt[...]("tests").parse_string("test test", parse_all=True).tests) + print(stmt[1, ...]("tests").parse_string("test test", parse_all=True).tests) + print( + pp.Optional(stmt[1, ...]("tests")) + .parse_string("test test", parse_all=True) + .tests + ) + print( + pp.Optional(stmt[1, ...])("tests") + .parse_string("test test", parse_all=True) + .tests + ) + print( + pp.Optional(pp.DelimitedList(stmt))("tests") + .parse_string("test,test", parse_all=True) + .tests + ) + self.assertEqual( + 2, + len(stmt[...]("tests").parse_string("test test", parse_all=True).tests), + "ZeroOrMore failure with set_results_name", + ) + self.assertEqual( + 2, + len(stmt[1, ...]("tests").parse_string("test test", parse_all=True).tests), + "OneOrMore failure with set_results_name", + ) + self.assertEqual( + 2, + len( + pp.Optional(stmt[1, ...]("tests")) + .parse_string("test test", parse_all=True) + .tests + ), + "OneOrMore failure with set_results_name", + ) + self.assertEqual( + 2, + len( + pp.Optional(pp.DelimitedList(stmt))("tests") + .parse_string("test,test", parse_all=True) + .tests + ), + "DelimitedList failure with set_results_name", + ) + self.assertEqual( + 2, + len((stmt * 2)("tests").parse_string("test test", parse_all=True).tests), + "multiplied(1) failure with set_results_name", + ) + self.assertEqual( + 2, + len(stmt[..., 2]("tests").parse_string("test test", parse_all=True).tests), + "multiplied(2) failure with set_results_name", + ) + self.assertEqual( + 2, + len(stmt[1, ...]("tests").parse_string("test test", parse_all=True).tests), + "multiplied(3) failure with set_results_name", + ) + self.assertEqual( + 2, + len(stmt[2, ...]("tests").parse_string("test test", parse_all=True).tests), + "multiplied(3) failure with set_results_name", + ) + + def testParseResultsReprWithResultsNames(self): + word = pp.Word(pp.printables)("word") + res = word[...].parse_string("test blub", parse_all=True) + + print(repr(res)) + print(res["word"]) + print(res.as_dict()) + + self.assertEqual( + "ParseResults(['test', 'blub'], {'word': 'blub'})", + repr(res), + "incorrect repr for ParseResults with list_all_matches=False", + ) + + word = pp.Word(pp.printables)("word*") + res = word[...].parse_string("test blub", parse_all=True) + + print(repr(res)) + print(res["word"]) + print(res.as_dict()) + + self.assertEqual( + "ParseResults(['test', 'blub'], {'word': ['test', 'blub']})", + repr(res), + "incorrect repr for ParseResults with list_all_matches=True", + ) + + def testWarnUsingLshiftForward(self): + print( + "verify that using '<<' operator with a Forward raises a warning if there is a dangling '|' operator" + ) + + fwd = pp.Forward() + print("unsafe << and |, but diag not enabled, should not warn") + fwd << pp.Word("a") | pp.Word("b") + + pp.enable_diag(pp.Diagnostics.warn_on_match_first_with_lshift_operator) + with self.assertWarns( + UserWarning, msg="failed to warn of using << and | operators" + ): + fwd = pp.Forward() + print("unsafe << and |, should warn") + fwd << pp.Word("a") | pp.Word("b") + + with self.assertWarns( + UserWarning, + msg="failed to warn of using << and | operators (within lambda)", + ): + fwd = pp.Forward() + print("unsafe << and |, should warn") + fwd_fn = lambda expr1, expr2: fwd << expr1 | expr2 + fwd_fn(pp.Word("a"), pp.Word("b")) + + fwd = pp.Forward() + print("safe <<= and |, should not warn") + fwd <<= pp.Word("a") | pp.Word("b") + c = fwd | pp.Word("c") + + print("safe << and (|), should not warn") + with self.assertDoesNotWarn( + "warning raised on safe use of << with Forward and MatchFirst" + ): + fwd = pp.Forward() + fwd << (pp.Word("a") | pp.Word("b")) + c = fwd | pp.Word("c") + + def testParseExpressionsWithRegex(self): + from itertools import product + + match_empty_regex = pp.Regex(r"[a-z]*") + match_nonempty_regex = pp.Regex(r"[a-z]+") + + parser_classes = pp.ParseExpression.__subclasses__() + test_string = "abc def" + expected = ["abc"] + for expr, cls in product( + (match_nonempty_regex, match_empty_regex), parser_classes + ): + print(expr, cls) + parser = cls([expr]) + parsed_result = parser.parse_string(test_string, parse_all=False) + print(parsed_result.dump()) + self.assertParseResultsEquals(parsed_result, expected) + + for expr, cls in product( + (match_nonempty_regex, match_empty_regex), (pp.MatchFirst, pp.Or) + ): + parser = cls([expr, expr]) + print(parser) + parsed_result = parser.parse_string(test_string, parse_all=False) + print(parsed_result.dump()) + self.assertParseResultsEquals(parsed_result, expected) + + def testAssertParseAndCheckDict(self): + """test assertParseAndCheckDict in test framework""" + + expr = pp.Word(pp.alphas)("item") + pp.Word(pp.nums)("qty") + self.assertParseAndCheckDict( + expr, "balloon 25", {"item": "balloon", "qty": "25"} + ) + + exprWithInt = pp.Word(pp.alphas)("item") + ppc.integer("qty") + self.assertParseAndCheckDict( + exprWithInt, "rucksack 49", {"item": "rucksack", "qty": 49} + ) + + def testOnlyOnce(self): + """test class OnlyOnce and its reset method""" + + # use a parse action to compute the sum of the parsed integers, + # and add it to the end + def append_sum(tokens): + tokens.append(sum(map(int, tokens))) + + pa = pp.OnlyOnce(append_sum) + expr = pp.OneOrMore(pp.Word(pp.nums)).add_parse_action(pa) + + result = expr.parse_string("0 123 321", parse_all=True) + print(result.dump()) + expected = ["0", "123", "321", 444] + self.assertParseResultsEquals( + result, expected, msg="issue with OnlyOnce first call" + ) + + with self.assertRaisesParseException( + msg="failed to raise exception calling OnlyOnce more than once" + ): + result2 = expr.parse_string("1 2 3 4 5", parse_all=True) + + pa.reset() + result = expr.parse_string("100 200 300") + print(result.dump()) + expected = ["100", "200", "300", 600] + self.assertParseResultsEquals( + result, expected, msg="issue with OnlyOnce after reset" + ) + + def testGoToColumn(self): + """tests for GoToColumn class""" + + dateExpr = pp.Regex(r"\d\d(\.\d\d){2}")("date") + numExpr = ppc.number("num") + + sample = """\ + date Not Important value NotImportant2 + 11.11.13 | useless . useless,21 useless 2 | 14.21 | asmdakldm + 21.12.12 | fmpaosmfpoamsp 4 | 41 | ajfa9si90""".splitlines() + + # Column number finds match + patt = dateExpr + pp.GoToColumn(70).ignore("|") + numExpr + pp.rest_of_line + + infile = iter(sample) + next(infile) + + expecteds = [["11.11.13", 14.21], ["21.12.12", 41]] + for line, expected in zip(infile, expecteds): + result = patt.parse_string(line, parse_all=True) + print(result) + + self.assertEqual( + expected, [result.date, result.num], msg="issue with GoToColumn" + ) + + # Column number does NOT match + patt = dateExpr("date") + pp.GoToColumn(30) + numExpr + pp.rest_of_line + + infile = iter(sample) + next(infile) + + for line in infile: + with self.assertRaisesParseException( + msg="issue with GoToColumn not finding match" + ): + result = patt.parse_string(line, parse_all=True) + + def testExceptionExplainVariations(self): + class Modifier: + def modify_upper(self, tokens): + tokens[:] = map(str.upper, tokens) + + modder = Modifier() + + # force an exception in the attached parse action + # integer has a parse action to convert to an int; + # this parse action should fail with a TypeError, since + # str.upper expects a str argument, not an int + grammar = ppc.integer().add_parse_action(modder.modify_upper) + + self_testcase_name = "tests.test_unit." + type(self).__name__ + + try: + grammar.parse_string("1000", parse_all=True) + except Exception as e: + # extract the exception explanation + explain_str = ParseException.explain_exception(e) + print(explain_str) + explain_str_lines = explain_str.splitlines() + + expected = [ + self_testcase_name, + "pyparsing.core.Word - integer", + "tests.test_unit.Modifier", + "pyparsing.results.ParseResults", + ] + + # verify the list of names shown in the explain "stack" + self.assertEqual( + expected, explain_str_lines[-len(expected) :], msg="invalid explain str" + ) + + # check type of raised exception matches explain output + # (actual exception text varies by Python version, and even + # by how the exception is raised, so we can only check the + # type name) + exception_line = explain_str_lines[-(len(expected) + 1)] + self.assertTrue( + exception_line.startswith("TypeError:"), + msg=f"unexpected exception line ({exception_line!r})", + ) + + def testExceptionMessageCustomization(self): + with resetting(pp.ParseBaseException, "formatted_message"): + + def custom_exception_message(exc) -> str: + found_phrase = f", found {exc.found}" if exc.found else "" + return f"{exc.lineno}:{exc.column} {exc.msg}{found_phrase}" + + pp.ParseBaseException.formatted_message = custom_exception_message + + try: + pp.Word(pp.nums).parse_string("ABC") + except ParseException as pe: + pe_msg = str(pe) + else: + pe_msg = "" + + self.assertEqual("1:1 Expected W:(0-9), found 'ABC'", pe_msg) + + def testForwardReferenceException(self): + token = pp.Forward() + num = pp.Word(pp.nums) + num.set_name("num") + text = pp.Word(pp.alphas) + text.set_name("text") + fail = pp.Regex(r"\\[A-Za-z]*")("name") + + def parse_fail(s, loc, toks): + raise pp.ParseFatalException(s, loc, f"Unknown symbol: {toks['name']}") + + fail.set_parse_action(parse_fail) + token <<= num | text | fail + + # If no name is given, do not intercept error messages + with self.assertRaises(pp.ParseFatalException, msg="Unknown symbol: \\fail"): + token.parse_string("\\fail") + + # If name is given, do intercept error messages + token.set_name("token") + with self.assertRaises(pp.ParseFatalException, msg="Expected token, found.*"): + token.parse_string("\\fail") + + def testForwardExceptionText(self): + wd = pp.Word(pp.alphas) + + ff = pp.Forward().set_name("fffff!") + ff <<= wd + pp.Opt(ff) + + with self.assertRaises(pp.ParseFatalException, msg="no numbers!"): + try: + ff.parse_string("123") + except pp.ParseException as pe: + raise pp.ParseSyntaxException("no numbers! just alphas!") from pe + + with self.assertRaises(pp.ParseException, msg="Expected W:(A-Za-z)"): + ff2 = pp.Forward() + ff2 <<= wd + ff2.parse_string("123") + + def testForwardExceptionText2(self): + """ + Test various expressions for error messages, under conditions in wrapped ParserElements + """ + v = "(omit closing paren" + w = "('omit closing quote)" + + for s, expr, expected in ( + (v, pp.nested_expr(), "Expected ')'"), + (v, pp.Combine(pp.nested_expr(), adjacent=False), "Expected ')'"), + ( + v, + pp.QuotedString("(", end_quote_char=")"), + "Expected quoted string, starting with ( ending with ), found '('", + ), + (w, pp.nested_expr(content=pp.sgl_quoted_string), "Expected ')'"), + ("", pp.nested_expr(), ""), + ("", pp.Word("A"), ""), + ): + print(repr(s)) + print(expr) + + with self.subTest("parse expr", expr=expr, s=s, expected=expected): + with self.assertRaisesParseException(expected_msg=expected) as ctx: + expr.parse_string(s, parse_all=True) + print(ctx.exception) + + with self.subTest("parse expr[1, ...]", expr=expr, s=s, expected=expected): + with self.assertRaisesParseException(expected_msg=expected) as ctx: + expr[1, ...].parse_string(s, parse_all=True) + print(ctx.exception) + + with self.subTest( + "parse DelimitedList(expr)", expr=expr, s=s, expected=expected + ): + with self.assertRaisesParseException(expected_msg=expected) as ctx: + pp.DelimitedList(expr).parse_string(s, parse_all=True) + print(ctx.exception) + + print() + + def testMiscellaneousExceptionBits(self): + pp.ParserElement.verbose_stacktrace = True + + self_testcase_name = "tests.test_unit." + type(self).__name__ + + # force a parsing exception - match an integer against "ABC" + try: + pp.Word(pp.nums).parse_string("ABC", parse_all=True) + except pp.ParseException as pe: + expected_str = "Expected W:(0-9), found 'ABC' (at char 0), (line:1, col:1)" + self.assertEqual(expected_str, str(pe), "invalid ParseException str") + self.assertEqual(expected_str, repr(pe), "invalid ParseException repr") + + self.assertEqual( + ">!<ABC", pe.mark_input_line(), "invalid default mark input line" + ) + self.assertEqual( + "ABC", pe.mark_input_line(""), "invalid mark input line with '' marker" + ) + + # test explain using depth=None, 0, 1 + depth_none_explain_str = pe.explain(depth=None) + depth_0_explain_str = pe.explain(depth=0) + depth_1_explain_str = pe.explain(depth=1) + print(depth_none_explain_str) + print() + print(depth_0_explain_str) + print() + print(depth_1_explain_str) + + expr_name = "pyparsing.core.Word - W:(0-9)" + for expected_function in [self_testcase_name, expr_name]: + self.assertTrue( + expected_function in depth_none_explain_str, + f"{expected_function!r} not found in ParseException.explain()", + ) + self.assertFalse( + expected_function in depth_0_explain_str, + f"{expected_function!r} found in ParseException.explain(depth=0)", + ) + + self.assertTrue( + expr_name in depth_1_explain_str, + f"{expected_function!r} not found in ParseException.explain()", + ) + self.assertFalse( + self_testcase_name in depth_1_explain_str, + f"{expected_function!r} not found in ParseException.explain()", + ) + + def testExpressionDefaultStrings(self): + expr = pp.Word(pp.nums) + print(expr) + self.assertEqual("W:(0-9)", repr(expr)) + + expr = pp.Word(pp.nums, exact=3) + print(expr) + self.assertEqual("W:(0-9){3}", repr(expr)) + + expr = pp.Word(pp.nums, min=2) + print(expr) + self.assertEqual("W:(0-9){2,...}", repr(expr)) + + expr = pp.Word(pp.nums, max=3) + print(expr) + self.assertEqual("W:(0-9){1,3}", repr(expr)) + + expr = pp.Word(pp.nums, min=2, max=3) + print(expr) + self.assertEqual("W:(0-9){2,3}", repr(expr)) + + expr = pp.Char(pp.nums) + print(expr) + self.assertEqual("(0-9)", repr(expr)) + + def testEmptyExpressionsAreHandledProperly(self): + try: + from pyparsing.diagram import to_railroad + except ModuleNotFoundError as mnfe: + print( + "Failed 'from pyparsing.diagram import to_railroad'" + f"\n {type(mnfe).__name__}: {mnfe}" + ) + if mnfe.__cause__: + print(f"\n {type(mnfe.__cause__).__name__}: {mnfe.__cause__}") + self.skipTest("Failed 'from pyparsing.diagram import to_railroad'") + + for cls in (pp.And, pp.Or, pp.MatchFirst, pp.Each): + print("testing empty", cls.__name__) + expr = cls([]) + expr.streamline() + to_railroad(expr) + + def testForwardsDoProperStreamlining(self): + wd = pp.Word(pp.alphas) + w3 = wd + wd + wd + # before streamlining, w3 is {{W:(A-Za-z) W:(A-Za-z)} W:(A-Za-z)} + self.assertIsInstance(w3.exprs[0], pp.And) + self.assertEqual(len(w3.exprs), 2) + + ff = pp.Forward() + ff <<= w3 + pp.Opt(ff) + # before streamlining, ff is {{{W:(A-Za-z) W:(A-Za-z)} W:(A-Za-z)} [Forward: None]} + self.assertEqual(len(ff.expr.exprs), 2) + + ff.streamline() + + # after streamlining: + # w3 is {W:(A-Za-z) W:(A-Za-z) W:(A-Za-z)} + # ff is {W:(A-Za-z) W:(A-Za-z) W:(A-Za-z) [Forward: None]} + self.assertEqual(len(ff.expr.exprs), 4) + self.assertEqual(len(w3.exprs), 3) + + test_exception_messages_tests = ( + (pp.Word(pp.alphas), "123", "Expected W:(A-Za-z), found '123'"), + (pp.Word(pp.alphas).set_name("word"), "123", "Expected word, found '123'"), + ( + pp.Group(pp.Word(pp.alphas).set_name("word")), + "123", + "Expected word, found '123'", + ), + ( + pp.OneOrMore(pp.Word(pp.alphas).set_name("word")), + "123", + "Expected word, found '123'", + ), + ( + pp.DelimitedList(pp.Word(pp.alphas).set_name("word")), + "123", + "Expected word, found '123'", + ), + ( + pp.Suppress(pp.Word(pp.alphas).set_name("word")), + "123", + "Expected word, found '123'", + ), + ( + pp.Forward() << pp.Word(pp.alphas).set_name("word"), + "123", + "Expected word, found '123'", + ), + ( + pp.Forward() << pp.Word(pp.alphas), + "123", + "Expected W:(A-Za-z), found '123'", + ), + ( + pp.Group(pp.Word(pp.alphas)), + "123", + "Expected W:(A-Za-z), found '123'", + ), + ( + "prefix" + (pp.Regex("a").set_name("a") | pp.Regex("b").set_name("b")), + "prefixc", + "Expected {a | b}, found 'c'", + ), + ( + "prefix" + (pp.Regex("a").set_name("a") | pp.Regex("b").set_name("b")), + "prefix c", + "Expected {a | b}, found 'c'", + ), + ( + "prefix" + (pp.Regex("a").set_name("a") ^ pp.Regex("b").set_name("b")), + "prefixc", + "Expected {a ^ b}, found 'c'", + ), + ( + "prefix" + (pp.Regex("a").set_name("a") ^ pp.Regex("b").set_name("b")), + "prefix c", + "Expected {a ^ b}, found 'c'", + ), + ) + + def test_exception_messages(self, tests=test_exception_messages_tests): + for expr, input_str, expected_msg in tests: + with self.subTest(expr=expr, input_str=input_str): + with self.assertRaisesParseException(expected_msg=expected_msg): + expr.parse_string(input_str) + + def test_exception_messages_with_exception_subclass(self): + class TooManyRepsException(pp.ParseFatalException): + pass + + @pp.trace_parse_action + def no_more_than_3(t): + if len(t) > 3: + raise TooManyRepsException(f"{len(t)} is too many, only 3 allowed") + + # parse an int followed by no more than 3 words + parser = pp.Word(pp.nums) + pp.Group( + pp.Word(pp.alphas)[...].add_parse_action(no_more_than_3) + ) + + # should succeed + result = parser.parse_string("1000 abc def ghi") + print(result.dump()) + + # should raise exception with local exception message + with self.assertRaisesParseException( + exc_type=ParseFatalException, + expected_msg="4 is too many, only 3 allowed", + msg="wrong exception message", + ) as pe_context: + result = parser.parse_string("2000 abc def ghi jkl") + + print(pe_context.exception) + + def test_pep8_synonyms(self): + """ + Test that staticmethods wrapped by replaced_by_pep8 wrapper are properly + callable as staticmethods. + """ + + def run_subtest(fn_name, expr=None, args=""): + bool_expr = pp.one_of("true false", as_keyword=True) + if expr is None: + expr = "bool_expr" + + # try calling a ParserElement staticmethod via a ParserElement instance + with self.subTest(fn_name=fn_name): + exec(f"{expr}.{fn_name}({args})", globals(), locals()) + + # access staticmethod synonyms using a ParserElement + parser_element_staticmethod_names = """ + enable_packrat disable_memoization enable_left_recursion reset_cache + """.split() + + if not ( + pp.ParserElement._packratEnabled or pp.ParserElement._left_recursion_enabled + ): + for name in parser_element_staticmethod_names: + run_subtest(name) + pp.ParserElement.disable_memoization() + + run_subtest("set_default_whitespace_chars", args="' '") + run_subtest("inline_literals_using", args="pp.Suppress") + + run_subtest( + "set_default_keyword_chars", expr="pp.Keyword('START')", args="'abcde'" + ) + pass + + +class Test04_WithPackrat(Test02_WithoutPackrat): + """ + rerun Test2 tests, now that packrat is enabled + """ + def setUp(self): + ParserElement.enable_packrat(force=True) + + def tearDown(self): + default_suite_context.restore() + + def test000_assert_packrat_status(self): + print("Packrat enabled:", ParserElement._packratEnabled) + print( + "Packrat cache:", + type(ParserElement.packrat_cache).__name__, + getattr(ParserElement.packrat_cache, "size", "- no size attribute -"), + ) + self.assertTrue(ParserElement._packratEnabled, "packrat not enabled") + self.assertEqual( + "_FifoCache", + type(ParserElement.packrat_cache).__name__, + msg="incorrect cache type", + ) + + +class Test06_WithBoundedPackrat(Test02_WithoutPackrat): + """ + rerun Test2 tests, now with bounded packrat cache + """ + def setUp(self): + ParserElement.enable_packrat(cache_size_limit=16, force=True) + + def tearDown(self): + default_suite_context.restore() + + def test000_assert_packrat_status(self): + print("Packrat enabled:", ParserElement._packratEnabled) + print( + "Packrat cache:", + type(ParserElement.packrat_cache).__name__, + getattr(ParserElement.packrat_cache, "size", "- no size attribute -"), + ) + self.assertTrue(ParserElement._packratEnabled, "packrat not enabled") + self.assertEqual( + "_FifoCache", + type(ParserElement.packrat_cache).__name__, + msg="incorrect cache type", + ) + + def test_exceeding_fifo_cache_size(self): + letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + letter_lit = pp.MatchFirst(pp.Literal.using_each(letters)) + result = letter_lit[...].parse_string(letters, parse_all=True) + self.assertEqual(list(result), list(letters)) + + +class Test08_WithUnboundedPackrat(Test02_WithoutPackrat): + """ + rerun Test2 tests, now with unbounded packrat cache + """ + def setUp(self): + ParserElement.enable_packrat(cache_size_limit=None, force=True) + + def tearDown(self): + default_suite_context.restore() + + + def test000_assert_packrat_status(self): + print("Packrat enabled:", ParserElement._packratEnabled) + print( + "Packrat cache:", + type(ParserElement.packrat_cache).__name__, + getattr(ParserElement.packrat_cache, "size", "- no size attribute -"), + ) + self.assertTrue(ParserElement._packratEnabled, "packrat not enabled") + self.assertEqual( + "_UnboundedCache", + type(ParserElement.packrat_cache).__name__, + msg="incorrect cache type", + ) + + +class Test09_WithLeftRecursionParsing(Test02_WithoutPackrat): + """ + rerun Test2 tests, now with unbounded left recursion cache + """ + + def setUp(self): + ParserElement.enable_left_recursion(force=True) + + def tearDown(self): + default_suite_context.restore() + + def test000_assert_packrat_status(self): + print("Left-Recursion enabled:", ParserElement._left_recursion_enabled) + self.assertTrue( + ParserElement._left_recursion_enabled, "left recursion not enabled" + ) + self.assertIsInstance(ParserElement.recursion_memos, pp.util.UnboundedMemo) + + +class Test10_WithLeftRecursionParsingBoundedMemo(Test02_WithoutPackrat): + """ + rerun Test2 tests, now with bounded left recursion cache + """ + + def setUp(self): + ParserElement.enable_left_recursion(cache_size_limit=4, force=True) + + def tearDown(self): + default_suite_context.restore() + + def test000_assert_packrat_status(self): + print("Left-Recursion enabled:", ParserElement._left_recursion_enabled) + self.assertTrue( + ParserElement._left_recursion_enabled, "left recursion not enabled" + ) + self.assertIsInstance(ParserElement.recursion_memos, pp.util.LRUMemo) + # check that the cache matches roughly what we expect + # – it may be larger due to action handling + self.assertLessEqual(ParserElement.recursion_memos._capacity, 4) + self.assertGreater(ParserElement.recursion_memos._capacity * 3, 4) + + +class Test11_LR1_Recursion(ppt.TestParseResultsAsserts, TestCase): + """ + Tests for recursive parsing + """ + + suite_context = None + save_suite_context = None + + def setUp(self): + recursion_suite_context.restore() + + def tearDown(self): + default_suite_context.restore() + + def test_repeat_as_recurse(self): + """repetition rules formulated with recursion""" + one_or_more = pp.Forward().set_name("one_or_more") + one_or_more <<= one_or_more + "a" | "a" + self.assertParseResultsEquals( + one_or_more.parse_string("a", parse_all=True), expected_list=["a"] + ) + self.assertParseResultsEquals( + one_or_more.parse_string("aaa aa", parse_all=True), + expected_list=["a", "a", "a", "a", "a"], + ) + DelimitedList = pp.Forward().set_name("DelimitedList") + DelimitedList <<= DelimitedList + pp.Suppress(",") + "b" | "b" + self.assertParseResultsEquals( + DelimitedList.parse_string("b", parse_all=True), expected_list=["b"] + ) + self.assertParseResultsEquals( + DelimitedList.parse_string("b,b", parse_all=True), expected_list=["b", "b"] + ) + self.assertParseResultsEquals( + DelimitedList.parse_string("b,b , b, b,b", parse_all=True), + expected_list=["b", "b", "b", "b", "b"], + ) + + def test_binary_recursive(self): + """parsing of single left-recursive binary operator""" + expr = pp.Forward().set_name("expr") + num = pp.Word(pp.nums) + expr <<= expr + "+" - num | num + self.assertParseResultsEquals( + expr.parse_string("1+2", parse_all=True), expected_list=["1", "+", "2"] + ) + self.assertParseResultsEquals( + expr.parse_string("1+2+3+4", parse_all=True), + expected_list=["1", "+", "2", "+", "3", "+", "4"], + ) + + def test_binary_associative(self): + """associative is preserved for single left-recursive binary operator""" + expr = pp.Forward().set_name("expr") + num = pp.Word(pp.nums) + expr <<= pp.Group(expr) + "+" - num | num + self.assertParseResultsEquals( + expr.parse_string("1+2", parse_all=True), expected_list=[["1"], "+", "2"] + ) + self.assertParseResultsEquals( + expr.parse_string("1+2+3+4", parse_all=True), + expected_list=[[[["1"], "+", "2"], "+", "3"], "+", "4"], + ) + + def test_add_sub(self): + """indirectly left-recursive/associative add/sub calculator""" + expr = pp.Forward().set_name("expr") + num = pp.Word(pp.nums).set_parse_action(lambda t: int(t[0])) + expr <<= ( + (expr + "+" - num).set_parse_action(lambda t: t[0] + t[2]) + | (expr + "-" - num).set_parse_action(lambda t: t[0] - t[2]) + | num + ) + self.assertEqual(expr.parse_string("1+2", parse_all=True)[0], 3) + self.assertEqual(expr.parse_string("1+2+3", parse_all=True)[0], 6) + self.assertEqual(expr.parse_string("1+2-3", parse_all=True)[0], 0) + self.assertEqual(expr.parse_string("1-2+3", parse_all=True)[0], 2) + self.assertEqual(expr.parse_string("1-2-3", parse_all=True)[0], -4) + + def test_math(self): + """precedence climbing parser for math""" + # named references + expr = pp.Forward().set_name("expr") + add_sub = pp.Forward().set_name("add_sub") + mul_div = pp.Forward().set_name("mul_div") + power = pp.Forward().set_name("power") + terminal = pp.Forward().set_name("terminal") + # concrete rules + number = pp.Word(pp.nums).set_parse_action(lambda t: int(t[0])) + signed = ("+" - expr) | ("-" - expr).set_parse_action(lambda t: -t[1]) + group = pp.Suppress("(") - expr - pp.Suppress(")") + add_sub <<= ( + (add_sub + "+" - mul_div).set_parse_action(lambda t: t[0] + t[2]) + | (add_sub + "-" - mul_div).set_parse_action(lambda t: t[0] - t[2]) + | mul_div + ) + mul_div <<= ( + (mul_div + "*" - power).set_parse_action(lambda t: t[0] * t[2]) + | (mul_div + "/" - power).set_parse_action(lambda t: t[0] / t[2]) + | power + ) + power <<= (terminal + "^" - power).set_parse_action( + lambda t: t[0] ** t[2] + ) | terminal + terminal <<= number | signed | group + expr <<= add_sub + # simple add_sub expressions + self.assertEqual(expr.parse_string("1+2", parse_all=True)[0], 3) + self.assertEqual(expr.parse_string("1+2+3", parse_all=True)[0], 6) + self.assertEqual(expr.parse_string("1+2-3", parse_all=True)[0], 0) + self.assertEqual(expr.parse_string("1-2+3", parse_all=True)[0], 2) + self.assertEqual(expr.parse_string("1-2-3", parse_all=True)[0], -4) + # precedence overwriting via parentheses + self.assertEqual(expr.parse_string("1+(2+3)", parse_all=True)[0], 6) + self.assertEqual(expr.parse_string("1+(2-3)", parse_all=True)[0], 0) + self.assertEqual(expr.parse_string("1-(2+3)", parse_all=True)[0], -4) + self.assertEqual(expr.parse_string("1-(2-3)", parse_all=True)[0], 2) + # complicated math expressions – same as Python expressions + self.assertEqual(expr.parse_string("1----3", parse_all=True)[0], 1 - ---3) + self.assertEqual(expr.parse_string("1+2*3", parse_all=True)[0], 1 + 2 * 3) + self.assertEqual(expr.parse_string("1*2+3", parse_all=True)[0], 1 * 2 + 3) + self.assertEqual(expr.parse_string("1*2^3", parse_all=True)[0], 1 * 2**3) + self.assertEqual(expr.parse_string("4^3^2^1", parse_all=True)[0], 4**3**2**1) + + def test_terminate_empty(self): + """Recursion with ``Empty`` terminates""" + empty = pp.Forward().set_name("e") + empty <<= empty + pp.Empty() | pp.Empty() + self.assertParseResultsEquals( + empty.parse_string("", parse_all=True), expected_list=[] + ) + + def test_non_peg(self): + """Recursion works for non-PEG operators""" + expr = pp.Forward().set_name("expr") + expr <<= expr + "a" ^ expr + "ab" ^ expr + "abc" ^ "." + self.assertParseResultsEquals( + expr.parse_string(".abcabaabc", parse_all=True), + expected_list=[".", "abc", "ab", "a", "abc"], + ) + + +class TestShowBestPractices(unittest.TestCase): + def test_loads_markdown_file(self): + # Mock the file read to simulate the Markdown content + mock_content = "## Test Best Practices\n- Example guideline" + mock_file = mock_open(read_data=mock_content) + + with patch("importlib.resources.files") as mock_files: + # mock path.open() to use our mock_file + mock_path = mock_files.return_value.joinpath.return_value + mock_path.open = mock_file + + result = pp.show_best_practices(file=None) + + self.assertEqual(result, mock_content) + + def test_fallback_when_file_missing(self): + # Patch files().joinpath().open to raise FileNotFoundError + with patch("importlib.resources.files") as mock_files: + mock_path = mock_files.return_value.joinpath.return_value + mock_path.open.side_effect = FileNotFoundError + + result = pp.show_best_practices(file=None) + + self.assertIn("## Planning", result) # Fallback contains "Planning" section + self.assertIn("## Implementing", result) + self.assertIn("## Testing", result) + self.assertIn("## Debugging", result) + + def test_cli_invocation_with_module_flag(self): + # Invoke the CLI the same way a user or AI would: + # python -m pyparsing.show_best_practices + cmd = [sys.executable, "-m", "pyparsing.ai.show_best_practices"] + subproc = subprocess.run(cmd, capture_output=True, text=True) + + self.assertEqual(subproc.returncode, 0, msg=f"stderr: {subproc.stderr}") + + subproc_stdout = subproc.stdout + # Should print the best practices markdown (either from file or fallback) + self.assertIn("## Planning", subproc_stdout) + self.assertIn("## Implementing", subproc_stdout) + self.assertIn("## Testing", subproc_stdout) + self.assertIn("## Debugging", subproc_stdout) + + +# force clear of packrat parsing flags before saving contexts +pp.ParserElement.disable_memoization() + +Test02_WithoutPackrat.suite_context = ppt.reset_pyparsing_context().save() +Test02_WithoutPackrat.save_suite_context = ppt.reset_pyparsing_context().save() + +default_suite_context = ppt.reset_pyparsing_context().save() +pp.ParserElement.enable_left_recursion() +recursion_suite_context = ppt.reset_pyparsing_context().save() +default_suite_context.restore()