diff --git a/AUTHORS.md b/AUTHORS.md index ef00fc43..4b5afb8a 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -1,6 +1,8 @@ Arpeggio - Parser interpreter based on PEG grammars -Author: Igor R. Dejanović +Arpeggio author: Igor R. Dejanović + +Original author of the state and actions subsystems: Andrey N. Dotsenko # Contributors diff --git a/arpeggio/__init__.py b/arpeggio/__init__.py index 77c35fd2..78f78456 100644 --- a/arpeggio/__init__.py +++ b/arpeggio/__init__.py @@ -1,8 +1,9 @@ ############################################################################### # Name: arpeggio.py # Purpose: PEG parser interpreter -# Author: Igor R. Dejanović -# Copyright: (c) 2009-2019 Igor R. Dejanović +# Author: Igor R. Dejanovic , Andrey N. Dotsenko +# Copyright: (c) 2009-2017 Igor R. Dejanovic +# Copyright: (c) 2025 Igor R. Dejanovic , Andrey N. Dotsenko # License: MIT License # # This is an implementation of packrat parser interpreter based on PEG @@ -10,12 +11,16 @@ # textual notation. ############################################################################### +import abc import bisect import codecs +import collections.abc import re import sys import types +import typing from collections import OrderedDict +import copy try: from importlib.metadata import version @@ -59,18 +64,21 @@ class NoMatch(Exception): match is not successful. Args: - rules (list of ParsingExpression): Rules that are tried at the position + rules: Rules or their wrappers that are tried at the position of the exception. - position (int): A position in the input stream where exception - occurred. - parser (Parser): An instance of a parser. + position: A position in the input stream where exception occurred. + parser: An instance of a parser. """ - def __init__(self, rules, position, parser): + def __init__( + self, + rules: typing.Union['ParsingExpression', 'ParserModelDescribable'], + position: int, + parser: 'Parser', + ): self.rules = rules self.position = position self.parser = parser - def eval_attrs(self): """ Call this to evaluate `message`, `context`, `line` and `col`. Called by __str__. @@ -79,7 +87,7 @@ def rule_to_exp_str(rule): if hasattr(rule, '_exp_str'): # Rule may override expected report string return rule._exp_str - elif rule.root: + elif hasattr(rule, 'root') and rule.root: return rule.rule_name elif isinstance(rule, Match) and \ not isinstance(rule, EndOfFile): @@ -159,10 +167,65 @@ def dprint(self, message, indent_change=0): # Parser Model (PEG Abstract Semantic Graph) elements -class ParsingExpression: +class ParserModelItem(abc.ABC): + """ + A basic class for all parser model classes. + + Represents the node of the parser model. All parser model classes should be descendants of this basic class + including any helper classes. + """ + + def resolve( + self, + resolve_cb: typing.Callable[['ParserModelItem'], 'ParserModelItem'] + ) -> 'ParserModelItem': + resolved_node = resolve_cb(self) + return resolved_node + + +class ParserModelDescribable(abc.ABC): + """ + A basic interface class for all parser model helper classes. + + This class is needed mainly for debugging purposes. It allows to get information from the inner parts of the rules + about parsing errors. + """ + + @property + @abc.abstractmethod + def name(self): + pass + + @property + def desc(self): + return self.name + ': ' + self.__class__.__name__ + + @property + def id(self): + return f'{self.name}: {id(self)}' + + +class ParsingStatement(ParserModelItem, ParserModelDescribable): + """ + A basic class for all parser model statement classes. + + Statements can parse text or do something else like manipulating the parser state system. + + All parser model statement classes (i.e. classes that participate in the parsing process) must be + descendants of this basic class. + """ + + @abc.abstractmethod + def parse(self, parser: 'Parser') -> typing.Optional['ParseTreeNode']: + pass + + +class ParsingExpression(ParsingStatement): """ An abstract class for all parsing expressions. - Represents the node of the Parser Model. + + The parsing expression differs from the parsing statement in such a way that it always involves + text parsing process. Attributes: elements: A list (or other python object) used as a staging structure @@ -181,6 +244,8 @@ class ParsingExpression: def __init__(self, *elements, **kwargs): + super().__init__() + if len(elements) == 1: elements = elements[0] self.elements = elements @@ -200,10 +265,12 @@ def __init__(self, *elements, **kwargs): # positions. self._result_cache = {} # position -> parse tree at the position + @typing.override @property def desc(self): return "{}{}".format(self.name, "-" if self.suppress else "") + @typing.override @property def name(self): if self.root: @@ -211,6 +278,7 @@ def name(self): else: return self.__class__.__name__ + @typing.override @property def id(self): if self.root: @@ -237,6 +305,7 @@ def _clear_cache(self, processed=None): processed.add(node) node._clear_cache(processed) + @typing.override def parse(self, parser): if parser.debug: @@ -345,6 +414,19 @@ def parse(self, parser): return result + def resolve(self, resolve_cb: typing.Callable[[ParserModelItem], ParserModelItem]) -> ParserModelItem: + for i, node in enumerate(typing.cast(list[ParsingExpression], self.nodes)): + self.nodes[i] = resolve_cb(node) + return super().resolve(resolve_cb) + + @property + def resolved_rule_name(self): + return self.rule_name + + @abc.abstractmethod + def _parse(self, parser: 'Parser') -> typing.Optional['ParseTreeNode']: + pass + class Sequence(ParsingExpression): """ @@ -356,6 +438,7 @@ def __init__(self, *elements, **kwargs): self.ws = kwargs.pop('ws', None) self.skipws = kwargs.pop('skipws', None) + @typing.override def _parse(self, parser): results = [] c_pos = parser.position @@ -371,6 +454,8 @@ def _parse(self, parser): # Prefetching append = results.append + state_snapshot = parser.take_state_snapshot() + try: for e in self.nodes: result = e.parse(parser) @@ -379,6 +464,7 @@ def _parse(self, parser): except NoMatch: parser.position = c_pos # Backtracking + parser.rollback_state_to_snapshot(state_snapshot) raise finally: @@ -396,6 +482,7 @@ class OrderedChoice(Sequence): Will match one of the parser expressions specified. Parser will try to match expressions in the order they are defined. """ + @typing.override def _parse(self, parser): result = None match = False @@ -411,6 +498,8 @@ def _parse(self, parser): try: for e in self.nodes: + state_snapshot = parser.take_state_snapshot() + try: result = e.parse(parser) match = True @@ -418,6 +507,8 @@ def _parse(self, parser): break except NoMatch: parser.position = c_pos # Backtracking + parser.rollback_state_to_snapshot(state_snapshot) + finally: if self.ws is not None: parser.ws = old_ws @@ -448,6 +539,7 @@ class Optional(Repetition): Optional will try to match parser expression specified and will not fail in case match is not successful. """ + @typing.override def _parse(self, parser): result = None c_pos = parser.position @@ -465,6 +557,7 @@ class ZeroOrMore(Repetition): ZeroOrMore will try to match parser expression specified zero or more times. It will never fail. """ + @typing.override def _parse(self, parser): results = [] @@ -480,7 +573,11 @@ def _parse(self, parser): sep = self.sep.parse if self.sep else None result = None + parser.state.push_repetition_layer() + while True: + state_snapshot = parser.take_state_snapshot() + try: c_pos = parser.position if sep and result: @@ -491,19 +588,30 @@ def _parse(self, parser): append(result) except NoMatch: parser.position = c_pos # Backtracking + parser.rollback_state_to_snapshot(state_snapshot) break + parser.state.pop_repetition_layer() + if self.eolterm: # Restore previous eolterm parser.eolterm = old_eolterm return results + @typing.override + def resolve(self, resolve_cb: typing.Callable[[ParserModelItem], ParserModelItem]) -> ParserModelItem: + node = super().resolve(resolve_cb) + if node.sep: + node.sep = node.sep.resolve(resolve_cb) + return node + class OneOrMore(Repetition): """ OneOrMore will try to match parser expression specified one or more times. """ + @typing.override def _parse(self, parser): results = [] first = True @@ -520,8 +628,12 @@ def _parse(self, parser): sep = self.sep.parse if self.sep else None result = None + parser.state.push_repetition_layer() + try: while True: + state_snapshot = parser.take_state_snapshot() + try: c_pos = parser.position if sep and result: @@ -533,23 +645,36 @@ def _parse(self, parser): first = False except NoMatch: parser.position = c_pos # Backtracking + parser.rollback_state_to_snapshot(state_snapshot) if first: raise break + finally: + if self.eolterm: # Restore previous eolterm parser.eolterm = old_eolterm + parser.state.pop_repetition_layer() + return results + @typing.override + def resolve(self, resolve_cb: typing.Callable[[ParserModelItem], ParserModelItem]) -> ParserModelItem: + node = super().resolve(resolve_cb) + if node.sep: + node.sep = node.sep.resolve(resolve_cb) + return node + class UnorderedGroup(Repetition): """ Will try to match all the parsing expressions in any order. """ + @typing.override def _parse(self, parser): results = [] c_pos = parser.position @@ -568,6 +693,10 @@ def _parse(self, parser): sep_result = None first = True + state_snapshot = parser.take_state_snapshot() + + parser.state.push_repetition_layer() + while nodes_to_try: sep_exc = None @@ -587,6 +716,8 @@ def _parse(self, parser): match = True all_optionals_fail = True for e in list(nodes_to_try): + curr_state_snapshot = parser.take_state_snapshot() + try: result = e.parse(parser) if result: @@ -604,6 +735,7 @@ def _parse(self, parser): except NoMatch: match = False parser.position = c_loc_pos # local backtracking + parser.rollback_state_to_snapshot(curr_state_snapshot) if not match or all_optionals_fail: # If sep is matched backtrack it @@ -614,9 +746,12 @@ def _parse(self, parser): # Restore previous eolterm parser.eolterm = old_eolterm + parser.state.pop_repetition_layer() + if not match: # Unsuccessful match of the whole PE - full backtracking parser.position = c_pos + parser.rollback_state_to_snapshot(state_snapshot) parser._nm_raise(self, c_pos, parser) if results: @@ -636,15 +771,21 @@ class And(SyntaxPredicate): This predicate will succeed if the specified expression matches current input. """ + @typing.override def _parse(self, parser): c_pos = parser.position - for e in self.nodes: - try: - e.parse(parser) - except NoMatch: - parser.position = c_pos - raise - parser.position = c_pos + state_snapshot = parser.take_state_snapshot() + + try: + for e in self.nodes: + try: + e.parse(parser) + except NoMatch: + parser.position = c_pos + raise + parser.position = c_pos + finally: + parser.rollback_state_to_snapshot(state_snapshot) class Not(SyntaxPredicate): @@ -652,9 +793,11 @@ class Not(SyntaxPredicate): This predicate will succeed if the specified expression doesn't match current input. """ + @typing.override def _parse(self, parser): c_pos = parser.position old_in_not = parser.in_not + parser.in_not = True try: for e in self.nodes: @@ -673,6 +816,7 @@ class Empty(SyntaxPredicate): """ This predicate will always succeed without consuming input. """ + @typing.override def _parse(self, parser): pass @@ -692,6 +836,7 @@ class Combine(Decorator): This rules will always return a Terminal parse tree node. Whitespaces will be preserved. Comments will not be matched. """ + @typing.override def _parse(self, parser): results = [] @@ -721,6 +866,7 @@ class Match(ParsingExpression): def __init__(self, rule_name, root=False, **kwargs): super().__init__(rule_name=rule_name, root=root, **kwargs) + @typing.override @property def name(self): if self.root: @@ -837,6 +983,7 @@ def __str__(self): def __unicode__(self): return self.__str__() + @typing.override def _parse(self, parser): c_pos = parser.position m = self.regex.match(parser.input, c_pos) @@ -870,6 +1017,7 @@ def __init__(self, to_match, rule_name='', root=False, ignore_case=None, self.to_match = to_match self.ignore_case = ignore_case + @typing.override def _parse(self, parser): c_pos = parser.position input_frag = parser.input[c_pos:c_pos+len(self.to_match)] @@ -908,7 +1056,6 @@ def __hash__(self): return hash(self.to_match) - # HACK: Kwd class is a bit hackish. Need to find a better way to # introduce different classes of string tokens. class Kwd(StrMatch): @@ -922,6 +1069,166 @@ def __init__(self, to_match): self.rule_name = 'keyword' +class ParsingState: + """ + A state that the parser could be in during the parsing process. + + Stores a name and an integer identifier of the state. Each state must have its own unique state identifier + across all the states of the parse model that the state is used in. Each state must also have its own unique name. + There must not be two or more states with the same name in the parse model. + """ + name: str + value: int + + def __init__(self, name: str, value: int): + self.name = name + self.value = value + + def __str__(self): + return f'{self.name} ({self.value})' + + def __eq__(self, other: 'ParsingState'): + return self.value == other.value + + +class ParsingStateStatement(ParsingStatement, abc.ABC): + """ + An abstract class for parsing state statements. + + Stores the parsing state and provides a property to get it. + """ + _parsing_state: ParsingState + + def __init__( + self, + parsing_state: ParsingState, + ): + super().__init__() + self._parsing_state = parsing_state + + @property + def state_name(self): + return self._parsing_state.name + + @property + def parsing_state(self): + return self._parsing_state + + +class MatchState(ParsingStateStatement): + """ + A statement to match the expected parsing state at the position the parser currently is in. + + If the current state doesn't match the expected state then the parsing process will fail. + """ + def parse(self, parser: 'Parser') -> typing.Optional['ParseTreeNode']: + c_pos = parser.position + curr_parsing_state = parser.state.parsing_state + if not curr_parsing_state: + if parser.debug: + parser.dprint( + f"-- The states stack is empty while matching `{self.parsing_state}` state at {c_pos} => " + f"'{parser.context()}'") + parser._nm_raise(self, c_pos, parser) + + if curr_parsing_state != self._parsing_state: + if parser.debug: + parser.dprint( + f"-- The current state (`{curr_parsing_state}`) doesn't match `{self.parsing_state}` state" + f" at {c_pos} => '{parser.context()}'") + parser._nm_raise(self, c_pos, parser) + + return None + + def __str__(self): + return '@' + self.state_name + + @typing.override + @property + def name(self): + return "@{}".format( + self.state_name, + ) + + +class PushState(ParsingStateStatement): + """ + A statement to push a new parsing state at the position the parser currently is in. + + This statement always passes. + """ + def parse(self, parser: 'Parser') -> typing.Optional['ParseTreeNode']: + parser.state.push_parsing_state(self._parsing_state) + return None + + def __str__(self): + return '+@' + self.state_name + + @typing.override + @property + def name(self): + return "+@{}".format( + self.state_name, + ) + + +class PopState(ParsingStateStatement): + """ + A statement to match and remove the expected parsing state at the position the parser currently is in. + + If the current statement doesn't match the expected statement, then the parsing process will fail. Otherwise, + the current parsing state will be removed from the top of the parsing states stack. + """ + def parse(self, parser: 'Parser') -> typing.Optional['ParseTreeNode']: + curr_parsing_state = parser.state.parsing_state + if curr_parsing_state != self._parsing_state: + c_pos = parser.position + if parser.debug: + parser.dprint( + f"-- The current state (`{curr_parsing_state}`) doesn't match `{self.parsing_state}` state" + f" at {c_pos} => '{parser.context()}'") + parser._nm_raise(self, c_pos, parser) + + parser.state.pop_parsing_state() + return None + + def __str__(self): + return '-@' + self.state_name + + @typing.override + @property + def name(self): + return "-@{}".format( + self.state_name, + ) + + +class StateWrapper(ParsingExpression): + """ + An expression wrapper that wraps an expression with a separate state layer. + + The expression will be parsed after a new parse state layer is pushed onto the top of the parse layers stack. + Finally, the parse state layer will be removed from the top of the stack. + """ + def __init__(self, node): + super().__init__(nodes=[node]) + + @typing.override + def _parse(self, parser: 'Parser') -> typing.Optional['ParseTreeNode']: + state_snapshot = parser.take_state_snapshot() + + parser.state.push_state_layer() + + try: + retval = self.nodes[0].parse(parser) + except: + parser.rollback_state_to_snapshot(state_snapshot) + raise + + parser.state.pop_state_layer() + return retval + + class EndOfFile(Match): """ The Match class that will succeed in case end of input is reached. @@ -929,10 +1236,12 @@ class EndOfFile(Match): def __init__(self): super().__init__("EOF") + @typing.override @property def name(self): return "EOF" + @typing.override def _parse(self, parser): c_pos = parser.position if len(parser.input) == c_pos: @@ -980,6 +1289,7 @@ def __init__(self, rule, position, error): self.error = error self.comments = None + @typing.override @property def name(self): return f"{self.rule_name} [{self.position}]" @@ -1054,6 +1364,7 @@ def __init__(self, rule, position, value, error=False, suppress=False, self.suppress = suppress self.extra_info = extra_info + @typing.override @property def desc(self): if self.value: @@ -1116,6 +1427,7 @@ def value(self): """Terminal protocol.""" return str(self) + @typing.override @property def desc(self): return self.name @@ -1376,6 +1688,250 @@ class SemanticActionToString(SemanticAction): def first_pass(self, parser, node, children): return str(node) + +class ParserStateLayer: + """ + A basic class for additional state parameters. + + Inherit from this class to add level-specific state parameters. + """ + states_stack: list[ParsingState] + + def __init__(self): + self.states_stack = [] + + def __deepcopy__(self, memo: dict = None): + copied = self.__class__() + copied.states_stack = copy.deepcopy(self.states_stack, memo) + return copied + + def queues_are_empty(self) -> bool: + return not self.states_stack + + def __str__(self): + return f"""States stack: +{str(self.states_stack)} +""" + + def __repr__(self): + return f'<{self.__class__.__name__}, id={id(self)}>{str(self)}' + + def __bool__(self): + return not self.states_stack + + +class HistoryItem(abc.ABC): + """ + An abstract class to store information needed to undo actions that were performed on the state system. + + In order to add a new data type to the state class that can be modified during parsing process, a new class + should be created by inheriting from this class to handle the undoing process. + """ + _object: typing.Any + _data: typing.Any + + def __init__(self, object: typing.Any, data: typing.Any): + self._object = object + self._data = data + + @abc.abstractmethod + def undo(self): + """ + Undo the operation information about which is stored in the class instance. + """ + pass + + def __deepcopy__(self, memo: dict = None): + return self.__class__(self._object, self._data) + + +class HistorySequencePush(HistoryItem): + """ + Mark appending a sequence (list, array, etc) with an item. + """ + _object: collections.abc.MutableSequence + + def __init__(self, obj: collections.abc.MutableSequence, data: typing.Any): + super().__init__(obj, data) + + @typing.override + def undo(self): + popped_data = self._object.pop() + assert(popped_data == self._data) + + +class HistorySequencePop(HistoryItem): + """ + Mark removing the last item from a sequence. + """ + _object: collections.abc.MutableSequence + + def __init__(self, obj: collections.abc.MutableSequence, data: typing.Any): + super().__init__(obj, data) + + @typing.override + def undo(self): + self._object.append(self._data) + + +class HistorySequencePopFront(HistoryItem): + """ + Mark removing the first item from a sequence. + """ + _object: collections.abc.MutableSequence + + def __init__(self, obj: collections.abc.MutableSequence, data: typing.Any): + super().__init__(obj, data) + + @typing.override + def undo(self): + self._object.insert(0, self._data) + + +class HistorySetAdd(HistoryItem): + """ + Mark adding an item to a set of items. + """ + _object: collections.abc.MutableSet + + def __init__(self, obj: collections.abc.MutableSet, data: typing.Any): + super().__init__(obj, data) + + @typing.override + def undo(self): + self._object.remove(self._data) + + +class HistorySetRemove(HistoryItem): + """ + Mark removing an item from a set of items. + """ + _object: collections.abc.MutableSet + + def __init__(self, object: collections.abc.MutableSet, data: typing.Any): + super().__init__(object, data) + + @typing.override + def undo(self): + self._object.add(self._data) + + +class ParserRepetitionStateLayer: + """ + A basic class for the parser repetition state layer. + + This class should store state information about the items that are inside a repetition (ZeroOrMore or OneOrMore). + """ + def __deepcopy__(self, memo: dict = None): + return self.__class__() + + +class ParserState: + """ + A basic class for the parser state system. + + The class instance should store all the data that is used between rules during the parsing process. + Inherit from this class to manage additional state functionality. + """ + _state_layer_class: ParserStateLayer = ParserStateLayer + _repetition_layer_class: ParserRepetitionStateLayer = ParserRepetitionStateLayer + + state_layers: list[_state_layer_class] + repetition_layers: list[_repetition_layer_class] + + _actions_history: list[HistoryItem] + + def __init__(self): + self.state_layers = [self._state_layer_class()] + self.repetition_layers = [] + self._actions_history = [] + + def __deepcopy__(self, memo: dict = None): + copied = self.__class__() + copied.state_layers = copy.deepcopy(self.state_layers, memo) + copied._actions_history = copy.deepcopy(self._actions_history, memo) + copied.repetition_layers = copy.deepcopy(self.repetition_layers, memo) + return copied + + def clear(self): + if len(self.state_layers) > 1 or self.state_layers[0]: + self.state_layers = [self._state_layer_class()] + self._actions_history = [] + + def load_from(self, other_state: 'ParserState'): + self.state_layers = other_state.state_layers + # Make the other state invalid to prevent possible errors: + other_state.state_layers = [] + + def push_parsing_state(self, parsing_state: ParsingState): + states_queue = self.state_layers[-1].states_stack + states_queue.append(parsing_state) + self._actions_history.append(HistorySequencePush(states_queue, parsing_state)) + + def pop_parsing_state(self) -> ParsingState: + states_queue = self.state_layers[-1].states_stack + parsing_state = states_queue.pop() + self._actions_history.append(HistorySequencePop(states_queue, parsing_state)) + return parsing_state + + @property + def parsing_state(self) -> ParsingState | None: + parsing_state = None + for state_layer in reversed(self.state_layers): + if state_layer.states_stack: + parsing_state = state_layer.states_stack[-1] + break + return parsing_state + + def push_state_layer(self): + """ + Push a new empty state layer onto the stack. + """ + layer = self._state_layer_class() + self.state_layers.append(layer) + self._actions_history.append(HistorySequencePush(self.state_layers, layer)) + + def pop_state_layer(self): + """ + Remove the last state layer from the stack. + + This function doesn't return anything because the state layer isn't supposed to be modified outside + the parser realization. + """ + if not self.state_layers[-1].queues_are_empty(): + raise GrammarError('One or more queues are not empty in the state layer that is being popped. ' + 'Probably, some grammar rules were not called to remove the items from the queues. ' + 'The parser state: ' + str(self)) + layer = self.state_layers.pop() + self._actions_history.append(HistorySequencePop(self.state_layers, layer)) + + def queues_are_empty(self) -> bool: + if len(self.state_layers) > 1: + return False + return self.state_layers[0].queues_are_empty() + + def push_repetition_layer(self): + """ + Push a new empty repetition layer onto the stack. + """ + layer = self._repetition_layer_class() + self.repetition_layers.append(layer) + + def pop_repetition_layer(self): + """ + Remove the last repetition layer from the stack. + """ + layer = self.repetition_layers.pop() + + def __str__(self) -> str: + return f"""State layers: +{self.state_layers} +""" + + +ParserStateSnapshot: typing.TypeAlias = int + + # ---------------------------------------------------- # Parsers @@ -1403,10 +1959,15 @@ class Parser(DebugPrinter): # Not marker for NoMatch rules list. Used if the first unsuccessful rule # match is Not. + _state_class: type[ParserState] = ParserState + _state: _state_class + + check_state_integrity: bool + FIRST_NOT = Not() def __init__(self, skipws=True, ws=None, reduce_tree=False, autokwd=False, - ignore_case=False, memoization=False, **kwargs): + ignore_case=False, memoization=False, check_state_integrity=True, **kwargs): """ Args: skipws (bool): Should the whitespace skipping be done. Default is @@ -1419,6 +1980,8 @@ def __init__(self, skipws=True, ws=None, reduce_tree=False, autokwd=False, ignore_case(bool): If case is ignored (default=False) memoization(bool): If memoization should be used (a.k.a. packrat parsing) + check_state_integrity: Raises an error if count of pushes isn't equal + the count of pops in the global state layer. """ super().__init__(**kwargs) @@ -1437,6 +2000,8 @@ def __init__(self, skipws=True, ws=None, reduce_tree=False, autokwd=False, self.autokwd = autokwd self.ignore_case = ignore_case self.memoization = memoization + self.check_state_integrity = check_state_integrity + self.comments_model = None self.comments = [] self.comment_positions = {} @@ -1450,6 +2015,8 @@ def __init__(self, skipws=True, ws=None, reduce_tree=False, autokwd=False, flags = re.IGNORECASE self.keyword_regex = re.compile(r'[^\d\W]\w*', flags) + self._state = self._state_class() + # Keep track of root rule we are currently in. # Used for debugging purposes self.in_rule = '' @@ -1501,6 +2068,7 @@ def parse(self, _input, file_name=None): file_name(str): If input is loaded from file this can be set to file name. It is used in error messages. """ + self.state.clear() self.position = 0 # Input position self.nm = None # Last NoMatch exception self.line_ends = [] @@ -1524,6 +2092,13 @@ def parse(self, _input, file_name=None): if self.memoization: self._clear_caches() + if self.check_state_integrity: + queues_are_empty = self.state.queues_are_empty() + if not queues_are_empty: + raise GrammarError('One or more queues are not empty. ' + 'Probably, some grammar rules were not called to remove the items from the queues. ' + 'The parser state: ' + str(self.state)) + # In debug mode export parse tree to dot file for # visualization if self.debug and self.parse_tree: @@ -1689,7 +2264,7 @@ def context(self, length=None, position=None): return retval.replace('\n', ' ').replace('\r', '') - def _nm_raise(self, *args): + def _nm_raise(self, *args) -> typing.NoReturn: """ Register new NoMatch object if the input is consumed from the last NoMatch and raise last NoMatch. @@ -1719,8 +2294,40 @@ def _clear_caches(self): if self.comments_model: self.comments_model._clear_cache() + @property + def state(self) -> _state_class: + return self._state + + def take_state_snapshot(self) -> ParserStateSnapshot: + """ + Take a snapshot of the parser state. + + Actually, returns the index of the current item in the history of operations made over the stack. + The behaviour might be changed in future so the returned type is not guaranteed to be int in the future versions + of the library. -class CrossRef: + Returns: + A snapshot. + """ + return len(self._state._actions_history) - 1 + + def rollback_state_to_snapshot(self, snapshot: ParserStateSnapshot): + """ + Rollback the state of the parser to a given state. + + This method undoes all the state changes that were made after the snapshot was taken. + + Parameters: + snapshot: + A previously taken snapshot to which the state is being rolled back. + """ + for i in range(len(self._state._actions_history) - 1, snapshot, -1): + history_item = self._state._actions_history[i] + history_item.undo() + del self._state._actions_history[i] + + +class CrossRef(ParserModelItem): ''' Used for rule reference resolving. ''' @@ -1878,6 +2485,9 @@ def inner_from_python(expression): if any(isinstance(x, CrossRef) for x in retval.nodes): __for_resolving.append(retval) + elif isinstance(expression, ParserModelItem): + retval = expression + else: raise GrammarError(f"Unrecognized grammar element '{expression}'.") diff --git a/arpeggio/cleanpeg.py b/arpeggio/cleanpeg.py index b462b2ab..f0f96518 100644 --- a/arpeggio/cleanpeg.py +++ b/arpeggio/cleanpeg.py @@ -3,8 +3,9 @@ # Purpose: This module is a variation of the original peg.py. # The syntax is slightly changed to be more readable and familiar to # python users. It is based on the Yash's suggestion - issue 11 -# Author: Igor R. Dejanovic -# Copyright: (c) 2014-2017 Igor R. Dejanovic +# Author: Igor R. Dejanovic , Andrey N. Dotsenko +# Copyright: (c) 2009-2017 Igor R. Dejanovic +# Copyright: (c) 2025 Igor R. Dejanovic , Andrey N. Dotsenko # License: MIT License ####################################################################### @@ -15,6 +16,7 @@ OneOrMore, Optional, ParserPython, + StrMatch, ZeroOrMore, visit_parse_tree, ) @@ -29,36 +31,187 @@ ASSIGNMENT = "=" ORDERED_CHOICE = "/" ZERO_OR_MORE = "*" -ONE_OR_MORE = "+" +ONE_OR_MORE_SYMBOL = '+' +ONE_OR_MORE = _('(? +# Author: Igor R. Dejanovic , Andrey N. Dotsenko # Copyright: (c) 2009-2017 Igor R. Dejanovic +# Copyright: (c) 2025 Igor R. Dejanovic , Andrey N. Dotsenko # License: MIT License ####################################################################### - +import abc import codecs +import collections.abc import copy +import enum import re +import typing from arpeggio import ( EOF, @@ -16,12 +20,21 @@ CrossRef, EndOfFile, GrammarError, + HistorySequencePush, + HistorySequencePop, + HistorySequencePopFront, + HistorySetAdd, Not, OneOrMore, Optional, OrderedChoice, + ParserState, + ParserStateLayer, + ParserRepetitionStateLayer, Parser, + ParserModelDescribable, ParserPython, + ParseTreeNode, PTNodeVisitor, SemanticError, Sequence, @@ -29,6 +42,14 @@ UnorderedGroup, ZeroOrMore, visit_parse_tree, + ParsingExpression, + ParsingStatement, + MatchState, + PushState, + PopState, + StateWrapper, + ParsingState, + ParserModelItem, ) from arpeggio import RegExMatch as _ @@ -38,37 +59,191 @@ LEFT_ARROW = "<-" ORDERED_CHOICE = "/" ZERO_OR_MORE = "*" -ONE_OR_MORE = "+" +ONE_OR_MORE_SYMBOL = '+' +ONE_OR_MORE = _('(? ParseTreeNode | None: + """ + This method must be implemented to run an action over the match result. + + Parameters: + parser + A parser used to parse the source code. + matched_result + The match result that need to be processed by the action. + c_pos + The parser position before the corresponding (the MatchActions child rule) rule was matched. + args + Additional arguments that were passed to the action. + + Returns: + A match result (usually the same as matched_result). + """ + pass + + @property + def command_str(self): + if self._command_hint: + if self._args: + args_str = ' ' + ' '.join(map(str, self._args)) + else: + args_str = '' + command_str = f'{self._command_hint}{args_str}' + else: + command_str = f'{self.__class__.__name__}({' '.join(map(str, self._args))})' + return command_str + + def __str__(self): + return f'{str(self._rule)}{{{self.command_str}}}' + + @typing.override + @property + def name(self): + return f'{self._rule.rule_name}{{..., {self.command_str}, ...}}' + +class ActionPush(MatchedAction): + """ + An action that is used to push a matched token onto the stack according to the rule name. + """ + @typing.override + def run( + self, + parser: 'ParserPEG', + matched_result: ParseTreeNode | None, + c_pos: int, + ) -> ParseTreeNode | None: + parser.state.push_rule_reference(self._rule.resolved_rule_name, str(matched_result)) + return matched_result + + +class ActionPop(MatchedAction): + """ + An action that is used to remove a matched token from the top of the matches list according to the rule name. + """ + @typing.override + def run( + self, + parser: 'ParserPEG', + matched_result: ParseTreeNode | None, + c_pos: int, + ) -> ParseTreeNode | None: + matched_str = str(matched_result) + rule_name = self._rule.resolved_rule_name + try: + removed = parser.state.pop_rule_reference(rule_name, matched_str) + except (IndexError, KeyError): + if parser.debug: + parser.dprint( + f"-- The stack for `{rule_name}` rule is empty at {c_pos} => " + f"'{parser.context(len(str(matched_result)))}'") + parser._nm_raise(self, c_pos, parser) + + if not removed: + if parser.debug: + match_against = parser.state.last_pushed_rule_reference(rule_name) + parser.dprint( + f"-- No match '{match_against}' at {c_pos} => " + f"'{parser.context(len(match_against))}'") + parser._nm_raise(self, c_pos, parser) + return matched_result + + +class ActionListAppend(MatchedAction): + """ + An action that is used to append the list of matched tokens with a matched token according to the rule name. + """ + @typing.override + def run( + self, + parser: 'ParserPEG', + matched_result: ParseTreeNode | None, + c_pos: int, + ) -> ParseTreeNode | None: + if matched_result is None: + matched_str = '' + else: + matched_str = str(matched_result) + parser.state.append_rule_reference(self._rule.resolved_rule_name, matched_str) + return matched_result + + +class ActionListLast(MatchedAction): + """ + An action that is used to match the last matched token from the top of the matches list according to the rule name. + """ + _state_scope: LayerScope = LayerScope.CURRENT + + @typing.override + def run( + self, + parser: 'ParserPEG', + matched_result: ParseTreeNode | None, + c_pos: int, + ) -> ParseTreeNode | None: + if matched_result is None: + matched_str = '' + else: + matched_str = str(matched_result) + + rule_name = self._rule.resolved_rule_name + try: + last = parser.state.last_rule_reference(rule_name, self._state_scope) + except (IndexError, KeyError): + if parser.debug: + parser.dprint( + f"-- The stack for `{rule_name}` rule is empty at {c_pos} => " + f"'{parser.context(len(str(matched_result)))}'") + parser._nm_raise(self, c_pos, parser) + + if matched_str != last: + if parser.debug: + parser.dprint( + f"-- No match '{last}' at {c_pos} => " + f"'{parser.context(len(last))}'") + parser._nm_raise(self, c_pos, parser) + + return matched_result + + +class ActionTryRemoveLast(MatchedAction): + """ + An action that is used to remove the last matched token from the list of matched tokens according to the rule name. + """ + @typing.override + def run( + self, + parser: 'ParserPEG', + matched_result: ParseTreeNode | None, + c_pos: int, + ) -> ParseTreeNode | None: + parser.state.try_remove_last_rule_reference(self._rule.resolved_rule_name) + return matched_result + + +class ActionParentListLast(ActionListLast): + """ + An action that is used to match the last matched token from the top of the matches list of the parent's stack layer + according to the rule name. + """ + _state_scope: LayerScope = LayerScope.PARENT + + +class ActionLonger(MatchedAction): + """ + An action that is used to match a longer token than the last matched token from the top of the matches list + according to the rule name. + """ + _state_scope: LayerScope = LayerScope.CURRENT + + @typing.override + def run( + self, + parser: 'ParserPEG', + matched_result: ParseTreeNode | None, + c_pos: int, + ) -> ParseTreeNode | None: + matched_str = str(matched_result) + rule_name =self._rule.resolved_rule_name + try: + last = parser.state.last_rule_reference(rule_name, self._state_scope) + except (IndexError, KeyError): + if parser.debug: + parser.dprint( + f"-- The stack for `{rule_name}` rule is empty or parent stack doesn't exist at {c_pos} => " + f"'{parser.context(len(str(matched_result)))}'") + parser._nm_raise(self, c_pos, parser) + + if len(matched_str) <= len(last): + if parser.debug: + parser.dprint( + f"-- Match '{matched_str}' is not longer than parent's '{last}' at {c_pos} => " + f"'{parser.context(len(matched_str))}'") + parser._nm_raise(self, c_pos, parser) + + return matched_result + + +class ActionParentListLonger(ActionLonger): + """ + An action that is used to match a longer token than the last matched token from the top of the matches list of + the parent's stack layer according to the rule name. + """ + _state_scope: LayerScope = LayerScope.PARENT + + +class ActionPopFront(MatchedAction): + """ + An action that is used to remove a matched token from the bottom of the matches list according to the rule name. + """ + @typing.override + def run( + self, + parser: 'ParserPEG', + matched_result: ParseTreeNode | None, + c_pos: int, + ) -> ParseTreeNode | None: + matched_str = str(matched_result) + rule_name =self._rule.resolved_rule_name + try: + removed = parser.state.pop_front_rule_reference(rule_name, matched_str) + except (IndexError, KeyError): + if parser.debug: + parser.dprint( + f"-- The stack for `{rule_name}` rule is empty at {c_pos} => " + f"'{parser.context(len(str(matched_result)))}'") + parser._nm_raise(self, c_pos, parser) + + if not removed: + if parser.debug: + match_against = parser.state.first_pushed_rule_reference(rule_name) + parser.dprint( + f"-- No match '{match_against}' at {c_pos} => " + f"'{parser.context(len(match_against))}'") + parser._nm_raise(self, c_pos, parser) + + return matched_result + + +class ActionAdd(MatchedAction): + """ + An action that is used to add a matched token to the set of matched tokens. + """ + @typing.override + def run( + self, + parser: 'ParserPEG', + matched_result: ParseTreeNode | None, + c_pos: int, + ) -> ParseTreeNode | None: + matched_str = str(matched_result) + parser.state.remember_rule_reference(self._rule.resolved_rule_name, matched_str) + return matched_result + + +class ActionParentAdd(MatchedAction): + """ + An action that is used to add a matched token to the set of matched tokens of the parent state layer. + """ + @typing.override + def run( + self, + parser: 'ParserPEG', + matched_result: ParseTreeNode | None, + c_pos: int, + ) -> ParseTreeNode | None: + matched_str = str(matched_result) + parser.state.remember_rule_reference( + self._rule.resolved_rule_name, + matched_str, + state_layer_scope = LayerScope.PARENT # noqa: E251 + ) + return matched_result + + +class ActionGlobalAdd(MatchedAction): + """ + An action that is used to add a matched token to the set of matched tokens of the global state layer. + """ + @typing.override + def run( + self, + parser: 'ParserPEG', + matched_result: ParseTreeNode | None, + c_pos: int, + ) -> ParseTreeNode | None: + matched_str = str(matched_result) + parser.state.remember_rule_reference( + self._rule.resolved_rule_name, + matched_str, + state_layer_scope = LayerScope.GLOBAL # noqa: E251 + ) + return matched_result + + +class ActionAny(MatchedAction): + """ + An action that is used to check if the matched token was previously added to the set of the matched tokens. + """ + @typing.override + def run( + self, + parser: 'ParserPEG', + matched_result: ParseTreeNode | None, + c_pos: int, + ) -> ParseTreeNode | None: + matched_str = str(matched_result) + + is_known = parser.state.rule_reference_is_known(self._rule.resolved_rule_name, matched_str) + if not is_known: + if parser.debug: + parser.dprint( + f"-- No known match for '{matched_str}' at {c_pos} => " + f"'{parser.context(len(matched_str))}'") + parser._nm_raise(self, c_pos, parser) + + return matched_result + + +class ActionSuppress(MatchedAction): + """ + An action that is used to suppress a rule. + """ + @typing.override + def run( + self, + parser: 'ParserPEG', + matched_result: ParseTreeNode | None, + c_pos: int, + ) -> ParseTreeNode | None: + return None + + +class ActionFirstLonger(MatchedAction): + """ + An action to check that the first matched token is longer than the token in the parent repetition. + + The main aim is to simplify validating indentation. + """ + @typing.override + def run( + self, + parser: 'ParserPEG', + matched_result: ParseTreeNode | None, + c_pos: int, + ) -> ParseTreeNode | None: + if matched_result is None: + matched_str = '' + else: + matched_str = str(matched_result) + + rule_name = self._rule.resolved_rule_name + last_value = parser.state.repetition_last_rule_reference(rule_name) + if last_value is None: + parent_value = parser.state.repetition_last_rule_reference(rule_name, LayerScope.PARENT) + + if ( + (parent_value is not None and len(matched_str) <= len(parent_value)) + or (parent_value is None and len(matched_str) > 0) + ): + if parser.debug: + parser.dprint( + f"-- Matched repetition '{matched_str}' token is not longer than " + f"parent token '{str(parent_value)}' at {c_pos} => '{parser.context(len(matched_str))}'") + parser._nm_raise(self, c_pos, parser) + + parser.state.repetition_set_rule_reference(rule_name, matched_str) + + return matched_result + + +class ActionOtherSame(MatchedAction): + """ + An action to check that the current matched token is longer than the previous token in the current repetition. + + The main aim is to simplify validating indentation. + """ + @typing.override + def run( + self, + parser: 'ParserPEG', + matched_result: ParseTreeNode | None, + c_pos: int, + ) -> ParseTreeNode | None: + if matched_result is None: + matched_str = '' + else: + matched_str = str(matched_result) + + rule_name = self._rule.resolved_rule_name + last_value = parser.state.repetition_last_rule_reference(rule_name) + if matched_str != last_value: + if parser.debug: + parser.dprint( + f"-- Matched repetition '{matched_str}' token is not the same as " + f"the previous '{last_value}' token at {c_pos} => '{parser.context(len(matched_str))}'") + parser._nm_raise(self, c_pos, parser) + + return matched_result + + +class MatchActions(ParsingExpression): + """ + Apply some actions to a matched rule. + Apply some actions to a matched rule. + + This rule parses his child rule and then runs stored actions over the result. Each action except the first one + receives the previous action result and returns its own result (usually the same). + """ + actions: list[MatchedAction] + + def __init__(self, rule: ParsingStatement, actions: list[MatchedAction]): + super().__init__(rule_name='', nodes=[rule]) + self.actions = actions + + @typing.override + def _parse(self, parser: 'ParserPEG'): + rule_node = self.nodes[0] + c_pos = parser.position + retval = rule_node.parse(parser) + for action in self.actions: + retval = action.run(parser, retval, c_pos) + return retval + + def __str__(self): + rule_node = self.nodes[0] + return str(rule_node) + + @typing.override + @property + def name(self): + actions_str = ', '.join(map(lambda action: ' '.join(action.command_str), self.actions)) + if self.rule_name: + self_name = f'{self.rule_name}=' + else: + self_name = '' + return f'{self_name}{str(self.nodes[0].resolved_rule_name)}{{{actions_str}}}' + + @typing.override + @property + def desc(self): + return "{}{}".format( + self.name, + "-" if self.suppress else "", + ) + + @typing.override + def resolve( + self, + resolve_cb: typing.Callable[[ParserModelItem], ParserModelItem] + ) -> 'MatchActions': + node = typing.cast(typing.Self, super().resolve(resolve_cb)) + for action in node.actions: + action._rule = node.nodes[0] + return node + + @typing.override + @property + def resolved_rule_name(self): + return self.rule_name or self.nodes[0].rule_name + + + +class ModifyConfig(ParsingExpression): + _modifiers: collections.abc.Sequence[tuple[str, typing.Any]] + + def __init__( + self, + node: ParsingExpression, + modifiers: collections.abc.Sequence[tuple[str, typing.Any]], + **kwargs, + ): + self._modifiers = modifiers + super().__init__(nodes=[node], **kwargs) + + @typing.override + def _parse(self, parser: 'Parser'): + old_values = [] + for modifier in self._modifiers: + old_values.append(getattr(parser, modifier[0])) + setattr(parser, modifier[0], modifier[1]) + + try: + retval = self.nodes[0].parse(parser) + finally: + for i, modifier in enumerate(self._modifiers): + setattr(parser, modifier[0], old_values[i]) + + return retval + + @typing.override + @property + def name(self): + modifiers_str = ', '. join(map(lambda modifier: modifier[0] + '=' + str(modifier[1]), self._modifiers)) + if self.rule_name: + return f'{self.rule_name}=[{modifiers_str}]{self.nodes[0].name}' + else: + return f'[{modifiers_str}]({self.nodes[0].name})' + + @typing.override + @property + def resolved_rule_name(self): + return self.rule_name or self.nodes[0].rule_name + + class PEGVisitor(PTNodeVisitor): """ Visitor that transforms parse tree to a PEG parser for the given language. """ + _parsing_state_by_name: dict[str, ParsingState] + _last_parsing_state_id: int + + matched_actions: dict[str, type[MatchedAction]] = { + 'push': ActionPush, + 'pop': ActionPop, + 'pop_front': ActionPopFront, + 'add': ActionAdd, + 'any': ActionAny, + 'list_append': ActionListAppend, + 'list_try_remove': ActionTryRemoveLast, + 'list_last': ActionListLast, + 'list_longer': ActionLonger, + 'parent_add': ActionParentAdd, + 'global_add': ActionGlobalAdd, + 'suppress': ActionSuppress, + 'parent_list_last': ActionParentListLast, + 'parent_list_longer': ActionParentListLonger, + 'first_longer': ActionFirstLonger, + 'other_same': ActionOtherSame, + } + matched_actions_aliases: dict[str, dict[str, str]] = { + 'state': { + 'push': 'push_state', + 'pop': 'pop_state', + }, + 'parent': { + 'add': 'parent_add', + 'list': { + 'parent_last': 'list_parent_last', + 'longer': 'parent_list_longer', + 'last': 'parent_list_last', + }, + }, + 'global': { + 'add': 'global_add', + }, + 'list': { + 'append': 'list_append', + 'last': 'list_last', + 'try': { + 'remove': 'list_try_remove', + }, + }, + 'first': { + 'longer': 'first_longer', + }, + 'other': { + 'same': 'other_same', + } + } + + modifiers_map = { + 'skip_whitespace': 'skipws', + 'whitespace': 'ws', + } def __init__(self, root_rule_name, comment_rule_name, ignore_case, *args, **kwargs): @@ -99,6 +871,22 @@ def __init__(self, root_rule_name, comment_rule_name, ignore_case, "EOF": EndOfFile() } + self._last_parsing_state_id = 0 + self._parsing_state_by_name = {} + + def register_parsing_state(self, state_name: str): + self._last_parsing_state_id += 1 + parsing_state_id = self._last_parsing_state_id + parsing_state = ParsingState(state_name, parsing_state_id) + self._parsing_state_by_name[state_name] = parsing_state + return parsing_state + + def get_state_by_name(self, state_name: str): + parsing_state = self._parsing_state_by_name.get(state_name) + if not parsing_state: + parsing_state = self.register_parsing_state(state_name) + return parsing_state + def visit_peggrammar(self, node, children): def _resolve(node): @@ -141,11 +929,9 @@ def resolve_rule_by_name(rule_name): if isinstance(node, CrossRef): # The root rule is a cross-ref resolved_rule = resolve_rule_by_name(node.target_rule_name) - return _resolve(resolved_rule) + return resolved_rule.resolve(_resolve) else: - # Resolve children nodes - for i, n in enumerate(node.nodes): - node.nodes[i] = _resolve(n) + node.resolve(_resolve) self.resolved.add(node) return node @@ -154,9 +940,9 @@ def resolve_rule_by_name(rule_name): comment_rule = None for rule in children: if rule.rule_name == self.root_rule_name: - root_rule = _resolve(rule) + root_rule = rule.resolve(_resolve) if rule.rule_name == self.comment_rule_name: - comment_rule = _resolve(rule) + comment_rule = rule.resolve(_resolve) assert root_rule, "Root rule not found!" return root_rule, comment_rule @@ -183,7 +969,85 @@ def visit_ordered_choice(self, node, children): retval = OrderedChoice(nodes=children[:]) if len(children) > 1 else children[0] return retval - def visit_prefix(self, node, children): + def postprocess_action_args(self, args): + action_name = args[0] + + if action_name in {'push_state', 'pop_state'}: + state_name = args[1] + parsing_state = self._parsing_state_by_name.get(state_name) + if not parsing_state: + parsing_state = self.register_parsing_state(state_name) + args[1] = parsing_state + + return args + + def preprocess_action_args(self, args) -> tuple[list[typing.Any], list[str]]: + if len(args) == 1: + return args, [args[0]] + + alias = self.matched_actions_aliases + original_command = [] + for i, arg in enumerate(args): + alias = alias.get(arg) + if alias is None: + return args, [args[0]] + + original_command.append(args[i]) + + if type(alias) is str: + del args[:i] + args[0] = alias + + return args, original_command + + @classmethod + def map_modifier(cls, name: str): + return cls.modifiers_map.get(name) or name + + def visit_parsing_expression_with_modifiers(self, node, children): + if len(children) == 1: + return children[0] + + modifiers = children[0] + modified_node = children[1] + return ModifyConfig(modified_node, modifiers) + + def visit_modifiers(self, node, children): + return children + + def visit_modifier(self, node, children): + return self.map_modifier(children[0]), children[1] + + def visit_true_literal(self, node, children): + return True + + def visit_false_literal(self, node, children): + return False + + def visit_expression_with_modifiers(self, node, children): + return self.visit_expression(node, children) + + def visit_expression(self, node, children): + if len(children) == 1: + return children[0] + + action_nodes = children[1] + actions = [] + rule_node = children[0] + for action_node in action_nodes: + action_args = [str(action_node[i]) for i in range(len(action_node))] + action_args, original_command = self.preprocess_action_args(action_args) + action_args = self.postprocess_action_args(action_args) + action_name = action_args[0] + action = self.matched_actions[action_name]( + rule_node, + action_args[1:], + command_hint = ' '.join(original_command), # noqa: E251 + ) + actions.append(action) + return MatchActions(children[0], actions) + + def visit_full_expression(self, node, children): if len(children) == 2: retval = Not() if children[0] == NOT else And() if isinstance(children[1], list): @@ -196,22 +1060,58 @@ def visit_prefix(self, node, children): return retval - def visit_sufix(self, node, children): - if len(children) == 2: - nodes = children[0] if isinstance(children[0], list) else [children[0]] - if children[1] == ZERO_OR_MORE: - retval = ZeroOrMore(nodes=nodes) - elif children[1] == ONE_OR_MORE: - retval = OneOrMore(nodes=nodes) - elif children[1] == OPTIONAL: - retval = Optional(nodes=nodes) - else: - retval = UnorderedGroup(nodes=nodes[0].nodes) + def visit_action_calls(self, node, children): + return children + + def visit_action_call(self, node, children): + return children + + def visit_quoted_string(self, node, children): + matched_str = str(node) + return self.decode_escaped_str(matched_str[1:-1]) + + def visit_parsing_state(self, node, children): + state_name = str(node) + parsing_state = self.get_state_by_name(state_name) + return MatchState(parsing_state) + + def visit_push_parsing_state(self, node, children): + state_name = str(node) + parsing_state = self.get_state_by_name(state_name) + return PushState(parsing_state) + + def visit_pop_parsing_state(self, node, children): + state_name = str(node) + parsing_state = self.get_state_by_name(state_name) + return PopState(parsing_state) + + def visit_wrapped_with_state_layer(self, node, children): + return StateWrapper(children[0]) + + def visit_repeated_expression(self, node, children): + if len(children) == 1: + return children[0] + + nodes = children[0] if isinstance(children[0], list) else [children[0]] + if children[1] == ZERO_OR_MORE: + retval = ZeroOrMore(nodes=nodes) + elif children[1] == ONE_OR_MORE_SYMBOL: + retval = OneOrMore(nodes=nodes) + elif children[1] == OPTIONAL: + retval = Optional(nodes=nodes) else: - retval = children[0] + retval = UnorderedGroup(nodes=nodes[0].nodes) return retval + def visit_grouped_parsing_expression(self, node, children): + if len(children) == 3: + if children[2] == ZERO_OR_MORE: + return ZeroOrMore(nodes=children[0], sep=children[1]) + else: + return OneOrMore(nodes=children[0], sep=children[1]) + return children[0] + def visit_rule_crossref(self, node, children): return CrossRef(node.value) @@ -220,9 +1120,7 @@ def visit_regex(self, node, children): match.compile() return match - def visit_str_match(self, node, children): - match_str = node.value[1:-1] - + def decode_escaped_str(self, s): # Scan the string literal, and sequentially match those escape # sequences which are syntactically valid Python. Attempt to convert # those, raising ``GrammarError`` for any semantically invalid ones. @@ -231,12 +1129,245 @@ def decode_escape(match): return codecs.decode(match.group(0), "unicode_escape") except UnicodeDecodeError as e: raise GrammarError(f"Invalid escape sequence '{match.group(0)}'.") from e - match_str = PEG_ESCAPE_SEQUENCES_RE.sub(decode_escape, match_str) + return PEG_ESCAPE_SEQUENCES_RE.sub(decode_escape, s) + + def visit_str_match(self, node, children): + match_str = node.value[1:-1] + match_str = self.decode_escaped_str(match_str) return StrMatch(match_str, ignore_case=self.ignore_case) +class ParserPEGStateLayer(ParserStateLayer): + """ + A class that holds additional data used in PEG expressions. + """ + rule_reference_stack: dict[str, str] + rule_reference_set: dict[str, str] + rule_reference_list: dict[str, str] + + def __init__(self): + super().__init__() + self.rule_reference_stack = {} + self.rule_reference_set = {} + self.rule_reference_list = {} + + def __deepcopy__(self, memo: dict = None): + copied = super().__deepcopy__(memo) + copied.rule_reference_stack = copy.deepcopy(self.rule_reference_stack, memo) + copied.rule_reference_set = copy.deepcopy(self.rule_reference_set, memo) + copied.rule_reference_list = copy.deepcopy(self.rule_reference_list, memo) + return copied + + def __bool__(self): + if super().__bool__(): + return True + + rule_reference_stack_is_empty = True + for key, value in self.rule_reference_stack.items(): + if value: + rule_reference_stack_is_empty = False + + rule_reference_set_is_empty = True + for key, value in self.rule_reference_set.items(): + if value: + rule_reference_set_is_empty = False + + rule_reference_list_is_empty = True + for key, value in self.rule_reference_list.items(): + if value: + rule_reference_list_is_empty = False + + if rule_reference_stack_is_empty and rule_reference_set_is_empty and rule_reference_list_is_empty: + return False + + return True + + + @typing.override + def queues_are_empty(self) -> bool: + if not super().queues_are_empty(): + return False + + rule_reference_stack_is_empty = True + for key, value in self.rule_reference_stack.items(): + if value: + rule_reference_stack_is_empty = False + + return rule_reference_stack_is_empty + + def __str__(self): + return f"""{super().__str__()} +Rule references queue: +{str(self.rule_reference_stack)} +Known rule references: +{str(self.rule_reference_set)} +""" + + +class ParserPEGRepetitionStateLayer(ParserRepetitionStateLayer): + """ + A class to store the state information about the items in a PEG repetition (* or +). + + Repetition layers are used to pass information between actions. + """ + last_rule_reference: dict[str, str] + + def __init__(self): + self.first_rule_reference = {} + self.last_rule_reference = {} + + def __deepcopy__(self, memo: dict = None): + copied = super().__deepcopy__(memo) + copied.first_rule_reference = copy.deepcopy(self.first_rule_reference, memo) + copied.last_rule_reference = copy.deepcopy(self.last_rule_reference, memo) + return copied + + +class ParserPEGState(ParserState): + """ + A class that manages additional data used in PEG expressions. + """ + _state_layer_class: ParserStateLayer = ParserPEGStateLayer + + _repetition_layer_class: ParserRepetitionStateLayer = ParserPEGRepetitionStateLayer + repetition_layers: list[_repetition_layer_class] + + def __init__(self): + super().__init__() + + def push_rule_reference( + self, + rule_name: str, + reference_name: str, + ): + stack = self.state_layers[-1].rule_reference_stack.setdefault(rule_name, []) + stack.append(reference_name) + self._actions_history.append(HistorySequencePush(stack, reference_name)) + + def pop_rule_reference( + self, + rule_name: str, + expected_reference_name: str = None, + ) -> str | None: + stack = self.state_layers[-1].rule_reference_stack[rule_name] + if expected_reference_name is not None and stack[-1] != expected_reference_name: + return None + reference_name = stack.pop() + self._actions_history.append(HistorySequencePop(stack, reference_name)) + return reference_name + + def pop_front_rule_reference( + self, + rule_name: str, + expected_reference_name: str = None, + ) -> str | None: + stack = self.state_layers[-1].rule_reference_stack[rule_name] + if expected_reference_name is not None and stack[0] != expected_reference_name: + return None + reference_name = stack.pop(0) + self._actions_history.append(HistorySequencePopFront(stack, reference_name)) + return reference_name + + def first_pushed_rule_reference(self, rule_name: str) -> str | None: + stack = self.state_layers[-1].rule_reference_stack.get(rule_name) + if not stack: + return None + return stack[0] + + def last_pushed_rule_reference( + self, + rule_name: str, + state_layer_scope: LayerScope = LayerScope.CURRENT, + ) -> str | None: + layer_num = state_layer_scope.value + stack = self.state_layers[layer_num].rule_reference_stack[rule_name] + if not stack: + return None + return stack[-1] + + def append_rule_reference( + self, + rule_name: str, + reference_name: str, + ): + stack = self.state_layers[-1].rule_reference_list.setdefault(rule_name, []) + stack.append(reference_name) + self._actions_history.append(HistorySequencePush(stack, reference_name)) + + def try_remove_last_rule_reference( + self, + rule_name: str, + ): + stack = self.state_layers[-1].rule_reference_list.setdefault(rule_name, []) + if not stack: + return # Was just trying. + + item = stack[-1] + del stack[-1] + self._actions_history.append(HistorySequencePop(stack, item)) + + def last_rule_reference( + self, + rule_name: str, + state_layer_scope: LayerScope = LayerScope.CURRENT, + ) -> str | None: + layer_num = state_layer_scope.value + stack = self.state_layers[layer_num].rule_reference_list[rule_name] + if not stack: + return None + return stack[-1] + + def remember_rule_reference( + self, + rule_name: str, + reference_name: str, + state_layer_scope: LayerScope = LayerScope.CURRENT + ): + layer_num = state_layer_scope.value + reference_set = self.state_layers[layer_num].rule_reference_set.setdefault(rule_name, set()) + reference_set.add(reference_name) + self._actions_history.append(HistorySetAdd(reference_set, reference_name)) + + def known_rule_references(self, rule_name: str) -> set[str]: + reference_set = self.state_layers[-1].rule_reference_set.get(rule_name) + return reference_set + + def rule_reference_is_known( + self, + rule_name: str, + reference_name: str + ) -> bool: + for state_layer in reversed(self.state_layers): + reference_set = state_layer.rule_reference_set.get(rule_name) + if reference_set is None: + continue + if reference_name in reference_set: + return True + return False + + def repetition_last_rule_reference( + self, + rule_name: str, + layer_scope: LayerScope = LayerScope.CURRENT, + ): + layer_num = layer_scope.value + if layer_num < 0 and len(self.repetition_layers) + layer_num < 0: + return None + last_reference_by_rule = self.repetition_layers[layer_num].last_rule_reference.get(rule_name) + return last_reference_by_rule + + def repetition_set_rule_reference( + self, + rule_name: str, + name: str, + ): + self.repetition_layers[-1].last_rule_reference[rule_name] = name + + class ParserPEG(Parser): + _state_class: type[ParserState] = ParserPEGState + _state: _state_class def __init__(self, language_def, root_rule_name, comment_rule_name=None, *args, **kwargs): @@ -250,6 +1381,7 @@ def __init__(self, language_def, root_rule_name, comment_rule_name=None, comment_rule_name(str): The name of the rule for comments. """ super().__init__(*args, **kwargs) + self.root_rule_name = root_rule_name self.comment_rule_name = comment_rule_name @@ -271,6 +1403,11 @@ def __init__(self, language_def, root_rule_name, comment_rule_name=None, def _parse(self): return self.parser_model.parse(self) + # Override just to fix the hinting issue because it's not possible to override a field: + @Parser.state.getter + def state(self) -> _state_class: + return self._state + def _from_peg(self, language_def): parser = ParserPython(peggrammar, comment, reduce_tree=False, debug=self.debug) diff --git a/arpeggio/tests/test_peg_actions_and_states.py b/arpeggio/tests/test_peg_actions_and_states.py new file mode 100644 index 00000000..1195a67a --- /dev/null +++ b/arpeggio/tests/test_peg_actions_and_states.py @@ -0,0 +1,638 @@ +####################################################################### +# Name: test_peg_actions_and_states +# Purpose: Test for parser constructed using PEG textual grammars using the actions and states system. +# Authors: Igor R. Dejanović , Andrey N. Dotsenko +# Copyright: (c) 2025 +# Igor R. Dejanović , +# Andrey N. Dotsenko +# License: MIT License +# +# This file is originally based on test_peg_parser.py +####################################################################### + +import pytest + +import arpeggio +from arpeggio.peg import ParserPEG +from arpeggio.cleanpeg import ParserPEG as ParserPEGClean +import enum + + +# Functions are used instead of variables to store grammar only to make parametrized test results readable +def get_grammar(): + return r''' +parser_entry <- program_element* EOF; + +program_element <- + anonymous_defer + / anonymous_deferred + / defer_call + / defer + / function + / alternative_function + / function_with_suppressed_keywords + / global_function + / function_call + / erroneous_non_closed_start + / erroneous_non_closed_end; + +function <- + @( + FUNCTION_START function_name{push, parent add} + program_element* + // Test And expression not changing the state of the parser + // Also, test quoted argument + FUNCTION_END &function_name{'pop'} function_name{pop} + ); + +global_function <- + GLOBAL + @( + FUNCTION_START function_name{push, global add} + program_element* + // Test setting multiple modifiers and setting a modifier to a quoted string value: + FUNCTION_END function_name{pop} + ); + +alternative_function <- + // Test branching with push action + // Also test resolving rule name by using a nested action + FUNCTION_START (function_name{push}){add} + @( + program_element* + // `*` operator is greedy so the closing `)` won't be matched until all the program_element statements are found + ) + '/' function_name{pop}; + +function_with_suppressed_keywords <- + @( + FUNCTION_START_SUPPRESSED function_name{push, parent add} + program_element* + FUNCTION_END_SUPPRESSED function_name{pop, 'suppress'} + ); + +function_call <- + function_name{any} + ARGUMENTS_START + ( + VALID_NAME + % ARGUMENTS_DELIMITER + )* + ARGUMENTS_END; + +defer_call <- + DEFER defer_name{push}; + +defer <- + defer_name{pop_front} DEFER_DELIMITER; + +defer_name <- VALID_NAME; + +anonymous_defer <- + ANONYMOUS_DEFER +@anonymous_defer; + +anonymous_deferred <- + @anonymous_defer DEFERRED + program_element* + END -@anonymous_defer; + +erroneous_non_closed_start <- + ERRONEOUS FUNCTION_START function_name{push, add}; + +erroneous_non_closed_end <- + ERRONEOUS FUNCTION_END function_name{pop}; + +FUNCTION_START_SUPPRESSED <- 'suppressed def'{suppress}; +FUNCTION_END_SUPPRESSED <- 'suppressed end of'{suppress}; + +FUNCTION_START <- 'def'; +function_name <- VALID_NAME; +FUNCTION_END <- 'end of'; +ARGUMENTS_START <- '('; +ARGUMENTS_END <- ')'; +VALID_NAME <- r'[a-zA-Z0-9_]+'; +ARGUMENTS_DELIMITER <- ','; +DEFER <- r'defer(?=\s)'; +DEFER_DELIMITER <- ':'; +ANONYMOUS_DEFER <- 'anonymous defer'; +DEFERRED <- 'deferred'; +END <- 'end'; +GLOBAL <- r'global(?=\s)'; +ERRONEOUS <- 'erroneous'; +''' + + +def get_clean_grammar(): + return get_grammar().replace('<-', '=').replace(';', '') + + +class Debugging(enum.Flag): + ENABLED = True + DISABLED = False + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_backreference(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 +end of function_name1 + +def function_name2 +end of function_name2 + """ + parser: ParserPEG = klass(grammar_cb(), 'parser_entry', debug=debug) + parser.parse(input_text) + + + with pytest.raises(Exception) as e: + parser.state.pop_rule_reference('function_name') + assert e is not None + + function_names = parser.state.known_rule_references('function_name') + assert isinstance(function_names, set) + assert len(function_names) == 2 + + if parser.debug: + output = capsys.readouterr() + assert 'states stack' in output.out + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_backreference_non_popped(klass, grammar_cb, debug, capsys): + input_text = """ +erroneous def function_name1 + """ + parser: ParserPEG = klass(grammar_cb(), 'parser_entry', debug=debug) + with pytest.raises(arpeggio.GrammarError): + parser.parse(input_text) + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_backreference_non_popped_in_state_layer(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name0 + erroneous def function_name1 +end of function_name0 + """ + parser: ParserPEG = klass(grammar_cb(), 'parser_entry', debug=debug) + with pytest.raises(arpeggio.GrammarError) as e: + parser.parse(input_text) + assert 'in the state layer' in e.value.message + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_backreference_non_pushed(klass, grammar_cb, debug, capsys): + input_text = """ +erroneous end of function_name1 + """ + parser: ParserPEG = klass(grammar_cb(), 'parser_entry', debug=debug) + with pytest.raises(arpeggio.NoMatch) as err_info: + parser.parse(input_text) + assert 'function_name{..., pop, ...}' in str(err_info.value) + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_backreference_any(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 +end of function_name1 + +def function_name2 + function_name1(arg1) +end of function_name2 + +function_name1(1, 2, 3) +function_name2(1, 2, 3) +""" + parser: ParserPEG = klass( + grammar_cb(), + 'parser_entry', + debug = debug, # noqa: E251 + reduce_tree = True, # noqa: E251 + ) + parse_tree = parser.parse(input_text) + assert parse_tree == [ + ["def", "function_name1", "end of", "function_name1"], + ["def", "function_name2", ["function_name1", "(", "arg1", ")"], "end of", "function_name2"], + ["function_name1", "(", "1", ",", "2", ",", "3", ")"], + ["function_name2", "(", "1", ",", "2", ",", "3", ")"], + "" # EOF + ] + + with pytest.raises(Exception) as e: + parser.state.pop_rule_reference('function_name') + assert e is not None + + if parser.debug: + output = capsys.readouterr() + assert 'states stack' in output.out + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_backreference_any_not_met(klass, grammar_cb, debug, capsys): + input_text = """ +function_name1(arg1) +""" + parser: ParserPEG = klass(grammar_cb(), 'parser_entry', debug=debug) + with pytest.raises(arpeggio.NoMatch): + parser.parse(input_text) + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_backreference_pop_front(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 +end of function_name1 + +def function_name2 +end of function_name2 + +def function_name3 + defer function_name1 + defer function_name2 + + function_name1: + function_name1(1, 2, 3) + function_name2: + function_name2(1, 2, 3) +end of function_name3 + +""" + parser: ParserPEG = klass( + grammar_cb(), + 'parser_entry', + debug = debug, # noqa: E251 + reduce_tree = True, # noqa: E251 + ) + parse_tree = parser.parse(input_text) + assert parse_tree == [ + ["def", "function_name1", "end of", "function_name1"], + ["def", "function_name2", "end of", "function_name2"], + [ + "def", "function_name3", + ["defer", "function_name1"], + ["defer", "function_name2"], + ["function_name1", ":"], + ["function_name1", "(", "1", ",", "2", ",", "3", ")"], + ["function_name2", ":"], + ["function_name2", "(", "1", ",", "2", ",", "3", ")"], + "end of", "function_name3" + ], + "" + ] + + with pytest.raises(Exception) as e: + parser.state.pop_rule_reference('function_name') + assert e is not None + + if parser.debug: + output = capsys.readouterr() + assert 'states stack' in output.out + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_backreference_with_state(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 +end of function_name1 + +def function_name2 +end of function_name2 + +def function_name3 + anonymous defer + anonymous defer + + deferred + function_name1(1, 2, 3) + end + + deferred + function_name2(1, 2, 3) + end +end of function_name3 + +""" + parser: ParserPEG = klass( + grammar_cb(), + 'parser_entry', + debug = debug, # noqa: E251 + reduce_tree = True, # noqa: E251 + ) + parse_tree = parser.parse(input_text) + assert parse_tree == [ + ["def", "function_name1", "end of", "function_name1"], + ["def", "function_name2", "end of", "function_name2"], + [ + "def", "function_name3", + "anonymous defer", + "anonymous defer", + ["deferred", ["function_name1", "(", "1", ",", "2", ",", "3", ")"], "end"], + ["deferred", ["function_name2", "(", "1", ",", "2", ",", "3", ")"], "end"], + "end of", "function_name3" + ], + "" + ] + + with pytest.raises(Exception) as e: + parser.state.pop_rule_reference('function_name') + assert e is not None + assert parser.state.parsing_state is None + + if parser.debug: + output = capsys.readouterr() + assert 'states stack' in output.out + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_backreference_with_wrong_state(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 +end of function_name1 + +deferred + function_name1(1, 2, 3) +end +""" + parser: ParserPEG = klass(grammar_cb(), 'parser_entry', debug=debug) + with pytest.raises(arpeggio.NoMatch): + parser.parse(input_text) + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_backreference_with_wrong_state_in_state_layer(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 +end of function_name1 + +def function_name2 + deferred + function_name1(1, 2, 3) + end +end of function_name2 +""" + parser: ParserPEG = klass(grammar_cb(), 'parser_entry', debug=debug) + with pytest.raises(arpeggio.NoMatch): + parser.parse(input_text) + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_backreference_with_not_popped_state_within_state_layer(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 + anonymous defer +end of function_name1 +""" + parser: ParserPEG = klass(grammar_cb(), 'parser_entry', debug=debug) + with pytest.raises(arpeggio.GrammarError): + parser.parse(input_text) + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_backreference_with_not_popped_state_within_global_state_layer(klass, grammar_cb, debug, capsys): + input_text = """ +anonymous defer +""" + parser: ParserPEG = klass(grammar_cb(), 'parser_entry', debug=debug) + with pytest.raises(arpeggio.GrammarError): + parser.parse(input_text) + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_backreference_with_lookahead(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 +/function_name1 + +def function_name2 +end of function_name2 + """ + parser: ParserPEG = klass(grammar_cb(), 'parser_entry', debug=debug) + parser.parse(input_text) + + with pytest.raises(Exception) as e: + parser.state.pop_rule_reference('function_name') + assert e is not None + + function_names = parser.state.known_rule_references('function_name') + assert isinstance(function_names, set) + assert len(function_names) == 2 + + if parser.debug: + output = capsys.readouterr() + assert 'states stack' in output.out + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_backreference_not_found(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 +end of function_name2 + """ + parser: ParserPEG = klass(grammar_cb(), 'parser_entry', debug=debug) + with pytest.raises(arpeggio.NoMatch): + parser.parse(input_text) + + with pytest.raises(Exception) as e: + parser.state.pop_rule_reference('function_name') + assert e is not None + + if parser.debug: + output = capsys.readouterr() + assert 'states stack' in output.out + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_backreference_any_not_found(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 + not_found(1, 2, 3) +end of function_name1 + """ + parser: ParserPEG = klass(grammar_cb(), 'parser_entry', debug=debug) + with pytest.raises(arpeggio.NoMatch): + parser.parse(input_text) + + with pytest.raises(Exception) as e: + parser.state.pop_rule_reference('function_name') + assert e is not None + + if parser.debug: + output = capsys.readouterr() + assert 'states stack' in output.out + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_backreference_global_add(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 + global def global_function_name + end of global_function_name +end of function_name1 + +global_function_name(1, 2, 3) + """ + parser: ParserPEG = klass(grammar_cb(), 'parser_entry', debug=debug) + parser.parse(input_text) + + with pytest.raises(Exception) as e: + parser.state.pop_rule_reference('function_name') + assert e is not None + + if parser.debug: + output = capsys.readouterr() + assert 'states stack' in output.out + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_wrapping_with_state_layer(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 + def local_function_name + end of local_function_name +end of function_name1 + +local_function_name(1, 2, 3) + """ + parser: ParserPEG = klass(grammar_cb(), 'parser_entry', debug=debug) + with pytest.raises(arpeggio.NoMatch): + parser.parse(input_text) + + with pytest.raises(Exception) as e: + parser.state.pop_rule_reference('function_name') + assert e is not None + + if parser.debug: + output = capsys.readouterr() + assert 'states stack' in output.out + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_state_reentrance(klass, grammar_cb, debug, capsys): + input_text1 = """ +erroneous def function_name1 + """ + parser: ParserPEG = klass(grammar_cb(), 'parser_entry', debug=debug) + + with pytest.raises(arpeggio.GrammarError): + parser.parse(input_text1) + + input_text2 = """ +erroneous end of function_name1 + """ + with pytest.raises(arpeggio.NoMatch): + # If parser.state is not cleared then this rule will pass, but the state should be cleared on every parse. + parser.parse(input_text2) + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_suppress_action(klass, grammar_cb, debug, capsys): + input_text = """ +suppressed def function_name1 +suppressed end of function_name1 + +suppressed def function_name2 + function_name1(arg1) +suppressed end of function_name2 + +function_name1(1, 2, 3) +function_name2(1, 2, 3) +""" + parser: ParserPEG = klass( + grammar_cb(), + 'parser_entry', + debug = debug, # noqa: E251 + reduce_tree = True, # noqa: E251 + ) + parse_tree = parser.parse(input_text) + assert parse_tree == [ + "function_name1", + ["function_name2", ["function_name1", "(", "arg1", ")"]], + ["function_name1", "(", "1", ",", "2", ",", "3", ")"], + ["function_name2", "(", "1", ",", "2", ",", "3", ")"], + "" # EOF + ] + + with pytest.raises(Exception) as e: + parser.state.pop_rule_reference('function_name') + assert e is not None + + if parser.debug: + output = capsys.readouterr() + assert 'states stack' in output.out diff --git a/arpeggio/tests/test_peg_config_modifiers.py b/arpeggio/tests/test_peg_config_modifiers.py new file mode 100644 index 00000000..8a7591f6 --- /dev/null +++ b/arpeggio/tests/test_peg_config_modifiers.py @@ -0,0 +1,146 @@ +####################################################################### +# Name: test_peg_actions_and_states +# Purpose: Test for parser constructed using PEG textual grammars using the actions and states system. +# Authors: Igor R. Dejanović , Andrey N. Dotsenko +# Copyright: (c) 2025 +# Igor R. Dejanović , +# Andrey N. Dotsenko +# License: MIT License +# +# This file is originally based on test_peg_actions_and_states.py +####################################################################### + +import pytest + +from arpeggio.peg import ParserPEG +from arpeggio.cleanpeg import ParserPEG as ParserPEGClean +import enum + + +# Functions are used instead of variables to store grammar only to make parametrized test results readable +def get_grammar(): + return r''' +parser_entry <- + (block_indentation program_element)* + EOF; + +program_element <- + function + / function_call; + +function <- + @( + FUNCTION_START function_name{push} + (block_indentation program_element)* + block_indentation FUNCTION_END [skip_whitespace=True, whitespace=' \t']function_name{pop} + ); + +function_call <- + function_name + !SPACE + ARGUMENTS_START + ( + VALID_NAME + % ARGUMENTS_DELIMITER + )* + ARGUMENTS_END; + +block_indentation <- INDENTATION{first longer, other same}; + +FUNCTION_START <- 'def'; +function_name <- VALID_NAME; +FUNCTION_END <- 'end of'; +ARGUMENTS_START <- '('{suppress}; +ARGUMENTS_END <- ')'{suppress}; +VALID_NAME <- r'[a-zA-Z0-9_]+'; +ARGUMENTS_DELIMITER <- ','{suppress}; +SPACE <- [skip_whitespace=False]r'[ \t]+'; +INDENTATION <- [whitespace='\n\r']r' *'; +''' + + +def get_clean_grammar(): + return get_grammar().replace('<-', '=').replace(';', '') + + +class Debugging(enum.Flag): + ENABLED = True + DISABLED = False + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_two_modifiers_with_string_modifier_and_backreference_action(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 +end of function_name1 + +def function_name2 +end of function_name2 + """ + parser: ParserPEG = klass( + grammar_cb(), + 'parser_entry', + debug = debug, # noqa: E251 + reduce_tree = True, # noqa: E251 + ) + parse_tree = parser.parse(input_text) + assert parse_tree == [ + ['def', 'function_name1', 'end of', 'function_name1'], + ['def', 'function_name2', 'end of', 'function_name2'], + '', + ] + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_single_modifier(klass, grammar_cb, debug, capsys): + input_text = """ +function_name1(1, 2, 3) +""" + parser: ParserPEG = klass( + grammar_cb(), + 'parser_entry', + debug = debug, # noqa: E251 + reduce_tree = True, # noqa: E251 + ) + parse_tree = parser.parse(input_text) + assert parse_tree == [ + ['function_name1', '1', '2', '3'], + '', + ] + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_indentation_with_spaces(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 +end of function_name1 + +def function_name2 + function_name1(1, 2, 3) + function_name2(4, 5, 6) + + def inner_function_name + function_name1(1, 2, 3) + function_name2(4, 5, 6) + end of inner_function_name +end of function_name2 +""" + parser: ParserPEG = klass( + grammar_cb(), + 'parser_entry', + debug = debug, # noqa: E251 + reduce_tree = True, # noqa: E251 + ) + parser.parse(input_text) diff --git a/arpeggio/tests/test_peg_indentation.py b/arpeggio/tests/test_peg_indentation.py new file mode 100644 index 00000000..abdd0631 --- /dev/null +++ b/arpeggio/tests/test_peg_indentation.py @@ -0,0 +1,163 @@ +####################################################################### +# Name: test_peg_indentation +# Purpose: Test actions that can be used to parse indentation. +# Authors: Igor R. Dejanović , Andrey N. Dotsenko +# Copyright: (c) 2025 +# Igor R. Dejanović , +# Andrey N. Dotsenko +# License: MIT License +# +# This file is originally based on test_peg_actions_and_states.py +####################################################################### + +import pytest + +from arpeggio import NoMatch +from arpeggio.peg import ParserPEG +from arpeggio.cleanpeg import ParserPEG as ParserPEGClean +import enum + + +# Functions are used instead of variables to store grammar only to make parametrized test results readable +def get_grammar(): + return r''' +parser_entry <- + ( + INDENTATION{list append, suppress} program_element + (INDENTATION{list last, suppress} program_element)* + )? + EOF; + +program_element <- + function_with_underscores + / function_call; + +function_with_underscores <- + @( + FUNCTION_START function_name{push, parent add} + ( + INDENTATION{parent list longer, list append, suppress} program_element + (INDENTATION{list last, suppress} program_element)* + )? + INDENTATION{parent list last, list try remove, suppress} FUNCTION_END function_name{pop} + ); + + +function_call <- + function_name{any} + ARGUMENTS_START + ( + VALID_NAME + % ARGUMENTS_DELIMITER + )* + ARGUMENTS_DELIMITER? + ARGUMENTS_END; + +FUNCTION_START <- 'def'; +function_name <- VALID_NAME; +FUNCTION_END <- 'end of'; +ARGUMENTS_START <- '('; +ARGUMENTS_END <- ')'; +VALID_NAME <- r'[a-zA-Z0-9_]+'; +ARGUMENTS_DELIMITER <- ','; +INDENTATION <- r'_*'; +''' + + +def get_clean_grammar(): + return get_grammar().replace('<-', '=').replace(';', '') + + +class Debugging(enum.Flag): + ENABLED = True + DISABLED = False + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_parent_last_last_longer(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 +end of function_name1 + +def function_name2 +____function_name1(1, 2, 3) +____function_name2(4, 5, 6) +____def inner_function_name +________function_name1(1, 2, 3) +________function_name2(4, 5, 6) +____end of inner_function_name +end of function_name2 +""" + parser: ParserPEG = klass( + grammar_cb(), + 'parser_entry', + debug = debug, # noqa: E251 + reduce_tree = True, # noqa: E251 + ) + parser.parse(input_text) + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_wrong_indentation(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 +end of function_name1 + +def function_name2 +____function_name1(1, 2, 3) +____function_name2(4, 5, 6) + +____def inner_function_name +____function_name1(1, 2, 3) +____function_name2(4, 5, 6) +____end of inner_function_name +end of function_name2 + +""" + parser: ParserPEG = klass( + grammar_cb(), + 'parser_entry', + debug = debug, # noqa: E251 + reduce_tree = True, # noqa: E251 + ) + with pytest.raises(NoMatch): + parser.parse(input_text) + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_wrong_indentation_at_end(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 +end of function_name1 + +def function_name2 +____function_name1(1, 2, 3) +____function_name2(4, 5, 6) + +____def inner_function_name +______function_name1(1, 2, 3) +______function_name2(4, 5, 6) +___end of inner_function_name +end of function_name2 + +""" + parser: ParserPEG = klass( + grammar_cb(), + 'parser_entry', + debug = debug, # noqa: E251 + reduce_tree = True, # noqa: E251 + ) + with pytest.raises(NoMatch): + parser.parse(input_text) diff --git a/arpeggio/tests/test_peg_indentation_using_same.py b/arpeggio/tests/test_peg_indentation_using_same.py new file mode 100644 index 00000000..1cfd5ae9 --- /dev/null +++ b/arpeggio/tests/test_peg_indentation_using_same.py @@ -0,0 +1,176 @@ +####################################################################### +# Name: test_peg_indentation_using_same +# Purpose: Test actions that can be used to parse indentation. +# Authors: Igor R. Dejanović , Andrey N. Dotsenko +# Copyright: (c) 2025 +# Igor R. Dejanović , +# Andrey N. Dotsenko +# License: MIT License +# +# This file is originally based on test_peg_indentation.py +####################################################################### + +import pytest + +from arpeggio import NoMatch +from arpeggio.peg import ParserPEG +from arpeggio.cleanpeg import ParserPEG as ParserPEGClean +import enum + + +# Functions are used instead of variables to store grammar only to make parametrized test results readable +def get_grammar(): + return r''' +parser_entry <- + (block_indentation program_element)* + EOF; + +program_element <- + function + / function_call; + +function <- + @( + FUNCTION_START function_name{push} + (block_indentation program_element)* + block_indentation FUNCTION_END function_name{pop} + ); + +function_call <- + function_name + ARGUMENTS_START + ( + VALID_NAME + % ARGUMENTS_DELIMITER + )* + ARGUMENTS_DELIMITER? + ARGUMENTS_END; + +block_indentation <- INDENTATION{first longer, other same}; + +FUNCTION_START <- 'def'; +function_name <- VALID_NAME; +FUNCTION_END <- 'end of'; +ARGUMENTS_START <- '('; +ARGUMENTS_END <- ')'; +VALID_NAME <- r'[a-zA-Z0-9_]+'; +ARGUMENTS_DELIMITER <- ','; +INDENTATION <- r'_*'; +''' + + +def get_clean_grammar(): + return get_grammar().replace('<-', '=').replace(';', '') + + +class Debugging(enum.Flag): + ENABLED = True + DISABLED = False + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_first_longer_and_other_same(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 +end of function_name1 + +def function_name2 +____function_name1(1, 2, 3) +____function_name2(4, 5, 6) +____def inner_function_name +________function_name1(1, 2, 3) +________function_name2(4, 5, 6) +____end of inner_function_name +end of function_name2 +""" + parser: ParserPEG = klass( + grammar_cb(), + 'parser_entry', + debug = debug, # noqa: E251 + reduce_tree = True, # noqa: E251 + ) + parser.parse(input_text) + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_wrong_indentation(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 +end of function_name1 + +def function_name2 +____function_name1(1, 2, 3) +____function_name2(4, 5, 6) + +____def inner_function_name +____function_name1(1, 2, 3) +____function_name2(4, 5, 6) +____end of inner_function_name +end of function_name2 + +""" + parser: ParserPEG = klass( + grammar_cb(), + 'parser_entry', + debug = debug, # noqa: E251 + reduce_tree = True, # noqa: E251 + ) + with pytest.raises(NoMatch): + parser.parse(input_text) + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_wrong_indentation_at_end(klass, grammar_cb, debug, capsys): + input_text = """ +def function_name1 +end of function_name1 + +def function_name2 +____function_name1(1, 2, 3) +____function_name2(4, 5, 6) + +____def inner_function_name +______function_name1(1, 2, 3) +______function_name2(4, 5, 6) +___end of inner_function_name +end of function_name2 + +""" + parser: ParserPEG = klass( + grammar_cb(), + 'parser_entry', + debug = debug, # noqa: E251 + reduce_tree = True, # noqa: E251 + ) + with pytest.raises(NoMatch): + parser.parse(input_text) + + +@pytest.mark.parametrize('klass, grammar_cb, debug', [ + (ParserPEGClean, get_clean_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.DISABLED), + (ParserPEG, get_grammar, Debugging.ENABLED), +]) +def test_wrong_indentation_at_program_start(klass, grammar_cb, debug, capsys): + input_text = """____def function_name1 +____end of function_name1""" + parser: ParserPEG = klass( + grammar_cb(), + 'parser_entry', + debug = debug, # noqa: E251 + reduce_tree = True, # noqa: E251 + ) + with pytest.raises(NoMatch): + parser.parse(input_text) diff --git a/docs/grammars.md b/docs/grammars.md index ffee1a37..beefd5aa 100644 --- a/docs/grammars.md +++ b/docs/grammars.md @@ -35,7 +35,7 @@ by end of input (`EOF`). `second` rule is ordered choice and will match either parse as far as it can, leaving the rest of the input unprocessed, and return without an error. So, be sure to always end your root rule sequence with `EOF` if you want a complete parse. - + During parsing each successfully matched rule will create a parse tree node. At the end of parsing a complete [parse tree](parse_trees.md) of the input will be @@ -92,7 +92,7 @@ Here is an example of arpeggio grammar for simple calculator: def calc(): return OneOrMore(expression), EOF Each rule is given in the form of Python function. Python function returns data -structure that maps to PEG expressions. +structure that maps to PEG expressions and state statements. - **Sequence** is represented as Python tuple. - **Ordered choice** is represented as Python list where each element is one @@ -107,6 +107,17 @@ structure that maps to PEG expressions. - **Not predicate** is represented as an instance of `Not` class. - **Literal string match** is represented as string or regular expression given as an instance of `RegExMatch` class. +- **Match state predicate** is represented as an instance of `MatchState` class. + A parsing state passed to the instance must be of `ParsingState` class. + The class checks if the current state is the same as the provided one. +- **Push state command** is represented as an instance of `PushState` class. + A parsing state passed to the instance must be of `ParsingState` class. +- **Pop state predicate** is represented as an instance of `PopState` class. + A parsing state passed to the instance must be of `ParsingState` class. + The class checks if the current state is the same as the provided one if any. +- **State layer wrapper** is represented as an instance of + `StateLayerWrapper` class. This wrapper is used to isolate the state + of a specific group of rules and check its integrity. - **End of string/file** is recognized by the `EOF` special rule. For example, the `calc` language consists of one or more `expression` and @@ -229,8 +240,11 @@ Each grammar rule is given as an assignment where the LHS is the rule name (e.g. - **Optional** expression is specified by `?`operator (e.g. `expression?`) and matches zero or one occurrence of *expression* - **Zero or more** expression is specified by `*` operator (e.g. `(( "*" / - "/" ) factor)*`). + "/" ) factor)*`). Additionally, a separator could be specified + by `%` operator (e.g. `(argument % ',')*`). - **One of more** is specified by `+` operator (e.g. `expression+`). + Additionally, a separator could be specified by `%` operator + (e.g. `(argument % ',')+`). - **Unordered group** is specified by `#` operator (e.g. `sequence#`). It has sense only if applied to the sequence expression. Elements of the sequence are matched in any order. @@ -238,8 +252,79 @@ Each grammar rule is given as an assignment where the LHS is the rule name (e.g. used in the grammar above). - **Not predicate** is specified by `!` operator (e.g. `!expression` - not used in the grammar above). +- **Match actions** are special commands written within `{` and `}` braces + after a rule (e.g. `some_rule{push, add}`). These commands will be executed + after the rule has been matched. Each command can have arguments separated + with the whitespace. If more than one command is specified, the commands + must be separated by comma `,` delimiter. See below for the list + of the provided actions. +- **State matches** are given as state names preceded by `@` operator + (e.g. `@some_state`). A match will succeed only if the current state is + the same as the provided by the operator state. +- **Push state** is used to push a state onto the stack of the states and + make it the current. It is specified using `+@` operator with a state name + after it (e.g. `+@some_state`). Later this state can be matched using + the `@` operator. This rule always succeeds. +- **Pop state** is used to pop a state form the top of the state stack. + It is specified using `-@` operator followed by the state name that should + be removed (e.g. `-@some_state`). If the current state doesn't match + the specified state then the match fails. +- **Wrapping with a state layer** is specified using `@(` and `)` operator and + allows one or more rules to be wrapped with a separate state layer + (e.g. `@( ((LOCAL variable_name{add}) / (variable_name{any} ASSIGN value))* )`). + It allows to store added by actions data in separate layers (for example, + to handle local variables). +- **Config modifiers** are specified before the target expression + using `[` and `]` operator that allows to override config parameters of + the parser with the specified ones (e.g. `[debug=True]expression`). + Parameters are separated from each other with comma + (`,`, e.g. `[skip_whitespace=True, whitespace=' \t']expression`). + Currently, only boolean (`True`, `False`) and string values are supported. + Useful config modifiers: `debug`, `skip_whitespace` (corresponds to + the parser's `skipws`) and `whitespace` (corresponds to the parser's `ws`). - A special rule `EOF` will match end of input string. +A set of basic **match actions** if provided: +- **push** to push a matched token onto the stack. This action always succeeds. +- **pop** to match against the token at the top of the stack corresponding + to the rule and pop that token from the stack. If the matched token and + the token at the top of the stack aren't the same, then the match will fail. +- **pop_front** to match against the token at the bottom of the stack + corresponding to the rule and remove that token from the stack. + If the matched token and the token at the bottom of the stack aren't + the same, then the match will fail. This action can be used to implement + FIFO (First In, First Out) rules. +- **add** to add a matched token to the set of the matched tokens + corresponding to the rule. This action always succeeds and can be used, + for example, to determine local variables. +- **parent add** to add a matched token to the set of the matched tokens of + the parent state layer corresponding to the rule. This action always succeeds + and can be used, for example, to determine local variables. +- **global add** to add a matched token to the set of the matched tokens of + the global state layer corresponding to the rule. This action always succeeds + and can be used, for example, to determine global variables. +- **any** to match any token corresponding to the rule that was previously + added to the set of matched tokens (by **add** action) across all the state + layers. If no token found, then the rule will fail to match. +- **list append** to add a token to the current state layer according to + the rule name. +- **list try remove** to try to remove the last token from the current + state layer if any according to the rule name. +- **list last** to match the last token added to the current state user list + according to the rule name. +- **list longer** to match a token that is longer than the last token added + to the current state user list according to the rule name. +- **parent list last** to match the last token added to the parent state user + list according to the rule name. +- **parent list longer** to match a token that is longer than the last token added + to the parent state user list according to the rule name. +- **other same** to match a token that is inside the repetition expression + only if it's always the same in every repetition. +- **first longer** to match a token only if it's length is longer than + the length of the last token from the parent repetition expression. +- **suppress** to suppress a rule so it wouldn't appear in the resulting + parsing tree. + In the RHS a rule reference is a name of another rule. Parser will try to match another rule at that location. diff --git a/examples/peg_peg/peg.peg b/examples/peg_peg/peg.peg index 426fda38..866ae0db 100644 --- a/examples/peg_peg/peg.peg +++ b/examples/peg_peg/peg.peg @@ -1,9 +1,9 @@ peggrammar <- rule+ EOF; rule <- rule_name LEFT_ARROW ordered_choice ';'; ordered_choice <- sequence (SLASH sequence)*; - sequence <- prefix+; - prefix <- (AND/NOT)? sufix; - sufix <- expression (QUESTION/STAR/PLUS)?; + sequence <- full_expression+; + full_expression <- (AND/NOT)? repeated_expression; + repeated_expression <- expression (QUESTION/STAR/PLUS)?; expression <- regex / rule_crossref / (OPEN ordered_choice CLOSE) / str_match; diff --git a/examples/peg_peg/peg_peg.py b/examples/peg_peg/peg_peg.py index 71cc9c46..696254f5 100644 --- a/examples/peg_peg/peg_peg.py +++ b/examples/peg_peg/peg_peg.py @@ -56,6 +56,6 @@ def main(debug=False): parser.parser_model = parser_model parser.parse(peg_grammar) + if __name__ == '__main__': main(debug=True) -