Optimize BPMN parser XML lookups

SpiffWorkflow/bpmn/parser/BpmnParser.py

@@ -47,6 +47,7 @@
 )
 from SpiffWorkflow.bpmn.specs.event_definitions.simple import NoneEventDefinition
 from SpiffWorkflow.bpmn.specs.event_definitions.timer import TimerEventDefinition
+from SpiffWorkflow.bpmn.specs.event_definitions.message import CorrelationProperty
 from SpiffWorkflow.bpmn.specs.mixins.subworkflow_task import SubWorkflowTask as SubWorkflowTaskMixin
 from SpiffWorkflow.bpmn.specs.mixins.events.start_event import StartEvent as StartEventMixin
 
@@ -161,8 +162,14 @@ def __init__(self, namespaces=None, validator=None, spec_descriptions=SPEC_DESCR
         self.collaborations = {}
         self.process_dependencies = set()
         self.messages = {}
+        self.signals = {}
+        self.errors = {}
+        self.escalations = {}
         self.correlations = {}
+        self.message_correlations = {}
         self.data_stores = {}
+        self.document_lanes = {}
+        self.document_positions = {}
 
     def _get_parser_class(self, tag):
         if tag in self.OVERRIDE_PARSER_CLASSES:
@@ -231,17 +238,55 @@ def add_bpmn_xml(self, bpmn, filename=None):
         # the parser instances, which need to know about the data stores to
         # resolve data references.
         self._add_data_stores(bpmn)
+        self._add_diagram_indexes(bpmn)
 
         self._add_processes(bpmn, filename)
         self._add_collaborations(bpmn)
         self._add_messages(bpmn)
+        self._add_signals(bpmn)
+        self._add_errors(bpmn)
+        self._add_escalations(bpmn)
         self._add_correlations(bpmn)
 
     def _add_processes(self, bpmn, filename=None):
         for process in bpmn.xpath('.//bpmn:process', namespaces=self.namespaces):
             self._find_dependencies(process)
             self.create_parser(process, filename)
 
+    def _document_key(self, bpmn):
+        root = bpmn.getroot() if hasattr(bpmn, 'getroot') else bpmn
+        return id(root)
+
+    def _add_diagram_indexes(self, bpmn):
+        document_key = self._document_key(bpmn)
+        if document_key in self.document_lanes:
+            return
+
+        lanes = {}
+        for flow_node_ref in bpmn.xpath('.//bpmn:flowNodeRef', namespaces=self.namespaces):
+            flow_node_id = flow_node_ref.text
+            if flow_node_id is not None:
+                lanes[flow_node_id] = flow_node_ref.getparent().get('name')
+
+        positions = {}
+        for shape in bpmn.xpath('.//bpmndi:BPMNShape[@bpmnElement]', namespaces=self.namespaces):
+            node_id = shape.get('bpmnElement')
+            bounds = first(shape.xpath('.//dc:Bounds', namespaces=self.namespaces))
+            if node_id is not None and bounds is not None:
+                positions[node_id] = {
+                    'x': float(bounds.get('x', 0)),
+                    'y': float(bounds.get('y', 0)),
+                }
+
+        self.document_lanes[document_key] = lanes
+        self.document_positions[document_key] = positions
+
+    def get_document_lanes(self, root):
+        return self.document_lanes.get(id(root), {})
+
+    def get_document_positions(self, root):
+        return self.document_positions.get(id(root), {})
+
     def _add_collaborations(self, bpmn):
         collaboration = first(bpmn.xpath('.//bpmn:collaboration', namespaces=self.namespaces))
         if collaboration is not None:
@@ -257,7 +302,41 @@ def _add_messages(self, bpmn):
                 )
             self.messages[message.attrib.get("id")] = message.attrib.get("name")
 
+    def _add_signals(self, bpmn):
+        for signal in bpmn.xpath('.//bpmn:signal', namespaces=self.namespaces):
+            signal_identifier = signal.attrib.get("id")
+            if signal_identifier is None:
+                raise ValidationException("Signal identifier is missing from bpmn xml")
+            self.signals[signal_identifier] = signal.attrib.get("name")
+
+    def _add_errors(self, bpmn):
+        for error in bpmn.xpath('.//bpmn:error', namespaces=self.namespaces):
+            error_identifier = error.attrib.get("id")
+            if error_identifier is None:
+                raise ValidationException("Error identifier is missing from bpmn xml")
+            self.errors[error_identifier] = {
+                "name": error.attrib.get("name"),
+                "error_code": error.attrib.get("errorCode"),
+            }
+
+    def _add_escalations(self, bpmn):
+        for escalation in bpmn.xpath('.//bpmn:escalation', namespaces=self.namespaces):
+            escalation_identifier = escalation.attrib.get("id")
+            if escalation_identifier is None:
+                raise ValidationException("Escalation identifier is missing from bpmn xml")
+            self.escalations[escalation_identifier] = {
+                "name": escalation.attrib.get("name"),
+                "escalation_code": escalation.attrib.get("escalationCode"),
+            }
+
     def _add_correlations(self, bpmn):
+        property_id_to_used_by = {}
+        for correlation_key in bpmn.xpath('.//bpmn:correlationKey', namespaces=self.namespaces):
+            used_by = correlation_key.attrib.get("name")
+            for property_ref in correlation_key.xpath('./bpmn:correlationPropertyRef', namespaces=self.namespaces):
+                if property_ref.text is not None:
+                    property_id_to_used_by.setdefault(property_ref.text, []).append(used_by)
+
         for correlation in bpmn.xpath('.//bpmn:correlationProperty', namespaces=self.namespaces):
             correlation_identifier = correlation.attrib.get("id")
             if correlation_identifier is None:
@@ -279,6 +358,14 @@ def _add_correlations(self, bpmn):
                 expression = children[0].text if len(children) > 0 else None
                 retrieval_expressions.append({"messageRef": message_model_identifier,
                                              "expression": expression})
+                if expression is not None:
+                    self.message_correlations.setdefault(message_model_identifier, []).append(
+                        CorrelationProperty(
+                            correlation_identifier,
+                            expression,
+                            property_id_to_used_by.get(correlation_identifier, []),
+                        )
+                    )
             self.correlations[correlation_identifier] = {
                 "name": correlation.attrib.get("name"),
                 "retrieval_expressions": retrieval_expressions

SpiffWorkflow/bpmn/parser/ProcessParser.py

@@ -41,20 +41,64 @@ def __init__(self, p, node, nsmap, data_stores, filename=None, lane=None):
         :param filename: the source BPMN filename (optional)
         :param lane: the lane of a subprocess (optional)
         """
-        super().__init__(node, nsmap, filename=filename, lane=lane)
         self.parser = p
+        super().__init__(node, nsmap, filename=filename, lane=lane)
         self.lane = lane
         self.spec = None
         self.process_executable = node.get('isExecutable', 'true') == 'true'
         self.data_stores = data_stores
         self.parent = None
+        self.document_root = node.getroottree().getroot()
+        self.nodes_by_id = {}
+        self.outgoing_sequence_flows = {}
+        self.boundary_events_by_attached = {}
+        self.data_object_references = {}
+        self.data_store_references = {}
+        self.positions_by_node_id = self.parser.get_document_positions(self.document_root)
+        self.lanes_by_node_id = self.parser.get_document_lanes(self.document_root)
+        self._index_process_nodes()
 
     def get_name(self):
         """
         Returns the process name (or ID, if no name is included in the file)
         """
         return self.node.get('name', default=self.bpmn_id)
 
+    def _index_process_nodes(self):
+        for node in self.xpath('.//*[@id]'):
+            node_id = node.get('id')
+            if node_id is not None:
+                self.nodes_by_id[node_id] = node
+            if node.tag.endswith('sequenceFlow'):
+                self.outgoing_sequence_flows.setdefault(node.get('sourceRef'), []).append(node)
+            elif node.tag.endswith('boundaryEvent'):
+                self.boundary_events_by_attached.setdefault(node.get('attachedToRef'), []).append(node)
+            elif node.tag.endswith('dataObjectReference'):
+                self.data_object_references[node_id] = node
+            elif node.tag.endswith('dataStoreReference'):
+                self.data_store_references[node_id] = node
+
+    def get_boundary_events(self, attached_to_ref):
+        return self.boundary_events_by_attached.get(attached_to_ref, [])
+
+    def get_outgoing_sequence_flows(self, source_ref):
+        return self.outgoing_sequence_flows.get(source_ref, [])
+
+    def get_node_by_id(self, node_id):
+        return self.nodes_by_id.get(node_id)
+
+    def get_data_object_reference(self, reference_id):
+        return self.data_object_references.get(reference_id)
+
+    def get_data_store_reference(self, reference_id):
+        return self.data_store_references.get(reference_id)
+
+    def get_position(self, node_id):
+        return self.positions_by_node_id.get(node_id, {'x': 0.0, 'y': 0.0})
+
+    def get_lane_name(self, node_id):
+        return self.lanes_by_node_id.get(node_id)
+
     def has_lanes(self) -> bool:
         """Returns true if this process has one or more named lanes """
         elements = self.xpath("//bpmn:lane")
@@ -67,7 +111,10 @@ def start_messages(self):
         """ This returns a list of message names that would cause this
             process to start. """
         message_names = []
-        messages = self.xpath("//bpmn:message")
+        messages_by_id = {
+            message.attrib.get('id'): message
+            for message in self.xpath("//bpmn:message")
+        }
         message_event_definitions = self.xpath(
             "//bpmn:startEvent/bpmn:messageEventDefinition")
         for message_event_definition in message_event_definitions:
@@ -80,7 +127,7 @@ def start_messages(self):
                     f"Could not find messageRef from message event definition: {med_id}"
                 )
             # Convert the id into a Message Name
-            message_name = next((m for m in messages if m.attrib.get('id') == message_model_identifier), None)
+            message_name = messages_by_id.get(message_model_identifier)
             message_names.append(message_name.attrib.get('name'))
 
         return message_names

SpiffWorkflow/bpmn/parser/TaskParser.py

@@ -59,9 +59,9 @@ def __init__(self, process_parser, spec_class, node, nsmap=None, lane=None):
           extending the TaskParser.
         :param node: the XML node for this task
         """
-        super().__init__(node, nsmap, filename=process_parser.filename, lane=lane)
         self.process_parser = process_parser
         self.spec_class = spec_class
+        super().__init__(node, nsmap, filename=process_parser.filename, lane=lane)
         self.spec = self.process_parser.spec
 
     def _copy_task_attrs(self, original, loop_characteristics=None):
@@ -199,19 +199,18 @@ def parse_node(self):
             if len(mi_loop_characteristics) > 0:
                 self._add_multiinstance_task(mi_loop_characteristics[0])
 
-            boundary_event_nodes = self.doc_xpath('.//bpmn:boundaryEvent[@attachedToRef="%s"]' % self.bpmn_id)
+            boundary_event_nodes = self.process_parser.get_boundary_events(self.bpmn_id)
             if boundary_event_nodes:
                 parent = self._add_boundary_event(boundary_event_nodes)
 
             children = []
-            outgoing = self.doc_xpath('.//bpmn:sequenceFlow[@sourceRef="%s"]' % self.bpmn_id)
+            outgoing = self.process_parser.get_outgoing_sequence_flows(self.bpmn_id)
             if len(outgoing) > 1 and not self.handles_multiple_outgoing():
                 self.raise_validation_exception('Multiple outgoing flows are not supported for tasks of type')
             for sequence_flow in outgoing:
                 target_ref = sequence_flow.get('targetRef')
-                try:
-                    target_node = one(self.doc_xpath('.//bpmn:*[@id="%s"]'% target_ref))
-                except Exception:
+                target_node = self.process_parser.get_node_by_id(target_ref)
+                if target_node is None:
                     self.raise_validation_exception('When looking for a task spec, we found two items, '
                         'perhaps a form has the same ID? (%s)' % target_ref)
 
@@ -268,4 +267,3 @@ def handles_multiple_outgoing(self):
         outgoing sequence flows.
         """
         return False
-

SpiffWorkflow/bpmn/parser/event_parsers.py

@@ -21,7 +21,7 @@
 
 from .ValidationException import ValidationException
 from .TaskParser import TaskParser
-from .util import first, one
+from .util import first
 
 from SpiffWorkflow.bpmn.specs.event_definitions.simple import (
     NoneEventDefinition,
@@ -38,10 +38,7 @@
     ErrorEventDefinition,
     EscalationEventDefinition
 )
-from SpiffWorkflow.bpmn.specs.event_definitions.message import (
-    MessageEventDefinition,
-    CorrelationProperty
-)
+from SpiffWorkflow.bpmn.specs.event_definitions.message import MessageEventDefinition
 from SpiffWorkflow.bpmn.specs.event_definitions.multiple import MultipleEventDefinition
 from SpiffWorkflow.bpmn.specs.event_definitions.conditional import ConditionalEventDefinition
 
@@ -89,11 +86,10 @@ def parse_error_event(self, error_event):
         """Parse the errorEventDefinition node and return an instance of ErrorEventDefinition."""
         error_ref = error_event.get('errorRef')
         if error_ref:
-            try:
-                error = one(self.doc_xpath('.//bpmn:error[@id="%s"]' % error_ref))
-            except Exception:
+            error = self.process_parser.parser.errors.get(error_ref)
+            if error is None:
                 self.raise_validation_exception('Expected an error node', node=error_event)
-            error_code = error.get('errorCode')
+            error_code = error.get('error_code')
             name = error.get('name')
         else:
             name, error_code = 'None Error Event', None
@@ -104,11 +100,10 @@ def parse_escalation_event(self, escalation_event):
 
         escalation_ref = escalation_event.get('escalationRef')
         if escalation_ref:
-            try:
-                escalation = one(self.doc_xpath('.//bpmn:escalation[@id="%s"]' % escalation_ref))
-            except Exception:
+            escalation = self.process_parser.parser.escalations.get(escalation_ref)
+            if escalation is None:
                 self.raise_validation_exception('Expected an Escalation node', node=escalation_event)
-            escalation_code = escalation.get('escalationCode')
+            escalation_code = escalation.get('escalation_code')
             name = escalation.get('name')
         else:
             name, escalation_code = 'None Escalation Event', None
@@ -118,11 +113,10 @@ def parse_message_event(self, message_event):
 
         message_ref = message_event.get('messageRef')
         if message_ref is not None:
-            try:
-                message = one(self.doc_xpath('.//bpmn:message[@id="%s"]' % message_ref))
-            except Exception:
+            message = self.process_parser.parser.messages.get(message_ref)
+            if message is None:
                 self.raise_validation_exception('Expected a Message node', node=message_event)
-            name = message.get('name')
+            name = message
             description = self.get_event_description(message_event)
             correlations = self.get_message_correlations(message_ref)
         else:
@@ -136,11 +130,10 @@ def parse_signal_event(self, signal_event):
 
         signal_ref = signal_event.get('signalRef')
         if signal_ref:
-            try:
-                signal = one(self.doc_xpath('.//bpmn:signal[@id="%s"]' % signal_ref))
-            except Exception:
+            signal = self.process_parser.parser.signals.get(signal_ref)
+            if signal is None:
                 self.raise_validation_exception('Expected a Signal node', node=signal_event)
-            name = signal.get('name')
+            name = signal
         else:
             name = signal_event.getparent().get('name')
         return SignalEventDefinition(name, description=self.get_event_description(signal_event))
@@ -168,17 +161,7 @@ def parse_timer_event(self, event):
             raise ValidationException("Time Specification Error. " + str(e), node=self.node, file_name=self.filename)
 
     def get_message_correlations(self, message_ref):
-
-        correlations = []
-        for correlation in self.doc_xpath(f".//bpmn:correlationPropertyRetrievalExpression[@messageRef='{message_ref}']"):
-            key = correlation.getparent().get('id')
-            children = correlation.getchildren()
-            expression = children[0].text if len(children) > 0 else None
-            used_by = [ e.getparent().get('name') for e in
-                self.doc_xpath(f".//bpmn:correlationKey/bpmn:correlationPropertyRef[text()='{key}']") ]
-            if key is not None and expression is not None:
-                correlations.append(CorrelationProperty(key, expression, used_by))
-        return correlations
+        return self.process_parser.parser.message_correlations.get(message_ref, [])
 
     def _create_task(self, event_definition, cancel_activity=None, parallel=None):