# iterparse -- event-driven parsing DEF __ITERPARSE_CHUNK_SIZE = 32768 cdef class iterparse: u"""iterparse(self, source, events=("end",), tag=None, \ attribute_defaults=False, dtd_validation=False, \ load_dtd=False, no_network=True, remove_blank_text=False, \ remove_comments=False, remove_pis=False, encoding=None, \ html=False, recover=None, huge_tree=False, schema=None) Incremental parser. Parses XML into a tree and generates tuples (event, element) in a SAX-like fashion. ``event`` is any of 'start', 'end', 'start-ns', 'end-ns'. For 'start' and 'end', ``element`` is the Element that the parser just found opening or closing. For 'start-ns', it is a tuple (prefix, URI) of a new namespace declaration. For 'end-ns', it is simply None. Note that all start and end events are guaranteed to be properly nested. The keyword argument ``events`` specifies a sequence of event type names that should be generated. By default, only 'end' events will be generated. The additional ``tag`` argument restricts the 'start' and 'end' events to those elements that match the given tag. The ``tag`` argument can also be a sequence of tags to allow matching more than one tag. By default, events are generated for all elements. Note that the 'start-ns' and 'end-ns' events are not impacted by this restriction. The other keyword arguments in the constructor are mainly based on the libxml2 parser configuration. A DTD will also be loaded if validation or attribute default values are requested. Available boolean keyword arguments: - attribute_defaults: read default attributes from DTD - dtd_validation: validate (if DTD is available) - load_dtd: use DTD for parsing - no_network: prevent network access for related files - remove_blank_text: discard blank text nodes - remove_comments: discard comments - remove_pis: discard processing instructions - strip_cdata: replace CDATA sections by normal text content (default: True) - compact: safe memory for short text content (default: True) - resolve_entities: replace entities by their text value (default: True) - huge_tree: disable security restrictions and support very deep trees and very long text content (only affects libxml2 2.7+) - html: parse input as HTML (default: XML) - recover: try hard to parse through broken input (default: True for HTML, False otherwise) Other keyword arguments: - encoding: override the document encoding - schema: an XMLSchema to validate against """ cdef _FeedParser _parser cdef object _tag cdef object _events cdef readonly object root cdef object _source cdef object _filename cdef object _error cdef bint _close_source_after_read def __init__(self, source, events=(u"end",), *, tag=None, attribute_defaults=False, dtd_validation=False, load_dtd=False, no_network=True, remove_blank_text=False, compact=True, resolve_entities=True, remove_comments=False, remove_pis=False, strip_cdata=True, encoding=None, html=False, recover=None, huge_tree=False, collect_ids=True, XMLSchema schema=None): if not hasattr(source, 'read'): source = _getFSPathOrObject(source) self._filename = source if python.IS_PYTHON2: source = _encodeFilename(source) source = open(source, 'rb') self._close_source_after_read = True else: self._filename = _getFilenameForFile(source) self._close_source_after_read = False if recover is None: recover = html if html: # make sure we're not looking for namespaces events = [event for event in events if event not in ('start-ns', 'end-ns')] parser = HTMLPullParser( events, tag=tag, recover=recover, base_url=self._filename, encoding=encoding, remove_blank_text=remove_blank_text, remove_comments=remove_comments, remove_pis=remove_pis, strip_cdata=strip_cdata, no_network=no_network, target=None, # TODO schema=schema, compact=compact) else: parser = XMLPullParser( events, tag=tag, recover=recover, base_url=self._filename, encoding=encoding, attribute_defaults=attribute_defaults, dtd_validation=dtd_validation, load_dtd=load_dtd, no_network=no_network, schema=schema, huge_tree=huge_tree, remove_blank_text=remove_blank_text, resolve_entities=resolve_entities, remove_comments=remove_comments, remove_pis=remove_pis, strip_cdata=strip_cdata, collect_ids=True, target=None, # TODO compact=compact) self._events = parser.read_events() self._parser = parser self._source = source @property def error_log(self): """The error log of the last (or current) parser run. """ return self._parser.feed_error_log @property def resolvers(self): """The custom resolver registry of the last (or current) parser run. """ return self._parser.resolvers @property def version(self): """The version of the underlying XML parser.""" return self._parser.version def set_element_class_lookup(self, ElementClassLookup lookup = None): u"""set_element_class_lookup(self, lookup = None) Set a lookup scheme for element classes generated from this parser. Reset it by passing None or nothing. """ self._parser.set_element_class_lookup(lookup) def makeelement(self, _tag, attrib=None, nsmap=None, **_extra): u"""makeelement(self, _tag, attrib=None, nsmap=None, **_extra) Creates a new element associated with this parser. """ self._parser.makeelement( _tag, attrib=None, nsmap=None, **_extra) @cython.final cdef _close_source(self): if self._source is None: return if not self._close_source_after_read: self._source = None return try: close = self._source.close except AttributeError: close = None finally: self._source = None if close is not None: close() def __iter__(self): return self def __next__(self): try: return next(self._events) except StopIteration: pass context = <_SaxParserContext>self._parser._getPushParserContext() if self._source is not None: done = False while not done: try: done = self._read_more_events(context) return next(self._events) except StopIteration: pass # no events yet except Exception as e: self._error = e self._close_source() try: return next(self._events) except StopIteration: break # nothing left to read or return if self._error is not None: error = self._error self._error = None raise error if (context._validator is not None and not context._validator.isvalid()): _raiseParseError(context._c_ctxt, self._filename, context._error_log) # no errors => all done raise StopIteration @cython.final cdef bint _read_more_events(self, _SaxParserContext context) except -123: data = self._source.read(__ITERPARSE_CHUNK_SIZE) if not isinstance(data, bytes): self._close_source() raise TypeError("reading file objects must return bytes objects") if not data: try: self.root = self._parser.close() finally: self._close_source() return True self._parser.feed(data) return False cdef enum _IterwalkSkipStates: IWSKIP_NEXT_IS_START IWSKIP_SKIP_NEXT IWSKIP_CAN_SKIP IWSKIP_CANNOT_SKIP cdef class iterwalk: u"""iterwalk(self, element_or_tree, events=("end",), tag=None) A tree walker that generates events from an existing tree as if it was parsing XML data with ``iterparse()``. Just as for ``iterparse()``, the ``tag`` argument can be a single tag or a sequence of tags. After receiving a 'start' or 'start-ns' event, the children and descendants of the current element can be excluded from iteration by calling the ``skip_subtree()`` method. """ cdef _MultiTagMatcher _matcher cdef list _node_stack cdef list _events cdef object _pop_event cdef object _include_siblings cdef int _index cdef int _event_filter cdef _IterwalkSkipStates _skip_state def __init__(self, element_or_tree, events=(u"end",), tag=None): cdef _Element root cdef int ns_count root = _rootNodeOrRaise(element_or_tree) self._event_filter = _buildParseEventFilter(events) if tag is None or tag == '*': self._matcher = None else: self._matcher = _MultiTagMatcher.__new__(_MultiTagMatcher, tag) self._node_stack = [] self._events = [] self._pop_event = self._events.pop self._skip_state = IWSKIP_CANNOT_SKIP # ignore all skip requests by default if self._event_filter: self._index = 0 if self._matcher is not None and self._event_filter & PARSE_EVENT_FILTER_START: self._matcher.cacheTags(root._doc) # When processing an ElementTree, add events for the preceding comments/PIs. if self._event_filter & (PARSE_EVENT_FILTER_COMMENT | PARSE_EVENT_FILTER_PI): if isinstance(element_or_tree, _ElementTree): self._include_siblings = root for elem in list(root.itersiblings(preceding=True))[::-1]: if self._event_filter & PARSE_EVENT_FILTER_COMMENT and elem.tag is Comment: self._events.append((u'comment', elem)) elif self._event_filter & PARSE_EVENT_FILTER_PI and elem.tag is PI: self._events.append((u'pi', elem)) ns_count = self._start_node(root) self._node_stack.append( (root, ns_count) ) else: self._index = -1 def __iter__(self): return self def __next__(self): cdef xmlNode* c_child cdef _Element node cdef _Element next_node cdef int ns_count = 0 if self._events: return self._next_event() if self._matcher is not None and self._index >= 0: node = self._node_stack[self._index][0] self._matcher.cacheTags(node._doc) # find next node while self._index >= 0: node = self._node_stack[self._index][0] if self._skip_state == IWSKIP_SKIP_NEXT: c_child = NULL else: c_child = self._process_non_elements( node._doc, _findChildForwards(node._c_node, 0)) self._skip_state = IWSKIP_CANNOT_SKIP while c_child is NULL: # back off through parents self._index -= 1 node = self._end_node() if self._index < 0: break c_child = self._process_non_elements( node._doc, _nextElement(node._c_node)) if c_child is not NULL: next_node = _elementFactory(node._doc, c_child) if self._event_filter & (PARSE_EVENT_FILTER_START | PARSE_EVENT_FILTER_START_NS): ns_count = self._start_node(next_node) elif self._event_filter & PARSE_EVENT_FILTER_END_NS: ns_count = _countNsDefs(next_node._c_node) self._node_stack.append( (next_node, ns_count) ) self._index += 1 if self._events: return self._next_event() if self._include_siblings is not None: node, self._include_siblings = self._include_siblings, None self._process_non_elements(node._doc, _nextElement(node._c_node)) if self._events: return self._next_event() raise StopIteration @cython.final cdef xmlNode* _process_non_elements(self, _Document doc, xmlNode* c_node): while c_node is not NULL and c_node.type != tree.XML_ELEMENT_NODE: if c_node.type == tree.XML_COMMENT_NODE: if self._event_filter & PARSE_EVENT_FILTER_COMMENT: self._events.append( (u"comment", _elementFactory(doc, c_node))) c_node = _nextElement(c_node) elif c_node.type == tree.XML_PI_NODE: if self._event_filter & PARSE_EVENT_FILTER_PI: self._events.append( (u"pi", _elementFactory(doc, c_node))) c_node = _nextElement(c_node) else: break return c_node @cython.final cdef _next_event(self): if self._skip_state == IWSKIP_NEXT_IS_START: if self._events[0][0] in (u'start', u'start-ns'): self._skip_state = IWSKIP_CAN_SKIP return self._pop_event(0) def skip_subtree(self): """Prevent descending into the current subtree. Instead, the next returned event will be the 'end' event of the current element (if included), ignoring any children or descendants. This has no effect right after an 'end' or 'end-ns' event. """ if self._skip_state == IWSKIP_CAN_SKIP: self._skip_state = IWSKIP_SKIP_NEXT @cython.final cdef int _start_node(self, _Element node) except -1: cdef int ns_count if self._event_filter & PARSE_EVENT_FILTER_START_NS: ns_count = _appendStartNsEvents(node._c_node, self._events) if self._events: self._skip_state = IWSKIP_NEXT_IS_START elif self._event_filter & PARSE_EVENT_FILTER_END_NS: ns_count = _countNsDefs(node._c_node) else: ns_count = 0 if self._event_filter & PARSE_EVENT_FILTER_START: if self._matcher is None or self._matcher.matches(node._c_node): self._events.append( (u"start", node) ) self._skip_state = IWSKIP_NEXT_IS_START return ns_count @cython.final cdef _Element _end_node(self): cdef _Element node cdef int i, ns_count node, ns_count = self._node_stack.pop() if self._event_filter & PARSE_EVENT_FILTER_END: if self._matcher is None or self._matcher.matches(node._c_node): self._events.append( (u"end", node) ) if self._event_filter & PARSE_EVENT_FILTER_END_NS and ns_count: event = (u"end-ns", None) for i in range(ns_count): self._events.append(event) return node cdef int _countNsDefs(xmlNode* c_node): cdef xmlNs* c_ns cdef int count count = 0 c_ns = c_node.nsDef while c_ns is not NULL: count += (c_ns.href is not NULL) c_ns = c_ns.next return count cdef int _appendStartNsEvents(xmlNode* c_node, list event_list) except -1: cdef xmlNs* c_ns cdef int count count = 0 c_ns = c_node.nsDef while c_ns is not NULL: if c_ns.href: ns_tuple = (funicodeOrEmpty(c_ns.prefix), funicode(c_ns.href)) event_list.append( (u"start-ns", ns_tuple) ) count += 1 c_ns = c_ns.next return count