# Proxy functions and low level node allocation stuff # Proxies represent elements, their reference is stored in the C # structure of the respective node to avoid multiple instantiation of # the Python class. @cython.linetrace(False) @cython.profile(False) cdef inline _Element getProxy(xmlNode* c_node): u"""Get a proxy for a given node. """ #print "getProxy for:", <int>c_node if c_node is not NULL and c_node._private is not NULL: return <_Element>c_node._private else: return None @cython.linetrace(False) @cython.profile(False) cdef inline bint hasProxy(xmlNode* c_node): if c_node._private is NULL: return False return True @cython.linetrace(False) @cython.profile(False) cdef inline int _registerProxy(_Element proxy, _Document doc, xmlNode* c_node) except -1: u"""Register a proxy and type for the node it's proxying for. """ #print "registering for:", <int>proxy._c_node assert not hasProxy(c_node), u"double registering proxy!" proxy._doc = doc proxy._c_node = c_node c_node._private = <void*>proxy return 0 @cython.linetrace(False) @cython.profile(False) cdef inline int _unregisterProxy(_Element proxy) except -1: u"""Unregister a proxy for the node it's proxying for. """ cdef xmlNode* c_node = proxy._c_node assert c_node._private is <void*>proxy, u"Tried to unregister unknown proxy" c_node._private = NULL return 0 ################################################################################ # temporarily make a node the root node of its document cdef xmlDoc* _fakeRootDoc(xmlDoc* c_base_doc, xmlNode* c_node) except NULL: return _plainFakeRootDoc(c_base_doc, c_node, 1) cdef xmlDoc* _plainFakeRootDoc(xmlDoc* c_base_doc, xmlNode* c_node, bint with_siblings) except NULL: # build a temporary document that has the given node as root node # note that copy and original must not be modified during its lifetime!! # always call _destroyFakeDoc() after use! cdef xmlNode* c_child cdef xmlNode* c_root cdef xmlNode* c_new_root cdef xmlDoc* c_doc if with_siblings or (c_node.prev is NULL and c_node.next is NULL): c_root = tree.xmlDocGetRootElement(c_base_doc) if c_root is c_node: # already the root node, no siblings return c_base_doc c_doc = _copyDoc(c_base_doc, 0) # non recursive! c_new_root = tree.xmlDocCopyNode(c_node, c_doc, 2) # non recursive! tree.xmlDocSetRootElement(c_doc, c_new_root) _copyParentNamespaces(c_node, c_new_root) c_new_root.children = c_node.children c_new_root.last = c_node.last c_new_root.next = c_new_root.prev = NULL # store original node c_doc._private = c_node # divert parent pointers of children c_child = c_new_root.children while c_child is not NULL: c_child.parent = c_new_root c_child = c_child.next c_doc.children = c_new_root return c_doc cdef void _destroyFakeDoc(xmlDoc* c_base_doc, xmlDoc* c_doc): # delete a temporary document cdef xmlNode* c_child cdef xmlNode* c_parent cdef xmlNode* c_root if c_doc is c_base_doc: return c_root = tree.xmlDocGetRootElement(c_doc) # restore parent pointers of children c_parent = <xmlNode*>c_doc._private c_child = c_root.children while c_child is not NULL: c_child.parent = c_parent c_child = c_child.next # prevent recursive removal of children c_root.children = c_root.last = NULL tree.xmlFreeDoc(c_doc) cdef _Element _fakeDocElementFactory(_Document doc, xmlNode* c_element): u"""Special element factory for cases where we need to create a fake root document, but still need to instantiate arbitrary nodes from it. If we instantiate the fake root node, things will turn bad when it's destroyed. Instead, if we are asked to instantiate the fake root node, we instantiate the original node instead. """ if c_element.doc is not doc._c_doc: if c_element.doc._private is not NULL: if c_element is c_element.doc.children: c_element = <xmlNode*>c_element.doc._private #assert c_element.type == tree.XML_ELEMENT_NODE return _elementFactory(doc, c_element) ################################################################################ # support for freeing tree elements when proxy objects are destroyed cdef int attemptDeallocation(xmlNode* c_node): u"""Attempt deallocation of c_node (or higher up in tree). """ cdef xmlNode* c_top # could be we actually aren't referring to the tree at all if c_node is NULL: #print "not freeing, node is NULL" return 0 c_top = getDeallocationTop(c_node) if c_top is not NULL: #print "freeing:", c_top.name _removeText(c_top.next) # tail tree.xmlFreeNode(c_top) return 1 return 0 cdef xmlNode* getDeallocationTop(xmlNode* c_node): u"""Return the top of the tree that can be deallocated, or NULL. """ cdef xmlNode* c_next #print "trying to do deallocating:", c_node.type if hasProxy(c_node): #print "Not freeing: proxies still exist" return NULL while c_node.parent is not NULL: c_node = c_node.parent #print "checking:", c_current.type if c_node.type == tree.XML_DOCUMENT_NODE or \ c_node.type == tree.XML_HTML_DOCUMENT_NODE: #print "not freeing: still in doc" return NULL # if we're still attached to the document, don't deallocate if hasProxy(c_node): #print "Not freeing: proxies still exist" return NULL # see whether we have children to deallocate if not canDeallocateChildNodes(c_node): return NULL # see whether we have siblings to deallocate c_next = c_node.prev while c_next: if _isElement(c_next): if hasProxy(c_next) or not canDeallocateChildNodes(c_next): return NULL c_next = c_next.prev c_next = c_node.next while c_next: if _isElement(c_next): if hasProxy(c_next) or not canDeallocateChildNodes(c_next): return NULL c_next = c_next.next return c_node cdef int canDeallocateChildNodes(xmlNode* c_parent): cdef xmlNode* c_node c_node = c_parent.children tree.BEGIN_FOR_EACH_ELEMENT_FROM(c_parent, c_node, 1) if hasProxy(c_node): return 0 tree.END_FOR_EACH_ELEMENT_FROM(c_node) return 1 ################################################################################ # fix _Document references and namespaces when a node changes documents cdef void _copyParentNamespaces(xmlNode* c_from_node, xmlNode* c_to_node) nogil: u"""Copy the namespaces of all ancestors of c_from_node to c_to_node. """ cdef xmlNode* c_parent cdef xmlNs* c_ns cdef xmlNs* c_new_ns cdef int prefix_known c_parent = c_from_node.parent while c_parent and (tree._isElementOrXInclude(c_parent) or c_parent.type == tree.XML_DOCUMENT_NODE): c_new_ns = c_parent.nsDef while c_new_ns: # libxml2 will check if the prefix is already defined tree.xmlNewNs(c_to_node, c_new_ns.href, c_new_ns.prefix) c_new_ns = c_new_ns.next c_parent = c_parent.parent ctypedef struct _ns_update_map: xmlNs* old xmlNs* new ctypedef struct _nscache: _ns_update_map* ns_map size_t size size_t last cdef int _growNsCache(_nscache* c_ns_cache) except -1: cdef _ns_update_map* ns_map_ptr if c_ns_cache.size == 0: c_ns_cache.size = 20 else: c_ns_cache.size *= 2 ns_map_ptr = <_ns_update_map*> python.lxml_realloc( c_ns_cache.ns_map, c_ns_cache.size, sizeof(_ns_update_map)) if not ns_map_ptr: python.lxml_free(c_ns_cache.ns_map) c_ns_cache.ns_map = NULL raise MemoryError() c_ns_cache.ns_map = ns_map_ptr return 0 cdef inline int _appendToNsCache(_nscache* c_ns_cache, xmlNs* c_old_ns, xmlNs* c_new_ns) except -1: if c_ns_cache.last >= c_ns_cache.size: _growNsCache(c_ns_cache) c_ns_cache.ns_map[c_ns_cache.last] = _ns_update_map(old=c_old_ns, new=c_new_ns) c_ns_cache.last += 1 cdef int _stripRedundantNamespaceDeclarations(xmlNode* c_element, _nscache* c_ns_cache, xmlNs** c_del_ns_list) except -1: u"""Removes namespace declarations from an element that are already defined in its parents. Does not free the xmlNs's, just prepends them to the c_del_ns_list. """ cdef xmlNs* c_ns cdef xmlNs* c_ns_next cdef xmlNs** c_nsdef # use a xmlNs** to handle assignments to "c_element.nsDef" correctly c_nsdef = &c_element.nsDef while c_nsdef[0] is not NULL: c_ns = tree.xmlSearchNsByHref( c_element.doc, c_element.parent, c_nsdef[0].href) if c_ns is NULL: # new namespace href => keep and cache the ns declaration _appendToNsCache(c_ns_cache, c_nsdef[0], c_nsdef[0]) c_nsdef = &c_nsdef[0].next else: # known namespace href => cache mapping and strip old ns _appendToNsCache(c_ns_cache, c_nsdef[0], c_ns) # cut out c_nsdef.next and prepend it to garbage chain c_ns_next = c_nsdef[0].next c_nsdef[0].next = c_del_ns_list[0] c_del_ns_list[0] = c_nsdef[0] c_nsdef[0] = c_ns_next return 0 cdef void _cleanUpFromNamespaceAdaptation(xmlNode* c_start_node, _nscache* c_ns_cache, xmlNs* c_del_ns_list): # Try to recover from exceptions with really bad timing. We were in the middle # of ripping out xmlNS-es and likely ran out of memory. Try to fix up the tree # by re-adding the original xmlNs declarations (which might still be used in some # places). if c_ns_cache.ns_map: python.lxml_free(c_ns_cache.ns_map) if c_del_ns_list: if not c_start_node.nsDef: c_start_node.nsDef = c_del_ns_list else: c_ns = c_start_node.nsDef while c_ns.next: c_ns = c_ns.next c_ns.next = c_del_ns_list cdef int moveNodeToDocument(_Document doc, xmlDoc* c_source_doc, xmlNode* c_element) except -1: u"""Fix the xmlNs pointers of a node and its subtree that were moved. Originally copied from libxml2's xmlReconciliateNs(). Expects libxml2 doc pointers of node to be correct already, but fixes _Document references. For each node in the subtree, we do this: 1) Remove redundant declarations of namespace that are already defined in its parents. 2) Replace namespaces that are *not* defined on the node or its parents by the equivalent namespace declarations that *are* defined on the node or its parents (possibly using a different prefix). If a namespace is unknown, declare a new one on the node. 3) Reassign the names of tags and attribute from the dict of the target document *iff* it is different from the dict used in the source subtree. 4) Set the Document reference to the new Document (if different). This is done on backtracking to keep the original Document alive as long as possible, until all its elements are updated. Note that the namespace declarations are removed from the tree in step 1), but freed only after the complete subtree was traversed and all occurrences were replaced by tree-internal pointers. """ cdef xmlNode* c_start_node cdef xmlNode* c_node cdef xmlDoc* c_doc = doc._c_doc cdef tree.xmlAttr* c_attr cdef char* c_name cdef _nscache c_ns_cache = [NULL, 0, 0] cdef xmlNs* c_del_ns_list = NULL cdef proxy_count = 0 if not tree._isElementOrXInclude(c_element): return 0 c_start_node = c_element tree.BEGIN_FOR_EACH_FROM(c_element, c_element, 1) if tree._isElementOrXInclude(c_element): if hasProxy(c_element): proxy_count += 1 # 1) cut out namespaces defined here that are already known by # the ancestors if c_element.nsDef is not NULL: try: _stripRedundantNamespaceDeclarations(c_element, &c_ns_cache, &c_del_ns_list) except: _cleanUpFromNamespaceAdaptation(c_start_node, &c_ns_cache, c_del_ns_list) raise # 2) make sure the namespaces of an element and its attributes # are declared in this document (i.e. on the node or its parents) if c_element.ns is not NULL: _fixCNs(doc, c_start_node, c_element, &c_ns_cache, c_del_ns_list) c_node = <xmlNode*>c_element.properties while c_node is not NULL: if c_node.ns is not NULL: _fixCNs(doc, c_start_node, c_node, &c_ns_cache, c_del_ns_list) c_node = c_node.next tree.END_FOR_EACH_FROM(c_element) # free now unused namespace declarations if c_del_ns_list is not NULL: tree.xmlFreeNsList(c_del_ns_list) # cleanup if c_ns_cache.ns_map is not NULL: python.lxml_free(c_ns_cache.ns_map) # 3) fix the names in the tree if we moved it from a different thread if doc._c_doc.dict is not c_source_doc.dict: fixThreadDictNames(c_start_node, c_source_doc.dict, doc._c_doc.dict) # 4) fix _Document references # (and potentially deallocate the source document) if proxy_count > 0: if proxy_count == 1 and c_start_node._private is not NULL: proxy = getProxy(c_start_node) if proxy is not None: if proxy._doc is not doc: proxy._doc = doc else: fixElementDocument(c_start_node, doc, proxy_count) else: fixElementDocument(c_start_node, doc, proxy_count) return 0 cdef void _setTreeDoc(xmlNode* c_node, xmlDoc* c_doc): """Adaptation of 'xmlSetTreeDoc()' that deep-fixes the document links iteratively. It avoids https://gitlab.gnome.org/GNOME/libxml2/issues/42 """ tree.BEGIN_FOR_EACH_FROM(c_node, c_node, 1) if c_node.type == tree.XML_ELEMENT_NODE: c_attr = <tree.xmlAttr*>c_node.properties while c_attr: if c_attr.atype == tree.XML_ATTRIBUTE_ID: tree.xmlRemoveID(c_node.doc, c_attr) c_attr.doc = c_doc _fixDocChildren(c_attr.children, c_doc) c_attr = c_attr.next # Set doc link for all nodes, not only elements. c_node.doc = c_doc tree.END_FOR_EACH_FROM(c_node) cdef inline void _fixDocChildren(xmlNode* c_child, xmlDoc* c_doc): while c_child: c_child.doc = c_doc if c_child.children: _fixDocChildren(c_child.children, c_doc) c_child = c_child.next cdef int _fixCNs(_Document doc, xmlNode* c_start_node, xmlNode* c_node, _nscache* c_ns_cache, xmlNs* c_del_ns_list) except -1: cdef xmlNs* c_ns = NULL cdef bint is_prefixed_attr = (c_node.type == tree.XML_ATTRIBUTE_NODE and c_node.ns.prefix) for ns_map in c_ns_cache.ns_map[:c_ns_cache.last]: if c_node.ns is ns_map.old: if is_prefixed_attr and not ns_map.new.prefix: # avoid dropping prefix from attributes continue c_ns = ns_map.new break if c_ns: c_node.ns = c_ns else: # not in cache or not acceptable # => find a replacement from this document try: c_ns = doc._findOrBuildNodeNs( c_start_node, c_node.ns.href, c_node.ns.prefix, c_node.type == tree.XML_ATTRIBUTE_NODE) c_node.ns = c_ns _appendToNsCache(c_ns_cache, c_node.ns, c_ns) except: _cleanUpFromNamespaceAdaptation(c_start_node, c_ns_cache, c_del_ns_list) raise return 0 cdef void fixElementDocument(xmlNode* c_element, _Document doc, size_t proxy_count): cdef xmlNode* c_node = c_element cdef _Element proxy = None # init-to-None required due to fake-loop below tree.BEGIN_FOR_EACH_FROM(c_element, c_node, 1) if c_node._private is not NULL: proxy = getProxy(c_node) if proxy is not None: if proxy._doc is not doc: proxy._doc = doc proxy_count -= 1 if proxy_count == 0: return tree.END_FOR_EACH_FROM(c_node) cdef void fixThreadDictNames(xmlNode* c_element, tree.xmlDict* c_src_dict, tree.xmlDict* c_dict) nogil: # re-assign the names of tags and attributes # # this should only be called when the element is based on a # different libxml2 tag name dictionary if c_element.type == tree.XML_DOCUMENT_NODE or \ c_element.type == tree.XML_HTML_DOCUMENT_NODE: # may define "xml" namespace fixThreadDictNsForNode(c_element, c_src_dict, c_dict) if c_element.doc.extSubset: fixThreadDictNamesForDtd(c_element.doc.extSubset, c_src_dict, c_dict) if c_element.doc.intSubset: fixThreadDictNamesForDtd(c_element.doc.intSubset, c_src_dict, c_dict) c_element = c_element.children while c_element is not NULL: fixThreadDictNamesForNode(c_element, c_src_dict, c_dict) c_element = c_element.next elif tree._isElementOrXInclude(c_element): fixThreadDictNamesForNode(c_element, c_src_dict, c_dict) cdef inline void _fixThreadDictPtr(const_xmlChar** c_ptr, tree.xmlDict* c_src_dict, tree.xmlDict* c_dict) nogil: c_str = c_ptr[0] if c_str and c_src_dict and tree.xmlDictOwns(c_src_dict, c_str): # return value can be NULL on memory error, but we don't handle that here c_str = tree.xmlDictLookup(c_dict, c_str, -1) if c_str: c_ptr[0] = c_str cdef void fixThreadDictNamesForNode(xmlNode* c_element, tree.xmlDict* c_src_dict, tree.xmlDict* c_dict) nogil: cdef xmlNode* c_node = c_element tree.BEGIN_FOR_EACH_FROM(c_element, c_node, 1) if c_node.type in (tree.XML_ELEMENT_NODE, tree.XML_XINCLUDE_START): fixThreadDictNamesForAttributes( c_node.properties, c_src_dict, c_dict) fixThreadDictNsForNode(c_node, c_src_dict, c_dict) _fixThreadDictPtr(&c_node.name, c_src_dict, c_dict) elif c_node.type == tree.XML_TEXT_NODE: # libxml2's SAX2 parser interns some indentation space fixThreadDictContentForNode(c_node, c_src_dict, c_dict) elif c_node.type == tree.XML_COMMENT_NODE: pass # don't touch c_node.name else: _fixThreadDictPtr(&c_node.name, c_src_dict, c_dict) tree.END_FOR_EACH_FROM(c_node) cdef inline void fixThreadDictNamesForAttributes(tree.xmlAttr* c_attr, tree.xmlDict* c_src_dict, tree.xmlDict* c_dict) nogil: cdef xmlNode* c_child cdef xmlNode* c_node = <xmlNode*>c_attr while c_node is not NULL: if c_node.type not in (tree.XML_TEXT_NODE, tree.XML_COMMENT_NODE): _fixThreadDictPtr(&c_node.name, c_src_dict, c_dict) # libxml2 keeps some (!) attribute values in the dict c_child = c_node.children while c_child is not NULL: fixThreadDictContentForNode(c_child, c_src_dict, c_dict) c_child = c_child.next c_node = c_node.next cdef inline void fixThreadDictContentForNode(xmlNode* c_node, tree.xmlDict* c_src_dict, tree.xmlDict* c_dict) nogil: if c_node.content is not NULL and \ c_node.content is not <xmlChar*>&c_node.properties: if tree.xmlDictOwns(c_src_dict, c_node.content): # result can be NULL on memory error, but we don't handle that here c_node.content = <xmlChar*>tree.xmlDictLookup(c_dict, c_node.content, -1) cdef inline void fixThreadDictNsForNode(xmlNode* c_node, tree.xmlDict* c_src_dict, tree.xmlDict* c_dict) nogil: cdef xmlNs* c_ns = c_node.nsDef while c_ns is not NULL: _fixThreadDictPtr(&c_ns.href, c_src_dict, c_dict) _fixThreadDictPtr(&c_ns.prefix, c_src_dict, c_dict) c_ns = c_ns.next cdef void fixThreadDictNamesForDtd(tree.xmlDtd* c_dtd, tree.xmlDict* c_src_dict, tree.xmlDict* c_dict) nogil: cdef xmlNode* c_node cdef tree.xmlElement* c_element cdef tree.xmlAttribute* c_attribute cdef tree.xmlEntity* c_entity c_node = c_dtd.children while c_node: if c_node.type == tree.XML_ELEMENT_DECL: c_element = <tree.xmlElement*>c_node if c_element.content: _fixThreadDictPtr(&c_element.content.name, c_src_dict, c_dict) _fixThreadDictPtr(&c_element.content.prefix, c_src_dict, c_dict) c_attribute = c_element.attributes while c_attribute: _fixThreadDictPtr(&c_attribute.defaultValue, c_src_dict, c_dict) _fixThreadDictPtr(&c_attribute.name, c_src_dict, c_dict) _fixThreadDictPtr(&c_attribute.prefix, c_src_dict, c_dict) _fixThreadDictPtr(&c_attribute.elem, c_src_dict, c_dict) c_attribute = c_attribute.nexth elif c_node.type == tree.XML_ENTITY_DECL: c_entity = <tree.xmlEntity*>c_node _fixThreadDictPtr(&c_entity.name, c_src_dict, c_dict) _fixThreadDictPtr(&c_entity.ExternalID, c_src_dict, c_dict) _fixThreadDictPtr(&c_entity.SystemID, c_src_dict, c_dict) _fixThreadDictPtr(<const_xmlChar**>&c_entity.content, c_src_dict, c_dict) c_node = c_node.next ################################################################################ # adopt an xmlDoc from an external libxml2 document source cdef _Document _adoptForeignDoc(xmlDoc* c_doc, _BaseParser parser=None, bint is_owned=True): """Convert and wrap an externally produced xmlDoc for use in lxml. Assures that all '_private' pointers are NULL to prevent accidental dereference into lxml proxy objects. """ if c_doc is NULL: raise ValueError("Illegal document provided: NULL") if c_doc.type not in (tree.XML_DOCUMENT_NODE, tree.XML_HTML_DOCUMENT_NODE): doc_type = c_doc.type if is_owned: tree.xmlFreeDoc(c_doc) raise ValueError(f"Illegal document provided: expected XML or HTML, found {doc_type}") cdef xmlNode* c_node = <xmlNode*>c_doc if is_owned: tree.BEGIN_FOR_EACH_FROM(<xmlNode*>c_doc, c_node, 1) c_node._private = NULL tree.END_FOR_EACH_FROM(c_node) else: # create a fresh copy that lxml owns c_doc = tree.xmlCopyDoc(c_doc, 1) if c_doc is NULL: raise MemoryError() return _documentFactory(c_doc, parser)