Added an original copy of pysolr.py so the custom changes can be worked out.

python/packages/html5lib/treewalkers/__init__.py

@@ -0,0 +1,52 @@
+"""A collection of modules for iterating through different kinds of
+tree, generating tokens identical to those produced by the tokenizer
+module.
+
+To create a tree walker for a new type of tree, you need to do
+implement a tree walker object (called TreeWalker by convention) that
+implements a 'serialize' method taking a tree as sole argument and
+returning an iterator generating tokens.
+"""
+
+treeWalkerCache = {}
+
+def getTreeWalker(treeType, implementation=None, **kwargs):
+    """Get a TreeWalker class for various types of tree with built-in support
+
+    treeType - the name of the tree type required (case-insensitive). Supported
+               values are "simpletree", "dom", "etree" and "beautifulsoup"
+
+               "simpletree" - a built-in DOM-ish tree type with support for some
+                              more pythonic idioms.
+                "dom" - The xml.dom.minidom DOM implementation
+                "pulldom" - The xml.dom.pulldom event stream
+                "etree" - A generic walker for tree implementations exposing an
+                          elementtree-like interface (known to work with
+                          ElementTree, cElementTree and lxml.etree).
+                "lxml" - Optimized walker for lxml.etree
+                "beautifulsoup" - Beautiful soup (if installed)
+                "genshi" - a Genshi stream
+
+    implementation - (Currently applies to the "etree" tree type only). A module
+                      implementing the tree type e.g. xml.etree.ElementTree or
+                      cElementTree."""
+
+    treeType = treeType.lower()
+    if treeType not in treeWalkerCache:
+        if treeType in ("dom", "pulldom", "simpletree"):
+            mod = __import__(treeType, globals())
+            treeWalkerCache[treeType] = mod.TreeWalker
+        elif treeType == "genshi":
+            import genshistream
+            treeWalkerCache[treeType] = genshistream.TreeWalker
+        elif treeType == "beautifulsoup":
+            import soup
+            treeWalkerCache[treeType] = soup.TreeWalker
+        elif treeType == "lxml":
+            import lxmletree
+            treeWalkerCache[treeType] = lxmletree.TreeWalker
+        elif treeType == "etree":
+            import etree
+            # XXX: NEVER cache here, caching is done in the etree submodule
+            return etree.getETreeModule(implementation, **kwargs).TreeWalker
+    return treeWalkerCache.get(treeType)

python/packages/html5lib/treewalkers/_base.py

@@ -0,0 +1,176 @@
+import gettext
+_ = gettext.gettext
+
+from html5lib.constants import voidElements, spaceCharacters
+spaceCharacters = u"".join(spaceCharacters)
+
+class TreeWalker(object):
+    def __init__(self, tree):
+        self.tree = tree
+
+    def __iter__(self):
+        raise NotImplementedError
+
+    def error(self, msg):
+        return {"type": "SerializeError", "data": msg}
+
+    def normalizeAttrs(self, attrs):
+        newattrs = {}
+        if attrs:
+            #TODO: treewalkers should always have attrs
+            for (namespace,name),value in attrs.iteritems():
+                namespace = unicode(namespace) if namespace else None
+                name = unicode(name)
+                value = unicode(value)
+                newattrs[(namespace,name)] = value
+        return newattrs
+
+    def emptyTag(self, namespace, name, attrs, hasChildren=False):
+        yield {"type": "EmptyTag", "name": unicode(name), 
+               "namespace":unicode(namespace),
+               "data": self.normalizeAttrs(attrs)}
+        if hasChildren:
+            yield self.error(_("Void element has children"))
+
+    def startTag(self, namespace, name, attrs):
+        return {"type": "StartTag", 
+                "name": unicode(name),
+                "namespace":unicode(namespace),
+                "data": self.normalizeAttrs(attrs)}
+
+    def endTag(self, namespace, name):
+        return {"type": "EndTag", 
+                "name": unicode(name),
+                "namespace":unicode(namespace),
+                "data": {}}
+
+    def text(self, data):
+        data = unicode(data)
+        middle = data.lstrip(spaceCharacters)
+        left = data[:len(data)-len(middle)]
+        if left:
+            yield {"type": "SpaceCharacters", "data": left}
+        data = middle
+        middle = data.rstrip(spaceCharacters)
+        right = data[len(middle):]
+        if middle:
+            yield {"type": "Characters", "data": middle}
+        if right:
+            yield {"type": "SpaceCharacters", "data": right}
+
+    def comment(self, data):
+        return {"type": "Comment", "data": unicode(data)}
+
+    def doctype(self, name, publicId=None, systemId=None, correct=True):
+        return {"type": "Doctype",
+                "name": name is not None and unicode(name) or u"",
+                "publicId": publicId,
+                "systemId": systemId,
+                "correct": correct}
+
+    def entity(self, name):
+        return {"type": "Entity", "name": unicode(name)}
+
+    def unknown(self, nodeType):
+        return self.error(_("Unknown node type: ") + nodeType)
+
+class RecursiveTreeWalker(TreeWalker):
+    def walkChildren(self, node):
+        raise NodeImplementedError
+
+    def element(self, node, namespace, name, attrs, hasChildren):
+        if name in voidElements:
+            for token in self.emptyTag(namespace, name, attrs, hasChildren):
+                yield token
+        else:
+            yield self.startTag(name, attrs)
+            if hasChildren:
+                for token in self.walkChildren(node):
+                    yield token
+            yield self.endTag(name)
+
+from xml.dom import Node
+
+DOCUMENT = Node.DOCUMENT_NODE
+DOCTYPE = Node.DOCUMENT_TYPE_NODE
+TEXT = Node.TEXT_NODE
+ELEMENT = Node.ELEMENT_NODE
+COMMENT = Node.COMMENT_NODE
+ENTITY = Node.ENTITY_NODE
+UNKNOWN = "<#UNKNOWN#>"
+
+class NonRecursiveTreeWalker(TreeWalker):
+    def getNodeDetails(self, node):
+        raise NotImplementedError
+    
+    def getFirstChild(self, node):
+        raise NotImplementedError
+    
+    def getNextSibling(self, node):
+        raise NotImplementedError
+    
+    def getParentNode(self, node):
+        raise NotImplementedError
+
+    def __iter__(self):
+        currentNode = self.tree
+        while currentNode is not None:
+            details = self.getNodeDetails(currentNode)
+            type, details = details[0], details[1:]
+            hasChildren = False
+            endTag = None
+
+            if type == DOCTYPE:
+                yield self.doctype(*details)
+
+            elif type == TEXT:
+                for token in self.text(*details):
+                    yield token
+
+            elif type == ELEMENT:
+                namespace, name, attributes, hasChildren = details
+                if name in voidElements:
+                    for token in self.emptyTag(namespace, name, attributes, 
+                                               hasChildren):
+                        yield token
+                    hasChildren = False
+                else:
+                    endTag = name
+                    yield self.startTag(namespace, name, attributes)
+
+            elif type == COMMENT:
+                yield self.comment(details[0])
+
+            elif type == ENTITY:
+                yield self.entity(details[0])
+
+            elif type == DOCUMENT:
+                hasChildren = True
+
+            else:
+                yield self.unknown(details[0])
+            
+            if hasChildren:
+                firstChild = self.getFirstChild(currentNode)
+            else:
+                firstChild = None
+            
+            if firstChild is not None:
+                currentNode = firstChild
+            else:
+                while currentNode is not None:
+                    details = self.getNodeDetails(currentNode)
+                    type, details = details[0], details[1:]
+                    if type == ELEMENT:
+                        namespace, name, attributes, hasChildren = details
+                        if name not in voidElements:
+                            yield self.endTag(namespace, name)
+                    if self.tree is currentNode:
+                        currentNode = None
+                        break
+                    nextSibling = self.getNextSibling(currentNode)
+                    if nextSibling is not None:
+                        currentNode = nextSibling
+                        break
+                    else:
+                        currentNode = self.getParentNode(currentNode)

python/packages/html5lib/treewalkers/dom.py

@@ -0,0 +1,41 @@
+from xml.dom import Node
+
+import gettext
+_ = gettext.gettext
+
+import _base
+from html5lib.constants import voidElements
+
+class TreeWalker(_base.NonRecursiveTreeWalker):
+    def getNodeDetails(self, node):
+        if node.nodeType == Node.DOCUMENT_TYPE_NODE:
+            return _base.DOCTYPE, node.name, node.publicId, node.systemId
+
+        elif node.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE):
+            return _base.TEXT, node.nodeValue
+
+        elif node.nodeType == Node.ELEMENT_NODE:
+            attrs = {}
+            for attr in node.attributes.keys():
+                attr = node.getAttributeNode(attr)
+                attrs[(attr.namespaceURI,attr.localName)] = attr.value
+            return (_base.ELEMENT, node.namespaceURI, node.nodeName, 
+                    attrs, node.hasChildNodes())
+
+        elif node.nodeType == Node.COMMENT_NODE:
+            return _base.COMMENT, node.nodeValue
+
+        elif node.nodeType in (Node.DOCUMENT_NODE, Node.DOCUMENT_FRAGMENT_NODE):
+            return (_base.DOCUMENT,)
+
+        else:
+            return _base.UNKNOWN, node.nodeType
+
+    def getFirstChild(self, node):
+        return node.firstChild
+
+    def getNextSibling(self, node):
+        return node.nextSibling
+
+    def getParentNode(self, node):
+        return node.parentNode

python/packages/html5lib/treewalkers/etree.py

@@ -0,0 +1,141 @@
+import gettext
+_ = gettext.gettext
+
+try:
+    from types import ModuleType
+except:
+    from new import module as ModuleType
+import copy
+import re
+
+import _base
+from html5lib.constants import voidElements
+
+tag_regexp = re.compile("{([^}]*)}(.*)")
+
+moduleCache = {}
+
+def getETreeModule(ElementTreeImplementation):
+    name = "_" + ElementTreeImplementation.__name__+"builder"
+    if name in moduleCache:
+        return moduleCache[name]
+    else:
+        mod = ModuleType("_" + ElementTreeImplementation.__name__+"builder")
+        objs = getETreeBuilder(ElementTreeImplementation)
+        mod.__dict__.update(objs)
+        moduleCache[name] = mod
+        return mod
+
+def getETreeBuilder(ElementTreeImplementation):
+    ElementTree = ElementTreeImplementation
+
+    class TreeWalker(_base.NonRecursiveTreeWalker):
+        """Given the particular ElementTree representation, this implementation,
+        to avoid using recursion, returns "nodes" as tuples with the following
+        content:
+
+        1. The current element
+        
+        2. The index of the element relative to its parent
+        
+        3. A stack of ancestor elements
+        
+        4. A flag "text", "tail" or None to indicate if the current node is a
+           text node; either the text or tail of the current element (1)
+        """
+        def getNodeDetails(self, node):
+            if isinstance(node, tuple): # It might be the root Element
+                elt, key, parents, flag = node
+                if flag in ("text", "tail"):
+                    return _base.TEXT, getattr(elt, flag)
+                else:
+                    node = elt
+
+            if not(hasattr(node, "tag")):
+                node = node.getroot()
+
+            if node.tag in ("<DOCUMENT_ROOT>", "<DOCUMENT_FRAGMENT>"):
+                return (_base.DOCUMENT,)
+
+            elif node.tag == "<!DOCTYPE>":
+                return (_base.DOCTYPE, node.text, 
+                        node.get("publicId"), node.get("systemId"))
+
+            elif node.tag == ElementTree.Comment:
+                return _base.COMMENT, node.text
+
+            else:
+                assert type(node.tag) in (str, unicode), type(node.tag)
+                #This is assumed to be an ordinary element
+                match = tag_regexp.match(node.tag)
+                if match:
+                    namespace, tag = match.groups()
+                else:
+                    namespace = None
+                    tag = node.tag
+                attrs = {}
+                for name, value in node.attrib.items():
+                    match = tag_regexp.match(name)
+                    if match:
+                        attrs[(match.group(1),match.group(2))] = value
+                    else:
+                        attrs[(None,name)] = value
+                return (_base.ELEMENT, namespace, tag, 
+                        attrs, len(node) or node.text)
+    
+        def getFirstChild(self, node):
+            if isinstance(node, tuple):
+                element, key, parents, flag = node
+            else:
+                element, key, parents, flag = node, None, [], None
+                
+            if flag in ("text", "tail"):
+                return None
+            else:
+                if element.text:
+                    return element, key, parents, "text"
+                elif len(element):
+                    parents.append(element)
+                    return element[0], 0, parents, None
+                else:
+                    return None
+        
+        def getNextSibling(self, node):
+            if isinstance(node, tuple):
+                element, key, parents, flag = node
+            else:
+                return None
+                
+            if flag == "text":
+                if len(element):
+                    parents.append(element)
+                    return element[0], 0, parents, None
+                else:
+                    return None
+            else:
+                if element.tail and flag != "tail":
+                    return element, key, parents, "tail"
+                elif key < len(parents[-1]) - 1:
+                    return parents[-1][key+1], key+1, parents, None
+                else:
+                    return None
+        
+        def getParentNode(self, node):
+            if isinstance(node, tuple):
+                element, key, parents, flag = node
+            else:
+                return None
+            
+            if flag == "text":
+                if not parents:
+                    return element
+                else:
+                    return element, key, parents, None
+            else:
+                parent = parents.pop()
+                if not parents:
+                    return parent
+                else:
+                    return parent, list(parents[-1]).index(parent), parents, None
+
+    return locals()

python/packages/html5lib/treewalkers/genshistream.py

@@ -0,0 +1,70 @@
+from genshi.core import START, END, XML_NAMESPACE, DOCTYPE, TEXT
+from genshi.core  import  START_NS, END_NS, START_CDATA, END_CDATA, PI, COMMENT
+from genshi.output import NamespaceFlattener
+
+import _base
+
+from html5lib.constants import voidElements
+
+class TreeWalker(_base.TreeWalker):
+    def __iter__(self):
+        depth = 0
+        ignore_until = None
+        previous = None
+        for event in self.tree:
+            if previous is not None:
+                if previous[0] == START:
+                    depth += 1
+                if ignore_until <= depth:
+                    ignore_until = None
+                if ignore_until is None:
+                    for token in self.tokens(previous, event):
+                        yield token
+                        if token["type"] == "EmptyTag":
+                            ignore_until = depth
+                if previous[0] == END:
+                    depth -= 1
+            previous = event
+        if previous is not None:
+            if ignore_until is None or ignore_until <= depth:
+                for token in self.tokens(previous, None):
+                    yield token
+            elif ignore_until is not None:
+                raise ValueError("Illformed DOM event stream: void element without END_ELEMENT")
+
+    def tokens(self, event, next):
+        kind, data, pos = event
+        if kind == START:
+            tag, attrib = data
+            name = tag.localname
+            namespace = tag.namespace
+            if tag in voidElements:
+                for token in self.emptyTag(namespace, name, list(attrib),
+                                           not next or next[0] != END 
+                                           or next[1] != tag):
+                    yield token
+            else:
+                yield self.startTag(namespace, name, list(attrib))
+
+        elif kind == END:
+            name = data.localname
+            namespace = data.namespace
+            if name not in voidElements:
+                yield self.endTag(namespace, name)
+
+        elif kind == COMMENT:
+            yield self.comment(data)
+
+        elif kind == TEXT:
+            for token in self.text(data):
+                yield token
+
+        elif kind == DOCTYPE:
+            yield self.doctype(*data)
+
+        elif kind in (XML_NAMESPACE, DOCTYPE, START_NS, END_NS, \
+          START_CDATA, END_CDATA, PI):
+            pass
+
+        else:
+            yield self.unknown(kind)

python/packages/html5lib/treewalkers/lxmletree.py

@@ -0,0 +1,186 @@
+from lxml import etree
+from html5lib.treebuilders.etree import tag_regexp
+
+from gettext import gettext
+_ = gettext
+
+import _base
+
+from html5lib.constants import voidElements
+from html5lib import ihatexml
+
+class Root(object):
+    def __init__(self, et):
+        self.elementtree = et
+        self.children = []
+        if et.docinfo.internalDTD:
+            self.children.append(Doctype(self, et.docinfo.root_name, 
+                                         et.docinfo.public_id, 
+                                         et.docinfo.system_url))
+        root = et.getroot()
+        node = root
+
+        while node.getprevious() is not None:
+            node = node.getprevious()
+        while node is not None:
+            self.children.append(node)
+            node = node.getnext()
+
+        self.text = None
+        self.tail = None
+    
+    def __getitem__(self, key):
+        return self.children[key]
+
+    def getnext(self):
+        return None
+
+    def __len__(self):
+        return 1
+
+class Doctype(object):
+    def __init__(self, root_node, name, public_id, system_id):
+        self.root_node = root_node
+        self.name = name
+        self.public_id = public_id
+        self.system_id = system_id
+        
+        self.text = None
+        self.tail = None
+
+    def getnext(self):
+        return self.root_node.children[1]
+
+class FragmentRoot(Root):
+    def __init__(self, children):
+        self.children = [FragmentWrapper(self, child) for child in children]
+        self.text = self.tail = None
+
+    def getnext(self):
+        return None
+
+class FragmentWrapper(object):
+    def __init__(self, fragment_root, obj):
+        self.root_node = fragment_root
+        self.obj = obj
+        if hasattr(self.obj, 'text'):
+            self.text = self.obj.text
+        else:
+            self.text = None
+        if hasattr(self.obj, 'tail'):
+            self.tail = self.obj.tail
+        else:
+            self.tail = None
+        self.isstring = isinstance(obj, basestring)
+        
+    def __getattr__(self, name):
+        return getattr(self.obj, name)
+    
+    def getnext(self):
+        siblings = self.root_node.children
+        idx = siblings.index(self)
+        if idx < len(siblings) - 1:
+            return siblings[idx + 1]
+        else:
+            return None
+
+    def __getitem__(self, key):
+        return self.obj[key]
+
+    def __nonzero__(self):
+        return bool(self.obj)
+
+    def getparent(self):
+        return None
+
+    def __str__(self):
+        return str(self.obj)
+
+    def __unicode__(self):
+        return unicode(self.obj)
+
+    def __len__(self):
+        return len(self.obj)
+
+        
+class TreeWalker(_base.NonRecursiveTreeWalker):
+    def __init__(self, tree):
+        if hasattr(tree, "getroot"):
+            tree = Root(tree)
+        elif isinstance(tree, list):
+            tree = FragmentRoot(tree)
+        _base.NonRecursiveTreeWalker.__init__(self, tree)
+        self.filter = ihatexml.InfosetFilter()
+    def getNodeDetails(self, node):
+        if isinstance(node, tuple): # Text node
+            node, key = node
+            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            return _base.TEXT, getattr(node, key)
+
+        elif isinstance(node, Root):
+            return (_base.DOCUMENT,)
+
+        elif isinstance(node, Doctype):
+            return _base.DOCTYPE, node.name, node.public_id, node.system_id
+
+        elif isinstance(node, FragmentWrapper) and node.isstring:
+            return _base.TEXT, node
+
+        elif node.tag == etree.Comment:
+            return _base.COMMENT, node.text
+
+        elif node.tag == etree.Entity:
+            return _base.ENTITY, node.text[1:-1] # strip &;
+
+        else:
+            #This is assumed to be an ordinary element
+            match = tag_regexp.match(node.tag)
+            if match:
+                namespace, tag = match.groups()
+            else:
+                namespace = None
+                tag = node.tag
+            attrs = {}
+            for name, value in node.attrib.items():
+                match = tag_regexp.match(name)
+                if match:
+                    attrs[(match.group(1),match.group(2))] = value
+                else:
+                    attrs[(None,name)] = value
+            return (_base.ELEMENT, namespace, self.filter.fromXmlName(tag), 
+                    attrs, len(node) > 0 or node.text)
+
+    def getFirstChild(self, node):
+        assert not isinstance(node, tuple), _("Text nodes have no children")
+
+        assert len(node) or node.text, "Node has no children"
+        if node.text:
+            return (node, "text")
+        else:
+            return node[0]
+
+    def getNextSibling(self, node):
+        if isinstance(node, tuple): # Text node
+            node, key = node
+            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            if key == "text":
+                # XXX: we cannot use a "bool(node) and node[0] or None" construct here
+                # because node[0] might evaluate to False if it has no child element
+                if len(node):
+                    return node[0]
+                else:
+                    return None
+            else: # tail
+                return node.getnext()
+
+        return node.tail and (node, "tail") or node.getnext()
+
+    def getParentNode(self, node):
+        if isinstance(node, tuple): # Text node
+            node, key = node
+            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            if key == "text":
+                return node
+            # else: fallback to "normal" processing
+
+        return node.getparent()

python/packages/html5lib/treewalkers/pulldom.py

@@ -0,0 +1,60 @@
+from xml.dom.pulldom import START_ELEMENT, END_ELEMENT, \
+    COMMENT, IGNORABLE_WHITESPACE, CHARACTERS
+
+import _base
+
+from html5lib.constants import voidElements
+
+class TreeWalker(_base.TreeWalker):
+    def __iter__(self):
+        ignore_until = None
+        previous = None
+        for event in self.tree:
+            if previous is not None and \
+              (ignore_until is None or previous[1] is ignore_until):
+                if previous[1] is ignore_until:
+                    ignore_until = None
+                for token in self.tokens(previous, event):
+                    yield token
+                    if token["type"] == "EmptyTag":
+                        ignore_until = previous[1]
+            previous = event
+        if ignore_until is None or previous[1] is ignore_until:
+            for token in self.tokens(previous, None):
+                yield token
+        elif ignore_until is not None:
+            raise ValueError("Illformed DOM event stream: void element without END_ELEMENT")
+
+    def tokens(self, event, next):
+        type, node = event
+        if type == START_ELEMENT:
+            name = node.nodeName
+            namespace = node.namespaceURI
+            attrs = {}
+            for attr in node.attributes.keys():
+                attr = node.getAttributeNode(attr)
+                attrs[(attr.namespaceURI,attr.localName)] = attr.value
+            if name in voidElements:
+                for token in self.emptyTag(namespace,
+                                           name,
+                                           attrs,
+                                           not next or next[1] is not node):
+                    yield token
+            else:
+                yield self.startTag(namespace, name, attrs)
+
+        elif type == END_ELEMENT:
+            name = node.nodeName
+            namespace = node.namespaceURI
+            if name not in voidElements:
+                yield self.endTag(namespace, name)
+
+        elif type == COMMENT:
+            yield self.comment(node.nodeValue)
+
+        elif type in (IGNORABLE_WHITESPACE, CHARACTERS):
+            for token in self.text(node.nodeValue):
+                yield token
+
+        else:
+            yield self.unknown(type)

python/packages/html5lib/treewalkers/simpletree.py

@@ -0,0 +1,78 @@
+import gettext
+_ = gettext.gettext
+
+import _base
+
+class TreeWalker(_base.NonRecursiveTreeWalker):
+    """Given that simpletree has no performant way of getting a node's
+    next sibling, this implementation returns "nodes" as tuples with the
+    following content:
+
+    1. The parent Node (Element, Document or DocumentFragment)
+
+    2. The child index of the current node in its parent's children list
+
+    3. A list used as a stack of all ancestors. It is a pair tuple whose
+       first item is a parent Node and second item is a child index.
+    """
+
+    def getNodeDetails(self, node):
+        if isinstance(node, tuple): # It might be the root Node
+            parent, idx, parents = node
+            node = parent.childNodes[idx]
+
+        # testing node.type allows us not to import treebuilders.simpletree
+        if node.type in (1, 2): # Document or DocumentFragment
+            return (_base.DOCUMENT,)
+
+        elif node.type == 3: # DocumentType
+            return _base.DOCTYPE, node.name, node.publicId, node.systemId
+
+        elif node.type == 4: # TextNode
+            return _base.TEXT, node.value
+
+        elif node.type == 5: # Element
+            attrs = {}
+            for name, value in node.attributes.items():
+                if isinstance(name, tuple):
+                    attrs[(name[2],name[1])] = value
+                else:
+                    attrs[(None,name)] = value
+            return (_base.ELEMENT, node.namespace, node.name, 
+                    attrs, node.hasContent())
+
+        elif node.type == 6: # CommentNode
+            return _base.COMMENT, node.data
+
+        else:
+            return _node.UNKNOWN, node.type
+
+    def getFirstChild(self, node):
+        if isinstance(node, tuple): # It might be the root Node
+            parent, idx, parents = node
+            parents.append((parent, idx))
+            node = parent.childNodes[idx]
+        else:
+            parents = []
+
+        assert node.hasContent(), "Node has no children"
+        return (node, 0, parents)
+
+    def getNextSibling(self, node):
+        assert isinstance(node, tuple), "Node is not a tuple: " + str(node)
+        parent, idx, parents = node
+        idx += 1
+        if len(parent.childNodes) > idx:
+            return (parent, idx, parents)
+        else:
+            return None
+
+    def getParentNode(self, node):
+        assert isinstance(node, tuple)
+        parent, idx, parents = node
+        if parents:
+            parent, idx = parents.pop()
+            return parent, idx, parents
+        else:
+            # HACK: We could return ``parent`` but None will stop the algorithm the same way
+            return None

python/packages/html5lib/treewalkers/soup.py

@@ -0,0 +1,60 @@
+import re
+import gettext
+_ = gettext.gettext
+
+from BeautifulSoup import BeautifulSoup, Declaration, Comment, Tag
+from html5lib.constants import namespaces
+import _base
+
+class TreeWalker(_base.NonRecursiveTreeWalker):
+    doctype_regexp = re.compile(
+        r'DOCTYPE\s+(?P<name>[^\s]*)(\s*PUBLIC\s*"(?P<publicId>.*)"\s*"(?P<systemId1>.*)"|\s*SYSTEM\s*"(?P<systemId2>.*)")?')
+    def getNodeDetails(self, node):
+        if isinstance(node, BeautifulSoup): # Document or DocumentFragment
+            return (_base.DOCUMENT,)
+
+        elif isinstance(node, Declaration): # DocumentType
+            string = unicode(node.string)
+            #Slice needed to remove markup added during unicode conversion,
+            #but only in some versions of BeautifulSoup/Python
+            if string.startswith('<!') and string.endswith('>'):
+                string = string[2:-1]
+            m = self.doctype_regexp.match(string)
+            #This regexp approach seems wrong and fragile
+            #but beautiful soup stores the doctype as a single thing and we want the seperate bits
+            #It should work as long as the tree is created by html5lib itself but may be wrong if it's
+            #been modified at all
+            #We could just feed to it a html5lib tokenizer, I guess...
+            assert m is not None, "DOCTYPE did not match expected format"
+
+            name = m.group('name')
+            publicId = m.group('publicId')
+            if publicId is not None:
+                systemId = m.group('systemId1')
+            else:
+                systemId = m.group('systemId2')
+            return _base.DOCTYPE, name, publicId or "", systemId or ""
+
+        elif isinstance(node, Comment):
+            string = unicode(node.string)
+            if string.startswith('<!--') and string.endswith('-->'):
+                string = string[4:-3]
+            return _base.COMMENT, string
+
+        elif isinstance(node, unicode): # TextNode
+            return _base.TEXT, node
+
+        elif isinstance(node, Tag): # Element
+            return (_base.ELEMENT, namespaces["html"], node.name,
+                    dict(node.attrs).items(), node.contents)
+        else:
+            return _base.UNKNOWN, node.__class__.__name__
+
+    def getFirstChild(self, node):
+        return node.contents[0]
+
+    def getNextSibling(self, node):
+        return node.nextSibling
+
+    def getParentNode(self, node):
+        return node.parent