| from __future__ import absolute_import, division, unicode_literals |
| |
| import json |
| import unittest |
| |
| from .support import get_data_files |
| |
| try: |
| unittest.TestCase.assertEqual |
| except AttributeError: |
| unittest.TestCase.assertEqual = unittest.TestCase.assertEquals |
| |
| import html5lib |
| from html5lib import constants |
| from html5lib.serializer import HTMLSerializer, serialize |
| from html5lib.treewalkers._base import TreeWalker |
| |
| optionals_loaded = [] |
| |
| try: |
| from lxml import etree |
| optionals_loaded.append("lxml") |
| except ImportError: |
| pass |
| |
| default_namespace = constants.namespaces["html"] |
| |
| |
| class JsonWalker(TreeWalker): |
| def __iter__(self): |
| for token in self.tree: |
| type = token[0] |
| if type == "StartTag": |
| if len(token) == 4: |
| namespace, name, attrib = token[1:4] |
| else: |
| namespace = default_namespace |
| name, attrib = token[1:3] |
| yield self.startTag(namespace, name, self._convertAttrib(attrib)) |
| elif type == "EndTag": |
| if len(token) == 3: |
| namespace, name = token[1:3] |
| else: |
| namespace = default_namespace |
| name = token[1] |
| yield self.endTag(namespace, name) |
| elif type == "EmptyTag": |
| if len(token) == 4: |
| namespace, name, attrib = token[1:] |
| else: |
| namespace = default_namespace |
| name, attrib = token[1:] |
| for token in self.emptyTag(namespace, name, self._convertAttrib(attrib)): |
| yield token |
| elif type == "Comment": |
| yield self.comment(token[1]) |
| elif type in ("Characters", "SpaceCharacters"): |
| for token in self.text(token[1]): |
| yield token |
| elif type == "Doctype": |
| if len(token) == 4: |
| yield self.doctype(token[1], token[2], token[3]) |
| elif len(token) == 3: |
| yield self.doctype(token[1], token[2]) |
| else: |
| yield self.doctype(token[1]) |
| else: |
| raise ValueError("Unknown token type: " + type) |
| |
| def _convertAttrib(self, attribs): |
| """html5lib tree-walkers use a dict of (namespace, name): value for |
| attributes, but JSON cannot represent this. Convert from the format |
| in the serializer tests (a list of dicts with "namespace", "name", |
| and "value" as keys) to html5lib's tree-walker format.""" |
| attrs = {} |
| for attrib in attribs: |
| name = (attrib["namespace"], attrib["name"]) |
| assert(name not in attrs) |
| attrs[name] = attrib["value"] |
| return attrs |
| |
| |
| def serialize_html(input, options): |
| options = dict([(str(k), v) for k, v in options.items()]) |
| stream = JsonWalker(input) |
| serializer = HTMLSerializer(alphabetical_attributes=True, **options) |
| return serializer.render(stream, options.get("encoding", None)) |
| |
| |
| def runSerializerTest(input, expected, options): |
| encoding = options.get("encoding", None) |
| |
| if encoding: |
| encode = lambda x: x.encode(encoding) |
| expected = list(map(encode, expected)) |
| |
| result = serialize_html(input, options) |
| if len(expected) == 1: |
| assert expected[0] == result, "Expected:\n%s\nActual:\n%s\nOptions:\n%s" % (expected[0], result, str(options)) |
| elif result not in expected: |
| assert False, "Expected: %s, Received: %s" % (expected, result) |
| |
| |
| class EncodingTestCase(unittest.TestCase): |
| def throwsWithLatin1(self, input): |
| self.assertRaises(UnicodeEncodeError, serialize_html, input, {"encoding": "iso-8859-1"}) |
| |
| def testDoctypeName(self): |
| self.throwsWithLatin1([["Doctype", "\u0101"]]) |
| |
| def testDoctypePublicId(self): |
| self.throwsWithLatin1([["Doctype", "potato", "\u0101"]]) |
| |
| def testDoctypeSystemId(self): |
| self.throwsWithLatin1([["Doctype", "potato", "potato", "\u0101"]]) |
| |
| def testCdataCharacters(self): |
| runSerializerTest([["StartTag", "http://www.w3.org/1999/xhtml", "style", {}], ["Characters", "\u0101"]], |
| ["<style>ā"], {"encoding": "iso-8859-1"}) |
| |
| def testCharacters(self): |
| runSerializerTest([["Characters", "\u0101"]], |
| ["ā"], {"encoding": "iso-8859-1"}) |
| |
| def testStartTagName(self): |
| self.throwsWithLatin1([["StartTag", "http://www.w3.org/1999/xhtml", "\u0101", []]]) |
| |
| def testEmptyTagName(self): |
| self.throwsWithLatin1([["EmptyTag", "http://www.w3.org/1999/xhtml", "\u0101", []]]) |
| |
| def testAttributeName(self): |
| self.throwsWithLatin1([["StartTag", "http://www.w3.org/1999/xhtml", "span", [{"namespace": None, "name": "\u0101", "value": "potato"}]]]) |
| |
| def testAttributeValue(self): |
| runSerializerTest([["StartTag", "http://www.w3.org/1999/xhtml", "span", |
| [{"namespace": None, "name": "potato", "value": "\u0101"}]]], |
| ["<span potato=ā>"], {"encoding": "iso-8859-1"}) |
| |
| def testEndTagName(self): |
| self.throwsWithLatin1([["EndTag", "http://www.w3.org/1999/xhtml", "\u0101"]]) |
| |
| def testComment(self): |
| self.throwsWithLatin1([["Comment", "\u0101"]]) |
| |
| |
| if "lxml" in optionals_loaded: |
| class LxmlTestCase(unittest.TestCase): |
| def setUp(self): |
| self.parser = etree.XMLParser(resolve_entities=False) |
| self.treewalker = html5lib.getTreeWalker("lxml") |
| self.serializer = HTMLSerializer() |
| |
| def testEntityReplacement(self): |
| doc = """<!DOCTYPE html SYSTEM "about:legacy-compat"><html>β</html>""" |
| tree = etree.fromstring(doc, parser=self.parser).getroottree() |
| result = serialize(tree, tree="lxml", omit_optional_tags=False) |
| self.assertEqual("""<!DOCTYPE html SYSTEM "about:legacy-compat"><html>\u03B2</html>""", result) |
| |
| def testEntityXML(self): |
| doc = """<!DOCTYPE html SYSTEM "about:legacy-compat"><html>></html>""" |
| tree = etree.fromstring(doc, parser=self.parser).getroottree() |
| result = serialize(tree, tree="lxml", omit_optional_tags=False) |
| self.assertEqual("""<!DOCTYPE html SYSTEM "about:legacy-compat"><html>></html>""", result) |
| |
| def testEntityNoResolve(self): |
| doc = """<!DOCTYPE html SYSTEM "about:legacy-compat"><html>β</html>""" |
| tree = etree.fromstring(doc, parser=self.parser).getroottree() |
| result = serialize(tree, tree="lxml", omit_optional_tags=False, |
| resolve_entities=False) |
| self.assertEqual("""<!DOCTYPE html SYSTEM "about:legacy-compat"><html>β</html>""", result) |
| |
| |
| def test_serializer(): |
| for filename in get_data_files('serializer', '*.test'): |
| with open(filename) as fp: |
| tests = json.load(fp) |
| for index, test in enumerate(tests['tests']): |
| yield runSerializerTest, test["input"], test["expected"], test.get("options", {}) |