import json

import html5lib

def parse(path="html5ents.xml"):
    return html5lib.parse(open(path), treebuilder="lxml")

def entity_table(tree):
    return dict((entity_name("".join(tr[0].xpath(".//text()"))),
                 entity_characters(tr[1].text))
                for tr in tree.xpath("//h:tbody/h:tr",
                                     namespaces={"h":"http://www.w3.org/1999/xhtml"}))

def entity_name(inp):
    return inp.strip()

def entity_characters(inp):
    return "".join(codepoint_to_character(item)
                    for item in inp.split()
                    if item)

def codepoint_to_character(inp):
    return ("\U000"+inp[2:]).decode("unicode-escape")

def make_tests_json(entities):
    test_list = make_test_list(entities)
    tests_json = {"tests":
                      [make_test(*item) for item in test_list]
                  }
    return tests_json

def make_test(name, characters, good):
    return {
        "description":test_description(name, good),
        "input":"&%s"%name,
        "output":test_expected(name, characters, good)
        }

def test_description(name, good):
    with_semicolon = name.endswith(";")
    semicolon_text = {True:"with a semi-colon",
                      False:"without a semi-colon"}[with_semicolon]
    if good:
        text = "Named entity: %s %s"%(name, semicolon_text)
    else:
        text = "Bad named entity: %s %s"%(name, semicolon_text)
    return text

def test_expected(name, characters, good):
    rv = []
    if not good or not name.endswith(";"):
        rv.append("ParseError")
    rv.append(["Character", characters])
    return rv

def make_test_list(entities):
    tests = []
    for entity_name, characters in entities.items():
        if entity_name.endswith(";") and not subentity_exists(entity_name, entities):
            tests.append((entity_name[:-1], "&" + entity_name[:-1], False))
        tests.append((entity_name, characters, True))
    return sorted(tests)

def subentity_exists(entity_name, entities):
    for i in range(1, len(entity_name)):
        if entity_name[:-i] in entities:
            return True
    return False

def make_entities_code(entities):
    entities_text = "\n".join("    \"%s\": u\"%s\","%(
            name, entities[name].encode(
                "unicode-escape").replace("\"", "\\\""))
                              for name in sorted(entities.keys()))
    return """entities = {
%s
}"""%entities_text

def main():
    entities = entity_table(parse())
    tests_json = make_tests_json(entities)
    json.dump(tests_json, open("namedEntities.test", "w"), indent=4)
    code = make_entities_code(entities)
    open("entities_constants.py", "w").write(code)

if __name__ == "__main__":
    main()