from __future__ import absolute_import, division, unicode_literals

import sys
import os
import json
import re

import html5lib
from . import support
from . import test_tokenizer

p = html5lib.HTMLParser()

unnamespaceExpected = re.compile(r"^(\|\s*)<html ([^>]+)>", re.M).sub


def main(out_path):
    if not os.path.exists(out_path):
        sys.stderr.write("Path %s does not exist" % out_path)
        sys.exit(1)

    for filename in support.get_data_files('tokenizer', '*.test'):
        run_file(filename, out_path)


def run_file(filename, out_path):
    try:
        tests_data = json.load(open(filename, "r"))
    except ValueError:
        sys.stderr.write("Failed to load %s\n" % filename)
        return
    name = os.path.splitext(os.path.split(filename)[1])[0]
    output_file = open(os.path.join(out_path, "tokenizer_%s.dat" % name), "w")

    if 'tests' in tests_data:
        for test_data in tests_data['tests']:
            if 'initialStates' not in test_data:
                test_data["initialStates"] = ["Data state"]

            for initial_state in test_data["initialStates"]:
                if initial_state != "Data state":
                    # don't support this yet
                    continue
                test = make_test(test_data)
                output_file.write(test)

    output_file.close()


def make_test(test_data):
    if 'doubleEscaped' in test_data:
        test_data = test_tokenizer.unescape_test(test_data)

    rv = []
    rv.append("#data")
    rv.append(test_data["input"].encode("utf8"))
    rv.append("#errors")
    tree = p.parse(test_data["input"])
    output = p.tree.testSerializer(tree)
    output = "\n".join(("| " + line[3:]) if line.startswith("|  ") else line
                       for line in output.split("\n"))
    output = unnamespaceExpected(r"\1<\2>", output)
    rv.append(output.encode("utf8"))
    rv.append("")
    return "\n".join(rv)

if __name__ == "__main__":
    main(sys.argv[1])