|
|
|
|
from __future__ import absolute_import, division, unicode_literals
|
|
|
|
|
|
|
|
|
|
import sys
|
|
|
|
|
import os
|
|
|
|
|
import json
|
|
|
|
|
import re
|
|
|
|
|
|
|
|
|
|
import html5lib
|
|
|
|
|
from . import support
|
|
|
|
|
from . import test_tokenizer
|
|
|
|
|
|
|
|
|
|
p = html5lib.HTMLParser()
|
|
|
|
|
|
|
|
|
|
unnamespaceExpected = re.compile(r"^(\|\s*)<html ([^>]+)>", re.M).sub
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main(out_path):
|
|
|
|
|
if not os.path.exists(out_path):
|
|
|
|
|
sys.stderr.write("Path %s does not exist" % out_path)
|
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
|
|
for filename in support.get_data_files('tokenizer', '*.test'):
|
|
|
|
|
run_file(filename, out_path)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def run_file(filename, out_path):
|
|
|
|
|
try:
|
|
|
|
|
tests_data = json.load(open(filename, "r"))
|
|
|
|
|
except ValueError:
|
|
|
|
|
sys.stderr.write("Failed to load %s\n" % filename)
|
|
|
|
|
return
|
|
|
|
|
name = os.path.splitext(os.path.split(filename)[1])[0]
|
|
|
|
|
output_file = open(os.path.join(out_path, "tokenizer_%s.dat" % name), "w")
|
|
|
|
|
|
|
|
|
|
if 'tests' in tests_data:
|
|
|
|
|
for test_data in tests_data['tests']:
|
|
|
|
|
if 'initialStates' not in test_data:
|
|
|
|
|
test_data["initialStates"] = ["Data state"]
|
|
|
|
|
|
|
|
|
|
for initial_state in test_data["initialStates"]:
|
|
|
|
|
if initial_state != "Data state":
|
|
|
|
|
# don't support this yet
|
|
|
|
|
continue
|
|
|
|
|
test = make_test(test_data)
|
|
|
|
|
output_file.write(test)
|
|
|
|
|
|
|
|
|
|
output_file.close()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def make_test(test_data):
|
|
|
|
|
if 'doubleEscaped' in test_data:
|
|
|
|
|
test_data = test_tokenizer.unescape_test(test_data)
|
|
|
|
|
|
|
|
|
|
rv = []
|
|
|
|
|
rv.append("#data")
|
|
|
|
|
rv.append(test_data["input"].encode("utf8"))
|
|
|
|
|
rv.append("#errors")
|
|
|
|
|
tree = p.parse(test_data["input"])
|
|
|
|
|
output = p.tree.testSerializer(tree)
|
|
|
|
|
output = "\n".join(("| " + line[3:]) if line.startswith("| ") else line
|
|
|
|
|
for line in output.split("\n"))
|
|
|
|
|
output = unnamespaceExpected(r"\1<\2>", output)
|
|
|
|
|
rv.append(output.encode("utf8"))
|
|
|
|
|
rv.append("")
|
|
|
|
|
return "\n".join(rv)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
main(sys.argv[1])
|
