You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
bazarr/libs/subzero/modification/dictionaries/make_data.py

172 lines
5.4 KiB

6 years ago
# coding=utf-8
import re
import os
import pprint
from collections import OrderedDict
from bs4 import BeautifulSoup
TEMPLATE = """\
import re
from collections import OrderedDict
data = """
TEMPLATE_END = """\
for lang, grps in data.iteritems():
for grp in grps.iterkeys():
if data[lang][grp]["pattern"]:
data[lang][grp]["pattern"] = re.compile(data[lang][grp]["pattern"])
"""
SZ_FIX_DATA = {
"eng": {
"PartialWordsAlways": {
u"°x°": u"%",
u"compiete": u"complete",
u"Âs": u"'s",
u"ÃÂs": u"'s",
u"a/ion": u"ation",
u"at/on": u"ation",
u"l/an": u"lian",
u"lljust": u"ll just",
u" L ": u" I ",
u" l ": u" I ",
u"'sjust": u"'s just",
u"'tjust": u"'t just",
u"\";": u"'s",
6 years ago
},
"WholeWords": {
u"I'11": u"I'll",
u"III'll": u"I'll",
u"Tun": u"Run",
u"pan'": u"part",
u"al'": u"at",
u"a re": u"are",
u"wail'": u"wait",
u"he)'": u"hey",
u"he)\"": u"hey",
u"He)'": u"Hey",
u"He)\"": u"Hey",
u"He)": u"Hey",
u"Yea h": u"Yeah",
u"yea h": u"yeah",
u"h is": u"his",
u" 're ": u"'re ",
u"LAst": u"Last",
u"forthis": u"for this",
u"Ls": u"Is",
u"Iam": u"I am",
u"Ican": u"I can",
},
"PartialLines": {
u"L know": u"I know",
u"L should": u"I should",
u"L do": u"I do",
u"L would": u"I would",
u"L could": u"I could",
u"L can": u"I can",
u"L happen": u"I happen",
u"L might": u"I might",
u"L have ": u"I have",
u"L had": u"I had",
u"L want": u"I want",
u"L was": u"I was",
u"L am": u"I am",
u"L will": u"I will",
u"L suggest": u"I suggest",
u"L think": u"I think",
u"L reckon": u"I reckon",
u"L like": u"I like",
u"L love": u"I love",
u"L don't": u"I don't",
u"L didn't": u"I didn't",
u"L wasn't": u"I wasnt't",
u"L haven't": u"I haven't",
u"L couldn't": u"I couldn't",
u"L won't": u"I won't",
u"H i": u"Hi",
},
"BeginLines": {
u"l ": u"I ",
u"L ": u"I ",
}
},
"nld": {
"PartialWordsAlways": {
u"ט": u"è",
u"י": u"é",
u"כ": u"ë",
u"צ": u"ë",
u"ן": u"ï",
u"ף": u"ó",
u"א": u"à",
u"": u"I",
u"č": u"è",
u"פ": u"o",
u"ם": u"i",
},
},
"swe": {
"PartialWordsAlways": {
u"ĺ": u"å",
u"Ĺ": u"Å",
}
}
}
SZ_FIX_DATA_GLOBAL = {
}
if __name__ == "__main__":
cur_dir = os.path.dirname(os.path.realpath(__file__))
xml_dir = os.path.join(cur_dir, "xml")
file_list = os.listdir(xml_dir)
data = {}
for fn in file_list:
if fn.endswith("_OCRFixReplaceList.xml"):
lang = fn.split("_")[0]
soup = BeautifulSoup(open(os.path.join(xml_dir, fn)), "xml")
fetch_data = (
# group, item_name, pattern
("WholeLines", "Line", None),
("WholeWords", "Word", lambda d: (ur"(?um)(\b|^)(?:" + u"|".join([re.escape(k) for k in d.keys()])
+ ur')(\b|$)') if d else None),
("PartialWordsAlways", "WordPart", None),
("PartialLines", "LinePart", lambda d: (ur"(?um)(?:(?<=\s)|(?<=^)|(?<=\b))(?:" +
u"|".join([re.escape(k) for k in d.keys()]) +
ur")(?:(?=\s)|(?=$)|(?=\b))") if d else None),
("BeginLines", "Beginning", lambda d: (ur"(?um)^(?:"+u"|".join([re.escape(k) for k in d.keys()])
+ ur')') if d else None),
("EndLines", "Ending", lambda d: (ur"(?um)(?:" + u"|".join([re.escape(k) for k in d.keys()]) +
ur")$") if d else None,),
)
data[lang] = dict((grp, {"data": OrderedDict(), "pattern": None}) for grp, item_name, pattern in fetch_data)
for grp, item_name, pattern in fetch_data:
for grp_data in soup.find_all(grp):
for line in grp_data.find_all(item_name):
data[lang][grp]["data"][line["from"]] = line["to"]
# add our own dictionaries
if lang in SZ_FIX_DATA and grp in SZ_FIX_DATA[lang]:
data[lang][grp]["data"].update(SZ_FIX_DATA[lang][grp])
if grp in SZ_FIX_DATA_GLOBAL:
data[lang][grp]["data"].update(SZ_FIX_DATA_GLOBAL[grp])
if pattern:
data[lang][grp]["pattern"] = pattern(data[lang][grp]["data"])
f = open(os.path.join(cur_dir, "data.py"), "w+")
f.write(TEMPLATE)
f.write(pprint.pformat(data, width=1))
f.write(TEMPLATE_END)
f.close()