|
|
"""Guess the natural language of a text
|
|
|
"""
|
|
|
# © 2012 spirit <hiddenspirit@gmail.com>
|
|
|
# https://bitbucket.org/spirit/guess_language
|
|
|
#
|
|
|
# Original Python package:
|
|
|
# Copyright (c) 2008, Kent S Johnson
|
|
|
# http://code.google.com/p/guess-language/
|
|
|
#
|
|
|
# Original C++ version for KDE:
|
|
|
# Copyright (c) 2006 Jacob R Rideout <kde@jacobrideout.net>
|
|
|
# http://websvn.kde.org/branches/work/sonnet-refactoring/common/nlp/guesslanguage.cpp?view=markup
|
|
|
#
|
|
|
# Original Language::Guess Perl module:
|
|
|
# Copyright (c) 2004-2006 Maciej Ceglowski
|
|
|
# http://web.archive.org/web/20090228163219/http://languid.cantbedone.org/
|
|
|
#
|
|
|
# Note: Language::Guess is GPL-licensed. KDE developers received permission
|
|
|
# from the author to distribute their port under LGPL:
|
|
|
# http://lists.kde.org/?l=kde-sonnet&m=116910092228811&w=2
|
|
|
#
|
|
|
# This program is free software: you can redistribute it and/or modify it
|
|
|
# under the terms of the GNU Lesser General Public License as published
|
|
|
# by the Free Software Foundation, either version 3 of the License,
|
|
|
# or (at your option) any later version.
|
|
|
#
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty
|
|
|
# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
|
# See the GNU Lesser General Public License for more details.
|
|
|
#
|
|
|
# You should have received a copy of the GNU Lesser General Public License
|
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
import functools
|
|
|
import re
|
|
|
import warnings
|
|
|
|
|
|
from collections import defaultdict, OrderedDict
|
|
|
|
|
|
from .data import BLOCKS, BLOCK_RSHIFT
|
|
|
|
|
|
|
|
|
__all__ = [
|
|
|
"guess_language", "use_enchant",
|
|
|
]
|
|
|
|
|
|
MAX_LENGTH = 4096
|
|
|
MIN_LENGTH = 20
|
|
|
MAX_GRAMS = 300
|
|
|
WORD_RE = re.compile(r"(?:[^\W\d_]|['’])+", re.U)
|
|
|
MODEL_ROOT = __name__ + ".data.models."
|
|
|
FALLBACK_LANGUAGE = "en_US"
|
|
|
|
|
|
BASIC_LATIN = {
|
|
|
"ceb", "en", "eu", "ha", "haw", "id", "la", "nr", "nso", "so", "ss", "st",
|
|
|
"sw", "tlh", "tn", "ts", "xh", "zu"
|
|
|
}
|
|
|
EXTENDED_LATIN = {
|
|
|
"af", "az", "ca", "cs", "cy", "da", "de", "eo", "es", "et", "fi", "fr",
|
|
|
"hr", "hu", "is", "it", "lt", "lv", "nb", "nl", "pl", "pt", "ro", "sk",
|
|
|
"sl", "sq", "sv", "tl", "tr", "ve", "vi"
|
|
|
}
|
|
|
ALL_LATIN = BASIC_LATIN.union(EXTENDED_LATIN)
|
|
|
CYRILLIC = {"bg", "kk", "ky", "mk", "mn", "ru", "sr", "uk", "uz"}
|
|
|
ARABIC = {"ar", "fa", "ps", "ur"}
|
|
|
DEVANAGARI = {"hi", "ne"}
|
|
|
PT = {"pt_BR", "pt_PT"}
|
|
|
|
|
|
# NOTE mn appears twice, once for mongolian script and once for CYRILLIC
|
|
|
SINGLETONS = [
|
|
|
("Armenian", "hy"),
|
|
|
("Hebrew", "he"),
|
|
|
("Bengali", "bn"),
|
|
|
("Gurmukhi", "pa"),
|
|
|
("Greek", "el"),
|
|
|
("Gujarati", "gu"),
|
|
|
("Oriya", "or"),
|
|
|
("Tamil", "ta"),
|
|
|
("Telugu", "te"),
|
|
|
("Kannada", "kn"),
|
|
|
("Malayalam", "ml"),
|
|
|
("Sinhala", "si"),
|
|
|
("Thai", "th"),
|
|
|
("Lao", "lo"),
|
|
|
("Tibetan", "bo"),
|
|
|
("Burmese", "my"),
|
|
|
("Georgian", "ka"),
|
|
|
("Mongolian", "mn-Mong"),
|
|
|
("Khmer", "km"),
|
|
|
]
|
|
|
|
|
|
NAME_MAP = {
|
|
|
"ab": "Abkhazian",
|
|
|
"af": "Afrikaans",
|
|
|
"ar": "Arabic",
|
|
|
"az": "Azeri",
|
|
|
"be": "Byelorussian",
|
|
|
"bg": "Bulgarian",
|
|
|
"bn": "Bengali",
|
|
|
"bo": "Tibetan",
|
|
|
"br": "Breton",
|
|
|
"ca": "Catalan",
|
|
|
"ceb": "Cebuano",
|
|
|
"cs": "Czech",
|
|
|
"cy": "Welsh",
|
|
|
"da": "Danish",
|
|
|
"de": "German",
|
|
|
"el": "Greek",
|
|
|
"en": "English",
|
|
|
"eo": "Esperanto",
|
|
|
"es": "Spanish",
|
|
|
"et": "Estonian",
|
|
|
"eu": "Basque",
|
|
|
"fa": "Farsi",
|
|
|
"fi": "Finnish",
|
|
|
"fo": "Faroese",
|
|
|
"fr": "French",
|
|
|
"fy": "Frisian",
|
|
|
"gd": "Scots Gaelic",
|
|
|
"gl": "Galician",
|
|
|
"gu": "Gujarati",
|
|
|
"ha": "Hausa",
|
|
|
"haw": "Hawaiian",
|
|
|
"he": "Hebrew",
|
|
|
"hi": "Hindi",
|
|
|
"hr": "Croatian",
|
|
|
"hu": "Hungarian",
|
|
|
"hy": "Armenian",
|
|
|
"id": "Indonesian",
|
|
|
"is": "Icelandic",
|
|
|
"it": "Italian",
|
|
|
"ja": "Japanese",
|
|
|
"ka": "Georgian",
|
|
|
"kk": "Kazakh",
|
|
|
"km": "Cambodian",
|
|
|
"ko": "Korean",
|
|
|
"ku": "Kurdish",
|
|
|
"ky": "Kyrgyz",
|
|
|
"la": "Latin",
|
|
|
"lt": "Lithuanian",
|
|
|
"lv": "Latvian",
|
|
|
"mg": "Malagasy",
|
|
|
"mk": "Macedonian",
|
|
|
"ml": "Malayalam",
|
|
|
"mn": "Mongolian",
|
|
|
"mr": "Marathi",
|
|
|
"ms": "Malay",
|
|
|
"nd": "Ndebele",
|
|
|
"ne": "Nepali",
|
|
|
"nl": "Dutch",
|
|
|
"nn": "Nynorsk",
|
|
|
"no": "Norwegian",
|
|
|
"nso": "Sepedi",
|
|
|
"pa": "Punjabi",
|
|
|
"pl": "Polish",
|
|
|
"ps": "Pashto",
|
|
|
"pt": "Portuguese",
|
|
|
"pt_PT": "Portuguese (Portugal)",
|
|
|
"pt_BR": "Portuguese (Brazil)",
|
|
|
"ro": "Romanian",
|
|
|
"ru": "Russian",
|
|
|
"sa": "Sanskrit",
|
|
|
"sh": "Serbo-Croatian",
|
|
|
"sk": "Slovak",
|
|
|
"sl": "Slovene",
|
|
|
"so": "Somali",
|
|
|
"sq": "Albanian",
|
|
|
"sr": "Serbian",
|
|
|
"sv": "Swedish",
|
|
|
"sw": "Swahili",
|
|
|
"ta": "Tamil",
|
|
|
"te": "Telugu",
|
|
|
"th": "Thai",
|
|
|
"tl": "Tagalog",
|
|
|
"tlh": "Klingon",
|
|
|
"tn": "Setswana",
|
|
|
"tr": "Turkish",
|
|
|
"ts": "Tsonga",
|
|
|
"tw": "Twi",
|
|
|
"uk": "Ukrainian",
|
|
|
"ur": "Urdu",
|
|
|
"uz": "Uzbek",
|
|
|
"ve": "Venda",
|
|
|
"vi": "Vietnamese",
|
|
|
"xh": "Xhosa",
|
|
|
"zh": "Chinese",
|
|
|
"zh_TW": "Traditional Chinese (Taiwan)",
|
|
|
"zu": "Zulu",
|
|
|
}
|
|
|
|
|
|
IANA_MAP = {
|
|
|
"ab": 12026,
|
|
|
"af": 40,
|
|
|
"ar": 26020,
|
|
|
"az": 26030,
|
|
|
"be": 11890,
|
|
|
"bg": 26050,
|
|
|
"bn": 26040,
|
|
|
"bo": 26601,
|
|
|
"br": 1361,
|
|
|
"ca": 3,
|
|
|
"ceb": 26060,
|
|
|
"cs": 26080,
|
|
|
"cy": 26560,
|
|
|
"da": 26090,
|
|
|
"de": 26160,
|
|
|
"el": 26165,
|
|
|
"en": 26110,
|
|
|
"eo": 11933,
|
|
|
"es": 26460,
|
|
|
"et": 26120,
|
|
|
"eu": 1232,
|
|
|
"fa": 26130,
|
|
|
"fi": 26140,
|
|
|
"fo": 11817,
|
|
|
"fr": 26150,
|
|
|
"fy": 1353,
|
|
|
"gd": 65555,
|
|
|
"gl": 1252,
|
|
|
"gu": 26599,
|
|
|
"ha": 26170,
|
|
|
"haw": 26180,
|
|
|
"he": 26592,
|
|
|
"hi": 26190,
|
|
|
"hr": 26070,
|
|
|
"hu": 26200,
|
|
|
"hy": 26597,
|
|
|
"id": 26220,
|
|
|
"is": 26210,
|
|
|
"it": 26230,
|
|
|
"ja": 26235,
|
|
|
"ka": 26600,
|
|
|
"kk": 26240,
|
|
|
"km": 1222,
|
|
|
"ko": 26255,
|
|
|
"ku": 11815,
|
|
|
"ky": 26260,
|
|
|
"la": 26280,
|
|
|
"lt": 26300,
|
|
|
"lv": 26290,
|
|
|
"mg": 1362,
|
|
|
"mk": 26310,
|
|
|
"ml": 26598,
|
|
|
"mn": 26320,
|
|
|
"mr": 1201,
|
|
|
"ms": 1147,
|
|
|
"ne": 26330,
|
|
|
"nl": 26100,
|
|
|
"nn": 172,
|
|
|
"no": 26340,
|
|
|
"pa": 65550,
|
|
|
"pl": 26380,
|
|
|
"ps": 26350,
|
|
|
"pt": 26390,
|
|
|
"ro": 26400,
|
|
|
"ru": 26410,
|
|
|
"sa": 1500,
|
|
|
"sh": 1399,
|
|
|
"sk": 26430,
|
|
|
"sl": 26440,
|
|
|
"so": 26450,
|
|
|
"sq": 26010,
|
|
|
"sr": 26420,
|
|
|
"sv": 26480,
|
|
|
"sw": 26470,
|
|
|
"ta": 26595,
|
|
|
"te": 26596,
|
|
|
"th": 26594,
|
|
|
"tl": 26490,
|
|
|
"tlh": 26250,
|
|
|
"tn": 65578,
|
|
|
"tr": 26500,
|
|
|
"tw": 1499,
|
|
|
"uk": 26520,
|
|
|
"ur": 26530,
|
|
|
"uz": 26540,
|
|
|
"vi": 26550,
|
|
|
"zh": 26065,
|
|
|
"zh_TW": 22,
|
|
|
}
|
|
|
|
|
|
models = {}
|
|
|
|
|
|
try:
|
|
|
from importlib import import_module
|
|
|
except ImportError:
|
|
|
import sys
|
|
|
|
|
|
def import_module(name):
|
|
|
"""Import a module.
|
|
|
"""
|
|
|
__import__(name)
|
|
|
return sys.modules[name]
|
|
|
|
|
|
try:
|
|
|
from collections import namedtuple
|
|
|
|
|
|
LanguageInfo = namedtuple("LanguageInfo", ["tag", "id", "name"])
|
|
|
except ImportError:
|
|
|
class LanguageInfo(tuple):
|
|
|
def __new__(cls, tag, id, name): #@ReservedAssignment
|
|
|
return tuple.__new__(cls, (tag, id, name))
|
|
|
|
|
|
def __init__(self, tag, id, name): #@ReservedAssignment
|
|
|
self.tag = tag
|
|
|
self.id = id
|
|
|
self.name = name
|
|
|
|
|
|
|
|
|
class UNKNOWN(str):
|
|
|
"""Unknown language
|
|
|
"""
|
|
|
def __bool__(self):
|
|
|
return False
|
|
|
|
|
|
|
|
|
UNKNOWN = UNKNOWN("UNKNOWN")
|
|
|
|
|
|
|
|
|
def guess_language(text: str, hints=None):
|
|
|
"""Return the ISO 639-1 language code.
|
|
|
"""
|
|
|
words = WORD_RE.findall(text[:MAX_LENGTH].replace("’", "'"))
|
|
|
return identify(words, find_runs(words), hints)
|
|
|
|
|
|
|
|
|
def guess_language_info(text: str, hints=None):
|
|
|
"""Return LanguageInfo(tag, id, name).
|
|
|
"""
|
|
|
tag = guess_language(text, hints)
|
|
|
|
|
|
if tag is UNKNOWN:
|
|
|
return LanguageInfo(UNKNOWN, UNKNOWN, UNKNOWN)
|
|
|
|
|
|
return LanguageInfo(tag, _get_id(tag), _get_name(tag))
|
|
|
|
|
|
|
|
|
# An alias for guess_language
|
|
|
guess_language_tag = guess_language
|
|
|
|
|
|
|
|
|
def guess_language_id(text: str, hints=None):
|
|
|
"""Return the language ID.
|
|
|
"""
|
|
|
return _get_id(guess_language(text, hints))
|
|
|
|
|
|
|
|
|
def guess_language_name(text: str, hints=None):
|
|
|
"""Return the language name (in English).
|
|
|
"""
|
|
|
return _get_name(guess_language(text, hints))
|
|
|
|
|
|
|
|
|
def _get_id(tag):
|
|
|
return IANA_MAP.get(tag, UNKNOWN)
|
|
|
|
|
|
|
|
|
def _get_name(tag):
|
|
|
return NAME_MAP.get(tag, UNKNOWN)
|
|
|
|
|
|
|
|
|
def find_runs(words):
|
|
|
"""Count the number of characters in each character block.
|
|
|
"""
|
|
|
run_types = defaultdict(int)
|
|
|
|
|
|
total_count = 0
|
|
|
|
|
|
for word in words:
|
|
|
for char in word:
|
|
|
block = BLOCKS[ord(char) >> BLOCK_RSHIFT]
|
|
|
run_types[block] += 1
|
|
|
total_count += 1
|
|
|
|
|
|
#pprint(run_types)
|
|
|
|
|
|
# return run types that used for 40% or more of the string
|
|
|
# return Basic Latin if found more than 15%
|
|
|
## and extended additional latin if over 10% (for Vietnamese)
|
|
|
relevant_runs = []
|
|
|
for key, value in run_types.items():
|
|
|
pct = value * 100 // total_count
|
|
|
if pct >= 40 or pct >= 15 and key == "Basic Latin":
|
|
|
relevant_runs.append(key)
|
|
|
#elif pct >= 10 and key == "Latin Extended Additional":
|
|
|
#relevant_runs.append(key)
|
|
|
|
|
|
return relevant_runs
|
|
|
|
|
|
|
|
|
def identify(words, scripts, hints=None):
|
|
|
"""Identify the language.
|
|
|
"""
|
|
|
if ("Hangul Syllables" in scripts or "Hangul Jamo" in scripts or
|
|
|
"Hangul Compatibility Jamo" in scripts or "Hangul" in scripts):
|
|
|
return "ko"
|
|
|
|
|
|
if "Greek and Coptic" in scripts:
|
|
|
return "el"
|
|
|
|
|
|
if "Kana" in scripts:
|
|
|
return "ja"
|
|
|
|
|
|
if ("CJK Unified Ideographs" in scripts or "Bopomofo" in scripts or
|
|
|
"Bopomofo Extended" in scripts or "KangXi Radicals" in scripts):
|
|
|
# This is in both Ceglowski and Rideout
|
|
|
# I can't imagine why...
|
|
|
# or "Arabic Presentation Forms-A" in scripts
|
|
|
return "zh"
|
|
|
|
|
|
if "Cyrillic" in scripts:
|
|
|
return check(words, filter_languages(CYRILLIC, hints))
|
|
|
|
|
|
if ("Arabic" in scripts or "Arabic Presentation Forms-A" in scripts or
|
|
|
"Arabic Presentation Forms-B" in scripts):
|
|
|
return check(words, filter_languages(ARABIC, hints))
|
|
|
|
|
|
if "Devanagari" in scripts:
|
|
|
return check(words, filter_languages(DEVANAGARI, hints))
|
|
|
|
|
|
# Try languages with unique scripts
|
|
|
for block_name, lang_name in SINGLETONS:
|
|
|
if block_name in scripts:
|
|
|
return lang_name
|
|
|
|
|
|
#if "Latin Extended Additional" in scripts:
|
|
|
#return "vi"
|
|
|
|
|
|
if "Extended Latin" in scripts:
|
|
|
latin_lang = check(words, filter_languages(EXTENDED_LATIN, hints))
|
|
|
if latin_lang == "pt":
|
|
|
return check(words, filter_languages(PT))
|
|
|
else:
|
|
|
return latin_lang
|
|
|
|
|
|
if "Basic Latin" in scripts:
|
|
|
return check(words, filter_languages(ALL_LATIN, hints))
|
|
|
|
|
|
return UNKNOWN
|
|
|
|
|
|
|
|
|
def filter_languages(languages, hints=None):
|
|
|
"""Filter languages.
|
|
|
"""
|
|
|
return languages.intersection(hints) if hints else languages
|
|
|
|
|
|
|
|
|
def check_with_all(words, languages):
|
|
|
"""Check what the best match is.
|
|
|
"""
|
|
|
return (check_with_enchant(words, languages) or
|
|
|
check_with_models(words, languages))
|
|
|
|
|
|
|
|
|
check = check_with_all
|
|
|
|
|
|
|
|
|
def use_enchant(use_enchant=True):
|
|
|
"""Enable or disable checking with PyEnchant.
|
|
|
"""
|
|
|
global check
|
|
|
check = check_with_all if use_enchant else check_with_models
|
|
|
|
|
|
|
|
|
def check_with_models(words, languages):
|
|
|
"""Check against known models.
|
|
|
"""
|
|
|
sample = " ".join(words)
|
|
|
|
|
|
if len(sample) < MIN_LENGTH:
|
|
|
return UNKNOWN
|
|
|
|
|
|
scores = []
|
|
|
model = create_ordered_model(sample) # QMap<int,QString>
|
|
|
|
|
|
for key in languages:
|
|
|
lkey = key.lower()
|
|
|
|
|
|
try:
|
|
|
known_model = models[lkey]
|
|
|
except KeyError:
|
|
|
try:
|
|
|
known_model = import_module(MODEL_ROOT + lkey).model
|
|
|
except ImportError:
|
|
|
known_model = None
|
|
|
models[lkey] = known_model
|
|
|
|
|
|
if known_model:
|
|
|
scores.append((distance(model, known_model), key))
|
|
|
|
|
|
if not scores:
|
|
|
return UNKNOWN
|
|
|
|
|
|
# we want the lowest score, less distance = greater chance of match
|
|
|
#pprint(sorted(scores))
|
|
|
return min(scores)[1]
|
|
|
|
|
|
|
|
|
def create_ordered_model(content):
|
|
|
"""Create a list of trigrams in content sorted by frequency.
|
|
|
"""
|
|
|
trigrams = defaultdict(int) # QHash<QString,int>
|
|
|
content = content.lower()
|
|
|
|
|
|
for i in range(len(content) - 2):
|
|
|
trigrams[content[i:i+3]] += 1
|
|
|
|
|
|
return sorted(trigrams.keys(), key=lambda k: (-trigrams[k], k))
|
|
|
|
|
|
|
|
|
def distance(model, known_model):
|
|
|
"""Calculate the distance to the known model.
|
|
|
"""
|
|
|
dist = 0
|
|
|
|
|
|
for i, value in enumerate(model[:MAX_GRAMS]):
|
|
|
if value in known_model:
|
|
|
dist += abs(i - known_model[value])
|
|
|
else:
|
|
|
dist += MAX_GRAMS
|
|
|
|
|
|
return dist
|
|
|
|
|
|
|
|
|
try:
|
|
|
import enchant
|
|
|
except ImportError:
|
|
|
warnings.warn("PyEnchant is unavailable", ImportWarning)
|
|
|
enchant = None
|
|
|
|
|
|
def check_with_enchant(*args, **kwargs):
|
|
|
return UNKNOWN
|
|
|
else:
|
|
|
import locale
|
|
|
|
|
|
enchant_base_languages_dict = None
|
|
|
|
|
|
def check_with_enchant(words, languages,
|
|
|
threshold=0.7, min_words=1, dictionaries={}):
|
|
|
"""Check against installed spelling dictionaries.
|
|
|
"""
|
|
|
if len(words) < min_words:
|
|
|
return UNKNOWN
|
|
|
|
|
|
best_score = 0
|
|
|
best_tag = UNKNOWN
|
|
|
|
|
|
for tag, enchant_tag in get_enchant_base_languages_dict().items():
|
|
|
if tag not in languages:
|
|
|
continue
|
|
|
try:
|
|
|
d = dictionaries[tag]
|
|
|
except KeyError:
|
|
|
d = dictionaries[tag] = enchant.Dict(enchant_tag)
|
|
|
score = sum([1 for word in words if d.check(word)])
|
|
|
if score > best_score:
|
|
|
best_score = score
|
|
|
best_tag = tag
|
|
|
|
|
|
if best_score / len(words) < threshold:
|
|
|
return UNKNOWN
|
|
|
|
|
|
return best_tag
|
|
|
|
|
|
def get_enchant_base_languages_dict():
|
|
|
"""Get ordered dictionary of enchant base languages.
|
|
|
|
|
|
locale_language, then "en", then the rest.
|
|
|
"""
|
|
|
global enchant_base_languages_dict
|
|
|
if enchant_base_languages_dict is None:
|
|
|
def get_language_sub_tag(tag):
|
|
|
return tag.split("_")[0]
|
|
|
enchant_base_languages_dict = OrderedDict()
|
|
|
enchant_languages = sorted(enchant.list_languages())
|
|
|
for full_tag in [get_locale_language(), FALLBACK_LANGUAGE]:
|
|
|
sub_tag = get_language_sub_tag(full_tag)
|
|
|
if sub_tag not in enchant_base_languages_dict:
|
|
|
for tag in [full_tag, sub_tag]:
|
|
|
try:
|
|
|
index = enchant_languages.index(tag)
|
|
|
except ValueError:
|
|
|
pass
|
|
|
else:
|
|
|
enchant_base_languages_dict[sub_tag] = tag
|
|
|
del enchant_languages[index]
|
|
|
break
|
|
|
for tag in enchant_languages:
|
|
|
sub_tag = get_language_sub_tag(tag)
|
|
|
if sub_tag not in enchant_base_languages_dict:
|
|
|
enchant_base_languages_dict[sub_tag] = tag
|
|
|
return enchant_base_languages_dict
|
|
|
|
|
|
def get_locale_language():
|
|
|
"""Get the language code for the current locale setting.
|
|
|
"""
|
|
|
return (locale.getlocale()[0] or locale.getdefaultlocale()[0] or
|
|
|
FALLBACK_LANGUAGE)
|
|
|
|
|
|
|
|
|
def deprecated(func):
|
|
|
"""This is a decorator which can be used to mark functions
|
|
|
as deprecated. It will result in a warning being emitted
|
|
|
when the function is used.
|
|
|
"""
|
|
|
@functools.wraps(func)
|
|
|
def new_func(*args, **kwargs):
|
|
|
warnings.warn(
|
|
|
"call to deprecated function %s()" % func.__name__,
|
|
|
category=DeprecationWarning,
|
|
|
stacklevel=2
|
|
|
)
|
|
|
return func(*args, **kwargs)
|
|
|
return new_func
|
|
|
|
|
|
|
|
|
@deprecated
|
|
|
def guessLanguage(text):
|
|
|
"""Deprecated function - use guess_language() instead.
|
|
|
"""
|
|
|
return guess_language(decode_text(text))
|
|
|
|
|
|
|
|
|
@deprecated
|
|
|
def guessLanguageTag(text):
|
|
|
"""Deprecated function - use guess_language_tag() instead.
|
|
|
"""
|
|
|
return guess_language_tag(decode_text(text))
|
|
|
|
|
|
|
|
|
@deprecated
|
|
|
def guessLanguageId(text):
|
|
|
"""Deprecated function - use guess_language_id() instead.
|
|
|
"""
|
|
|
return guess_language_id(decode_text(text))
|
|
|
|
|
|
|
|
|
@deprecated
|
|
|
def guessLanguageName(text):
|
|
|
"""Deprecated function - use guess_language_name() instead.
|
|
|
"""
|
|
|
return guess_language_name(decode_text(text))
|
|
|
|
|
|
|
|
|
@deprecated
|
|
|
def guessLanguageInfo(text):
|
|
|
"""Deprecated function - use guess_language_info() instead.
|
|
|
"""
|
|
|
return guess_language_info(decode_text(text))
|
|
|
|
|
|
|
|
|
def decode_text(text, encoding="utf-8"):
|
|
|
"""Decode text if needed (for deprecated functions).
|
|
|
"""
|
|
|
if not isinstance(text, str):
|
|
|
warnings.warn("passing an encoded string is deprecated",
|
|
|
DeprecationWarning, 4)
|
|
|
text = text.decode(encoding)
|
|
|
return text
|
