""" Guess the natural language of a text
"""
# © 2012 spirit <hiddenspirit@gmail.com>
# https://bitbucket.org/spirit/guess_language
#
# Original Python package:
# Copyright (c) 2008, Kent S Johnson
# http://code.google.com/p/guess-language/
#
# Original C++ version for KDE:
# Copyright (c) 2006 Jacob R Rideout <kde@jacobrideout.net>
# http://websvn.kde.org/branches/work/sonnet-refactoring/common/nlp/guesslanguage.cpp?view=markup
#
# Original Language::Guess Perl module:
# Copyright (c) 2004-2006 Maciej Ceglowski
# http://web.archive.org/web/20090228163219/http://languid.cantbedone.org/
#
# Note: Language::Guess is GPL-licensed. KDE developers received permission
# from the author to distribute their port under LGPL:
# http://lists.kde.org/?l=kde-sonnet&m=116910092228811&w=2
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 3 of the License,
# or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty
# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import functools
import re
import warnings
from collections import defaultdict , OrderedDict
from . data import BLOCKS , BLOCK_RSHIFT
__all__ = [
" guess_language " , " use_enchant " ,
]
MAX_LENGTH = 4096
MIN_LENGTH = 20
MAX_GRAMS = 300
WORD_RE = re . compile ( r " (?:[^ \ W \ d_]|[ ' ’ " , re . U )
MODEL_ROOT = __name__ + " .data.models. "
FALLBACK_LANGUAGE = " en_US "
BASIC_LATIN = {
" ceb " , " en " , " eu " , " ha " , " haw " , " id " , " la " , " nr " , " nso " , " so " , " ss " , " st " ,
" sw " , " tlh " , " tn " , " ts " , " xh " , " zu "
}
EXTENDED_LATIN = {
" af " , " az " , " ca " , " cs " , " cy " , " da " , " de " , " eo " , " es " , " et " , " fi " , " fr " ,
" hr " , " hu " , " is " , " it " , " lt " , " lv " , " nb " , " nl " , " pl " , " pt " , " ro " , " sk " ,
" sl " , " sq " , " sv " , " tl " , " tr " , " ve " , " vi "
}
ALL_LATIN = BASIC_LATIN . union ( EXTENDED_LATIN )
CYRILLIC = { " bg " , " kk " , " ky " , " mk " , " mn " , " ru " , " sr " , " uk " , " uz " }
ARABIC = { " ar " , " fa " , " ps " , " ur " }
DEVANAGARI = { " hi " , " ne " }
PT = { " pt_BR " , " pt_PT " }
# NOTE mn appears twice, once for mongolian script and once for CYRILLIC
SINGLETONS = [
( " Armenian " , " hy " ) ,
( " Hebrew " , " he " ) ,
( " Bengali " , " bn " ) ,
( " Gurmukhi " , " pa " ) ,
( " Greek " , " el " ) ,
( " Gujarati " , " gu " ) ,
( " Oriya " , " or " ) ,
( " Tamil " , " ta " ) ,
( " Telugu " , " te " ) ,
( " Kannada " , " kn " ) ,
( " Malayalam " , " ml " ) ,
( " Sinhala " , " si " ) ,
( " Thai " , " th " ) ,
( " Lao " , " lo " ) ,
( " Tibetan " , " bo " ) ,
( " Burmese " , " my " ) ,
( " Georgian " , " ka " ) ,
( " Mongolian " , " mn-Mong " ) ,
( " Khmer " , " km " ) ,
]
NAME_MAP = {
" ab " : " Abkhazian " ,
" af " : " Afrikaans " ,
" ar " : " Arabic " ,
" az " : " Azeri " ,
" be " : " Byelorussian " ,
" bg " : " Bulgarian " ,
" bn " : " Bengali " ,
" bo " : " Tibetan " ,
" br " : " Breton " ,
" ca " : " Catalan " ,
" ceb " : " Cebuano " ,
" cs " : " Czech " ,
" cy " : " Welsh " ,
" da " : " Danish " ,
" de " : " German " ,
" el " : " Greek " ,
" en " : " English " ,
" eo " : " Esperanto " ,
" es " : " Spanish " ,
" et " : " Estonian " ,
" eu " : " Basque " ,
" fa " : " Farsi " ,
" fi " : " Finnish " ,
" fo " : " Faroese " ,
" fr " : " French " ,
" fy " : " Frisian " ,
" gd " : " Scots Gaelic " ,
" gl " : " Galician " ,
" gu " : " Gujarati " ,
" ha " : " Hausa " ,
" haw " : " Hawaiian " ,
" he " : " Hebrew " ,
" hi " : " Hindi " ,
" hr " : " Croatian " ,
" hu " : " Hungarian " ,
" hy " : " Armenian " ,
" id " : " Indonesian " ,
" is " : " Icelandic " ,
" it " : " Italian " ,
" ja " : " Japanese " ,
" ka " : " Georgian " ,
" kk " : " Kazakh " ,
" km " : " Cambodian " ,
" ko " : " Korean " ,
" ku " : " Kurdish " ,
" ky " : " Kyrgyz " ,
" la " : " Latin " ,
" lt " : " Lithuanian " ,
" lv " : " Latvian " ,
" mg " : " Malagasy " ,
" mk " : " Macedonian " ,
" ml " : " Malayalam " ,
" mn " : " Mongolian " ,
" mr " : " Marathi " ,
" ms " : " Malay " ,
" nd " : " Ndebele " ,
" ne " : " Nepali " ,
" nl " : " Dutch " ,
" nn " : " Nynorsk " ,
" no " : " Norwegian " ,
" nso " : " Sepedi " ,
" pa " : " Punjabi " ,
" pl " : " Polish " ,
" ps " : " Pashto " ,
" pt " : " Portuguese " ,
" pt_PT " : " Portuguese (Portugal) " ,
" pt_BR " : " Portuguese (Brazil) " ,
" ro " : " Romanian " ,
" ru " : " Russian " ,
" sa " : " Sanskrit " ,
" sh " : " Serbo-Croatian " ,
" sk " : " Slovak " ,
" sl " : " Slovene " ,
" so " : " Somali " ,
" sq " : " Albanian " ,
" sr " : " Serbian " ,
" sv " : " Swedish " ,
" sw " : " Swahili " ,
" ta " : " Tamil " ,
" te " : " Telugu " ,
" th " : " Thai " ,
" tl " : " Tagalog " ,
" tlh " : " Klingon " ,
" tn " : " Setswana " ,
" tr " : " Turkish " ,
" ts " : " Tsonga " ,
" tw " : " Twi " ,
" uk " : " Ukrainian " ,
" ur " : " Urdu " ,
" uz " : " Uzbek " ,
" ve " : " Venda " ,
" vi " : " Vietnamese " ,
" xh " : " Xhosa " ,
" zh " : " Chinese " ,
" zh_TW " : " Traditional Chinese (Taiwan) " ,
" zu " : " Zulu " ,
}
IANA_MAP = {
" ab " : 12026 ,
" af " : 40 ,
" ar " : 26020 ,
" az " : 26030 ,
" be " : 11890 ,
" bg " : 26050 ,
" bn " : 26040 ,
" bo " : 26601 ,
" br " : 1361 ,
" ca " : 3 ,
" ceb " : 26060 ,
" cs " : 26080 ,
" cy " : 26560 ,
" da " : 26090 ,
" de " : 26160 ,
" el " : 26165 ,
" en " : 26110 ,
" eo " : 11933 ,
" es " : 26460 ,
" et " : 26120 ,
" eu " : 1232 ,
" fa " : 26130 ,
" fi " : 26140 ,
" fo " : 11817 ,
" fr " : 26150 ,
" fy " : 1353 ,
" gd " : 65555 ,
" gl " : 1252 ,
" gu " : 26599 ,
" ha " : 26170 ,
" haw " : 26180 ,
" he " : 26592 ,
" hi " : 26190 ,
" hr " : 26070 ,
" hu " : 26200 ,
" hy " : 26597 ,
" id " : 26220 ,
" is " : 26210 ,
" it " : 26230 ,
" ja " : 26235 ,
" ka " : 26600 ,
" kk " : 26240 ,
" km " : 1222 ,
" ko " : 26255 ,
" ku " : 11815 ,
" ky " : 26260 ,
" la " : 26280 ,
" lt " : 26300 ,
" lv " : 26290 ,
" mg " : 1362 ,
" mk " : 26310 ,
" ml " : 26598 ,
" mn " : 26320 ,
" mr " : 1201 ,
" ms " : 1147 ,
" ne " : 26330 ,
" nl " : 26100 ,
" nn " : 172 ,
" no " : 26340 ,
" pa " : 65550 ,
" pl " : 26380 ,
" ps " : 26350 ,
" pt " : 26390 ,
" ro " : 26400 ,
" ru " : 26410 ,
" sa " : 1500 ,
" sh " : 1399 ,
" sk " : 26430 ,
" sl " : 26440 ,
" so " : 26450 ,
" sq " : 26010 ,
" sr " : 26420 ,
" sv " : 26480 ,
" sw " : 26470 ,
" ta " : 26595 ,
" te " : 26596 ,
" th " : 26594 ,
" tl " : 26490 ,
" tlh " : 26250 ,
" tn " : 65578 ,
" tr " : 26500 ,
" tw " : 1499 ,
" uk " : 26520 ,
" ur " : 26530 ,
" uz " : 26540 ,
" vi " : 26550 ,
" zh " : 26065 ,
" zh_TW " : 22 ,
}
models = { }
try :
from importlib import import_module
except ImportError :
import sys
def import_module ( name ) :
""" Import a module.
"""
__import__ ( name )
return sys . modules [ name ]
try :
from collections import namedtuple
LanguageInfo = namedtuple ( " LanguageInfo " , [ " tag " , " id " , " name " ] )
except ImportError :
class LanguageInfo ( tuple ) :
def __new__ ( cls , tag , id , name ) : #@ReservedAssignment
return tuple . __new__ ( cls , ( tag , id , name ) )
def __init__ ( self , tag , id , name ) : #@ReservedAssignment
self . tag = tag
self . id = id
self . name = name
class UNKNOWN ( str ) :
""" Unknown language
"""
def __bool__ ( self ) :
return False
UNKNOWN = UNKNOWN ( " UNKNOWN " )
def guess_language ( text : str , hints = None ) :
""" Return the ISO 639-1 language code.
"""
words = WORD_RE . findall ( text [ : MAX_LENGTH ] . replace ( " ’ " , " ' " ) )
return identify ( words , find_runs ( words ) , hints )
def guess_language_info ( text : str , hints = None ) :
""" Return LanguageInfo(tag, id, name).
"""
tag = guess_language ( text , hints )
if tag is UNKNOWN :
return LanguageInfo ( UNKNOWN , UNKNOWN , UNKNOWN )
return LanguageInfo ( tag , _get_id ( tag ) , _get_name ( tag ) )
# An alias for guess_language
guess_language_tag = guess_language
def guess_language_id ( text : str , hints = None ) :
""" Return the language ID.
"""
return _get_id ( guess_language ( text , hints ) )
def guess_language_name ( text : str , hints = None ) :
""" Return the language name (in English).
"""
return _get_name ( guess_language ( text , hints ) )
def _get_id ( tag ) :
return IANA_MAP . get ( tag , UNKNOWN )
def _get_name ( tag ) :
return NAME_MAP . get ( tag , UNKNOWN )
def find_runs ( words ) :
""" Count the number of characters in each character block.
"""
run_types = defaultdict ( int )
total_count = 0
for word in words :
for char in word :
block = BLOCKS [ ord ( char ) >> BLOCK_RSHIFT ]
run_types [ block ] + = 1
total_count + = 1
#pprint(run_types)
# return run types that used for 40% or more of the string
# return Basic Latin if found more than 15%
## and extended additional latin if over 10% (for Vietnamese)
relevant_runs = [ ]
for key , value in run_types . items ( ) :
pct = value * 100 / / total_count
if pct > = 40 or pct > = 15 and key == " Basic Latin " :
relevant_runs . append ( key )
#elif pct >= 10 and key == "Latin Extended Additional":
#relevant_runs.append(key)
return relevant_runs
def identify ( words , scripts , hints = None ) :
""" Identify the language.
"""
if ( " Hangul Syllables " in scripts or " Hangul Jamo " in scripts or
" Hangul Compatibility Jamo " in scripts or " Hangul " in scripts ) :
return " ko "
if " Greek and Coptic " in scripts :
return " el "
if " Kana " in scripts :
return " ja "
if ( " CJK Unified Ideographs " in scripts or " Bopomofo " in scripts or
" Bopomofo Extended " in scripts or " KangXi Radicals " in scripts ) :
# This is in both Ceglowski and Rideout
# I can't imagine why...
# or "Arabic Presentation Forms-A" in scripts
return " zh "
if " Cyrillic " in scripts :
return check ( words , filter_languages ( CYRILLIC , hints ) )
if ( " Arabic " in scripts or " Arabic Presentation Forms-A " in scripts or
" Arabic Presentation Forms-B " in scripts ) :
return check ( words , filter_languages ( ARABIC , hints ) )
if " Devanagari " in scripts :
return check ( words , filter_languages ( DEVANAGARI , hints ) )
# Try languages with unique scripts
for block_name , lang_name in SINGLETONS :
if block_name in scripts :
return lang_name
#if "Latin Extended Additional" in scripts:
#return "vi"
if " Extended Latin " in scripts :
latin_lang = check ( words , filter_languages ( EXTENDED_LATIN , hints ) )
if latin_lang == " pt " :
return check ( words , filter_languages ( PT ) )
else :
return latin_lang
if " Basic Latin " in scripts :
return check ( words , filter_languages ( ALL_LATIN , hints ) )
return UNKNOWN
def filter_languages ( languages , hints = None ) :
""" Filter languages.
"""
return languages . intersection ( hints ) if hints else languages
def check_with_all ( words , languages ) :
""" Check what the best match is.
"""
return ( check_with_enchant ( words , languages ) or
check_with_models ( words , languages ) )
check = check_with_all
def use_enchant ( use_enchant = True ) :
""" Enable or disable checking with PyEnchant.
"""
global check
check = check_with_all if use_enchant else check_with_models
def check_with_models ( words , languages ) :
""" Check against known models.
"""
sample = " " . join ( words )
if len ( sample ) < MIN_LENGTH :
return UNKNOWN
scores = [ ]
model = create_ordered_model ( sample ) # QMap<int,QString>
for key in languages :
lkey = key . lower ( )
try :
known_model = models [ lkey ]
except KeyError :
try :
known_model = import_module ( MODEL_ROOT + lkey ) . model
except ImportError :
known_model = None
models [ lkey ] = known_model
if known_model :
scores . append ( ( distance ( model , known_model ) , key ) )
if not scores :
return UNKNOWN
# we want the lowest score, less distance = greater chance of match
#pprint(sorted(scores))
return min ( scores ) [ 1 ]
def create_ordered_model ( content ) :
""" Create a list of trigrams in content sorted by frequency.
"""
trigrams = defaultdict ( int ) # QHash<QString,int>
content = content . lower ( )
for i in range ( len ( content ) - 2 ) :
trigrams [ content [ i : i + 3 ] ] + = 1
return sorted ( trigrams . keys ( ) , key = lambda k : ( - trigrams [ k ] , k ) )
def distance ( model , known_model ) :
""" Calculate the distance to the known model.
"""
dist = 0
for i , value in enumerate ( model [ : MAX_GRAMS ] ) :
if value in known_model :
dist + = abs ( i - known_model [ value ] )
else :
dist + = MAX_GRAMS
return dist
try :
import enchant
except ImportError :
warnings . warn ( " PyEnchant is unavailable " , ImportWarning )
enchant = None
def check_with_enchant ( * args , * * kwargs ) :
return UNKNOWN
else :
import locale
enchant_base_languages_dict = None
def check_with_enchant ( words , languages ,
threshold = 0.7 , min_words = 1 , dictionaries = { } ) :
""" Check against installed spelling dictionaries.
"""
if len ( words ) < min_words :
return UNKNOWN
best_score = 0
best_tag = UNKNOWN
for tag , enchant_tag in get_enchant_base_languages_dict ( ) . items ( ) :
if tag not in languages :
continue
try :
d = dictionaries [ tag ]
except KeyError :
d = dictionaries [ tag ] = enchant . Dict ( enchant_tag )
score = sum ( [ 1 for word in words if d . check ( word ) ] )
if score > best_score :
best_score = score
best_tag = tag
if best_score / len ( words ) < threshold :
return UNKNOWN
return best_tag
def get_enchant_base_languages_dict ( ) :
""" Get ordered dictionary of enchant base languages.
locale_language , then " en " , then the rest .
"""
global enchant_base_languages_dict
if enchant_base_languages_dict is None :
def get_language_sub_tag ( tag ) :
return tag . split ( " _ " ) [ 0 ]
enchant_base_languages_dict = OrderedDict ( )
enchant_languages = sorted ( enchant . list_languages ( ) )
for full_tag in [ get_locale_language ( ) , FALLBACK_LANGUAGE ] :
sub_tag = get_language_sub_tag ( full_tag )
if sub_tag not in enchant_base_languages_dict :
for tag in [ full_tag , sub_tag ] :
try :
index = enchant_languages . index ( tag )
except ValueError :
pass
else :
enchant_base_languages_dict [ sub_tag ] = tag
del enchant_languages [ index ]
break
for tag in enchant_languages :
sub_tag = get_language_sub_tag ( tag )
if sub_tag not in enchant_base_languages_dict :
enchant_base_languages_dict [ sub_tag ] = tag
return enchant_base_languages_dict
def get_locale_language ( ) :
""" Get the language code for the current locale setting.
"""
return ( locale . getlocale ( ) [ 0 ] or locale . getdefaultlocale ( ) [ 0 ] or
FALLBACK_LANGUAGE )
def deprecated ( func ) :
""" This is a decorator which can be used to mark functions
as deprecated . It will result in a warning being emitted
when the function is used .
"""
@functools.wraps ( func )
def new_func ( * args , * * kwargs ) :
warnings . warn (
" call to deprecated function %s () " % func . __name__ ,
category = DeprecationWarning ,
stacklevel = 2
)
return func ( * args , * * kwargs )
return new_func
@deprecated
def guessLanguage ( text ) :
""" Deprecated function - use guess_language() instead.
"""
return guess_language ( decode_text ( text ) )
@deprecated
def guessLanguageTag ( text ) :
""" Deprecated function - use guess_language_tag() instead.
"""
return guess_language_tag ( decode_text ( text ) )
@deprecated
def guessLanguageId ( text ) :
""" Deprecated function - use guess_language_id() instead.
"""
return guess_language_id ( decode_text ( text ) )
@deprecated
def guessLanguageName ( text ) :
""" Deprecated function - use guess_language_name() instead.
"""
return guess_language_name ( decode_text ( text ) )
@deprecated
def guessLanguageInfo ( text ) :
""" Deprecated function - use guess_language_info() instead.
"""
return guess_language_info ( decode_text ( text ) )
def decode_text ( text , encoding = " utf-8 " ) :
""" Decode text if needed (for deprecated functions).
"""
if not isinstance ( text , str ) :
warnings . warn ( " passing an encoded string is deprecated " ,
DeprecationWarning , 4 )
text = text . decode ( encoding )
return text
