|
|
"""
|
|
|
ftfy: fixes text for you
|
|
|
|
|
|
This is a module for making text less broken. See the `fix_text` function
|
|
|
for more information.
|
|
|
"""
|
|
|
|
|
|
import unicodedata
|
|
|
import warnings
|
|
|
from typing import List, NamedTuple, Optional, Tuple, Union, no_type_check
|
|
|
|
|
|
from ftfy import bad_codecs
|
|
|
from ftfy import chardata, fixes
|
|
|
from ftfy.badness import is_bad
|
|
|
from ftfy.formatting import display_ljust
|
|
|
|
|
|
__version__ = "6.1.1"
|
|
|
|
|
|
|
|
|
# Though this function does nothing, it lets linters know that we're using
|
|
|
# ftfy.bad_codecs. See the docstring in `bad_codecs/__init__.py` for more.
|
|
|
bad_codecs.ok()
|
|
|
|
|
|
|
|
|
class ExplainedText(NamedTuple):
|
|
|
"""
|
|
|
The return type from ftfy's functions that provide an "explanation" of which
|
|
|
steps it applied to fix the text, such as :func:`fix_and_explain()`.
|
|
|
|
|
|
When the 'explain' option is disabled, these functions return the same
|
|
|
type, but the `explanation` will be None.
|
|
|
"""
|
|
|
|
|
|
text: str
|
|
|
explanation: Optional[List[Tuple[str, str]]]
|
|
|
|
|
|
|
|
|
class TextFixerConfig(NamedTuple):
|
|
|
r"""
|
|
|
A TextFixerConfig object stores configuration options for ftfy.
|
|
|
|
|
|
It's implemented as a namedtuple with defaults, so you can instantiate
|
|
|
it by providing the values to change from their defaults as keyword arguments.
|
|
|
For example, to disable 'unescape_html' and keep the rest of the defaults::
|
|
|
|
|
|
TextFixerConfig(unescape_html=False)
|
|
|
|
|
|
Here are the options and their default values:
|
|
|
|
|
|
- `unescape_html`: "auto"
|
|
|
|
|
|
Configures whether to replace HTML entities such as & with the character
|
|
|
they represent. "auto" says to do this by default, but disable it when a
|
|
|
literal < character appears, indicating that the input is actual HTML and
|
|
|
entities should be preserved. The value can be True, to always enable this
|
|
|
fixer, or False, to always disable it.
|
|
|
|
|
|
- `remove_terminal_escapes`: True
|
|
|
|
|
|
Removes "ANSI" terminal escapes, such as for changing the color of text in a
|
|
|
terminal window.
|
|
|
|
|
|
- `fix_encoding`: True
|
|
|
|
|
|
Detect mojibake and attempt to fix it by decoding the text in a different
|
|
|
encoding standard.
|
|
|
|
|
|
The following four options affect `fix_encoding` works, and do nothing if
|
|
|
`fix_encoding` is False:
|
|
|
|
|
|
- `restore_byte_a0`: True
|
|
|
|
|
|
Allow a literal space (U+20) to be interpreted as a non-breaking space
|
|
|
(U+A0) when that would make it part of a fixable mojibake string.
|
|
|
|
|
|
Because spaces are very common characters, this could lead to false
|
|
|
positives, but we try to apply it only when there's strong evidence for
|
|
|
mojibake. Disabling `restore_byte_a0` is safer from false positives,
|
|
|
but creates false negatives.
|
|
|
|
|
|
- `replace_lossy_sequences`: True
|
|
|
|
|
|
Detect mojibake that has been partially replaced by the characters
|
|
|
'<EFBFBD>' or '?'. If the mojibake could be decoded otherwise, replace the
|
|
|
detected sequence with '<EFBFBD>'.
|
|
|
|
|
|
- `decode_inconsistent_utf8`: True
|
|
|
|
|
|
When we see sequences that distinctly look like UTF-8 mojibake, but
|
|
|
there's no consistent way to reinterpret the string in a new encoding,
|
|
|
replace the mojibake with the appropriate UTF-8 characters anyway.
|
|
|
|
|
|
This helps to decode strings that are concatenated from different
|
|
|
encodings.
|
|
|
|
|
|
- `fix_c1_controls`: True
|
|
|
|
|
|
Replace C1 control characters (the useless characters U+80 - U+9B that
|
|
|
come from Latin-1) with their Windows-1252 equivalents, like HTML5 does,
|
|
|
even if the whole string doesn't decode as Latin-1.
|
|
|
|
|
|
- `fix_latin_ligatures`: True
|
|
|
|
|
|
Replace common Latin-alphabet ligatures, such as ``fi``, with the
|
|
|
letters they're made of.
|
|
|
|
|
|
- `fix_character_width`: True
|
|
|
|
|
|
Replace fullwidth Latin characters and halfwidth Katakana with
|
|
|
their more standard widths.
|
|
|
|
|
|
- `uncurl_quotes`: True
|
|
|
|
|
|
Replace curly quotes with straight quotes.
|
|
|
|
|
|
- `fix_line_breaks`: True
|
|
|
|
|
|
Replace various forms of line breaks with the standard Unix line
|
|
|
break, ``\n``.
|
|
|
|
|
|
- `fix_surrogates`: True
|
|
|
|
|
|
Replace sequences of UTF-16 surrogate codepoints with the character
|
|
|
they were meant to encode. This fixes text that was decoded with the
|
|
|
obsolete UCS-2 standard, and allows it to support high-numbered
|
|
|
codepoints such as emoji.
|
|
|
|
|
|
- `remove_control_chars`: True
|
|
|
|
|
|
Remove certain control characters that have no displayed effect on text.
|
|
|
|
|
|
- `normalization`: "NFC"
|
|
|
|
|
|
Choose what kind of Unicode normalization is applied. Usually, we apply
|
|
|
NFC normalization, so that letters followed by combining characters become
|
|
|
single combined characters.
|
|
|
|
|
|
Changing this to "NFKC" applies more compatibility conversions, such as
|
|
|
replacing the 'micro sign' with a standard Greek lowercase mu, which looks
|
|
|
identical. However, some NFKC normalizations change the meaning of text,
|
|
|
such as converting "10³" to "103".
|
|
|
|
|
|
`normalization` can be None, to apply no normalization.
|
|
|
|
|
|
- `max_decode_length`: 1_000_000
|
|
|
|
|
|
The maximum size of "segment" that ftfy will try to fix all at once.
|
|
|
|
|
|
- `explain`: True
|
|
|
|
|
|
Whether to compute 'explanations', lists describing what ftfy changed.
|
|
|
When this is False, the explanation will be None, and the code that
|
|
|
builds the explanation will be skipped, possibly saving time.
|
|
|
|
|
|
Functions that accept TextFixerConfig and don't return an explanation
|
|
|
will automatically set `explain` to False.
|
|
|
"""
|
|
|
unescape_html: Union[str, bool] = "auto"
|
|
|
remove_terminal_escapes: bool = True
|
|
|
fix_encoding: bool = True
|
|
|
restore_byte_a0: bool = True
|
|
|
replace_lossy_sequences: bool = True
|
|
|
decode_inconsistent_utf8: bool = True
|
|
|
fix_c1_controls: bool = True
|
|
|
fix_latin_ligatures: bool = True
|
|
|
fix_character_width: bool = True
|
|
|
uncurl_quotes: bool = True
|
|
|
fix_line_breaks: bool = True
|
|
|
fix_surrogates: bool = True
|
|
|
remove_control_chars: bool = True
|
|
|
normalization: Optional[str] = "NFC"
|
|
|
max_decode_length: int = 1000000
|
|
|
explain: bool = True
|
|
|
|
|
|
|
|
|
def _config_from_kwargs(config: TextFixerConfig, kwargs: dict) -> TextFixerConfig:
|
|
|
"""
|
|
|
Handle parameters provided as keyword arguments to ftfy's top-level
|
|
|
functions, converting them into a TextFixerConfig.
|
|
|
"""
|
|
|
if "fix_entities" in kwargs:
|
|
|
warnings.warn(
|
|
|
"`fix_entities` has been renamed to `unescape_html`", DeprecationWarning
|
|
|
)
|
|
|
kwargs = kwargs.copy()
|
|
|
kwargs["unescape_html"] = kwargs["fix_entities"]
|
|
|
del kwargs["fix_entities"]
|
|
|
config = config._replace(**kwargs)
|
|
|
return config
|
|
|
|
|
|
|
|
|
FIXERS = {
|
|
|
"unescape_html": fixes.unescape_html,
|
|
|
"remove_terminal_escapes": fixes.remove_terminal_escapes,
|
|
|
"restore_byte_a0": fixes.restore_byte_a0,
|
|
|
"replace_lossy_sequences": fixes.replace_lossy_sequences,
|
|
|
"decode_inconsistent_utf8": fixes.decode_inconsistent_utf8,
|
|
|
"fix_c1_controls": fixes.fix_c1_controls,
|
|
|
"fix_latin_ligatures": fixes.fix_latin_ligatures,
|
|
|
"fix_character_width": fixes.fix_character_width,
|
|
|
"uncurl_quotes": fixes.uncurl_quotes,
|
|
|
"fix_line_breaks": fixes.fix_line_breaks,
|
|
|
"fix_surrogates": fixes.fix_surrogates,
|
|
|
"remove_control_chars": fixes.remove_control_chars,
|
|
|
}
|
|
|
|
|
|
|
|
|
BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode.
|
|
|
|
|
|
ftfy is designed to fix problems with text. Treating bytes like they're
|
|
|
interchangeable with Unicode text is usually something that introduces
|
|
|
problems with text.
|
|
|
|
|
|
You should first decode these bytes from the encoding you think they're in.
|
|
|
If you're not sure what encoding they're in:
|
|
|
|
|
|
- First, try to find out. 'utf-8' is a good assumption.
|
|
|
- If the encoding is simply unknowable, try running your bytes through
|
|
|
ftfy.guess_bytes. As the name implies, this may not always be accurate.
|
|
|
|
|
|
For more information on the distinction between bytes and text, read the
|
|
|
Python Unicode HOWTO:
|
|
|
|
|
|
http://docs.python.org/3/howto/unicode.html
|
|
|
"""
|
|
|
|
|
|
|
|
|
def _try_fix(
|
|
|
fixer_name: str, text: str, config: TextFixerConfig, steps: Optional[list]
|
|
|
) -> str:
|
|
|
"""
|
|
|
A helper function used across several 'fixer' steps, deciding whether to
|
|
|
apply the fix and whether to record the fix in `steps`.
|
|
|
"""
|
|
|
if getattr(config, fixer_name):
|
|
|
fixer = FIXERS[fixer_name]
|
|
|
fixed = fixer(text)
|
|
|
if steps is not None and fixed != text:
|
|
|
steps.append(("apply", fixer_name))
|
|
|
return fixed
|
|
|
|
|
|
return text
|
|
|
|
|
|
|
|
|
def fix_text(text: str, config: Optional[TextFixerConfig] = None, **kwargs) -> str:
|
|
|
r"""
|
|
|
Given Unicode text as input, fix inconsistencies and glitches in it,
|
|
|
such as mojibake (text that was decoded in the wrong encoding).
|
|
|
|
|
|
Let's start with some examples:
|
|
|
|
|
|
>>> fix_text('✔ No problems')
|
|
|
'✔ No problems'
|
|
|
|
|
|
>>> print(fix_text("¯\\_(ã\x83\x84)_/¯"))
|
|
|
¯\_(ツ)_/¯
|
|
|
|
|
|
>>> fix_text('Broken text… it’s flubberific!')
|
|
|
"Broken text... it's flubberific!"
|
|
|
|
|
|
>>> fix_text('ＬＯＵＤ ＮＯＩＳＥＳ')
|
|
|
'LOUD NOISES'
|
|
|
|
|
|
ftfy applies a number of different fixes to the text, and can accept
|
|
|
configuration to select which fixes to apply.
|
|
|
|
|
|
The configuration takes the form of a :class:`TextFixerConfig` object,
|
|
|
and you can see a description of the options in that class's docstring
|
|
|
or in the full documentation at ftfy.readthedocs.org.
|
|
|
|
|
|
For convenience and backward compatibility, the configuration can also
|
|
|
take the form of keyword arguments, which will set the equivalently-named
|
|
|
fields of the TextFixerConfig object.
|
|
|
|
|
|
For example, here are two ways to fix text but skip the "uncurl_quotes"
|
|
|
step::
|
|
|
|
|
|
fix_text(text, TextFixerConfig(uncurl_quotes=False))
|
|
|
fix_text(text, uncurl_quotes=False)
|
|
|
|
|
|
This function fixes text in independent segments, which are usually lines
|
|
|
of text, or arbitrarily broken up every 1 million codepoints (configurable
|
|
|
with `config.max_decode_length`) if there aren't enough line breaks. The
|
|
|
bound on segment lengths helps to avoid unbounded slowdowns.
|
|
|
|
|
|
ftfy can also provide an 'explanation', a list of transformations it applied
|
|
|
to the text that would fix more text like it. This function doesn't provide
|
|
|
explanations (because there may be different fixes for different segments
|
|
|
of text).
|
|
|
|
|
|
To get an explanation, use the :func:`fix_and_explain()` function, which
|
|
|
fixes the string in one segment and explains what it fixed.
|
|
|
"""
|
|
|
|
|
|
if config is None:
|
|
|
config = TextFixerConfig(explain=False)
|
|
|
config = _config_from_kwargs(config, kwargs)
|
|
|
if isinstance(text, bytes):
|
|
|
raise UnicodeError(BYTES_ERROR_TEXT)
|
|
|
|
|
|
out = []
|
|
|
pos = 0
|
|
|
while pos < len(text):
|
|
|
textbreak = text.find("\n", pos) + 1
|
|
|
if textbreak == 0:
|
|
|
textbreak = len(text)
|
|
|
if (textbreak - pos) > config.max_decode_length:
|
|
|
textbreak = pos + config.max_decode_length
|
|
|
|
|
|
segment = text[pos:textbreak]
|
|
|
if config.unescape_html == "auto" and "<" in segment:
|
|
|
config = config._replace(unescape_html=False)
|
|
|
fixed_segment, _ = fix_and_explain(segment, config)
|
|
|
out.append(fixed_segment)
|
|
|
pos = textbreak
|
|
|
return "".join(out)
|
|
|
|
|
|
|
|
|
def fix_and_explain(
|
|
|
text: str, config: Optional[TextFixerConfig] = None, **kwargs
|
|
|
) -> ExplainedText:
|
|
|
"""
|
|
|
Fix text as a single segment, returning the fixed text and an explanation
|
|
|
of what was fixed.
|
|
|
|
|
|
The explanation is a list of steps that can be applied with
|
|
|
:func:`apply_plan`, or if config.explain is False, it will be None.
|
|
|
"""
|
|
|
if config is None:
|
|
|
config = TextFixerConfig()
|
|
|
if isinstance(text, bytes):
|
|
|
raise UnicodeError(BYTES_ERROR_TEXT)
|
|
|
config = _config_from_kwargs(config, kwargs)
|
|
|
|
|
|
if config.unescape_html == "auto" and "<" in text:
|
|
|
config = config._replace(unescape_html=False)
|
|
|
|
|
|
if config.explain:
|
|
|
steps: Optional[List[Tuple[str, str]]] = []
|
|
|
else:
|
|
|
# If explanations aren't desired, `steps` will be None
|
|
|
steps = None
|
|
|
|
|
|
while True:
|
|
|
origtext = text
|
|
|
|
|
|
text = _try_fix("unescape_html", text, config, steps)
|
|
|
|
|
|
if config.fix_encoding:
|
|
|
if steps is None:
|
|
|
text = fix_encoding(text)
|
|
|
else:
|
|
|
text, encoding_steps = fix_encoding_and_explain(text, config)
|
|
|
if encoding_steps is not None:
|
|
|
steps.extend(encoding_steps)
|
|
|
|
|
|
for fixer in [
|
|
|
"fix_c1_controls",
|
|
|
"fix_latin_ligatures",
|
|
|
"fix_character_width",
|
|
|
"uncurl_quotes",
|
|
|
"fix_line_breaks",
|
|
|
"fix_surrogates",
|
|
|
"remove_terminal_escapes",
|
|
|
"remove_control_chars",
|
|
|
]:
|
|
|
text = _try_fix(fixer, text, config, steps)
|
|
|
|
|
|
if config.normalization is not None:
|
|
|
fixed = unicodedata.normalize(config.normalization, text)
|
|
|
if steps is not None and fixed != text:
|
|
|
steps.append(("normalize", config.normalization))
|
|
|
text = fixed
|
|
|
|
|
|
if text == origtext:
|
|
|
return ExplainedText(text, steps)
|
|
|
|
|
|
|
|
|
def fix_encoding_and_explain(
|
|
|
text: str, config: Optional[TextFixerConfig] = None, **kwargs
|
|
|
) -> ExplainedText:
|
|
|
"""
|
|
|
Apply the steps of ftfy that detect mojibake and fix it. Returns the fixed
|
|
|
text and a list explaining what was fixed.
|
|
|
|
|
|
This includes fixing text by encoding and decoding it in different encodings,
|
|
|
as well as the subordinate fixes `restore_byte_a0`, `replace_lossy_sequences`,
|
|
|
`decode_inconsistent_utf8`, and `fix_c1_controls`.
|
|
|
|
|
|
Examples::
|
|
|
|
|
|
>>> fix_encoding_and_explain("só")
|
|
|
ExplainedText(text='só', explanation=[('encode', 'latin-1'), ('decode', 'utf-8')])
|
|
|
|
|
|
>>> result = fix_encoding_and_explain("voilà le travail")
|
|
|
>>> result.text
|
|
|
'voilà le travail'
|
|
|
>>> result.explanation
|
|
|
[('encode', 'latin-1'), ('transcode', 'restore_byte_a0'), ('decode', 'utf-8')]
|
|
|
|
|
|
"""
|
|
|
if config is None:
|
|
|
config = TextFixerConfig()
|
|
|
if isinstance(text, bytes):
|
|
|
raise UnicodeError(BYTES_ERROR_TEXT)
|
|
|
config = _config_from_kwargs(config, kwargs)
|
|
|
|
|
|
if not config.fix_encoding:
|
|
|
# A weird trivial case: we're asked to fix the encoding, but skip
|
|
|
# fixing the encoding
|
|
|
return ExplainedText(text, [])
|
|
|
|
|
|
plan_so_far: List[Tuple[str, str]] = []
|
|
|
while True:
|
|
|
prevtext = text
|
|
|
text, plan = _fix_encoding_one_step_and_explain(text, config)
|
|
|
if plan is not None:
|
|
|
plan_so_far.extend(plan)
|
|
|
if text == prevtext:
|
|
|
return ExplainedText(text, plan_so_far)
|
|
|
|
|
|
|
|
|
def _fix_encoding_one_step_and_explain(
|
|
|
text: str, config: TextFixerConfig
|
|
|
) -> ExplainedText:
|
|
|
"""
|
|
|
Perform one step of fixing the encoding of text.
|
|
|
"""
|
|
|
if config is None:
|
|
|
config = TextFixerConfig()
|
|
|
|
|
|
if len(text) == 0:
|
|
|
return ExplainedText(text, [])
|
|
|
|
|
|
# The first plan is to return ASCII text unchanged, as well as text
|
|
|
# that doesn't look like it contains mojibake
|
|
|
if chardata.possible_encoding(text, "ascii") or not is_bad(text):
|
|
|
return ExplainedText(text, [])
|
|
|
|
|
|
# As we go through the next step, remember the possible encodings
|
|
|
# that we encounter but don't successfully fix yet. We may need them
|
|
|
# later.
|
|
|
possible_1byte_encodings = []
|
|
|
|
|
|
# Suppose the text was supposed to be UTF-8, but it was decoded using
|
|
|
# a single-byte encoding instead. When these cases can be fixed, they
|
|
|
# are usually the correct thing to do, so try them next.
|
|
|
for encoding in chardata.CHARMAP_ENCODINGS:
|
|
|
if chardata.possible_encoding(text, encoding):
|
|
|
possible_1byte_encodings.append(encoding)
|
|
|
encoded_bytes = text.encode(encoding)
|
|
|
encode_step = ("encode", encoding)
|
|
|
transcode_steps = []
|
|
|
|
|
|
# Now, find out if it's UTF-8 (or close enough). Otherwise,
|
|
|
# remember the encoding for later.
|
|
|
try:
|
|
|
decoding = "utf-8"
|
|
|
# Check encoded_bytes for sequences that would be UTF-8,
|
|
|
# except they have b' ' where b'\xa0' would belong.
|
|
|
if config.restore_byte_a0 and chardata.ALTERED_UTF8_RE.search(
|
|
|
encoded_bytes
|
|
|
):
|
|
|
replaced_bytes = fixes.restore_byte_a0(encoded_bytes)
|
|
|
if replaced_bytes != encoded_bytes:
|
|
|
transcode_steps.append(("transcode", "restore_byte_a0"))
|
|
|
encoded_bytes = replaced_bytes
|
|
|
|
|
|
# Replace sequences where information has been lost
|
|
|
if config.replace_lossy_sequences and encoding.startswith("sloppy"):
|
|
|
replaced_bytes = fixes.replace_lossy_sequences(encoded_bytes)
|
|
|
if replaced_bytes != encoded_bytes:
|
|
|
transcode_steps.append(("transcode", "replace_lossy_sequences"))
|
|
|
encoded_bytes = replaced_bytes
|
|
|
|
|
|
if 0xED in encoded_bytes or 0xC0 in encoded_bytes:
|
|
|
decoding = "utf-8-variants"
|
|
|
|
|
|
decode_step = ("decode", decoding)
|
|
|
steps = [encode_step] + transcode_steps + [decode_step]
|
|
|
fixed = encoded_bytes.decode(decoding)
|
|
|
return ExplainedText(fixed, steps)
|
|
|
|
|
|
except UnicodeDecodeError:
|
|
|
pass
|
|
|
|
|
|
# Look for a-hat-euro sequences that remain, and fix them in isolation.
|
|
|
if config.decode_inconsistent_utf8 and chardata.UTF8_DETECTOR_RE.search(text):
|
|
|
steps = [("apply", "decode_inconsistent_utf8")]
|
|
|
fixed = fixes.decode_inconsistent_utf8(text)
|
|
|
if fixed != text:
|
|
|
return ExplainedText(fixed, steps)
|
|
|
|
|
|
# The next most likely case is that this is Latin-1 that was intended to
|
|
|
# be read as Windows-1252, because those two encodings in particular are
|
|
|
# easily confused.
|
|
|
if "latin-1" in possible_1byte_encodings:
|
|
|
if "windows-1252" in possible_1byte_encodings:
|
|
|
# This text is in the intersection of Latin-1 and
|
|
|
# Windows-1252, so it's probably legit.
|
|
|
return ExplainedText(text, [])
|
|
|
else:
|
|
|
# Otherwise, it means we have characters that are in Latin-1 but
|
|
|
# not in Windows-1252. Those are C1 control characters. Nobody
|
|
|
# wants those. Assume they were meant to be Windows-1252.
|
|
|
try:
|
|
|
fixed = text.encode("latin-1").decode("windows-1252")
|
|
|
if fixed != text:
|
|
|
steps = [("encode", "latin-1"), ("decode", "windows-1252")]
|
|
|
return ExplainedText(fixed, steps)
|
|
|
except UnicodeDecodeError:
|
|
|
pass
|
|
|
|
|
|
# Fix individual characters of Latin-1 with a less satisfying explanation
|
|
|
if config.fix_c1_controls and chardata.C1_CONTROL_RE.search(text):
|
|
|
steps = [("transcode", "fix_c1_controls")]
|
|
|
fixed = fixes.fix_c1_controls(text)
|
|
|
return ExplainedText(fixed, steps)
|
|
|
|
|
|
# The cases that remain are mixups between two different single-byte
|
|
|
# encodings, and not the common case of Latin-1 vs. Windows-1252.
|
|
|
#
|
|
|
# With the new heuristic in 6.0, it's possible that we're closer to solving
|
|
|
# these in some cases. It would require a lot of testing and tuning, though.
|
|
|
# For now, we leave the text unchanged in these cases.
|
|
|
return ExplainedText(text, [])
|
|
|
|
|
|
|
|
|
def fix_encoding(text: str, config: TextFixerConfig = None, **kwargs):
|
|
|
"""
|
|
|
Apply just the encoding-fixing steps of ftfy to this text. Returns the
|
|
|
fixed text, discarding the explanation.
|
|
|
|
|
|
>>> fix_encoding("ó")
|
|
|
'ó'
|
|
|
>>> fix_encoding("&ATILDE;&SUP3;")
|
|
|
'&ATILDE;&SUP3;'
|
|
|
"""
|
|
|
if config is None:
|
|
|
config = TextFixerConfig(explain=False)
|
|
|
config = _config_from_kwargs(config, kwargs)
|
|
|
fixed, _explan = fix_encoding_and_explain(text, config)
|
|
|
return fixed
|
|
|
|
|
|
|
|
|
# Some alternate names for the main functions
|
|
|
ftfy = fix_text
|
|
|
|
|
|
|
|
|
def fix_text_segment(text: str, config: TextFixerConfig = None, **kwargs):
|
|
|
"""
|
|
|
Fix text as a single segment, with a consistent sequence of steps that
|
|
|
are applied to fix the text. Discard the explanation.
|
|
|
"""
|
|
|
if config is None:
|
|
|
config = TextFixerConfig(explain=False)
|
|
|
config = _config_from_kwargs(config, kwargs)
|
|
|
fixed, _explan = fix_and_explain(text, config)
|
|
|
return fixed
|
|
|
|
|
|
|
|
|
def fix_file(input_file, encoding=None, config=None, **kwargs):
|
|
|
"""
|
|
|
Fix text that is found in a file.
|
|
|
|
|
|
If the file is being read as Unicode text, use that. If it's being read as
|
|
|
bytes, then we hope an encoding was supplied. If not, unfortunately, we
|
|
|
have to guess what encoding it is. We'll try a few common encodings, but we
|
|
|
make no promises. See the `guess_bytes` function for how this is done.
|
|
|
|
|
|
The output is a stream of fixed lines of text.
|
|
|
"""
|
|
|
if config is None:
|
|
|
config = TextFixerConfig()
|
|
|
config = _config_from_kwargs(config, kwargs)
|
|
|
|
|
|
for line in input_file:
|
|
|
if isinstance(line, bytes):
|
|
|
if encoding is None:
|
|
|
line, encoding = guess_bytes(line)
|
|
|
else:
|
|
|
line = line.decode(encoding)
|
|
|
if config.unescape_html == "auto" and "<" in line:
|
|
|
config = config._replace(unescape_html=False)
|
|
|
|
|
|
fixed_line, _explan = fix_and_explain(line, config)
|
|
|
yield fixed_line
|
|
|
|
|
|
|
|
|
def guess_bytes(bstring):
|
|
|
"""
|
|
|
NOTE: Using `guess_bytes` is not the recommended way of using ftfy. ftfy
|
|
|
is not designed to be an encoding detector.
|
|
|
|
|
|
In the unfortunate situation that you have some bytes in an unknown
|
|
|
encoding, ftfy can guess a reasonable strategy for decoding them, by trying
|
|
|
a few common encodings that can be distinguished from each other.
|
|
|
|
|
|
Unlike the rest of ftfy, this may not be accurate, and it may *create*
|
|
|
Unicode problems instead of solving them!
|
|
|
|
|
|
The encodings we try here are:
|
|
|
|
|
|
- UTF-16 with a byte order mark, because a UTF-16 byte order mark looks
|
|
|
like nothing else
|
|
|
- UTF-8, because it's the global standard, which has been used by a
|
|
|
majority of the Web since 2008
|
|
|
- "utf-8-variants", or buggy implementations of UTF-8
|
|
|
- MacRoman, because Microsoft Office thinks it's still a thing, and it
|
|
|
can be distinguished by its line breaks. (If there are no line breaks in
|
|
|
the string, though, you're out of luck.)
|
|
|
- "sloppy-windows-1252", the Latin-1-like encoding that is the most common
|
|
|
single-byte encoding.
|
|
|
"""
|
|
|
if isinstance(bstring, str):
|
|
|
raise UnicodeError(
|
|
|
"This string was already decoded as Unicode. You should pass "
|
|
|
"bytes to guess_bytes, not Unicode."
|
|
|
)
|
|
|
|
|
|
if bstring.startswith(b"\xfe\xff") or bstring.startswith(b"\xff\xfe"):
|
|
|
return bstring.decode("utf-16"), "utf-16"
|
|
|
|
|
|
byteset = set(bstring)
|
|
|
try:
|
|
|
if 0xED in byteset or 0xC0 in byteset:
|
|
|
# Byte 0xed can be used to encode a range of codepoints that
|
|
|
# are UTF-16 surrogates. UTF-8 does not use UTF-16 surrogates,
|
|
|
# so when we see 0xed, it's very likely we're being asked to
|
|
|
# decode CESU-8, the variant that encodes UTF-16 surrogates
|
|
|
# instead of the original characters themselves.
|
|
|
#
|
|
|
# This will occasionally trigger on standard UTF-8, as there
|
|
|
# are some Korean characters that also use byte 0xed, but that's
|
|
|
# not harmful because standard UTF-8 characters will decode the
|
|
|
# same way in our 'utf-8-variants' codec.
|
|
|
#
|
|
|
# Byte 0xc0 is impossible because, numerically, it would only
|
|
|
# encode characters lower than U+0040. Those already have
|
|
|
# single-byte representations, and UTF-8 requires using the
|
|
|
# shortest possible representation. However, Java hides the null
|
|
|
# codepoint, U+0000, in a non-standard longer representation -- it
|
|
|
# encodes it as 0xc0 0x80 instead of 0x00, guaranteeing that 0x00
|
|
|
# will never appear in the encoded bytes.
|
|
|
#
|
|
|
# The 'utf-8-variants' decoder can handle both of these cases, as
|
|
|
# well as standard UTF-8, at the cost of a bit of speed.
|
|
|
return bstring.decode("utf-8-variants"), "utf-8-variants"
|
|
|
else:
|
|
|
return bstring.decode("utf-8"), "utf-8"
|
|
|
except UnicodeDecodeError:
|
|
|
pass
|
|
|
|
|
|
if 0x0D in byteset and 0x0A not in byteset:
|
|
|
# Files that contain CR and not LF are likely to be MacRoman.
|
|
|
return bstring.decode("macroman"), "macroman"
|
|
|
|
|
|
return bstring.decode("sloppy-windows-1252"), "sloppy-windows-1252"
|
|
|
|
|
|
|
|
|
@no_type_check
|
|
|
def apply_plan(text: str, plan: List[Tuple[str, str]]):
|
|
|
"""
|
|
|
Apply a plan for fixing the encoding of text.
|
|
|
|
|
|
The plan is a list of tuples of the form (operation, arg).
|
|
|
|
|
|
`operation` is one of:
|
|
|
|
|
|
- `'encode'`: convert a string to bytes, using `arg` as the encoding
|
|
|
- `'decode'`: convert bytes to a string, using `arg` as the encoding
|
|
|
- `'transcode'`: convert bytes to bytes, using the function named `arg`
|
|
|
- `'apply'`: convert a string to a string, using the function named `arg`
|
|
|
|
|
|
The functions that can be applied by 'transcode' and 'apply' are
|
|
|
specifically those that appear in the dictionary named `FIXERS`. They
|
|
|
can also can be imported from the `ftfy.fixes` module.
|
|
|
|
|
|
Example::
|
|
|
|
|
|
>>> mojibake = "schön"
|
|
|
>>> text, plan = fix_and_explain(mojibake)
|
|
|
>>> apply_plan(mojibake, plan)
|
|
|
'schön'
|
|
|
"""
|
|
|
obj = text
|
|
|
for operation, encoding in plan:
|
|
|
if operation == "encode":
|
|
|
obj = obj.encode(encoding)
|
|
|
elif operation == "decode":
|
|
|
obj = obj.decode(encoding)
|
|
|
elif operation in ("transcode", "apply"):
|
|
|
if encoding in FIXERS:
|
|
|
obj = FIXERS[encoding](obj)
|
|
|
else:
|
|
|
raise ValueError("Unknown function to apply: %s" % encoding)
|
|
|
else:
|
|
|
raise ValueError("Unknown plan step: %s" % operation)
|
|
|
|
|
|
return obj
|
|
|
|
|
|
|
|
|
def explain_unicode(text: str):
|
|
|
"""
|
|
|
A utility method that's useful for debugging mysterious Unicode.
|
|
|
|
|
|
It breaks down a string, showing you for each codepoint its number in
|
|
|
hexadecimal, its glyph, its category in the Unicode standard, and its name
|
|
|
in the Unicode standard.
|
|
|
|
|
|
>>> explain_unicode('(╯°□°)╯︵ ┻━┻')
|
|
|
U+0028 ( [Ps] LEFT PARENTHESIS
|
|
|
U+256F ╯ [So] BOX DRAWINGS LIGHT ARC UP AND LEFT
|
|
|
U+00B0 ° [So] DEGREE SIGN
|
|
|
U+25A1 □ [So] WHITE SQUARE
|
|
|
U+00B0 ° [So] DEGREE SIGN
|
|
|
U+0029 ) [Pe] RIGHT PARENTHESIS
|
|
|
U+256F ╯ [So] BOX DRAWINGS LIGHT ARC UP AND LEFT
|
|
|
U+FE35 ︵ [Ps] PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS
|
|
|
U+0020 [Zs] SPACE
|
|
|
U+253B ┻ [So] BOX DRAWINGS HEAVY UP AND HORIZONTAL
|
|
|
U+2501 ━ [So] BOX DRAWINGS HEAVY HORIZONTAL
|
|
|
U+253B ┻ [So] BOX DRAWINGS HEAVY UP AND HORIZONTAL
|
|
|
"""
|
|
|
for char in text:
|
|
|
if char.isprintable():
|
|
|
display = char
|
|
|
else:
|
|
|
display = char.encode("unicode-escape").decode("ascii")
|
|
|
print(
|
|
|
"U+{code:04X} {display} [{category}] {name}".format(
|
|
|
display=display_ljust(display, 7),
|
|
|
code=ord(char),
|
|
|
category=unicodedata.category(char),
|
|
|
name=unicodedata.name(char, "<unknown>"),
|
|
|
)
|
|
|
)
|
