# encoding: utf-8
""" Helper classes for tests. """
# Use of this source code is governed by the MIT license.
__license__ = " MIT "
import pickle
import copy
import functools
import unittest
import warnings
from unittest import TestCase
from bs4 import BeautifulSoup
from bs4 . element import (
CharsetMetaAttributeValue ,
Comment ,
ContentMetaAttributeValue ,
Doctype ,
PYTHON_SPECIFIC_ENCODINGS ,
SoupStrainer ,
Script ,
Stylesheet ,
Tag
)
from bs4 . builder import HTMLParserTreeBuilder
default_builder = HTMLParserTreeBuilder
BAD_DOCUMENT = """ A bare string
< ! DOCTYPE xsl : stylesheet SYSTEM " htmlent.dtd " >
< ! DOCTYPE xsl : stylesheet PUBLIC " htmlent.dtd " >
< div > < ! [ CDATA [ A CDATA section where it doesn ' t belong]]></div>
< div > < svg > < ! [ CDATA [ HTML5 does allow CDATA sections in SVG ] ] > < / svg > < / div >
< div > A < meta > tag < / div >
< div > A < br > tag that supposedly has contents . < / br > < / div >
< div > AT & T < / div >
< div > < textarea > Within a textarea , markup like < b > tags and < & < & amp ; should be treated as literal < / textarea > < / div >
< div > < script > if ( i < 2 ) { alert ( " <b>Markup within script tags should be treated as literal.</b> " ) ; } < / script > < / div >
< div > This numeric entity is missing the final semicolon : < x t = " piñata " > < / div >
< div > < a href = " http://example.com/</a> that attribute value never got closed</div>
< div > < a href = " foo</a>, </a><a href= " bar " >that attribute value was closed by the subsequent tag</a></div>
< ! This document starts with a bogus declaration > < div > a < / div >
< div > This document contains < ! an incomplete declaration < div > ( do you see it ? ) < / div >
< div > This document ends with < ! an incomplete declaration
< div > < a style = { height : 21 px ; } > That attribute value was bogus < / a > < / div >
< ! DOCTYPE html PUBLIC " -//W3C//DTD XHTML 1.0 Transitional//EN " > The doctype is invalid because it contains extra whitespace
< div > < table > < td nowrap > That boolean attribute had no value < / td > < / table > < / div >
< div > Here ' s a nonexistent entity: &#foo; (do you see it?)</div>
< div > This document ends before the entity finishes : & gt
< div > < p > Paragraphs shouldn ' t contain block display elements, but this one does: <dl><dt>you see?</dt></p>
< b b = " 20 " a = " 1 " b = " 10 " a = " 2 " a = " 3 " a = " 4 " > Multiple values for the same attribute . < / b >
< div > < table > < tr > < td > Here ' s a table</td></tr></table></div>
< div > < table id = " 1 " > < tr > < td > Here ' s a nested table:<table id= " 2 " ><tr><td>foo</td></tr></table></td></div>
< div > This tag contains nothing but whitespace : < b > < / b > < / div >
< div > < blockquote > < p > < b > This p tag is cut off by < / blockquote > < / p > the end of the blockquote tag < / div >
< div > < table > < div > This table contains bare markup < / div > < / table > < / div >
< div > < div id = " 1 " > \n < a href = " link1 " > This link is never closed . \n < / div > \n < div id = " 2 " > \n < div id = " 3 " > \n < a href = " link2 " > This link is closed . < / a > \n < / div > \n < / div > < / div >
< div > This document contains a < ! DOCTYPE surprise > surprise doctype < / div >
< div > < a > < B > < Cd > < EFG > Mixed case tags are folded to lowercase < / efg > < / CD > < / b > < / A > < / div >
< div > < our \u2603 > Tag name contains Unicode characters < / our \u2603 > < / div >
< div > < a \u2603 = " snowman " > Attribute name contains Unicode characters < / a > < / div >
< ! DOCTYPE html PUBLIC " -//W3C//DTD XHTML 1.0 Transitional//EN " " http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd " >
"""
class SoupTest ( unittest . TestCase ) :
@property
def default_builder ( self ) :
return default_builder
def soup ( self , markup , * * kwargs ) :
""" Build a Beautiful Soup object from markup. """
builder = kwargs . pop ( ' builder ' , self . default_builder )
return BeautifulSoup ( markup , builder = builder , * * kwargs )
def document_for ( self , markup , * * kwargs ) :
""" Turn an HTML fragment into a document.
The details depend on the builder .
"""
return self . default_builder ( * * kwargs ) . test_fragment_to_document ( markup )
def assertSoupEquals ( self , to_parse , compare_parsed_to = None ) :
builder = self . default_builder
obj = BeautifulSoup ( to_parse , builder = builder )
if compare_parsed_to is None :
compare_parsed_to = to_parse
# Verify that the documents come out the same.
self . assertEqual ( obj . decode ( ) , self . document_for ( compare_parsed_to ) )
# Also run some checks on the BeautifulSoup object itself:
# Verify that every tag that was opened was eventually closed.
# There are no tags in the open tag counter.
assert all ( v == 0 for v in list ( obj . open_tag_counter . values ( ) ) )
# The only tag in the tag stack is the one for the root
# document.
self . assertEqual (
[ obj . ROOT_TAG_NAME ] , [ x . name for x in obj . tagStack ]
)
def assertConnectedness ( self , element ) :
""" Ensure that next_element and previous_element are properly
set for all descendants of the given element .
"""
earlier = None
for e in element . descendants :
if earlier :
self . assertEqual ( e , earlier . next_element )
self . assertEqual ( earlier , e . previous_element )
earlier = e
def linkage_validator ( self , el , _recursive_call = False ) :
""" Ensure proper linkage throughout the document. """
descendant = None
# Document element should have no previous element or previous sibling.
# It also shouldn't have a next sibling.
if el . parent is None :
assert el . previous_element is None , \
" Bad previous_element \n NODE: {} \n PREV: {} \n EXPECTED: {} " . format (
el , el . previous_element , None
)
assert el . previous_sibling is None , \
" Bad previous_sibling \n NODE: {} \n PREV: {} \n EXPECTED: {} " . format (
el , el . previous_sibling , None
)
assert el . next_sibling is None , \
" Bad next_sibling \n NODE: {} \n NEXT: {} \n EXPECTED: {} " . format (
el , el . next_sibling , None
)
idx = 0
child = None
last_child = None
last_idx = len ( el . contents ) - 1
for child in el . contents :
descendant = None
# Parent should link next element to their first child
# That child should have no previous sibling
if idx == 0 :
if el . parent is not None :
assert el . next_element is child , \
" Bad next_element \n NODE: {} \n NEXT: {} \n EXPECTED: {} " . format (
el , el . next_element , child
)
assert child . previous_element is el , \
" Bad previous_element \n NODE: {} \n PREV: {} \n EXPECTED: {} " . format (
child , child . previous_element , el
)
assert child . previous_sibling is None , \
" Bad previous_sibling \n NODE: {} \n PREV {} \n EXPECTED: {} " . format (
child , child . previous_sibling , None
)
# If not the first child, previous index should link as sibling to this index
# Previous element should match the last index or the last bubbled up descendant
else :
assert child . previous_sibling is el . contents [ idx - 1 ] , \
" Bad previous_sibling \n NODE: {} \n PREV {} \n EXPECTED {} " . format (
child , child . previous_sibling , el . contents [ idx - 1 ]
)
assert el . contents [ idx - 1 ] . next_sibling is child , \
" Bad next_sibling \n NODE: {} \n NEXT {} \n EXPECTED {} " . format (
el . contents [ idx - 1 ] , el . contents [ idx - 1 ] . next_sibling , child
)
if last_child is not None :
assert child . previous_element is last_child , \
" Bad previous_element \n NODE: {} \n PREV {} \n EXPECTED {} \n CONTENTS {} " . format (
child , child . previous_element , last_child , child . parent . contents
)
assert last_child . next_element is child , \
" Bad next_element \n NODE: {} \n NEXT {} \n EXPECTED {} " . format (
last_child , last_child . next_element , child
)
if isinstance ( child , Tag ) and child . contents :
descendant = self . linkage_validator ( child , True )
# A bubbled up descendant should have no next siblings
assert descendant . next_sibling is None , \
" Bad next_sibling \n NODE: {} \n NEXT {} \n EXPECTED {} " . format (
descendant , descendant . next_sibling , None
)
# Mark last child as either the bubbled up descendant or the current child
if descendant is not None :
last_child = descendant
else :
last_child = child
# If last child, there are non next siblings
if idx == last_idx :
assert child . next_sibling is None , \
" Bad next_sibling \n NODE: {} \n NEXT {} \n EXPECTED {} " . format (
child , child . next_sibling , None
)
idx + = 1
child = descendant if descendant is not None else child
if child is None :
child = el
if not _recursive_call and child is not None :
target = el
while True :
if target is None :
assert child . next_element is None , \
" Bad next_element \n NODE: {} \n NEXT {} \n EXPECTED {} " . format (
child , child . next_element , None
)
break
elif target . next_sibling is not None :
assert child . next_element is target . next_sibling , \
" Bad next_element \n NODE: {} \n NEXT {} \n EXPECTED {} " . format (
child , child . next_element , target . next_sibling
)
break
target = target . parent
# We are done, so nothing to return
return None
else :
# Return the child to the recursive caller
return child
class TreeBuilderSmokeTest ( object ) :
# Tests that are common to HTML and XML tree builders.
def test_fuzzed_input ( self ) :
# This test centralizes in one place the various fuzz tests
# for Beautiful Soup created by the oss-fuzz project.
# These strings superficially resemble markup, but they
# generally can't be parsed into anything. The best we can
# hope for is that parsing these strings won't crash the
# parser.
#
# n.b. This markup is commented out because these fuzz tests
# _do_ crash the parser. However the crashes are due to bugs
# in html.parser, not Beautiful Soup -- otherwise I'd fix the
# bugs!
bad_markup = [
# https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=28873
# https://github.com/guidovranken/python-library-fuzzers/blob/master/corp-html/519e5b4269a01185a0d5e76295251921da2f0700
# https://bugs.python.org/issue37747
#
#b'\n<![\xff\xfe\xfe\xcd\x00',
#https://github.com/guidovranken/python-library-fuzzers/blob/master/corp-html/de32aa55785be29bbc72a1a8e06b00611fb3d9f8
# https://bugs.python.org/issue34480
#
#b'<![n\x00'
]
for markup in bad_markup :
with warnings . catch_warnings ( record = False ) :
soup = self . soup ( markup )
class HTMLTreeBuilderSmokeTest ( TreeBuilderSmokeTest ) :
""" A basic test of a treebuilder ' s competence.
Any HTML treebuilder , present or future , should be able to pass
these tests . With invalid markup , there ' s room for interpretation,
and different parsers can handle it differently . But with the
markup in these tests , there ' s not much room for interpretation.
"""
def test_empty_element_tags ( self ) :
""" Verify that all HTML4 and HTML5 empty element (aka void element) tags
are handled correctly .
"""
for name in [
' area ' , ' base ' , ' br ' , ' col ' , ' embed ' , ' hr ' , ' img ' , ' input ' , ' keygen ' , ' link ' , ' menuitem ' , ' meta ' , ' param ' , ' source ' , ' track ' , ' wbr ' ,
' spacer ' , ' frame '
] :
soup = self . soup ( " " )
new_tag = soup . new_tag ( name )
self . assertEqual ( True , new_tag . is_empty_element )
def test_special_string_containers ( self ) :
soup = self . soup (
" <style>Some CSS</style><script>Some Javascript</script> "
)
assert isinstance ( soup . style . string , Stylesheet )
assert isinstance ( soup . script . string , Script )
soup = self . soup (
" <style><!--Some CSS--></style> "
)
assert isinstance ( soup . style . string , Stylesheet )
# The contents of the style tag resemble an HTML comment, but
# it's not treated as a comment.
self . assertEqual ( " <!--Some CSS--> " , soup . style . string )
assert isinstance ( soup . style . string , Stylesheet )
def test_pickle_and_unpickle_identity ( self ) :
# Pickling a tree, then unpickling it, yields a tree identical
# to the original.
tree = self . soup ( " <a><b>foo</a> " )
dumped = pickle . dumps ( tree , 2 )
loaded = pickle . loads ( dumped )
self . assertEqual ( loaded . __class__ , BeautifulSoup )
self . assertEqual ( loaded . decode ( ) , tree . decode ( ) )
def assertDoctypeHandled ( self , doctype_fragment ) :
""" Assert that a given doctype string is handled correctly. """
doctype_str , soup = self . _document_with_doctype ( doctype_fragment )
# Make sure a Doctype object was created.
doctype = soup . contents [ 0 ]
self . assertEqual ( doctype . __class__ , Doctype )
self . assertEqual ( doctype , doctype_fragment )
self . assertEqual (
soup . encode ( " utf8 " ) [ : len ( doctype_str ) ] ,
doctype_str
)
# Make sure that the doctype was correctly associated with the
# parse tree and that the rest of the document parsed.
self . assertEqual ( soup . p . contents [ 0 ] , ' foo ' )
def _document_with_doctype ( self , doctype_fragment , doctype_string = " DOCTYPE " ) :
""" Generate and parse a document with the given doctype. """
doctype = ' <! %s %s > ' % ( doctype_string , doctype_fragment )
markup = doctype + ' \n <p>foo</p> '
soup = self . soup ( markup )
return doctype . encode ( " utf8 " ) , soup
def test_normal_doctypes ( self ) :
""" Make sure normal, everyday HTML doctypes are handled correctly. """
self . assertDoctypeHandled ( " html " )
self . assertDoctypeHandled (
' html PUBLIC " -//W3C//DTD XHTML 1.0 Transitional//EN " ' )
def test_empty_doctype ( self ) :
soup = self . soup ( " <!DOCTYPE> " )
doctype = soup . contents [ 0 ]
self . assertEqual ( " " , doctype . strip ( ) )
def test_mixed_case_doctype ( self ) :
# A lowercase or mixed-case doctype becomes a Doctype.
for doctype_fragment in ( " doctype " , " DocType " ) :
doctype_str , soup = self . _document_with_doctype (
" html " , doctype_fragment
)
# Make sure a Doctype object was created and that the DOCTYPE
# is uppercase.
doctype = soup . contents [ 0 ]
self . assertEqual ( doctype . __class__ , Doctype )
self . assertEqual ( doctype , " html " )
self . assertEqual (
soup . encode ( " utf8 " ) [ : len ( doctype_str ) ] ,
b " <!DOCTYPE html> "
)
# Make sure that the doctype was correctly associated with the
# parse tree and that the rest of the document parsed.
self . assertEqual ( soup . p . contents [ 0 ] , ' foo ' )
def test_public_doctype_with_url ( self ) :
doctype = ' html PUBLIC " -//W3C//DTD XHTML 1.0 Transitional//EN " " http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd " '
self . assertDoctypeHandled ( doctype )
def test_system_doctype ( self ) :
self . assertDoctypeHandled ( ' foo SYSTEM " http://www.example.com/ " ' )
def test_namespaced_system_doctype ( self ) :
# We can handle a namespaced doctype with a system ID.
self . assertDoctypeHandled ( ' xsl:stylesheet SYSTEM " htmlent.dtd " ' )
def test_namespaced_public_doctype ( self ) :
# Test a namespaced doctype with a public id.
self . assertDoctypeHandled ( ' xsl:stylesheet PUBLIC " htmlent.dtd " ' )
def test_real_xhtml_document ( self ) :
""" A real XHTML document should come out more or less the same as it went in. """
markup = b """ <?xml version= " 1.0 " encoding= " utf-8 " ?>
< ! DOCTYPE html PUBLIC " -//W3C//DTD XHTML 1.0 Transitional//EN " >
< html xmlns = " http://www.w3.org/1999/xhtml " >
< head > < title > Hello . < / title > < / head >
< body > Goodbye . < / body >
< / html > """
soup = self . soup ( markup )
self . assertEqual (
soup . encode ( " utf-8 " ) . replace ( b " \n " , b " " ) ,
markup . replace ( b " \n " , b " " ) )
def test_namespaced_html ( self ) :
""" When a namespaced XML document is parsed as HTML it should
be treated as HTML with weird tag names .
"""
markup = b """ <ns1:foo>content</ns1:foo><ns1:foo/><ns2:foo/> """
soup = self . soup ( markup )
self . assertEqual ( 2 , len ( soup . find_all ( " ns1:foo " ) ) )
def test_processing_instruction ( self ) :
# We test both Unicode and bytestring to verify that
# process_markup correctly sets processing_instruction_class
# even when the markup is already Unicode and there is no
# need to process anything.
markup = """ <?PITarget PIContent?> """
soup = self . soup ( markup )
self . assertEqual ( markup , soup . decode ( ) )
markup = b """ <?PITarget PIContent?> """
soup = self . soup ( markup )
self . assertEqual ( markup , soup . encode ( " utf8 " ) )
def test_deepcopy ( self ) :
""" Make sure you can copy the tree builder.
This is important because the builder is part of a
BeautifulSoup object , and we want to be able to copy that .
"""
copy . deepcopy ( self . default_builder )
def test_p_tag_is_never_empty_element ( self ) :
""" A <p> tag is never designated as an empty-element tag.
Even if the markup shows it as an empty - element tag , it
shouldn ' t be presented that way.
"""
soup = self . soup ( " <p/> " )
self . assertFalse ( soup . p . is_empty_element )
self . assertEqual ( str ( soup . p ) , " <p></p> " )
def test_unclosed_tags_get_closed ( self ) :
""" A tag that ' s not closed by the end of the document should be closed.
This applies to all tags except empty - element tags .
"""
self . assertSoupEquals ( " <p> " , " <p></p> " )
self . assertSoupEquals ( " <b> " , " <b></b> " )
self . assertSoupEquals ( " <br> " , " <br/> " )
def test_br_is_always_empty_element_tag ( self ) :
""" A <br> tag is designated as an empty-element tag.
Some parsers treat < br > < / br > as one < br / > tag , some parsers as
two tags , but it should always be an empty - element tag .
"""
soup = self . soup ( " <br></br> " )
self . assertTrue ( soup . br . is_empty_element )
self . assertEqual ( str ( soup . br ) , " <br/> " )
def test_nested_formatting_elements ( self ) :
self . assertSoupEquals ( " <em><em></em></em> " )
def test_double_head ( self ) :
html = ''' <!DOCTYPE html>
< html >
< head >
< title > Ordinary HEAD element test < / title >
< / head >
< script type = " text/javascript " >
alert ( " Help! " ) ;
< / script >
< body >
Hello , world !
< / body >
< / html >
'''
soup = self . soup ( html )
self . assertEqual ( " text/javascript " , soup . find ( ' script ' ) [ ' type ' ] )
def test_comment ( self ) :
# Comments are represented as Comment objects.
markup = " <p>foo<!--foobar-->baz</p> "
self . assertSoupEquals ( markup )
soup = self . soup ( markup )
comment = soup . find ( text = " foobar " )
self . assertEqual ( comment . __class__ , Comment )
# The comment is properly integrated into the tree.
foo = soup . find ( text = " foo " )
self . assertEqual ( comment , foo . next_element )
baz = soup . find ( text = " baz " )
self . assertEqual ( comment , baz . previous_element )
def test_preserved_whitespace_in_pre_and_textarea ( self ) :
""" Whitespace must be preserved in <pre> and <textarea> tags,
even if that would mean not prettifying the markup .
"""
pre_markup = " <pre> </pre> "
textarea_markup = " <textarea> woo \n woo </textarea> "
self . assertSoupEquals ( pre_markup )
self . assertSoupEquals ( textarea_markup )
soup = self . soup ( pre_markup )
self . assertEqual ( soup . pre . prettify ( ) , pre_markup )
soup = self . soup ( textarea_markup )
self . assertEqual ( soup . textarea . prettify ( ) , textarea_markup )
soup = self . soup ( " <textarea></textarea> " )
self . assertEqual ( soup . textarea . prettify ( ) , " <textarea></textarea> " )
def test_nested_inline_elements ( self ) :
""" Inline elements can be nested indefinitely. """
b_tag = " <b>Inside a B tag</b> "
self . assertSoupEquals ( b_tag )
nested_b_tag = " <p>A <i>nested <b>tag</b></i></p> "
self . assertSoupEquals ( nested_b_tag )
double_nested_b_tag = " <p>A <a>doubly <i>nested <b>tag</b></i></a></p> "
self . assertSoupEquals ( nested_b_tag )
def test_nested_block_level_elements ( self ) :
""" Block elements can be nested. """
soup = self . soup ( ' <blockquote><p><b>Foo</b></p></blockquote> ' )
blockquote = soup . blockquote
self . assertEqual ( blockquote . p . b . string , ' Foo ' )
self . assertEqual ( blockquote . b . string , ' Foo ' )
def test_correctly_nested_tables ( self ) :
""" One table can go inside another one. """
markup = ( ' <table id= " 1 " > '
' <tr> '
" <td>Here ' s another table: "
' <table id= " 2 " > '
' <tr><td>foo</td></tr> '
' </table></td> ' )
self . assertSoupEquals (
markup ,
' <table id= " 1 " ><tr><td>Here \' s another table: '
' <table id= " 2 " ><tr><td>foo</td></tr></table> '
' </td></tr></table> ' )
self . assertSoupEquals (
" <table><thead><tr><td>Foo</td></tr></thead> "
" <tbody><tr><td>Bar</td></tr></tbody> "
" <tfoot><tr><td>Baz</td></tr></tfoot></table> " )
def test_multivalued_attribute_with_whitespace ( self ) :
# Whitespace separating the values of a multi-valued attribute
# should be ignored.
markup = ' <div class= " foo bar " ></a> '
soup = self . soup ( markup )
self . assertEqual ( [ ' foo ' , ' bar ' ] , soup . div [ ' class ' ] )
# If you search by the literal name of the class it's like the whitespace
# wasn't there.
self . assertEqual ( soup . div , soup . find ( ' div ' , class_ = " foo bar " ) )
def test_deeply_nested_multivalued_attribute ( self ) :
# html5lib can set the attributes of the same tag many times
# as it rearranges the tree. This has caused problems with
# multivalued attributes.
markup = ' <table><div><div class= " css " ></div></div></table> '
soup = self . soup ( markup )
self . assertEqual ( [ " css " ] , soup . div . div [ ' class ' ] )
def test_multivalued_attribute_on_html ( self ) :
# html5lib uses a different API to set the attributes ot the
# <html> tag. This has caused problems with multivalued
# attributes.
markup = ' <html class= " a b " ></html> '
soup = self . soup ( markup )
self . assertEqual ( [ " a " , " b " ] , soup . html [ ' class ' ] )
def test_angle_brackets_in_attribute_values_are_escaped ( self ) :
self . assertSoupEquals ( ' <a b= " <a> " ></a> ' , ' <a b= " <a> " ></a> ' )
def test_strings_resembling_character_entity_references ( self ) :
# "&T" and "&p" look like incomplete character entities, but they are
# not.
self . assertSoupEquals (
" <p>• AT&T is in the s&p 500</p> " ,
" <p> \u2022 AT&T is in the s&p 500</p> "
)
def test_apos_entity ( self ) :
self . assertSoupEquals (
" <p>Bob's Bar</p> " ,
" <p>Bob ' s Bar</p> " ,
)
def test_entities_in_foreign_document_encoding ( self ) :
# “ and ” are invalid numeric entities referencing
# Windows-1252 characters. - references a character common
# to Windows-1252 and Unicode, and ☃ references a
# character only found in Unicode.
#
# All of these entities should be converted to Unicode
# characters.
markup = " <p>“Hello” -☃</p> "
soup = self . soup ( markup )
self . assertEqual ( " “Hello” -☃ " , soup . p . string )
def test_entities_in_attributes_converted_to_unicode ( self ) :
expect = ' <p id= " pi \N{LATIN SMALL LETTER N WITH TILDE} ata " ></p> '
self . assertSoupEquals ( ' <p id= " piñata " ></p> ' , expect )
self . assertSoupEquals ( ' <p id= " piñata " ></p> ' , expect )
self . assertSoupEquals ( ' <p id= " piñata " ></p> ' , expect )
self . assertSoupEquals ( ' <p id= " piñata " ></p> ' , expect )
def test_entities_in_text_converted_to_unicode ( self ) :
expect = ' <p>pi \N{LATIN SMALL LETTER N WITH TILDE} ata</p> '
self . assertSoupEquals ( " <p>piñata</p> " , expect )
self . assertSoupEquals ( " <p>piñata</p> " , expect )
self . assertSoupEquals ( " <p>piñata</p> " , expect )
self . assertSoupEquals ( " <p>piñata</p> " , expect )
def test_quot_entity_converted_to_quotation_mark ( self ) :
self . assertSoupEquals ( " <p>I said "good day!"</p> " ,
' <p>I said " good day! " </p> ' )
def test_out_of_range_entity ( self ) :
expect = " \N{REPLACEMENT CHARACTER} "
self . assertSoupEquals ( " � " , expect )
self . assertSoupEquals ( " � " , expect )
self . assertSoupEquals ( " � " , expect )
def test_multipart_strings ( self ) :
" Mostly to prevent a recurrence of a bug in the html5lib treebuilder. "
soup = self . soup ( " <html><h2> \n foo</h2><p></p></html> " )
self . assertEqual ( " p " , soup . h2 . string . next_element . name )
self . assertEqual ( " p " , soup . p . name )
self . assertConnectedness ( soup )
def test_empty_element_tags ( self ) :
""" Verify consistent handling of empty-element tags,
no matter how they come in through the markup .
"""
self . assertSoupEquals ( ' <br/><br/><br/> ' , " <br/><br/><br/> " )
self . assertSoupEquals ( ' <br /><br /><br /> ' , " <br/><br/><br/> " )
def test_head_tag_between_head_and_body ( self ) :
" Prevent recurrence of a bug in the html5lib treebuilder. "
content = """ <html><head></head>
< link > < / link >
< body > foo < / body >
< / html >
"""
soup = self . soup ( content )
self . assertNotEqual ( None , soup . html . body )
self . assertConnectedness ( soup )
def test_multiple_copies_of_a_tag ( self ) :
" Prevent recurrence of a bug in the html5lib treebuilder. "
content = """ <!DOCTYPE html>
< html >
< body >
< article id = " a " >
< div > < a href = " 1 " > < / div >
< footer >
< a href = " 2 " > < / a >
< / footer >
< / article >
< / body >
< / html >
"""
soup = self . soup ( content )
self . assertConnectedness ( soup . article )
def test_basic_namespaces ( self ) :
""" Parsers don ' t need to *understand* namespaces, but at the
very least they should not choke on namespaces or lose
data . """
markup = b ' <html xmlns= " http://www.w3.org/1999/xhtml " xmlns:mathml= " http://www.w3.org/1998/Math/MathML " xmlns:svg= " http://www.w3.org/2000/svg " ><head></head><body><mathml:msqrt>4</mathml:msqrt><b svg:fill= " red " ></b></body></html> '
soup = self . soup ( markup )
self . assertEqual ( markup , soup . encode ( ) )
html = soup . html
self . assertEqual ( ' http://www.w3.org/1999/xhtml ' , soup . html [ ' xmlns ' ] )
self . assertEqual (
' http://www.w3.org/1998/Math/MathML ' , soup . html [ ' xmlns:mathml ' ] )
self . assertEqual (
' http://www.w3.org/2000/svg ' , soup . html [ ' xmlns:svg ' ] )
def test_multivalued_attribute_value_becomes_list ( self ) :
markup = b ' <a class= " foo bar " > '
soup = self . soup ( markup )
self . assertEqual ( [ ' foo ' , ' bar ' ] , soup . a [ ' class ' ] )
#
# Generally speaking, tests below this point are more tests of
# Beautiful Soup than tests of the tree builders. But parsers are
# weird, so we run these tests separately for every tree builder
# to detect any differences between them.
#
def test_can_parse_unicode_document ( self ) :
# A seemingly innocuous document... but it's in Unicode! And
# it contains characters that can't be represented in the
# encoding found in the declaration! The horror!
markup = ' <html><head><meta encoding= " euc-jp " ></head><body>Sacr \N{LATIN SMALL LETTER E WITH ACUTE} bleu!</body> '
soup = self . soup ( markup )
self . assertEqual ( ' Sacr \xe9 bleu! ' , soup . body . string )
def test_soupstrainer ( self ) :
""" Parsers should be able to work with SoupStrainers. """
strainer = SoupStrainer ( " b " )
soup = self . soup ( " A <b>bold</b> <meta/> <i>statement</i> " ,
parse_only = strainer )
self . assertEqual ( soup . decode ( ) , " <b>bold</b> " )
def test_single_quote_attribute_values_become_double_quotes ( self ) :
self . assertSoupEquals ( " <foo attr= ' bar ' ></foo> " ,
' <foo attr= " bar " ></foo> ' )
def test_attribute_values_with_nested_quotes_are_left_alone ( self ) :
text = """ <foo attr= ' bar " brawls " happen ' >a</foo> """
self . assertSoupEquals ( text )
def test_attribute_values_with_double_nested_quotes_get_quoted ( self ) :
text = """ <foo attr= ' bar " brawls " happen ' >a</foo> """
soup = self . soup ( text )
soup . foo [ ' attr ' ] = ' Brawls happen at " Bob \' s Bar " '
self . assertSoupEquals (
soup . foo . decode ( ) ,
""" <foo attr= " Brawls happen at "Bob \' s Bar" " >a</foo> """ )
def test_ampersand_in_attribute_value_gets_escaped ( self ) :
self . assertSoupEquals ( ' <this is= " really messed up & stuff " ></this> ' ,
' <this is= " really messed up & stuff " ></this> ' )
self . assertSoupEquals (
' <a href= " http://example.org?a=1&b=2;3 " >foo</a> ' ,
' <a href= " http://example.org?a=1&b=2;3 " >foo</a> ' )
def test_escaped_ampersand_in_attribute_value_is_left_alone ( self ) :
self . assertSoupEquals ( ' <a href= " http://example.org?a=1&b=2;3 " ></a> ' )
def test_entities_in_strings_converted_during_parsing ( self ) :
# Both XML and HTML entities are converted to Unicode characters
# during parsing.
text = " <p><<sacré bleu!>></p> "
expected = " <p><<sacr \N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></p> "
self . assertSoupEquals ( text , expected )
def test_smart_quotes_converted_on_the_way_in ( self ) :
# Microsoft smart quotes are converted to Unicode characters during
# parsing.
quote = b " <p> \x91 Foo \x92 </p> "
soup = self . soup ( quote )
self . assertEqual (
soup . p . string ,
" \N{LEFT SINGLE QUOTATION MARK} Foo \N{RIGHT SINGLE QUOTATION MARK} " )
def test_non_breaking_spaces_converted_on_the_way_in ( self ) :
soup = self . soup ( " <a> </a> " )
self . assertEqual ( soup . a . string , " \N{NO-BREAK SPACE} " * 2 )
def test_entities_converted_on_the_way_out ( self ) :
text = " <p><<sacré bleu!>></p> "
expected = " <p><<sacr \N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></p> " . encode ( " utf-8 " )
soup = self . soup ( text )
self . assertEqual ( soup . p . encode ( " utf-8 " ) , expected )
def test_real_iso_latin_document ( self ) :
# Smoke test of interrelated functionality, using an
# easy-to-understand document.
# Here it is in Unicode. Note that it claims to be in ISO-Latin-1.
unicode_html = ' <html><head><meta content= " text/html; charset=ISO-Latin-1 " http-equiv= " Content-type " /></head><body><p>Sacr \N{LATIN SMALL LETTER E WITH ACUTE} bleu!</p></body></html> '
# That's because we're going to encode it into ISO-Latin-1, and use
# that to test.
iso_latin_html = unicode_html . encode ( " iso-8859-1 " )
# Parse the ISO-Latin-1 HTML.
soup = self . soup ( iso_latin_html )
# Encode it to UTF-8.
result = soup . encode ( " utf-8 " )
# What do we expect the result to look like? Well, it would
# look like unicode_html, except that the META tag would say
# UTF-8 instead of ISO-Latin-1.
expected = unicode_html . replace ( " ISO-Latin-1 " , " utf-8 " )
# And, of course, it would be in UTF-8, not Unicode.
expected = expected . encode ( " utf-8 " )
# Ta-da!
self . assertEqual ( result , expected )
def test_real_shift_jis_document ( self ) :
# Smoke test to make sure the parser can handle a document in
# Shift-JIS encoding, without choking.
shift_jis_html = (
b ' <html><head></head><body><pre> '
b ' \x82 \xb1 \x82 \xea \x82 \xcd Shift-JIS \x82 \xc5 \x83 R \x81 [ \x83 f '
b ' \x83 B \x83 \x93 \x83 O \x82 \xb3 \x82 \xea \x82 \xbd \x93 \xfa \x96 { \x8c '
b ' \xea \x82 \xcc \x83 t \x83 @ \x83 C \x83 \x8b \x82 \xc5 \x82 \xb7 \x81 B '
b ' </pre></body></html> ' )
unicode_html = shift_jis_html . decode ( " shift-jis " )
soup = self . soup ( unicode_html )
# Make sure the parse tree is correctly encoded to various
# encodings.
self . assertEqual ( soup . encode ( " utf-8 " ) , unicode_html . encode ( " utf-8 " ) )
self . assertEqual ( soup . encode ( " euc_jp " ) , unicode_html . encode ( " euc_jp " ) )
def test_real_hebrew_document ( self ) :
# A real-world test to make sure we can convert ISO-8859-9 (a
# Hebrew encoding) to UTF-8.
hebrew_document = b ' <html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1> \xed \xe5 \xec \xf9 </body></html> '
soup = self . soup (
hebrew_document , from_encoding = " iso8859-8 " )
# Some tree builders call it iso8859-8, others call it iso-8859-9.
# That's not a difference we really care about.
assert soup . original_encoding in ( ' iso8859-8 ' , ' iso-8859-8 ' )
self . assertEqual (
soup . encode ( ' utf-8 ' ) ,
hebrew_document . decode ( " iso8859-8 " ) . encode ( " utf-8 " ) )
def test_meta_tag_reflects_current_encoding ( self ) :
# Here's the <meta> tag saying that a document is
# encoded in Shift-JIS.
meta_tag = ( ' <meta content= " text/html; charset=x-sjis " '
' http-equiv= " Content-type " /> ' )
# Here's a document incorporating that meta tag.
shift_jis_html = (
' <html><head> \n %s \n '
' <meta http-equiv= " Content-language " content= " ja " /> '
' </head><body>Shift-JIS markup goes here. ' ) % meta_tag
soup = self . soup ( shift_jis_html )
# Parse the document, and the charset is seemingly unaffected.
parsed_meta = soup . find ( ' meta ' , { ' http-equiv ' : ' Content-type ' } )
content = parsed_meta [ ' content ' ]
self . assertEqual ( ' text/html; charset=x-sjis ' , content )
# But that value is actually a ContentMetaAttributeValue object.
self . assertTrue ( isinstance ( content , ContentMetaAttributeValue ) )
# And it will take on a value that reflects its current
# encoding.
self . assertEqual ( ' text/html; charset=utf8 ' , content . encode ( " utf8 " ) )
# For the rest of the story, see TestSubstitutions in
# test_tree.py.
def test_html5_style_meta_tag_reflects_current_encoding ( self ) :
# Here's the <meta> tag saying that a document is
# encoded in Shift-JIS.
meta_tag = ( ' <meta id= " encoding " charset= " x-sjis " /> ' )
# Here's a document incorporating that meta tag.
shift_jis_html = (
' <html><head> \n %s \n '
' <meta http-equiv= " Content-language " content= " ja " /> '
' </head><body>Shift-JIS markup goes here. ' ) % meta_tag
soup = self . soup ( shift_jis_html )
# Parse the document, and the charset is seemingly unaffected.
parsed_meta = soup . find ( ' meta ' , id = " encoding " )
charset = parsed_meta [ ' charset ' ]
self . assertEqual ( ' x-sjis ' , charset )
# But that value is actually a CharsetMetaAttributeValue object.
self . assertTrue ( isinstance ( charset , CharsetMetaAttributeValue ) )
# And it will take on a value that reflects its current
# encoding.
self . assertEqual ( ' utf8 ' , charset . encode ( " utf8 " ) )
def test_python_specific_encodings_not_used_in_charset ( self ) :
# You can encode an HTML document using a Python-specific
# encoding, but that encoding won't be mentioned _inside_ the
# resulting document. Instead, the document will appear to
# have no encoding.
for markup in [
b ' <meta charset= " utf8 " ></head> '
b ' <meta id= " encoding " charset= " utf-8 " /> '
] :
soup = self . soup ( markup )
for encoding in PYTHON_SPECIFIC_ENCODINGS :
if encoding in (
' idna ' , ' mbcs ' , ' oem ' , ' undefined ' ,
' string_escape ' , ' string-escape '
) :
# For one reason or another, these will raise an
# exception if we actually try to use them, so don't
# bother.
continue
encoded = soup . encode ( encoding )
assert b ' meta charset= " " ' in encoded
assert encoding . encode ( " ascii " ) not in encoded
def test_tag_with_no_attributes_can_have_attributes_added ( self ) :
data = self . soup ( " <a>text</a> " )
data . a [ ' foo ' ] = ' bar '
self . assertEqual ( ' <a foo= " bar " >text</a> ' , data . a . decode ( ) )
def test_closing_tag_with_no_opening_tag ( self ) :
# Without BeautifulSoup.open_tag_counter, the </span> tag will
# cause _popToTag to be called over and over again as we look
# for a <span> tag that wasn't there. The result is that 'text2'
# will show up outside the body of the document.
soup = self . soup ( " <body><div><p>text1</p></span>text2</div></body> " )
self . assertEqual (
" <body><div><p>text1</p>text2</div></body> " , soup . body . decode ( )
)
def test_worst_case ( self ) :
""" Test the worst case (currently) for linking issues. """
soup = self . soup ( BAD_DOCUMENT )
self . linkage_validator ( soup )
class XMLTreeBuilderSmokeTest ( TreeBuilderSmokeTest ) :
def test_pickle_and_unpickle_identity ( self ) :
# Pickling a tree, then unpickling it, yields a tree identical
# to the original.
tree = self . soup ( " <a><b>foo</a> " )
dumped = pickle . dumps ( tree , 2 )
loaded = pickle . loads ( dumped )
self . assertEqual ( loaded . __class__ , BeautifulSoup )
self . assertEqual ( loaded . decode ( ) , tree . decode ( ) )
def test_docstring_generated ( self ) :
soup = self . soup ( " <root/> " )
self . assertEqual (
soup . encode ( ) , b ' <?xml version= " 1.0 " encoding= " utf-8 " ?> \n <root/> ' )
def test_xml_declaration ( self ) :
markup = b """ <?xml version= " 1.0 " encoding= " utf8 " ?> \n <foo/> """
soup = self . soup ( markup )
self . assertEqual ( markup , soup . encode ( " utf8 " ) )
def test_python_specific_encodings_not_used_in_xml_declaration ( self ) :
# You can encode an XML document using a Python-specific
# encoding, but that encoding won't be mentioned _inside_ the
# resulting document.
markup = b """ <?xml version= " 1.0 " ?> \n <foo/> """
soup = self . soup ( markup )
for encoding in PYTHON_SPECIFIC_ENCODINGS :
if encoding in (
' idna ' , ' mbcs ' , ' oem ' , ' undefined ' ,
' string_escape ' , ' string-escape '
) :
# For one reason or another, these will raise an
# exception if we actually try to use them, so don't
# bother.
continue
encoded = soup . encode ( encoding )
assert b ' <?xml version= " 1.0 " ?> ' in encoded
assert encoding . encode ( " ascii " ) not in encoded
def test_processing_instruction ( self ) :
markup = b """ <?xml version= " 1.0 " encoding= " utf8 " ?> \n <?PITarget PIContent?> """
soup = self . soup ( markup )
self . assertEqual ( markup , soup . encode ( " utf8 " ) )
def test_real_xhtml_document ( self ) :
""" A real XHTML document should come out *exactly* the same as it went in. """
markup = b """ <?xml version= " 1.0 " encoding= " utf-8 " ?>
< ! DOCTYPE html PUBLIC " -//W3C//DTD XHTML 1.0 Transitional//EN " >
< html xmlns = " http://www.w3.org/1999/xhtml " >
< head > < title > Hello . < / title > < / head >
< body > Goodbye . < / body >
< / html > """
soup = self . soup ( markup )
self . assertEqual (
soup . encode ( " utf-8 " ) , markup )
def test_nested_namespaces ( self ) :
doc = b """ <?xml version= " 1.0 " encoding= " utf-8 " ?>
< ! DOCTYPE html PUBLIC " -//W3C//DTD XHTML 1.1//EN " " http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd " >
< parent xmlns = " http://ns1/ " >
< child xmlns = " http://ns2/ " xmlns : ns3 = " http://ns3/ " >
< grandchild ns3 : attr = " value " xmlns = " http://ns4/ " / >
< / child >
< / parent > """
soup = self . soup ( doc )
self . assertEqual ( doc , soup . encode ( ) )
def test_formatter_processes_script_tag_for_xml_documents ( self ) :
doc = """
< script type = " text/javascript " >
< / script >
"""
soup = BeautifulSoup ( doc , " lxml-xml " )
# lxml would have stripped this while parsing, but we can add
# it later.
soup . script . string = ' console.log( " < < hey > > " ); '
encoded = soup . encode ( )
self . assertTrue ( b " < < hey > > " in encoded )
def test_can_parse_unicode_document ( self ) :
markup = ' <?xml version= " 1.0 " encoding= " euc-jp " ><root>Sacr \N{LATIN SMALL LETTER E WITH ACUTE} bleu!</root> '
soup = self . soup ( markup )
self . assertEqual ( ' Sacr \xe9 bleu! ' , soup . root . string )
def test_popping_namespaced_tag ( self ) :
markup = ' <rss xmlns:dc= " foo " ><dc:creator>b</dc:creator><dc:date>2012-07-02T20:33:42Z</dc:date><dc:rights>c</dc:rights><image>d</image></rss> '
soup = self . soup ( markup )
self . assertEqual (
str ( soup . rss ) , markup )
def test_docstring_includes_correct_encoding ( self ) :
soup = self . soup ( " <root/> " )
self . assertEqual (
soup . encode ( " latin1 " ) ,
b ' <?xml version= " 1.0 " encoding= " latin1 " ?> \n <root/> ' )
def test_large_xml_document ( self ) :
""" A large XML document should come out the same as it went in. """
markup = ( b ' <?xml version= " 1.0 " encoding= " utf-8 " ?> \n <root> '
+ b ' 0 ' * ( 2 * * 12 )
+ b ' </root> ' )
soup = self . soup ( markup )
self . assertEqual ( soup . encode ( " utf-8 " ) , markup )
def test_tags_are_empty_element_if_and_only_if_they_are_empty ( self ) :
self . assertSoupEquals ( " <p> " , " <p/> " )
self . assertSoupEquals ( " <p>foo</p> " )
def test_namespaces_are_preserved ( self ) :
markup = ' <root xmlns:a= " http://example.com/ " xmlns:b= " http://example.net/ " ><a:foo>This tag is in the a namespace</a:foo><b:foo>This tag is in the b namespace</b:foo></root> '
soup = self . soup ( markup )
root = soup . root
self . assertEqual ( " http://example.com/ " , root [ ' xmlns:a ' ] )
self . assertEqual ( " http://example.net/ " , root [ ' xmlns:b ' ] )
def test_closing_namespaced_tag ( self ) :
markup = ' <p xmlns:dc= " http://purl.org/dc/elements/1.1/ " ><dc:date>20010504</dc:date></p> '
soup = self . soup ( markup )
self . assertEqual ( str ( soup . p ) , markup )
def test_namespaced_attributes ( self ) :
markup = ' <foo xmlns:xsi= " http://www.w3.org/2001/XMLSchema-instance " ><bar xsi:schemaLocation= " http://www.example.com " /></foo> '
soup = self . soup ( markup )
self . assertEqual ( str ( soup . foo ) , markup )
def test_namespaced_attributes_xml_namespace ( self ) :
markup = ' <foo xml:lang= " fr " >bar</foo> '
soup = self . soup ( markup )
self . assertEqual ( str ( soup . foo ) , markup )
def test_find_by_prefixed_name ( self ) :
doc = """ <?xml version= " 1.0 " encoding= " utf-8 " ?>
< Document xmlns = " http://example.com/ns0 "
xmlns : ns1 = " http://example.com/ns1 "
xmlns : ns2 = " http://example.com/ns2 "
< ns1 : tag > foo < / ns1 : tag >
< ns1 : tag > bar < / ns1 : tag >
< ns2 : tag key = " value " > baz < / ns2 : tag >
< / Document >
"""
soup = self . soup ( doc )
# There are three <tag> tags.
self . assertEqual ( 3 , len ( soup . find_all ( ' tag ' ) ) )
# But two of them are ns1:tag and one of them is ns2:tag.
self . assertEqual ( 2 , len ( soup . find_all ( ' ns1:tag ' ) ) )
self . assertEqual ( 1 , len ( soup . find_all ( ' ns2:tag ' ) ) )
self . assertEqual ( 1 , len ( soup . find_all ( ' ns2:tag ' , key = ' value ' ) ) )
self . assertEqual ( 3 , len ( soup . find_all ( [ ' ns1:tag ' , ' ns2:tag ' ] ) ) )
def test_copy_tag_preserves_namespace ( self ) :
xml = """ <?xml version= " 1.0 " encoding= " UTF-8 " standalone= " yes " ?>
< w : document xmlns : w = " http://example.com/ns0 " / > """
soup = self . soup ( xml )
tag = soup . document
duplicate = copy . copy ( tag )
# The two tags have the same namespace prefix.
self . assertEqual ( tag . prefix , duplicate . prefix )
def test_worst_case ( self ) :
""" Test the worst case (currently) for linking issues. """
soup = self . soup ( BAD_DOCUMENT )
self . linkage_validator ( soup )
class HTML5TreeBuilderSmokeTest ( HTMLTreeBuilderSmokeTest ) :
""" Smoke test for a tree builder that supports HTML5. """
def test_real_xhtml_document ( self ) :
# Since XHTML is not HTML5, HTML5 parsers are not tested to handle
# XHTML documents in any particular way.
pass
def test_html_tags_have_namespace ( self ) :
markup = " <a> "
soup = self . soup ( markup )
self . assertEqual ( " http://www.w3.org/1999/xhtml " , soup . a . namespace )
def test_svg_tags_have_namespace ( self ) :
markup = ' <svg><circle/></svg> '
soup = self . soup ( markup )
namespace = " http://www.w3.org/2000/svg "
self . assertEqual ( namespace , soup . svg . namespace )
self . assertEqual ( namespace , soup . circle . namespace )
def test_mathml_tags_have_namespace ( self ) :
markup = ' <math><msqrt>5</msqrt></math> '
soup = self . soup ( markup )
namespace = ' http://www.w3.org/1998/Math/MathML '
self . assertEqual ( namespace , soup . math . namespace )
self . assertEqual ( namespace , soup . msqrt . namespace )
def test_xml_declaration_becomes_comment ( self ) :
markup = ' <?xml version= " 1.0 " encoding= " utf-8 " ?><html></html> '
soup = self . soup ( markup )
self . assertTrue ( isinstance ( soup . contents [ 0 ] , Comment ) )
self . assertEqual ( soup . contents [ 0 ] , ' ?xml version= " 1.0 " encoding= " utf-8 " ? ' )
self . assertEqual ( " html " , soup . contents [ 0 ] . next_element . name )
def skipIf ( condition , reason ) :
def nothing ( test , * args , * * kwargs ) :
return None
def decorator ( test_item ) :
if condition :
return nothing
else :
return test_item
return decorator
