3
d[< @ sl d dl Z d dlZd dlZd dlmZ d dlmZmZmZ G dd deZ G dd deZ
G dd d eZ dS )
N)
BeautifulSoup)EntitySubstitutionEncodingDetector
UnicodeDammitc @ sf e Zd ZdZdd Zejjdd d d d gdd Zdd Z dd Z
dd Z dd Z dd Z
dd ZdS )!TestUnicodeDammitz"Standalone tests of UnicodeDammit.c C s d}t |}|j|kstd S )Nu I'm already Unicode! ☃)r unicode_markupAssertionError)selfmarkupdammit r ?/tmp/pip-build-8z3xcdsh/beautifulsoup4/bs4/tests/test_dammit.pytest_unicode_input s z$TestUnicodeDammit.test_unicode_inputz smart_quotes_to,expect_convertedN ‘’“”xml ‘’“”html ‘’“”asciiz''z""c C s, d}t |dg|dj}|dj|ks(tdS )zbVerify the functionality of the smart_quotes_to argument
to the UnicodeDammit constructor.s z windows-1252)known_definite_encodingssmart_quotes_toz
{}N)r r formatr )r r Zexpect_convertedr
Z convertedr r r
test_smart_quotes_to s
z&TestUnicodeDammit.test_smart_quotes_toc C s0 d}t |}|jj dks t|jdks,td S )Ns Sacré bleu! ☃zutf-8u Sacré bleu! ☃)r original_encodinglowerr r )r utf8r r r r
test_detect_utf8& s z"TestUnicodeDammit.test_detect_utf8c C s4 d}t |dg}|jj dks"t|jdks0td S )Ns z
iso-8859-8u םולש)r r r r r )r hebrewr r r r
test_convert_hebrew, s z%TestUnicodeDammit.test_convert_hebrewc C s6 d}t |}|jj dks t|jjd|ks2td S )Ns ケータイ Watchzutf-8)r r r r r encode)r utf_8r r r r
/test_dont_see_smart_quotes_where_there_are_none2 s zATestUnicodeDammit.test_dont_see_smart_quotes_where_there_are_nonec C s, dj d}t|dg}|jj dks(td S )Nu
Räksmörgåszutf-8z
iso-8859-8)r r r r r )r utf8_datar r r r
test_ignore_inappropriate_codecs8 s
z2TestUnicodeDammit.test_ignore_inappropriate_codecsc C s: dj d}x*dD ]"}t||g}|jj dkstqW d S )Nu
Räksmörgåszutf-8.utf8...
utF---16.!)r$ r% r& )r r r r r )r r" Z bad_encodingr r r r
test_ignore_invalid_codecs= s
z,TestUnicodeDammit.test_ignore_invalid_codecsc C sL dj d}t|dgd}|jj dks*tt|ddgd}|jd ksHtd S )Nu
Räksmörgåszutf-8)Zexclude_encodingsz windows-1252)r r r r r )r r" r r r r
test_exclude_encodingsC s
z(TestUnicodeDammit.test_exclude_encodings)Nr )r r )r r ''"")r r) )__name__
__module__ __qualname____doc__r pytestmark parametrizer r r r! r# r' r( r r r r
r s
r c @ sT e Zd Zdd Zdd Zdd Zdd Zd d
Zd d Zd
d Z dd Z
dd Z dS )TestEncodingDetectorc C s" t d}t|j}d|ks td S )Ns' u utf-�)r list encodingsr )r Zdetectedr3 r r r
Ptest_encoding_detector_replaces_junk_in_encoding_name_with_replacement_characterS s
zeTestEncodingDetector.test_encoding_detector_replaces_junk_in_encoding_name_with_replacement_characterc C s, x&dD ] }t |dd}d|jkstqW d S ) N& & $ # T)is_htmlzeuc-jp)r5 r6 r7 r8 )r r r )r datar r r r
test_detect_html5_style_meta_tagY s z5TestEncodingDetector.test_detect_html5_style_meta_tagc C s d}t jj}tjtj zLdd }|t j_t|}d|jks@td|j ksNtt
|d}|jsbtW d tjtj |t j_X d S )NsT
بتر
ѐc S s d S )Nr )strr r r
noopy s zETestEncodingDetector.test_last_ditch_entity_replacement..noopTu �z html.parser) bs4r Zchardet_dammitloggingdisableWARNINGr Z contains_replacement_charactersr r r NOTSET)r docchardetr= r Zsoupr r r
"test_last_ditch_entity_replacementc s
z7TestEncodingDetector.test_last_ditch_entity_replacementc C s, d}t |}d|jkstd|jks(td S )Ns < a > < / a > u áézutf-16le)r r r r )r r: r r r r
test_byte_order_mark_removed s z1TestEncodingDetector.test_byte_order_mark_removedc C s d}t |}t |dgd}d|jks(tt |dgd}d|jksDtdgdd |jD ks^td }t |dgd
gd }d
|jkstdd
gd d |jD kstd S )
Ns < a > < / a > zutf-16)r zutf-8)user_encodingszutf-16lec S s g | ] }|d qS )r r ).0xr r r
s zRTestEncodingDetector.test_known_definite_versus_user_encodings..s z
iso-8859-8)r rG c S s g | ] }|d qS )r r )rH rI r r r
rJ s )r r r tried_encodings)r r: r beforeafterr r r r
)test_known_definite_versus_user_encodings s
z>TestEncodingDetector.test_known_definite_versus_user_encodingsc C sJ d}t |dgdgdgd}d|jks(tdddgdd |jD ksFtd S )Ns z shift-jiszutf-8z
iso-8859-8)r Zoverride_encodingsrG c S s g | ] }|d qS )r r )rH rI r r r
rJ s zKTestEncodingDetector.test_deprecated_override_encodings..)r r r rK )r r r r r r
"test_deprecated_override_encodings s
z7TestEncodingDetector.test_deprecated_override_encodingsc
C s` dj d}dj d}|| | }tjt |jd W d Q R X tj|}d|jdks\td S )Nu ☃ r u “Hi, I like Windows!” windows_1252u+ ☃☃☃“Hi, I like Windows!”☃☃☃u ☃☃☃)r r. ZraisesUnicodeDecodeErrordecoder detwingler )r r rQ rC Zfixedr r r
test_detwingle s
z#TestEncodingDetector.test_detwinglec C s@ x:dD ]2}|j d}|jds"ttj|}||kstqW d S )N œ ₓ ðr )rV rW rX )r endswithr r rT )r Ztricky_unicode_charinputoutputr r r
+test_detwingle_ignores_multibyte_characters s
z@TestEncodingDetector.test_detwingle_ignores_multibyte_charactersc C s d}|j d}d}|j d}tj}||ddd ks6td||ddksJtd||ddks^td||ksntd||ks~td d
}||| d kst||| d kst||| ddd dkst||dd dkst|d | dd dkst|d
| dd d kstd S )Nz0r z,F)r9 zutf-8Tz
iso-8859-1 i )r9 search_entire_document)r_ a)r r Zfind_declared_encodingr )r Z html_unicodeZ
html_bytesZ xml_unicodeZ xml_bytesmZspacerr r r
test_find_declared_encoding s$
z0TestEncodingDetector.test_find_declared_encodingN) r* r+ r, r4 r; rE rF rN rO rU r] rb r r r r
r1 Q s
# r1 c @ s e Zd ZdZdd Zejjdd$d%gd d
Zd d Z d
d Z
dd Z dd Z dd Z
dd Zdd Zdd Zdd Zd d Zd d Zd!d" Zd#S )&TestEntitySubstitutionz1Standalone tests of the EntitySubstitution class.c C s
t | _d S )N)r sub)r r r r
setup_method s z#TestEntitySubstitution.setup_methodzoriginal,substituted foo∀☃õbar foo∀☃õbar ‘’foo“” ‘’foo“”c C s | j j||kstd S )N)rd substitute_htmlr )r originalZ substitutedr r r
test_substitute_html s z+TestEntitySubstitution.test_substitute_htmlc C s> x8d D ]0\}}d}|| }|| }| j j||kstqW d S ) N⊧ ⊧𝔑 𝔑≧̸ ≧̸¬ ¬⫬ ⫬||fj>><<&&z3 %s 4rm rn ro rp rq rr rs rt ru rv rx rx ry rz r{ r| r} r~ )
r r r r r rw r r r r )rd rj r )r entityutemplaterawZ
with_entitiesr r r
test_html5_entity% s z(TestEntitySubstitution.test_html5_entityc C s< d}d}| j j||ks td}d}| j j||ks8td S )Nu fjords ⊔ penguinszfjords ⊔ penguinsu fjords ⊔︀ penguinszfjords ⊔︀ penguins)rd rj r )r r: r
r r r
)test_html5_entity_with_variation_selectorC s z@TestEntitySubstitution.test_html5_entity_with_variation_selectorc C s d}| j j|d|kstd S )NzWelcome to "my bar"F)rd substitute_xmlr )r sr r r
Itest_xml_converstion_includes_no_quotes_if_make_quoted_attribute_is_falseO s z`TestEntitySubstitution.test_xml_converstion_includes_no_quotes_if_make_quoted_attribute_is_falsec C s0 | j jdddkst| j jdddks,td S )NZWelcomeTz "Welcome"z Bob's Barz "Bob's Bar")rd r r )r r r r
6test_xml_attribute_quoting_normally_uses_double_quotesS s zMTestEntitySubstitution.test_xml_attribute_quoting_normally_uses_double_quotesc C s d}| j j|ddkstd S )NzWelcome to "my bar"Tz'Welcome to "my bar"')rd r r )r r r r r
Otest_xml_attribute_quoting_uses_single_quotes_when_value_contains_double_quotesW s zfTestEntitySubstitution.test_xml_attribute_quoting_uses_single_quotes_when_value_contains_double_quotesc C s d}| j j|ddkstd S )NzWelcome to "Bob's Bar"Tz""Welcome to "Bob's Bar"")rd r r )r r r r r
btest_xml_attribute_quoting_escapes_single_quotes_when_value_contains_both_single_and_double_quotes[ s zyTestEntitySubstitution.test_xml_attribute_quoting_escapes_single_quotes_when_value_contains_both_single_and_double_quotesc C s d}| j j||kstd S )NzWelcome to "Bob's Bar")rd r r )r quotedr r r
zfoo<bar>)rd r r )r r r r
'test_xml_quoting_handles_angle_bracketsc s z>TestEntitySubstitution.test_xml_quoting_handles_angle_bracketsc C s | j jddkstd S )NzAT&TzAT&T)rd r r )r r r r
#test_xml_quoting_handles_ampersandsf s z:TestEntitySubstitution.test_xml_quoting_handles_ampersandsc C s | j jddkstd S )Nz ÁT&TzÁT&T)rd r r )r r r r
Etest_xml_quoting_including_ampersands_when_they_are_part_of_an_entityi s z\TestEntitySubstitution.test_xml_quoting_including_ampersands_when_they_are_part_of_an_entityc C s | j jddkstd S )Nz ÁT&TzÁT&T)rd Z"substitute_xml_containing_entitiesr )r r r r
Dtest_xml_quoting_ignoring_ampersands_when_they_are_part_of_an_entityl s z[TestEntitySubstitution.test_xml_quoting_ignoring_ampersands_when_they_are_part_of_an_entityc C s d}| j j||kstdS )z:There's no need to do this except inside attribute values.z Bob's "bar"N)rd rj r )r textr r r
test_quotes_not_html_substitutedo s z7TestEntitySubstitution.test_quotes_not_html_substitutedN)rf rg )rh ri )r* r+ r, r- re r. r/ r0 rl r r r r r r r r r r r r r r r r
rc s& rc ) r. r? r> r Z
bs4.dammitr r r objectr r1 rc r r r r
s E ?