Yanz Mini Shell
[_]
[-]
[X]
[
HomeShell 1
] [
HomeShell 2
] [
Upload
] [
Command Shell
] [
Scripting
] [
About
]
[ Directory ] =>
/
home
veronikagstoette
public_html
wp-content
Action
[*]
New File
[*]
New Folder
Sensitive File
[*]
/etc/passwd
[*]
/etc/shadow
[*]
/etc/resolv.conf
[
Delete
] [
Edit
] [
Rename
] [
Back
]
testing.py 0000644 00000077325 15246412105 0006611 0 ustar 00 # encoding: utf-8 """Helper classes for tests.""" # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. __license__ = "MIT" import pickle import copy import functools import unittest from unittest import TestCase from bs4 import BeautifulSoup from bs4.element import ( CharsetMetaAttributeValue, Comment, ContentMetaAttributeValue, Doctype, SoupStrainer, ) from bs4.builder import HTMLParserTreeBuilder default_builder = HTMLParserTreeBuilder class SoupTest(unittest.TestCase): @property def default_builder(self): return default_builder() def soup(self, markup, **kwargs): """Build a Beautiful Soup object from markup.""" builder = kwargs.pop('builder', self.default_builder) return BeautifulSoup(markup, builder=builder, **kwargs) def document_for(self, markup): """Turn an HTML fragment into a document. The details depend on the builder. """ return self.default_builder.test_fragment_to_document(markup) def assertSoupEquals(self, to_parse, compare_parsed_to=None): builder = self.default_builder obj = BeautifulSoup(to_parse, builder=builder) if compare_parsed_to is None: compare_parsed_to = to_parse self.assertEqual(obj.decode(), self.document_for(compare_parsed_to)) def assertConnectedness(self, element): """Ensure that next_element and previous_element are properly set for all descendants of the given element. """ earlier = None for e in element.descendants: if earlier: self.assertEqual(e, earlier.next_element) self.assertEqual(earlier, e.previous_element) earlier = e class HTMLTreeBuilderSmokeTest(object): """A basic test of a treebuilder's competence. Any HTML treebuilder, present or future, should be able to pass these tests. With invalid markup, there's room for interpretation, and different parsers can handle it differently. But with the markup in these tests, there's not much room for interpretation. """ def test_empty_element_tags(self): """Verify that all HTML4 and HTML5 empty element (aka void element) tags are handled correctly. """ for name in [ 'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr', 'spacer', 'frame' ]: soup = self.soup("") new_tag = soup.new_tag(name) self.assertEqual(True, new_tag.is_empty_element) def test_pickle_and_unpickle_identity(self): # Pickling a tree, then unpickling it, yields a tree identical # to the original. tree = self.soup("<a><b>foo</a>") dumped = pickle.dumps(tree, 2) loaded = pickle.loads(dumped) self.assertEqual(loaded.__class__, BeautifulSoup) self.assertEqual(loaded.decode(), tree.decode()) def assertDoctypeHandled(self, doctype_fragment): """Assert that a given doctype string is handled correctly.""" doctype_str, soup = self._document_with_doctype(doctype_fragment) # Make sure a Doctype object was created. doctype = soup.contents[0] self.assertEqual(doctype.__class__, Doctype) self.assertEqual(doctype, doctype_fragment) self.assertEqual(str(soup)[:len(doctype_str)], doctype_str) # Make sure that the doctype was correctly associated with the # parse tree and that the rest of the document parsed. self.assertEqual(soup.p.contents[0], 'foo') def _document_with_doctype(self, doctype_fragment): """Generate and parse a document with the given doctype.""" doctype = '<!DOCTYPE %s>' % doctype_fragment markup = doctype + '\n<p>foo</p>' soup = self.soup(markup) return doctype, soup def test_normal_doctypes(self): """Make sure normal, everyday HTML doctypes are handled correctly.""" self.assertDoctypeHandled("html") self.assertDoctypeHandled( 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"') def test_empty_doctype(self): soup = self.soup("<!DOCTYPE>") doctype = soup.contents[0] self.assertEqual("", doctype.strip()) def test_public_doctype_with_url(self): doctype = 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"' self.assertDoctypeHandled(doctype) def test_system_doctype(self): self.assertDoctypeHandled('foo SYSTEM "http://www.example.com/"') def test_namespaced_system_doctype(self): # We can handle a namespaced doctype with a system ID. self.assertDoctypeHandled('xsl:stylesheet SYSTEM "htmlent.dtd"') def test_namespaced_public_doctype(self): # Test a namespaced doctype with a public id. self.assertDoctypeHandled('xsl:stylesheet PUBLIC "htmlent.dtd"') def test_real_xhtml_document(self): """A real XHTML document should come out more or less the same as it went in.""" markup = b"""<?xml version="1.0" encoding="utf-8"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"> <html xmlns="http://www.w3.org/1999/xhtml"> <head><title>Hello.</title></head> <body>Goodbye.</body> </html>""" soup = self.soup(markup) self.assertEqual( soup.encode("utf-8").replace(b"\n", b""), markup.replace(b"\n", b"")) def test_namespaced_html(self): """When a namespaced XML document is parsed as HTML it should be treated as HTML with weird tag names. """ markup = b"""<ns1:foo>content</ns1:foo><ns1:foo/><ns2:foo/>""" soup = self.soup(markup) self.assertEqual(2, len(soup.find_all("ns1:foo"))) def test_processing_instruction(self): # We test both Unicode and bytestring to verify that # process_markup correctly sets processing_instruction_class # even when the markup is already Unicode and there is no # need to process anything. markup = """<?PITarget PIContent?>""" soup = self.soup(markup) self.assertEqual(markup, soup.decode()) markup = b"""<?PITarget PIContent?>""" soup = self.soup(markup) self.assertEqual(markup, soup.encode("utf8")) def test_deepcopy(self): """Make sure you can copy the tree builder. This is important because the builder is part of a BeautifulSoup object, and we want to be able to copy that. """ copy.deepcopy(self.default_builder) def test_p_tag_is_never_empty_element(self): """A <p> tag is never designated as an empty-element tag. Even if the markup shows it as an empty-element tag, it shouldn't be presented that way. """ soup = self.soup("<p/>") self.assertFalse(soup.p.is_empty_element) self.assertEqual(str(soup.p), "<p></p>") def test_unclosed_tags_get_closed(self): """A tag that's not closed by the end of the document should be closed. This applies to all tags except empty-element tags. """ self.assertSoupEquals("<p>", "<p></p>") self.assertSoupEquals("<b>", "<b></b>") self.assertSoupEquals("<br>", "<br/>") def test_br_is_always_empty_element_tag(self): """A <br> tag is designated as an empty-element tag. Some parsers treat <br></br> as one <br/> tag, some parsers as two tags, but it should always be an empty-element tag. """ soup = self.soup("<br></br>") self.assertTrue(soup.br.is_empty_element) self.assertEqual(str(soup.br), "<br/>") def test_nested_formatting_elements(self): self.assertSoupEquals("<em><em></em></em>") def test_double_head(self): html = '''<!DOCTYPE html> <html> <head> <title>Ordinary HEAD element test</title> </head> <script type="text/javascript"> alert("Help!"); </script> <body> Hello, world! </body> </html> ''' soup = self.soup(html) self.assertEqual("text/javascript", soup.find('script')['type']) def test_comment(self): # Comments are represented as Comment objects. markup = "<p>foo<!--foobar-->baz</p>" self.assertSoupEquals(markup) soup = self.soup(markup) comment = soup.find(text="foobar") self.assertEqual(comment.__class__, Comment) # The comment is properly integrated into the tree. foo = soup.find(text="foo") self.assertEqual(comment, foo.next_element) baz = soup.find(text="baz") self.assertEqual(comment, baz.previous_element) def test_preserved_whitespace_in_pre_and_textarea(self): """Whitespace must be preserved in <pre> and <textarea> tags, even if that would mean not prettifying the markup. """ pre_markup = "<pre> </pre>" textarea_markup = "<textarea> woo\nwoo </textarea>" self.assertSoupEquals(pre_markup) self.assertSoupEquals(textarea_markup) soup = self.soup(pre_markup) self.assertEqual(soup.pre.prettify(), pre_markup) soup = self.soup(textarea_markup) self.assertEqual(soup.textarea.prettify(), textarea_markup) soup = self.soup("<textarea></textarea>") self.assertEqual(soup.textarea.prettify(), "<textarea></textarea>") def test_nested_inline_elements(self): """Inline elements can be nested indefinitely.""" b_tag = "<b>Inside a B tag</b>" self.assertSoupEquals(b_tag) nested_b_tag = "<p>A <i>nested <b>tag</b></i></p>" self.assertSoupEquals(nested_b_tag) double_nested_b_tag = "<p>A <a>doubly <i>nested <b>tag</b></i></a></p>" self.assertSoupEquals(nested_b_tag) def test_nested_block_level_elements(self): """Block elements can be nested.""" soup = self.soup('<blockquote><p><b>Foo</b></p></blockquote>') blockquote = soup.blockquote self.assertEqual(blockquote.p.b.string, 'Foo') self.assertEqual(blockquote.b.string, 'Foo') def test_correctly_nested_tables(self): """One table can go inside another one.""" markup = ('<table id="1">' '<tr>' "<td>Here's another table:" '<table id="2">' '<tr><td>foo</td></tr>' '</table></td>') self.assertSoupEquals( markup, '<table id="1"><tr><td>Here\'s another table:' '<table id="2"><tr><td>foo</td></tr></table>' '</td></tr></table>') self.assertSoupEquals( "<table><thead><tr><td>Foo</td></tr></thead>" "<tbody><tr><td>Bar</td></tr></tbody>" "<tfoot><tr><td>Baz</td></tr></tfoot></table>") def test_deeply_nested_multivalued_attribute(self): # html5lib can set the attributes of the same tag many times # as it rearranges the tree. This has caused problems with # multivalued attributes. markup = '<table><div><div class="css"></div></div></table>' soup = self.soup(markup) self.assertEqual(["css"], soup.div.div['class']) def test_multivalued_attribute_on_html(self): # html5lib uses a different API to set the attributes ot the # <html> tag. This has caused problems with multivalued # attributes. markup = '<html class="a b"></html>' soup = self.soup(markup) self.assertEqual(["a", "b"], soup.html['class']) def test_angle_brackets_in_attribute_values_are_escaped(self): self.assertSoupEquals('<a b="<a>"></a>', '<a b="<a>"></a>') def test_strings_resembling_character_entity_references(self): # "&T" and "&p" look like incomplete character entities, but they are # not. self.assertSoupEquals( "<p>• AT&T is in the s&p 500</p>", "<p>\\u2022 AT&T is in the s&p 500</p>" ) def test_entities_in_foreign_document_encoding(self): # “ and ” are invalid numeric entities referencing # Windows-1252 characters. - references a character common # to Windows-1252 and Unicode, and ☃ references a # character only found in Unicode. # # All of these entities should be converted to Unicode # characters. markup = "<p>“Hello” -☃</p>" soup = self.soup(markup) self.assertEqual("“Hello” -☃", soup.p.string) def test_entities_in_attributes_converted_to_unicode(self): expect = '<p id="pi\N{LATIN SMALL LETTER N WITH TILDE}ata"></p>' self.assertSoupEquals('<p id="piñata"></p>', expect) self.assertSoupEquals('<p id="piñata"></p>', expect) self.assertSoupEquals('<p id="piñata"></p>', expect) self.assertSoupEquals('<p id="piñata"></p>', expect) def test_entities_in_text_converted_to_unicode(self): expect = '<p>pi\N{LATIN SMALL LETTER N WITH TILDE}ata</p>' self.assertSoupEquals("<p>piñata</p>", expect) self.assertSoupEquals("<p>piñata</p>", expect) self.assertSoupEquals("<p>piñata</p>", expect) self.assertSoupEquals("<p>piñata</p>", expect) def test_quot_entity_converted_to_quotation_mark(self): self.assertSoupEquals("<p>I said "good day!"</p>", '<p>I said "good day!"</p>') def test_out_of_range_entity(self): expect = "\N{REPLACEMENT CHARACTER}" self.assertSoupEquals("�", expect) self.assertSoupEquals("�", expect) self.assertSoupEquals("�", expect) def test_multipart_strings(self): "Mostly to prevent a recurrence of a bug in the html5lib treebuilder." soup = self.soup("<html><h2>\nfoo</h2><p></p></html>") self.assertEqual("p", soup.h2.string.next_element.name) self.assertEqual("p", soup.p.name) self.assertConnectedness(soup) def test_empty_element_tags(self): """Verify consistent handling of empty-element tags, no matter how they come in through the markup. """ self.assertSoupEquals('<br/><br/><br/>', "<br/><br/><br/>") self.assertSoupEquals('<br /><br /><br />', "<br/><br/><br/>") def test_head_tag_between_head_and_body(self): "Prevent recurrence of a bug in the html5lib treebuilder." content = """<html><head></head> <link></link> <body>foo</body> </html> """ soup = self.soup(content) self.assertNotEqual(None, soup.html.body) self.assertConnectedness(soup) def test_multiple_copies_of_a_tag(self): "Prevent recurrence of a bug in the html5lib treebuilder." content = """<!DOCTYPE html> <html> <body> <article id="a" > <div><a href="1"></div> <footer> <a href="2"></a> </footer> </article> </body> </html> """ soup = self.soup(content) self.assertConnectedness(soup.article) def test_basic_namespaces(self): """Parsers don't need to *understand* namespaces, but at the very least they should not choke on namespaces or lose data.""" markup = b'<html xmlns="http://www.w3.org/1999/xhtml" xmlns:mathml="http://www.w3.org/1998/Math/MathML" xmlns:svg="http://www.w3.org/2000/svg"><head></head><body><mathml:msqrt>4</mathml:msqrt><b svg:fill="red"></b></body></html>' soup = self.soup(markup) self.assertEqual(markup, soup.encode()) html = soup.html self.assertEqual('http://www.w3.org/1999/xhtml', soup.html['xmlns']) self.assertEqual( 'http://www.w3.org/1998/Math/MathML', soup.html['xmlns:mathml']) self.assertEqual( 'http://www.w3.org/2000/svg', soup.html['xmlns:svg']) def test_multivalued_attribute_value_becomes_list(self): markup = b'<a class="foo bar">' soup = self.soup(markup) self.assertEqual(['foo', 'bar'], soup.a['class']) # # Generally speaking, tests below this point are more tests of # Beautiful Soup than tests of the tree builders. But parsers are # weird, so we run these tests separately for every tree builder # to detect any differences between them. # def test_can_parse_unicode_document(self): # A seemingly innocuous document... but it's in Unicode! And # it contains characters that can't be represented in the # encoding found in the declaration! The horror! markup = '<html><head><meta encoding="euc-jp"></head><body>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</body>' soup = self.soup(markup) self.assertEqual('Sacr\xe9 bleu!', soup.body.string) def test_soupstrainer(self): """Parsers should be able to work with SoupStrainers.""" strainer = SoupStrainer("b") soup = self.soup("A <b>bold</b> <meta/> <i>statement</i>", parse_only=strainer) self.assertEqual(soup.decode(), "<b>bold</b>") def test_single_quote_attribute_values_become_double_quotes(self): self.assertSoupEquals("<foo attr='bar'></foo>", '<foo attr="bar"></foo>') def test_attribute_values_with_nested_quotes_are_left_alone(self): text = """<foo attr='bar "brawls" happen'>a</foo>""" self.assertSoupEquals(text) def test_attribute_values_with_double_nested_quotes_get_quoted(self): text = """<foo attr='bar "brawls" happen'>a</foo>""" soup = self.soup(text) soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"' self.assertSoupEquals( soup.foo.decode(), """<foo attr="Brawls happen at "Bob\'s Bar"">a</foo>""") def test_ampersand_in_attribute_value_gets_escaped(self): self.assertSoupEquals('<this is="really messed up & stuff"></this>', '<this is="really messed up & stuff"></this>') self.assertSoupEquals( '<a href="http://example.org?a=1&b=2;3">foo</a>', '<a href="http://example.org?a=1&b=2;3">foo</a>') def test_escaped_ampersand_in_attribute_value_is_left_alone(self): self.assertSoupEquals('<a href="http://example.org?a=1&b=2;3"></a>') def test_entities_in_strings_converted_during_parsing(self): # Both XML and HTML entities are converted to Unicode characters # during parsing. text = "<p><<sacré bleu!>></p>" expected = "<p><<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></p>" self.assertSoupEquals(text, expected) def test_smart_quotes_converted_on_the_way_in(self): # Microsoft smart quotes are converted to Unicode characters during # parsing. quote = b"<p>\x91Foo\x92</p>" soup = self.soup(quote) self.assertEqual( soup.p.string, "\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}") def test_non_breaking_spaces_converted_on_the_way_in(self): soup = self.soup("<a> </a>") self.assertEqual(soup.a.string, "\N{NO-BREAK SPACE}" * 2) def test_entities_converted_on_the_way_out(self): text = "<p><<sacré bleu!>></p>" expected = "<p><<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></p>".encode("utf-8") soup = self.soup(text) self.assertEqual(soup.p.encode("utf-8"), expected) def test_real_iso_latin_document(self): # Smoke test of interrelated functionality, using an # easy-to-understand document. # Here it is in Unicode. Note that it claims to be in ISO-Latin-1. unicode_html = '<html><head><meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type"/></head><body><p>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</p></body></html>' # That's because we're going to encode it into ISO-Latin-1, and use # that to test. iso_latin_html = unicode_html.encode("iso-8859-1") # Parse the ISO-Latin-1 HTML. soup = self.soup(iso_latin_html) # Encode it to UTF-8. result = soup.encode("utf-8") # What do we expect the result to look like? Well, it would # look like unicode_html, except that the META tag would say # UTF-8 instead of ISO-Latin-1. expected = unicode_html.replace("ISO-Latin-1", "utf-8") # And, of course, it would be in UTF-8, not Unicode. expected = expected.encode("utf-8") # Ta-da! self.assertEqual(result, expected) def test_real_shift_jis_document(self): # Smoke test to make sure the parser can handle a document in # Shift-JIS encoding, without choking. shift_jis_html = ( b'<html><head></head><body><pre>' b'\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f' b'\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c' b'\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B' b'</pre></body></html>') unicode_html = shift_jis_html.decode("shift-jis") soup = self.soup(unicode_html) # Make sure the parse tree is correctly encoded to various # encodings. self.assertEqual(soup.encode("utf-8"), unicode_html.encode("utf-8")) self.assertEqual(soup.encode("euc_jp"), unicode_html.encode("euc_jp")) def test_real_hebrew_document(self): # A real-world test to make sure we can convert ISO-8859-9 (a # Hebrew encoding) to UTF-8. hebrew_document = b'<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>' soup = self.soup( hebrew_document, from_encoding="iso8859-8") # Some tree builders call it iso8859-8, others call it iso-8859-9. # That's not a difference we really care about. assert soup.original_encoding in ('iso8859-8', 'iso-8859-8') self.assertEqual( soup.encode('utf-8'), hebrew_document.decode("iso8859-8").encode("utf-8")) def test_meta_tag_reflects_current_encoding(self): # Here's the <meta> tag saying that a document is # encoded in Shift-JIS. meta_tag = ('<meta content="text/html; charset=x-sjis" ' 'http-equiv="Content-type"/>') # Here's a document incorporating that meta tag. shift_jis_html = ( '<html><head>\n%s\n' '<meta http-equiv="Content-language" content="ja"/>' '</head><body>Shift-JIS markup goes here.') % meta_tag soup = self.soup(shift_jis_html) # Parse the document, and the charset is seemingly unaffected. parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'}) content = parsed_meta['content'] self.assertEqual('text/html; charset=x-sjis', content) # But that value is actually a ContentMetaAttributeValue object. self.assertTrue(isinstance(content, ContentMetaAttributeValue)) # And it will take on a value that reflects its current # encoding. self.assertEqual('text/html; charset=utf8', content.encode("utf8")) # For the rest of the story, see TestSubstitutions in # test_tree.py. def test_html5_style_meta_tag_reflects_current_encoding(self): # Here's the <meta> tag saying that a document is # encoded in Shift-JIS. meta_tag = ('<meta id="encoding" charset="x-sjis" />') # Here's a document incorporating that meta tag. shift_jis_html = ( '<html><head>\n%s\n' '<meta http-equiv="Content-language" content="ja"/>' '</head><body>Shift-JIS markup goes here.') % meta_tag soup = self.soup(shift_jis_html) # Parse the document, and the charset is seemingly unaffected. parsed_meta = soup.find('meta', id="encoding") charset = parsed_meta['charset'] self.assertEqual('x-sjis', charset) # But that value is actually a CharsetMetaAttributeValue object. self.assertTrue(isinstance(charset, CharsetMetaAttributeValue)) # And it will take on a value that reflects its current # encoding. self.assertEqual('utf8', charset.encode("utf8")) def test_tag_with_no_attributes_can_have_attributes_added(self): data = self.soup("<a>text</a>") data.a['foo'] = 'bar' self.assertEqual('<a foo="bar">text</a>', data.a.decode()) class XMLTreeBuilderSmokeTest(object): def test_pickle_and_unpickle_identity(self): # Pickling a tree, then unpickling it, yields a tree identical # to the original. tree = self.soup("<a><b>foo</a>") dumped = pickle.dumps(tree, 2) loaded = pickle.loads(dumped) self.assertEqual(loaded.__class__, BeautifulSoup) self.assertEqual(loaded.decode(), tree.decode()) def test_docstring_generated(self): soup = self.soup("<root/>") self.assertEqual( soup.encode(), b'<?xml version="1.0" encoding="utf-8"?>\n<root/>') def test_xml_declaration(self): markup = b"""<?xml version="1.0" encoding="utf8"?>\n<foo/>""" soup = self.soup(markup) self.assertEqual(markup, soup.encode("utf8")) def test_processing_instruction(self): markup = b"""<?xml version="1.0" encoding="utf8"?>\n<?PITarget PIContent?>""" soup = self.soup(markup) self.assertEqual(markup, soup.encode("utf8")) def test_real_xhtml_document(self): """A real XHTML document should come out *exactly* the same as it went in.""" markup = b"""<?xml version="1.0" encoding="utf-8"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"> <html xmlns="http://www.w3.org/1999/xhtml"> <head><title>Hello.</title></head> <body>Goodbye.</body> </html>""" soup = self.soup(markup) self.assertEqual( soup.encode("utf-8"), markup) def test_nested_namespaces(self): doc = b"""<?xml version="1.0" encoding="utf-8"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> <parent xmlns="http://ns1/"> <child xmlns="http://ns2/" xmlns:ns3="http://ns3/"> <grandchild ns3:attr="value" xmlns="http://ns4/"/> </child> </parent>""" soup = self.soup(doc) self.assertEqual(doc, soup.encode()) def test_formatter_processes_script_tag_for_xml_documents(self): doc = """ <script type="text/javascript"> </script> """ soup = BeautifulSoup(doc, "lxml-xml") # lxml would have stripped this while parsing, but we can add # it later. soup.script.string = 'console.log("< < hey > > ");' encoded = soup.encode() self.assertTrue(b"< < hey > >" in encoded) def test_can_parse_unicode_document(self): markup = '<?xml version="1.0" encoding="euc-jp"><root>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</root>' soup = self.soup(markup) self.assertEqual('Sacr\xe9 bleu!', soup.root.string) def test_popping_namespaced_tag(self): markup = '<rss xmlns:dc="foo"><dc:creator>b</dc:creator><dc:date>2012-07-02T20:33:42Z</dc:date><dc:rights>c</dc:rights><image>d</image></rss>' soup = self.soup(markup) self.assertEqual( str(soup.rss), markup) def test_docstring_includes_correct_encoding(self): soup = self.soup("<root/>") self.assertEqual( soup.encode("latin1"), b'<?xml version="1.0" encoding="latin1"?>\n<root/>') def test_large_xml_document(self): """A large XML document should come out the same as it went in.""" markup = (b'<?xml version="1.0" encoding="utf-8"?>\n<root>' + b'0' * (2**12) + b'</root>') soup = self.soup(markup) self.assertEqual(soup.encode("utf-8"), markup) def test_tags_are_empty_element_if_and_only_if_they_are_empty(self): self.assertSoupEquals("<p>", "<p/>") self.assertSoupEquals("<p>foo</p>") def test_namespaces_are_preserved(self): markup = '<root xmlns:a="http://example.com/" xmlns:b="http://example.net/"><a:foo>This tag is in the a namespace</a:foo><b:foo>This tag is in the b namespace</b:foo></root>' soup = self.soup(markup) root = soup.root self.assertEqual("http://example.com/", root['xmlns:a']) self.assertEqual("http://example.net/", root['xmlns:b']) def test_closing_namespaced_tag(self): markup = '<p xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:date>20010504</dc:date></p>' soup = self.soup(markup) self.assertEqual(str(soup.p), markup) def test_namespaced_attributes(self): markup = '<foo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><bar xsi:schemaLocation="http://www.example.com"/></foo>' soup = self.soup(markup) self.assertEqual(str(soup.foo), markup) def test_namespaced_attributes_xml_namespace(self): markup = '<foo xml:lang="fr">bar</foo>' soup = self.soup(markup) self.assertEqual(str(soup.foo), markup) def test_find_by_prefixed_name(self): doc = """<?xml version="1.0" encoding="utf-8"?> <Document xmlns="http://example.com/ns0" xmlns:ns1="http://example.com/ns1" xmlns:ns2="http://example.com/ns2" <ns1:tag>foo</ns1:tag> <ns1:tag>bar</ns1:tag> <ns2:tag key="value">baz</ns2:tag> </Document> """ soup = self.soup(doc) # There are three <tag> tags. self.assertEqual(3, len(soup.find_all('tag'))) # But two of them are ns1:tag and one of them is ns2:tag. self.assertEqual(2, len(soup.find_all('ns1:tag'))) self.assertEqual(1, len(soup.find_all('ns2:tag'))) self.assertEqual(1, len(soup.find_all('ns2:tag', key='value'))) self.assertEqual(3, len(soup.find_all(['ns1:tag', 'ns2:tag']))) def test_copy_tag_preserves_namespace(self): xml = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <w:document xmlns:w="http://example.com/ns0"/>""" soup = self.soup(xml) tag = soup.document duplicate = copy.copy(tag) # The two tags have the same namespace prefix. self.assertEqual(tag.prefix, duplicate.prefix) class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest): """Smoke test for a tree builder that supports HTML5.""" def test_real_xhtml_document(self): # Since XHTML is not HTML5, HTML5 parsers are not tested to handle # XHTML documents in any particular way. pass def test_html_tags_have_namespace(self): markup = "<a>" soup = self.soup(markup) self.assertEqual("http://www.w3.org/1999/xhtml", soup.a.namespace) def test_svg_tags_have_namespace(self): markup = '<svg><circle/></svg>' soup = self.soup(markup) namespace = "http://www.w3.org/2000/svg" self.assertEqual(namespace, soup.svg.namespace) self.assertEqual(namespace, soup.circle.namespace) def test_mathml_tags_have_namespace(self): markup = '<math><msqrt>5</msqrt></math>' soup = self.soup(markup) namespace = 'http://www.w3.org/1998/Math/MathML' self.assertEqual(namespace, soup.math.namespace) self.assertEqual(namespace, soup.msqrt.namespace) def test_xml_declaration_becomes_comment(self): markup = '<?xml version="1.0" encoding="utf-8"?><html></html>' soup = self.soup(markup) self.assertTrue(isinstance(soup.contents[0], Comment)) self.assertEqual(soup.contents[0], '?xml version="1.0" encoding="utf-8"?') self.assertEqual("html", soup.contents[0].next_element.name) def skipIf(condition, reason): def nothing(test, *args, **kwargs): return None def decorator(test_item): if condition: return nothing else: return test_item return decorator tests/test_builder_registry.py 0000644 00000012716 15246412105 0012704 0 ustar 00 """Tests of the builder registry.""" import unittest import warnings from bs4 import BeautifulSoup from bs4.builder import ( builder_registry as registry, HTMLParserTreeBuilder, TreeBuilderRegistry, ) try: from bs4.builder import HTML5TreeBuilder HTML5LIB_PRESENT = True except ImportError: HTML5LIB_PRESENT = False try: from bs4.builder import ( LXMLTreeBuilderForXML, LXMLTreeBuilder, ) LXML_PRESENT = True except ImportError: LXML_PRESENT = False class BuiltInRegistryTest(unittest.TestCase): """Test the built-in registry with the default builders registered.""" def test_combination(self): if LXML_PRESENT: self.assertEqual(registry.lookup('fast', 'html'), LXMLTreeBuilder) if LXML_PRESENT: self.assertEqual(registry.lookup('permissive', 'xml'), LXMLTreeBuilderForXML) self.assertEqual(registry.lookup('strict', 'html'), HTMLParserTreeBuilder) if HTML5LIB_PRESENT: self.assertEqual(registry.lookup('html5lib', 'html'), HTML5TreeBuilder) def test_lookup_by_markup_type(self): if LXML_PRESENT: self.assertEqual(registry.lookup('html'), LXMLTreeBuilder) self.assertEqual(registry.lookup('xml'), LXMLTreeBuilderForXML) else: self.assertEqual(registry.lookup('xml'), None) if HTML5LIB_PRESENT: self.assertEqual(registry.lookup('html'), HTML5TreeBuilder) else: self.assertEqual(registry.lookup('html'), HTMLParserTreeBuilder) def test_named_library(self): if LXML_PRESENT: self.assertEqual(registry.lookup('lxml', 'xml'), LXMLTreeBuilderForXML) self.assertEqual(registry.lookup('lxml', 'html'), LXMLTreeBuilder) if HTML5LIB_PRESENT: self.assertEqual(registry.lookup('html5lib'), HTML5TreeBuilder) self.assertEqual(registry.lookup('html.parser'), HTMLParserTreeBuilder) def test_beautifulsoup_constructor_does_lookup(self): with warnings.catch_warnings(record=True) as w: # This will create a warning about not explicitly # specifying a parser, but we'll ignore it. # You can pass in a string. BeautifulSoup("", features="html") # Or a list of strings. BeautifulSoup("", features=["html", "fast"]) # You'll get an exception if BS can't find an appropriate # builder. self.assertRaises(ValueError, BeautifulSoup, "", features="no-such-feature") class RegistryTest(unittest.TestCase): """Test the TreeBuilderRegistry class in general.""" def setUp(self): self.registry = TreeBuilderRegistry() def builder_for_features(self, *feature_list): cls = type('Builder_' + '_'.join(feature_list), (object,), {'features' : feature_list}) self.registry.register(cls) return cls def test_register_with_no_features(self): builder = self.builder_for_features() # Since the builder advertises no features, you can't find it # by looking up features. self.assertEqual(self.registry.lookup('foo'), None) # But you can find it by doing a lookup with no features, if # this happens to be the only registered builder. self.assertEqual(self.registry.lookup(), builder) def test_register_with_features_makes_lookup_succeed(self): builder = self.builder_for_features('foo', 'bar') self.assertEqual(self.registry.lookup('foo'), builder) self.assertEqual(self.registry.lookup('bar'), builder) def test_lookup_fails_when_no_builder_implements_feature(self): builder = self.builder_for_features('foo', 'bar') self.assertEqual(self.registry.lookup('baz'), None) def test_lookup_gets_most_recent_registration_when_no_feature_specified(self): builder1 = self.builder_for_features('foo') builder2 = self.builder_for_features('bar') self.assertEqual(self.registry.lookup(), builder2) def test_lookup_fails_when_no_tree_builders_registered(self): self.assertEqual(self.registry.lookup(), None) def test_lookup_gets_most_recent_builder_supporting_all_features(self): has_one = self.builder_for_features('foo') has_the_other = self.builder_for_features('bar') has_both_early = self.builder_for_features('foo', 'bar', 'baz') has_both_late = self.builder_for_features('foo', 'bar', 'quux') lacks_one = self.builder_for_features('bar') has_the_other = self.builder_for_features('foo') # There are two builders featuring 'foo' and 'bar', but # the one that also features 'quux' was registered later. self.assertEqual(self.registry.lookup('foo', 'bar'), has_both_late) # There is only one builder featuring 'foo', 'bar', and 'baz'. self.assertEqual(self.registry.lookup('foo', 'bar', 'baz'), has_both_early) def test_lookup_fails_when_cannot_reconcile_requested_features(self): builder1 = self.builder_for_features('foo', 'bar') builder2 = self.builder_for_features('foo', 'baz') self.assertEqual(self.registry.lookup('bar', 'baz'), None) tests/test_lxml.py 0000644 00000005125 15246412105 0010276 0 ustar 00 """Tests to ensure that the lxml tree builder generates good trees.""" import re import warnings try: import lxml.etree LXML_PRESENT = True LXML_VERSION = lxml.etree.LXML_VERSION except ImportError as e: LXML_PRESENT = False LXML_VERSION = (0,) if LXML_PRESENT: from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML from bs4 import ( BeautifulSoup, BeautifulStoneSoup, ) from bs4.element import Comment, Doctype, SoupStrainer from bs4.testing import skipIf from bs4.tests import test_htmlparser from bs4.testing import ( HTMLTreeBuilderSmokeTest, XMLTreeBuilderSmokeTest, SoupTest, skipIf, ) @skipIf( not LXML_PRESENT, "lxml seems not to be present, not testing its tree builder.") class LXMLTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest): """See ``HTMLTreeBuilderSmokeTest``.""" @property def default_builder(self): return LXMLTreeBuilder() def test_out_of_range_entity(self): self.assertSoupEquals( "<p>foo�bar</p>", "<p>foobar</p>") self.assertSoupEquals( "<p>foo�bar</p>", "<p>foobar</p>") self.assertSoupEquals( "<p>foo�bar</p>", "<p>foobar</p>") def test_entities_in_foreign_document_encoding(self): # We can't implement this case correctly because by the time we # hear about markup like "“", it's been (incorrectly) converted into # a string like u'\x93' pass # In lxml < 2.3.5, an empty doctype causes a segfault. Skip this # test if an old version of lxml is installed. @skipIf( not LXML_PRESENT or LXML_VERSION < (2,3,5,0), "Skipping doctype test for old version of lxml to avoid segfault.") def test_empty_doctype(self): soup = self.soup("<!DOCTYPE>") doctype = soup.contents[0] self.assertEqual("", doctype.strip()) def test_beautifulstonesoup_is_xml_parser(self): # Make sure that the deprecated BSS class uses an xml builder # if one is installed. with warnings.catch_warnings(record=True) as w: soup = BeautifulStoneSoup("<b />") self.assertEqual("<b/>", str(soup.b)) self.assertTrue("BeautifulStoneSoup class is deprecated" in str(w[0].message)) @skipIf( not LXML_PRESENT, "lxml seems not to be present, not testing its XML tree builder.") class LXMLXMLTreeBuilderSmokeTest(SoupTest, XMLTreeBuilderSmokeTest): """See ``HTMLTreeBuilderSmokeTest``.""" @property def default_builder(self): return LXMLTreeBuilderForXML() tests/test_tree.py 0000644 00000234315 15246412105 0010266 0 ustar 00 # -*- coding: utf-8 -*- """Tests for Beautiful Soup's tree traversal methods. The tree traversal methods are the main advantage of using Beautiful Soup over just using a parser. Different parsers will build different Beautiful Soup trees given the same markup, but all Beautiful Soup trees can be traversed with the methods tested here. """ from pdb import set_trace import copy import pickle import re import warnings from bs4 import BeautifulSoup from bs4.builder import ( builder_registry, HTMLParserTreeBuilder, ) from bs4.element import ( PY3K, CData, Comment, Declaration, Doctype, NavigableString, SoupStrainer, Tag, ) from bs4.testing import ( SoupTest, skipIf, ) XML_BUILDER_PRESENT = (builder_registry.lookup("xml") is not None) LXML_PRESENT = (builder_registry.lookup("lxml") is not None) class TreeTest(SoupTest): def assertSelects(self, tags, should_match): """Make sure that the given tags have the correct text. This is used in tests that define a bunch of tags, each containing a single string, and then select certain strings by some mechanism. """ self.assertEqual([tag.string for tag in tags], should_match) def assertSelectsIDs(self, tags, should_match): """Make sure that the given tags have the correct IDs. This is used in tests that define a bunch of tags, each containing a single string, and then select certain strings by some mechanism. """ self.assertEqual([tag['id'] for tag in tags], should_match) class TestFind(TreeTest): """Basic tests of the find() method. find() just calls find_all() with limit=1, so it's not tested all that thouroughly here. """ def test_find_tag(self): soup = self.soup("<a>1</a><b>2</b><a>3</a><b>4</b>") self.assertEqual(soup.find("b").string, "2") def test_unicode_text_find(self): soup = self.soup('<h1>Räksmörgås</h1>') self.assertEqual(soup.find(string='Räksmörgås'), 'Räksmörgås') def test_unicode_attribute_find(self): soup = self.soup('<h1 id="Räksmörgås">here it is</h1>') str(soup) self.assertEqual("here it is", soup.find(id='Räksmörgås').text) def test_find_everything(self): """Test an optimization that finds all tags.""" soup = self.soup("<a>foo</a><b>bar</b>") self.assertEqual(2, len(soup.find_all())) def test_find_everything_with_name(self): """Test an optimization that finds all tags with a given name.""" soup = self.soup("<a>foo</a><b>bar</b><a>baz</a>") self.assertEqual(2, len(soup.find_all('a'))) class TestFindAll(TreeTest): """Basic tests of the find_all() method.""" def test_find_all_text_nodes(self): """You can search the tree for text nodes.""" soup = self.soup("<html>Foo<b>bar</b>\xbb</html>") # Exact match. self.assertEqual(soup.find_all(string="bar"), ["bar"]) self.assertEqual(soup.find_all(text="bar"), ["bar"]) # Match any of a number of strings. self.assertEqual( soup.find_all(text=["Foo", "bar"]), ["Foo", "bar"]) # Match a regular expression. self.assertEqual(soup.find_all(text=re.compile('.*')), ["Foo", "bar", '\xbb']) # Match anything. self.assertEqual(soup.find_all(text=True), ["Foo", "bar", '\xbb']) def test_find_all_limit(self): """You can limit the number of items returned by find_all.""" soup = self.soup("<a>1</a><a>2</a><a>3</a><a>4</a><a>5</a>") self.assertSelects(soup.find_all('a', limit=3), ["1", "2", "3"]) self.assertSelects(soup.find_all('a', limit=1), ["1"]) self.assertSelects( soup.find_all('a', limit=10), ["1", "2", "3", "4", "5"]) # A limit of 0 means no limit. self.assertSelects( soup.find_all('a', limit=0), ["1", "2", "3", "4", "5"]) def test_calling_a_tag_is_calling_findall(self): soup = self.soup("<a>1</a><b>2<a id='foo'>3</a></b>") self.assertSelects(soup('a', limit=1), ["1"]) self.assertSelects(soup.b(id="foo"), ["3"]) def test_find_all_with_self_referential_data_structure_does_not_cause_infinite_recursion(self): soup = self.soup("<a></a>") # Create a self-referential list. l = [] l.append(l) # Without special code in _normalize_search_value, this would cause infinite # recursion. self.assertEqual([], soup.find_all(l)) def test_find_all_resultset(self): """All find_all calls return a ResultSet""" soup = self.soup("<a></a>") result = soup.find_all("a") self.assertTrue(hasattr(result, "source")) result = soup.find_all(True) self.assertTrue(hasattr(result, "source")) result = soup.find_all(text="foo") self.assertTrue(hasattr(result, "source")) class TestFindAllBasicNamespaces(TreeTest): def test_find_by_namespaced_name(self): soup = self.soup('<mathml:msqrt>4</mathml:msqrt><a svg:fill="red">') self.assertEqual("4", soup.find("mathml:msqrt").string) self.assertEqual("a", soup.find(attrs= { "svg:fill" : "red" }).name) class TestFindAllByName(TreeTest): """Test ways of finding tags by tag name.""" def setUp(self): super(TreeTest, self).setUp() self.tree = self.soup("""<a>First tag.</a> <b>Second tag.</b> <c>Third <a>Nested tag.</a> tag.</c>""") def test_find_all_by_tag_name(self): # Find all the <a> tags. self.assertSelects( self.tree.find_all('a'), ['First tag.', 'Nested tag.']) def test_find_all_by_name_and_text(self): self.assertSelects( self.tree.find_all('a', text='First tag.'), ['First tag.']) self.assertSelects( self.tree.find_all('a', text=True), ['First tag.', 'Nested tag.']) self.assertSelects( self.tree.find_all('a', text=re.compile("tag")), ['First tag.', 'Nested tag.']) def test_find_all_on_non_root_element(self): # You can call find_all on any node, not just the root. self.assertSelects(self.tree.c.find_all('a'), ['Nested tag.']) def test_calling_element_invokes_find_all(self): self.assertSelects(self.tree('a'), ['First tag.', 'Nested tag.']) def test_find_all_by_tag_strainer(self): self.assertSelects( self.tree.find_all(SoupStrainer('a')), ['First tag.', 'Nested tag.']) def test_find_all_by_tag_names(self): self.assertSelects( self.tree.find_all(['a', 'b']), ['First tag.', 'Second tag.', 'Nested tag.']) def test_find_all_by_tag_dict(self): self.assertSelects( self.tree.find_all({'a' : True, 'b' : True}), ['First tag.', 'Second tag.', 'Nested tag.']) def test_find_all_by_tag_re(self): self.assertSelects( self.tree.find_all(re.compile('^[ab]$')), ['First tag.', 'Second tag.', 'Nested tag.']) def test_find_all_with_tags_matching_method(self): # You can define an oracle method that determines whether # a tag matches the search. def id_matches_name(tag): return tag.name == tag.get('id') tree = self.soup("""<a id="a">Match 1.</a> <a id="1">Does not match.</a> <b id="b">Match 2.</a>""") self.assertSelects( tree.find_all(id_matches_name), ["Match 1.", "Match 2."]) def test_find_with_multi_valued_attribute(self): soup = self.soup( "<div class='a b'>1</div><div class='a c'>2</div><div class='a d'>3</div>" ) r1 = soup.find('div', 'a d'); r2 = soup.find('div', re.compile(r'a d')); r3, r4 = soup.find_all('div', ['a b', 'a d']); self.assertEqual('3', r1.string) self.assertEqual('3', r2.string) self.assertEqual('1', r3.string) self.assertEqual('3', r4.string) class TestFindAllByAttribute(TreeTest): def test_find_all_by_attribute_name(self): # You can pass in keyword arguments to find_all to search by # attribute. tree = self.soup(""" <a id="first">Matching a.</a> <a id="second"> Non-matching <b id="first">Matching b.</b>a. </a>""") self.assertSelects(tree.find_all(id='first'), ["Matching a.", "Matching b."]) def test_find_all_by_utf8_attribute_value(self): peace = "םולש".encode("utf8") data = '<a title="םולש"></a>'.encode("utf8") soup = self.soup(data) self.assertEqual([soup.a], soup.find_all(title=peace)) self.assertEqual([soup.a], soup.find_all(title=peace.decode("utf8"))) self.assertEqual([soup.a], soup.find_all(title=[peace, "something else"])) def test_find_all_by_attribute_dict(self): # You can pass in a dictionary as the argument 'attrs'. This # lets you search for attributes like 'name' (a fixed argument # to find_all) and 'class' (a reserved word in Python.) tree = self.soup(""" <a name="name1" class="class1">Name match.</a> <a name="name2" class="class2">Class match.</a> <a name="name3" class="class3">Non-match.</a> <name1>A tag called 'name1'.</name1> """) # This doesn't do what you want. self.assertSelects(tree.find_all(name='name1'), ["A tag called 'name1'."]) # This does what you want. self.assertSelects(tree.find_all(attrs={'name' : 'name1'}), ["Name match."]) self.assertSelects(tree.find_all(attrs={'class' : 'class2'}), ["Class match."]) def test_find_all_by_class(self): tree = self.soup(""" <a class="1">Class 1.</a> <a class="2">Class 2.</a> <b class="1">Class 1.</b> <c class="3 4">Class 3 and 4.</c> """) # Passing in the class_ keyword argument will search against # the 'class' attribute. self.assertSelects(tree.find_all('a', class_='1'), ['Class 1.']) self.assertSelects(tree.find_all('c', class_='3'), ['Class 3 and 4.']) self.assertSelects(tree.find_all('c', class_='4'), ['Class 3 and 4.']) # Passing in a string to 'attrs' will also search the CSS class. self.assertSelects(tree.find_all('a', '1'), ['Class 1.']) self.assertSelects(tree.find_all(attrs='1'), ['Class 1.', 'Class 1.']) self.assertSelects(tree.find_all('c', '3'), ['Class 3 and 4.']) self.assertSelects(tree.find_all('c', '4'), ['Class 3 and 4.']) def test_find_by_class_when_multiple_classes_present(self): tree = self.soup("<gar class='foo bar'>Found it</gar>") f = tree.find_all("gar", class_=re.compile("o")) self.assertSelects(f, ["Found it"]) f = tree.find_all("gar", class_=re.compile("a")) self.assertSelects(f, ["Found it"]) # If the search fails to match the individual strings "foo" and "bar", # it will be tried against the combined string "foo bar". f = tree.find_all("gar", class_=re.compile("o b")) self.assertSelects(f, ["Found it"]) def test_find_all_with_non_dictionary_for_attrs_finds_by_class(self): soup = self.soup("<a class='bar'>Found it</a>") self.assertSelects(soup.find_all("a", re.compile("ba")), ["Found it"]) def big_attribute_value(value): return len(value) > 3 self.assertSelects(soup.find_all("a", big_attribute_value), []) def small_attribute_value(value): return len(value) <= 3 self.assertSelects( soup.find_all("a", small_attribute_value), ["Found it"]) def test_find_all_with_string_for_attrs_finds_multiple_classes(self): soup = self.soup('<a class="foo bar"></a><a class="foo"></a>') a, a2 = soup.find_all("a") self.assertEqual([a, a2], soup.find_all("a", "foo")) self.assertEqual([a], soup.find_all("a", "bar")) # If you specify the class as a string that contains a # space, only that specific value will be found. self.assertEqual([a], soup.find_all("a", class_="foo bar")) self.assertEqual([a], soup.find_all("a", "foo bar")) self.assertEqual([], soup.find_all("a", "bar foo")) def test_find_all_by_attribute_soupstrainer(self): tree = self.soup(""" <a id="first">Match.</a> <a id="second">Non-match.</a>""") strainer = SoupStrainer(attrs={'id' : 'first'}) self.assertSelects(tree.find_all(strainer), ['Match.']) def test_find_all_with_missing_attribute(self): # You can pass in None as the value of an attribute to find_all. # This will match tags that do not have that attribute set. tree = self.soup("""<a id="1">ID present.</a> <a>No ID present.</a> <a id="">ID is empty.</a>""") self.assertSelects(tree.find_all('a', id=None), ["No ID present."]) def test_find_all_with_defined_attribute(self): # You can pass in None as the value of an attribute to find_all. # This will match tags that have that attribute set to any value. tree = self.soup("""<a id="1">ID present.</a> <a>No ID present.</a> <a id="">ID is empty.</a>""") self.assertSelects( tree.find_all(id=True), ["ID present.", "ID is empty."]) def test_find_all_with_numeric_attribute(self): # If you search for a number, it's treated as a string. tree = self.soup("""<a id=1>Unquoted attribute.</a> <a id="1">Quoted attribute.</a>""") expected = ["Unquoted attribute.", "Quoted attribute."] self.assertSelects(tree.find_all(id=1), expected) self.assertSelects(tree.find_all(id="1"), expected) def test_find_all_with_list_attribute_values(self): # You can pass a list of attribute values instead of just one, # and you'll get tags that match any of the values. tree = self.soup("""<a id="1">1</a> <a id="2">2</a> <a id="3">3</a> <a>No ID.</a>""") self.assertSelects(tree.find_all(id=["1", "3", "4"]), ["1", "3"]) def test_find_all_with_regular_expression_attribute_value(self): # You can pass a regular expression as an attribute value, and # you'll get tags whose values for that attribute match the # regular expression. tree = self.soup("""<a id="a">One a.</a> <a id="aa">Two as.</a> <a id="ab">Mixed as and bs.</a> <a id="b">One b.</a> <a>No ID.</a>""") self.assertSelects(tree.find_all(id=re.compile("^a+$")), ["One a.", "Two as."]) def test_find_by_name_and_containing_string(self): soup = self.soup("<b>foo</b><b>bar</b><a>foo</a>") a = soup.a self.assertEqual([a], soup.find_all("a", text="foo")) self.assertEqual([], soup.find_all("a", text="bar")) self.assertEqual([], soup.find_all("a", text="bar")) def test_find_by_name_and_containing_string_when_string_is_buried(self): soup = self.soup("<a>foo</a><a><b><c>foo</c></b></a>") self.assertEqual(soup.find_all("a"), soup.find_all("a", text="foo")) def test_find_by_attribute_and_containing_string(self): soup = self.soup('<b id="1">foo</b><a id="2">foo</a>') a = soup.a self.assertEqual([a], soup.find_all(id=2, text="foo")) self.assertEqual([], soup.find_all(id=1, text="bar")) class TestIndex(TreeTest): """Test Tag.index""" def test_index(self): tree = self.soup("""<div> <a>Identical</a> <b>Not identical</b> <a>Identical</a> <c><d>Identical with child</d></c> <b>Also not identical</b> <c><d>Identical with child</d></c> </div>""") div = tree.div for i, element in enumerate(div.contents): self.assertEqual(i, div.index(element)) self.assertRaises(ValueError, tree.index, 1) class TestParentOperations(TreeTest): """Test navigation and searching through an element's parents.""" def setUp(self): super(TestParentOperations, self).setUp() self.tree = self.soup('''<ul id="empty"></ul> <ul id="top"> <ul id="middle"> <ul id="bottom"> <b>Start here</b> </ul> </ul>''') self.start = self.tree.b def test_parent(self): self.assertEqual(self.start.parent['id'], 'bottom') self.assertEqual(self.start.parent.parent['id'], 'middle') self.assertEqual(self.start.parent.parent.parent['id'], 'top') def test_parent_of_top_tag_is_soup_object(self): top_tag = self.tree.contents[0] self.assertEqual(top_tag.parent, self.tree) def test_soup_object_has_no_parent(self): self.assertEqual(None, self.tree.parent) def test_find_parents(self): self.assertSelectsIDs( self.start.find_parents('ul'), ['bottom', 'middle', 'top']) self.assertSelectsIDs( self.start.find_parents('ul', id="middle"), ['middle']) def test_find_parent(self): self.assertEqual(self.start.find_parent('ul')['id'], 'bottom') self.assertEqual(self.start.find_parent('ul', id='top')['id'], 'top') def test_parent_of_text_element(self): text = self.tree.find(text="Start here") self.assertEqual(text.parent.name, 'b') def test_text_element_find_parent(self): text = self.tree.find(text="Start here") self.assertEqual(text.find_parent('ul')['id'], 'bottom') def test_parent_generator(self): parents = [parent['id'] for parent in self.start.parents if parent is not None and 'id' in parent.attrs] self.assertEqual(parents, ['bottom', 'middle', 'top']) class ProximityTest(TreeTest): def setUp(self): super(TreeTest, self).setUp() self.tree = self.soup( '<html id="start"><head></head><body><b id="1">One</b><b id="2">Two</b><b id="3">Three</b></body></html>') class TestNextOperations(ProximityTest): def setUp(self): super(TestNextOperations, self).setUp() self.start = self.tree.b def test_next(self): self.assertEqual(self.start.next_element, "One") self.assertEqual(self.start.next_element.next_element['id'], "2") def test_next_of_last_item_is_none(self): last = self.tree.find(text="Three") self.assertEqual(last.next_element, None) def test_next_of_root_is_none(self): # The document root is outside the next/previous chain. self.assertEqual(self.tree.next_element, None) def test_find_all_next(self): self.assertSelects(self.start.find_all_next('b'), ["Two", "Three"]) self.start.find_all_next(id=3) self.assertSelects(self.start.find_all_next(id=3), ["Three"]) def test_find_next(self): self.assertEqual(self.start.find_next('b')['id'], '2') self.assertEqual(self.start.find_next(text="Three"), "Three") def test_find_next_for_text_element(self): text = self.tree.find(text="One") self.assertEqual(text.find_next("b").string, "Two") self.assertSelects(text.find_all_next("b"), ["Two", "Three"]) def test_next_generator(self): start = self.tree.find(text="Two") successors = [node for node in start.next_elements] # There are two successors: the final <b> tag and its text contents. tag, contents = successors self.assertEqual(tag['id'], '3') self.assertEqual(contents, "Three") class TestPreviousOperations(ProximityTest): def setUp(self): super(TestPreviousOperations, self).setUp() self.end = self.tree.find(text="Three") def test_previous(self): self.assertEqual(self.end.previous_element['id'], "3") self.assertEqual(self.end.previous_element.previous_element, "Two") def test_previous_of_first_item_is_none(self): first = self.tree.find('html') self.assertEqual(first.previous_element, None) def test_previous_of_root_is_none(self): # The document root is outside the next/previous chain. # XXX This is broken! #self.assertEqual(self.tree.previous_element, None) pass def test_find_all_previous(self): # The <b> tag containing the "Three" node is the predecessor # of the "Three" node itself, which is why "Three" shows up # here. self.assertSelects( self.end.find_all_previous('b'), ["Three", "Two", "One"]) self.assertSelects(self.end.find_all_previous(id=1), ["One"]) def test_find_previous(self): self.assertEqual(self.end.find_previous('b')['id'], '3') self.assertEqual(self.end.find_previous(text="One"), "One") def test_find_previous_for_text_element(self): text = self.tree.find(text="Three") self.assertEqual(text.find_previous("b").string, "Three") self.assertSelects( text.find_all_previous("b"), ["Three", "Two", "One"]) def test_previous_generator(self): start = self.tree.find(text="One") predecessors = [node for node in start.previous_elements] # There are four predecessors: the <b> tag containing "One" # the <body> tag, the <head> tag, and the <html> tag. b, body, head, html = predecessors self.assertEqual(b['id'], '1') self.assertEqual(body.name, "body") self.assertEqual(head.name, "head") self.assertEqual(html.name, "html") class SiblingTest(TreeTest): def setUp(self): super(SiblingTest, self).setUp() markup = '''<html> <span id="1"> <span id="1.1"></span> </span> <span id="2"> <span id="2.1"></span> </span> <span id="3"> <span id="3.1"></span> </span> <span id="4"></span> </html>''' # All that whitespace looks good but makes the tests more # difficult. Get rid of it. markup = re.compile(r"\n\s*").sub("", markup) self.tree = self.soup(markup) class TestNextSibling(SiblingTest): def setUp(self): super(TestNextSibling, self).setUp() self.start = self.tree.find(id="1") def test_next_sibling_of_root_is_none(self): self.assertEqual(self.tree.next_sibling, None) def test_next_sibling(self): self.assertEqual(self.start.next_sibling['id'], '2') self.assertEqual(self.start.next_sibling.next_sibling['id'], '3') # Note the difference between next_sibling and next_element. self.assertEqual(self.start.next_element['id'], '1.1') def test_next_sibling_may_not_exist(self): self.assertEqual(self.tree.html.next_sibling, None) nested_span = self.tree.find(id="1.1") self.assertEqual(nested_span.next_sibling, None) last_span = self.tree.find(id="4") self.assertEqual(last_span.next_sibling, None) def test_find_next_sibling(self): self.assertEqual(self.start.find_next_sibling('span')['id'], '2') def test_next_siblings(self): self.assertSelectsIDs(self.start.find_next_siblings("span"), ['2', '3', '4']) self.assertSelectsIDs(self.start.find_next_siblings(id='3'), ['3']) def test_next_sibling_for_text_element(self): soup = self.soup("Foo<b>bar</b>baz") start = soup.find(text="Foo") self.assertEqual(start.next_sibling.name, 'b') self.assertEqual(start.next_sibling.next_sibling, 'baz') self.assertSelects(start.find_next_siblings('b'), ['bar']) self.assertEqual(start.find_next_sibling(text="baz"), "baz") self.assertEqual(start.find_next_sibling(text="nonesuch"), None) class TestPreviousSibling(SiblingTest): def setUp(self): super(TestPreviousSibling, self).setUp() self.end = self.tree.find(id="4") def test_previous_sibling_of_root_is_none(self): self.assertEqual(self.tree.previous_sibling, None) def test_previous_sibling(self): self.assertEqual(self.end.previous_sibling['id'], '3') self.assertEqual(self.end.previous_sibling.previous_sibling['id'], '2') # Note the difference between previous_sibling and previous_element. self.assertEqual(self.end.previous_element['id'], '3.1') def test_previous_sibling_may_not_exist(self): self.assertEqual(self.tree.html.previous_sibling, None) nested_span = self.tree.find(id="1.1") self.assertEqual(nested_span.previous_sibling, None) first_span = self.tree.find(id="1") self.assertEqual(first_span.previous_sibling, None) def test_find_previous_sibling(self): self.assertEqual(self.end.find_previous_sibling('span')['id'], '3') def test_previous_siblings(self): self.assertSelectsIDs(self.end.find_previous_siblings("span"), ['3', '2', '1']) self.assertSelectsIDs(self.end.find_previous_siblings(id='1'), ['1']) def test_previous_sibling_for_text_element(self): soup = self.soup("Foo<b>bar</b>baz") start = soup.find(text="baz") self.assertEqual(start.previous_sibling.name, 'b') self.assertEqual(start.previous_sibling.previous_sibling, 'Foo') self.assertSelects(start.find_previous_siblings('b'), ['bar']) self.assertEqual(start.find_previous_sibling(text="Foo"), "Foo") self.assertEqual(start.find_previous_sibling(text="nonesuch"), None) class TestTagCreation(SoupTest): """Test the ability to create new tags.""" def test_new_tag(self): soup = self.soup("") new_tag = soup.new_tag("foo", bar="baz", attrs={"name": "a name"}) self.assertTrue(isinstance(new_tag, Tag)) self.assertEqual("foo", new_tag.name) self.assertEqual(dict(bar="baz", name="a name"), new_tag.attrs) self.assertEqual(None, new_tag.parent) def test_tag_inherits_self_closing_rules_from_builder(self): if XML_BUILDER_PRESENT: xml_soup = BeautifulSoup("", "lxml-xml") xml_br = xml_soup.new_tag("br") xml_p = xml_soup.new_tag("p") # Both the <br> and <p> tag are empty-element, just because # they have no contents. self.assertEqual(b"<br/>", xml_br.encode()) self.assertEqual(b"<p/>", xml_p.encode()) html_soup = BeautifulSoup("", "html.parser") html_br = html_soup.new_tag("br") html_p = html_soup.new_tag("p") # The HTML builder users HTML's rules about which tags are # empty-element tags, and the new tags reflect these rules. self.assertEqual(b"<br/>", html_br.encode()) self.assertEqual(b"<p></p>", html_p.encode()) def test_new_string_creates_navigablestring(self): soup = self.soup("") s = soup.new_string("foo") self.assertEqual("foo", s) self.assertTrue(isinstance(s, NavigableString)) def test_new_string_can_create_navigablestring_subclass(self): soup = self.soup("") s = soup.new_string("foo", Comment) self.assertEqual("foo", s) self.assertTrue(isinstance(s, Comment)) class TestTreeModification(SoupTest): def test_attribute_modification(self): soup = self.soup('<a id="1"></a>') soup.a['id'] = 2 self.assertEqual(soup.decode(), self.document_for('<a id="2"></a>')) del(soup.a['id']) self.assertEqual(soup.decode(), self.document_for('<a></a>')) soup.a['id2'] = 'foo' self.assertEqual(soup.decode(), self.document_for('<a id2="foo"></a>')) def test_new_tag_creation(self): builder = builder_registry.lookup('html')() soup = self.soup("<body></body>", builder=builder) a = Tag(soup, builder, 'a') ol = Tag(soup, builder, 'ol') a['href'] = 'http://foo.com/' soup.body.insert(0, a) soup.body.insert(1, ol) self.assertEqual( soup.body.encode(), b'<body><a href="http://foo.com/"></a><ol></ol></body>') def test_append_to_contents_moves_tag(self): doc = """<p id="1">Don't leave me <b>here</b>.</p> <p id="2">Don\'t leave!</p>""" soup = self.soup(doc) second_para = soup.find(id='2') bold = soup.b # Move the <b> tag to the end of the second paragraph. soup.find(id='2').append(soup.b) # The <b> tag is now a child of the second paragraph. self.assertEqual(bold.parent, second_para) self.assertEqual( soup.decode(), self.document_for( '<p id="1">Don\'t leave me .</p>\n' '<p id="2">Don\'t leave!<b>here</b></p>')) def test_replace_with_returns_thing_that_was_replaced(self): text = "<a></a><b><c></c></b>" soup = self.soup(text) a = soup.a new_a = a.replace_with(soup.c) self.assertEqual(a, new_a) def test_unwrap_returns_thing_that_was_replaced(self): text = "<a><b></b><c></c></a>" soup = self.soup(text) a = soup.a new_a = a.unwrap() self.assertEqual(a, new_a) def test_replace_with_and_unwrap_give_useful_exception_when_tag_has_no_parent(self): soup = self.soup("<a><b>Foo</b></a><c>Bar</c>") a = soup.a a.extract() self.assertEqual(None, a.parent) self.assertRaises(ValueError, a.unwrap) self.assertRaises(ValueError, a.replace_with, soup.c) def test_replace_tag_with_itself(self): text = "<a><b></b><c>Foo<d></d></c></a><a><e></e></a>" soup = self.soup(text) c = soup.c soup.c.replace_with(c) self.assertEqual(soup.decode(), self.document_for(text)) def test_replace_tag_with_its_parent_raises_exception(self): text = "<a><b></b></a>" soup = self.soup(text) self.assertRaises(ValueError, soup.b.replace_with, soup.a) def test_insert_tag_into_itself_raises_exception(self): text = "<a><b></b></a>" soup = self.soup(text) self.assertRaises(ValueError, soup.a.insert, 0, soup.a) def test_insert_beautifulsoup_object_inserts_children(self): """Inserting one BeautifulSoup object into another actually inserts all of its children -- you'll never combine BeautifulSoup objects. """ soup = self.soup("<p>And now, a word:</p><p>And we're back.</p>") text = "<p>p2</p><p>p3</p>" to_insert = self.soup(text) soup.insert(1, to_insert) for i in soup.descendants: assert not isinstance(i, BeautifulSoup) p1, p2, p3, p4 = list(soup.children) self.assertEqual("And now, a word:", p1.string) self.assertEqual("p2", p2.string) self.assertEqual("p3", p3.string) self.assertEqual("And we're back.", p4.string) def test_replace_with_maintains_next_element_throughout(self): soup = self.soup('<p><a>one</a><b>three</b></p>') a = soup.a b = a.contents[0] # Make it so the <a> tag has two text children. a.insert(1, "two") # Now replace each one with the empty string. left, right = a.contents left.replaceWith('') right.replaceWith('') # The <b> tag is still connected to the tree. self.assertEqual("three", soup.b.string) def test_replace_final_node(self): soup = self.soup("<b>Argh!</b>") soup.find(text="Argh!").replace_with("Hooray!") new_text = soup.find(text="Hooray!") b = soup.b self.assertEqual(new_text.previous_element, b) self.assertEqual(new_text.parent, b) self.assertEqual(new_text.previous_element.next_element, new_text) self.assertEqual(new_text.next_element, None) def test_consecutive_text_nodes(self): # A builder should never create two consecutive text nodes, # but if you insert one next to another, Beautiful Soup will # handle it correctly. soup = self.soup("<a><b>Argh!</b><c></c></a>") soup.b.insert(1, "Hooray!") self.assertEqual( soup.decode(), self.document_for( "<a><b>Argh!Hooray!</b><c></c></a>")) new_text = soup.find(text="Hooray!") self.assertEqual(new_text.previous_element, "Argh!") self.assertEqual(new_text.previous_element.next_element, new_text) self.assertEqual(new_text.previous_sibling, "Argh!") self.assertEqual(new_text.previous_sibling.next_sibling, new_text) self.assertEqual(new_text.next_sibling, None) self.assertEqual(new_text.next_element, soup.c) def test_insert_string(self): soup = self.soup("<a></a>") soup.a.insert(0, "bar") soup.a.insert(0, "foo") # The string were added to the tag. self.assertEqual(["foo", "bar"], soup.a.contents) # And they were converted to NavigableStrings. self.assertEqual(soup.a.contents[0].next_element, "bar") def test_insert_tag(self): builder = self.default_builder soup = self.soup( "<a><b>Find</b><c>lady!</c><d></d></a>", builder=builder) magic_tag = Tag(soup, builder, 'magictag') magic_tag.insert(0, "the") soup.a.insert(1, magic_tag) self.assertEqual( soup.decode(), self.document_for( "<a><b>Find</b><magictag>the</magictag><c>lady!</c><d></d></a>")) # Make sure all the relationships are hooked up correctly. b_tag = soup.b self.assertEqual(b_tag.next_sibling, magic_tag) self.assertEqual(magic_tag.previous_sibling, b_tag) find = b_tag.find(text="Find") self.assertEqual(find.next_element, magic_tag) self.assertEqual(magic_tag.previous_element, find) c_tag = soup.c self.assertEqual(magic_tag.next_sibling, c_tag) self.assertEqual(c_tag.previous_sibling, magic_tag) the = magic_tag.find(text="the") self.assertEqual(the.parent, magic_tag) self.assertEqual(the.next_element, c_tag) self.assertEqual(c_tag.previous_element, the) def test_append_child_thats_already_at_the_end(self): data = "<a><b></b></a>" soup = self.soup(data) soup.a.append(soup.b) self.assertEqual(data, soup.decode()) def test_move_tag_to_beginning_of_parent(self): data = "<a><b></b><c></c><d></d></a>" soup = self.soup(data) soup.a.insert(0, soup.d) self.assertEqual("<a><d></d><b></b><c></c></a>", soup.decode()) def test_insert_works_on_empty_element_tag(self): # This is a little strange, since most HTML parsers don't allow # markup like this to come through. But in general, we don't # know what the parser would or wouldn't have allowed, so # I'm letting this succeed for now. soup = self.soup("<br/>") soup.br.insert(1, "Contents") self.assertEqual(str(soup.br), "<br>Contents</br>") def test_insert_before(self): soup = self.soup("<a>foo</a><b>bar</b>") soup.b.insert_before("BAZ") soup.a.insert_before("QUUX") self.assertEqual( soup.decode(), self.document_for("QUUX<a>foo</a>BAZ<b>bar</b>")) soup.a.insert_before(soup.b) self.assertEqual( soup.decode(), self.document_for("QUUX<b>bar</b><a>foo</a>BAZ")) def test_insert_after(self): soup = self.soup("<a>foo</a><b>bar</b>") soup.b.insert_after("BAZ") soup.a.insert_after("QUUX") self.assertEqual( soup.decode(), self.document_for("<a>foo</a>QUUX<b>bar</b>BAZ")) soup.b.insert_after(soup.a) self.assertEqual( soup.decode(), self.document_for("QUUX<b>bar</b><a>foo</a>BAZ")) def test_insert_after_raises_exception_if_after_has_no_meaning(self): soup = self.soup("") tag = soup.new_tag("a") string = soup.new_string("") self.assertRaises(ValueError, string.insert_after, tag) self.assertRaises(NotImplementedError, soup.insert_after, tag) self.assertRaises(ValueError, tag.insert_after, tag) def test_insert_before_raises_notimplementederror_if_before_has_no_meaning(self): soup = self.soup("") tag = soup.new_tag("a") string = soup.new_string("") self.assertRaises(ValueError, string.insert_before, tag) self.assertRaises(NotImplementedError, soup.insert_before, tag) self.assertRaises(ValueError, tag.insert_before, tag) def test_replace_with(self): soup = self.soup( "<p>There's <b>no</b> business like <b>show</b> business</p>") no, show = soup.find_all('b') show.replace_with(no) self.assertEqual( soup.decode(), self.document_for( "<p>There's business like <b>no</b> business</p>")) self.assertEqual(show.parent, None) self.assertEqual(no.parent, soup.p) self.assertEqual(no.next_element, "no") self.assertEqual(no.next_sibling, " business") def test_replace_first_child(self): data = "<a><b></b><c></c></a>" soup = self.soup(data) soup.b.replace_with(soup.c) self.assertEqual("<a><c></c></a>", soup.decode()) def test_replace_last_child(self): data = "<a><b></b><c></c></a>" soup = self.soup(data) soup.c.replace_with(soup.b) self.assertEqual("<a><b></b></a>", soup.decode()) def test_nested_tag_replace_with(self): soup = self.soup( """<a>We<b>reserve<c>the</c><d>right</d></b></a><e>to<f>refuse</f><g>service</g></e>""") # Replace the entire <b> tag and its contents ("reserve the # right") with the <f> tag ("refuse"). remove_tag = soup.b move_tag = soup.f remove_tag.replace_with(move_tag) self.assertEqual( soup.decode(), self.document_for( "<a>We<f>refuse</f></a><e>to<g>service</g></e>")) # The <b> tag is now an orphan. self.assertEqual(remove_tag.parent, None) self.assertEqual(remove_tag.find(text="right").next_element, None) self.assertEqual(remove_tag.previous_element, None) self.assertEqual(remove_tag.next_sibling, None) self.assertEqual(remove_tag.previous_sibling, None) # The <f> tag is now connected to the <a> tag. self.assertEqual(move_tag.parent, soup.a) self.assertEqual(move_tag.previous_element, "We") self.assertEqual(move_tag.next_element.next_element, soup.e) self.assertEqual(move_tag.next_sibling, None) # The gap where the <f> tag used to be has been mended, and # the word "to" is now connected to the <g> tag. to_text = soup.find(text="to") g_tag = soup.g self.assertEqual(to_text.next_element, g_tag) self.assertEqual(to_text.next_sibling, g_tag) self.assertEqual(g_tag.previous_element, to_text) self.assertEqual(g_tag.previous_sibling, to_text) def test_unwrap(self): tree = self.soup(""" <p>Unneeded <em>formatting</em> is unneeded</p> """) tree.em.unwrap() self.assertEqual(tree.em, None) self.assertEqual(tree.p.text, "Unneeded formatting is unneeded") def test_wrap(self): soup = self.soup("I wish I was bold.") value = soup.string.wrap(soup.new_tag("b")) self.assertEqual(value.decode(), "<b>I wish I was bold.</b>") self.assertEqual( soup.decode(), self.document_for("<b>I wish I was bold.</b>")) def test_wrap_extracts_tag_from_elsewhere(self): soup = self.soup("<b></b>I wish I was bold.") soup.b.next_sibling.wrap(soup.b) self.assertEqual( soup.decode(), self.document_for("<b>I wish I was bold.</b>")) def test_wrap_puts_new_contents_at_the_end(self): soup = self.soup("<b>I like being bold.</b>I wish I was bold.") soup.b.next_sibling.wrap(soup.b) self.assertEqual(2, len(soup.b.contents)) self.assertEqual( soup.decode(), self.document_for( "<b>I like being bold.I wish I was bold.</b>")) def test_extract(self): soup = self.soup( '<html><body>Some content. <div id="nav">Nav crap</div> More content.</body></html>') self.assertEqual(len(soup.body.contents), 3) extracted = soup.find(id="nav").extract() self.assertEqual( soup.decode(), "<html><body>Some content. More content.</body></html>") self.assertEqual(extracted.decode(), '<div id="nav">Nav crap</div>') # The extracted tag is now an orphan. self.assertEqual(len(soup.body.contents), 2) self.assertEqual(extracted.parent, None) self.assertEqual(extracted.previous_element, None) self.assertEqual(extracted.next_element.next_element, None) # The gap where the extracted tag used to be has been mended. content_1 = soup.find(text="Some content. ") content_2 = soup.find(text=" More content.") self.assertEqual(content_1.next_element, content_2) self.assertEqual(content_1.next_sibling, content_2) self.assertEqual(content_2.previous_element, content_1) self.assertEqual(content_2.previous_sibling, content_1) def test_extract_distinguishes_between_identical_strings(self): soup = self.soup("<a>foo</a><b>bar</b>") foo_1 = soup.a.string bar_1 = soup.b.string foo_2 = soup.new_string("foo") bar_2 = soup.new_string("bar") soup.a.append(foo_2) soup.b.append(bar_2) # Now there are two identical strings in the <a> tag, and two # in the <b> tag. Let's remove the first "foo" and the second # "bar". foo_1.extract() bar_2.extract() self.assertEqual(foo_2, soup.a.string) self.assertEqual(bar_2, soup.b.string) def test_extract_multiples_of_same_tag(self): soup = self.soup(""" <html> <head> <script>foo</script> </head> <body> <script>bar</script> <a></a> </body> <script>baz</script> </html>""") [soup.script.extract() for i in soup.find_all("script")] self.assertEqual("<body>\n\n<a></a>\n</body>", str(soup.body)) def test_extract_works_when_element_is_surrounded_by_identical_strings(self): soup = self.soup( '<html>\n' '<body>hi</body>\n' '</html>') soup.find('body').extract() self.assertEqual(None, soup.find('body')) def test_clear(self): """Tag.clear()""" soup = self.soup("<p><a>String <em>Italicized</em></a> and another</p>") # clear using extract() a = soup.a soup.p.clear() self.assertEqual(len(soup.p.contents), 0) self.assertTrue(hasattr(a, "contents")) # clear using decompose() em = a.em a.clear(decompose=True) self.assertEqual(0, len(em.contents)) def test_string_set(self): """Tag.string = 'string'""" soup = self.soup("<a></a> <b><c></c></b>") soup.a.string = "foo" self.assertEqual(soup.a.contents, ["foo"]) soup.b.string = "bar" self.assertEqual(soup.b.contents, ["bar"]) def test_string_set_does_not_affect_original_string(self): soup = self.soup("<a><b>foo</b><c>bar</c>") soup.b.string = soup.c.string self.assertEqual(soup.a.encode(), b"<a><b>bar</b><c>bar</c></a>") def test_set_string_preserves_class_of_string(self): soup = self.soup("<a></a>") cdata = CData("foo") soup.a.string = cdata self.assertTrue(isinstance(soup.a.string, CData)) class TestElementObjects(SoupTest): """Test various features of element objects.""" def test_len(self): """The length of an element is its number of children.""" soup = self.soup("<top>1<b>2</b>3</top>") # The BeautifulSoup object itself contains one element: the # <top> tag. self.assertEqual(len(soup.contents), 1) self.assertEqual(len(soup), 1) # The <top> tag contains three elements: the text node "1", the # <b> tag, and the text node "3". self.assertEqual(len(soup.top), 3) self.assertEqual(len(soup.top.contents), 3) def test_member_access_invokes_find(self): """Accessing a Python member .foo invokes find('foo')""" soup = self.soup('<b><i></i></b>') self.assertEqual(soup.b, soup.find('b')) self.assertEqual(soup.b.i, soup.find('b').find('i')) self.assertEqual(soup.a, None) def test_deprecated_member_access(self): soup = self.soup('<b><i></i></b>') with warnings.catch_warnings(record=True) as w: tag = soup.bTag self.assertEqual(soup.b, tag) self.assertEqual( '.bTag is deprecated, use .find("b") instead. If you really were looking for a tag called bTag, use .find("bTag")', str(w[0].message)) def test_has_attr(self): """has_attr() checks for the presence of an attribute. Please note note: has_attr() is different from __in__. has_attr() checks the tag's attributes and __in__ checks the tag's chidlren. """ soup = self.soup("<foo attr='bar'>") self.assertTrue(soup.foo.has_attr('attr')) self.assertFalse(soup.foo.has_attr('attr2')) def test_attributes_come_out_in_alphabetical_order(self): markup = '<b a="1" z="5" m="3" f="2" y="4"></b>' self.assertSoupEquals(markup, '<b a="1" f="2" m="3" y="4" z="5"></b>') def test_string(self): # A tag that contains only a text node makes that node # available as .string. soup = self.soup("<b>foo</b>") self.assertEqual(soup.b.string, 'foo') def test_empty_tag_has_no_string(self): # A tag with no children has no .stirng. soup = self.soup("<b></b>") self.assertEqual(soup.b.string, None) def test_tag_with_multiple_children_has_no_string(self): # A tag with no children has no .string. soup = self.soup("<a>foo<b></b><b></b></b>") self.assertEqual(soup.b.string, None) soup = self.soup("<a>foo<b></b>bar</b>") self.assertEqual(soup.b.string, None) # Even if all the children are strings, due to trickery, # it won't work--but this would be a good optimization. soup = self.soup("<a>foo</b>") soup.a.insert(1, "bar") self.assertEqual(soup.a.string, None) def test_tag_with_recursive_string_has_string(self): # A tag with a single child which has a .string inherits that # .string. soup = self.soup("<a><b>foo</b></a>") self.assertEqual(soup.a.string, "foo") self.assertEqual(soup.string, "foo") def test_lack_of_string(self): """Only a tag containing a single text node has a .string.""" soup = self.soup("<b>f<i>e</i>o</b>") self.assertFalse(soup.b.string) soup = self.soup("<b></b>") self.assertFalse(soup.b.string) def test_all_text(self): """Tag.text and Tag.get_text(sep=u"") -> all child text, concatenated""" soup = self.soup("<a>a<b>r</b> <r> t </r></a>") self.assertEqual(soup.a.text, "ar t ") self.assertEqual(soup.a.get_text(strip=True), "art") self.assertEqual(soup.a.get_text(","), "a,r, , t ") self.assertEqual(soup.a.get_text(",", strip=True), "a,r,t") def test_get_text_ignores_comments(self): soup = self.soup("foo<!--IGNORE-->bar") self.assertEqual(soup.get_text(), "foobar") self.assertEqual( soup.get_text(types=(NavigableString, Comment)), "fooIGNOREbar") self.assertEqual( soup.get_text(types=None), "fooIGNOREbar") def test_all_strings_ignores_comments(self): soup = self.soup("foo<!--IGNORE-->bar") self.assertEqual(['foo', 'bar'], list(soup.strings)) class TestCDAtaListAttributes(SoupTest): """Testing cdata-list attributes like 'class'. """ def test_single_value_becomes_list(self): soup = self.soup("<a class='foo'>") self.assertEqual(["foo"],soup.a['class']) def test_multiple_values_becomes_list(self): soup = self.soup("<a class='foo bar'>") self.assertEqual(["foo", "bar"], soup.a['class']) def test_multiple_values_separated_by_weird_whitespace(self): soup = self.soup("<a class='foo\tbar\nbaz'>") self.assertEqual(["foo", "bar", "baz"],soup.a['class']) def test_attributes_joined_into_string_on_output(self): soup = self.soup("<a class='foo\tbar'>") self.assertEqual(b'<a class="foo bar"></a>', soup.a.encode()) def test_get_attribute_list(self): soup = self.soup("<a id='abc def'>") self.assertEqual(['abc def'], soup.a.get_attribute_list('id')) def test_accept_charset(self): soup = self.soup('<form accept-charset="ISO-8859-1 UTF-8">') self.assertEqual(['ISO-8859-1', 'UTF-8'], soup.form['accept-charset']) def test_cdata_attribute_applying_only_to_one_tag(self): data = '<a accept-charset="ISO-8859-1 UTF-8"></a>' soup = self.soup(data) # We saw in another test that accept-charset is a cdata-list # attribute for the <form> tag. But it's not a cdata-list # attribute for any other tag. self.assertEqual('ISO-8859-1 UTF-8', soup.a['accept-charset']) def test_string_has_immutable_name_property(self): string = self.soup("s").string self.assertEqual(None, string.name) def t(): string.name = 'foo' self.assertRaises(AttributeError, t) class TestPersistence(SoupTest): "Testing features like pickle and deepcopy." def setUp(self): super(TestPersistence, self).setUp() self.page = """<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/transitional.dtd"> <html> <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> <title>Beautiful Soup: We called him Tortoise because he taught us.</title> <link rev="made" href="mailto:leonardr@segfault.org"> <meta name="Description" content="Beautiful Soup: an HTML parser optimized for screen-scraping."> <meta name="generator" content="Markov Approximation 1.4 (module: leonardr)"> <meta name="author" content="Leonard Richardson"> </head> <body> <a href="foo">foo</a> <a href="foo"><b>bar</b></a> </body> </html>""" self.tree = self.soup(self.page) def test_pickle_and_unpickle_identity(self): # Pickling a tree, then unpickling it, yields a tree identical # to the original. dumped = pickle.dumps(self.tree, 2) loaded = pickle.loads(dumped) self.assertEqual(loaded.__class__, BeautifulSoup) self.assertEqual(loaded.decode(), self.tree.decode()) def test_deepcopy_identity(self): # Making a deepcopy of a tree yields an identical tree. copied = copy.deepcopy(self.tree) self.assertEqual(copied.decode(), self.tree.decode()) def test_copy_preserves_encoding(self): soup = BeautifulSoup(b'<p> </p>', 'html.parser') encoding = soup.original_encoding copy = soup.__copy__() self.assertEqual("<p> </p>", str(copy)) self.assertEqual(encoding, copy.original_encoding) def test_unicode_pickle(self): # A tree containing Unicode characters can be pickled. html = "<b>\N{SNOWMAN}</b>" soup = self.soup(html) dumped = pickle.dumps(soup, pickle.HIGHEST_PROTOCOL) loaded = pickle.loads(dumped) self.assertEqual(loaded.decode(), soup.decode()) def test_copy_navigablestring_is_not_attached_to_tree(self): html = "<b>Foo<a></a></b><b>Bar</b>" soup = self.soup(html) s1 = soup.find(string="Foo") s2 = copy.copy(s1) self.assertEqual(s1, s2) self.assertEqual(None, s2.parent) self.assertEqual(None, s2.next_element) self.assertNotEqual(None, s1.next_sibling) self.assertEqual(None, s2.next_sibling) self.assertEqual(None, s2.previous_element) def test_copy_navigablestring_subclass_has_same_type(self): html = "<b><!--Foo--></b>" soup = self.soup(html) s1 = soup.string s2 = copy.copy(s1) self.assertEqual(s1, s2) self.assertTrue(isinstance(s2, Comment)) def test_copy_entire_soup(self): html = "<div><b>Foo<a></a></b><b>Bar</b></div>end" soup = self.soup(html) soup_copy = copy.copy(soup) self.assertEqual(soup, soup_copy) def test_copy_tag_copies_contents(self): html = "<div><b>Foo<a></a></b><b>Bar</b></div>end" soup = self.soup(html) div = soup.div div_copy = copy.copy(div) # The two tags look the same, and evaluate to equal. self.assertEqual(str(div), str(div_copy)) self.assertEqual(div, div_copy) # But they're not the same object. self.assertFalse(div is div_copy) # And they don't have the same relation to the parse tree. The # copy is not associated with a parse tree at all. self.assertEqual(None, div_copy.parent) self.assertEqual(None, div_copy.previous_element) self.assertEqual(None, div_copy.find(string='Bar').next_element) self.assertNotEqual(None, div.find(string='Bar').next_element) class TestSubstitutions(SoupTest): def test_default_formatter_is_minimal(self): markup = "<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>" soup = self.soup(markup) decoded = soup.decode(formatter="minimal") # The < is converted back into < but the e-with-acute is left alone. self.assertEqual( decoded, self.document_for( "<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>")) def test_formatter_html(self): markup = "<br><b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>" soup = self.soup(markup) decoded = soup.decode(formatter="html") self.assertEqual( decoded, self.document_for("<br/><b><<Sacré bleu!>></b>")) def test_formatter_html5(self): markup = "<br><b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>" soup = self.soup(markup) decoded = soup.decode(formatter="html5") self.assertEqual( decoded, self.document_for("<br><b><<Sacré bleu!>></b>")) def test_formatter_minimal(self): markup = "<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>" soup = self.soup(markup) decoded = soup.decode(formatter="minimal") # The < is converted back into < but the e-with-acute is left alone. self.assertEqual( decoded, self.document_for( "<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>")) def test_formatter_null(self): markup = "<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>" soup = self.soup(markup) decoded = soup.decode(formatter=None) # Neither the angle brackets nor the e-with-acute are converted. # This is not valid HTML, but it's what the user wanted. self.assertEqual(decoded, self.document_for("<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>")) def test_formatter_custom(self): markup = "<b><foo></b><b>bar</b><br/>" soup = self.soup(markup) decoded = soup.decode(formatter = lambda x: x.upper()) # Instead of normal entity conversion code, the custom # callable is called on every string. self.assertEqual( decoded, self.document_for("<b><FOO></b><b>BAR</b><br>")) def test_formatter_is_run_on_attribute_values(self): markup = '<a href="http://a.com?a=b&c=é">e</a>' soup = self.soup(markup) a = soup.a expect_minimal = '<a href="http://a.com?a=b&c=é">e</a>' self.assertEqual(expect_minimal, a.decode()) self.assertEqual(expect_minimal, a.decode(formatter="minimal")) expect_html = '<a href="http://a.com?a=b&c=é">e</a>' self.assertEqual(expect_html, a.decode(formatter="html")) self.assertEqual(markup, a.decode(formatter=None)) expect_upper = '<a href="HTTP://A.COM?A=B&C=É">E</a>' self.assertEqual(expect_upper, a.decode(formatter=lambda x: x.upper())) def test_formatter_skips_script_tag_for_html_documents(self): doc = """ <script type="text/javascript"> console.log("< < hey > > "); </script> """ encoded = BeautifulSoup(doc, 'html.parser').encode() self.assertTrue(b"< < hey > >" in encoded) def test_formatter_skips_style_tag_for_html_documents(self): doc = """ <style type="text/css"> console.log("< < hey > > "); </style> """ encoded = BeautifulSoup(doc, 'html.parser').encode() self.assertTrue(b"< < hey > >" in encoded) def test_prettify_leaves_preformatted_text_alone(self): soup = self.soup("<div> foo <pre> \tbar\n \n </pre> baz ") # Everything outside the <pre> tag is reformatted, but everything # inside is left alone. self.assertEqual( '<div>\n foo\n <pre> \tbar\n \n </pre>\n baz\n</div>', soup.div.prettify()) def test_prettify_accepts_formatter_function(self): soup = BeautifulSoup("<html><body>foo</body></html>", 'html.parser') pretty = soup.prettify(formatter = lambda x: x.upper()) self.assertTrue("FOO" in pretty) def test_prettify_outputs_unicode_by_default(self): soup = self.soup("<a></a>") self.assertEqual(str, type(soup.prettify())) def test_prettify_can_encode_data(self): soup = self.soup("<a></a>") self.assertEqual(bytes, type(soup.prettify("utf-8"))) def test_html_entity_substitution_off_by_default(self): markup = "<b>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</b>" soup = self.soup(markup) encoded = soup.b.encode("utf-8") self.assertEqual(encoded, markup.encode('utf-8')) def test_encoding_substitution(self): # Here's the <meta> tag saying that a document is # encoded in Shift-JIS. meta_tag = ('<meta content="text/html; charset=x-sjis" ' 'http-equiv="Content-type"/>') soup = self.soup(meta_tag) # Parse the document, and the charset apprears unchanged. self.assertEqual(soup.meta['content'], 'text/html; charset=x-sjis') # Encode the document into some encoding, and the encoding is # substituted into the meta tag. utf_8 = soup.encode("utf-8") self.assertTrue(b"charset=utf-8" in utf_8) euc_jp = soup.encode("euc_jp") self.assertTrue(b"charset=euc_jp" in euc_jp) shift_jis = soup.encode("shift-jis") self.assertTrue(b"charset=shift-jis" in shift_jis) utf_16_u = soup.encode("utf-16").decode("utf-16") self.assertTrue("charset=utf-16" in utf_16_u) def test_encoding_substitution_doesnt_happen_if_tag_is_strained(self): markup = ('<head><meta content="text/html; charset=x-sjis" ' 'http-equiv="Content-type"/></head><pre>foo</pre>') # Beautiful Soup used to try to rewrite the meta tag even if the # meta tag got filtered out by the strainer. This test makes # sure that doesn't happen. strainer = SoupStrainer('pre') soup = self.soup(markup, parse_only=strainer) self.assertEqual(soup.contents[0].name, 'pre') class TestEncoding(SoupTest): """Test the ability to encode objects into strings.""" def test_unicode_string_can_be_encoded(self): html = "<b>\N{SNOWMAN}</b>" soup = self.soup(html) self.assertEqual(soup.b.string.encode("utf-8"), "\N{SNOWMAN}".encode("utf-8")) def test_tag_containing_unicode_string_can_be_encoded(self): html = "<b>\N{SNOWMAN}</b>" soup = self.soup(html) self.assertEqual( soup.b.encode("utf-8"), html.encode("utf-8")) def test_encoding_substitutes_unrecognized_characters_by_default(self): html = "<b>\N{SNOWMAN}</b>" soup = self.soup(html) self.assertEqual(soup.b.encode("ascii"), b"<b>☃</b>") def test_encoding_can_be_made_strict(self): html = "<b>\N{SNOWMAN}</b>" soup = self.soup(html) self.assertRaises( UnicodeEncodeError, soup.encode, "ascii", errors="strict") def test_decode_contents(self): html = "<b>\N{SNOWMAN}</b>" soup = self.soup(html) self.assertEqual("\N{SNOWMAN}", soup.b.decode_contents()) def test_encode_contents(self): html = "<b>\N{SNOWMAN}</b>" soup = self.soup(html) self.assertEqual( "\N{SNOWMAN}".encode("utf8"), soup.b.encode_contents( encoding="utf8")) def test_deprecated_renderContents(self): html = "<b>\N{SNOWMAN}</b>" soup = self.soup(html) self.assertEqual( "\N{SNOWMAN}".encode("utf8"), soup.b.renderContents()) def test_repr(self): html = "<b>\N{SNOWMAN}</b>" soup = self.soup(html) if PY3K: self.assertEqual(html, repr(soup)) else: self.assertEqual(b'<b>\\u2603</b>', repr(soup)) class TestNavigableStringSubclasses(SoupTest): def test_cdata(self): # None of the current builders turn CDATA sections into CData # objects, but you can create them manually. soup = self.soup("") cdata = CData("foo") soup.insert(1, cdata) self.assertEqual(str(soup), "<![CDATA[foo]]>") self.assertEqual(soup.find(text="foo"), "foo") self.assertEqual(soup.contents[0], "foo") def test_cdata_is_never_formatted(self): """Text inside a CData object is passed into the formatter. But the return value is ignored. """ self.count = 0 def increment(*args): self.count += 1 return "BITTER FAILURE" soup = self.soup("") cdata = CData("<><><>") soup.insert(1, cdata) self.assertEqual( b"<![CDATA[<><><>]]>", soup.encode(formatter=increment)) self.assertEqual(1, self.count) def test_doctype_ends_in_newline(self): # Unlike other NavigableString subclasses, a DOCTYPE always ends # in a newline. doctype = Doctype("foo") soup = self.soup("") soup.insert(1, doctype) self.assertEqual(soup.encode(), b"<!DOCTYPE foo>\n") def test_declaration(self): d = Declaration("foo") self.assertEqual("<?foo?>", d.output_ready()) class TestSoupSelector(TreeTest): HTML = """ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> <html> <head> <title>The title</title> <link rel="stylesheet" href="blah.css" type="text/css" id="l1"> </head> <body> <custom-dashed-tag class="dashed" id="dash1">Hello there.</custom-dashed-tag> <div id="main" class="fancy"> <div id="inner"> <h1 id="header1">An H1</h1> <p>Some text</p> <p class="onep" id="p1">Some more text</p> <h2 id="header2">An H2</h2> <p class="class1 class2 class3" id="pmulti">Another</p> <a href="http://bob.example.org/" rel="friend met" id="bob">Bob</a> <h2 id="header3">Another H2</h2> <a id="me" href="http://simonwillison.net/" rel="me">me</a> <span class="s1"> <a href="#" id="s1a1">span1a1</a> <a href="#" id="s1a2">span1a2 <span id="s1a2s1">test</span></a> <span class="span2"> <a href="#" id="s2a1">span2a1</a> </span> <span class="span3"></span> <custom-dashed-tag class="dashed" id="dash2"/> <div data-tag="dashedvalue" id="data1"/> </span> </div> <x id="xid"> <z id="zida"/> <z id="zidab"/> <z id="zidac"/> </x> <y id="yid"> <z id="zidb"/> </y> <p lang="en" id="lang-en">English</p> <p lang="en-gb" id="lang-en-gb">English UK</p> <p lang="en-us" id="lang-en-us">English US</p> <p lang="fr" id="lang-fr">French</p> </div> <div id="footer"> </div> """ def setUp(self): self.soup = BeautifulSoup(self.HTML, 'html.parser') def assertSelects(self, selector, expected_ids, **kwargs): el_ids = [el['id'] for el in self.soup.select(selector, **kwargs)] el_ids.sort() expected_ids.sort() self.assertEqual(expected_ids, el_ids, "Selector %s, expected [%s], got [%s]" % ( selector, ', '.join(expected_ids), ', '.join(el_ids) ) ) assertSelect = assertSelects def assertSelectMultiple(self, *tests): for selector, expected_ids in tests: self.assertSelect(selector, expected_ids) def test_one_tag_one(self): els = self.soup.select('title') self.assertEqual(len(els), 1) self.assertEqual(els[0].name, 'title') self.assertEqual(els[0].contents, ['The title']) def test_one_tag_many(self): els = self.soup.select('div') self.assertEqual(len(els), 4) for div in els: self.assertEqual(div.name, 'div') el = self.soup.select_one('div') self.assertEqual('main', el['id']) def test_select_one_returns_none_if_no_match(self): match = self.soup.select_one('nonexistenttag') self.assertEqual(None, match) def test_tag_in_tag_one(self): els = self.soup.select('div div') self.assertSelects('div div', ['inner', 'data1']) def test_tag_in_tag_many(self): for selector in ('html div', 'html body div', 'body div'): self.assertSelects(selector, ['data1', 'main', 'inner', 'footer']) def test_limit(self): self.assertSelects('html div', ['main'], limit=1) self.assertSelects('html body div', ['inner', 'main'], limit=2) self.assertSelects('body div', ['data1', 'main', 'inner', 'footer'], limit=10) def test_tag_no_match(self): self.assertEqual(len(self.soup.select('del')), 0) def test_invalid_tag(self): self.assertRaises(ValueError, self.soup.select, 'tag%t') def test_select_dashed_tag_ids(self): self.assertSelects('custom-dashed-tag', ['dash1', 'dash2']) def test_select_dashed_by_id(self): dashed = self.soup.select('custom-dashed-tag[id=\"dash2\"]') self.assertEqual(dashed[0].name, 'custom-dashed-tag') self.assertEqual(dashed[0]['id'], 'dash2') def test_dashed_tag_text(self): self.assertEqual(self.soup.select('body > custom-dashed-tag')[0].text, 'Hello there.') def test_select_dashed_matches_find_all(self): self.assertEqual(self.soup.select('custom-dashed-tag'), self.soup.find_all('custom-dashed-tag')) def test_header_tags(self): self.assertSelectMultiple( ('h1', ['header1']), ('h2', ['header2', 'header3']), ) def test_class_one(self): for selector in ('.onep', 'p.onep', 'html p.onep'): els = self.soup.select(selector) self.assertEqual(len(els), 1) self.assertEqual(els[0].name, 'p') self.assertEqual(els[0]['class'], ['onep']) def test_class_mismatched_tag(self): els = self.soup.select('div.onep') self.assertEqual(len(els), 0) def test_one_id(self): for selector in ('div#inner', '#inner', 'div div#inner'): self.assertSelects(selector, ['inner']) def test_bad_id(self): els = self.soup.select('#doesnotexist') self.assertEqual(len(els), 0) def test_items_in_id(self): els = self.soup.select('div#inner p') self.assertEqual(len(els), 3) for el in els: self.assertEqual(el.name, 'p') self.assertEqual(els[1]['class'], ['onep']) self.assertFalse(els[0].has_attr('class')) def test_a_bunch_of_emptys(self): for selector in ('div#main del', 'div#main div.oops', 'div div#main'): self.assertEqual(len(self.soup.select(selector)), 0) def test_multi_class_support(self): for selector in ('.class1', 'p.class1', '.class2', 'p.class2', '.class3', 'p.class3', 'html p.class2', 'div#inner .class2'): self.assertSelects(selector, ['pmulti']) def test_multi_class_selection(self): for selector in ('.class1.class3', '.class3.class2', '.class1.class2.class3'): self.assertSelects(selector, ['pmulti']) def test_child_selector(self): self.assertSelects('.s1 > a', ['s1a1', 's1a2']) self.assertSelects('.s1 > a span', ['s1a2s1']) def test_child_selector_id(self): self.assertSelects('.s1 > a#s1a2 span', ['s1a2s1']) def test_attribute_equals(self): self.assertSelectMultiple( ('p[class="onep"]', ['p1']), ('p[id="p1"]', ['p1']), ('[class="onep"]', ['p1']), ('[id="p1"]', ['p1']), ('link[rel="stylesheet"]', ['l1']), ('link[type="text/css"]', ['l1']), ('link[href="blah.css"]', ['l1']), ('link[href="no-blah.css"]', []), ('[rel="stylesheet"]', ['l1']), ('[type="text/css"]', ['l1']), ('[href="blah.css"]', ['l1']), ('[href="no-blah.css"]', []), ('p[href="no-blah.css"]', []), ('[href="no-blah.css"]', []), ) def test_attribute_tilde(self): self.assertSelectMultiple( ('p[class~="class1"]', ['pmulti']), ('p[class~="class2"]', ['pmulti']), ('p[class~="class3"]', ['pmulti']), ('[class~="class1"]', ['pmulti']), ('[class~="class2"]', ['pmulti']), ('[class~="class3"]', ['pmulti']), ('a[rel~="friend"]', ['bob']), ('a[rel~="met"]', ['bob']), ('[rel~="friend"]', ['bob']), ('[rel~="met"]', ['bob']), ) def test_attribute_startswith(self): self.assertSelectMultiple( ('[rel^="style"]', ['l1']), ('link[rel^="style"]', ['l1']), ('notlink[rel^="notstyle"]', []), ('[rel^="notstyle"]', []), ('link[rel^="notstyle"]', []), ('link[href^="bla"]', ['l1']), ('a[href^="http://"]', ['bob', 'me']), ('[href^="http://"]', ['bob', 'me']), ('[id^="p"]', ['pmulti', 'p1']), ('[id^="m"]', ['me', 'main']), ('div[id^="m"]', ['main']), ('a[id^="m"]', ['me']), ('div[data-tag^="dashed"]', ['data1']) ) def test_attribute_endswith(self): self.assertSelectMultiple( ('[href$=".css"]', ['l1']), ('link[href$=".css"]', ['l1']), ('link[id$="1"]', ['l1']), ('[id$="1"]', ['data1', 'l1', 'p1', 'header1', 's1a1', 's2a1', 's1a2s1', 'dash1']), ('div[id$="1"]', ['data1']), ('[id$="noending"]', []), ) def test_attribute_contains(self): self.assertSelectMultiple( # From test_attribute_startswith ('[rel*="style"]', ['l1']), ('link[rel*="style"]', ['l1']), ('notlink[rel*="notstyle"]', []), ('[rel*="notstyle"]', []), ('link[rel*="notstyle"]', []), ('link[href*="bla"]', ['l1']), ('[href*="http://"]', ['bob', 'me']), ('[id*="p"]', ['pmulti', 'p1']), ('div[id*="m"]', ['main']), ('a[id*="m"]', ['me']), # From test_attribute_endswith ('[href*=".css"]', ['l1']), ('link[href*=".css"]', ['l1']), ('link[id*="1"]', ['l1']), ('[id*="1"]', ['data1', 'l1', 'p1', 'header1', 's1a1', 's1a2', 's2a1', 's1a2s1', 'dash1']), ('div[id*="1"]', ['data1']), ('[id*="noending"]', []), # New for this test ('[href*="."]', ['bob', 'me', 'l1']), ('a[href*="."]', ['bob', 'me']), ('link[href*="."]', ['l1']), ('div[id*="n"]', ['main', 'inner']), ('div[id*="nn"]', ['inner']), ('div[data-tag*="edval"]', ['data1']) ) def test_attribute_exact_or_hypen(self): self.assertSelectMultiple( ('p[lang|="en"]', ['lang-en', 'lang-en-gb', 'lang-en-us']), ('[lang|="en"]', ['lang-en', 'lang-en-gb', 'lang-en-us']), ('p[lang|="fr"]', ['lang-fr']), ('p[lang|="gb"]', []), ) def test_attribute_exists(self): self.assertSelectMultiple( ('[rel]', ['l1', 'bob', 'me']), ('link[rel]', ['l1']), ('a[rel]', ['bob', 'me']), ('[lang]', ['lang-en', 'lang-en-gb', 'lang-en-us', 'lang-fr']), ('p[class]', ['p1', 'pmulti']), ('[blah]', []), ('p[blah]', []), ('div[data-tag]', ['data1']) ) def test_quoted_space_in_selector_name(self): html = """<div style="display: wrong">nope</div> <div style="display: right">yes</div> """ soup = BeautifulSoup(html, 'html.parser') [chosen] = soup.select('div[style="display: right"]') self.assertEqual("yes", chosen.string) def test_unsupported_pseudoclass(self): self.assertRaises( NotImplementedError, self.soup.select, "a:no-such-pseudoclass") self.assertRaises( NotImplementedError, self.soup.select, "a:nth-of-type(a)") def test_nth_of_type(self): # Try to select first paragraph els = self.soup.select('div#inner p:nth-of-type(1)') self.assertEqual(len(els), 1) self.assertEqual(els[0].string, 'Some text') # Try to select third paragraph els = self.soup.select('div#inner p:nth-of-type(3)') self.assertEqual(len(els), 1) self.assertEqual(els[0].string, 'Another') # Try to select (non-existent!) fourth paragraph els = self.soup.select('div#inner p:nth-of-type(4)') self.assertEqual(len(els), 0) # Pass in an invalid value. self.assertRaises( ValueError, self.soup.select, 'div p:nth-of-type(0)') def test_nth_of_type_direct_descendant(self): els = self.soup.select('div#inner > p:nth-of-type(1)') self.assertEqual(len(els), 1) self.assertEqual(els[0].string, 'Some text') def test_id_child_selector_nth_of_type(self): self.assertSelects('#inner > p:nth-of-type(2)', ['p1']) def test_select_on_element(self): # Other tests operate on the tree; this operates on an element # within the tree. inner = self.soup.find("div", id="main") selected = inner.select("div") # The <div id="inner"> tag was selected. The <div id="footer"> # tag was not. self.assertSelectsIDs(selected, ['inner', 'data1']) def test_overspecified_child_id(self): self.assertSelects(".fancy #inner", ['inner']) self.assertSelects(".normal #inner", []) def test_adjacent_sibling_selector(self): self.assertSelects('#p1 + h2', ['header2']) self.assertSelects('#p1 + h2 + p', ['pmulti']) self.assertSelects('#p1 + #header2 + .class1', ['pmulti']) self.assertEqual([], self.soup.select('#p1 + p')) def test_general_sibling_selector(self): self.assertSelects('#p1 ~ h2', ['header2', 'header3']) self.assertSelects('#p1 ~ #header2', ['header2']) self.assertSelects('#p1 ~ h2 + a', ['me']) self.assertSelects('#p1 ~ h2 + [rel="me"]', ['me']) self.assertEqual([], self.soup.select('#inner ~ h2')) def test_dangling_combinator(self): self.assertRaises(ValueError, self.soup.select, 'h1 >') def test_sibling_combinator_wont_select_same_tag_twice(self): self.assertSelects('p[lang] ~ p', ['lang-en-gb', 'lang-en-us', 'lang-fr']) # Test the selector grouping operator (the comma) def test_multiple_select(self): self.assertSelects('x, y', ['xid', 'yid']) def test_multiple_select_with_no_space(self): self.assertSelects('x,y', ['xid', 'yid']) def test_multiple_select_with_more_space(self): self.assertSelects('x, y', ['xid', 'yid']) def test_multiple_select_duplicated(self): self.assertSelects('x, x', ['xid']) def test_multiple_select_sibling(self): self.assertSelects('x, y ~ p[lang=fr]', ['xid', 'lang-fr']) def test_multiple_select_tag_and_direct_descendant(self): self.assertSelects('x, y > z', ['xid', 'zidb']) def test_multiple_select_direct_descendant_and_tags(self): self.assertSelects('div > x, y, z', ['xid', 'yid', 'zida', 'zidb', 'zidab', 'zidac']) def test_multiple_select_indirect_descendant(self): self.assertSelects('div x,y, z', ['xid', 'yid', 'zida', 'zidb', 'zidab', 'zidac']) def test_invalid_multiple_select(self): self.assertRaises(ValueError, self.soup.select, ',x, y') self.assertRaises(ValueError, self.soup.select, 'x,,y') def test_multiple_select_attrs(self): self.assertSelects('p[lang=en], p[lang=en-gb]', ['lang-en', 'lang-en-gb']) def test_multiple_select_ids(self): self.assertSelects('x, y > z[id=zida], z[id=zidab], z[id=zidb]', ['xid', 'zidb', 'zidab']) def test_multiple_select_nested(self): self.assertSelects('body > div > x, y > z', ['xid', 'zidb']) def test_select_duplicate_elements(self): # When markup contains duplicate elements, a multiple select # will find all of them. markup = '<div class="c1"/><div class="c2"/><div class="c1"/>' soup = BeautifulSoup(markup, 'html.parser') selected = soup.select(".c1, .c2") self.assertEqual(3, len(selected)) # Verify that find_all finds the same elements, though because # of an implementation detail it finds them in a different # order. for element in soup.find_all(class_=['c1', 'c2']): assert element in selected tests/test_html5lib.py 0000644 00000011453 15246412105 0011043 0 ustar 00 """Tests to ensure that the html5lib tree builder generates good trees.""" import warnings try: from bs4.builder import HTML5TreeBuilder HTML5LIB_PRESENT = True except ImportError as e: HTML5LIB_PRESENT = False from bs4.element import SoupStrainer from bs4.testing import ( HTML5TreeBuilderSmokeTest, SoupTest, skipIf, ) @skipIf( not HTML5LIB_PRESENT, "html5lib seems not to be present, not testing its tree builder.") class HTML5LibBuilderSmokeTest(SoupTest, HTML5TreeBuilderSmokeTest): """See ``HTML5TreeBuilderSmokeTest``.""" @property def default_builder(self): return HTML5TreeBuilder() def test_soupstrainer(self): # The html5lib tree builder does not support SoupStrainers. strainer = SoupStrainer("b") markup = "<p>A <b>bold</b> statement.</p>" with warnings.catch_warnings(record=True) as w: soup = self.soup(markup, parse_only=strainer) self.assertEqual( soup.decode(), self.document_for(markup)) self.assertTrue( "the html5lib tree builder doesn't support parse_only" in str(w[0].message)) def test_correctly_nested_tables(self): """html5lib inserts <tbody> tags where other parsers don't.""" markup = ('<table id="1">' '<tr>' "<td>Here's another table:" '<table id="2">' '<tr><td>foo</td></tr>' '</table></td>') self.assertSoupEquals( markup, '<table id="1"><tbody><tr><td>Here\'s another table:' '<table id="2"><tbody><tr><td>foo</td></tr></tbody></table>' '</td></tr></tbody></table>') self.assertSoupEquals( "<table><thead><tr><td>Foo</td></tr></thead>" "<tbody><tr><td>Bar</td></tr></tbody>" "<tfoot><tr><td>Baz</td></tr></tfoot></table>") def test_xml_declaration_followed_by_doctype(self): markup = '''<?xml version="1.0" encoding="utf-8"?> <!DOCTYPE html> <html> <head> </head> <body> <p>foo</p> </body> </html>''' soup = self.soup(markup) # Verify that we can reach the <p> tag; this means the tree is connected. self.assertEqual(b"<p>foo</p>", soup.p.encode()) def test_reparented_markup(self): markup = '<p><em>foo</p>\n<p>bar<a></a></em></p>' soup = self.soup(markup) self.assertEqual("<body><p><em>foo</em></p><em>\n</em><p><em>bar<a></a></em></p></body>", soup.body.decode()) self.assertEqual(2, len(soup.find_all('p'))) def test_reparented_markup_ends_with_whitespace(self): markup = '<p><em>foo</p>\n<p>bar<a></a></em></p>\n' soup = self.soup(markup) self.assertEqual("<body><p><em>foo</em></p><em>\n</em><p><em>bar<a></a></em></p>\n</body>", soup.body.decode()) self.assertEqual(2, len(soup.find_all('p'))) def test_reparented_markup_containing_identical_whitespace_nodes(self): """Verify that we keep the two whitespace nodes in this document distinct when reparenting the adjacent <tbody> tags. """ markup = '<table> <tbody><tbody><ims></tbody> </table>' soup = self.soup(markup) space1, space2 = soup.find_all(string=' ') tbody1, tbody2 = soup.find_all('tbody') assert space1.next_element is tbody1 assert tbody2.next_element is space2 def test_reparented_markup_containing_children(self): markup = '<div><a>aftermath<p><noscript>target</noscript>aftermath</a></p></div>' soup = self.soup(markup) noscript = soup.noscript self.assertEqual("target", noscript.next_element) target = soup.find(string='target') # The 'aftermath' string was duplicated; we want the second one. final_aftermath = soup.find_all(string='aftermath')[-1] # The <noscript> tag was moved beneath a copy of the <a> tag, # but the 'target' string within is still connected to the # (second) 'aftermath' string. self.assertEqual(final_aftermath, target.next_element) self.assertEqual(target, final_aftermath.previous_element) def test_processing_instruction(self): """Processing instructions become comments.""" markup = b"""<?PITarget PIContent?>""" soup = self.soup(markup) assert str(soup).startswith("<!--?PITarget PIContent?-->") def test_cloned_multivalue_node(self): markup = b"""<a class="my_class"><p></a>""" soup = self.soup(markup) a1, a2 = soup.find_all('a') self.assertEqual(a1, a2) assert a1 is not a2 def test_foster_parenting(self): markup = b"""<table><td></tbody>A""" soup = self.soup(markup) self.assertEqual("<body>A<table><tbody><tr><td></td></tr></tbody></table></body>", soup.body.decode()) tests/test_htmlparser.py 0000644 00000003300 15246412105 0011474 0 ustar 00 """Tests to ensure that the html.parser tree builder generates good trees.""" from pdb import set_trace import pickle from bs4.testing import SoupTest, HTMLTreeBuilderSmokeTest from bs4.builder import HTMLParserTreeBuilder from bs4.builder._htmlparser import BeautifulSoupHTMLParser class HTMLParserTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest): @property def default_builder(self): return HTMLParserTreeBuilder() def test_namespaced_system_doctype(self): # html.parser can't handle namespaced doctypes, so skip this one. pass def test_namespaced_public_doctype(self): # html.parser can't handle namespaced doctypes, so skip this one. pass def test_builder_is_pickled(self): """Unlike most tree builders, HTMLParserTreeBuilder and will be restored after pickling. """ tree = self.soup("<a><b>foo</a>") dumped = pickle.dumps(tree, 2) loaded = pickle.loads(dumped) self.assertTrue(isinstance(loaded.builder, type(tree.builder))) def test_redundant_empty_element_closing_tags(self): self.assertSoupEquals('<br></br><br></br><br></br>', "<br/><br/><br/>") self.assertSoupEquals('</br></br></br>', "") def test_empty_element(self): # This verifies that any buffered data present when the parser # finishes working is handled. self.assertSoupEquals("foo &# bar", "foo &# bar") class TestHTMLParserSubclass(SoupTest): def test_error(self): """Verify that our HTMLParser subclass implements error() in a way that doesn't cause a crash. """ parser = BeautifulSoupHTMLParser() parser.error("don't crash") tests/__init__.py 0000644 00000000033 15246412105 0010013 0 ustar 00 "The beautifulsoup tests." tests/test_soup.py 0000644 00000047544 15246412105 0010323 0 ustar 00 # -*- coding: utf-8 -*- """Tests of Beautiful Soup as a whole.""" from pdb import set_trace import logging import unittest import sys import tempfile from bs4 import ( BeautifulSoup, BeautifulStoneSoup, ) from bs4.element import ( CharsetMetaAttributeValue, ContentMetaAttributeValue, SoupStrainer, NamespacedAttribute, ) import bs4.dammit from bs4.dammit import ( EntitySubstitution, UnicodeDammit, EncodingDetector, ) from bs4.testing import ( SoupTest, skipIf, ) import warnings try: from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML LXML_PRESENT = True except ImportError as e: LXML_PRESENT = False PYTHON_3_PRE_3_2 = (sys.version_info[0] == 3 and sys.version_info < (3,2)) class TestConstructor(SoupTest): def test_short_unicode_input(self): data = "<h1>éé</h1>" soup = self.soup(data) self.assertEqual("éé", soup.h1.string) def test_embedded_null(self): data = "<h1>foo\0bar</h1>" soup = self.soup(data) self.assertEqual("foo\0bar", soup.h1.string) def test_exclude_encodings(self): utf8_data = "Räksmörgås".encode("utf-8") soup = self.soup(utf8_data, exclude_encodings=["utf-8"]) self.assertEqual("windows-1252", soup.original_encoding) class TestWarnings(SoupTest): def _no_parser_specified(self, s, is_there=True): v = s.startswith(BeautifulSoup.NO_PARSER_SPECIFIED_WARNING[:80]) self.assertTrue(v) def test_warning_if_no_parser_specified(self): with warnings.catch_warnings(record=True) as w: soup = self.soup("<a><b></b></a>") msg = str(w[0].message) self._assert_no_parser_specified(msg) def test_warning_if_parser_specified_too_vague(self): with warnings.catch_warnings(record=True) as w: soup = self.soup("<a><b></b></a>", "html") msg = str(w[0].message) self._assert_no_parser_specified(msg) def test_no_warning_if_explicit_parser_specified(self): with warnings.catch_warnings(record=True) as w: soup = self.soup("<a><b></b></a>", "html.parser") self.assertEqual([], w) def test_parseOnlyThese_renamed_to_parse_only(self): with warnings.catch_warnings(record=True) as w: soup = self.soup("<a><b></b></a>", parseOnlyThese=SoupStrainer("b")) msg = str(w[0].message) self.assertTrue("parseOnlyThese" in msg) self.assertTrue("parse_only" in msg) self.assertEqual(b"<b></b>", soup.encode()) def test_fromEncoding_renamed_to_from_encoding(self): with warnings.catch_warnings(record=True) as w: utf8 = b"\xc3\xa9" soup = self.soup(utf8, fromEncoding="utf8") msg = str(w[0].message) self.assertTrue("fromEncoding" in msg) self.assertTrue("from_encoding" in msg) self.assertEqual("utf8", soup.original_encoding) def test_unrecognized_keyword_argument(self): self.assertRaises( TypeError, self.soup, "<a>", no_such_argument=True) class TestWarnings(SoupTest): def test_disk_file_warning(self): filehandle = tempfile.NamedTemporaryFile() filename = filehandle.name try: with warnings.catch_warnings(record=True) as w: soup = self.soup(filename) msg = str(w[0].message) self.assertTrue("looks like a filename" in msg) finally: filehandle.close() # The file no longer exists, so Beautiful Soup will no longer issue the warning. with warnings.catch_warnings(record=True) as w: soup = self.soup(filename) self.assertEqual(0, len(w)) def test_url_warning_with_bytes_url(self): with warnings.catch_warnings(record=True) as warning_list: soup = self.soup(b"http://www.crummybytes.com/") # Be aware this isn't the only warning that can be raised during # execution.. self.assertTrue(any("looks like a URL" in str(w.message) for w in warning_list)) def test_url_warning_with_unicode_url(self): with warnings.catch_warnings(record=True) as warning_list: # note - this url must differ from the bytes one otherwise # python's warnings system swallows the second warning soup = self.soup("http://www.crummyunicode.com/") self.assertTrue(any("looks like a URL" in str(w.message) for w in warning_list)) def test_url_warning_with_bytes_and_space(self): with warnings.catch_warnings(record=True) as warning_list: soup = self.soup(b"http://www.crummybytes.com/ is great") self.assertFalse(any("looks like a URL" in str(w.message) for w in warning_list)) def test_url_warning_with_unicode_and_space(self): with warnings.catch_warnings(record=True) as warning_list: soup = self.soup("http://www.crummyuncode.com/ is great") self.assertFalse(any("looks like a URL" in str(w.message) for w in warning_list)) class TestSelectiveParsing(SoupTest): def test_parse_with_soupstrainer(self): markup = "No<b>Yes</b><a>No<b>Yes <c>Yes</c></b>" strainer = SoupStrainer("b") soup = self.soup(markup, parse_only=strainer) self.assertEqual(soup.encode(), b"<b>Yes</b><b>Yes <c>Yes</c></b>") class TestEntitySubstitution(unittest.TestCase): """Standalone tests of the EntitySubstitution class.""" def setUp(self): self.sub = EntitySubstitution def test_simple_html_substitution(self): # Unicode characters corresponding to named HTML entites # are substituted, and no others. s = "foo\\u2200\N{SNOWMAN}\\u00f5bar" self.assertEqual(self.sub.substitute_html(s), "foo∀\N{SNOWMAN}õbar") def test_smart_quote_substitution(self): # MS smart quotes are a common source of frustration, so we # give them a special test. quotes = b"\x91\x92foo\x93\x94" dammit = UnicodeDammit(quotes) self.assertEqual(self.sub.substitute_html(dammit.markup), "‘’foo“”") def test_xml_converstion_includes_no_quotes_if_make_quoted_attribute_is_false(self): s = 'Welcome to "my bar"' self.assertEqual(self.sub.substitute_xml(s, False), s) def test_xml_attribute_quoting_normally_uses_double_quotes(self): self.assertEqual(self.sub.substitute_xml("Welcome", True), '"Welcome"') self.assertEqual(self.sub.substitute_xml("Bob's Bar", True), '"Bob\'s Bar"') def test_xml_attribute_quoting_uses_single_quotes_when_value_contains_double_quotes(self): s = 'Welcome to "my bar"' self.assertEqual(self.sub.substitute_xml(s, True), "'Welcome to \"my bar\"'") def test_xml_attribute_quoting_escapes_single_quotes_when_value_contains_both_single_and_double_quotes(self): s = 'Welcome to "Bob\'s Bar"' self.assertEqual( self.sub.substitute_xml(s, True), '"Welcome to "Bob\'s Bar""') def test_xml_quotes_arent_escaped_when_value_is_not_being_quoted(self): quoted = 'Welcome to "Bob\'s Bar"' self.assertEqual(self.sub.substitute_xml(quoted), quoted) def test_xml_quoting_handles_angle_brackets(self): self.assertEqual( self.sub.substitute_xml("foo<bar>"), "foo<bar>") def test_xml_quoting_handles_ampersands(self): self.assertEqual(self.sub.substitute_xml("AT&T"), "AT&T") def test_xml_quoting_including_ampersands_when_they_are_part_of_an_entity(self): self.assertEqual( self.sub.substitute_xml("ÁT&T"), "&Aacute;T&T") def test_xml_quoting_ignoring_ampersands_when_they_are_part_of_an_entity(self): self.assertEqual( self.sub.substitute_xml_containing_entities("ÁT&T"), "ÁT&T") def test_quotes_not_html_substituted(self): """There's no need to do this except inside attribute values.""" text = 'Bob\'s "bar"' self.assertEqual(self.sub.substitute_html(text), text) class TestEncodingConversion(SoupTest): # Test Beautiful Soup's ability to decode and encode from various # encodings. def setUp(self): super(TestEncodingConversion, self).setUp() self.unicode_data = '<html><head><meta charset="utf-8"/></head><body><foo>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</foo></body></html>' self.utf8_data = self.unicode_data.encode("utf-8") # Just so you know what it looks like. self.assertEqual( self.utf8_data, b'<html><head><meta charset="utf-8"/></head><body><foo>Sacr\xc3\xa9 bleu!</foo></body></html>') def test_ascii_in_unicode_out(self): # ASCII input is converted to Unicode. The original_encoding # attribute is set to 'utf-8', a superset of ASCII. chardet = bs4.dammit.chardet_dammit logging.disable(logging.WARNING) try: def noop(str): return None # Disable chardet, which will realize that the ASCII is ASCII. bs4.dammit.chardet_dammit = noop ascii = b"<foo>a</foo>" soup_from_ascii = self.soup(ascii) unicode_output = soup_from_ascii.decode() self.assertTrue(isinstance(unicode_output, str)) self.assertEqual(unicode_output, self.document_for(ascii.decode())) self.assertEqual(soup_from_ascii.original_encoding.lower(), "utf-8") finally: logging.disable(logging.NOTSET) bs4.dammit.chardet_dammit = chardet def test_unicode_in_unicode_out(self): # Unicode input is left alone. The original_encoding attribute # is not set. soup_from_unicode = self.soup(self.unicode_data) self.assertEqual(soup_from_unicode.decode(), self.unicode_data) self.assertEqual(soup_from_unicode.foo.string, 'Sacr\xe9 bleu!') self.assertEqual(soup_from_unicode.original_encoding, None) def test_utf8_in_unicode_out(self): # UTF-8 input is converted to Unicode. The original_encoding # attribute is set. soup_from_utf8 = self.soup(self.utf8_data) self.assertEqual(soup_from_utf8.decode(), self.unicode_data) self.assertEqual(soup_from_utf8.foo.string, 'Sacr\xe9 bleu!') def test_utf8_out(self): # The internal data structures can be encoded as UTF-8. soup_from_unicode = self.soup(self.unicode_data) self.assertEqual(soup_from_unicode.encode('utf-8'), self.utf8_data) @skipIf( PYTHON_3_PRE_3_2, "Bad HTMLParser detected; skipping test of non-ASCII characters in attribute name.") def test_attribute_name_containing_unicode_characters(self): markup = '<div><a \N{SNOWMAN}="snowman"></a></div>' self.assertEqual(self.soup(markup).div.encode("utf8"), markup.encode("utf8")) class TestUnicodeDammit(unittest.TestCase): """Standalone tests of UnicodeDammit.""" def test_unicode_input(self): markup = "I'm already Unicode! \N{SNOWMAN}" dammit = UnicodeDammit(markup) self.assertEqual(dammit.unicode_markup, markup) def test_smart_quotes_to_unicode(self): markup = b"<foo>\x91\x92\x93\x94</foo>" dammit = UnicodeDammit(markup) self.assertEqual( dammit.unicode_markup, "<foo>\\u2018\\u2019\\u201c\\u201d</foo>") def test_smart_quotes_to_xml_entities(self): markup = b"<foo>\x91\x92\x93\x94</foo>" dammit = UnicodeDammit(markup, smart_quotes_to="xml") self.assertEqual( dammit.unicode_markup, "<foo>‘’“”</foo>") def test_smart_quotes_to_html_entities(self): markup = b"<foo>\x91\x92\x93\x94</foo>" dammit = UnicodeDammit(markup, smart_quotes_to="html") self.assertEqual( dammit.unicode_markup, "<foo>‘’“”</foo>") def test_smart_quotes_to_ascii(self): markup = b"<foo>\x91\x92\x93\x94</foo>" dammit = UnicodeDammit(markup, smart_quotes_to="ascii") self.assertEqual( dammit.unicode_markup, """<foo>''""</foo>""") def test_detect_utf8(self): utf8 = b"Sacr\xc3\xa9 bleu! \xe2\x98\x83" dammit = UnicodeDammit(utf8) self.assertEqual(dammit.original_encoding.lower(), 'utf-8') self.assertEqual(dammit.unicode_markup, 'Sacr\xe9 bleu! \N{SNOWMAN}') def test_convert_hebrew(self): hebrew = b"\xed\xe5\xec\xf9" dammit = UnicodeDammit(hebrew, ["iso-8859-8"]) self.assertEqual(dammit.original_encoding.lower(), 'iso-8859-8') self.assertEqual(dammit.unicode_markup, '\\u05dd\\u05d5\\u05dc\\u05e9') def test_dont_see_smart_quotes_where_there_are_none(self): utf_8 = b"\343\202\261\343\203\274\343\202\277\343\202\244 Watch" dammit = UnicodeDammit(utf_8) self.assertEqual(dammit.original_encoding.lower(), 'utf-8') self.assertEqual(dammit.unicode_markup.encode("utf-8"), utf_8) def test_ignore_inappropriate_codecs(self): utf8_data = "Räksmörgås".encode("utf-8") dammit = UnicodeDammit(utf8_data, ["iso-8859-8"]) self.assertEqual(dammit.original_encoding.lower(), 'utf-8') def test_ignore_invalid_codecs(self): utf8_data = "Räksmörgås".encode("utf-8") for bad_encoding in ['.utf8', '...', 'utF---16.!']: dammit = UnicodeDammit(utf8_data, [bad_encoding]) self.assertEqual(dammit.original_encoding.lower(), 'utf-8') def test_exclude_encodings(self): # This is UTF-8. utf8_data = "Räksmörgås".encode("utf-8") # But if we exclude UTF-8 from consideration, the guess is # Windows-1252. dammit = UnicodeDammit(utf8_data, exclude_encodings=["utf-8"]) self.assertEqual(dammit.original_encoding.lower(), 'windows-1252') # And if we exclude that, there is no valid guess at all. dammit = UnicodeDammit( utf8_data, exclude_encodings=["utf-8", "windows-1252"]) self.assertEqual(dammit.original_encoding, None) def test_encoding_detector_replaces_junk_in_encoding_name_with_replacement_character(self): detected = EncodingDetector( b'<?xml version="1.0" encoding="UTF-\xdb" ?>') encodings = list(detected.encodings) assert 'utf-\N{REPLACEMENT CHARACTER}' in encodings def test_detect_html5_style_meta_tag(self): for data in ( b'<html><meta charset="euc-jp" /></html>', b"<html><meta charset='euc-jp' /></html>", b"<html><meta charset=euc-jp /></html>", b"<html><meta charset=euc-jp/></html>"): dammit = UnicodeDammit(data, is_html=True) self.assertEqual( "euc-jp", dammit.original_encoding) def test_last_ditch_entity_replacement(self): # This is a UTF-8 document that contains bytestrings # completely incompatible with UTF-8 (ie. encoded with some other # encoding). # # Since there is no consistent encoding for the document, # Unicode, Dammit will eventually encode the document as UTF-8 # and encode the incompatible characters as REPLACEMENT # CHARACTER. # # If chardet is installed, it will detect that the document # can be converted into ISO-8859-1 without errors. This happens # to be the wrong encoding, but it is a consistent encoding, so the # code we're testing here won't run. # # So we temporarily disable chardet if it's present. doc = b"""\357\273\277<?xml version="1.0" encoding="UTF-8"?> <html><b>\330\250\330\252\330\261</b> <i>\310\322\321\220\312\321\355\344</i></html>""" chardet = bs4.dammit.chardet_dammit logging.disable(logging.WARNING) try: def noop(str): return None bs4.dammit.chardet_dammit = noop dammit = UnicodeDammit(doc) self.assertEqual(True, dammit.contains_replacement_characters) self.assertTrue("\\ufffd" in dammit.unicode_markup) soup = BeautifulSoup(doc, "html.parser") self.assertTrue(soup.contains_replacement_characters) finally: logging.disable(logging.NOTSET) bs4.dammit.chardet_dammit = chardet def test_byte_order_mark_removed(self): # A document written in UTF-16LE will have its byte order marker stripped. data = b'\xff\xfe<\x00a\x00>\x00\xe1\x00\xe9\x00<\x00/\x00a\x00>\x00' dammit = UnicodeDammit(data) self.assertEqual("<a>áé</a>", dammit.unicode_markup) self.assertEqual("utf-16le", dammit.original_encoding) def test_detwingle(self): # Here's a UTF8 document. utf8 = ("\N{SNOWMAN}" * 3).encode("utf8") # Here's a Windows-1252 document. windows_1252 = ( "\N{LEFT DOUBLE QUOTATION MARK}Hi, I like Windows!" "\N{RIGHT DOUBLE QUOTATION MARK}").encode("windows_1252") # Through some unholy alchemy, they've been stuck together. doc = utf8 + windows_1252 + utf8 # The document can't be turned into UTF-8: self.assertRaises(UnicodeDecodeError, doc.decode, "utf8") # Unicode, Dammit thinks the whole document is Windows-1252, # and decodes it into "☃☃☃“Hi, I like Windows!”☃☃☃" # But if we run it through fix_embedded_windows_1252, it's fixed: fixed = UnicodeDammit.detwingle(doc) self.assertEqual( "☃☃☃“Hi, I like Windows!”☃☃☃", fixed.decode("utf8")) def test_detwingle_ignores_multibyte_characters(self): # Each of these characters has a UTF-8 representation ending # in \x93. \x93 is a smart quote if interpreted as # Windows-1252. But our code knows to skip over multibyte # UTF-8 characters, so they'll survive the process unscathed. for tricky_unicode_char in ( "\N{LATIN SMALL LIGATURE OE}", # 2-byte char '\xc5\x93' "\N{LATIN SUBSCRIPT SMALL LETTER X}", # 3-byte char '\xe2\x82\x93' "\xf0\x90\x90\x93", # This is a CJK character, not sure which one. ): input = tricky_unicode_char.encode("utf8") self.assertTrue(input.endswith(b'\x93')) output = UnicodeDammit.detwingle(input) self.assertEqual(output, input) class TestNamedspacedAttribute(SoupTest): def test_name_may_be_none(self): a = NamespacedAttribute("xmlns", None) self.assertEqual(a, "xmlns") def test_attribute_is_equivalent_to_colon_separated_string(self): a = NamespacedAttribute("a", "b") self.assertEqual("a:b", a) def test_attributes_are_equivalent_if_prefix_and_name_identical(self): a = NamespacedAttribute("a", "b", "c") b = NamespacedAttribute("a", "b", "c") self.assertEqual(a, b) # The actual namespace is not considered. c = NamespacedAttribute("a", "b", None) self.assertEqual(a, c) # But name and prefix are important. d = NamespacedAttribute("a", "z", "c") self.assertNotEqual(a, d) e = NamespacedAttribute("z", "b", "c") self.assertNotEqual(a, e) class TestAttributeValueWithCharsetSubstitution(unittest.TestCase): def test_content_meta_attribute_value(self): value = CharsetMetaAttributeValue("euc-jp") self.assertEqual("euc-jp", value) self.assertEqual("euc-jp", value.original_value) self.assertEqual("utf8", value.encode("utf8")) def test_content_meta_attribute_value(self): value = ContentMetaAttributeValue("text/html; charset=euc-jp") self.assertEqual("text/html; charset=euc-jp", value) self.assertEqual("text/html; charset=euc-jp", value.original_value) self.assertEqual("text/html; charset=utf8", value.encode("utf8")) tests/test_docs.py 0000644 00000002053 15246412105 0010247 0 ustar 00 "Test harness for doctests." # pylint: disable-msg=E0611,W0142 __metaclass__ = type __all__ = [ 'additional_tests', ] import atexit import doctest import os #from pkg_resources import ( # resource_filename, resource_exists, resource_listdir, cleanup_resources) import unittest DOCTEST_FLAGS = ( doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF) # def additional_tests(): # "Run the doc tests (README.txt and docs/*, if any exist)" # doctest_files = [ # os.path.abspath(resource_filename('bs4', 'README.txt'))] # if resource_exists('bs4', 'docs'): # for name in resource_listdir('bs4', 'docs'): # if name.endswith('.txt'): # doctest_files.append( # os.path.abspath( # resource_filename('bs4', 'docs/%s' % name))) # kwargs = dict(module_relative=False, optionflags=DOCTEST_FLAGS) # atexit.register(cleanup_resources) # return unittest.TestSuite(( # doctest.DocFileSuite(*doctest_files, **kwargs))) tests/__pycache__/test_soup.cpython-36.opt-1.pyc 0000644 00000052470 15246412105 0015540 0 ustar 00 3 *6]dO � @ s~ d Z ddlmZ ddlZddlZddlZddlZddlmZm Z ddl m Z m Z m Z mZ ddlZddlmZmZmZ ddlmZmZ ddlZyddlmZmZ d ZW n& ek r� Z z d ZW Y ddZ [ X nX ej d d ko�ej d k Z G d d� de�Z G dd� de�Z G dd� de�Z G dd� de�Z!G dd� dej"�Z#G dd� de�Z$G dd� dej"�Z%G dd� de�Z&G d d � d ej"�Z'dS ) z#Tests of Beautiful Soup as a whole.� )� set_traceN)� BeautifulSoup�BeautifulStoneSoup)�CharsetMetaAttributeValue�ContentMetaAttributeValue� SoupStrainer�NamespacedAttribute)�EntitySubstitution� UnicodeDammit�EncodingDetector)�SoupTest�skipIf)�LXMLTreeBuilder�LXMLTreeBuilderForXMLTF� � c @ s$ e Zd Zdd� Zdd� Zdd� ZdS )�TestConstructorc C s" d}| j |�}| jd|jj� d S )Nu <h1>éé</h1>u éé)�soup� assertEqual�h1�string)�self�datar � r � /usr/lib/python3.6/test_soup.py�test_short_unicode_input* s z(TestConstructor.test_short_unicode_inputc C s" d}| j |�}| jd|jj� d S )Nz<h1>foo bar</h1>zfoo bar)r r r r )r r r r r r �test_embedded_null/ s z"TestConstructor.test_embedded_nullc C s, dj d�}| j|dgd�}| jd|j� d S )Nu Räksmörgåszutf-8)�exclude_encodingsz windows-1252)�encoder r �original_encoding)r � utf8_datar r r r �test_exclude_encodings4 s z&TestConstructor.test_exclude_encodingsN)�__name__� __module__� __qualname__r r r! r r r r r ( s r c @ sF e Zd Zddd�Zdd� Zdd� Zdd � Zd d � Zd d � Zdd� Z dS )� TestWarningsTc C s" |j tjd d� �}| j|� d S )N�P )� startswithr ZNO_PARSER_SPECIFIED_WARNING� assertTrue)r �sZis_there�vr r r �_no_parser_specified< s z!TestWarnings._no_parser_specifiedc C s> t jdd��}| jd�}W d Q R X t|d j�}| j|� d S )NT)�recordz<a><b></b></a>r )�warnings�catch_warningsr �str�message�_assert_no_parser_specified)r �wr �msgr r r �#test_warning_if_no_parser_specified@ s z0TestWarnings.test_warning_if_no_parser_specifiedc C s@ t jdd��}| jdd�}W d Q R X t|d j�}| j|� d S )NT)r, z<a><b></b></a>�htmlr )r- r. r r/ r0 r1 )r r2 r r3 r r r �*test_warning_if_parser_specified_too_vagueF s z7TestWarnings.test_warning_if_parser_specified_too_vaguec C s4 t jdd��}| jdd�}W d Q R X | jg |� d S )NT)r, z<a><b></b></a>z html.parser)r- r. r r )r r2 r r r r �,test_no_warning_if_explicit_parser_specifiedL s z9TestWarnings.test_no_warning_if_explicit_parser_specifiedc C sh t jdd��}| jdtd�d�}W d Q R X t|d j�}| jd|k� | jd|k� | jd |j� � d S ) NT)r, z<a><b></b></a>�b)�parseOnlyTheser r9 � parse_onlys <b></b>) r- r. r r r/ r0 r( r r )r r2 r r3 r r r �)test_parseOnlyThese_renamed_to_parse_onlyQ s z6TestWarnings.test_parseOnlyThese_renamed_to_parse_onlyc C sf t jdd��}d}| j|dd�}W d Q R X t|d j�}| jd|k� | jd|k� | jd|j� d S ) NT)r, s é�utf8)� fromEncodingr r= Z from_encoding)r- r. r r/ r0 r( r r )r r2 r< r r3 r r r �*test_fromEncoding_renamed_to_from_encodingY s z7TestWarnings.test_fromEncoding_renamed_to_from_encodingc C s | j t| jddd� d S )Nz<a>T)Zno_such_argument)� assertRaises� TypeErrorr )r r r r �"test_unrecognized_keyword_argumentb s z/TestWarnings.test_unrecognized_keyword_argumentN)T) r" r# r$ r+ r4 r6 r7 r; r> rA r r r r r% : s r% c @ s4 e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd S ) r% c C s� t j� }|j}zBtjdd��}| j|�}W d Q R X t|d j�}| jd|k� W d |j � X tjdd��}| j|�}W d Q R X | j dt |�� d S )NT)r, r zlooks like a filename) �tempfileZNamedTemporaryFile�namer- r. r r/ r0 r( �closer �len)r Z filehandle�filenamer2 r r3 r r r �test_disk_file_warningh s z#TestWarnings.test_disk_file_warningc C s> t jdd��}| jd�}W d Q R X | jtdd� |D ��� d S )NT)r, s http://www.crummybytes.com/c s s | ]}d t |j�kV qdS )zlooks like a URLN)r/ r0 )�.0r2 r r r � <genexpr>} s z?TestWarnings.test_url_warning_with_bytes_url.<locals>.<genexpr>)r- r. r r( �any)r � warning_listr r r r � test_url_warning_with_bytes_urlx s z,TestWarnings.test_url_warning_with_bytes_urlc C s> t jdd��}| jd�}W d Q R X | jtdd� |D ��� d S )NT)r, z http://www.crummyunicode.com/c s s | ]}d t |j�kV qdS )zlooks like a URLN)r/ r0 )rH r2 r r r rI � s zATestWarnings.test_url_warning_with_unicode_url.<locals>.<genexpr>)r- r. r r( rJ )r rK r r r r �!test_url_warning_with_unicode_url� s z.TestWarnings.test_url_warning_with_unicode_urlc C s> t jdd��}| jd�}W d Q R X | jtdd� |D ��� d S )NT)r, s$ http://www.crummybytes.com/ is greatc s s | ]}d t |j�kV qdS )zlooks like a URLN)r/ r0 )rH r2 r r r rI � s zETestWarnings.test_url_warning_with_bytes_and_space.<locals>.<genexpr>)r- r. r � assertFalserJ )r rK r r r r �%test_url_warning_with_bytes_and_space� s z2TestWarnings.test_url_warning_with_bytes_and_spacec C s> t jdd��}| jd�}W d Q R X | jtdd� |D ��� d S )NT)r, z%http://www.crummyuncode.com/ is greatc s s | ]}d t |j�kV qdS )zlooks like a URLN)r/ r0 )rH r2 r r r rI � s zGTestWarnings.test_url_warning_with_unicode_and_space.<locals>.<genexpr>)r- r. r rN rJ )r rK r r r r �'test_url_warning_with_unicode_and_space� s z4TestWarnings.test_url_warning_with_unicode_and_spaceN)r" r# r$ rG rL rM rO rP r r r r r% f s c @ s e Zd Zdd� ZdS )�TestSelectiveParsingc C s. d}t d�}| j||d�}| j|j� d� d S )Nz&No<b>Yes</b><a>No<b>Yes <c>Yes</c></b>r8 )r: s <b>Yes</b><b>Yes <c>Yes</c></b>)r r r r )r �markupZstrainerr r r r � test_parse_with_soupstrainer� s z1TestSelectiveParsing.test_parse_with_soupstrainerN)r" r# r$ rS r r r r rQ � s rQ c @ sx e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd� Zd S ) �TestEntitySubstitutionz1Standalone tests of the EntitySubstitution class.c C s t | _d S )N)r �sub)r r r r �setUp� s z TestEntitySubstitution.setUpc C s d}| j | jj|�d� d S )Nu foo\u2200☃\u00f5baru foo∀☃õbar)r rU �substitute_html)r r) r r r � test_simple_html_substitution� s z4TestEntitySubstitution.test_simple_html_substitutionc C s& d}t |�}| j| jj|j�d� d S )Ns ��foo��z ‘’foo“”)r r rU rW rR )r Zquotes�dammitr r r � test_smart_quote_substitution� s z4TestEntitySubstitution.test_smart_quote_substitutionc C s d}| j | jj|d�|� d S )NzWelcome to "my bar"F)r rU �substitute_xml)r r) r r r �Itest_xml_converstion_includes_no_quotes_if_make_quoted_attribute_is_false� s z`TestEntitySubstitution.test_xml_converstion_includes_no_quotes_if_make_quoted_attribute_is_falsec C s0 | j | jjdd�d� | j | jjdd�d� d S )NZWelcomeTz "Welcome"z Bob's Barz "Bob's Bar")r rU r[ )r r r r �6test_xml_attribute_quoting_normally_uses_double_quotes� s zMTestEntitySubstitution.test_xml_attribute_quoting_normally_uses_double_quotesc C s d}| j | jj|d�d� d S )NzWelcome to "my bar"Tz'Welcome to "my bar"')r rU r[ )r r) r r r �Otest_xml_attribute_quoting_uses_single_quotes_when_value_contains_double_quotes� s zfTestEntitySubstitution.test_xml_attribute_quoting_uses_single_quotes_when_value_contains_double_quotesc C s d}| j | jj|d�d� d S )NzWelcome to "Bob's Bar"Tz""Welcome to "Bob's Bar"")r rU r[ )r r) r r r �btest_xml_attribute_quoting_escapes_single_quotes_when_value_contains_both_single_and_double_quotes� s zyTestEntitySubstitution.test_xml_attribute_quoting_escapes_single_quotes_when_value_contains_both_single_and_double_quotesc C s d}| j | jj|�|� d S )NzWelcome to "Bob's Bar")r rU r[ )r Zquotedr r r �<test_xml_quotes_arent_escaped_when_value_is_not_being_quoted� s zSTestEntitySubstitution.test_xml_quotes_arent_escaped_when_value_is_not_being_quotedc C s | j | jjd�d� d S )Nzfoo<bar>zfoo<bar>)r rU r[ )r r r r �'test_xml_quoting_handles_angle_brackets� s z>TestEntitySubstitution.test_xml_quoting_handles_angle_bracketsc C s | j | jjd�d� d S )NzAT&TzAT&T)r rU r[ )r r r r �#test_xml_quoting_handles_ampersands� s z:TestEntitySubstitution.test_xml_quoting_handles_ampersandsc C s | j | jjd�d� d S )Nz ÁT&Tz&Aacute;T&T)r rU r[ )r r r r �Etest_xml_quoting_including_ampersands_when_they_are_part_of_an_entity� s z\TestEntitySubstitution.test_xml_quoting_including_ampersands_when_they_are_part_of_an_entityc C s | j | jjd�d� d S )Nz ÁT&TzÁT&T)r rU Z"substitute_xml_containing_entities)r r r r �Dtest_xml_quoting_ignoring_ampersands_when_they_are_part_of_an_entity� s z[TestEntitySubstitution.test_xml_quoting_ignoring_ampersands_when_they_are_part_of_an_entityc C s d}| j | jj|�|� dS )z:There's no need to do this except inside attribute values.z Bob's "bar"N)r rU rW )r �textr r r � test_quotes_not_html_substituted� s z7TestEntitySubstitution.test_quotes_not_html_substitutedN)r" r# r$ �__doc__rV rX rZ r\ r] r^ r_ r` ra rb rc rd rf r r r r rT � s rT c sN e Zd Z� fdd�Zdd� Zdd� Zdd� Zd d � Zee d �d d � �Z � Z S )�TestEncodingConversionc s4 t t| �j� d| _| jjd�| _| j| jd� d S )NuU <html><head><meta charset="utf-8"/></head><body><foo>Sacré bleu!</foo></body></html>zutf-8sU <html><head><meta charset="utf-8"/></head><body><foo>Sacré bleu!</foo></body></html>)�superrh rV � unicode_datar r r )r )� __class__r r rV � s z TestEncodingConversion.setUpc C s� t jj}tjtj� zbdd� }|t j_d}| j|�}|j� }| jt |t �� | j || j |j� �� | j |j j� d� W d tjtj� |t j_X d S )Nc S s d S )Nr )r/ r r r �noop� s z>TestEncodingConversion.test_ascii_in_unicode_out.<locals>.noops <foo>a</foo>zutf-8)�bs4rY �chardet_dammit�logging�disable�WARNINGr �decoder( � isinstancer/ r Z document_forr �lower�NOTSET)r �chardetrl �asciiZsoup_from_asciiZunicode_outputr r r �test_ascii_in_unicode_out� s z0TestEncodingConversion.test_ascii_in_unicode_outc C s@ | j | j�}| j|j� | j� | j|jjd� | j|jd � d S )Nu Sacré bleu!)r rj r rr �foor r )r �soup_from_unicoder r r �test_unicode_in_unicode_out s z2TestEncodingConversion.test_unicode_in_unicode_outc C s2 | j | j�}| j|j� | j� | j|jjd� d S )Nu Sacré bleu!)r r r rr rj ry r )r Zsoup_from_utf8r r r �test_utf8_in_unicode_out s z/TestEncodingConversion.test_utf8_in_unicode_outc C s$ | j | j�}| j|jd�| j� d S )Nzutf-8)r rj r r r )r rz r r r � test_utf8_out s z$TestEncodingConversion.test_utf8_outzQBad HTMLParser detected; skipping test of non-ASCII characters in attribute name.c C s( d}| j | j|�jjd�|jd�� d S )Nu <div><a ☃="snowman"></a></div>r< )r r Zdivr )r rR r r r �1test_attribute_name_containing_unicode_characters s zHTestEncodingConversion.test_attribute_name_containing_unicode_characters) r" r# r$ rV rx r{ r| r} r �PYTHON_3_PRE_3_2r~ � __classcell__r r )rk r rh � s rh c @ s� e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd� Zd d � Zd d � Zd d!� Zd"d#� Zd$S )%�TestUnicodeDammitz"Standalone tests of UnicodeDammit.c C s d}t |�}| j|j|� d S )Nu I'm already Unicode! ☃)r r �unicode_markup)r rR rY r r r �test_unicode_input" s z$TestUnicodeDammit.test_unicode_inputc C s d}t |�}| j|jd� d S )Ns <foo>����</foo>z#<foo>\u2018\u2019\u201c\u201d</foo>)r r r� )r rR rY r r r � test_smart_quotes_to_unicode' s z.TestUnicodeDammit.test_smart_quotes_to_unicodec C s" d}t |dd�}| j|jd� d S )Ns <foo>����</foo>Zxml)�smart_quotes_toz+<foo>‘’“”</foo>)r r r� )r rR rY r r r �!test_smart_quotes_to_xml_entities- s z3TestUnicodeDammit.test_smart_quotes_to_xml_entitiesc C s" d}t |dd�}| j|jd� d S )Ns <foo>����</foo>r5 )r� z'<foo>‘’“”</foo>)r r r� )r rR rY r r r �"test_smart_quotes_to_html_entities3 s z4TestUnicodeDammit.test_smart_quotes_to_html_entitiesc C s" d}t |dd�}| j|jd� d S )Ns <foo>����</foo>rw )r� z<foo>''""</foo>)r r r� )r rR rY r r r �test_smart_quotes_to_ascii9 s z,TestUnicodeDammit.test_smart_quotes_to_asciic C s0 d}t |�}| j|jj� d� | j|jd� d S )Ns Sacré bleu! ☃zutf-8u Sacré bleu! ☃)r r r rt r� )r r< rY r r r �test_detect_utf8? s z"TestUnicodeDammit.test_detect_utf8c C s4 d}t |dg�}| j|jj� d� | j|jd� d S )Ns ���z iso-8859-8z\u05dd\u05d5\u05dc\u05e9)r r r rt r� )r �hebrewrY r r r �test_convert_hebrewF s z%TestUnicodeDammit.test_convert_hebrewc C s6 d}t |�}| j|jj� d� | j|jjd�|� d S )Ns ケータイ Watchzutf-8)r r r rt r� r )r �utf_8rY r r r �/test_dont_see_smart_quotes_where_there_are_noneL s zATestUnicodeDammit.test_dont_see_smart_quotes_where_there_are_nonec C s, dj d�}t|dg�}| j|jj� d� d S )Nu Räksmörgåszutf-8z iso-8859-8)r r r r rt )r r rY r r r � test_ignore_inappropriate_codecsR s z2TestUnicodeDammit.test_ignore_inappropriate_codecsc C s: dj d�}x*dD ]"}t||g�}| j|jj� d� qW d S )Nu Räksmörgåszutf-8�.utf8�...� utF---16.!)r� r� r� )r r r r rt )r r Z bad_encodingrY r r r �test_ignore_invalid_codecsW s z,TestUnicodeDammit.test_ignore_invalid_codecsc C sL dj d�}t|dgd�}| j|jj� d� t|ddgd�}| j|jd � d S )Nu Räksmörgåszutf-8)r z windows-1252)r r r r rt )r r rY r r r r! ] s z(TestUnicodeDammit.test_exclude_encodingsc C s t d�}t|j�}d S )Ns' <?xml version="1.0" encoding="UTF-�" ?>)r �list� encodings)r Zdetectedr� r r r �Ptest_encoding_detector_replaces_junk_in_encoding_name_with_replacement_characterk s zbTestUnicodeDammit.test_encoding_detector_replaces_junk_in_encoding_name_with_replacement_characterc C s, x&dD ] }t |dd�}| jd|j� qW d S ) N�& <html><meta charset="euc-jp" /></html>�& <html><meta charset='euc-jp' /></html>�$ <html><meta charset=euc-jp /></html>�# <html><meta charset=euc-jp/></html>T)Zis_htmlzeuc-jp)r� r� r� r� )r r r )r r rY r r r � test_detect_html5_style_meta_tagq s z2TestUnicodeDammit.test_detect_html5_style_meta_tagc C s� d}t jj}tjtj� zPdd� }|t j_t|�}| jd|j� | j d|j k� t |d�}| j |j� W d tjtj � |t j_X d S )NsT <?xml version="1.0" encoding="UTF-8"?> <html><b>بتر</b> <i>��ѐ����</i></html>c S s d S )Nr )r/ r r r rl � s zBTestUnicodeDammit.test_last_ditch_entity_replacement.<locals>.noopTz\ufffdz html.parser) rm rY rn ro rp rq r r Z contains_replacement_charactersr( r� r ru )r �docrv rl rY r r r r �"test_last_ditch_entity_replacement| s z4TestUnicodeDammit.test_last_ditch_entity_replacementc C s, d}t |�}| jd|j� | jd|j� d S )Ns ��< a > � � < / a > u <a>áé</a>zutf-16le)r r r� r )r r rY r r r � test_byte_order_mark_removed� s z.TestUnicodeDammit.test_byte_order_mark_removedc C sP dj d�}dj d�}|| | }| jt|jd� tj|�}| jd|jd�� d S )Nu ☃r r< u “Hi, I like Windows!”� windows_1252u+ ☃☃☃“Hi, I like Windows!”☃☃☃u ☃☃☃)r r? �UnicodeDecodeErrorrr r � detwingler )r r< r� r� Zfixedr r r �test_detwingle� s z TestUnicodeDammit.test_detwinglec C sB x<dD ]4}|j d�}| j|jd�� tj|�}| j||� qW d S )N� œ� ₓ� ðr< � �)r� r� r� )r r( �endswithr r� r )r Ztricky_unicode_char�input�outputr r r �+test_detwingle_ignores_multibyte_characters� s z=TestUnicodeDammit.test_detwingle_ignores_multibyte_charactersN)r" r# r$ rg r� r� r� r� r� r� r� r� r� r� r! r� r� r� r� r� r� r r r r r� s$ #r� c @ s$ e Zd Zdd� Zdd� Zdd� ZdS )�TestNamedspacedAttributec C s t dd �}| j|d� d S )NZxmlns)r r )r �ar r r �test_name_may_be_none� s z.TestNamedspacedAttribute.test_name_may_be_nonec C s t dd�}| jd|� d S )Nr� r8 za:b)r r )r r� r r r �6test_attribute_is_equivalent_to_colon_separated_string� s zOTestNamedspacedAttribute.test_attribute_is_equivalent_to_colon_separated_stringc C sp t ddd�}t ddd�}| j||� t ddd �}| j||� t ddd�}| j||� t ddd�}| j||� d S )Nr� r8 �c�z)r r ZassertNotEqual)r r� r8 r� �d�er r r �;test_attributes_are_equivalent_if_prefix_and_name_identical� s zTTestNamedspacedAttribute.test_attributes_are_equivalent_if_prefix_and_name_identicalN)r" r# r$ r� r� r� r r r r r� � s r� c @ s e Zd Zdd� Zdd� ZdS )�)TestAttributeValueWithCharsetSubstitutionc C s8 t d�}| jd|� | jd|j� | jd|jd�� d S )Nzeuc-jpr< )r r �original_valuer )r �valuer r r �!test_content_meta_attribute_value� s zKTestAttributeValueWithCharsetSubstitution.test_content_meta_attribute_valuec C s8 t d�}| jd|� | jd|j� | jd|jd�� d S )Nztext/html; charset=euc-jpztext/html; charset=utf8r< )r r r� r )r r� r r r r� � s N)r" r# r$ r� r r r r r� � s r� )r r )(rg Zpdbr ro Zunittest�sysrB rm r r Z bs4.elementr r r r Z bs4.dammitr r r Z bs4.testingr r r- Z bs4.builderr r Z LXML_PRESENT� ImportErrorr� � version_infor r r% rQ ZTestCaserT rh r� r� r� r r r r �<module> s6 ,/ E< / tests/__pycache__/test_soup.cpython-36.pyc 0000644 00000052540 15246412105 0014577 0 ustar 00 3 *6]dO � @ s~ d Z ddlmZ ddlZddlZddlZddlZddlmZm Z ddl m Z m Z m Z mZ ddlZddlmZmZmZ ddlmZmZ ddlZyddlmZmZ d ZW n& ek r� Z z d ZW Y ddZ [ X nX ej d d ko�ej d k Z G d d� de�Z G dd� de�Z G dd� de�Z G dd� de�Z!G dd� dej"�Z#G dd� de�Z$G dd� dej"�Z%G dd� de�Z&G d d � d ej"�Z'dS ) z#Tests of Beautiful Soup as a whole.� )� set_traceN)� BeautifulSoup�BeautifulStoneSoup)�CharsetMetaAttributeValue�ContentMetaAttributeValue� SoupStrainer�NamespacedAttribute)�EntitySubstitution� UnicodeDammit�EncodingDetector)�SoupTest�skipIf)�LXMLTreeBuilder�LXMLTreeBuilderForXMLTF� � c @ s$ e Zd Zdd� Zdd� Zdd� ZdS )�TestConstructorc C s" d}| j |�}| jd|jj� d S )Nu <h1>éé</h1>u éé)�soup� assertEqual�h1�string)�self�datar � r � /usr/lib/python3.6/test_soup.py�test_short_unicode_input* s z(TestConstructor.test_short_unicode_inputc C s" d}| j |�}| jd|jj� d S )Nz<h1>foo bar</h1>zfoo bar)r r r r )r r r r r r �test_embedded_null/ s z"TestConstructor.test_embedded_nullc C s, dj d�}| j|dgd�}| jd|j� d S )Nu Räksmörgåszutf-8)�exclude_encodingsz windows-1252)�encoder r �original_encoding)r � utf8_datar r r r �test_exclude_encodings4 s z&TestConstructor.test_exclude_encodingsN)�__name__� __module__� __qualname__r r r! r r r r r ( s r c @ sF e Zd Zddd�Zdd� Zdd� Zdd � Zd d � Zd d � Zdd� Z dS )� TestWarningsTc C s" |j tjd d� �}| j|� d S )N�P )� startswithr ZNO_PARSER_SPECIFIED_WARNING� assertTrue)r �sZis_there�vr r r �_no_parser_specified< s z!TestWarnings._no_parser_specifiedc C s> t jdd��}| jd�}W d Q R X t|d j�}| j|� d S )NT)�recordz<a><b></b></a>r )�warnings�catch_warningsr �str�message�_assert_no_parser_specified)r �wr �msgr r r �#test_warning_if_no_parser_specified@ s z0TestWarnings.test_warning_if_no_parser_specifiedc C s@ t jdd��}| jdd�}W d Q R X t|d j�}| j|� d S )NT)r, z<a><b></b></a>�htmlr )r- r. r r/ r0 r1 )r r2 r r3 r r r �*test_warning_if_parser_specified_too_vagueF s z7TestWarnings.test_warning_if_parser_specified_too_vaguec C s4 t jdd��}| jdd�}W d Q R X | jg |� d S )NT)r, z<a><b></b></a>z html.parser)r- r. r r )r r2 r r r r �,test_no_warning_if_explicit_parser_specifiedL s z9TestWarnings.test_no_warning_if_explicit_parser_specifiedc C sh t jdd��}| jdtd�d�}W d Q R X t|d j�}| jd|k� | jd|k� | jd |j� � d S ) NT)r, z<a><b></b></a>�b)�parseOnlyTheser r9 � parse_onlys <b></b>) r- r. r r r/ r0 r( r r )r r2 r r3 r r r �)test_parseOnlyThese_renamed_to_parse_onlyQ s z6TestWarnings.test_parseOnlyThese_renamed_to_parse_onlyc C sf t jdd��}d}| j|dd�}W d Q R X t|d j�}| jd|k� | jd|k� | jd|j� d S ) NT)r, s é�utf8)� fromEncodingr r= Z from_encoding)r- r. r r/ r0 r( r r )r r2 r< r r3 r r r �*test_fromEncoding_renamed_to_from_encodingY s z7TestWarnings.test_fromEncoding_renamed_to_from_encodingc C s | j t| jddd� d S )Nz<a>T)Zno_such_argument)� assertRaises� TypeErrorr )r r r r �"test_unrecognized_keyword_argumentb s z/TestWarnings.test_unrecognized_keyword_argumentN)T) r" r# r$ r+ r4 r6 r7 r; r> rA r r r r r% : s r% c @ s4 e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd S ) r% c C s� t j� }|j}zBtjdd��}| j|�}W d Q R X t|d j�}| jd|k� W d |j � X tjdd��}| j|�}W d Q R X | j dt |�� d S )NT)r, r zlooks like a filename) �tempfileZNamedTemporaryFile�namer- r. r r/ r0 r( �closer �len)r Z filehandle�filenamer2 r r3 r r r �test_disk_file_warningh s z#TestWarnings.test_disk_file_warningc C s> t jdd��}| jd�}W d Q R X | jtdd� |D ��� d S )NT)r, s http://www.crummybytes.com/c s s | ]}d t |j�kV qdS )zlooks like a URLN)r/ r0 )�.0r2 r r r � <genexpr>} s z?TestWarnings.test_url_warning_with_bytes_url.<locals>.<genexpr>)r- r. r r( �any)r � warning_listr r r r � test_url_warning_with_bytes_urlx s z,TestWarnings.test_url_warning_with_bytes_urlc C s> t jdd��}| jd�}W d Q R X | jtdd� |D ��� d S )NT)r, z http://www.crummyunicode.com/c s s | ]}d t |j�kV qdS )zlooks like a URLN)r/ r0 )rH r2 r r r rI � s zATestWarnings.test_url_warning_with_unicode_url.<locals>.<genexpr>)r- r. r r( rJ )r rK r r r r �!test_url_warning_with_unicode_url� s z.TestWarnings.test_url_warning_with_unicode_urlc C s> t jdd��}| jd�}W d Q R X | jtdd� |D ��� d S )NT)r, s$ http://www.crummybytes.com/ is greatc s s | ]}d t |j�kV qdS )zlooks like a URLN)r/ r0 )rH r2 r r r rI � s zETestWarnings.test_url_warning_with_bytes_and_space.<locals>.<genexpr>)r- r. r � assertFalserJ )r rK r r r r �%test_url_warning_with_bytes_and_space� s z2TestWarnings.test_url_warning_with_bytes_and_spacec C s> t jdd��}| jd�}W d Q R X | jtdd� |D ��� d S )NT)r, z%http://www.crummyuncode.com/ is greatc s s | ]}d t |j�kV qdS )zlooks like a URLN)r/ r0 )rH r2 r r r rI � s zGTestWarnings.test_url_warning_with_unicode_and_space.<locals>.<genexpr>)r- r. r rN rJ )r rK r r r r �'test_url_warning_with_unicode_and_space� s z4TestWarnings.test_url_warning_with_unicode_and_spaceN)r" r# r$ rG rL rM rO rP r r r r r% f s c @ s e Zd Zdd� ZdS )�TestSelectiveParsingc C s. d}t d�}| j||d�}| j|j� d� d S )Nz&No<b>Yes</b><a>No<b>Yes <c>Yes</c></b>r8 )r: s <b>Yes</b><b>Yes <c>Yes</c></b>)r r r r )r �markupZstrainerr r r r � test_parse_with_soupstrainer� s z1TestSelectiveParsing.test_parse_with_soupstrainerN)r" r# r$ rS r r r r rQ � s rQ c @ sx e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd� Zd S ) �TestEntitySubstitutionz1Standalone tests of the EntitySubstitution class.c C s t | _d S )N)r �sub)r r r r �setUp� s z TestEntitySubstitution.setUpc C s d}| j | jj|�d� d S )Nu foo\u2200☃\u00f5baru foo∀☃õbar)r rU �substitute_html)r r) r r r � test_simple_html_substitution� s z4TestEntitySubstitution.test_simple_html_substitutionc C s& d}t |�}| j| jj|j�d� d S )Ns ��foo��z ‘’foo“”)r r rU rW rR )r Zquotes�dammitr r r � test_smart_quote_substitution� s z4TestEntitySubstitution.test_smart_quote_substitutionc C s d}| j | jj|d�|� d S )NzWelcome to "my bar"F)r rU �substitute_xml)r r) r r r �Itest_xml_converstion_includes_no_quotes_if_make_quoted_attribute_is_false� s z`TestEntitySubstitution.test_xml_converstion_includes_no_quotes_if_make_quoted_attribute_is_falsec C s0 | j | jjdd�d� | j | jjdd�d� d S )NZWelcomeTz "Welcome"z Bob's Barz "Bob's Bar")r rU r[ )r r r r �6test_xml_attribute_quoting_normally_uses_double_quotes� s zMTestEntitySubstitution.test_xml_attribute_quoting_normally_uses_double_quotesc C s d}| j | jj|d�d� d S )NzWelcome to "my bar"Tz'Welcome to "my bar"')r rU r[ )r r) r r r �Otest_xml_attribute_quoting_uses_single_quotes_when_value_contains_double_quotes� s zfTestEntitySubstitution.test_xml_attribute_quoting_uses_single_quotes_when_value_contains_double_quotesc C s d}| j | jj|d�d� d S )NzWelcome to "Bob's Bar"Tz""Welcome to "Bob's Bar"")r rU r[ )r r) r r r �btest_xml_attribute_quoting_escapes_single_quotes_when_value_contains_both_single_and_double_quotes� s zyTestEntitySubstitution.test_xml_attribute_quoting_escapes_single_quotes_when_value_contains_both_single_and_double_quotesc C s d}| j | jj|�|� d S )NzWelcome to "Bob's Bar")r rU r[ )r Zquotedr r r �<test_xml_quotes_arent_escaped_when_value_is_not_being_quoted� s zSTestEntitySubstitution.test_xml_quotes_arent_escaped_when_value_is_not_being_quotedc C s | j | jjd�d� d S )Nzfoo<bar>zfoo<bar>)r rU r[ )r r r r �'test_xml_quoting_handles_angle_brackets� s z>TestEntitySubstitution.test_xml_quoting_handles_angle_bracketsc C s | j | jjd�d� d S )NzAT&TzAT&T)r rU r[ )r r r r �#test_xml_quoting_handles_ampersands� s z:TestEntitySubstitution.test_xml_quoting_handles_ampersandsc C s | j | jjd�d� d S )Nz ÁT&Tz&Aacute;T&T)r rU r[ )r r r r �Etest_xml_quoting_including_ampersands_when_they_are_part_of_an_entity� s z\TestEntitySubstitution.test_xml_quoting_including_ampersands_when_they_are_part_of_an_entityc C s | j | jjd�d� d S )Nz ÁT&TzÁT&T)r rU Z"substitute_xml_containing_entities)r r r r �Dtest_xml_quoting_ignoring_ampersands_when_they_are_part_of_an_entity� s z[TestEntitySubstitution.test_xml_quoting_ignoring_ampersands_when_they_are_part_of_an_entityc C s d}| j | jj|�|� dS )z:There's no need to do this except inside attribute values.z Bob's "bar"N)r rU rW )r �textr r r � test_quotes_not_html_substituted� s z7TestEntitySubstitution.test_quotes_not_html_substitutedN)r" r# r$ �__doc__rV rX rZ r\ r] r^ r_ r` ra rb rc rd rf r r r r rT � s rT c sN e Zd Z� fdd�Zdd� Zdd� Zdd� Zd d � Zee d �d d � �Z � Z S )�TestEncodingConversionc s4 t t| �j� d| _| jjd�| _| j| jd� d S )NuU <html><head><meta charset="utf-8"/></head><body><foo>Sacré bleu!</foo></body></html>zutf-8sU <html><head><meta charset="utf-8"/></head><body><foo>Sacré bleu!</foo></body></html>)�superrh rV � unicode_datar r r )r )� __class__r r rV � s z TestEncodingConversion.setUpc C s� t jj}tjtj� zbdd� }|t j_d}| j|�}|j� }| jt |t �� | j || j |j� �� | j |j j� d� W d tjtj� |t j_X d S )Nc S s d S )Nr )r/ r r r �noop� s z>TestEncodingConversion.test_ascii_in_unicode_out.<locals>.noops <foo>a</foo>zutf-8)�bs4rY �chardet_dammit�logging�disable�WARNINGr �decoder( � isinstancer/ r Z document_forr �lower�NOTSET)r �chardetrl �asciiZsoup_from_asciiZunicode_outputr r r �test_ascii_in_unicode_out� s z0TestEncodingConversion.test_ascii_in_unicode_outc C s@ | j | j�}| j|j� | j� | j|jjd� | j|jd � d S )Nu Sacré bleu!)r rj r rr �foor r )r �soup_from_unicoder r r �test_unicode_in_unicode_out s z2TestEncodingConversion.test_unicode_in_unicode_outc C s2 | j | j�}| j|j� | j� | j|jjd� d S )Nu Sacré bleu!)r r r rr rj ry r )r Zsoup_from_utf8r r r �test_utf8_in_unicode_out s z/TestEncodingConversion.test_utf8_in_unicode_outc C s$ | j | j�}| j|jd�| j� d S )Nzutf-8)r rj r r r )r rz r r r � test_utf8_out s z$TestEncodingConversion.test_utf8_outzQBad HTMLParser detected; skipping test of non-ASCII characters in attribute name.c C s( d}| j | j|�jjd�|jd�� d S )Nu <div><a ☃="snowman"></a></div>r< )r r Zdivr )r rR r r r �1test_attribute_name_containing_unicode_characters s zHTestEncodingConversion.test_attribute_name_containing_unicode_characters) r" r# r$ rV rx r{ r| r} r �PYTHON_3_PRE_3_2r~ � __classcell__r r )rk r rh � s rh c @ s� e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd� Zd d � Zd d � Zd d!� Zd"d#� Zd$S )%�TestUnicodeDammitz"Standalone tests of UnicodeDammit.c C s d}t |�}| j|j|� d S )Nu I'm already Unicode! ☃)r r �unicode_markup)r rR rY r r r �test_unicode_input" s z$TestUnicodeDammit.test_unicode_inputc C s d}t |�}| j|jd� d S )Ns <foo>����</foo>z#<foo>\u2018\u2019\u201c\u201d</foo>)r r r� )r rR rY r r r � test_smart_quotes_to_unicode' s z.TestUnicodeDammit.test_smart_quotes_to_unicodec C s" d}t |dd�}| j|jd� d S )Ns <foo>����</foo>Zxml)�smart_quotes_toz+<foo>‘’“”</foo>)r r r� )r rR rY r r r �!test_smart_quotes_to_xml_entities- s z3TestUnicodeDammit.test_smart_quotes_to_xml_entitiesc C s" d}t |dd�}| j|jd� d S )Ns <foo>����</foo>r5 )r� z'<foo>‘’“”</foo>)r r r� )r rR rY r r r �"test_smart_quotes_to_html_entities3 s z4TestUnicodeDammit.test_smart_quotes_to_html_entitiesc C s" d}t |dd�}| j|jd� d S )Ns <foo>����</foo>rw )r� z<foo>''""</foo>)r r r� )r rR rY r r r �test_smart_quotes_to_ascii9 s z,TestUnicodeDammit.test_smart_quotes_to_asciic C s0 d}t |�}| j|jj� d� | j|jd� d S )Ns Sacré bleu! ☃zutf-8u Sacré bleu! ☃)r r r rt r� )r r< rY r r r �test_detect_utf8? s z"TestUnicodeDammit.test_detect_utf8c C s4 d}t |dg�}| j|jj� d� | j|jd� d S )Ns ���z iso-8859-8z\u05dd\u05d5\u05dc\u05e9)r r r rt r� )r �hebrewrY r r r �test_convert_hebrewF s z%TestUnicodeDammit.test_convert_hebrewc C s6 d}t |�}| j|jj� d� | j|jjd�|� d S )Ns ケータイ Watchzutf-8)r r r rt r� r )r �utf_8rY r r r �/test_dont_see_smart_quotes_where_there_are_noneL s zATestUnicodeDammit.test_dont_see_smart_quotes_where_there_are_nonec C s, dj d�}t|dg�}| j|jj� d� d S )Nu Räksmörgåszutf-8z iso-8859-8)r r r r rt )r r rY r r r � test_ignore_inappropriate_codecsR s z2TestUnicodeDammit.test_ignore_inappropriate_codecsc C s: dj d�}x*dD ]"}t||g�}| j|jj� d� qW d S )Nu Räksmörgåszutf-8�.utf8�...� utF---16.!)r� r� r� )r r r r rt )r r Z bad_encodingrY r r r �test_ignore_invalid_codecsW s z,TestUnicodeDammit.test_ignore_invalid_codecsc C sL dj d�}t|dgd�}| j|jj� d� t|ddgd�}| j|jd � d S )Nu Räksmörgåszutf-8)r z windows-1252)r r r r rt )r r rY r r r r! ] s z(TestUnicodeDammit.test_exclude_encodingsc C s" t d�}t|j�}d|ks t�d S )Ns' <?xml version="1.0" encoding="UTF-�" ?>u utf-�)r �list� encodings�AssertionError)r Zdetectedr� r r r �Ptest_encoding_detector_replaces_junk_in_encoding_name_with_replacement_characterk s zbTestUnicodeDammit.test_encoding_detector_replaces_junk_in_encoding_name_with_replacement_characterc C s, x&dD ] }t |dd�}| jd|j� qW d S ) N�& <html><meta charset="euc-jp" /></html>�& <html><meta charset='euc-jp' /></html>�$ <html><meta charset=euc-jp /></html>�# <html><meta charset=euc-jp/></html>T)Zis_htmlzeuc-jp)r� r� r� r� )r r r )r r rY r r r � test_detect_html5_style_meta_tagq s z2TestUnicodeDammit.test_detect_html5_style_meta_tagc C s� d}t jj}tjtj� zPdd� }|t j_t|�}| jd|j� | j d|j k� t |d�}| j |j� W d tjtj � |t j_X d S )NsT <?xml version="1.0" encoding="UTF-8"?> <html><b>بتر</b> <i>��ѐ����</i></html>c S s d S )Nr )r/ r r r rl � s zBTestUnicodeDammit.test_last_ditch_entity_replacement.<locals>.noopTz\ufffdz html.parser) rm rY rn ro rp rq r r Z contains_replacement_charactersr( r� r ru )r �docrv rl rY r r r r �"test_last_ditch_entity_replacement| s z4TestUnicodeDammit.test_last_ditch_entity_replacementc C s, d}t |�}| jd|j� | jd|j� d S )Ns ��< a > � � < / a > u <a>áé</a>zutf-16le)r r r� r )r r rY r r r � test_byte_order_mark_removed� s z.TestUnicodeDammit.test_byte_order_mark_removedc C sP dj d�}dj d�}|| | }| jt|jd� tj|�}| jd|jd�� d S )Nu ☃r r< u “Hi, I like Windows!”� windows_1252u+ ☃☃☃“Hi, I like Windows!”☃☃☃u ☃☃☃)r r? �UnicodeDecodeErrorrr r � detwingler )r r< r� r� Zfixedr r r �test_detwingle� s z TestUnicodeDammit.test_detwinglec C sB x<dD ]4}|j d�}| j|jd�� tj|�}| j||� qW d S )N� œ� ₓ� ðr< � �)r� r� r� )r r( �endswithr r� r )r Ztricky_unicode_char�input�outputr r r �+test_detwingle_ignores_multibyte_characters� s z=TestUnicodeDammit.test_detwingle_ignores_multibyte_charactersN)r" r# r$ rg r� r� r� r� r� r� r� r� r� r� r! r� r� r� r� r� r� r r r r r� s$ #r� c @ s$ e Zd Zdd� Zdd� Zdd� ZdS )�TestNamedspacedAttributec C s t dd �}| j|d� d S )NZxmlns)r r )r �ar r r �test_name_may_be_none� s z.TestNamedspacedAttribute.test_name_may_be_nonec C s t dd�}| jd|� d S )Nr� r8 za:b)r r )r r� r r r �6test_attribute_is_equivalent_to_colon_separated_string� s zOTestNamedspacedAttribute.test_attribute_is_equivalent_to_colon_separated_stringc C sp t ddd�}t ddd�}| j||� t ddd �}| j||� t ddd�}| j||� t ddd�}| j||� d S )Nr� r8 �c�z)r r ZassertNotEqual)r r� r8 r� �d�er r r �;test_attributes_are_equivalent_if_prefix_and_name_identical� s zTTestNamedspacedAttribute.test_attributes_are_equivalent_if_prefix_and_name_identicalN)r" r# r$ r� r� r� r r r r r� � s r� c @ s e Zd Zdd� Zdd� ZdS )�)TestAttributeValueWithCharsetSubstitutionc C s8 t d�}| jd|� | jd|j� | jd|jd�� d S )Nzeuc-jpr< )r r �original_valuer )r �valuer r r �!test_content_meta_attribute_value� s zKTestAttributeValueWithCharsetSubstitution.test_content_meta_attribute_valuec C s8 t d�}| jd|� | jd|j� | jd|jd�� d S )Nztext/html; charset=euc-jpztext/html; charset=utf8r< )r r r� r )r r� r r r r� � s N)r" r# r$ r� r r r r r� � s r� )r r )(rg Zpdbr ro Zunittest�sysrB rm r r Z bs4.elementr r r r Z bs4.dammitr r r Z bs4.testingr r r- Z bs4.builderr r Z LXML_PRESENT� ImportErrorr� � version_infor r r% rQ ZTestCaserT rh r� r� r� r r r r �<module> s6 ,/ E< / tests/__pycache__/test_html5lib.cpython-36.opt-1.pyc 0000644 00000012132 15246412105 0016261 0 ustar 00 3 6]+ � @ s� d Z ddlZyddlmZ dZW n& ek rF Z z dZW Y ddZ[X nX ddlmZ ddl m Z m Z m Z e e d�G d d � d e e ��Z dS ) zDTests to ensure that the html5lib tree builder generates good trees.� N)�HTML5TreeBuilderTF)� SoupStrainer)�HTML5TreeBuilderSmokeTest�SoupTest�skipIfz?html5lib seems not to be present, not testing its tree builder.c @ sl e Zd ZdZedd� �Zdd� Zdd� Zdd � Zd d � Z d d � Z dd� Z dd� Z dd� Z dd� Zdd� ZdS )�HTML5LibBuilderSmokeTestz"See ``HTML5TreeBuilderSmokeTest``.c C s t � S )N)r )�self� r �#/usr/lib/python3.6/test_html5lib.py�default_builder s z(HTML5LibBuilderSmokeTest.default_builderc C sd t d�}d}tjdd��}| j||d�}W d Q R X | j|j� | j|�� | jdt|d j �k� d S )N�bz <p>A <b>bold</b> statement.</p>T)�record)Z parse_onlyz4the html5lib tree builder doesn't support parse_onlyr ) r �warnings�catch_warnings�soup� assertEqual�decodeZ document_forZ assertTrue�str�message)r Zstrainer�markup�wr r r r �test_soupstrainer s z*HTML5LibBuilderSmokeTest.test_soupstrainerc C s d}| j |d� | j d� dS )z8html5lib inserts <tbody> tags where other parsers don't.z[<table id="1"><tr><td>Here's another table:<table id="2"><tr><td>foo</td></tr></table></td>z�<table id="1"><tbody><tr><td>Here's another table:<table id="2"><tbody><tr><td>foo</td></tr></tbody></table></td></tr></tbody></table>z{<table><thead><tr><td>Foo</td></tr></thead><tbody><tr><td>Bar</td></tr></tbody><tfoot><tr><td>Baz</td></tr></tfoot></table>N)ZassertSoupEquals)r r r r r � test_correctly_nested_tables( s z5HTML5LibBuilderSmokeTest.test_correctly_nested_tablesc C s$ d}| j |�}| jd|jj� � d S )Nzy<?xml version="1.0" encoding="utf-8"?> <!DOCTYPE html> <html> <head> </head> <body> <p>foo</p> </body> </html>s <p>foo</p>)r r �p�encode)r r r r r r �(test_xml_declaration_followed_by_doctype< s zAHTML5LibBuilderSmokeTest.test_xml_declaration_followed_by_doctypec C s: d}| j |�}| jd|jj� � | jdt|jd��� d S )Nz%<p><em>foo</p> <p>bar<a></a></em></p>zD<body><p><em>foo</em></p><em> </em><p><em>bar<a></a></em></p></body>� r )r r �bodyr �len�find_all)r r r r r r �test_reparented_markupJ s z/HTML5LibBuilderSmokeTest.test_reparented_markupc C s: d}| j |�}| jd|jj� � | jdt|jd��� d S )Nz&<p><em>foo</p> <p>bar<a></a></em></p> zE<body><p><em>foo</em></p><em> </em><p><em>bar<a></a></em></p> </body>r r )r r r r r r )r r r r r r �+test_reparented_markup_ends_with_whitespaceQ s zDHTML5LibBuilderSmokeTest.test_reparented_markup_ends_with_whitespacec C s0 d}| j |�}|jdd�\}}|jd�\}}dS )z�Verify that we keep the two whitespace nodes in this document distinct when reparenting the adjacent <tbody> tags. z,<table> <tbody><tbody><ims></tbody> </table>� )�stringZtbodyN)r r )r r r Zspace1Zspace2Ztbody1Ztbody2r r r �<test_reparented_markup_containing_identical_whitespace_nodesW s zUHTML5LibBuilderSmokeTest.test_reparented_markup_containing_identical_whitespace_nodesc C s^ d}| j |�}|j}| jd|j� |jdd�}|jdd�d }| j||j� | j||j� d S )NzF<div><a>aftermath<p><noscript>target</noscript>aftermath</a></p></div>�target)r# Z aftermath� ���)r �noscriptr Z next_element�findr Zprevious_element)r r r r( r% Zfinal_aftermathr r r �*test_reparented_markup_containing_childrenb s zCHTML5LibBuilderSmokeTest.test_reparented_markup_containing_childrenc C s d}| j |�}dS )z(Processing instructions become comments.s <?PITarget PIContent?>N)r )r r r r r r �test_processing_instructionr s z4HTML5LibBuilderSmokeTest.test_processing_instructionc C s, d}| j |�}|jd�\}}| j||� d S )Ns <a class="my_class"><p></a>�a)r r r )r r r Za1Za2r r r �test_cloned_multivalue_nodex s z4HTML5LibBuilderSmokeTest.test_cloned_multivalue_nodec C s$ d}| j |�}| jd|jj� � d S )Ns <table><td></tbody>Az><body>A<table><tbody><tr><td></td></tr></tbody></table></body>)r r r r )r r r r r r �test_foster_parenting s z.HTML5LibBuilderSmokeTest.test_foster_parentingN)�__name__� __module__� __qualname__�__doc__�propertyr r r r r r! r$ r* r+ r- r. r r r r r s r )r2 r Z bs4.builderr ZHTML5LIB_PRESENT� ImportError�eZ bs4.elementr Z bs4.testingr r r r r r r r �<module> s tests/__pycache__/__init__.cpython-36.pyc 0000644 00000000225 15246412105 0014302 0 ustar 00 3 ��CN � @ s d Z dS )zThe beautifulsoup tests.N)�__doc__� r r � /usr/lib/python3.6/__init__.py�<module> s tests/__pycache__/test_htmlparser.cpython-36.pyc 0000644 00000004774 15246412105 0016000 0 ustar 00 3 Y=K[� � @ sf d Z ddlmZ ddlZddlmZmZ ddlmZ ddl m Z G dd� dee�Z G d d � d e�Z dS ) zGTests to ensure that the html.parser tree builder generates good trees.� )� set_traceN)�SoupTest�HTMLTreeBuilderSmokeTest)�HTMLParserTreeBuilder)�BeautifulSoupHTMLParserc @ s@ e Zd Zedd� �Zdd� Zdd� Zdd� Zd d � Zd d � Z d S )� HTMLParserTreeBuilderSmokeTestc C s t � S )N)r )�self� r �%/usr/lib/python3.6/test_htmlparser.py�default_builder s z.HTMLParserTreeBuilderSmokeTest.default_builderc C s d S )Nr )r r r r � test_namespaced_system_doctype s z=HTMLParserTreeBuilderSmokeTest.test_namespaced_system_doctypec C s d S )Nr )r r r r � test_namespaced_public_doctype s z=HTMLParserTreeBuilderSmokeTest.test_namespaced_public_doctypec C s< | j d�}tj|d�}tj|�}| jt|jt|j��� dS )zfUnlike most tree builders, HTMLParserTreeBuilder and will be restored after pickling. z <a><b>foo</a>� N)Zsoup�pickle�dumps�loadsZ assertTrue� isinstanceZbuilder�type)r ZtreeZdumpedZloadedr r r �test_builder_is_pickled s z6HTMLParserTreeBuilderSmokeTest.test_builder_is_pickledc C s | j dd� | j dd� d S )Nz<br></br><br></br><br></br>z<br/><br/><br/>z</br></br></br>� )�assertSoupEquals)r r r r �)test_redundant_empty_element_closing_tags! s zHHTMLParserTreeBuilderSmokeTest.test_redundant_empty_element_closing_tagsc C s | j dd� d S )Nz foo &# barzfoo &# bar)r )r r r r �test_empty_element% s z1HTMLParserTreeBuilderSmokeTest.test_empty_elementN) �__name__� __module__� __qualname__�propertyr r r r r r r r r r r s r c @ s e Zd Zdd� ZdS )�TestHTMLParserSubclassc C s t � }|jd� dS )zlVerify that our HTMLParser subclass implements error() in a way that doesn't cause a crash. z don't crashN)r �error)r �parserr r r � test_error, s z!TestHTMLParserSubclass.test_errorN)r r r r r r r r r + s r ) �__doc__Zpdbr r Z bs4.testingr r Z bs4.builderr Zbs4.builder._htmlparserr r r r r r r �<module> s ! tests/__pycache__/test_tree.cpython-36.pyc 0000644 00000254024 15246412105 0014551 0 ustar 00 3 6]�8 � @ s d Z ddlmZ ddlZddlZddlZddlZddlmZ ddl m Z m Z ddl m Z mZmZmZmZmZmZmZ ddlmZmZ e jd�dk Ze jd �dk ZG d d � d e�ZG d d � d e�Z G dd� de�Z G dd� de�Z G dd� de�Z G dd� de�Z G dd� de�Z!G dd� de�Z"G dd� de�Z#G d d � d e#�Z$G d d � d e#�Z%G d d!� d!e�Z&G d"d#� d#e&�Z'G d$d%� d%e&�Z(G d&d'� d'e�Z)G d(d)� d)e�Z*G d*d+� d+e�Z+G d,d-� d-e�Z,G d.d/� d/e�Z-G d0d1� d1e�Z.G d2d3� d3e�Z/G d4d5� d5e�Z0G d6d7� d7e�Z1dS )8a8 Tests for Beautiful Soup's tree traversal methods. The tree traversal methods are the main advantage of using Beautiful Soup over just using a parser. Different parsers will build different Beautiful Soup trees given the same markup, but all Beautiful Soup trees can be traversed with the methods tested here. � )� set_traceN)� BeautifulSoup)�builder_registry�HTMLParserTreeBuilder)�PY3K�CData�Comment� Declaration�Doctype�NavigableString� SoupStrainer�Tag)�SoupTest�skipIfZxmlZlxmlc @ s e Zd Zdd� Zdd� ZdS )�TreeTestc C s | j dd� |D �|� dS )z�Make sure that the given tags have the correct text. This is used in tests that define a bunch of tags, each containing a single string, and then select certain strings by some mechanism. c S s g | ] }|j �qS � )�string)�.0�tagr r � /usr/lib/python3.6/test_tree.py� <listcomp>2 s z*TreeTest.assertSelects.<locals>.<listcomp>N)� assertEqual)�self�tags� should_matchr r r � assertSelects+ s zTreeTest.assertSelectsc C s | j dd� |D �|� dS )z�Make sure that the given tags have the correct IDs. This is used in tests that define a bunch of tags, each containing a single string, and then select certain strings by some mechanism. c S s g | ] }|d �qS )�idr )r r r r r r ; s z-TreeTest.assertSelectsIDs.<locals>.<listcomp>N)r )r r r r r r �assertSelectsIDs4 s zTreeTest.assertSelectsIDsN)�__name__� __module__� __qualname__r r r r r r r ) s r c @ s8 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd S ) �TestFindz�Basic tests of the find() method. find() just calls find_all() with limit=1, so it's not tested all that thouroughly here. c C s" | j d�}| j|jd�jd� d S )Nz <a>1</a><b>2</b><a>3</a><b>4</b>�b�2)�soupr �findr )r r$ r r r � test_find_tagE s zTestFind.test_find_tagc C s" | j d�}| j|jdd�d� d S )Nu <h1>Räksmörgås</h1>u Räksmörgås)r )r$ r r% )r r$ r r r �test_unicode_text_findI s z TestFind.test_unicode_text_findc C s, | j d�}t|� | jd|jdd�j� d S )Nu& <h1 id="Räksmörgås">here it is</h1>z here it isu Räksmörgås)r )r$ �strr r% �text)r r$ r r r �test_unicode_attribute_findM s z$TestFind.test_unicode_attribute_findc C s" | j d�}| jdt|j� �� dS )z)Test an optimization that finds all tags.z<a>foo</a><b>bar</b>� N)r$ r �len�find_all)r r$ r r r �test_find_everythingS s z TestFind.test_find_everythingc C s$ | j d�}| jdt|jd��� dS )z;Test an optimization that finds all tags with a given name.z <a>foo</a><b>bar</b><a>baz</a>r+ �aN)r$ r r, r- )r r$ r r r � test_find_everything_with_nameX s z'TestFind.test_find_everything_with_nameN) r r r �__doc__r& r' r* r. r0 r r r r r! > s r! c @ s8 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd S ) � TestFindAllz%Basic tests of the find_all() method.c C s� | j d�}| j|jdd�dg� | j|jdd�dg� | j|jddgd�ddg� | j|jtjd�d�dddg� | j|jdd�dddg� d S ) z'You can search the tree for text nodes.u <html>Foo<b>bar</b>»</html>�bar)r )r) �Fooz.*� »TN)r$ r r- �re�compile)r r$ r r r �test_find_all_text_nodes` s z$TestFindAll.test_find_all_text_nodesc C s� | j d�}| j|jddd�dddg� | j|jddd�dg� | j|jdd d�dddd d g� | j|jdd d�dddd d g� d S )z7You can limit the number of items returned by find_all.z(<a>1</a><a>2</a><a>3</a><a>4</a><a>5</a>r/ � )�limit�1r# �3� � �4�5r N)r$ r r- )r r$ r r r �test_find_all_limitp s z TestFindAll.test_find_all_limitc C s: | j d�}| j|ddd�dg� | j|jdd�dg� d S ) Nz!<a>1</a><b>2<a id='foo'>3</a></b>r/ r= )r: r; �foo)r r< )r$ r r" )r r$ r r r �%test_calling_a_tag_is_calling_findall| s z1TestFindAll.test_calling_a_tag_is_calling_findallc C s. | j d�}g }|j|� | jg |j|�� d S )Nz<a></a>)r$ �appendr r- )r r$ �lr r r �Ttest_find_all_with_self_referential_data_structure_does_not_cause_infinite_recursion� s z`TestFindAll.test_find_all_with_self_referential_data_structure_does_not_cause_infinite_recursionc C s^ | j d�}|jd�}| jt|d�� |jd�}| jt|d�� |jdd�}| jt|d�� dS )z%All find_all calls return a ResultSetz<a></a>r/ �sourceTrB )r) N)r$ r- � assertTrue�hasattr)r r$ �resultr r r �test_find_all_resultset� s z#TestFindAll.test_find_all_resultsetN) r r r r1 r8 rA rC rF rK r r r r r2 ] s r2 c @ s e Zd Zdd� ZdS )�TestFindAllBasicNamespacesc C s< | j d�}| jd|jd�j� | jd|jddid�j� d S )Nz0<mathml:msqrt>4</mathml:msqrt><a svg:fill="red">r? z mathml:msqrtr/ zsvg:fillZred)�attrs)r$ r r% r �name)r r$ r r r � test_find_by_namespaced_name� s z7TestFindAllBasicNamespaces.test_find_by_namespaced_nameN)r r r rO r r r r rL � s rL c sp e Zd ZdZ� fdd�Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dd� Z dd� Z� ZS )�TestFindAllByNamez&Test ways of finding tags by tag name.c s t t| �j� | jd�| _d S )Nz�<a>First tag.</a> <b>Second tag.</b> <c>Third <a>Nested tag.</a> tag.</c>)�superr �setUpr$ �tree)r )� __class__r r rR � s zTestFindAllByName.setUpc C s | j | jjd�ddg� d S )Nr/ z First tag.z Nested tag.)r rS r- )r r r r �test_find_all_by_tag_name� s z+TestFindAllByName.test_find_all_by_tag_namec C s\ | j | jjddd�dg� | j | jjddd�ddg� | j | jjdtjd�d�ddg� d S )Nr/ z First tag.)r) Tz Nested tag.r )r rS r- r6 r7 )r r r r � test_find_all_by_name_and_text� s z0TestFindAllByName.test_find_all_by_name_and_textc C s | j | jjjd�dg� d S )Nr/ z Nested tag.)r rS �cr- )r r r r �!test_find_all_on_non_root_element� s z3TestFindAllByName.test_find_all_on_non_root_elementc C s | j | jd�ddg� d S )Nr/ z First tag.z Nested tag.)r rS )r r r r �%test_calling_element_invokes_find_all� s z7TestFindAllByName.test_calling_element_invokes_find_allc C s | j | jjtd��ddg� d S )Nr/ z First tag.z Nested tag.)r rS r- r )r r r r � test_find_all_by_tag_strainer� s z/TestFindAllByName.test_find_all_by_tag_strainerc C s" | j | jjddg�dddg� d S )Nr/ r" z First tag.z Second tag.z Nested tag.)r rS r- )r r r r �test_find_all_by_tag_names� s z,TestFindAllByName.test_find_all_by_tag_namesc C s$ | j | jjddd��dddg� d S )NT)r/ r" z First tag.z Second tag.z Nested tag.)r rS r- )r r r r �test_find_all_by_tag_dict� s z+TestFindAllByName.test_find_all_by_tag_dictc C s$ | j | jjtjd��dddg� d S )Nz^[ab]$z First tag.z Second tag.z Nested tag.)r rS r- r6 r7 )r r r r �test_find_all_by_tag_re� s z)TestFindAllByName.test_find_all_by_tag_rec C s, dd� }| j d�}| j|j|�ddg� d S )Nc S s | j | jd�kS )Nr )rN �get)r r r r �id_matches_name� s zRTestFindAllByName.test_find_all_with_tags_matching_method.<locals>.id_matches_namez�<a id="a">Match 1.</a> <a id="1">Does not match.</a> <b id="b">Match 2.</a>zMatch 1.zMatch 2.)r$ r r- )r r_ rS r r r �'test_find_all_with_tags_matching_method� s z9TestFindAllByName.test_find_all_with_tags_matching_methodc C sx | j d�}|jdd�}|jdtjd��}|jdddg�\}}| jd|j� | jd|j� | jd|j� | jd|j� d S )NzH<div class='a b'>1</div><div class='a c'>2</div><div class='a d'>3</div>�divza dza br< r; )r$ r% r6 r7 r- r r )r r$ Zr1Zr2Zr3Zr4r r r �%test_find_with_multi_valued_attribute� s z7TestFindAllByName.test_find_with_multi_valued_attribute)r r r r1 rR rU rV rX rY rZ r[ r\ r] r` rb � __classcell__r r )rT r rP � s rP c @ s� e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd � Zd d � Zd d � Zd!S )"�TestFindAllByAttributec C s& | j d�}| j|jdd�ddg� d S )Nz� <a id="first">Matching a.</a> <a id="second"> Non-matching <b id="first">Matching b.</b>a. </a>�first)r z Matching a.z Matching b.)r$ r r- )r rS r r r � test_find_all_by_attribute_name� s z6TestFindAllByAttribute.test_find_all_by_attribute_namec C st dj d�}dj d�}| j|�}| j|jg|j|d�� | j|jg|j|jd�d�� | j|jg|j|dgd�� d S )Nu םולש�utf8u <a title="םולש"></a>)�titlezsomething else)�encoder$ r r/ r- �decode)r Zpeace�datar$ r r r �%test_find_all_by_utf8_attribute_value� s z<TestFindAllByAttribute.test_find_all_by_utf8_attribute_valuec C sX | j d�}| j|jdd�dg� | j|jddid�dg� | j|jdd id�d g� d S ) Na0 <a name="name1" class="class1">Name match.</a> <a name="name2" class="class2">Class match.</a> <a name="name3" class="class3">Non-match.</a> <name1>A tag called 'name1'.</name1> Zname1)rN zA tag called 'name1'.rN )rM z Name match.�classZclass2z Class match.)r$ r r- )r rS r r r � test_find_all_by_attribute_dict s z6TestFindAllByAttribute.test_find_all_by_attribute_dictc C s� | j d�}| j|jddd�dg� | j|jddd�dg� | j|jdd d�dg� | j|jdd�dg� | j|jdd �ddg� | j|jdd�dg� | j|jdd �dg� d S ) Nz� <a class="1">Class 1.</a> <a class="2">Class 2.</a> <b class="1">Class 1.</b> <c class="3 4">Class 3 and 4.</c> r/ r; )�class_zClass 1.rW r< zClass 3 and 4.r? )rM )r$ r r- )r rS r r r �test_find_all_by_class s z-TestFindAllByAttribute.test_find_all_by_classc C st | j d�}|jdtjd�d�}| j|dg� |jdtjd�d�}| j|dg� |jdtjd�d�}| j|dg� d S )Nz#<gar class='foo bar'>Found it</gar>Zgar�o)ro zFound itr/ zo b)r$ r- r6 r7 r )r rS �fr r r �0test_find_by_class_when_multiple_classes_present- s zGTestFindAllByAttribute.test_find_by_class_when_multiple_classes_presentc C sd | j d�}| j|jdtjd��dg� dd� }| j|jd|�g � dd� }| j|jd|�dg� d S ) Nz<a class='bar'>Found it</a>r/ ZbazFound itc S s t | �dkS )Nr9 )r, )�valuer r r �big_attribute_value@ s znTestFindAllByAttribute.test_find_all_with_non_dictionary_for_attrs_finds_by_class.<locals>.big_attribute_valuec S s t | �dkS )Nr9 )r, )rt r r r �small_attribute_valueE s zpTestFindAllByAttribute.test_find_all_with_non_dictionary_for_attrs_finds_by_class.<locals>.small_attribute_value)r$ r r- r6 r7 )r r$ ru rv r r r �:test_find_all_with_non_dictionary_for_attrs_finds_by_class; s zQTestFindAllByAttribute.test_find_all_with_non_dictionary_for_attrs_finds_by_classc C s� | j d�}|jd�\}}| j||g|jdd�� | j|g|jdd�� | j|g|jddd�� | j|g|jdd�� | jg |jdd�� d S )Nz*<a class="foo bar"></a><a class="foo"></a>r/ rB r3 zfoo bar)ro zbar foo)r$ r- r )r r$ r/ Za2r r r �:test_find_all_with_string_for_attrs_finds_multiple_classesK s zQTestFindAllByAttribute.test_find_all_with_string_for_attrs_finds_multiple_classesc C s0 | j d�}tddid�}| j|j|�dg� d S )Nzi <a id="first">Match.</a> <a id="second">Non-match.</a>r re )rM zMatch.)r$ r r r- )r rS �strainerr r r �'test_find_all_by_attribute_soupstrainerW s z>TestFindAllByAttribute.test_find_all_by_attribute_soupstrainerc C s& | j d�}| j|jdd d�dg� d S )Nz�<a id="1">ID present.</a> <a>No ID present.</a> <a id="">ID is empty.</a>r/ )r zNo ID present.)r$ r r- )r rS r r r �$test_find_all_with_missing_attribute_ s z;TestFindAllByAttribute.test_find_all_with_missing_attributec C s& | j d�}| j|jdd�ddg� d S )Nz�<a id="1">ID present.</a> <a>No ID present.</a> <a id="">ID is empty.</a>T)r z ID present.z ID is empty.)r$ r r- )r rS r r r �$test_find_all_with_defined_attributeg s z;TestFindAllByAttribute.test_find_all_with_defined_attributec C s> | j d�}ddg}| j|jdd�|� | j|jdd�|� d S )Nz[<a id=1>Unquoted attribute.</a> <a id="1">Quoted attribute.</a>zUnquoted attribute.zQuoted attribute.r= )r r; )r$ r r- )r rS Zexpectedr r r �$test_find_all_with_numeric_attributep s z;TestFindAllByAttribute.test_find_all_with_numeric_attributec C s, | j d�}| j|jdddgd�ddg� d S )Nz�<a id="1">1</a> <a id="2">2</a> <a id="3">3</a> <a>No ID.</a>r; r< r? )r )r$ r r- )r rS r r r �(test_find_all_with_list_attribute_valuesy s z?TestFindAllByAttribute.test_find_all_with_list_attribute_valuesc C s, | j d�}| j|jtjd�d�ddg� d S )Nz�<a id="a">One a.</a> <a id="aa">Two as.</a> <a id="ab">Mixed as and bs.</a> <a id="b">One b.</a> <a>No ID.</a>z^a+$)r zOne a.zTwo as.)r$ r r- r6 r7 )r rS r r r �5test_find_all_with_regular_expression_attribute_value� s zLTestFindAllByAttribute.test_find_all_with_regular_expression_attribute_valuec C sX | j d�}|j}| j|g|jddd�� | jg |jddd�� | jg |jddd�� d S )Nz <b>foo</b><b>bar</b><a>foo</a>r/ rB )r) r3 )r$ r/ r r- )r r$ r/ r r r �'test_find_by_name_and_containing_string� s z>TestFindAllByAttribute.test_find_by_name_and_containing_stringc C s* | j d�}| j|jd�|jddd�� d S )Nz"<a>foo</a><a><b><c>foo</c></b></a>r/ rB )r) )r$ r r- )r r$ r r r �=test_find_by_name_and_containing_string_when_string_is_buried� s zTTestFindAllByAttribute.test_find_by_name_and_containing_string_when_string_is_buriedc C sB | j d�}|j}| j|g|jddd�� | jg |jddd�� d S )Nz"<b id="1">foo</b><a id="2">foo</a>r+ rB )r r) r= r3 )r$ r/ r r- )r r$ r/ r r r �,test_find_by_attribute_and_containing_string� s zCTestFindAllByAttribute.test_find_by_attribute_and_containing_stringN)r r r rf rl rn rp rs rw rx rz r{ r| r} r~ r r� r� r� r r r r rd � s rd c @ s e Zd ZdZdd� ZdS )� TestIndexzTest Tag.indexc C sN | j d�}|j}x(t|j�D ]\}}| j||j|�� q W | jt|jd� d S )Nah <div> <a>Identical</a> <b>Not identical</b> <a>Identical</a> <c><d>Identical with child</d></c> <b>Also not identical</b> <c><d>Identical with child</d></c> </div>r= )r$ ra � enumerate�contentsr �index� assertRaises� ValueError)r rS ra �i�elementr r r � test_index� s zTestIndex.test_indexN)r r r r1 r� r r r r r� � s r� c s` e Zd ZdZ� fdd�Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z � Z S )�TestParentOperationsz;Test navigation and searching through an element's parents.c s( t t| �j� | jd�| _| jj| _d S )Na1 <ul id="empty"></ul> <ul id="top"> <ul id="middle"> <ul id="bottom"> <b>Start here</b> </ul> </ul>)rQ r� rR r$ rS r" �start)r )rT r r rR � s zTestParentOperations.setUpc C sF | j | jjd d� | j | jjjd d� | j | jjjjd d� d S )Nr �bottom�middle�top)r r� �parent)r r r r � test_parent� s z TestParentOperations.test_parentc C s | j jd }| j|j| j � d S )Nr )rS r� r r� )r Ztop_tagr r r �%test_parent_of_top_tag_is_soup_object� s z:TestParentOperations.test_parent_of_top_tag_is_soup_objectc C s | j d | jj� d S )N)r rS r� )r r r r � test_soup_object_has_no_parent� s z3TestParentOperations.test_soup_object_has_no_parentc C s8 | j | jjd�dddg� | j | jjddd�dg� d S )N�ulr� r� r� )r )r r� Z find_parents)r r r r �test_find_parents� s z&TestParentOperations.test_find_parentsc C s8 | j | jjd�d d� | j | jjddd�d d� d S )Nr� r r� r� )r )r r� � find_parent)r r r r �test_find_parent� s z%TestParentOperations.test_find_parentc C s" | j jdd�}| j|jjd� d S )Nz Start here)r) r" )rS r% r r� rN )r r) r r r �test_parent_of_text_element� s z0TestParentOperations.test_parent_of_text_elementc C s( | j jdd�}| j|jd�d d� d S )Nz Start here)r) r� r r� )rS r% r r� )r r) r r r � test_text_element_find_parent� s z2TestParentOperations.test_text_element_find_parentc C s( dd� | j jD �}| j|dddg� d S )Nc S s& g | ] }|d k rd|j kr|d �qS )Nr )rM )r r� r r r r � s z>TestParentOperations.test_parent_generator.<locals>.<listcomp>r� r� r� )r� �parentsr )r r� r r r �test_parent_generator� s z*TestParentOperations.test_parent_generator)r r r r1 rR r� r� r� r� r� r� r� r� rc r r )rT r r� � s r� c s e Zd Z� fdd�Z� ZS )� ProximityTestc s t t| �j� | jd�| _d S )Nzg<html id="start"><head></head><body><b id="1">One</b><b id="2">Two</b><b id="3">Three</b></body></html>)rQ r rR r$ rS )r )rT r r rR � s zProximityTest.setUp)r r r rR rc r r )rT r r� � s r� c sT e Zd Z� fdd�Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z � Z S )�TestNextOperationsc s t t| �j� | jj| _d S )N)rQ r� rR rS r" r� )r )rT r r rR � s zTestNextOperations.setUpc C s* | j | jjd� | j | jjjd d� d S )N�Oner r# )r r� � next_element)r r r r � test_next� s z TestNextOperations.test_nextc C s | j jdd�}| j|jd � d S )N�Three)r) )rS r% r r� )r Zlastr r r � test_next_of_last_item_is_none� s z1TestNextOperations.test_next_of_last_item_is_nonec C s | j | jjd � d S )N)r rS r� )r r r r �test_next_of_root_is_none s z,TestNextOperations.test_next_of_root_is_nonec C sB | j | jjd�ddg� | jjdd� | j | jjdd�dg� d S )Nr" �Twor� r9 )r )r r� � find_all_next)r r r r �test_find_all_next s z%TestNextOperations.test_find_all_nextc C s2 | j | jjd�d d� | j | jjdd�d� d S )Nr" r r# r� )r) )r r� � find_next)r r r r �test_find_next s z!TestNextOperations.test_find_nextc C s< | j jdd�}| j|jd�jd� | j|jd�ddg� d S )Nr� )r) r" r� r� )rS r% r r� r r r� )r r) r r r � test_find_next_for_text_element s z2TestNextOperations.test_find_next_for_text_elementc C sF | j jdd�}dd� |jD �}|\}}| j|d d� | j|d� d S )Nr� )r) c S s g | ]}|�qS r r )r �noder r r r s z:TestNextOperations.test_next_generator.<locals>.<listcomp>r r< r� )rS r% Z next_elementsr )r r� Z successorsr r� r r r �test_next_generator s z&TestNextOperations.test_next_generator) r r r rR r� r� r� r� r� r� r� rc r r )rT r r� � s r� c sT e Zd Z� fdd�Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z � Z S )�TestPreviousOperationsc s" t t| �j� | jjdd�| _d S )Nr� )r) )rQ r� rR rS r% �end)r )rT r r rR s z TestPreviousOperations.setUpc C s* | j | jjd d� | j | jjjd� d S )Nr r< r� )r r� �previous_element)r r r r � test_previous! s z$TestPreviousOperations.test_previousc C s | j jd�}| j|jd � d S )N�html)rS r% r r� )r re r r r �#test_previous_of_first_item_is_none% s z:TestPreviousOperations.test_previous_of_first_item_is_nonec C s d S )Nr )r r r r � test_previous_of_root_is_none) s z4TestPreviousOperations.test_previous_of_root_is_nonec C s6 | j | jjd�dddg� | j | jjdd�dg� d S )Nr" r� r� r� r= )r )r r� �find_all_previous)r r r r �test_find_all_previous/ s z-TestPreviousOperations.test_find_all_previousc C s2 | j | jjd�d d� | j | jjdd�d� d S )Nr" r r< r� )r) )r r� � find_previous)r r r r �test_find_previous7 s z)TestPreviousOperations.test_find_previousc C s> | j jdd�}| j|jd�jd� | j|jd�dddg� d S )Nr� )r) r" r� r� )rS r% r r� r r r� )r r) r r r �#test_find_previous_for_text_element; s z:TestPreviousOperations.test_find_previous_for_text_elementc C sh | j jdd�}dd� |jD �}|\}}}}| j|d d� | j|jd� | j|jd� | j|jd � d S ) Nr� )r) c S s g | ]}|�qS r r )r r� r r r r C s zBTestPreviousOperations.test_previous_generator.<locals>.<listcomp>r r; �body�headr� )rS r% Zprevious_elementsr rN )r r� Z predecessorsr" r� r� r� r r r �test_previous_generatorA s z.TestPreviousOperations.test_previous_generator) r r r rR r� r� r� r� r� r� r� rc r r )rT r r� s r� c s e Zd Z� fdd�Z� ZS )� SiblingTestc s4 t t| �j� d}tjd�jd|�}| j|�| _d S )Na� <html> <span id="1"> <span id="1.1"></span> </span> <span id="2"> <span id="2.1"></span> </span> <span id="3"> <span id="3.1"></span> </span> <span id="4"></span> </html>z\n\s*� )rQ r� rR r6 r7 �subr$ rS )r �markup)rT r r rR P s zSiblingTest.setUp)r r r rR rc r r )rT r r� N s r� c sL e Zd Z� fdd�Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z � Z S )�TestNextSiblingc s" t t| �j� | jjdd�| _d S )Nr; )r )rQ r� rR rS r% r� )r )rT r r rR f s zTestNextSibling.setUpc C s | j | jjd � d S )N)r rS � next_sibling)r r r r �!test_next_sibling_of_root_is_nonej s z1TestNextSibling.test_next_sibling_of_root_is_nonec C sB | j | jjd d� | j | jjjd d� | j | jjd d� d S )Nr r# r< z1.1)r r� r� r� )r r r r �test_next_siblingm s z!TestNextSibling.test_next_siblingc C sN | j | jjjd � | jjdd�}| j |jd � | jjdd�}| j |jd � d S )Nz1.1)r r? )r rS r� r� r% )r � nested_spanZ last_spanr r r � test_next_sibling_may_not_existt s z/TestNextSibling.test_next_sibling_may_not_existc C s | j | jjd�d d� d S )N�spanr r# )r r� �find_next_sibling)r r r r �test_find_next_sibling} s z&TestNextSibling.test_find_next_siblingc C s6 | j | jjd�dddg� | j | jjdd�dg� d S )Nr� r# r< r? )r )r r� �find_next_siblings)r r r r �test_next_siblings� s z"TestNextSibling.test_next_siblingsc C sv | j d�}|jdd�}| j|jjd� | j|jjd� | j|jd�dg� | j|jdd�d� | j|jdd�d � d S )NzFoo<b>bar</b>bazr4 )r) r" �bazr3 �nonesuch)r$ r% r r� rN r r� r� )r r$ r� r r r �"test_next_sibling_for_text_element� s z2TestNextSibling.test_next_sibling_for_text_element) r r r rR r� r� r� r� r� r� rc r r )rT r r� d s r� c sL e Zd Z� fdd�Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z � Z S )�TestPreviousSiblingc s" t t| �j� | jjdd�| _d S )Nr? )r )rQ r� rR rS r% r� )r )rT r r rR � s zTestPreviousSibling.setUpc C s | j | jjd � d S )N)r rS �previous_sibling)r r r r �%test_previous_sibling_of_root_is_none� s z9TestPreviousSibling.test_previous_sibling_of_root_is_nonec C sB | j | jjd d� | j | jjjd d� | j | jjd d� d S )Nr r< r# z3.1)r r� r� r� )r r r r �test_previous_sibling� s z)TestPreviousSibling.test_previous_siblingc C sN | j | jjjd � | jjdd�}| j |jd � | jjdd�}| j |jd � d S )Nz1.1)r r; )r rS r� r� r% )r r� Z first_spanr r r �#test_previous_sibling_may_not_exist� s z7TestPreviousSibling.test_previous_sibling_may_not_existc C s | j | jjd�d d� d S )Nr� r r< )r r� �find_previous_sibling)r r r r �test_find_previous_sibling� s z.TestPreviousSibling.test_find_previous_siblingc C s6 | j | jjd�dddg� | j | jjdd�dg� d S )Nr� r< r# r; )r )r r� �find_previous_siblings)r r r r �test_previous_siblings� s z*TestPreviousSibling.test_previous_siblingsc C sv | j d�}|jdd�}| j|jjd� | j|jjd� | j|jd�dg� | j|jdd�d� | j|jdd�d � d S )NzFoo<b>bar</b>bazr� )r) r" r4 r3 r� )r$ r% r r� rN r r� r� )r r$ r� r r r �&test_previous_sibling_for_text_element� s z:TestPreviousSibling.test_previous_sibling_for_text_element) r r r rR r� r� r� r� r� r� rc r r )rT r r� � s r� c @ s0 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd S ) �TestTagCreationz$Test the ability to create new tags.c C sd | j d�}|jddddid�}| jt|t�� | jd|j� | jtddd�|j� | jd |j � d S )Nr� rB r� rN za name)r3 rM )r3 rN ) r$ �new_tagrH � isinstancer r rN �dictrM r� )r r$ r� r r r � test_new_tag� s z TestTagCreation.test_new_tagc C s� t rBtdd�}|jd�}|jd�}| jd|j� � | jd|j� � tdd�}|jd�}|jd�}| jd|j� � | jd|j� � d S ) Nr� zlxml-xml�br�ps <br/>s <p/>z html.parsers <p></p>)�XML_BUILDER_PRESENTr r� r ri )r Zxml_soupZxml_brZxml_pZ html_soupZhtml_brZhtml_pr r r �1test_tag_inherits_self_closing_rules_from_builder� s zATestTagCreation.test_tag_inherits_self_closing_rules_from_builderc C s4 | j d�}|jd�}| jd|� | jt|t�� d S )Nr� rB )r$ � new_stringr rH r� r )r r$ �sr r r �'test_new_string_creates_navigablestring� s z7TestTagCreation.test_new_string_creates_navigablestringc C s6 | j d�}|jdt�}| jd|� | jt|t�� d S )Nr� rB )r$ r� r r rH r� )r r$ r� r r r �3test_new_string_can_create_navigablestring_subclass� s zCTestTagCreation.test_new_string_can_create_navigablestring_subclassN)r r r r1 r� r� r� r� r r r r r� � s r� c @ s< e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Zd'd(� Zd)d*� Zd+d,� Zd-d.� Zd/d0� Zd1d2� Zd3d4� Z d5d6� Z d7d8� Z d9d:� Z d;d<� Z d=d>� Z!d?d@� Z"dAdB� Z#dCdD� Z$dEdF� Z%dGdH� Z&dIdJ� Z'dKdL� Z(dMS )N�TestTreeModificationc C sl | j d�}d|jd< | j|j� | jd�� |jd= | j|j� | jd�� d|jd< | j|j� | jd�� d S ) Nz<a id="1"></a>r+ r z<a id="2"></a>z<a></a>rB Zid2z<a id2="foo"></a>)r$ r/ r rj � document_for)r r$ r r r �test_attribute_modification� s z0TestTreeModification.test_attribute_modificationc C sl t jd�� }| jd|d�}t||d�}t||d�}d|d< |jjd|� |jjd |� | j|jj� d � d S ) Nr� z <body></body>)�builderr/ �olzhttp://foo.com/Zhrefr r= s4 <body><a href="http://foo.com/"></a><ol></ol></body>)r �lookupr$ r r� �insertr ri )r r� r$ r/ r� r r r �test_new_tag_creation� s z*TestTreeModification.test_new_tag_creationc C s\ d}| j |�}|jdd�}|j}|jdd�j|j� | j|j|� | j|j� | jd�� d S )NzT<p id="1">Don't leave me <b>here</b>.</p> <p id="2">Don't leave!</p>r# )r zD<p id="1">Don't leave me .</p> <p id="2">Don't leave!<b>here</b></p>)r$ r% r" rD r r� rj r� )r �docr$ Z second_paraZboldr r r �!test_append_to_contents_moves_tag� s z6TestTreeModification.test_append_to_contents_moves_tagc C s0 d}| j |�}|j}|j|j�}| j||� d S )Nz<a></a><b><c></c></b>)r$ r/ � replace_withrW r )r r) r$ r/ �new_ar r r �1test_replace_with_returns_thing_that_was_replaced s zFTestTreeModification.test_replace_with_returns_thing_that_was_replacedc C s, d}| j |�}|j}|j� }| j||� d S )Nz<a><b></b><c></c></a>)r$ r/ �unwrapr )r r) r$ r/ r� r r r �+test_unwrap_returns_thing_that_was_replaced s z@TestTreeModification.test_unwrap_returns_thing_that_was_replacedc C sJ | j d�}|j}|j� | jd |j� | jt|j� | jt|j|j � d S )Nz<a><b>Foo</b></a><c>Bar</c>) r$ r/ �extractr r� r� r� r� r� rW )r r$ r/ r r r �Itest_replace_with_and_unwrap_give_useful_exception_when_tag_has_no_parent s z^TestTreeModification.test_replace_with_and_unwrap_give_useful_exception_when_tag_has_no_parentc C s: d}| j |�}|j}|jj|� | j|j� | j|�� d S )Nz-<a><b></b><c>Foo<d></d></c></a><a><e></e></a>)r$ rW r� r rj r� )r r) r$ rW r r r � test_replace_tag_with_itself' s z1TestTreeModification.test_replace_tag_with_itselfc C s& d}| j |�}| jt|jj|j� d S )Nz<a><b></b></a>)r$ r� r� r" r� r/ )r r) r$ r r r �1test_replace_tag_with_its_parent_raises_exception. s zFTestTreeModification.test_replace_tag_with_its_parent_raises_exceptionc C s( d}| j |�}| jt|jjd|j� d S )Nz<a><b></b></a>r )r$ r� r� r/ r� )r r) r$ r r r �,test_insert_tag_into_itself_raises_exception3 s zATestTreeModification.test_insert_tag_into_itself_raises_exceptionc C s� | j d�}d}| j |�}|jd|� x |jD ]}t|t� s,t�q,W t|j�\}}}}| jd|j � | jd|j � | jd|j � | jd|j � dS ) z�Inserting one BeautifulSoup object into another actually inserts all of its children -- you'll never combine BeautifulSoup objects. z-<p>And now, a word:</p><p>And we're back.</p>z<p>p2</p><p>p3</p>r= zAnd now, a word:�p2�p3zAnd we're back.N) r$ r� Z descendantsr� r �AssertionError�listZchildrenr r ) r r$ r) Z to_insertr� �p1r� r� Zp4r r r �1test_insert_beautifulsoup_object_inserts_children8 s zFTestTreeModification.test_insert_beautifulsoup_object_inserts_childrenc C sX | j d�}|j}|jd }|jdd� |j\}}|jd� |jd� | jd|jj� d S )Nz <p><a>one</a><b>three</b></p>r r= Ztwor� Zthree)r$ r/ r� r� Z replaceWithr r" r )r r$ r/ r" �left�rightr r r �3test_replace_with_maintains_next_element_throughoutL s zHTestTreeModification.test_replace_with_maintains_next_element_throughoutc C sl | j d�}|jdd�jd� |jdd�}|j}| j|j|� | j|j|� | j|jj|� | j|jd � d S )Nz <b>Argh!</b>zArgh!)r) zHooray!)r$ r% r� r" r r� r� r� )r r$ �new_textr" r r r �test_replace_final_node[ s z,TestTreeModification.test_replace_final_nodec C s� | j d�}|jjdd� | j|j� | jd�� |jdd�}| j|jd� | j|jj|� | j|j d� | j|j j |� | j|j d � | j|j|j � d S )Nz<a><b>Argh!</b><c></c></a>r= zHooray!z!<a><b>Argh!Hooray!</b><c></c></a>)r) zArgh!) r$ r" r� r rj r� r% r� r� r� r� rW )r r$ r r r r �test_consecutive_text_nodese s z0TestTreeModification.test_consecutive_text_nodesc C sT | j d�}|jjdd� |jjdd� | jddg|jj� | j|jjd jd� d S )Nz<a></a>r r3 rB )r$ r/ r� r r� r� )r r$ r r r �test_insert_stringz s z'TestTreeModification.test_insert_stringc C s� | j }| jd|d�}t||d�}|jdd� |jjd|� | j|j� | jd�� |j}| j|j |� | j|j |� |j dd �}| j|j |� | j|j |� |j}| j|j |� | j|j |� |j dd �}| j|j|� | j|j |� | j|j |� d S ) Nz%<a><b>Find</b><c>lady!</c><d></d></a>)r� Zmagictagr �ther= z=<a><b>Find</b><magictag>the</magictag><c>lady!</c><d></d></a>ZFind)r) )Zdefault_builderr$ r r� r/ r rj r� r" r� r� r% r� r� rW r� )r r� r$ Z magic_tagZb_tagr% Zc_tagr r r r �test_insert_tag� s, z$TestTreeModification.test_insert_tagc C s0 d}| j |�}|jj|j� | j||j� � d S )Nz<a><b></b></a>)r$ r/ rD r" r rj )r rk r$ r r r �*test_append_child_thats_already_at_the_end� s z?TestTreeModification.test_append_child_thats_already_at_the_endc C s2 d}| j |�}|jjd|j� | jd|j� � d S )Nz <a><b></b><c></c><d></d></a>r z <a><d></d><b></b><c></c></a>)r$ r/ r� �dr rj )r rk r$ r r r �$test_move_tag_to_beginning_of_parent� s z9TestTreeModification.test_move_tag_to_beginning_of_parentc C s. | j d�}|jjdd� | jt|j�d� d S )Nz<br/>r= ZContentsz<br>Contents</br>)r$ r� r� r r( )r r$ r r r �&test_insert_works_on_empty_element_tag� s z;TestTreeModification.test_insert_works_on_empty_element_tagc C s` | j d�}|jjd� |jjd� | j|j� | jd�� |jj|j� | j|j� | jd�� d S )Nz<a>foo</a><b>bar</b>�BAZ�QUUXzQUUX<a>foo</a>BAZ<b>bar</b>zQUUX<b>bar</b><a>foo</a>BAZ)r$ r" � insert_beforer/ r rj r� )r r$ r r r �test_insert_before� s z'TestTreeModification.test_insert_beforec C s` | j d�}|jjd� |jjd� | j|j� | jd�� |jj|j� | j|j� | jd�� d S )Nz<a>foo</a><b>bar</b>r r z<a>foo</a>QUUX<b>bar</b>BAZzQUUX<b>bar</b><a>foo</a>BAZ)r$ r" � insert_afterr/ r rj r� )r r$ r r r �test_insert_after� s z&TestTreeModification.test_insert_afterc C sR | j d�}|jd�}|jd�}| jt|j|� | jt|j|� | jt|j|� d S )Nr� r/ )r$ r� r� r� r� r �NotImplementedError)r r$ r r r r r �:test_insert_after_raises_exception_if_after_has_no_meaning� s zOTestTreeModification.test_insert_after_raises_exception_if_after_has_no_meaningc C sR | j d�}|jd�}|jd�}| jt|j|� | jt|j|� | jt|j|� d S )Nr� r/ )r$ r� r� r� r� r r )r r$ r r r r r �Ftest_insert_before_raises_notimplementederror_if_before_has_no_meaning� s z[TestTreeModification.test_insert_before_raises_notimplementederror_if_before_has_no_meaningc C sv | j d�}|jd�\}}|j|� | j|j� | jd�� | j|jd � | j|j|j� | j|jd� | j|j d� d S )Nz;<p>There's <b>no</b> business like <b>show</b> business</p>r" z0<p>There's business like <b>no</b> business</p>�noz business) r$ r- r� r rj r� r� r� r� r� )r r$ r Zshowr r r �test_replace_with� s z&TestTreeModification.test_replace_withc C s0 d}| j |�}|jj|j� | jd|j� � d S )Nz<a><b></b><c></c></a>z<a><c></c></a>)r$ r" r� rW r rj )r rk r$ r r r �test_replace_first_child� s z-TestTreeModification.test_replace_first_childc C s0 d}| j |�}|jj|j� | jd|j� � d S )Nz<a><b></b><c></c></a>z<a><b></b></a>)r$ rW r� r" r rj )r rk r$ r r r �test_replace_last_child� s z,TestTreeModification.test_replace_last_childc C s | j d�}|j}|j}|j|� | j|j� | jd�� | j|jd � | j|jdd�j d � | j|j d � | j|j d � | j|j d � | j|j|j � | j|j d� | j|j j |j� | j|j d � |jdd�}|j}| j|j |� | j|j |� | j|j |� | j|j |� d S )NzQ<a>We<b>reserve<c>the</c><d>right</d></b></a><e>to<f>refuse</f><g>service</g></e>z-<a>We<f>refuse</f></a><e>to<g>service</g></e>r )r) ZWeZto)r$ r" rr r� r rj r� r� r% r� r� r� r� r/ �e�g)r r$ Z remove_tagZmove_tagZto_textZg_tagr r r � test_nested_tag_replace_with� s. z1TestTreeModification.test_nested_tag_replace_withc C s6 | j d�}|jj� | j|jd � | j|jjd� d S )NzI <p>Unneeded <em>formatting</em> is unneeded</p> z Unneeded formatting is unneeded)r$ �emr� r r� r) )r rS r r r � test_unwrap s z TestTreeModification.test_unwrapc C sF | j d�}|jj|jd��}| j|j� d� | j|j� | jd�� d S )NzI wish I was bold.r" z<b>I wish I was bold.</b>)r$ r �wrapr� r rj r� )r r$ rt r r r � test_wrap" s z TestTreeModification.test_wrapc C s4 | j d�}|jjj|j� | j|j� | jd�� d S )Nz<b></b>I wish I was bold.z<b>I wish I was bold.</b>)r$ r" r� r r rj r� )r r$ r r r �%test_wrap_extracts_tag_from_elsewhere) s z:TestTreeModification.test_wrap_extracts_tag_from_elsewherec C sH | j d�}|jjj|j� | jdt|jj�� | j|j� | jd�� d S )Nz+<b>I like being bold.</b>I wish I was bold.r+ z+<b>I like being bold.I wish I was bold.</b>) r$ r" r� r r r, r� rj r� )r r$ r r r �&test_wrap_puts_new_contents_at_the_end/ s z;TestTreeModification.test_wrap_puts_new_contents_at_the_endc C s� | j d�}| jt|jj�d� |jdd�j� }| j|j� d� | j|j� d� | jt|jj�d� | j|jd � | j|j d � | j|j j d � |jdd �}|jd d �}| j|j |� | j|j |� | j|j |� | j|j |� d S ) NzR<html><body>Some content. <div id="nav">Nav crap</div> More content.</body></html>r9 Znav)r z6<html><body>Some content. More content.</body></html>z <div id="nav">Nav crap</div>r+ zSome content. )r) z More content.) r$ r r, r� r� r% r� rj r� r� r� r� r� )r r$ Z extractedZ content_1Z content_2r r r � test_extract7 s" z!TestTreeModification.test_extractc C sz | j d�}|jj}|jj}|jd�}|jd�}|jj|� |jj|� |j� |j� | j||jj� | j||jj� d S )Nz<a>foo</a><b>bar</b>rB r3 )r$ r/ r r" r� rD r� r )r r$ Zfoo_1Zbar_1Zfoo_2Zbar_2r r r �4test_extract_distinguishes_between_identical_stringsP s zITestTreeModification.test_extract_distinguishes_between_identical_stringsc s8 | j d�� � fdd�� jd�D � | jdt� j�� d S )Nzv <html> <head> <script>foo</script> </head> <body> <script>bar</script> <a></a> </body> <script>baz</script> </html>c s g | ]}� j j� �qS r )�scriptr� )r r� )r$ r r r m s zKTestTreeModification.test_extract_multiples_of_same_tag.<locals>.<listcomp>r$ z<body> <a></a> </body>)r$ r- r r( r� )r r )r$ r �"test_extract_multiples_of_same_taga s z7TestTreeModification.test_extract_multiples_of_same_tagc C s. | j d�}|jd�j� | jd |jd�� d S )Nz <html> <body>hi</body> </html>r� )r$ r% r� r )r r$ r r r �Btest_extract_works_when_element_is_surrounded_by_identical_stringsq s zWTestTreeModification.test_extract_works_when_element_is_surrounded_by_identical_stringsc C sf | j d�}|j}|jj� | jt|jj�d� | jt|d�� |j }|jdd� | jdt|j�� dS )z Tag.clear()z4<p><a>String <em>Italicized</em></a> and another</p>r r� T)Z decomposeN) r$ r/ r� �clearr r, r� rH rI r )r r$ r/ r r r r � test_clearz s z TestTreeModification.test_clearc C sB | j d�}d|j_| j|jjdg� d|j_| j|jjdg� dS )zTag.string = 'string'z<a></a> <b><c></c></b>rB r3 N)r$ r/ r r r� r" )r r$ r r r �test_string_set� s z$TestTreeModification.test_string_setc C s, | j d�}|jj|j_| j|jj� d� d S )Nz<a><b>foo</b><c>bar</c>s <a><b>bar</b><c>bar</c></a>)r$ rW r r" r r/ ri )r r$ r r r �/test_string_set_does_not_affect_original_string� s zDTestTreeModification.test_string_set_does_not_affect_original_stringc C s2 | j d�}td�}||j_| jt|jjt�� d S )Nz<a></a>rB )r$ r r/ r rH r� )r r$ �cdatar r r �)test_set_string_preserves_class_of_string� s z>TestTreeModification.test_set_string_preserves_class_of_stringN))r r r r� r� r� r� r� r� r� r� r� r� r r r r r r r r r r r r r r r r r r r r! r" r# r% r& r( r) r* r, r r r r r� � sL $ r� c @ sx e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd� Zd S ) �TestElementObjectsz)Test various features of element objects.c C sV | j d�}| jt|j�d� | jt|�d� | jt|j�d� | jt|jj�d� dS )z3The length of an element is its number of children.z<top>1<b>2</b>3</top>r= r9 N)r$ r r, r� r� )r r$ r r r �test_len� s zTestElementObjects.test_lenc C sL | j d�}| j|j|jd�� | j|jj|jd�jd�� | j|jd� dS )z2Accessing a Python member .foo invokes find('foo')z<b><i></i></b>r" r� N)r$ r r" r% r� r/ )r r$ r r r � test_member_access_invokes_find� s z2TestElementObjects.test_member_access_invokes_findc C sP | j d�}tjdd�� }|j}W d Q R X | j|j|� | jdt|d j�� d S )Nz<b><i></i></b>T)�recordzp.bTag is deprecated, use .find("b") instead. If you really were looking for a tag called bTag, use .find("bTag")r )r$ �warnings�catch_warningsZbTagr r" r( �message)r r$ �wr r r r � test_deprecated_member_access� s z0TestElementObjects.test_deprecated_member_accessc C s2 | j d�}| j|jjd�� | j|jjd�� dS )z�has_attr() checks for the presence of an attribute. Please note note: has_attr() is different from __in__. has_attr() checks the tag's attributes and __in__ checks the tag's chidlren. z<foo attr='bar'>�attrZattr2N)r$ rH rB �has_attr� assertFalse)r r$ r r r � test_has_attr� s z TestElementObjects.test_has_attrc C s d}| j |d� d S )Nz%<b a="1" z="5" m="3" f="2" y="4"></b>z%<b a="1" f="2" m="3" y="4" z="5"></b>)ZassertSoupEquals)r r� r r r �.test_attributes_come_out_in_alphabetical_order� s zATestElementObjects.test_attributes_come_out_in_alphabetical_orderc C s | j d�}| j|jjd� d S )Nz <b>foo</b>rB )r$ r r" r )r r$ r r r � test_string� s z TestElementObjects.test_stringc C s | j d�}| j|jjd � d S )Nz<b></b>)r$ r r" r )r r$ r r r � test_empty_tag_has_no_string� s z/TestElementObjects.test_empty_tag_has_no_stringc C s` | j d�}| j|jjd � | j d�}| j|jjd � | j d�}|jjdd� | j|jjd � d S )Nz<a>foo<b></b><b></b></b>z<a>foo<b></b>bar</b>z <a>foo</b>r= r3 )r$ r r" r r/ r� )r r$ r r r �-test_tag_with_multiple_children_has_no_string� s z@TestElementObjects.test_tag_with_multiple_children_has_no_stringc C s, | j d�}| j|jjd� | j|jd� d S )Nz<a><b>foo</b></a>rB )r$ r r/ r )r r$ r r r �)test_tag_with_recursive_string_has_string� s z<TestElementObjects.test_tag_with_recursive_string_has_stringc C s4 | j d�}| j|jj� | j d�}| j|jj� dS )z7Only a tag containing a single text node has a .string.z<b>f<i>e</i>o</b>z<b></b>N)r$ r8 r" r )r r$ r r r �test_lack_of_string� s z&TestElementObjects.test_lack_of_stringc C s` | j d�}| j|jjd� | j|jjdd�d� | j|jjd�d� | j|jjddd�d� d S ) zBTag.text and Tag.get_text(sep=u"") -> all child text, concatenatedz <a>a<b>r</b> <r> t </r></a>zar t T)�stripZart�,z a,r, , t za,r,tN)r$ r r/ r) �get_text)r r$ r r r � test_all_text� s z TestElementObjects.test_all_textc C sJ | j d�}| j|j� d� | j|jttfd�d� | j|jd d�d� d S )Nzfoo<!--IGNORE-->barZfoobar)�typesZ fooIGNOREbar)r$ r rB r r )r r$ r r r � test_get_text_ignores_comments� s z1TestElementObjects.test_get_text_ignores_commentsc C s$ | j d�}| jddgt|j�� d S )Nzfoo<!--IGNORE-->barrB r3 )r$ r r� Zstrings)r r$ r r r �!test_all_strings_ignores_comments s z4TestElementObjects.test_all_strings_ignores_commentsN)r r r r1 r. r/ r5 r9 r: r; r< r= r> r? rC rE rF r r r r r- � s r- c @ sP e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dS )�TestCDAtaListAttributesz0Testing cdata-list attributes like 'class'. c C s" | j d�}| jdg|jd � d S )Nz<a class='foo'>rB rm )r$ r r/ )r r$ r r r � test_single_value_becomes_list s z6TestCDAtaListAttributes.test_single_value_becomes_listc C s$ | j d�}| jddg|jd � d S )Nz<a class='foo bar'>rB r3 rm )r$ r r/ )r r$ r r r �!test_multiple_values_becomes_list s z9TestCDAtaListAttributes.test_multiple_values_becomes_listc C s& | j d�}| jdddg|jd � d S )Nz<a class='foo bar baz'>rB r3 r� rm )r$ r r/ )r r$ r r r �2test_multiple_values_separated_by_weird_whitespace s zJTestCDAtaListAttributes.test_multiple_values_separated_by_weird_whitespacec C s | j d�}| jd|jj� � d S )Nz<a class='foo bar'>s <a class="foo bar"></a>)r$ r r/ ri )r r$ r r r �,test_attributes_joined_into_string_on_output s zDTestCDAtaListAttributes.test_attributes_joined_into_string_on_outputc C s$ | j d�}| jdg|jjd�� d S )Nz<a id='abc def'>zabc defr )r$ r r/ Zget_attribute_list)r r$ r r r �test_get_attribute_list s z/TestCDAtaListAttributes.test_get_attribute_listc C s$ | j d�}| jddg|jd � d S )Nz(<form accept-charset="ISO-8859-1 UTF-8">z ISO-8859-1zUTF-8zaccept-charset)r$ r Zform)r r$ r r r �test_accept_charset! s z+TestCDAtaListAttributes.test_accept_charsetc C s$ d}| j |�}| jd|jd � d S )Nz)<a accept-charset="ISO-8859-1 UTF-8"></a>zISO-8859-1 UTF-8zaccept-charset)r$ r r/ )r rk r$ r r r �-test_cdata_attribute_applying_only_to_one_tag% s zETestCDAtaListAttributes.test_cdata_attribute_applying_only_to_one_tagc s6 | j d�j� | jd � j� � fdd�}| jt|� d S )Nr� c s d� _ d S )NrB )rN r )r r r �t0 s zJTestCDAtaListAttributes.test_string_has_immutable_name_property.<locals>.t)r$ r r rN r� �AttributeError)r rO r )r r �'test_string_has_immutable_name_property- s z?TestCDAtaListAttributes.test_string_has_immutable_name_propertyN) r r r r1 rH rI rJ rK rL rM rN rQ r r r r rG s rG c s` e Zd ZdZ� fdd�Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z � Z S )�TestPersistencez*Testing features like pickle and deepcopy.c s& t t| �j� d| _| j| j�| _d S )Nay <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/transitional.dtd"> <html> <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> <title>Beautiful Soup: We called him Tortoise because he taught us.</title> <link rev="made" href="mailto:leonardr@segfault.org"> <meta name="Description" content="Beautiful Soup: an HTML parser optimized for screen-scraping."> <meta name="generator" content="Markov Approximation 1.4 (module: leonardr)"> <meta name="author" content="Leonard Richardson"> </head> <body> <a href="foo">foo</a> <a href="foo"><b>bar</b></a> </body> </html>)rQ rR rR Zpager$ rS )r )rT r r rR 7 s zTestPersistence.setUpc C s@ t j| jd�}t j|�}| j|jt� | j|j� | jj� � d S )Nr+ )�pickle�dumpsrS �loadsr rT r rj )r �dumped�loadedr r r �!test_pickle_and_unpickle_identityK s z1TestPersistence.test_pickle_and_unpickle_identityc C s& t j| j�}| j|j� | jj� � d S )N)�copy�deepcopyrS r rj )r Zcopiedr r r �test_deepcopy_identityS s z&TestPersistence.test_deepcopy_identityc C s: t dd�}|j}|j� }| jdt|�� | j||j� d S )Ns <p> </p>z html.parseru <p> </p>)r Zoriginal_encoding�__copy__r r( )r r$ �encodingrY r r r � test_copy_preserves_encodingX s z,TestPersistence.test_copy_preserves_encodingc C s> d}| j |�}tj|tj�}tj|�}| j|j� |j� � d S )Nu <b>☃</b>)r$ rS rT ZHIGHEST_PROTOCOLrU r rj )r r� r$ rV rW r r r �test_unicode_pickle_ s z#TestPersistence.test_unicode_picklec C sz d}| j |�}|jdd�}tj|�}| j||� | jd |j� | jd |j� | jd |j� | jd |j� | jd |j� d S )Nz<b>Foo<a></a></b><b>Bar</b>r4 )r ) r$ r% rY r r� r� �assertNotEqualr� r� )r r� r$ �s1�s2r r r �1test_copy_navigablestring_is_not_attached_to_treeg s zATestPersistence.test_copy_navigablestring_is_not_attached_to_treec C s> d}| j |�}|j}tj|�}| j||� | jt|t�� d S )Nz<b><!--Foo--></b>)r$ r rY r rH r� r )r r� r$ ra rb r r r �0test_copy_navigablestring_subclass_has_same_types s z@TestPersistence.test_copy_navigablestring_subclass_has_same_typec C s( d}| j |�}tj|�}| j||� d S )Nz)<div><b>Foo<a></a></b><b>Bar</b></div>end)r$ rY r )r r� r$ Z soup_copyr r r �test_copy_entire_soup{ s z%TestPersistence.test_copy_entire_soupc C s� d}| j |�}|j}tj|�}| jt|�t|�� | j||� | j||k� | jd |j� | jd |j� | jd |jdd�j � | j d |jdd�j � d S )Nz)<div><b>Foo<a></a></b><b>Bar</b></div>endZBar)r ) r$ ra rY r r( r8 r� r� r% r� r` )r r� r$ ra Zdiv_copyr r r � test_copy_tag_copies_contents� s z-TestPersistence.test_copy_tag_copies_contents)r r r r1 rR rX r[ r^ r_ rc rd re rf rc r r )rT r rR 4 s rR c @ s� e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd � Zd d � Zd d � Zd!S )"�TestSubstitutionsc C s0 d}| j |�}|jdd�}| j|| jd�� d S )Nu# <b><<Sacré bleu!>></b>�minimal)� formatter)r$ rj r r� )r r� r$ �decodedr r r �!test_default_formatter_is_minimal� s z3TestSubstitutions.test_default_formatter_is_minimalc C s0 d}| j |�}|jdd�}| j|| jd�� d S )Nu' <br><b><<Sacré bleu!>></b>r� )ri z.<br/><b><<Sacré bleu!>></b>)r$ rj r r� )r r� r$ rj r r r �test_formatter_html� s z%TestSubstitutions.test_formatter_htmlc C s0 d}| j |�}|jdd�}| j|| jd�� d S )Nu' <br><b><<Sacré bleu!>></b>Zhtml5)ri z-<br><b><<Sacré bleu!>></b>)r$ rj r r� )r r� r$ rj r r r �test_formatter_html5� s z&TestSubstitutions.test_formatter_html5c C s0 d}| j |�}|jdd�}| j|| jd�� d S )Nu# <b><<Sacré bleu!>></b>rh )ri )r$ rj r r� )r r� r$ rj r r r �test_formatter_minimal� s z(TestSubstitutions.test_formatter_minimalc C s0 d}| j |�}|jd d�}| j|| jd�� d S )Nu# <b><<Sacré bleu!>></b>)ri u <b><<Sacré bleu!>></b>)r$ rj r r� )r r� r$ rj r r r �test_formatter_null� s z%TestSubstitutions.test_formatter_nullc C s4 d}| j |�}|jdd� d�}| j|| jd�� d S )Nz!<b><foo></b><b>bar</b><br/>c S s | j � S )N)�upper)�xr r r �<lambda>� s z9TestSubstitutions.test_formatter_custom.<locals>.<lambda>)ri z<b><FOO></b><b>BAR</b><br>)r$ rj r r� )r r� r$ rj r r r �test_formatter_custom� s z'TestSubstitutions.test_formatter_customc C s� d}| j |�}|j}d}| j||j� � | j||jdd�� d}| j||jdd�� | j||jd d�� d}| j||jdd � d�� d S ) Nu% <a href="http://a.com?a=b&c=é">e</a>u) <a href="http://a.com?a=b&c=é">e</a>rh )ri z/<a href="http://a.com?a=b&c=é">e</a>r� u% <a href="HTTP://A.COM?A=B&C=É">E</a>c S s | j � S )N)rp )rq r r r rr � s zMTestSubstitutions.test_formatter_is_run_on_attribute_values.<locals>.<lambda>)r$ r/ r rj )r r� r$ r/ Zexpect_minimalZ expect_htmlZ expect_upperr r r �)test_formatter_is_run_on_attribute_values� s z;TestSubstitutions.test_formatter_is_run_on_attribute_valuesc C s$ d}t |d�j� }| jd|k� d S )NzO <script type="text/javascript"> console.log("< < hey > > "); </script> z html.parsers < < hey > >)r ri rH )r r� �encodedr r r �2test_formatter_skips_script_tag_for_html_documents� s zDTestSubstitutions.test_formatter_skips_script_tag_for_html_documentsc C s$ d}t |d�j� }| jd|k� d S )NzF <style type="text/css"> console.log("< < hey > > "); </style> z html.parsers < < hey > >)r ri rH )r r� ru r r r �1test_formatter_skips_style_tag_for_html_documents� s zCTestSubstitutions.test_formatter_skips_style_tag_for_html_documentsc C s | j d�}| jd|jj� � d S )Nz*<div> foo <pre> bar </pre> baz z/<div> foo <pre> bar </pre> baz </div>)r$ r ra �prettify)r r$ r r r �,test_prettify_leaves_preformatted_text_alone� s z>TestSubstitutions.test_prettify_leaves_preformatted_text_alonec C s, t dd�}|jdd� d�}| jd|k� d S )Nz <html><body>foo</body></html>z html.parserc S s | j � S )N)rp )rq r r r rr � s zLTestSubstitutions.test_prettify_accepts_formatter_function.<locals>.<lambda>)ri ZFOO)r rx rH )r r$ Zprettyr r r �(test_prettify_accepts_formatter_function� s z:TestSubstitutions.test_prettify_accepts_formatter_functionc C s" | j d�}| jtt|j� �� d S )Nz<a></a>)r$ r r( �typerx )r r$ r r r �(test_prettify_outputs_unicode_by_default� s z:TestSubstitutions.test_prettify_outputs_unicode_by_defaultc C s$ | j d�}| jtt|jd��� d S )Nz<a></a>zutf-8)r$ r �bytesr{ rx )r r$ r r r � test_prettify_can_encode_data s z/TestSubstitutions.test_prettify_can_encode_datac C s0 d}| j |�}|jjd�}| j||jd�� d S )Nu <b>Sacré bleu!</b>zutf-8)r$ r" ri r )r r� r$ ru r r r �,test_html_entity_substitution_off_by_default s z>TestSubstitutions.test_html_entity_substitution_off_by_defaultc C s� d}| j |�}| j|jd d� |jd�}| jd|k� |jd�}| jd|k� |jd�}| jd |k� |jd �jd �}| jd |k� d S ) NzE<meta content="text/html; charset=x-sjis" http-equiv="Content-type"/>Zcontentztext/html; charset=x-sjiszutf-8s charset=utf-8�euc_jps charset=euc_jpz shift-jiss charset=shift-jiszutf-16zcharset=utf-16)r$ r �metari rH rj )r Zmeta_tagr$ �utf_8r� � shift_jisZutf_16_ur r r �test_encoding_substitution s z,TestSubstitutions.test_encoding_substitutionc C s2 d}t d�}| j||d�}| j|jd jd� d S )Nz`<head><meta content="text/html; charset=x-sjis" http-equiv="Content-type"/></head><pre>foo</pre>Zpre)Z parse_onlyr )r r$ r r� rN )r r� ry r$ r r r �;test_encoding_substitution_doesnt_happen_if_tag_is_strained$ s zMTestSubstitutions.test_encoding_substitution_doesnt_happen_if_tag_is_strainedN)r r r rk rl rm rn ro rs rt rv rw ry rz r| r~ r r� r� r r r r rg � s rg c @ sP e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dS )� TestEncodingz0Test the ability to encode objects into strings.c C s. d}| j |�}| j|jjjd�djd�� d S )Nu <b>☃</b>zutf-8u ☃)r$ r r" r ri )r r� r$ r r r �"test_unicode_string_can_be_encoded2 s z/TestEncoding.test_unicode_string_can_be_encodedc C s, d}| j |�}| j|jjd�|jd�� d S )Nu <b>☃</b>zutf-8)r$ r r" ri )r r� r$ r r r �1test_tag_containing_unicode_string_can_be_encoded8 s z>TestEncoding.test_tag_containing_unicode_string_can_be_encodedc C s& d}| j |�}| j|jjd�d� d S )Nu <b>☃</b>�asciis <b>☃</b>)r$ r r" ri )r r� r$ r r r �<test_encoding_substitutes_unrecognized_characters_by_default> s zITestEncoding.test_encoding_substitutes_unrecognized_characters_by_defaultc C s& d}| j |�}| jt|jddd� d S )Nu <b>☃</b>r� �strict)�errors)r$ r� �UnicodeEncodeErrorri )r r� r$ r r r � test_encoding_can_be_made_strictC s z-TestEncoding.test_encoding_can_be_made_strictc C s$ d}| j |�}| jd|jj� � d S )Nu <b>☃</b>u ☃)r$ r r" Zdecode_contents)r r� r$ r r r �test_decode_contentsI s z!TestEncoding.test_decode_contentsc C s. d}| j |�}| jdjd�|jjdd�� d S )Nu <b>☃</b>u ☃rg )r] )r$ r ri r" Zencode_contents)r r� r$ r r r �test_encode_contentsN s z!TestEncoding.test_encode_contentsc C s* d}| j |�}| jdjd�|jj� � d S )Nu <b>☃</b>u ☃rg )r$ r ri r" ZrenderContents)r r� r$ r r r � test_deprecated_renderContentsU s z+TestEncoding.test_deprecated_renderContentsc C s8 d}| j |�}tr$| j|t|�� n| jdt|�� d S )Nu <b>☃</b>s <b>\u2603</b>)r$ r r �repr)r r� r$ r r r � test_repr[ s zTestEncoding.test_reprN) r r r r1 r� r� r� r� r� r� r� r� r r r r r� / s r� c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S ) � TestNavigableStringSubclassesc C sX | j d�}td�}|jd|� | jt|�d� | j|jdd�d� | j|jd d� d S )Nr� rB r= z<![CDATA[foo]]>)r) r )r$ r r� r r( r% r� )r r$ r+ r r r � test_cdatae s z(TestNavigableStringSubclasses.test_cdatac sV d� _ � fdd�}� jd�}td�}|jd|� � jd|j|d�� � jd� j � d S ) zkText inside a CData object is passed into the formatter. But the return value is ignored. r c s � j d7 _ dS )Nr= zBITTER FAILURE)�count)�args)r r r � incrementv s zNTestNavigableStringSubclasses.test_cdata_is_never_formatted.<locals>.incrementr� z<><><>r= s <![CDATA[<><><>]]>)ri N)r� r$ r r� r ri )r r� r$ r+ r )r r � test_cdata_is_never_formattedo s z;TestNavigableStringSubclasses.test_cdata_is_never_formattedc C s2 t d�}| jd�}|jd|� | j|j� d� d S )NrB r� r= s <!DOCTYPE foo> )r r$ r� r ri )r Zdoctyper$ r r r � test_doctype_ends_in_newline� s z:TestNavigableStringSubclasses.test_doctype_ends_in_newlinec C s t d�}| jd|j� � d S )NrB z<?foo?>)r r Z output_ready)r r r r r �test_declaration� s z.TestNavigableStringSubclasses.test_declarationN)r r r r� r� r� r� r r r r r� c s r� c @ s� e Zd ZdZdd� Zdd� ZeZdd� Zdd � Zd d � Z d d � Z dd� Z dd� Z dd� Z dd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d!� Zd"d#� Zd$d%� Zd&d'� Zd(d)� Zd*d+� Zd,d-� Zd.d/� Zd0d1� Z d2d3� Z d4d5� Z d6d7� Z d8d9� Z d:d;� Z!d<d=� Z"d>d?� Z#d@dA� Z$dBdC� Z%dDdE� Z&dFdG� Z'dHdI� Z(dJdK� Z)dLdM� Z*dNdO� Z+dPdQ� Z,dRdS� Z-dTdU� Z.dVdW� Z/dXdY� Z0dZd[� Z1d\d]� Z2d^d_� Z3d`da� Z4dbdc� Z5ddde� Z6dfdg� Z7dhdi� Z8djdk� Z9dldm� Z:dndo� Z;dpdq� Z<drds� Z=dtS )u�TestSoupSelectora� <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> <html> <head> <title>The title</title> <link rel="stylesheet" href="blah.css" type="text/css" id="l1"> </head> <body> <custom-dashed-tag class="dashed" id="dash1">Hello there.</custom-dashed-tag> <div id="main" class="fancy"> <div id="inner"> <h1 id="header1">An H1</h1> <p>Some text</p> <p class="onep" id="p1">Some more text</p> <h2 id="header2">An H2</h2> <p class="class1 class2 class3" id="pmulti">Another</p> <a href="http://bob.example.org/" rel="friend met" id="bob">Bob</a> <h2 id="header3">Another H2</h2> <a id="me" href="http://simonwillison.net/" rel="me">me</a> <span class="s1"> <a href="#" id="s1a1">span1a1</a> <a href="#" id="s1a2">span1a2 <span id="s1a2s1">test</span></a> <span class="span2"> <a href="#" id="s2a1">span2a1</a> </span> <span class="span3"></span> <custom-dashed-tag class="dashed" id="dash2"/> <div data-tag="dashedvalue" id="data1"/> </span> </div> <x id="xid"> <z id="zida"/> <z id="zidab"/> <z id="zidac"/> </x> <y id="yid"> <z id="zidb"/> </y> <p lang="en" id="lang-en">English</p> <p lang="en-gb" id="lang-en-gb">English UK</p> <p lang="en-us" id="lang-en-us">English US</p> <p lang="fr" id="lang-fr">French</p> </div> <div id="footer"> </div> c C s t | jd�| _d S )Nz html.parser)r �HTMLr$ )r r r r rR � s zTestSoupSelector.setUpc K sR dd� | j j|f|�D �}|j� |j� | j||d|dj|�dj|�f � d S )Nc S s g | ] }|d �qS )r r )r �elr r r r � s z2TestSoupSelector.assertSelects.<locals>.<listcomp>z$Selector %s, expected [%s], got [%s]z, )r$ �select�sortr �join)r �selector� expected_ids�kwargsZel_idsr r r r � s z TestSoupSelector.assertSelectsc G s" x |D ]\}}| j ||� qW d S )N)� assertSelect)r Ztestsr� r� r r r �assertSelectMultiple� s z%TestSoupSelector.assertSelectMultiplec C sF | j jd�}| jt|�d� | j|d jd� | j|d jdg� d S )Nrh r= r z The title)r$ r� r r, rN r� )r �elsr r r �test_one_tag_one� s z!TestSoupSelector.test_one_tag_onec C sX | j jd�}| jt|�d� x|D ]}| j|jd� q"W | j jd�}| jd|d � d S )Nra � �mainr )r$ r� r r, rN � select_one)r r� ra r� r r r �test_one_tag_many� s z"TestSoupSelector.test_one_tag_manyc C s | j jd�}| jd |� d S )NZnonexistenttag)r$ r� r )r �matchr r r �(test_select_one_returns_none_if_no_match� s z9TestSoupSelector.test_select_one_returns_none_if_no_matchc C s | j jd�}| jdddg� d S )Nzdiv div�inner�data1)r$ r� r )r r� r r r �test_tag_in_tag_one� s z$TestSoupSelector.test_tag_in_tag_onec C s& x dD ]}| j |ddddg� qW d S ) N�html div� html body div�body divr� r� r� �footer)r� r� r� )r )r r� r r r �test_tag_in_tag_many� s z%TestSoupSelector.test_tag_in_tag_manyc C sB | j ddgdd� | j dddgdd� | j dd ddd gd d� d S ) Nzhtml divr� r= )r: z html body divr� r+ zbody divr� r� r> )r )r r r r � test_limit� s zTestSoupSelector.test_limitc C s | j t| jjd��d� d S )N�delr )r r, r$ r� )r r r r �test_tag_no_match� s z"TestSoupSelector.test_tag_no_matchc C s | j t| jjd� d S )Nztag%t)r� r� r$ r� )r r r r �test_invalid_tag� s z!TestSoupSelector.test_invalid_tagc C s | j dddg� d S )Nzcustom-dashed-tag�dash1�dash2)r )r r r r �test_select_dashed_tag_ids� s z+TestSoupSelector.test_select_dashed_tag_idsc C s6 | j jd�}| j|d jd� | j|d d d� d S )Nz custom-dashed-tag[id="dash2"]r zcustom-dashed-tagr r� )r$ r� r rN )r Zdashedr r r �test_select_dashed_by_id� s z)TestSoupSelector.test_select_dashed_by_idc C s | j | jjd�d jd� d S )Nzbody > custom-dashed-tagr z Hello there.)r r$ r� r) )r r r r �test_dashed_tag_text s z%TestSoupSelector.test_dashed_tag_textc C s | j | jjd�| jjd�� d S )Nzcustom-dashed-tag)r r$ r� r- )r r r r �#test_select_dashed_matches_find_all s z4TestSoupSelector.test_select_dashed_matches_find_allc C s | j ddgfdddgf� d S )NZh1�header1Zh2�header2�header3)r� )r r r r �test_header_tags s z!TestSoupSelector.test_header_tagsc C sV xPd D ]H}| j j|�}| jt|�d� | j|d jd� | j|d d dg� qW d S ) N�.onep�p.onep� html p.onepr= r r� rm �onep)r� r� r� )r$ r� r r, rN )r r� r� r r r �test_class_one s z TestSoupSelector.test_class_onec C s | j jd�}| jt|�d� d S )Nzdiv.onepr )r$ r� r r, )r r� r r r �test_class_mismatched_tag s z*TestSoupSelector.test_class_mismatched_tagc C s xdD ]}| j |dg� qW d S )N� div#inner�#inner� div div#innerr� )r� r� r� )r )r r� r r r � test_one_id s z TestSoupSelector.test_one_idc C s | j jd�}| jt|�d� d S )Nz #doesnotexistr )r$ r� r r, )r r� r r r � test_bad_id s z TestSoupSelector.test_bad_idc C sf | j jd�}| jt|�d� x|D ]}| j|jd� q"W | j|d d dg� | j|d jd�� d S )Nz div#inner pr9 r� r= rm r� r )r$ r� r r, rN r8 r7 )r r� r� r r r �test_items_in_id# s z!TestSoupSelector.test_items_in_idc C s* x$dD ] }| j t| jj|��d� qW d S )N� div#main del�div#main div.oops� div div#mainr )r� r� r� )r r, r$ r� )r r� r r r �test_a_bunch_of_emptys+ s z'TestSoupSelector.test_a_bunch_of_emptysc C s xd D ]}| j |d g� qW d S ) N�.class1�p.class1�.class2�p.class2�.class3�p.class3� html p.class2�div#inner .class2�pmulti)r� r� r� r� r� r� r� r� )r )r r� r r r �test_multi_class_support/ s z)TestSoupSelector.test_multi_class_supportc C s xdD ]}| j |dg� qW d S )N�.class1.class3�.class3.class2�.class1.class2.class3r� )r� r� r� )r )r r� r r r �test_multi_class_selection4 s z+TestSoupSelector.test_multi_class_selectionc C s" | j dddg� | j ddg� d S )Nz.s1 > a�s1a1�s1a2z .s1 > a span�s1a2s1)r )r r r r �test_child_selector9 s z$TestSoupSelector.test_child_selectorc C s | j ddg� d S )Nz.s1 > a#s1a2 spanr� )r )r r r r �test_child_selector_id= s z'TestSoupSelector.test_child_selector_idc C st | j ddgfddgfddgfddgfddgfddgfd dgfd g fd dgfd dgfd dgfdg fdg fdg f� d S )Nzp[class="onep"]r� z p[id="p1"]z[class="onep"]z [id="p1"]zlink[rel="stylesheet"]�l1zlink[type="text/css"]zlink[href="blah.css"]zlink[href="no-blah.css"]z[rel="stylesheet"]z[type="text/css"]z[href="blah.css"]z[href="no-blah.css"]zp[href="no-blah.css"])r� )r r r r �test_attribute_equals@ s z&TestSoupSelector.test_attribute_equalsc C s\ | j ddgfddgfddgfddgfddgfddgfdd gfd d gfd d gfd d gf� d S ) Nzp[class~="class1"]r� zp[class~="class2"]zp[class~="class3"]z[class~="class1"]z[class~="class2"]z[class~="class3"]za[rel~="friend"]�bobz a[rel~="met"]z[rel~="friend"]z [rel~="met"])r� )r r r r �test_attribute_tildeR s z%TestSoupSelector.test_attribute_tildec C sv | j ddgfddgfdg fdg fdg fddgfdd d gfd d d gfd d dgfdd dgfddgfdd gfddgf� d S )Nz[rel^="style"]r� zlink[rel^="style"]znotlink[rel^="notstyle"]z[rel^="notstyle"]zlink[rel^="notstyle"]zlink[href^="bla"]za[href^="http://"]r� �mez[href^="http://"]z [id^="p"]r� r� z [id^="m"]r� z div[id^="m"]z a[id^="m"]zdiv[data-tag^="dashed"]r� )r� )r r r r �test_attribute_startswith` s z*TestSoupSelector.test_attribute_startswithc C sH | j ddgfddgfddgfdddddd d d d gfd dgfdg f� d S )Nz[href$=".css"]r� zlink[href$=".css"]z link[id$="1"]z [id$="1"]r� r� r� r� �s2a1r� r� z div[id$="1"]z[id$="noending"])r� )r r r r �test_attribute_endswithq s z(TestSoupSelector.test_attribute_endswithc C s� | j ddgfddgfdg fdg fdg fddgfdd d gfd d d gfddgfdd gfddgfddgfddgfdddd ddddddg fd dgfd g fd d d dgfd d d gfd dgfd!dd"gfd#d"gfd$dgf� d S )%Nz[rel*="style"]r� zlink[rel*="style"]znotlink[rel*="notstyle"]z[rel*="notstyle"]zlink[rel*="notstyle"]zlink[href*="bla"]z[href*="http://"]r� r� z [id*="p"]r� r� z div[id*="m"]r� z a[id*="m"]z[href*=".css"]zlink[href*=".css"]z link[id*="1"]z [id*="1"]r� r� r� r� r� r� r� z div[id*="1"]z[id*="noending"]z [href*="."]z a[href*="."]zlink[href*="."]z div[id*="n"]r� z div[id*="nn"]zdiv[data-tag*="edval"])r� )r r r r �test_attribute_contains{ s. z(TestSoupSelector.test_attribute_containsc C s2 | j ddddgfddddgfddgfdg f� d S ) Nz p[lang|="en"]zlang-enz lang-en-gbz lang-en-usz [lang|="en"]z p[lang|="fr"]zlang-frz p[lang|="gb"])r� )r r r r � test_attribute_exact_or_hypen� s z.TestSoupSelector.test_attribute_exact_or_hypenc C sV | j ddddgfddgfdddgfddd d d gfd d dgfdg fdg fddgf� d S )Nz[rel]r� r� r� z link[rel]za[rel]z[lang]zlang-enz lang-en-gbz lang-en-uszlang-frzp[class]r� r� z[blah]zp[blah]z div[data-tag]r� )r� )r r r r �test_attribute_exists� s z&TestSoupSelector.test_attribute_existsc C s, d}t |d�}|jd�\}| jd|j� d S )Nz]<div style="display: wrong">nope</div> <div style="display: right">yes</div> z html.parserzdiv[style="display: right"]�yes)r r� r r )r r� r$ Zchosenr r r �"test_quoted_space_in_selector_name� s z3TestSoupSelector.test_quoted_space_in_selector_namec C s( | j t| jjd� | j t| jjd� d S )Nza:no-such-pseudoclassza:nth-of-type(a))r� r r$ r� )r r r r � test_unsupported_pseudoclass� s z-TestSoupSelector.test_unsupported_pseudoclassc C s� | j jd�}| jt|�d� | j|d jd� | j jd�}| jt|�d� | j|d jd� | j jd�}| jt|�d� | jt| j jd� d S ) Nzdiv#inner p:nth-of-type(1)r= r z Some textzdiv#inner p:nth-of-type(3)ZAnotherzdiv#inner p:nth-of-type(4)zdiv p:nth-of-type(0))r$ r� r r, r r� r� )r r� r r r �test_nth_of_type� s z!TestSoupSelector.test_nth_of_typec C s2 | j jd�}| jt|�d� | j|d jd� d S )Nz div#inner > p:nth-of-type(1)r= r z Some text)r$ r� r r, r )r r� r r r �"test_nth_of_type_direct_descendant� s z3TestSoupSelector.test_nth_of_type_direct_descendantc C s | j ddg� d S )Nz#inner > p:nth-of-type(2)r� )r )r r r r �"test_id_child_selector_nth_of_type� s z3TestSoupSelector.test_id_child_selector_nth_of_typec C s. | j jddd�}|jd�}| j|ddg� d S )Nra r� )r r� r� )r$ r% r� r )r r� �selectedr r r �test_select_on_element� s z'TestSoupSelector.test_select_on_elementc C s | j ddg� | j dg � d S )Nz .fancy #innerr� z.normal #inner)r )r r r r �test_overspecified_child_id� s z,TestSoupSelector.test_overspecified_child_idc C sB | j ddg� | j ddg� | j ddg� | jg | jjd�� d S )Nz#p1 + h2r� z #p1 + h2 + pr� z#p1 + #header2 + .class1z#p1 + p)r r r$ r� )r r r r � test_adjacent_sibling_selector� s z/TestSoupSelector.test_adjacent_sibling_selectorc C sR | j dddg� | j ddg� | j ddg� | j ddg� | jg | jjd�� d S ) Nz#p1 ~ h2r� r� z#p1 ~ #header2z #p1 ~ h2 + ar� z#p1 ~ h2 + [rel="me"]z #inner ~ h2)r r r$ r� )r r r r � test_general_sibling_selector� s z.TestSoupSelector.test_general_sibling_selectorc C s | j t| jjd� d S )Nzh1 >)r� r� r$ r� )r r r r �test_dangling_combinator� s z)TestSoupSelector.test_dangling_combinatorc C s | j ddddg� d S )Nz p[lang] ~ pz lang-en-gbz lang-en-uszlang-fr)r )r r r r �2test_sibling_combinator_wont_select_same_tag_twice� s zCTestSoupSelector.test_sibling_combinator_wont_select_same_tag_twicec C s | j dddg� d S )Nzx, y�xid�yid)r )r r r r �test_multiple_select� s z%TestSoupSelector.test_multiple_selectc C s | j dddg� d S )Nzx,yr r )r )r r r r �"test_multiple_select_with_no_space� s z3TestSoupSelector.test_multiple_select_with_no_spacec C s | j dddg� d S )Nzx, yr r )r )r r r r �$test_multiple_select_with_more_space� s z5TestSoupSelector.test_multiple_select_with_more_spacec C s | j ddg� d S )Nzx, xr )r )r r r r � test_multiple_select_duplicated s z0TestSoupSelector.test_multiple_select_duplicatedc C s | j dddg� d S )Nzx, y ~ p[lang=fr]r zlang-fr)r )r r r r � test_multiple_select_sibling s z-TestSoupSelector.test_multiple_select_siblingc C s | j dddg� d S )Nzx, y > zr �zidb)r )r r r r �.test_multiple_select_tag_and_direct_descendant s z?TestSoupSelector.test_multiple_select_tag_and_direct_descendantc C s | j dddddddg� d S )Nz div > x, y, zr r �zidar �zidab�zidac)r )r r r r �/test_multiple_select_direct_descendant_and_tags s z@TestSoupSelector.test_multiple_select_direct_descendant_and_tagsc C s | j dddddddg� d S )Nz div x,y, zr r r r r r )r )r r r r �(test_multiple_select_indirect_descendant s z9TestSoupSelector.test_multiple_select_indirect_descendantc C s( | j t| jjd� | j t| jjd� d S )Nz,x, yzx,,y)r� r� r$ r� )r r r r � test_invalid_multiple_select s z-TestSoupSelector.test_invalid_multiple_selectc C s | j dddg� d S )Nzp[lang=en], p[lang=en-gb]zlang-enz lang-en-gb)r )r r r r �test_multiple_select_attrs s z+TestSoupSelector.test_multiple_select_attrsc C s | j ddddg� d S )Nz*x, y > z[id=zida], z[id=zidab], z[id=zidb]r r r )r )r r r r �test_multiple_select_ids s z)TestSoupSelector.test_multiple_select_idsc C s | j dddg� d S )Nzbody > div > x, y > zr r )r )r r r r �test_multiple_select_nested s z,TestSoupSelector.test_multiple_select_nestedc C sR d}t |d�}|jd�}| jdt|�� x$|jddgd�D ]}||ks:t�q:W d S )Nz3<div class="c1"/><div class="c2"/><div class="c1"/>z html.parserz.c1, .c2r9 Zc1Zc2)ro )r r� r r, r- r� )r r� r$ r� r� r r r � test_select_duplicate_elements s z/TestSoupSelector.test_select_duplicate_elementsN)>r r r r� rR r r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r r r r r r r r r r r r r r r� � sv 1 r� )2r1 Zpdbr rY rS r6 r1 Zbs4r Z bs4.builderr r Z bs4.elementr r r r r r r r Z bs4.testingr r r� r� Z LXML_PRESENTr r! r2 rL rP rd r� r� r� r� r� r� r� r� r� r� r- rG rR rg r� r� r� r r r r �<module> sN ( ;O 83(3--* 6n+a 4* tests/__pycache__/__init__.cpython-36.opt-1.pyc 0000644 00000000225 15246412105 0015241 0 ustar 00 3 ��CN � @ s d Z dS )zThe beautifulsoup tests.N)�__doc__� r r � /usr/lib/python3.6/__init__.py�<module> s tests/__pycache__/test_docs.cpython-36.opt-1.pyc 0000644 00000000565 15246412105 0015500 0 ustar 00 3 ���O+ � @ sD d Z eZdgZddlZddlZddlZddlZejej B ej B Z dS )zTest harness for doctests.Zadditional_tests� N) �__doc__�typeZ __metaclass__�__all__�atexitZdoctest�osZunittest�ELLIPSISZNORMALIZE_WHITESPACEZ REPORT_NDIFFZ DOCTEST_FLAGS� r r � /usr/lib/python3.6/test_docs.py�<module> s tests/__pycache__/test_builder_registry.cpython-36.opt-1.pyc 0000644 00000011553 15246412105 0020125 0 ustar 00 3 c�T� � @ s� d Z ddlZddlZddlmZ ddlmZmZm Z yddlm Z dZ W n e k r` dZ Y nX yddlm Z mZ dZW n e k r� dZY nX G d d � d ej�ZG d d � d ej�ZdS ) z Tests of the builder registry.� N)� BeautifulSoup)�builder_registry�HTMLParserTreeBuilder�TreeBuilderRegistry)�HTML5TreeBuilderTF)�LXMLTreeBuilderForXML�LXMLTreeBuilderc @ s0 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd S ) �BuiltInRegistryTestz@Test the built-in registry with the default builders registered.c C s` t r| jtjdd�t� t r0| jtjdd�t� | jtjdd�t� tr\| jtjdd�t� d S )N�fast�htmlZ permissive�xml�strict�html5lib) � LXML_PRESENT� assertEqual�registry�lookupr r r �HTML5LIB_PRESENTr )�self� r �+/usr/lib/python3.6/test_builder_registry.py�test_combination s z$BuiltInRegistryTest.test_combinationc C sj t r*| jtjd�t� | jtjd�t� n<| jtjd�d � trT| jtjd�t� n| jtjd�t� d S )Nr r ) r r r r r r r r r )r r r r �test_lookup_by_markup_type. s z.BuiltInRegistryTest.test_lookup_by_markup_typec C sX t r,| jtjdd�t� | jtjdd�t� trB| jtjd�t� | jtjd�t� d S )NZlxmlr r r z html.parser) r r r r r r r r r )r r r r �test_named_library9 s z&BuiltInRegistryTest.test_named_libraryc C sJ t jdd��"}tddd� tdddgd� W d Q R X | jttddd� d S )NT)�record� r )�featuresr zno-such-feature)�warnings�catch_warningsr Z assertRaises� ValueError)r �wr r r �*test_beautifulsoup_constructor_does_lookupF s z>BuiltInRegistryTest.test_beautifulsoup_constructor_does_lookupN)�__name__� __module__� __qualname__�__doc__r r r r! r r r r r s r c @ sX e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dS )� RegistryTestz.Test the TreeBuilderRegistry class in general.c C s t � | _d S )N)r r )r r r r �setUpY s zRegistryTest.setUpc G s, t ddj|� tfd|i�}| jj|� |S )NZBuilder_�_r )�type�join�objectr �register)r Z feature_list�clsr r r �builder_for_features\ s z!RegistryTest.builder_for_featuresc C s2 | j � }| j| jjd�d � | j| jj� |� d S )N�foo)r. r r r )r �builderr r r � test_register_with_no_featuresc s z+RegistryTest.test_register_with_no_featuresc C s8 | j dd�}| j| jjd�|� | j| jjd�|� d S )Nr/ �bar)r. r r r )r r0 r r r �0test_register_with_features_makes_lookup_succeedn s z=RegistryTest.test_register_with_features_makes_lookup_succeedc C s$ | j dd�}| j| jjd�d � d S )Nr/ r2 �baz)r. r r r )r r0 r r r �4test_lookup_fails_when_no_builder_implements_features s zARegistryTest.test_lookup_fails_when_no_builder_implements_featurec C s* | j d�}| j d�}| j| jj� |� d S )Nr/ r2 )r. r r r )r �builder1�builder2r r r �Ctest_lookup_gets_most_recent_registration_when_no_feature_specifiedw s zPRegistryTest.test_lookup_gets_most_recent_registration_when_no_feature_specifiedc C s | j | jj� d � d S )N)r r r )r r r r �2test_lookup_fails_when_no_tree_builders_registered| s z?RegistryTest.test_lookup_fails_when_no_tree_builders_registeredc C sv | j d�}| j d�}| j ddd�}| j ddd�}| j d�}| j d�}| j| jjdd�|� | j| jjddd�|� d S )Nr/ r2 r4 Zquux)r. r r r )r Zhas_oneZ has_the_otherZhas_both_earlyZ has_both_lateZ lacks_oner r r �<test_lookup_gets_most_recent_builder_supporting_all_features s zIRegistryTest.test_lookup_gets_most_recent_builder_supporting_all_featuresc C s2 | j dd�}| j dd�}| j| jjdd�d � d S )Nr/ r2 r4 )r. r r r )r r6 r7 r r r �:test_lookup_fails_when_cannot_reconcile_requested_features� s zGRegistryTest.test_lookup_fails_when_cannot_reconcile_requested_featuresN) r" r# r$ r% r' r. r1 r3 r5 r8 r9 r: r; r r r r r& V s r& )r% Zunittestr Zbs4r Z bs4.builderr r r r r r � ImportErrorr r r ZTestCaser r& r r r r �<module> s 9 tests/__pycache__/test_html5lib.cpython-36.pyc 0000644 00000012343 15246412105 0015326 0 ustar 00 3 6]+ � @ s� d Z ddlZyddlmZ dZW n& ek rF Z z dZW Y ddZ[X nX ddlmZ ddl m Z m Z m Z e e d�G d d � d e e ��Z dS ) zDTests to ensure that the html5lib tree builder generates good trees.� N)�HTML5TreeBuilderTF)� SoupStrainer)�HTML5TreeBuilderSmokeTest�SoupTest�skipIfz?html5lib seems not to be present, not testing its tree builder.c @ sl e Zd ZdZedd� �Zdd� Zdd� Zdd � Zd d � Z d d � Z dd� Z dd� Z dd� Z dd� Zdd� ZdS )�HTML5LibBuilderSmokeTestz"See ``HTML5TreeBuilderSmokeTest``.c C s t � S )N)r )�self� r �#/usr/lib/python3.6/test_html5lib.py�default_builder s z(HTML5LibBuilderSmokeTest.default_builderc C sd t d�}d}tjdd��}| j||d�}W d Q R X | j|j� | j|�� | jdt|d j �k� d S )N�bz <p>A <b>bold</b> statement.</p>T)�record)Z parse_onlyz4the html5lib tree builder doesn't support parse_onlyr ) r �warnings�catch_warnings�soup� assertEqual�decodeZ document_forZ assertTrue�str�message)r Zstrainer�markup�wr r r r �test_soupstrainer s z*HTML5LibBuilderSmokeTest.test_soupstrainerc C s d}| j |d� | j d� dS )z8html5lib inserts <tbody> tags where other parsers don't.z[<table id="1"><tr><td>Here's another table:<table id="2"><tr><td>foo</td></tr></table></td>z�<table id="1"><tbody><tr><td>Here's another table:<table id="2"><tbody><tr><td>foo</td></tr></tbody></table></td></tr></tbody></table>z{<table><thead><tr><td>Foo</td></tr></thead><tbody><tr><td>Bar</td></tr></tbody><tfoot><tr><td>Baz</td></tr></tfoot></table>N)ZassertSoupEquals)r r r r r � test_correctly_nested_tables( s z5HTML5LibBuilderSmokeTest.test_correctly_nested_tablesc C s$ d}| j |�}| jd|jj� � d S )Nzy<?xml version="1.0" encoding="utf-8"?> <!DOCTYPE html> <html> <head> </head> <body> <p>foo</p> </body> </html>s <p>foo</p>)r r �p�encode)r r r r r r �(test_xml_declaration_followed_by_doctype< s zAHTML5LibBuilderSmokeTest.test_xml_declaration_followed_by_doctypec C s: d}| j |�}| jd|jj� � | jdt|jd��� d S )Nz%<p><em>foo</p> <p>bar<a></a></em></p>zD<body><p><em>foo</em></p><em> </em><p><em>bar<a></a></em></p></body>� r )r r �bodyr �len�find_all)r r r r r r �test_reparented_markupJ s z/HTML5LibBuilderSmokeTest.test_reparented_markupc C s: d}| j |�}| jd|jj� � | jdt|jd��� d S )Nz&<p><em>foo</p> <p>bar<a></a></em></p> zE<body><p><em>foo</em></p><em> </em><p><em>bar<a></a></em></p> </body>r r )r r r r r r )r r r r r r �+test_reparented_markup_ends_with_whitespaceQ s zDHTML5LibBuilderSmokeTest.test_reparented_markup_ends_with_whitespacec C sL d}| j |�}|jdd�\}}|jd�\}}|j|ks:t�|j|ksHt�dS )z�Verify that we keep the two whitespace nodes in this document distinct when reparenting the adjacent <tbody> tags. z,<table> <tbody><tbody><ims></tbody> </table>� )�stringZtbodyN)r r � next_element�AssertionError)r r r Zspace1Zspace2Ztbody1Ztbody2r r r �<test_reparented_markup_containing_identical_whitespace_nodesW s zUHTML5LibBuilderSmokeTest.test_reparented_markup_containing_identical_whitespace_nodesc C s^ d}| j |�}|j}| jd|j� |jdd�}|jdd�d }| j||j� | j||j� d S )NzF<div><a>aftermath<p><noscript>target</noscript>aftermath</a></p></div>�target)r# Z aftermath� ���)r �noscriptr r$ �findr Zprevious_element)r r r r* r' Zfinal_aftermathr r r �*test_reparented_markup_containing_childrenb s zCHTML5LibBuilderSmokeTest.test_reparented_markup_containing_childrenc C s$ d}| j |�}t|�jd�s t�dS )z(Processing instructions become comments.s <?PITarget PIContent?>z<!--?PITarget PIContent?-->N)r r � startswithr% )r r r r r r �test_processing_instructionr s z4HTML5LibBuilderSmokeTest.test_processing_instructionc C s8 d}| j |�}|jd�\}}| j||� ||k s4t�d S )Ns <a class="my_class"><p></a>�a)r r r r% )r r r Za1Za2r r r �test_cloned_multivalue_nodex s z4HTML5LibBuilderSmokeTest.test_cloned_multivalue_nodec C s$ d}| j |�}| jd|jj� � d S )Ns <table><td></tbody>Az><body>A<table><tbody><tr><td></td></tr></tbody></table></body>)r r r r )r r r r r r �test_foster_parenting s z.HTML5LibBuilderSmokeTest.test_foster_parentingN)�__name__� __module__� __qualname__�__doc__�propertyr r r r r r! r& r, r. r0 r1 r r r r r s r )r5 r Z bs4.builderr ZHTML5LIB_PRESENT� ImportError�eZ bs4.elementr Z bs4.testingr r r r r r r r �<module> s tests/__pycache__/test_lxml.cpython-36.pyc 0000644 00000005761 15246412105 0014570 0 ustar 00 3 6]U � @ s d Z ddlZddlZyddlZdZejjZW n* ek rV Z zdZdZW Y ddZ [ X nX erlddl m Z m Z ddl mZmZ ddlmZmZmZ ddlmZ dd lmZ dd lmZmZmZmZ ee d �G d d � d ee��Zee d�G dd� dee��Z dS )z@Tests to ensure that the lxml tree builder generates good trees.� NTF)�LXMLTreeBuilder�LXMLTreeBuilderForXML)� BeautifulSoup�BeautifulStoneSoup)�Comment�Doctype� SoupStrainer)�skipIf)�test_htmlparser)�HTMLTreeBuilderSmokeTest�XMLTreeBuilderSmokeTest�SoupTestr z;lxml seems not to be present, not testing its tree builder.c @ sP e Zd ZdZedd� �Zdd� Zdd� Zee p6e dk d �d d� �Z dd� Z dS )�LXMLTreeBuilderSmokeTestz!See ``HTMLTreeBuilderSmokeTest``.c C s t � S )N)r )�self� r � /usr/lib/python3.6/test_lxml.py�default_builder% s z(LXMLTreeBuilderSmokeTest.default_builderc C s( | j dd� | j dd� | j dd� d S )Nz <p>foo�bar</p>z <p>foobar</p>z <p>foo�bar</p>z<p>foo�bar</p>)ZassertSoupEquals)r r r r �test_out_of_range_entity) s z1LXMLTreeBuilderSmokeTest.test_out_of_range_entityc C s d S )Nr )r r r r �*test_entities_in_foreign_document_encoding1 s zCLXMLTreeBuilderSmokeTest.test_entities_in_foreign_document_encoding� � � r z@Skipping doctype test for old version of lxml to avoid segfault.c C s( | j d�}|jd }| jd|j� � d S )Nz <!DOCTYPE>r � )�soup�contents� assertEqual�strip)r r Zdoctyper r r �test_empty_doctype: s z+LXMLTreeBuilderSmokeTest.test_empty_doctypec C sN t jdd��}td�}W d Q R X | jdt|j�� | jdt|d j�k� d S )NT)�recordz<b />z<b/>z&BeautifulStoneSoup class is deprecatedr )�warnings�catch_warningsr r �str�bZ assertTrue�message)r �wr r r r �%test_beautifulstonesoup_is_xml_parserB s z>LXMLTreeBuilderSmokeTest.test_beautifulstonesoup_is_xml_parserN)r r r r ) �__name__� __module__� __qualname__�__doc__�propertyr r r r � LXML_PRESENT� LXML_VERSIONr r% r r r r r s r z?lxml seems not to be present, not testing its XML tree builder.c @ s e Zd ZdZedd� �ZdS )�LXMLXMLTreeBuilderSmokeTestz!See ``HTMLTreeBuilderSmokeTest``.c C s t � S )N)r )r r r r r P s z+LXMLXMLTreeBuilderSmokeTest.default_builderN)r& r' r( r) r* r r r r r r- J s r- )r ) r) �rer Z lxml.etreeZlxmlr+ Zetreer, � ImportError�eZ bs4.builderr r Zbs4r r Z bs4.elementr r r Z bs4.testingr Z bs4.testsr r r r r r- r r r r �<module> s0 ( tests/__pycache__/test_htmlparser.cpython-36.opt-1.pyc 0000644 00000004774 15246412105 0016737 0 ustar 00 3 Y=K[� � @ sf d Z ddlmZ ddlZddlmZmZ ddlmZ ddl m Z G dd� dee�Z G d d � d e�Z dS ) zGTests to ensure that the html.parser tree builder generates good trees.� )� set_traceN)�SoupTest�HTMLTreeBuilderSmokeTest)�HTMLParserTreeBuilder)�BeautifulSoupHTMLParserc @ s@ e Zd Zedd� �Zdd� Zdd� Zdd� Zd d � Zd d � Z d S )� HTMLParserTreeBuilderSmokeTestc C s t � S )N)r )�self� r �%/usr/lib/python3.6/test_htmlparser.py�default_builder s z.HTMLParserTreeBuilderSmokeTest.default_builderc C s d S )Nr )r r r r � test_namespaced_system_doctype s z=HTMLParserTreeBuilderSmokeTest.test_namespaced_system_doctypec C s d S )Nr )r r r r � test_namespaced_public_doctype s z=HTMLParserTreeBuilderSmokeTest.test_namespaced_public_doctypec C s< | j d�}tj|d�}tj|�}| jt|jt|j��� dS )zfUnlike most tree builders, HTMLParserTreeBuilder and will be restored after pickling. z <a><b>foo</a>� N)Zsoup�pickle�dumps�loadsZ assertTrue� isinstanceZbuilder�type)r ZtreeZdumpedZloadedr r r �test_builder_is_pickled s z6HTMLParserTreeBuilderSmokeTest.test_builder_is_pickledc C s | j dd� | j dd� d S )Nz<br></br><br></br><br></br>z<br/><br/><br/>z</br></br></br>� )�assertSoupEquals)r r r r �)test_redundant_empty_element_closing_tags! s zHHTMLParserTreeBuilderSmokeTest.test_redundant_empty_element_closing_tagsc C s | j dd� d S )Nz foo &# barzfoo &# bar)r )r r r r �test_empty_element% s z1HTMLParserTreeBuilderSmokeTest.test_empty_elementN) �__name__� __module__� __qualname__�propertyr r r r r r r r r r r s r c @ s e Zd Zdd� ZdS )�TestHTMLParserSubclassc C s t � }|jd� dS )zlVerify that our HTMLParser subclass implements error() in a way that doesn't cause a crash. z don't crashN)r �error)r �parserr r r � test_error, s z!TestHTMLParserSubclass.test_errorN)r r r r r r r r r + s r ) �__doc__Zpdbr r Z bs4.testingr r Z bs4.builderr Zbs4.builder._htmlparserr r r r r r r �<module> s ! tests/__pycache__/test_lxml.cpython-36.opt-1.pyc 0000644 00000005761 15246412105 0015527 0 ustar 00 3 6]U � @ s d Z ddlZddlZyddlZdZejjZW n* ek rV Z zdZdZW Y ddZ [ X nX erlddl m Z m Z ddl mZmZ ddlmZmZmZ ddlmZ dd lmZ dd lmZmZmZmZ ee d �G d d � d ee��Zee d�G dd� dee��Z dS )z@Tests to ensure that the lxml tree builder generates good trees.� NTF)�LXMLTreeBuilder�LXMLTreeBuilderForXML)� BeautifulSoup�BeautifulStoneSoup)�Comment�Doctype� SoupStrainer)�skipIf)�test_htmlparser)�HTMLTreeBuilderSmokeTest�XMLTreeBuilderSmokeTest�SoupTestr z;lxml seems not to be present, not testing its tree builder.c @ sP e Zd ZdZedd� �Zdd� Zdd� Zee p6e dk d �d d� �Z dd� Z dS )�LXMLTreeBuilderSmokeTestz!See ``HTMLTreeBuilderSmokeTest``.c C s t � S )N)r )�self� r � /usr/lib/python3.6/test_lxml.py�default_builder% s z(LXMLTreeBuilderSmokeTest.default_builderc C s( | j dd� | j dd� | j dd� d S )Nz <p>foo�bar</p>z <p>foobar</p>z <p>foo�bar</p>z<p>foo�bar</p>)ZassertSoupEquals)r r r r �test_out_of_range_entity) s z1LXMLTreeBuilderSmokeTest.test_out_of_range_entityc C s d S )Nr )r r r r �*test_entities_in_foreign_document_encoding1 s zCLXMLTreeBuilderSmokeTest.test_entities_in_foreign_document_encoding� � � r z@Skipping doctype test for old version of lxml to avoid segfault.c C s( | j d�}|jd }| jd|j� � d S )Nz <!DOCTYPE>r � )�soup�contents� assertEqual�strip)r r Zdoctyper r r �test_empty_doctype: s z+LXMLTreeBuilderSmokeTest.test_empty_doctypec C sN t jdd��}td�}W d Q R X | jdt|j�� | jdt|d j�k� d S )NT)�recordz<b />z<b/>z&BeautifulStoneSoup class is deprecatedr )�warnings�catch_warningsr r �str�bZ assertTrue�message)r �wr r r r �%test_beautifulstonesoup_is_xml_parserB s z>LXMLTreeBuilderSmokeTest.test_beautifulstonesoup_is_xml_parserN)r r r r ) �__name__� __module__� __qualname__�__doc__�propertyr r r r � LXML_PRESENT� LXML_VERSIONr r% r r r r r s r z?lxml seems not to be present, not testing its XML tree builder.c @ s e Zd ZdZedd� �ZdS )�LXMLXMLTreeBuilderSmokeTestz!See ``HTMLTreeBuilderSmokeTest``.c C s t � S )N)r )r r r r r P s z+LXMLXMLTreeBuilderSmokeTest.default_builderN)r& r' r( r) r* r r r r r r- J s r- )r ) r) �rer Z lxml.etreeZlxmlr+ Zetreer, � ImportError�eZ bs4.builderr r Zbs4r r Z bs4.elementr r r Z bs4.testingr Z bs4.testsr r r r r r- r r r r �<module> s0 ( tests/__pycache__/test_docs.cpython-36.pyc 0000644 00000000565 15246412105 0014541 0 ustar 00 3 ���O+ � @ sD d Z eZdgZddlZddlZddlZddlZejej B ej B Z dS )zTest harness for doctests.Zadditional_tests� N) �__doc__�typeZ __metaclass__�__all__�atexitZdoctest�osZunittest�ELLIPSISZNORMALIZE_WHITESPACEZ REPORT_NDIFFZ DOCTEST_FLAGS� r r � /usr/lib/python3.6/test_docs.py�<module> s tests/__pycache__/test_tree.cpython-36.opt-1.pyc 0000644 00000253731 15246412105 0015514 0 ustar 00 3 6]�8 � @ s d Z ddlmZ ddlZddlZddlZddlZddlmZ ddl m Z m Z ddl m Z mZmZmZmZmZmZmZ ddlmZmZ e jd�dk Ze jd �dk ZG d d � d e�ZG d d � d e�Z G dd� de�Z G dd� de�Z G dd� de�Z G dd� de�Z G dd� de�Z!G dd� de�Z"G dd� de�Z#G d d � d e#�Z$G d d � d e#�Z%G d d!� d!e�Z&G d"d#� d#e&�Z'G d$d%� d%e&�Z(G d&d'� d'e�Z)G d(d)� d)e�Z*G d*d+� d+e�Z+G d,d-� d-e�Z,G d.d/� d/e�Z-G d0d1� d1e�Z.G d2d3� d3e�Z/G d4d5� d5e�Z0G d6d7� d7e�Z1dS )8a8 Tests for Beautiful Soup's tree traversal methods. The tree traversal methods are the main advantage of using Beautiful Soup over just using a parser. Different parsers will build different Beautiful Soup trees given the same markup, but all Beautiful Soup trees can be traversed with the methods tested here. � )� set_traceN)� BeautifulSoup)�builder_registry�HTMLParserTreeBuilder)�PY3K�CData�Comment� Declaration�Doctype�NavigableString� SoupStrainer�Tag)�SoupTest�skipIfZxmlZlxmlc @ s e Zd Zdd� Zdd� ZdS )�TreeTestc C s | j dd� |D �|� dS )z�Make sure that the given tags have the correct text. This is used in tests that define a bunch of tags, each containing a single string, and then select certain strings by some mechanism. c S s g | ] }|j �qS � )�string)�.0�tagr r � /usr/lib/python3.6/test_tree.py� <listcomp>2 s z*TreeTest.assertSelects.<locals>.<listcomp>N)� assertEqual)�self�tags� should_matchr r r � assertSelects+ s zTreeTest.assertSelectsc C s | j dd� |D �|� dS )z�Make sure that the given tags have the correct IDs. This is used in tests that define a bunch of tags, each containing a single string, and then select certain strings by some mechanism. c S s g | ] }|d �qS )�idr )r r r r r r ; s z-TreeTest.assertSelectsIDs.<locals>.<listcomp>N)r )r r r r r r �assertSelectsIDs4 s zTreeTest.assertSelectsIDsN)�__name__� __module__� __qualname__r r r r r r r ) s r c @ s8 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd S ) �TestFindz�Basic tests of the find() method. find() just calls find_all() with limit=1, so it's not tested all that thouroughly here. c C s" | j d�}| j|jd�jd� d S )Nz <a>1</a><b>2</b><a>3</a><b>4</b>�b�2)�soupr �findr )r r$ r r r � test_find_tagE s zTestFind.test_find_tagc C s" | j d�}| j|jdd�d� d S )Nu <h1>Räksmörgås</h1>u Räksmörgås)r )r$ r r% )r r$ r r r �test_unicode_text_findI s z TestFind.test_unicode_text_findc C s, | j d�}t|� | jd|jdd�j� d S )Nu& <h1 id="Räksmörgås">here it is</h1>z here it isu Räksmörgås)r )r$ �strr r% �text)r r$ r r r �test_unicode_attribute_findM s z$TestFind.test_unicode_attribute_findc C s" | j d�}| jdt|j� �� dS )z)Test an optimization that finds all tags.z<a>foo</a><b>bar</b>� N)r$ r �len�find_all)r r$ r r r �test_find_everythingS s z TestFind.test_find_everythingc C s$ | j d�}| jdt|jd��� dS )z;Test an optimization that finds all tags with a given name.z <a>foo</a><b>bar</b><a>baz</a>r+ �aN)r$ r r, r- )r r$ r r r � test_find_everything_with_nameX s z'TestFind.test_find_everything_with_nameN) r r r �__doc__r& r' r* r. r0 r r r r r! > s r! c @ s8 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd S ) � TestFindAllz%Basic tests of the find_all() method.c C s� | j d�}| j|jdd�dg� | j|jdd�dg� | j|jddgd�ddg� | j|jtjd�d�dddg� | j|jdd�dddg� d S ) z'You can search the tree for text nodes.u <html>Foo<b>bar</b>»</html>�bar)r )r) �Fooz.*� »TN)r$ r r- �re�compile)r r$ r r r �test_find_all_text_nodes` s z$TestFindAll.test_find_all_text_nodesc C s� | j d�}| j|jddd�dddg� | j|jddd�dg� | j|jdd d�dddd d g� | j|jdd d�dddd d g� d S )z7You can limit the number of items returned by find_all.z(<a>1</a><a>2</a><a>3</a><a>4</a><a>5</a>r/ � )�limit�1r# �3� � �4�5r N)r$ r r- )r r$ r r r �test_find_all_limitp s z TestFindAll.test_find_all_limitc C s: | j d�}| j|ddd�dg� | j|jdd�dg� d S ) Nz!<a>1</a><b>2<a id='foo'>3</a></b>r/ r= )r: r; �foo)r r< )r$ r r" )r r$ r r r �%test_calling_a_tag_is_calling_findall| s z1TestFindAll.test_calling_a_tag_is_calling_findallc C s. | j d�}g }|j|� | jg |j|�� d S )Nz<a></a>)r$ �appendr r- )r r$ �lr r r �Ttest_find_all_with_self_referential_data_structure_does_not_cause_infinite_recursion� s z`TestFindAll.test_find_all_with_self_referential_data_structure_does_not_cause_infinite_recursionc C s^ | j d�}|jd�}| jt|d�� |jd�}| jt|d�� |jdd�}| jt|d�� dS )z%All find_all calls return a ResultSetz<a></a>r/ �sourceTrB )r) N)r$ r- � assertTrue�hasattr)r r$ �resultr r r �test_find_all_resultset� s z#TestFindAll.test_find_all_resultsetN) r r r r1 r8 rA rC rF rK r r r r r2 ] s r2 c @ s e Zd Zdd� ZdS )�TestFindAllBasicNamespacesc C s< | j d�}| jd|jd�j� | jd|jddid�j� d S )Nz0<mathml:msqrt>4</mathml:msqrt><a svg:fill="red">r? z mathml:msqrtr/ zsvg:fillZred)�attrs)r$ r r% r �name)r r$ r r r � test_find_by_namespaced_name� s z7TestFindAllBasicNamespaces.test_find_by_namespaced_nameN)r r r rO r r r r rL � s rL c sp e Zd ZdZ� fdd�Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dd� Z dd� Z� ZS )�TestFindAllByNamez&Test ways of finding tags by tag name.c s t t| �j� | jd�| _d S )Nz�<a>First tag.</a> <b>Second tag.</b> <c>Third <a>Nested tag.</a> tag.</c>)�superr �setUpr$ �tree)r )� __class__r r rR � s zTestFindAllByName.setUpc C s | j | jjd�ddg� d S )Nr/ z First tag.z Nested tag.)r rS r- )r r r r �test_find_all_by_tag_name� s z+TestFindAllByName.test_find_all_by_tag_namec C s\ | j | jjddd�dg� | j | jjddd�ddg� | j | jjdtjd�d�ddg� d S )Nr/ z First tag.)r) Tz Nested tag.r )r rS r- r6 r7 )r r r r � test_find_all_by_name_and_text� s z0TestFindAllByName.test_find_all_by_name_and_textc C s | j | jjjd�dg� d S )Nr/ z Nested tag.)r rS �cr- )r r r r �!test_find_all_on_non_root_element� s z3TestFindAllByName.test_find_all_on_non_root_elementc C s | j | jd�ddg� d S )Nr/ z First tag.z Nested tag.)r rS )r r r r �%test_calling_element_invokes_find_all� s z7TestFindAllByName.test_calling_element_invokes_find_allc C s | j | jjtd��ddg� d S )Nr/ z First tag.z Nested tag.)r rS r- r )r r r r � test_find_all_by_tag_strainer� s z/TestFindAllByName.test_find_all_by_tag_strainerc C s" | j | jjddg�dddg� d S )Nr/ r" z First tag.z Second tag.z Nested tag.)r rS r- )r r r r �test_find_all_by_tag_names� s z,TestFindAllByName.test_find_all_by_tag_namesc C s$ | j | jjddd��dddg� d S )NT)r/ r" z First tag.z Second tag.z Nested tag.)r rS r- )r r r r �test_find_all_by_tag_dict� s z+TestFindAllByName.test_find_all_by_tag_dictc C s$ | j | jjtjd��dddg� d S )Nz^[ab]$z First tag.z Second tag.z Nested tag.)r rS r- r6 r7 )r r r r �test_find_all_by_tag_re� s z)TestFindAllByName.test_find_all_by_tag_rec C s, dd� }| j d�}| j|j|�ddg� d S )Nc S s | j | jd�kS )Nr )rN �get)r r r r �id_matches_name� s zRTestFindAllByName.test_find_all_with_tags_matching_method.<locals>.id_matches_namez�<a id="a">Match 1.</a> <a id="1">Does not match.</a> <b id="b">Match 2.</a>zMatch 1.zMatch 2.)r$ r r- )r r_ rS r r r �'test_find_all_with_tags_matching_method� s z9TestFindAllByName.test_find_all_with_tags_matching_methodc C sx | j d�}|jdd�}|jdtjd��}|jdddg�\}}| jd|j� | jd|j� | jd|j� | jd|j� d S )NzH<div class='a b'>1</div><div class='a c'>2</div><div class='a d'>3</div>�divza dza br< r; )r$ r% r6 r7 r- r r )r r$ Zr1Zr2Zr3Zr4r r r �%test_find_with_multi_valued_attribute� s z7TestFindAllByName.test_find_with_multi_valued_attribute)r r r r1 rR rU rV rX rY rZ r[ r\ r] r` rb � __classcell__r r )rT r rP � s rP c @ s� e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd � Zd d � Zd d � Zd!S )"�TestFindAllByAttributec C s& | j d�}| j|jdd�ddg� d S )Nz� <a id="first">Matching a.</a> <a id="second"> Non-matching <b id="first">Matching b.</b>a. </a>�first)r z Matching a.z Matching b.)r$ r r- )r rS r r r � test_find_all_by_attribute_name� s z6TestFindAllByAttribute.test_find_all_by_attribute_namec C st dj d�}dj d�}| j|�}| j|jg|j|d�� | j|jg|j|jd�d�� | j|jg|j|dgd�� d S )Nu םולש�utf8u <a title="םולש"></a>)�titlezsomething else)�encoder$ r r/ r- �decode)r Zpeace�datar$ r r r �%test_find_all_by_utf8_attribute_value� s z<TestFindAllByAttribute.test_find_all_by_utf8_attribute_valuec C sX | j d�}| j|jdd�dg� | j|jddid�dg� | j|jdd id�d g� d S ) Na0 <a name="name1" class="class1">Name match.</a> <a name="name2" class="class2">Class match.</a> <a name="name3" class="class3">Non-match.</a> <name1>A tag called 'name1'.</name1> Zname1)rN zA tag called 'name1'.rN )rM z Name match.�classZclass2z Class match.)r$ r r- )r rS r r r � test_find_all_by_attribute_dict s z6TestFindAllByAttribute.test_find_all_by_attribute_dictc C s� | j d�}| j|jddd�dg� | j|jddd�dg� | j|jdd d�dg� | j|jdd�dg� | j|jdd �ddg� | j|jdd�dg� | j|jdd �dg� d S ) Nz� <a class="1">Class 1.</a> <a class="2">Class 2.</a> <b class="1">Class 1.</b> <c class="3 4">Class 3 and 4.</c> r/ r; )�class_zClass 1.rW r< zClass 3 and 4.r? )rM )r$ r r- )r rS r r r �test_find_all_by_class s z-TestFindAllByAttribute.test_find_all_by_classc C st | j d�}|jdtjd�d�}| j|dg� |jdtjd�d�}| j|dg� |jdtjd�d�}| j|dg� d S )Nz#<gar class='foo bar'>Found it</gar>Zgar�o)ro zFound itr/ zo b)r$ r- r6 r7 r )r rS �fr r r �0test_find_by_class_when_multiple_classes_present- s zGTestFindAllByAttribute.test_find_by_class_when_multiple_classes_presentc C sd | j d�}| j|jdtjd��dg� dd� }| j|jd|�g � dd� }| j|jd|�dg� d S ) Nz<a class='bar'>Found it</a>r/ ZbazFound itc S s t | �dkS )Nr9 )r, )�valuer r r �big_attribute_value@ s znTestFindAllByAttribute.test_find_all_with_non_dictionary_for_attrs_finds_by_class.<locals>.big_attribute_valuec S s t | �dkS )Nr9 )r, )rt r r r �small_attribute_valueE s zpTestFindAllByAttribute.test_find_all_with_non_dictionary_for_attrs_finds_by_class.<locals>.small_attribute_value)r$ r r- r6 r7 )r r$ ru rv r r r �:test_find_all_with_non_dictionary_for_attrs_finds_by_class; s zQTestFindAllByAttribute.test_find_all_with_non_dictionary_for_attrs_finds_by_classc C s� | j d�}|jd�\}}| j||g|jdd�� | j|g|jdd�� | j|g|jddd�� | j|g|jdd�� | jg |jdd�� d S )Nz*<a class="foo bar"></a><a class="foo"></a>r/ rB r3 zfoo bar)ro zbar foo)r$ r- r )r r$ r/ Za2r r r �:test_find_all_with_string_for_attrs_finds_multiple_classesK s zQTestFindAllByAttribute.test_find_all_with_string_for_attrs_finds_multiple_classesc C s0 | j d�}tddid�}| j|j|�dg� d S )Nzi <a id="first">Match.</a> <a id="second">Non-match.</a>r re )rM zMatch.)r$ r r r- )r rS �strainerr r r �'test_find_all_by_attribute_soupstrainerW s z>TestFindAllByAttribute.test_find_all_by_attribute_soupstrainerc C s& | j d�}| j|jdd d�dg� d S )Nz�<a id="1">ID present.</a> <a>No ID present.</a> <a id="">ID is empty.</a>r/ )r zNo ID present.)r$ r r- )r rS r r r �$test_find_all_with_missing_attribute_ s z;TestFindAllByAttribute.test_find_all_with_missing_attributec C s& | j d�}| j|jdd�ddg� d S )Nz�<a id="1">ID present.</a> <a>No ID present.</a> <a id="">ID is empty.</a>T)r z ID present.z ID is empty.)r$ r r- )r rS r r r �$test_find_all_with_defined_attributeg s z;TestFindAllByAttribute.test_find_all_with_defined_attributec C s> | j d�}ddg}| j|jdd�|� | j|jdd�|� d S )Nz[<a id=1>Unquoted attribute.</a> <a id="1">Quoted attribute.</a>zUnquoted attribute.zQuoted attribute.r= )r r; )r$ r r- )r rS Zexpectedr r r �$test_find_all_with_numeric_attributep s z;TestFindAllByAttribute.test_find_all_with_numeric_attributec C s, | j d�}| j|jdddgd�ddg� d S )Nz�<a id="1">1</a> <a id="2">2</a> <a id="3">3</a> <a>No ID.</a>r; r< r? )r )r$ r r- )r rS r r r �(test_find_all_with_list_attribute_valuesy s z?TestFindAllByAttribute.test_find_all_with_list_attribute_valuesc C s, | j d�}| j|jtjd�d�ddg� d S )Nz�<a id="a">One a.</a> <a id="aa">Two as.</a> <a id="ab">Mixed as and bs.</a> <a id="b">One b.</a> <a>No ID.</a>z^a+$)r zOne a.zTwo as.)r$ r r- r6 r7 )r rS r r r �5test_find_all_with_regular_expression_attribute_value� s zLTestFindAllByAttribute.test_find_all_with_regular_expression_attribute_valuec C sX | j d�}|j}| j|g|jddd�� | jg |jddd�� | jg |jddd�� d S )Nz <b>foo</b><b>bar</b><a>foo</a>r/ rB )r) r3 )r$ r/ r r- )r r$ r/ r r r �'test_find_by_name_and_containing_string� s z>TestFindAllByAttribute.test_find_by_name_and_containing_stringc C s* | j d�}| j|jd�|jddd�� d S )Nz"<a>foo</a><a><b><c>foo</c></b></a>r/ rB )r) )r$ r r- )r r$ r r r �=test_find_by_name_and_containing_string_when_string_is_buried� s zTTestFindAllByAttribute.test_find_by_name_and_containing_string_when_string_is_buriedc C sB | j d�}|j}| j|g|jddd�� | jg |jddd�� d S )Nz"<b id="1">foo</b><a id="2">foo</a>r+ rB )r r) r= r3 )r$ r/ r r- )r r$ r/ r r r �,test_find_by_attribute_and_containing_string� s zCTestFindAllByAttribute.test_find_by_attribute_and_containing_stringN)r r r rf rl rn rp rs rw rx rz r{ r| r} r~ r r� r� r� r r r r rd � s rd c @ s e Zd ZdZdd� ZdS )� TestIndexzTest Tag.indexc C sN | j d�}|j}x(t|j�D ]\}}| j||j|�� q W | jt|jd� d S )Nah <div> <a>Identical</a> <b>Not identical</b> <a>Identical</a> <c><d>Identical with child</d></c> <b>Also not identical</b> <c><d>Identical with child</d></c> </div>r= )r$ ra � enumerate�contentsr �index� assertRaises� ValueError)r rS ra �i�elementr r r � test_index� s zTestIndex.test_indexN)r r r r1 r� r r r r r� � s r� c s` e Zd ZdZ� fdd�Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z � Z S )�TestParentOperationsz;Test navigation and searching through an element's parents.c s( t t| �j� | jd�| _| jj| _d S )Na1 <ul id="empty"></ul> <ul id="top"> <ul id="middle"> <ul id="bottom"> <b>Start here</b> </ul> </ul>)rQ r� rR r$ rS r" �start)r )rT r r rR � s zTestParentOperations.setUpc C sF | j | jjd d� | j | jjjd d� | j | jjjjd d� d S )Nr �bottom�middle�top)r r� �parent)r r r r � test_parent� s z TestParentOperations.test_parentc C s | j jd }| j|j| j � d S )Nr )rS r� r r� )r Ztop_tagr r r �%test_parent_of_top_tag_is_soup_object� s z:TestParentOperations.test_parent_of_top_tag_is_soup_objectc C s | j d | jj� d S )N)r rS r� )r r r r � test_soup_object_has_no_parent� s z3TestParentOperations.test_soup_object_has_no_parentc C s8 | j | jjd�dddg� | j | jjddd�dg� d S )N�ulr� r� r� )r )r r� Z find_parents)r r r r �test_find_parents� s z&TestParentOperations.test_find_parentsc C s8 | j | jjd�d d� | j | jjddd�d d� d S )Nr� r r� r� )r )r r� � find_parent)r r r r �test_find_parent� s z%TestParentOperations.test_find_parentc C s" | j jdd�}| j|jjd� d S )Nz Start here)r) r" )rS r% r r� rN )r r) r r r �test_parent_of_text_element� s z0TestParentOperations.test_parent_of_text_elementc C s( | j jdd�}| j|jd�d d� d S )Nz Start here)r) r� r r� )rS r% r r� )r r) r r r � test_text_element_find_parent� s z2TestParentOperations.test_text_element_find_parentc C s( dd� | j jD �}| j|dddg� d S )Nc S s& g | ] }|d k rd|j kr|d �qS )Nr )rM )r r� r r r r � s z>TestParentOperations.test_parent_generator.<locals>.<listcomp>r� r� r� )r� �parentsr )r r� r r r �test_parent_generator� s z*TestParentOperations.test_parent_generator)r r r r1 rR r� r� r� r� r� r� r� r� rc r r )rT r r� � s r� c s e Zd Z� fdd�Z� ZS )� ProximityTestc s t t| �j� | jd�| _d S )Nzg<html id="start"><head></head><body><b id="1">One</b><b id="2">Two</b><b id="3">Three</b></body></html>)rQ r rR r$ rS )r )rT r r rR � s zProximityTest.setUp)r r r rR rc r r )rT r r� � s r� c sT e Zd Z� fdd�Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z � Z S )�TestNextOperationsc s t t| �j� | jj| _d S )N)rQ r� rR rS r" r� )r )rT r r rR � s zTestNextOperations.setUpc C s* | j | jjd� | j | jjjd d� d S )N�Oner r# )r r� � next_element)r r r r � test_next� s z TestNextOperations.test_nextc C s | j jdd�}| j|jd � d S )N�Three)r) )rS r% r r� )r Zlastr r r � test_next_of_last_item_is_none� s z1TestNextOperations.test_next_of_last_item_is_nonec C s | j | jjd � d S )N)r rS r� )r r r r �test_next_of_root_is_none s z,TestNextOperations.test_next_of_root_is_nonec C sB | j | jjd�ddg� | jjdd� | j | jjdd�dg� d S )Nr" �Twor� r9 )r )r r� � find_all_next)r r r r �test_find_all_next s z%TestNextOperations.test_find_all_nextc C s2 | j | jjd�d d� | j | jjdd�d� d S )Nr" r r# r� )r) )r r� � find_next)r r r r �test_find_next s z!TestNextOperations.test_find_nextc C s< | j jdd�}| j|jd�jd� | j|jd�ddg� d S )Nr� )r) r" r� r� )rS r% r r� r r r� )r r) r r r � test_find_next_for_text_element s z2TestNextOperations.test_find_next_for_text_elementc C sF | j jdd�}dd� |jD �}|\}}| j|d d� | j|d� d S )Nr� )r) c S s g | ]}|�qS r r )r �noder r r r s z:TestNextOperations.test_next_generator.<locals>.<listcomp>r r< r� )rS r% Z next_elementsr )r r� Z successorsr r� r r r �test_next_generator s z&TestNextOperations.test_next_generator) r r r rR r� r� r� r� r� r� r� rc r r )rT r r� � s r� c sT e Zd Z� fdd�Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z � Z S )�TestPreviousOperationsc s" t t| �j� | jjdd�| _d S )Nr� )r) )rQ r� rR rS r% �end)r )rT r r rR s z TestPreviousOperations.setUpc C s* | j | jjd d� | j | jjjd� d S )Nr r< r� )r r� �previous_element)r r r r � test_previous! s z$TestPreviousOperations.test_previousc C s | j jd�}| j|jd � d S )N�html)rS r% r r� )r re r r r �#test_previous_of_first_item_is_none% s z:TestPreviousOperations.test_previous_of_first_item_is_nonec C s d S )Nr )r r r r � test_previous_of_root_is_none) s z4TestPreviousOperations.test_previous_of_root_is_nonec C s6 | j | jjd�dddg� | j | jjdd�dg� d S )Nr" r� r� r� r= )r )r r� �find_all_previous)r r r r �test_find_all_previous/ s z-TestPreviousOperations.test_find_all_previousc C s2 | j | jjd�d d� | j | jjdd�d� d S )Nr" r r< r� )r) )r r� � find_previous)r r r r �test_find_previous7 s z)TestPreviousOperations.test_find_previousc C s> | j jdd�}| j|jd�jd� | j|jd�dddg� d S )Nr� )r) r" r� r� )rS r% r r� r r r� )r r) r r r �#test_find_previous_for_text_element; s z:TestPreviousOperations.test_find_previous_for_text_elementc C sh | j jdd�}dd� |jD �}|\}}}}| j|d d� | j|jd� | j|jd� | j|jd � d S ) Nr� )r) c S s g | ]}|�qS r r )r r� r r r r C s zBTestPreviousOperations.test_previous_generator.<locals>.<listcomp>r r; �body�headr� )rS r% Zprevious_elementsr rN )r r� Z predecessorsr" r� r� r� r r r �test_previous_generatorA s z.TestPreviousOperations.test_previous_generator) r r r rR r� r� r� r� r� r� r� rc r r )rT r r� s r� c s e Zd Z� fdd�Z� ZS )� SiblingTestc s4 t t| �j� d}tjd�jd|�}| j|�| _d S )Na� <html> <span id="1"> <span id="1.1"></span> </span> <span id="2"> <span id="2.1"></span> </span> <span id="3"> <span id="3.1"></span> </span> <span id="4"></span> </html>z\n\s*� )rQ r� rR r6 r7 �subr$ rS )r �markup)rT r r rR P s zSiblingTest.setUp)r r r rR rc r r )rT r r� N s r� c sL e Zd Z� fdd�Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z � Z S )�TestNextSiblingc s" t t| �j� | jjdd�| _d S )Nr; )r )rQ r� rR rS r% r� )r )rT r r rR f s zTestNextSibling.setUpc C s | j | jjd � d S )N)r rS � next_sibling)r r r r �!test_next_sibling_of_root_is_nonej s z1TestNextSibling.test_next_sibling_of_root_is_nonec C sB | j | jjd d� | j | jjjd d� | j | jjd d� d S )Nr r# r< z1.1)r r� r� r� )r r r r �test_next_siblingm s z!TestNextSibling.test_next_siblingc C sN | j | jjjd � | jjdd�}| j |jd � | jjdd�}| j |jd � d S )Nz1.1)r r? )r rS r� r� r% )r � nested_spanZ last_spanr r r � test_next_sibling_may_not_existt s z/TestNextSibling.test_next_sibling_may_not_existc C s | j | jjd�d d� d S )N�spanr r# )r r� �find_next_sibling)r r r r �test_find_next_sibling} s z&TestNextSibling.test_find_next_siblingc C s6 | j | jjd�dddg� | j | jjdd�dg� d S )Nr� r# r< r? )r )r r� �find_next_siblings)r r r r �test_next_siblings� s z"TestNextSibling.test_next_siblingsc C sv | j d�}|jdd�}| j|jjd� | j|jjd� | j|jd�dg� | j|jdd�d� | j|jdd�d � d S )NzFoo<b>bar</b>bazr4 )r) r" �bazr3 �nonesuch)r$ r% r r� rN r r� r� )r r$ r� r r r �"test_next_sibling_for_text_element� s z2TestNextSibling.test_next_sibling_for_text_element) r r r rR r� r� r� r� r� r� rc r r )rT r r� d s r� c sL e Zd Z� fdd�Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z � Z S )�TestPreviousSiblingc s" t t| �j� | jjdd�| _d S )Nr? )r )rQ r� rR rS r% r� )r )rT r r rR � s zTestPreviousSibling.setUpc C s | j | jjd � d S )N)r rS �previous_sibling)r r r r �%test_previous_sibling_of_root_is_none� s z9TestPreviousSibling.test_previous_sibling_of_root_is_nonec C sB | j | jjd d� | j | jjjd d� | j | jjd d� d S )Nr r< r# z3.1)r r� r� r� )r r r r �test_previous_sibling� s z)TestPreviousSibling.test_previous_siblingc C sN | j | jjjd � | jjdd�}| j |jd � | jjdd�}| j |jd � d S )Nz1.1)r r; )r rS r� r� r% )r r� Z first_spanr r r �#test_previous_sibling_may_not_exist� s z7TestPreviousSibling.test_previous_sibling_may_not_existc C s | j | jjd�d d� d S )Nr� r r< )r r� �find_previous_sibling)r r r r �test_find_previous_sibling� s z.TestPreviousSibling.test_find_previous_siblingc C s6 | j | jjd�dddg� | j | jjdd�dg� d S )Nr� r< r# r; )r )r r� �find_previous_siblings)r r r r �test_previous_siblings� s z*TestPreviousSibling.test_previous_siblingsc C sv | j d�}|jdd�}| j|jjd� | j|jjd� | j|jd�dg� | j|jdd�d� | j|jdd�d � d S )NzFoo<b>bar</b>bazr� )r) r" r4 r3 r� )r$ r% r r� rN r r� r� )r r$ r� r r r �&test_previous_sibling_for_text_element� s z:TestPreviousSibling.test_previous_sibling_for_text_element) r r r rR r� r� r� r� r� r� rc r r )rT r r� � s r� c @ s0 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd S ) �TestTagCreationz$Test the ability to create new tags.c C sd | j d�}|jddddid�}| jt|t�� | jd|j� | jtddd�|j� | jd |j � d S )Nr� rB r� rN za name)r3 rM )r3 rN ) r$ �new_tagrH � isinstancer r rN �dictrM r� )r r$ r� r r r � test_new_tag� s z TestTagCreation.test_new_tagc C s� t rBtdd�}|jd�}|jd�}| jd|j� � | jd|j� � tdd�}|jd�}|jd�}| jd|j� � | jd|j� � d S ) Nr� zlxml-xml�br�ps <br/>s <p/>z html.parsers <p></p>)�XML_BUILDER_PRESENTr r� r ri )r Zxml_soupZxml_brZxml_pZ html_soupZhtml_brZhtml_pr r r �1test_tag_inherits_self_closing_rules_from_builder� s zATestTagCreation.test_tag_inherits_self_closing_rules_from_builderc C s4 | j d�}|jd�}| jd|� | jt|t�� d S )Nr� rB )r$ � new_stringr rH r� r )r r$ �sr r r �'test_new_string_creates_navigablestring� s z7TestTagCreation.test_new_string_creates_navigablestringc C s6 | j d�}|jdt�}| jd|� | jt|t�� d S )Nr� rB )r$ r� r r rH r� )r r$ r� r r r �3test_new_string_can_create_navigablestring_subclass� s zCTestTagCreation.test_new_string_can_create_navigablestring_subclassN)r r r r1 r� r� r� r� r r r r r� � s r� c @ s< e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Zd'd(� Zd)d*� Zd+d,� Zd-d.� Zd/d0� Zd1d2� Zd3d4� Z d5d6� Z d7d8� Z d9d:� Z d;d<� Z d=d>� Z!d?d@� Z"dAdB� Z#dCdD� Z$dEdF� Z%dGdH� Z&dIdJ� Z'dKdL� Z(dMS )N�TestTreeModificationc C sl | j d�}d|jd< | j|j� | jd�� |jd= | j|j� | jd�� d|jd< | j|j� | jd�� d S ) Nz<a id="1"></a>r+ r z<a id="2"></a>z<a></a>rB Zid2z<a id2="foo"></a>)r$ r/ r rj � document_for)r r$ r r r �test_attribute_modification� s z0TestTreeModification.test_attribute_modificationc C sl t jd�� }| jd|d�}t||d�}t||d�}d|d< |jjd|� |jjd |� | j|jj� d � d S ) Nr� z <body></body>)�builderr/ �olzhttp://foo.com/Zhrefr r= s4 <body><a href="http://foo.com/"></a><ol></ol></body>)r �lookupr$ r r� �insertr ri )r r� r$ r/ r� r r r �test_new_tag_creation� s z*TestTreeModification.test_new_tag_creationc C s\ d}| j |�}|jdd�}|j}|jdd�j|j� | j|j|� | j|j� | jd�� d S )NzT<p id="1">Don't leave me <b>here</b>.</p> <p id="2">Don't leave!</p>r# )r zD<p id="1">Don't leave me .</p> <p id="2">Don't leave!<b>here</b></p>)r$ r% r" rD r r� rj r� )r �docr$ Z second_paraZboldr r r �!test_append_to_contents_moves_tag� s z6TestTreeModification.test_append_to_contents_moves_tagc C s0 d}| j |�}|j}|j|j�}| j||� d S )Nz<a></a><b><c></c></b>)r$ r/ � replace_withrW r )r r) r$ r/ �new_ar r r �1test_replace_with_returns_thing_that_was_replaced s zFTestTreeModification.test_replace_with_returns_thing_that_was_replacedc C s, d}| j |�}|j}|j� }| j||� d S )Nz<a><b></b><c></c></a>)r$ r/ �unwrapr )r r) r$ r/ r� r r r �+test_unwrap_returns_thing_that_was_replaced s z@TestTreeModification.test_unwrap_returns_thing_that_was_replacedc C sJ | j d�}|j}|j� | jd |j� | jt|j� | jt|j|j � d S )Nz<a><b>Foo</b></a><c>Bar</c>) r$ r/ �extractr r� r� r� r� r� rW )r r$ r/ r r r �Itest_replace_with_and_unwrap_give_useful_exception_when_tag_has_no_parent s z^TestTreeModification.test_replace_with_and_unwrap_give_useful_exception_when_tag_has_no_parentc C s: d}| j |�}|j}|jj|� | j|j� | j|�� d S )Nz-<a><b></b><c>Foo<d></d></c></a><a><e></e></a>)r$ rW r� r rj r� )r r) r$ rW r r r � test_replace_tag_with_itself' s z1TestTreeModification.test_replace_tag_with_itselfc C s& d}| j |�}| jt|jj|j� d S )Nz<a><b></b></a>)r$ r� r� r" r� r/ )r r) r$ r r r �1test_replace_tag_with_its_parent_raises_exception. s zFTestTreeModification.test_replace_tag_with_its_parent_raises_exceptionc C s( d}| j |�}| jt|jjd|j� d S )Nz<a><b></b></a>r )r$ r� r� r/ r� )r r) r$ r r r �,test_insert_tag_into_itself_raises_exception3 s zATestTreeModification.test_insert_tag_into_itself_raises_exceptionc C s� | j d�}d}| j |�}|jd|� x|jD ]}q,W t|j�\}}}}| jd|j� | jd|j� | jd|j� | jd|j� dS ) z�Inserting one BeautifulSoup object into another actually inserts all of its children -- you'll never combine BeautifulSoup objects. z-<p>And now, a word:</p><p>And we're back.</p>z<p>p2</p><p>p3</p>r= zAnd now, a word:�p2�p3zAnd we're back.N)r$ r� Z descendants�listZchildrenr r ) r r$ r) Z to_insertr� �p1r� r� Zp4r r r �1test_insert_beautifulsoup_object_inserts_children8 s zFTestTreeModification.test_insert_beautifulsoup_object_inserts_childrenc C sX | j d�}|j}|jd }|jdd� |j\}}|jd� |jd� | jd|jj� d S )Nz <p><a>one</a><b>three</b></p>r r= Ztwor� Zthree)r$ r/ r� r� Z replaceWithr r" r )r r$ r/ r" �left�rightr r r �3test_replace_with_maintains_next_element_throughoutL s zHTestTreeModification.test_replace_with_maintains_next_element_throughoutc C sl | j d�}|jdd�jd� |jdd�}|j}| j|j|� | j|j|� | j|jj|� | j|jd � d S )Nz <b>Argh!</b>zArgh!)r) zHooray!)r$ r% r� r" r r� r� r� )r r$ �new_textr" r r r �test_replace_final_node[ s z,TestTreeModification.test_replace_final_nodec C s� | j d�}|jjdd� | j|j� | jd�� |jdd�}| j|jd� | j|jj|� | j|j d� | j|j j |� | j|j d � | j|j|j � d S )Nz<a><b>Argh!</b><c></c></a>r= zHooray!z!<a><b>Argh!Hooray!</b><c></c></a>)r) zArgh!) r$ r" r� r rj r� r% r� r� r� r� rW )r r$ r r r r �test_consecutive_text_nodese s z0TestTreeModification.test_consecutive_text_nodesc C sT | j d�}|jjdd� |jjdd� | jddg|jj� | j|jjd jd� d S )Nz<a></a>r r3 rB )r$ r/ r� r r� r� )r r$ r r r �test_insert_stringz s z'TestTreeModification.test_insert_stringc C s� | j }| jd|d�}t||d�}|jdd� |jjd|� | j|j� | jd�� |j}| j|j |� | j|j |� |j dd �}| j|j |� | j|j |� |j}| j|j |� | j|j |� |j dd �}| j|j|� | j|j |� | j|j |� d S ) Nz%<a><b>Find</b><c>lady!</c><d></d></a>)r� Zmagictagr �ther= z=<a><b>Find</b><magictag>the</magictag><c>lady!</c><d></d></a>ZFind)r) )Zdefault_builderr$ r r� r/ r rj r� r" r� r� r% r� r� rW r� )r r� r$ Z magic_tagZb_tagr% Zc_tagr r r r �test_insert_tag� s, z$TestTreeModification.test_insert_tagc C s0 d}| j |�}|jj|j� | j||j� � d S )Nz<a><b></b></a>)r$ r/ rD r" r rj )r rk r$ r r r �*test_append_child_thats_already_at_the_end� s z?TestTreeModification.test_append_child_thats_already_at_the_endc C s2 d}| j |�}|jjd|j� | jd|j� � d S )Nz <a><b></b><c></c><d></d></a>r z <a><d></d><b></b><c></c></a>)r$ r/ r� �dr rj )r rk r$ r r r �$test_move_tag_to_beginning_of_parent� s z9TestTreeModification.test_move_tag_to_beginning_of_parentc C s. | j d�}|jjdd� | jt|j�d� d S )Nz<br/>r= ZContentsz<br>Contents</br>)r$ r� r� r r( )r r$ r r r �&test_insert_works_on_empty_element_tag� s z;TestTreeModification.test_insert_works_on_empty_element_tagc C s` | j d�}|jjd� |jjd� | j|j� | jd�� |jj|j� | j|j� | jd�� d S )Nz<a>foo</a><b>bar</b>�BAZ�QUUXzQUUX<a>foo</a>BAZ<b>bar</b>zQUUX<b>bar</b><a>foo</a>BAZ)r$ r" � insert_beforer/ r rj r� )r r$ r r r �test_insert_before� s z'TestTreeModification.test_insert_beforec C s` | j d�}|jjd� |jjd� | j|j� | jd�� |jj|j� | j|j� | jd�� d S )Nz<a>foo</a><b>bar</b>r r z<a>foo</a>QUUX<b>bar</b>BAZzQUUX<b>bar</b><a>foo</a>BAZ)r$ r" � insert_afterr/ r rj r� )r r$ r r r �test_insert_after� s z&TestTreeModification.test_insert_afterc C sR | j d�}|jd�}|jd�}| jt|j|� | jt|j|� | jt|j|� d S )Nr� r/ )r$ r� r� r� r� r �NotImplementedError)r r$ r r r r r �:test_insert_after_raises_exception_if_after_has_no_meaning� s zOTestTreeModification.test_insert_after_raises_exception_if_after_has_no_meaningc C sR | j d�}|jd�}|jd�}| jt|j|� | jt|j|� | jt|j|� d S )Nr� r/ )r$ r� r� r� r� r r )r r$ r r r r r �Ftest_insert_before_raises_notimplementederror_if_before_has_no_meaning� s z[TestTreeModification.test_insert_before_raises_notimplementederror_if_before_has_no_meaningc C sv | j d�}|jd�\}}|j|� | j|j� | jd�� | j|jd � | j|j|j� | j|jd� | j|j d� d S )Nz;<p>There's <b>no</b> business like <b>show</b> business</p>r" z0<p>There's business like <b>no</b> business</p>�noz business) r$ r- r� r rj r� r� r� r� r� )r r$ r Zshowr r r �test_replace_with� s z&TestTreeModification.test_replace_withc C s0 d}| j |�}|jj|j� | jd|j� � d S )Nz<a><b></b><c></c></a>z<a><c></c></a>)r$ r" r� rW r rj )r rk r$ r r r �test_replace_first_child� s z-TestTreeModification.test_replace_first_childc C s0 d}| j |�}|jj|j� | jd|j� � d S )Nz<a><b></b><c></c></a>z<a><b></b></a>)r$ rW r� r" r rj )r rk r$ r r r �test_replace_last_child� s z,TestTreeModification.test_replace_last_childc C s | j d�}|j}|j}|j|� | j|j� | jd�� | j|jd � | j|jdd�j d � | j|j d � | j|j d � | j|j d � | j|j|j � | j|j d� | j|j j |j� | j|j d � |jdd�}|j}| j|j |� | j|j |� | j|j |� | j|j |� d S )NzQ<a>We<b>reserve<c>the</c><d>right</d></b></a><e>to<f>refuse</f><g>service</g></e>z-<a>We<f>refuse</f></a><e>to<g>service</g></e>r� )r) ZWeZto)r$ r" rr r� r rj r� r� r% r� r� r� r� r/ �e�g)r r$ Z remove_tagZmove_tagZto_textZg_tagr r r � test_nested_tag_replace_with� s. z1TestTreeModification.test_nested_tag_replace_withc C s6 | j d�}|jj� | j|jd � | j|jjd� d S )NzI <p>Unneeded <em>formatting</em> is unneeded</p> z Unneeded formatting is unneeded)r$ �emr� r r� r) )r rS r r r � test_unwrap s z TestTreeModification.test_unwrapc C sF | j d�}|jj|jd��}| j|j� d� | j|j� | jd�� d S )NzI wish I was bold.r" z<b>I wish I was bold.</b>)r$ r �wrapr� r rj r� )r r$ rt r r r � test_wrap" s z TestTreeModification.test_wrapc C s4 | j d�}|jjj|j� | j|j� | jd�� d S )Nz<b></b>I wish I was bold.z<b>I wish I was bold.</b>)r$ r" r� r r rj r� )r r$ r r r �%test_wrap_extracts_tag_from_elsewhere) s z:TestTreeModification.test_wrap_extracts_tag_from_elsewherec C sH | j d�}|jjj|j� | jdt|jj�� | j|j� | jd�� d S )Nz+<b>I like being bold.</b>I wish I was bold.r+ z+<b>I like being bold.I wish I was bold.</b>) r$ r" r� r r r, r� rj r� )r r$ r r r �&test_wrap_puts_new_contents_at_the_end/ s z;TestTreeModification.test_wrap_puts_new_contents_at_the_endc C s� | j d�}| jt|jj�d� |jdd�j� }| j|j� d� | j|j� d� | jt|jj�d� | j|jd � | j|j d � | j|j j d � |jdd �}|jd d �}| j|j |� | j|j |� | j|j |� | j|j |� d S ) NzR<html><body>Some content. <div id="nav">Nav crap</div> More content.</body></html>r9 Znav)r z6<html><body>Some content. More content.</body></html>z <div id="nav">Nav crap</div>r+ zSome content. )r) z More content.) r$ r r, r� r� r% r� rj r� r� r� r� r� )r r$ Z extractedZ content_1Z content_2r r r � test_extract7 s" z!TestTreeModification.test_extractc C sz | j d�}|jj}|jj}|jd�}|jd�}|jj|� |jj|� |j� |j� | j||jj� | j||jj� d S )Nz<a>foo</a><b>bar</b>rB r3 )r$ r/ r r" r� rD r� r )r r$ Zfoo_1Zbar_1Zfoo_2Zbar_2r r r �4test_extract_distinguishes_between_identical_stringsP s zITestTreeModification.test_extract_distinguishes_between_identical_stringsc s8 | j d�� � fdd�� jd�D � | jdt� j�� d S )Nzv <html> <head> <script>foo</script> </head> <body> <script>bar</script> <a></a> </body> <script>baz</script> </html>c s g | ]}� j j� �qS r )�scriptr� )r r� )r$ r r r m s zKTestTreeModification.test_extract_multiples_of_same_tag.<locals>.<listcomp>r# z<body> <a></a> </body>)r$ r- r r( r� )r r )r$ r �"test_extract_multiples_of_same_taga s z7TestTreeModification.test_extract_multiples_of_same_tagc C s. | j d�}|jd�j� | jd |jd�� d S )Nz <html> <body>hi</body> </html>r� )r$ r% r� r )r r$ r r r �Btest_extract_works_when_element_is_surrounded_by_identical_stringsq s zWTestTreeModification.test_extract_works_when_element_is_surrounded_by_identical_stringsc C sf | j d�}|j}|jj� | jt|jj�d� | jt|d�� |j }|jdd� | jdt|j�� dS )z Tag.clear()z4<p><a>String <em>Italicized</em></a> and another</p>r r� T)Z decomposeN) r$ r/ r� �clearr r, r� rH rI r )r r$ r/ r r r r � test_clearz s z TestTreeModification.test_clearc C sB | j d�}d|j_| j|jjdg� d|j_| j|jjdg� dS )zTag.string = 'string'z<a></a> <b><c></c></b>rB r3 N)r$ r/ r r r� r" )r r$ r r r �test_string_set� s z$TestTreeModification.test_string_setc C s, | j d�}|jj|j_| j|jj� d� d S )Nz<a><b>foo</b><c>bar</c>s <a><b>bar</b><c>bar</c></a>)r$ rW r r" r r/ ri )r r$ r r r �/test_string_set_does_not_affect_original_string� s zDTestTreeModification.test_string_set_does_not_affect_original_stringc C s2 | j d�}td�}||j_| jt|jjt�� d S )Nz<a></a>rB )r$ r r/ r rH r� )r r$ �cdatar r r �)test_set_string_preserves_class_of_string� s z>TestTreeModification.test_set_string_preserves_class_of_stringN))r r r r� r� r� r� r� r� r� r� r� r� r r r r r r r r r r r r r r r r r r r r r! r" r$ r% r' r( r) r+ r r r r r� � sL $ r� c @ sx e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd� Zd S ) �TestElementObjectsz)Test various features of element objects.c C sV | j d�}| jt|j�d� | jt|�d� | jt|j�d� | jt|jj�d� dS )z3The length of an element is its number of children.z<top>1<b>2</b>3</top>r= r9 N)r$ r r, r� r� )r r$ r r r �test_len� s zTestElementObjects.test_lenc C sL | j d�}| j|j|jd�� | j|jj|jd�jd�� | j|jd� dS )z2Accessing a Python member .foo invokes find('foo')z<b><i></i></b>r" r� N)r$ r r" r% r� r/ )r r$ r r r � test_member_access_invokes_find� s z2TestElementObjects.test_member_access_invokes_findc C sP | j d�}tjdd�� }|j}W d Q R X | j|j|� | jdt|d j�� d S )Nz<b><i></i></b>T)�recordzp.bTag is deprecated, use .find("b") instead. If you really were looking for a tag called bTag, use .find("bTag")r )r$ �warnings�catch_warningsZbTagr r" r( �message)r r$ �wr r r r � test_deprecated_member_access� s z0TestElementObjects.test_deprecated_member_accessc C s2 | j d�}| j|jjd�� | j|jjd�� dS )z�has_attr() checks for the presence of an attribute. Please note note: has_attr() is different from __in__. has_attr() checks the tag's attributes and __in__ checks the tag's chidlren. z<foo attr='bar'>�attrZattr2N)r$ rH rB �has_attr� assertFalse)r r$ r r r � test_has_attr� s z TestElementObjects.test_has_attrc C s d}| j |d� d S )Nz%<b a="1" z="5" m="3" f="2" y="4"></b>z%<b a="1" f="2" m="3" y="4" z="5"></b>)ZassertSoupEquals)r r� r r r �.test_attributes_come_out_in_alphabetical_order� s zATestElementObjects.test_attributes_come_out_in_alphabetical_orderc C s | j d�}| j|jjd� d S )Nz <b>foo</b>rB )r$ r r" r )r r$ r r r � test_string� s z TestElementObjects.test_stringc C s | j d�}| j|jjd � d S )Nz<b></b>)r$ r r" r )r r$ r r r � test_empty_tag_has_no_string� s z/TestElementObjects.test_empty_tag_has_no_stringc C s` | j d�}| j|jjd � | j d�}| j|jjd � | j d�}|jjdd� | j|jjd � d S )Nz<a>foo<b></b><b></b></b>z<a>foo<b></b>bar</b>z <a>foo</b>r= r3 )r$ r r" r r/ r� )r r$ r r r �-test_tag_with_multiple_children_has_no_string� s z@TestElementObjects.test_tag_with_multiple_children_has_no_stringc C s, | j d�}| j|jjd� | j|jd� d S )Nz<a><b>foo</b></a>rB )r$ r r/ r )r r$ r r r �)test_tag_with_recursive_string_has_string� s z<TestElementObjects.test_tag_with_recursive_string_has_stringc C s4 | j d�}| j|jj� | j d�}| j|jj� dS )z7Only a tag containing a single text node has a .string.z<b>f<i>e</i>o</b>z<b></b>N)r$ r7 r" r )r r$ r r r �test_lack_of_string� s z&TestElementObjects.test_lack_of_stringc C s` | j d�}| j|jjd� | j|jjdd�d� | j|jjd�d� | j|jjddd�d� d S ) zBTag.text and Tag.get_text(sep=u"") -> all child text, concatenatedz <a>a<b>r</b> <r> t </r></a>zar t T)�stripZart�,z a,r, , t za,r,tN)r$ r r/ r) �get_text)r r$ r r r � test_all_text� s z TestElementObjects.test_all_textc C sJ | j d�}| j|j� d� | j|jttfd�d� | j|jd d�d� d S )Nzfoo<!--IGNORE-->barZfoobar)�typesZ fooIGNOREbar)r$ r rA r r )r r$ r r r � test_get_text_ignores_comments� s z1TestElementObjects.test_get_text_ignores_commentsc C s$ | j d�}| jddgt|j�� d S )Nzfoo<!--IGNORE-->barrB r3 )r$ r r� Zstrings)r r$ r r r �!test_all_strings_ignores_comments s z4TestElementObjects.test_all_strings_ignores_commentsN)r r r r1 r- r. r4 r8 r9 r: r; r< r= r> rB rD rE r r r r r, � s r, c @ sP e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dS )�TestCDAtaListAttributesz0Testing cdata-list attributes like 'class'. c C s" | j d�}| jdg|jd � d S )Nz<a class='foo'>rB rm )r$ r r/ )r r$ r r r � test_single_value_becomes_list s z6TestCDAtaListAttributes.test_single_value_becomes_listc C s$ | j d�}| jddg|jd � d S )Nz<a class='foo bar'>rB r3 rm )r$ r r/ )r r$ r r r �!test_multiple_values_becomes_list s z9TestCDAtaListAttributes.test_multiple_values_becomes_listc C s& | j d�}| jdddg|jd � d S )Nz<a class='foo bar baz'>rB r3 r� rm )r$ r r/ )r r$ r r r �2test_multiple_values_separated_by_weird_whitespace s zJTestCDAtaListAttributes.test_multiple_values_separated_by_weird_whitespacec C s | j d�}| jd|jj� � d S )Nz<a class='foo bar'>s <a class="foo bar"></a>)r$ r r/ ri )r r$ r r r �,test_attributes_joined_into_string_on_output s zDTestCDAtaListAttributes.test_attributes_joined_into_string_on_outputc C s$ | j d�}| jdg|jjd�� d S )Nz<a id='abc def'>zabc defr )r$ r r/ Zget_attribute_list)r r$ r r r �test_get_attribute_list s z/TestCDAtaListAttributes.test_get_attribute_listc C s$ | j d�}| jddg|jd � d S )Nz(<form accept-charset="ISO-8859-1 UTF-8">z ISO-8859-1zUTF-8zaccept-charset)r$ r Zform)r r$ r r r �test_accept_charset! s z+TestCDAtaListAttributes.test_accept_charsetc C s$ d}| j |�}| jd|jd � d S )Nz)<a accept-charset="ISO-8859-1 UTF-8"></a>zISO-8859-1 UTF-8zaccept-charset)r$ r r/ )r rk r$ r r r �-test_cdata_attribute_applying_only_to_one_tag% s zETestCDAtaListAttributes.test_cdata_attribute_applying_only_to_one_tagc s6 | j d�j� | jd � j� � fdd�}| jt|� d S )Nr� c s d� _ d S )NrB )rN r )r r r �t0 s zJTestCDAtaListAttributes.test_string_has_immutable_name_property.<locals>.t)r$ r r rN r� �AttributeError)r rN r )r r �'test_string_has_immutable_name_property- s z?TestCDAtaListAttributes.test_string_has_immutable_name_propertyN) r r r r1 rG rH rI rJ rK rL rM rP r r r r rF s rF c s` e Zd ZdZ� fdd�Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z � Z S )�TestPersistencez*Testing features like pickle and deepcopy.c s& t t| �j� d| _| j| j�| _d S )Nay <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/transitional.dtd"> <html> <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> <title>Beautiful Soup: We called him Tortoise because he taught us.</title> <link rev="made" href="mailto:leonardr@segfault.org"> <meta name="Description" content="Beautiful Soup: an HTML parser optimized for screen-scraping."> <meta name="generator" content="Markov Approximation 1.4 (module: leonardr)"> <meta name="author" content="Leonard Richardson"> </head> <body> <a href="foo">foo</a> <a href="foo"><b>bar</b></a> </body> </html>)rQ rQ rR Zpager$ rS )r )rT r r rR 7 s zTestPersistence.setUpc C s@ t j| jd�}t j|�}| j|jt� | j|j� | jj� � d S )Nr+ )�pickle�dumpsrS �loadsr rT r rj )r �dumped�loadedr r r �!test_pickle_and_unpickle_identityK s z1TestPersistence.test_pickle_and_unpickle_identityc C s& t j| j�}| j|j� | jj� � d S )N)�copy�deepcopyrS r rj )r Zcopiedr r r �test_deepcopy_identityS s z&TestPersistence.test_deepcopy_identityc C s: t dd�}|j}|j� }| jdt|�� | j||j� d S )Ns <p> </p>z html.parseru <p> </p>)r Zoriginal_encoding�__copy__r r( )r r$ �encodingrX r r r � test_copy_preserves_encodingX s z,TestPersistence.test_copy_preserves_encodingc C s> d}| j |�}tj|tj�}tj|�}| j|j� |j� � d S )Nu <b>☃</b>)r$ rR rS ZHIGHEST_PROTOCOLrT r rj )r r� r$ rU rV r r r �test_unicode_pickle_ s z#TestPersistence.test_unicode_picklec C sz d}| j |�}|jdd�}tj|�}| j||� | jd |j� | jd |j� | jd |j� | jd |j� | jd |j� d S )Nz<b>Foo<a></a></b><b>Bar</b>r4 )r ) r$ r% rX r r� r� �assertNotEqualr� r� )r r� r$ �s1�s2r r r �1test_copy_navigablestring_is_not_attached_to_treeg s zATestPersistence.test_copy_navigablestring_is_not_attached_to_treec C s> d}| j |�}|j}tj|�}| j||� | jt|t�� d S )Nz<b><!--Foo--></b>)r$ r rX r rH r� r )r r� r$ r` ra r r r �0test_copy_navigablestring_subclass_has_same_types s z@TestPersistence.test_copy_navigablestring_subclass_has_same_typec C s( d}| j |�}tj|�}| j||� d S )Nz)<div><b>Foo<a></a></b><b>Bar</b></div>end)r$ rX r )r r� r$ Z soup_copyr r r �test_copy_entire_soup{ s z%TestPersistence.test_copy_entire_soupc C s� d}| j |�}|j}tj|�}| jt|�t|�� | j||� | j||k� | jd |j� | jd |j� | jd |jdd�j � | j d |jdd�j � d S )Nz)<div><b>Foo<a></a></b><b>Bar</b></div>endZBar)r ) r$ ra rX r r( r7 r� r� r% r� r_ )r r� r$ ra Zdiv_copyr r r � test_copy_tag_copies_contents� s z-TestPersistence.test_copy_tag_copies_contents)r r r r1 rR rW rZ r] r^ rb rc rd re rc r r )rT r rQ 4 s rQ c @ s� e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd � Zd d � Zd d � Zd!S )"�TestSubstitutionsc C s0 d}| j |�}|jdd�}| j|| jd�� d S )Nu# <b><<Sacré bleu!>></b>�minimal)� formatter)r$ rj r r� )r r� r$ �decodedr r r �!test_default_formatter_is_minimal� s z3TestSubstitutions.test_default_formatter_is_minimalc C s0 d}| j |�}|jdd�}| j|| jd�� d S )Nu' <br><b><<Sacré bleu!>></b>r� )rh z.<br/><b><<Sacré bleu!>></b>)r$ rj r r� )r r� r$ ri r r r �test_formatter_html� s z%TestSubstitutions.test_formatter_htmlc C s0 d}| j |�}|jdd�}| j|| jd�� d S )Nu' <br><b><<Sacré bleu!>></b>Zhtml5)rh z-<br><b><<Sacré bleu!>></b>)r$ rj r r� )r r� r$ ri r r r �test_formatter_html5� s z&TestSubstitutions.test_formatter_html5c C s0 d}| j |�}|jdd�}| j|| jd�� d S )Nu# <b><<Sacré bleu!>></b>rg )rh )r$ rj r r� )r r� r$ ri r r r �test_formatter_minimal� s z(TestSubstitutions.test_formatter_minimalc C s0 d}| j |�}|jd d�}| j|| jd�� d S )Nu# <b><<Sacré bleu!>></b>)rh u <b><<Sacré bleu!>></b>)r$ rj r r� )r r� r$ ri r r r �test_formatter_null� s z%TestSubstitutions.test_formatter_nullc C s4 d}| j |�}|jdd� d�}| j|| jd�� d S )Nz!<b><foo></b><b>bar</b><br/>c S s | j � S )N)�upper)�xr r r �<lambda>� s z9TestSubstitutions.test_formatter_custom.<locals>.<lambda>)rh z<b><FOO></b><b>BAR</b><br>)r$ rj r r� )r r� r$ ri r r r �test_formatter_custom� s z'TestSubstitutions.test_formatter_customc C s� d}| j |�}|j}d}| j||j� � | j||jdd�� d}| j||jdd�� | j||jd d�� d}| j||jdd � d�� d S ) Nu% <a href="http://a.com?a=b&c=é">e</a>u) <a href="http://a.com?a=b&c=é">e</a>rg )rh z/<a href="http://a.com?a=b&c=é">e</a>r� u% <a href="HTTP://A.COM?A=B&C=É">E</a>c S s | j � S )N)ro )rp r r r rq � s zMTestSubstitutions.test_formatter_is_run_on_attribute_values.<locals>.<lambda>)r$ r/ r rj )r r� r$ r/ Zexpect_minimalZ expect_htmlZ expect_upperr r r �)test_formatter_is_run_on_attribute_values� s z;TestSubstitutions.test_formatter_is_run_on_attribute_valuesc C s$ d}t |d�j� }| jd|k� d S )NzO <script type="text/javascript"> console.log("< < hey > > "); </script> z html.parsers < < hey > >)r ri rH )r r� �encodedr r r �2test_formatter_skips_script_tag_for_html_documents� s zDTestSubstitutions.test_formatter_skips_script_tag_for_html_documentsc C s$ d}t |d�j� }| jd|k� d S )NzF <style type="text/css"> console.log("< < hey > > "); </style> z html.parsers < < hey > >)r ri rH )r r� rt r r r �1test_formatter_skips_style_tag_for_html_documents� s zCTestSubstitutions.test_formatter_skips_style_tag_for_html_documentsc C s | j d�}| jd|jj� � d S )Nz*<div> foo <pre> bar </pre> baz z/<div> foo <pre> bar </pre> baz </div>)r$ r ra �prettify)r r$ r r r �,test_prettify_leaves_preformatted_text_alone� s z>TestSubstitutions.test_prettify_leaves_preformatted_text_alonec C s, t dd�}|jdd� d�}| jd|k� d S )Nz <html><body>foo</body></html>z html.parserc S s | j � S )N)ro )rp r r r rq � s zLTestSubstitutions.test_prettify_accepts_formatter_function.<locals>.<lambda>)rh ZFOO)r rw rH )r r$ Zprettyr r r �(test_prettify_accepts_formatter_function� s z:TestSubstitutions.test_prettify_accepts_formatter_functionc C s" | j d�}| jtt|j� �� d S )Nz<a></a>)r$ r r( �typerw )r r$ r r r �(test_prettify_outputs_unicode_by_default� s z:TestSubstitutions.test_prettify_outputs_unicode_by_defaultc C s$ | j d�}| jtt|jd��� d S )Nz<a></a>zutf-8)r$ r �bytesrz rw )r r$ r r r � test_prettify_can_encode_data s z/TestSubstitutions.test_prettify_can_encode_datac C s0 d}| j |�}|jjd�}| j||jd�� d S )Nu <b>Sacré bleu!</b>zutf-8)r$ r" ri r )r r� r$ rt r r r �,test_html_entity_substitution_off_by_default s z>TestSubstitutions.test_html_entity_substitution_off_by_defaultc C s� d}| j |�}| j|jd d� |jd�}| jd|k� |jd�}| jd|k� |jd�}| jd |k� |jd �jd �}| jd |k� d S ) NzE<meta content="text/html; charset=x-sjis" http-equiv="Content-type"/>Zcontentztext/html; charset=x-sjiszutf-8s charset=utf-8�euc_jps charset=euc_jpz shift-jiss charset=shift-jiszutf-16zcharset=utf-16)r$ r �metari rH rj )r Zmeta_tagr$ �utf_8r � shift_jisZutf_16_ur r r �test_encoding_substitution s z,TestSubstitutions.test_encoding_substitutionc C s2 d}t d�}| j||d�}| j|jd jd� d S )Nz`<head><meta content="text/html; charset=x-sjis" http-equiv="Content-type"/></head><pre>foo</pre>Zpre)Z parse_onlyr )r r$ r r� rN )r r� ry r$ r r r �;test_encoding_substitution_doesnt_happen_if_tag_is_strained$ s zMTestSubstitutions.test_encoding_substitution_doesnt_happen_if_tag_is_strainedN)r r r rj rk rl rm rn rr rs ru rv rx ry r{ r} r~ r� r� r r r r rf � s rf c @ sP e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dS )� TestEncodingz0Test the ability to encode objects into strings.c C s. d}| j |�}| j|jjjd�djd�� d S )Nu <b>☃</b>zutf-8u ☃)r$ r r" r ri )r r� r$ r r r �"test_unicode_string_can_be_encoded2 s z/TestEncoding.test_unicode_string_can_be_encodedc C s, d}| j |�}| j|jjd�|jd�� d S )Nu <b>☃</b>zutf-8)r$ r r" ri )r r� r$ r r r �1test_tag_containing_unicode_string_can_be_encoded8 s z>TestEncoding.test_tag_containing_unicode_string_can_be_encodedc C s& d}| j |�}| j|jjd�d� d S )Nu <b>☃</b>�asciis <b>☃</b>)r$ r r" ri )r r� r$ r r r �<test_encoding_substitutes_unrecognized_characters_by_default> s zITestEncoding.test_encoding_substitutes_unrecognized_characters_by_defaultc C s& d}| j |�}| jt|jddd� d S )Nu <b>☃</b>r� �strict)�errors)r$ r� �UnicodeEncodeErrorri )r r� r$ r r r � test_encoding_can_be_made_strictC s z-TestEncoding.test_encoding_can_be_made_strictc C s$ d}| j |�}| jd|jj� � d S )Nu <b>☃</b>u ☃)r$ r r" Zdecode_contents)r r� r$ r r r �test_decode_contentsI s z!TestEncoding.test_decode_contentsc C s. d}| j |�}| jdjd�|jjdd�� d S )Nu <b>☃</b>u ☃rg )r\ )r$ r ri r" Zencode_contents)r r� r$ r r r �test_encode_contentsN s z!TestEncoding.test_encode_contentsc C s* d}| j |�}| jdjd�|jj� � d S )Nu <b>☃</b>u ☃rg )r$ r ri r" ZrenderContents)r r� r$ r r r � test_deprecated_renderContentsU s z+TestEncoding.test_deprecated_renderContentsc C s8 d}| j |�}tr$| j|t|�� n| jdt|�� d S )Nu <b>☃</b>s <b>\u2603</b>)r$ r r �repr)r r� r$ r r r � test_repr[ s zTestEncoding.test_reprN) r r r r1 r� r� r� r� r� r� r� r� r r r r r� / s r� c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S ) � TestNavigableStringSubclassesc C sX | j d�}td�}|jd|� | jt|�d� | j|jdd�d� | j|jd d� d S )Nr� rB r= z<![CDATA[foo]]>)r) r )r$ r r� r r( r% r� )r r$ r* r r r � test_cdatae s z(TestNavigableStringSubclasses.test_cdatac sV d� _ � fdd�}� jd�}td�}|jd|� � jd|j|d�� � jd� j � d S ) zkText inside a CData object is passed into the formatter. But the return value is ignored. r c s � j d7 _ dS )Nr= zBITTER FAILURE)�count)�args)r r r � incrementv s zNTestNavigableStringSubclasses.test_cdata_is_never_formatted.<locals>.incrementr� z<><><>r= s <![CDATA[<><><>]]>)rh N)r� r$ r r� r ri )r r� r$ r* r )r r � test_cdata_is_never_formattedo s z;TestNavigableStringSubclasses.test_cdata_is_never_formattedc C s2 t d�}| jd�}|jd|� | j|j� d� d S )NrB r� r= s <!DOCTYPE foo> )r r$ r� r ri )r Zdoctyper$ r r r � test_doctype_ends_in_newline� s z:TestNavigableStringSubclasses.test_doctype_ends_in_newlinec C s t d�}| jd|j� � d S )NrB z<?foo?>)r r Z output_ready)r r r r r �test_declaration� s z.TestNavigableStringSubclasses.test_declarationN)r r r r� r� r� r� r r r r r� c s r� c @ s� e Zd ZdZdd� Zdd� ZeZdd� Zdd � Zd d � Z d d � Z dd� Z dd� Z dd� Z dd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d!� Zd"d#� Zd$d%� Zd&d'� Zd(d)� Zd*d+� Zd,d-� Zd.d/� Zd0d1� Z d2d3� Z d4d5� Z d6d7� Z d8d9� Z d:d;� Z!d<d=� Z"d>d?� Z#d@dA� Z$dBdC� Z%dDdE� Z&dFdG� Z'dHdI� Z(dJdK� Z)dLdM� Z*dNdO� Z+dPdQ� Z,dRdS� Z-dTdU� Z.dVdW� Z/dXdY� Z0dZd[� Z1d\d]� Z2d^d_� Z3d`da� Z4dbdc� Z5ddde� Z6dfdg� Z7dhdi� Z8djdk� Z9dldm� Z:dndo� Z;dpdq� Z<drds� Z=dtS )u�TestSoupSelectora� <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> <html> <head> <title>The title</title> <link rel="stylesheet" href="blah.css" type="text/css" id="l1"> </head> <body> <custom-dashed-tag class="dashed" id="dash1">Hello there.</custom-dashed-tag> <div id="main" class="fancy"> <div id="inner"> <h1 id="header1">An H1</h1> <p>Some text</p> <p class="onep" id="p1">Some more text</p> <h2 id="header2">An H2</h2> <p class="class1 class2 class3" id="pmulti">Another</p> <a href="http://bob.example.org/" rel="friend met" id="bob">Bob</a> <h2 id="header3">Another H2</h2> <a id="me" href="http://simonwillison.net/" rel="me">me</a> <span class="s1"> <a href="#" id="s1a1">span1a1</a> <a href="#" id="s1a2">span1a2 <span id="s1a2s1">test</span></a> <span class="span2"> <a href="#" id="s2a1">span2a1</a> </span> <span class="span3"></span> <custom-dashed-tag class="dashed" id="dash2"/> <div data-tag="dashedvalue" id="data1"/> </span> </div> <x id="xid"> <z id="zida"/> <z id="zidab"/> <z id="zidac"/> </x> <y id="yid"> <z id="zidb"/> </y> <p lang="en" id="lang-en">English</p> <p lang="en-gb" id="lang-en-gb">English UK</p> <p lang="en-us" id="lang-en-us">English US</p> <p lang="fr" id="lang-fr">French</p> </div> <div id="footer"> </div> c C s t | jd�| _d S )Nz html.parser)r �HTMLr$ )r r r r rR � s zTestSoupSelector.setUpc K sR dd� | j j|f|�D �}|j� |j� | j||d|dj|�dj|�f � d S )Nc S s g | ] }|d �qS )r r )r �elr r r r � s z2TestSoupSelector.assertSelects.<locals>.<listcomp>z$Selector %s, expected [%s], got [%s]z, )r$ �select�sortr �join)r �selector� expected_ids�kwargsZel_idsr r r r � s z TestSoupSelector.assertSelectsc G s" x |D ]\}}| j ||� qW d S )N)� assertSelect)r Ztestsr� r� r r r �assertSelectMultiple� s z%TestSoupSelector.assertSelectMultiplec C sF | j jd�}| jt|�d� | j|d jd� | j|d jdg� d S )Nrh r= r z The title)r$ r� r r, rN r� )r �elsr r r �test_one_tag_one� s z!TestSoupSelector.test_one_tag_onec C sX | j jd�}| jt|�d� x|D ]}| j|jd� q"W | j jd�}| jd|d � d S )Nra � �mainr )r$ r� r r, rN � select_one)r r� ra r� r r r �test_one_tag_many� s z"TestSoupSelector.test_one_tag_manyc C s | j jd�}| jd |� d S )NZnonexistenttag)r$ r� r )r �matchr r r �(test_select_one_returns_none_if_no_match� s z9TestSoupSelector.test_select_one_returns_none_if_no_matchc C s | j jd�}| jdddg� d S )Nzdiv div�inner�data1)r$ r� r )r r� r r r �test_tag_in_tag_one� s z$TestSoupSelector.test_tag_in_tag_onec C s& x dD ]}| j |ddddg� qW d S ) N�html div� html body div�body divr� r� r� �footer)r� r� r� )r )r r� r r r �test_tag_in_tag_many� s z%TestSoupSelector.test_tag_in_tag_manyc C sB | j ddgdd� | j dddgdd� | j dd ddd gd d� d S ) Nzhtml divr� r= )r: z html body divr� r+ zbody divr� r� r> )r )r r r r � test_limit� s zTestSoupSelector.test_limitc C s | j t| jjd��d� d S )N�delr )r r, r$ r� )r r r r �test_tag_no_match� s z"TestSoupSelector.test_tag_no_matchc C s | j t| jjd� d S )Nztag%t)r� r� r$ r� )r r r r �test_invalid_tag� s z!TestSoupSelector.test_invalid_tagc C s | j dddg� d S )Nzcustom-dashed-tag�dash1�dash2)r )r r r r �test_select_dashed_tag_ids� s z+TestSoupSelector.test_select_dashed_tag_idsc C s6 | j jd�}| j|d jd� | j|d d d� d S )Nz custom-dashed-tag[id="dash2"]r zcustom-dashed-tagr r� )r$ r� r rN )r Zdashedr r r �test_select_dashed_by_id� s z)TestSoupSelector.test_select_dashed_by_idc C s | j | jjd�d jd� d S )Nzbody > custom-dashed-tagr z Hello there.)r r$ r� r) )r r r r �test_dashed_tag_text s z%TestSoupSelector.test_dashed_tag_textc C s | j | jjd�| jjd�� d S )Nzcustom-dashed-tag)r r$ r� r- )r r r r �#test_select_dashed_matches_find_all s z4TestSoupSelector.test_select_dashed_matches_find_allc C s | j ddgfdddgf� d S )NZh1�header1Zh2�header2�header3)r� )r r r r �test_header_tags s z!TestSoupSelector.test_header_tagsc C sV xPd D ]H}| j j|�}| jt|�d� | j|d jd� | j|d d dg� qW d S ) N�.onep�p.onep� html p.onepr= r r� rm �onep)r� r� r� )r$ r� r r, rN )r r� r� r r r �test_class_one s z TestSoupSelector.test_class_onec C s | j jd�}| jt|�d� d S )Nzdiv.onepr )r$ r� r r, )r r� r r r �test_class_mismatched_tag s z*TestSoupSelector.test_class_mismatched_tagc C s xdD ]}| j |dg� qW d S )N� div#inner�#inner� div div#innerr� )r� r� r� )r )r r� r r r � test_one_id s z TestSoupSelector.test_one_idc C s | j jd�}| jt|�d� d S )Nz #doesnotexistr )r$ r� r r, )r r� r r r � test_bad_id s z TestSoupSelector.test_bad_idc C sf | j jd�}| jt|�d� x|D ]}| j|jd� q"W | j|d d dg� | j|d jd�� d S )Nz div#inner pr9 r� r= rm r� r )r$ r� r r, rN r7 r6 )r r� r� r r r �test_items_in_id# s z!TestSoupSelector.test_items_in_idc C s* x$dD ] }| j t| jj|��d� qW d S )N� div#main del�div#main div.oops� div div#mainr )r� r� r� )r r, r$ r� )r r� r r r �test_a_bunch_of_emptys+ s z'TestSoupSelector.test_a_bunch_of_emptysc C s xd D ]}| j |d g� qW d S ) N�.class1�p.class1�.class2�p.class2�.class3�p.class3� html p.class2�div#inner .class2�pmulti)r� r� r� r� r� r� r� r� )r )r r� r r r �test_multi_class_support/ s z)TestSoupSelector.test_multi_class_supportc C s xdD ]}| j |dg� qW d S )N�.class1.class3�.class3.class2�.class1.class2.class3r� )r� r� r� )r )r r� r r r �test_multi_class_selection4 s z+TestSoupSelector.test_multi_class_selectionc C s" | j dddg� | j ddg� d S )Nz.s1 > a�s1a1�s1a2z .s1 > a span�s1a2s1)r )r r r r �test_child_selector9 s z$TestSoupSelector.test_child_selectorc C s | j ddg� d S )Nz.s1 > a#s1a2 spanr� )r )r r r r �test_child_selector_id= s z'TestSoupSelector.test_child_selector_idc C st | j ddgfddgfddgfddgfddgfddgfd dgfd g fd dgfd dgfd dgfdg fdg fdg f� d S )Nzp[class="onep"]r� z p[id="p1"]z[class="onep"]z [id="p1"]zlink[rel="stylesheet"]�l1zlink[type="text/css"]zlink[href="blah.css"]zlink[href="no-blah.css"]z[rel="stylesheet"]z[type="text/css"]z[href="blah.css"]z[href="no-blah.css"]zp[href="no-blah.css"])r� )r r r r �test_attribute_equals@ s z&TestSoupSelector.test_attribute_equalsc C s\ | j ddgfddgfddgfddgfddgfddgfdd gfd d gfd d gfd d gf� d S ) Nzp[class~="class1"]r� zp[class~="class2"]zp[class~="class3"]z[class~="class1"]z[class~="class2"]z[class~="class3"]za[rel~="friend"]�bobz a[rel~="met"]z[rel~="friend"]z [rel~="met"])r� )r r r r �test_attribute_tildeR s z%TestSoupSelector.test_attribute_tildec C sv | j ddgfddgfdg fdg fdg fddgfdd d gfd d d gfd d dgfdd dgfddgfdd gfddgf� d S )Nz[rel^="style"]r� zlink[rel^="style"]znotlink[rel^="notstyle"]z[rel^="notstyle"]zlink[rel^="notstyle"]zlink[href^="bla"]za[href^="http://"]r� �mez[href^="http://"]z [id^="p"]r� r� z [id^="m"]r� z div[id^="m"]z a[id^="m"]zdiv[data-tag^="dashed"]r� )r� )r r r r �test_attribute_startswith` s z*TestSoupSelector.test_attribute_startswithc C sH | j ddgfddgfddgfdddddd d d d gfd dgfdg f� d S )Nz[href$=".css"]r� zlink[href$=".css"]z link[id$="1"]z [id$="1"]r� r� r� r� �s2a1r� r� z div[id$="1"]z[id$="noending"])r� )r r r r �test_attribute_endswithq s z(TestSoupSelector.test_attribute_endswithc C s� | j ddgfddgfdg fdg fdg fddgfdd d gfd d d gfddgfdd gfddgfddgfddgfdddd ddddddg fd dgfd g fd d d dgfd d d gfd dgfd!dd"gfd#d"gfd$dgf� d S )%Nz[rel*="style"]r� zlink[rel*="style"]znotlink[rel*="notstyle"]z[rel*="notstyle"]zlink[rel*="notstyle"]zlink[href*="bla"]z[href*="http://"]r� r� z [id*="p"]r� r� z div[id*="m"]r� z a[id*="m"]z[href*=".css"]zlink[href*=".css"]z link[id*="1"]z [id*="1"]r� r� r� r� r� r� r� z div[id*="1"]z[id*="noending"]z [href*="."]z a[href*="."]zlink[href*="."]z div[id*="n"]r� z div[id*="nn"]zdiv[data-tag*="edval"])r� )r r r r �test_attribute_contains{ s. z(TestSoupSelector.test_attribute_containsc C s2 | j ddddgfddddgfddgfdg f� d S ) Nz p[lang|="en"]zlang-enz lang-en-gbz lang-en-usz [lang|="en"]z p[lang|="fr"]zlang-frz p[lang|="gb"])r� )r r r r � test_attribute_exact_or_hypen� s z.TestSoupSelector.test_attribute_exact_or_hypenc C sV | j ddddgfddgfdddgfddd d d gfd d dgfdg fdg fddgf� d S )Nz[rel]r� r� r� z link[rel]za[rel]z[lang]zlang-enz lang-en-gbz lang-en-uszlang-frzp[class]r� r� z[blah]zp[blah]z div[data-tag]r� )r� )r r r r �test_attribute_exists� s z&TestSoupSelector.test_attribute_existsc C s, d}t |d�}|jd�\}| jd|j� d S )Nz]<div style="display: wrong">nope</div> <div style="display: right">yes</div> z html.parserzdiv[style="display: right"]�yes)r r� r r )r r� r$ Zchosenr r r �"test_quoted_space_in_selector_name� s z3TestSoupSelector.test_quoted_space_in_selector_namec C s( | j t| jjd� | j t| jjd� d S )Nza:no-such-pseudoclassza:nth-of-type(a))r� r r$ r� )r r r r � test_unsupported_pseudoclass� s z-TestSoupSelector.test_unsupported_pseudoclassc C s� | j jd�}| jt|�d� | j|d jd� | j jd�}| jt|�d� | j|d jd� | j jd�}| jt|�d� | jt| j jd� d S ) Nzdiv#inner p:nth-of-type(1)r= r z Some textzdiv#inner p:nth-of-type(3)ZAnotherzdiv#inner p:nth-of-type(4)zdiv p:nth-of-type(0))r$ r� r r, r r� r� )r r� r r r �test_nth_of_type� s z!TestSoupSelector.test_nth_of_typec C s2 | j jd�}| jt|�d� | j|d jd� d S )Nz div#inner > p:nth-of-type(1)r= r z Some text)r$ r� r r, r )r r� r r r �"test_nth_of_type_direct_descendant� s z3TestSoupSelector.test_nth_of_type_direct_descendantc C s | j ddg� d S )Nz#inner > p:nth-of-type(2)r� )r )r r r r �"test_id_child_selector_nth_of_type� s z3TestSoupSelector.test_id_child_selector_nth_of_typec C s. | j jddd�}|jd�}| j|ddg� d S )Nra r� )r r� r� )r$ r% r� r )r r� �selectedr r r �test_select_on_element� s z'TestSoupSelector.test_select_on_elementc C s | j ddg� | j dg � d S )Nz .fancy #innerr� z.normal #inner)r )r r r r �test_overspecified_child_id� s z,TestSoupSelector.test_overspecified_child_idc C sB | j ddg� | j ddg� | j ddg� | jg | jjd�� d S )Nz#p1 + h2r� z #p1 + h2 + pr� z#p1 + #header2 + .class1z#p1 + p)r r r$ r� )r r r r � test_adjacent_sibling_selector� s z/TestSoupSelector.test_adjacent_sibling_selectorc C sR | j dddg� | j ddg� | j ddg� | j ddg� | jg | jjd�� d S ) Nz#p1 ~ h2r� r� z#p1 ~ #header2z #p1 ~ h2 + ar� z#p1 ~ h2 + [rel="me"]z #inner ~ h2)r r r$ r� )r r r r � test_general_sibling_selector� s z.TestSoupSelector.test_general_sibling_selectorc C s | j t| jjd� d S )Nzh1 >)r� r� r$ r� )r r r r �test_dangling_combinator� s z)TestSoupSelector.test_dangling_combinatorc C s | j ddddg� d S )Nz p[lang] ~ pz lang-en-gbz lang-en-uszlang-fr)r )r r r r �2test_sibling_combinator_wont_select_same_tag_twice� s zCTestSoupSelector.test_sibling_combinator_wont_select_same_tag_twicec C s | j dddg� d S )Nzx, y�xid�yid)r )r r r r �test_multiple_select� s z%TestSoupSelector.test_multiple_selectc C s | j dddg� d S )Nzx,yr� r )r )r r r r �"test_multiple_select_with_no_space� s z3TestSoupSelector.test_multiple_select_with_no_spacec C s | j dddg� d S )Nzx, yr� r )r )r r r r �$test_multiple_select_with_more_space� s z5TestSoupSelector.test_multiple_select_with_more_spacec C s | j ddg� d S )Nzx, xr� )r )r r r r � test_multiple_select_duplicated s z0TestSoupSelector.test_multiple_select_duplicatedc C s | j dddg� d S )Nzx, y ~ p[lang=fr]r� zlang-fr)r )r r r r � test_multiple_select_sibling s z-TestSoupSelector.test_multiple_select_siblingc C s | j dddg� d S )Nzx, y > zr� �zidb)r )r r r r �.test_multiple_select_tag_and_direct_descendant s z?TestSoupSelector.test_multiple_select_tag_and_direct_descendantc C s | j dddddddg� d S )Nz div > x, y, zr� r �zidar �zidab�zidac)r )r r r r �/test_multiple_select_direct_descendant_and_tags s z@TestSoupSelector.test_multiple_select_direct_descendant_and_tagsc C s | j dddddddg� d S )Nz div x,y, zr� r r r r r )r )r r r r �(test_multiple_select_indirect_descendant s z9TestSoupSelector.test_multiple_select_indirect_descendantc C s( | j t| jjd� | j t| jjd� d S )Nz,x, yzx,,y)r� r� r$ r� )r r r r � test_invalid_multiple_select s z-TestSoupSelector.test_invalid_multiple_selectc C s | j dddg� d S )Nzp[lang=en], p[lang=en-gb]zlang-enz lang-en-gb)r )r r r r �test_multiple_select_attrs s z+TestSoupSelector.test_multiple_select_attrsc C s | j ddddg� d S )Nz*x, y > z[id=zida], z[id=zidab], z[id=zidb]r� r r )r )r r r r �test_multiple_select_ids s z)TestSoupSelector.test_multiple_select_idsc C s | j dddg� d S )Nzbody > div > x, y > zr� r )r )r r r r �test_multiple_select_nested s z,TestSoupSelector.test_multiple_select_nestedc C sF d}t |d�}|jd�}| jdt|�� x|jddgd�D ]}q:W d S )Nz3<div class="c1"/><div class="c2"/><div class="c1"/>z html.parserz.c1, .c2r9 Zc1Zc2)ro )r r� r r, r- )r r� r$ r� r� r r r � test_select_duplicate_elements s z/TestSoupSelector.test_select_duplicate_elementsN)>r r r r� rR r r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r r r r r r r r r r r r r r r� � sv 1 r� )2r1 Zpdbr rX rR r6 r0 Zbs4r Z bs4.builderr r Z bs4.elementr r r r r r r r Z bs4.testingr r r� r� Z LXML_PRESENTr r! r2 rL rP rd r� r� r� r� r� r� r� r� r� r� r, rF rQ rf r� r� r� r r r r �<module> sN ( ;O 83(3--* 6n+a 4* tests/__pycache__/test_builder_registry.cpython-36.pyc 0000644 00000011553 15246412105 0017166 0 ustar 00 3 c�T� � @ s� d Z ddlZddlZddlmZ ddlmZmZm Z yddlm Z dZ W n e k r` dZ Y nX yddlm Z mZ dZW n e k r� dZY nX G d d � d ej�ZG d d � d ej�ZdS ) z Tests of the builder registry.� N)� BeautifulSoup)�builder_registry�HTMLParserTreeBuilder�TreeBuilderRegistry)�HTML5TreeBuilderTF)�LXMLTreeBuilderForXML�LXMLTreeBuilderc @ s0 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd S ) �BuiltInRegistryTestz@Test the built-in registry with the default builders registered.c C s` t r| jtjdd�t� t r0| jtjdd�t� | jtjdd�t� tr\| jtjdd�t� d S )N�fast�htmlZ permissive�xml�strict�html5lib) � LXML_PRESENT� assertEqual�registry�lookupr r r �HTML5LIB_PRESENTr )�self� r �+/usr/lib/python3.6/test_builder_registry.py�test_combination s z$BuiltInRegistryTest.test_combinationc C sj t r*| jtjd�t� | jtjd�t� n<| jtjd�d � trT| jtjd�t� n| jtjd�t� d S )Nr r ) r r r r r r r r r )r r r r �test_lookup_by_markup_type. s z.BuiltInRegistryTest.test_lookup_by_markup_typec C sX t r,| jtjdd�t� | jtjdd�t� trB| jtjd�t� | jtjd�t� d S )NZlxmlr r r z html.parser) r r r r r r r r r )r r r r �test_named_library9 s z&BuiltInRegistryTest.test_named_libraryc C sJ t jdd��"}tddd� tdddgd� W d Q R X | jttddd� d S )NT)�record� r )�featuresr zno-such-feature)�warnings�catch_warningsr Z assertRaises� ValueError)r �wr r r �*test_beautifulsoup_constructor_does_lookupF s z>BuiltInRegistryTest.test_beautifulsoup_constructor_does_lookupN)�__name__� __module__� __qualname__�__doc__r r r r! r r r r r s r c @ sX e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dS )� RegistryTestz.Test the TreeBuilderRegistry class in general.c C s t � | _d S )N)r r )r r r r �setUpY s zRegistryTest.setUpc G s, t ddj|� tfd|i�}| jj|� |S )NZBuilder_�_r )�type�join�objectr �register)r Z feature_list�clsr r r �builder_for_features\ s z!RegistryTest.builder_for_featuresc C s2 | j � }| j| jjd�d � | j| jj� |� d S )N�foo)r. r r r )r �builderr r r � test_register_with_no_featuresc s z+RegistryTest.test_register_with_no_featuresc C s8 | j dd�}| j| jjd�|� | j| jjd�|� d S )Nr/ �bar)r. r r r )r r0 r r r �0test_register_with_features_makes_lookup_succeedn s z=RegistryTest.test_register_with_features_makes_lookup_succeedc C s$ | j dd�}| j| jjd�d � d S )Nr/ r2 �baz)r. r r r )r r0 r r r �4test_lookup_fails_when_no_builder_implements_features s zARegistryTest.test_lookup_fails_when_no_builder_implements_featurec C s* | j d�}| j d�}| j| jj� |� d S )Nr/ r2 )r. r r r )r �builder1�builder2r r r �Ctest_lookup_gets_most_recent_registration_when_no_feature_specifiedw s zPRegistryTest.test_lookup_gets_most_recent_registration_when_no_feature_specifiedc C s | j | jj� d � d S )N)r r r )r r r r �2test_lookup_fails_when_no_tree_builders_registered| s z?RegistryTest.test_lookup_fails_when_no_tree_builders_registeredc C sv | j d�}| j d�}| j ddd�}| j ddd�}| j d�}| j d�}| j| jjdd�|� | j| jjddd�|� d S )Nr/ r2 r4 Zquux)r. r r r )r Zhas_oneZ has_the_otherZhas_both_earlyZ has_both_lateZ lacks_oner r r �<test_lookup_gets_most_recent_builder_supporting_all_features s zIRegistryTest.test_lookup_gets_most_recent_builder_supporting_all_featuresc C s2 | j dd�}| j dd�}| j| jjdd�d � d S )Nr/ r2 r4 )r. r r r )r r6 r7 r r r �:test_lookup_fails_when_cannot_reconcile_requested_features� s zGRegistryTest.test_lookup_fails_when_cannot_reconcile_requested_featuresN) r" r# r$ r% r' r. r1 r3 r5 r8 r9 r: r; r r r r r& V s r& )r% Zunittestr Zbs4r Z bs4.builderr r r r r r � ImportErrorr r r ZTestCaser r& r r r r �<module> s 9 builder/_htmlparser.py 0000644 00000031615 15246412105 0011072 0 ustar 00 # encoding: utf-8 """Use the HTMLParser library to parse HTML files that aren't too bad.""" # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. __all__ = [ 'HTMLParserTreeBuilder', ] from html.parser import HTMLParser try: from html.parser import HTMLParseError except ImportError as e: # HTMLParseError is removed in Python 3.5. Since it can never be # thrown in 3.5, we can just define our own class as a placeholder. class HTMLParseError(Exception): pass import sys import warnings # Starting in Python 3.2, the HTMLParser constructor takes a 'strict' # argument, which we'd like to set to False. Unfortunately, # http://bugs.python.org/issue13273 makes strict=True a better bet # before Python 3.2.3. # # At the end of this file, we monkeypatch HTMLParser so that # strict=True works well on Python 3.2.2. major, minor, release = sys.version_info[:3] CONSTRUCTOR_TAKES_STRICT = major == 3 and minor == 2 and release >= 3 CONSTRUCTOR_STRICT_IS_DEPRECATED = major == 3 and minor == 3 CONSTRUCTOR_TAKES_CONVERT_CHARREFS = major == 3 and minor >= 4 from bs4.element import ( CData, Comment, Declaration, Doctype, ProcessingInstruction, ) from bs4.dammit import EntitySubstitution, UnicodeDammit from bs4.builder import ( HTML, HTMLTreeBuilder, STRICT, ) HTMLPARSER = 'html.parser' class BeautifulSoupHTMLParser(HTMLParser): def __init__(self, *args, **kwargs): HTMLParser.__init__(self, *args, **kwargs) # Keep a list of empty-element tags that were encountered # without an explicit closing tag. If we encounter a closing tag # of this type, we'll associate it with one of those entries. # # This isn't a stack because we don't care about the # order. It's a list of closing tags we've already handled and # will ignore, assuming they ever show up. self.already_closed_empty_element = [] def error(self, msg): """In Python 3, HTMLParser subclasses must implement error(), although this requirement doesn't appear to be documented. In Python 2, HTMLParser implements error() as raising an exception. In any event, this method is called only on very strange markup and our best strategy is to pretend it didn't happen and keep going. """ warnings.warn(msg) def handle_startendtag(self, name, attrs): # This is only called when the markup looks like # <tag/>. # is_startend() tells handle_starttag not to close the tag # just because its name matches a known empty-element tag. We # know that this is an empty-element tag and we want to call # handle_endtag ourselves. tag = self.handle_starttag(name, attrs, handle_empty_element=False) self.handle_endtag(name) def handle_starttag(self, name, attrs, handle_empty_element=True): # XXX namespace attr_dict = {} for key, value in attrs: # Change None attribute values to the empty string # for consistency with the other tree builders. if value is None: value = '' attr_dict[key] = value attrvalue = '""' #print "START", name tag = self.soup.handle_starttag(name, None, None, attr_dict) if tag and tag.is_empty_element and handle_empty_element: # Unlike other parsers, html.parser doesn't send separate end tag # events for empty-element tags. (It's handled in # handle_startendtag, but only if the original markup looked like # <tag/>.) # # So we need to call handle_endtag() ourselves. Since we # know the start event is identical to the end event, we # don't want handle_endtag() to cross off any previous end # events for tags of this name. self.handle_endtag(name, check_already_closed=False) # But we might encounter an explicit closing tag for this tag # later on. If so, we want to ignore it. self.already_closed_empty_element.append(name) def handle_endtag(self, name, check_already_closed=True): #print "END", name if check_already_closed and name in self.already_closed_empty_element: # This is a redundant end tag for an empty-element tag. # We've already called handle_endtag() for it, so just # check it off the list. # print "ALREADY CLOSED", name self.already_closed_empty_element.remove(name) else: self.soup.handle_endtag(name) def handle_data(self, data): self.soup.handle_data(data) def handle_charref(self, name): # XXX workaround for a bug in HTMLParser. Remove this once # it's fixed in all supported versions. # http://bugs.python.org/issue13633 if name.startswith('x'): real_name = int(name.lstrip('x'), 16) elif name.startswith('X'): real_name = int(name.lstrip('X'), 16) else: real_name = int(name) data = None if real_name < 256: # HTML numeric entities are supposed to reference Unicode # code points, but sometimes they reference code points in # some other encoding (ahem, Windows-1252). E.g. “ # instead of É for LEFT DOUBLE QUOTATION MARK. This # code tries to detect this situation and compensate. for encoding in (self.soup.original_encoding, 'windows-1252'): if not encoding: continue try: data = bytearray([real_name]).decode(encoding) except UnicodeDecodeError as e: pass if not data: try: data = chr(real_name) except (ValueError, OverflowError) as e: pass data = data or "\N{REPLACEMENT CHARACTER}" self.handle_data(data) def handle_entityref(self, name): character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name) if character is not None: data = character else: # If this were XML, it would be ambiguous whether "&foo" # was an character entity reference with a missing # semicolon or the literal string "&foo". Since this is # HTML, we have a complete list of all character entity references, # and this one wasn't found, so assume it's the literal string "&foo". data = "&%s" % name self.handle_data(data) def handle_comment(self, data): self.soup.endData() self.soup.handle_data(data) self.soup.endData(Comment) def handle_decl(self, data): self.soup.endData() if data.startswith("DOCTYPE "): data = data[len("DOCTYPE "):] elif data == 'DOCTYPE': # i.e. "<!DOCTYPE>" data = '' self.soup.handle_data(data) self.soup.endData(Doctype) def unknown_decl(self, data): if data.upper().startswith('CDATA['): cls = CData data = data[len('CDATA['):] else: cls = Declaration self.soup.endData() self.soup.handle_data(data) self.soup.endData(cls) def handle_pi(self, data): self.soup.endData() self.soup.handle_data(data) self.soup.endData(ProcessingInstruction) class HTMLParserTreeBuilder(HTMLTreeBuilder): is_xml = False picklable = True NAME = HTMLPARSER features = [NAME, HTML, STRICT] def __init__(self, *args, **kwargs): if CONSTRUCTOR_TAKES_STRICT and not CONSTRUCTOR_STRICT_IS_DEPRECATED: kwargs['strict'] = False if CONSTRUCTOR_TAKES_CONVERT_CHARREFS: kwargs['convert_charrefs'] = False self.parser_args = (args, kwargs) def prepare_markup(self, markup, user_specified_encoding=None, document_declared_encoding=None, exclude_encodings=None): """ :return: A 4-tuple (markup, original encoding, encoding declared within markup, whether any characters had to be replaced with REPLACEMENT CHARACTER). """ if isinstance(markup, str): yield (markup, None, None, False) return try_encodings = [user_specified_encoding, document_declared_encoding] dammit = UnicodeDammit(markup, try_encodings, is_html=True, exclude_encodings=exclude_encodings) yield (dammit.markup, dammit.original_encoding, dammit.declared_html_encoding, dammit.contains_replacement_characters) def feed(self, markup): args, kwargs = self.parser_args parser = BeautifulSoupHTMLParser(*args, **kwargs) parser.soup = self.soup try: parser.feed(markup) parser.close() except HTMLParseError as e: warnings.warn(RuntimeWarning( "Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help.")) raise e parser.already_closed_empty_element = [] # Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some # 3.2.3 code. This ensures they don't treat markup like <p></p> as a # string. # # XXX This code can be removed once most Python 3 users are on 3.2.3. if major == 3 and minor == 2 and not CONSTRUCTOR_TAKES_STRICT: import re attrfind_tolerant = re.compile( r'\s*((?<=[\'"\s])[^\s/>][^\s/=>]*)(\s*=+\s*' r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?') HTMLParserTreeBuilder.attrfind_tolerant = attrfind_tolerant locatestarttagend = re.compile(r""" <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name (?:\s+ # whitespace before attribute name (?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name (?:\s*=\s* # value indicator (?:'[^']*' # LITA-enclosed value |\"[^\"]*\" # LIT-enclosed value |[^'\">\s]+ # bare value ) )? ) )* \s* # trailing whitespace """, re.VERBOSE) BeautifulSoupHTMLParser.locatestarttagend = locatestarttagend from html.parser import tagfind, attrfind def parse_starttag(self, i): self.__starttag_text = None endpos = self.check_for_whole_start_tag(i) if endpos < 0: return endpos rawdata = self.rawdata self.__starttag_text = rawdata[i:endpos] # Now parse the data between i+1 and j into a tag and attrs attrs = [] match = tagfind.match(rawdata, i+1) assert match, 'unexpected call to parse_starttag()' k = match.end() self.lasttag = tag = rawdata[i+1:k].lower() while k < endpos: if self.strict: m = attrfind.match(rawdata, k) else: m = attrfind_tolerant.match(rawdata, k) if not m: break attrname, rest, attrvalue = m.group(1, 2, 3) if not rest: attrvalue = None elif attrvalue[:1] == '\'' == attrvalue[-1:] or \ attrvalue[:1] == '"' == attrvalue[-1:]: attrvalue = attrvalue[1:-1] if attrvalue: attrvalue = self.unescape(attrvalue) attrs.append((attrname.lower(), attrvalue)) k = m.end() end = rawdata[k:endpos].strip() if end not in (">", "/>"): lineno, offset = self.getpos() if "\n" in self.__starttag_text: lineno = lineno + self.__starttag_text.count("\n") offset = len(self.__starttag_text) \ - self.__starttag_text.rfind("\n") else: offset = offset + len(self.__starttag_text) if self.strict: self.error("junk characters in start tag: %r" % (rawdata[k:endpos][:20],)) self.handle_data(rawdata[i:endpos]) return endpos if end.endswith('/>'): # XHTML-style empty tag: <span attr="value" /> self.handle_startendtag(tag, attrs) else: self.handle_starttag(tag, attrs) if tag in self.CDATA_CONTENT_ELEMENTS: self.set_cdata_mode(tag) return endpos def set_cdata_mode(self, elem): self.cdata_elem = elem.lower() self.interesting = re.compile(r'</\s*%s\s*>' % self.cdata_elem, re.I) BeautifulSoupHTMLParser.parse_starttag = parse_starttag BeautifulSoupHTMLParser.set_cdata_mode = set_cdata_mode CONSTRUCTOR_TAKES_STRICT = True builder/_lxml.py 0000644 00000022575 15246412105 0007672 0 ustar 00 # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. __all__ = [ 'LXMLTreeBuilderForXML', 'LXMLTreeBuilder', ] try: from collections.abc import Callable # Python 3.6 except ImportError as e: from collections import Callable from io import BytesIO from io import StringIO from lxml import etree from bs4.element import ( Comment, Doctype, NamespacedAttribute, ProcessingInstruction, XMLProcessingInstruction, ) from bs4.builder import ( FAST, HTML, HTMLTreeBuilder, PERMISSIVE, ParserRejectedMarkup, TreeBuilder, XML) from bs4.dammit import EncodingDetector LXML = 'lxml' class LXMLTreeBuilderForXML(TreeBuilder): DEFAULT_PARSER_CLASS = etree.XMLParser is_xml = True processing_instruction_class = XMLProcessingInstruction NAME = "lxml-xml" ALTERNATE_NAMES = ["xml"] # Well, it's permissive by XML parser standards. features = [NAME, LXML, XML, FAST, PERMISSIVE] CHUNK_SIZE = 512 # This namespace mapping is specified in the XML Namespace # standard. DEFAULT_NSMAPS = {'http://www.w3.org/XML/1998/namespace' : "xml"} def default_parser(self, encoding): # This can either return a parser object or a class, which # will be instantiated with default arguments. if self._default_parser is not None: return self._default_parser return etree.XMLParser( target=self, strip_cdata=False, recover=True, encoding=encoding) def parser_for(self, encoding): # Use the default parser. parser = self.default_parser(encoding) if isinstance(parser, Callable): # Instantiate the parser with default arguments parser = parser(target=self, strip_cdata=False, encoding=encoding) return parser def __init__(self, parser=None, empty_element_tags=None): # TODO: Issue a warning if parser is present but not a # callable, since that means there's no way to create new # parsers for different encodings. self._default_parser = parser if empty_element_tags is not None: self.empty_element_tags = set(empty_element_tags) self.soup = None self.nsmaps = [self.DEFAULT_NSMAPS] def _getNsTag(self, tag): # Split the namespace URL out of a fully-qualified lxml tag # name. Copied from lxml's src/lxml/sax.py. if tag[0] == '{': return tuple(tag[1:].split('}', 1)) else: return (None, tag) def prepare_markup(self, markup, user_specified_encoding=None, exclude_encodings=None, document_declared_encoding=None): """ :yield: A series of 4-tuples. (markup, encoding, declared encoding, has undergone character replacement) Each 4-tuple represents a strategy for parsing the document. """ # Instead of using UnicodeDammit to convert the bytestring to # Unicode using different encodings, use EncodingDetector to # iterate over the encodings, and tell lxml to try to parse # the document as each one in turn. is_html = not self.is_xml if is_html: self.processing_instruction_class = ProcessingInstruction else: self.processing_instruction_class = XMLProcessingInstruction if isinstance(markup, str): # We were given Unicode. Maybe lxml can parse Unicode on # this system? yield markup, None, document_declared_encoding, False if isinstance(markup, str): # No, apparently not. Convert the Unicode to UTF-8 and # tell lxml to parse it as UTF-8. yield (markup.encode("utf8"), "utf8", document_declared_encoding, False) try_encodings = [user_specified_encoding, document_declared_encoding] detector = EncodingDetector( markup, try_encodings, is_html, exclude_encodings) for encoding in detector.encodings: yield (detector.markup, encoding, document_declared_encoding, False) def feed(self, markup): if isinstance(markup, bytes): markup = BytesIO(markup) elif isinstance(markup, str): markup = StringIO(markup) # Call feed() at least once, even if the markup is empty, # or the parser won't be initialized. data = markup.read(self.CHUNK_SIZE) try: self.parser = self.parser_for(self.soup.original_encoding) self.parser.feed(data) while len(data) != 0: # Now call feed() on the rest of the data, chunk by chunk. data = markup.read(self.CHUNK_SIZE) if len(data) != 0: self.parser.feed(data) self.parser.close() except (UnicodeDecodeError, LookupError, etree.ParserError) as e: raise ParserRejectedMarkup(str(e)) def close(self): self.nsmaps = [self.DEFAULT_NSMAPS] def start(self, name, attrs, nsmap={}): # Make sure attrs is a mutable dict--lxml may send an immutable dictproxy. attrs = dict(attrs) nsprefix = None # Invert each namespace map as it comes in. if len(nsmap) == 0 and len(self.nsmaps) > 1: # There are no new namespaces for this tag, but # non-default namespaces are in play, so we need a # separate tag stack to know when they end. self.nsmaps.append(None) elif len(nsmap) > 0: # A new namespace mapping has come into play. inverted_nsmap = dict((value, key) for key, value in list(nsmap.items())) self.nsmaps.append(inverted_nsmap) # Also treat the namespace mapping as a set of attributes on the # tag, so we can recreate it later. attrs = attrs.copy() for prefix, namespace in list(nsmap.items()): attribute = NamespacedAttribute( "xmlns", prefix, "http://www.w3.org/2000/xmlns/") attrs[attribute] = namespace # Namespaces are in play. Find any attributes that came in # from lxml with namespaces attached to their names, and # turn then into NamespacedAttribute objects. new_attrs = {} for attr, value in list(attrs.items()): namespace, attr = self._getNsTag(attr) if namespace is None: new_attrs[attr] = value else: nsprefix = self._prefix_for_namespace(namespace) attr = NamespacedAttribute(nsprefix, attr, namespace) new_attrs[attr] = value attrs = new_attrs namespace, name = self._getNsTag(name) nsprefix = self._prefix_for_namespace(namespace) self.soup.handle_starttag(name, namespace, nsprefix, attrs) def _prefix_for_namespace(self, namespace): """Find the currently active prefix for the given namespace.""" if namespace is None: return None for inverted_nsmap in reversed(self.nsmaps): if inverted_nsmap is not None and namespace in inverted_nsmap: return inverted_nsmap[namespace] return None def end(self, name): self.soup.endData() completed_tag = self.soup.tagStack[-1] namespace, name = self._getNsTag(name) nsprefix = None if namespace is not None: for inverted_nsmap in reversed(self.nsmaps): if inverted_nsmap is not None and namespace in inverted_nsmap: nsprefix = inverted_nsmap[namespace] break self.soup.handle_endtag(name, nsprefix) if len(self.nsmaps) > 1: # This tag, or one of its parents, introduced a namespace # mapping, so pop it off the stack. self.nsmaps.pop() def pi(self, target, data): self.soup.endData() self.soup.handle_data(target + ' ' + data) self.soup.endData(self.processing_instruction_class) def data(self, content): self.soup.handle_data(content) def doctype(self, name, pubid, system): self.soup.endData() doctype = Doctype.for_name_and_ids(name, pubid, system) self.soup.object_was_parsed(doctype) def comment(self, content): "Handle comments as Comment objects." self.soup.endData() self.soup.handle_data(content) self.soup.endData(Comment) def test_fragment_to_document(self, fragment): """See `TreeBuilder`.""" return '<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML): NAME = LXML ALTERNATE_NAMES = ["lxml-html"] features = ALTERNATE_NAMES + [NAME, HTML, FAST, PERMISSIVE] is_xml = False processing_instruction_class = ProcessingInstruction def default_parser(self, encoding): return etree.HTMLParser def feed(self, markup): encoding = self.soup.original_encoding try: self.parser = self.parser_for(encoding) self.parser.feed(markup) self.parser.close() except (UnicodeDecodeError, LookupError, etree.ParserError) as e: raise ParserRejectedMarkup(str(e)) def test_fragment_to_document(self, fragment): """See `TreeBuilder`.""" return '<html><body>%s</body></html>' % fragment builder/__init__.py 0000644 00000027706 15246412105 0010317 0 ustar 00 # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. from collections import defaultdict import itertools import sys from bs4.element import ( CharsetMetaAttributeValue, ContentMetaAttributeValue, HTMLAwareEntitySubstitution, whitespace_re ) __all__ = [ 'HTMLTreeBuilder', 'SAXTreeBuilder', 'TreeBuilder', 'TreeBuilderRegistry', ] # Some useful features for a TreeBuilder to have. FAST = 'fast' PERMISSIVE = 'permissive' STRICT = 'strict' XML = 'xml' HTML = 'html' HTML_5 = 'html5' class TreeBuilderRegistry(object): def __init__(self): self.builders_for_feature = defaultdict(list) self.builders = [] def register(self, treebuilder_class): """Register a treebuilder based on its advertised features.""" for feature in treebuilder_class.features: self.builders_for_feature[feature].insert(0, treebuilder_class) self.builders.insert(0, treebuilder_class) def lookup(self, *features): if len(self.builders) == 0: # There are no builders at all. return None if len(features) == 0: # They didn't ask for any features. Give them the most # recently registered builder. return self.builders[0] # Go down the list of features in order, and eliminate any builders # that don't match every feature. features = list(features) features.reverse() candidates = None candidate_set = None while len(features) > 0: feature = features.pop() we_have_the_feature = self.builders_for_feature.get(feature, []) if len(we_have_the_feature) > 0: if candidates is None: candidates = we_have_the_feature candidate_set = set(candidates) else: # Eliminate any candidates that don't have this feature. candidate_set = candidate_set.intersection( set(we_have_the_feature)) # The only valid candidates are the ones in candidate_set. # Go through the original list of candidates and pick the first one # that's in candidate_set. if candidate_set is None: return None for candidate in candidates: if candidate in candidate_set: return candidate return None # The BeautifulSoup class will take feature lists from developers and use them # to look up builders in this registry. builder_registry = TreeBuilderRegistry() class TreeBuilder(object): """Turn a document into a Beautiful Soup object tree.""" NAME = "[Unknown tree builder]" ALTERNATE_NAMES = [] features = [] is_xml = False picklable = False preserve_whitespace_tags = set() empty_element_tags = None # A tag will be considered an empty-element # tag when and only when it has no contents. # A value for these tag/attribute combinations is a space- or # comma-separated list of CDATA, rather than a single CDATA. cdata_list_attributes = {} def __init__(self): self.soup = None def reset(self): pass def can_be_empty_element(self, tag_name): """Might a tag with this name be an empty-element tag? The final markup may or may not actually present this tag as self-closing. For instance: an HTMLBuilder does not consider a <p> tag to be an empty-element tag (it's not in HTMLBuilder.empty_element_tags). This means an empty <p> tag will be presented as "<p></p>", not "<p />". The default implementation has no opinion about which tags are empty-element tags, so a tag will be presented as an empty-element tag if and only if it has no contents. "<foo></foo>" will become "<foo />", and "<foo>bar</foo>" will be left alone. """ if self.empty_element_tags is None: return True return tag_name in self.empty_element_tags def feed(self, markup): raise NotImplementedError() def prepare_markup(self, markup, user_specified_encoding=None, document_declared_encoding=None): return markup, None, None, False def test_fragment_to_document(self, fragment): """Wrap an HTML fragment to make it look like a document. Different parsers do this differently. For instance, lxml introduces an empty <head> tag, and html5lib doesn't. Abstracting this away lets us write simple tests which run HTML fragments through the parser and compare the results against other HTML fragments. This method should not be used outside of tests. """ return fragment def set_up_substitutions(self, tag): return False def _replace_cdata_list_attribute_values(self, tag_name, attrs): """Replaces class="foo bar" with class=["foo", "bar"] Modifies its input in place. """ if not attrs: return attrs if self.cdata_list_attributes: universal = self.cdata_list_attributes.get('*', []) tag_specific = self.cdata_list_attributes.get( tag_name.lower(), None) for attr in list(attrs.keys()): if attr in universal or (tag_specific and attr in tag_specific): # We have a "class"-type attribute whose string # value is a whitespace-separated list of # values. Split it into a list. value = attrs[attr] if isinstance(value, str): values = whitespace_re.split(value) else: # html5lib sometimes calls setAttributes twice # for the same tag when rearranging the parse # tree. On the second call the attribute value # here is already a list. If this happens, # leave the value alone rather than trying to # split it again. values = value attrs[attr] = values return attrs class SAXTreeBuilder(TreeBuilder): """A Beautiful Soup treebuilder that listens for SAX events.""" def feed(self, markup): raise NotImplementedError() def close(self): pass def startElement(self, name, attrs): attrs = dict((key[1], value) for key, value in list(attrs.items())) #print "Start %s, %r" % (name, attrs) self.soup.handle_starttag(name, attrs) def endElement(self, name): #print "End %s" % name self.soup.handle_endtag(name) def startElementNS(self, nsTuple, nodeName, attrs): # Throw away (ns, nodeName) for now. self.startElement(nodeName, attrs) def endElementNS(self, nsTuple, nodeName): # Throw away (ns, nodeName) for now. self.endElement(nodeName) #handler.endElementNS((ns, node.nodeName), node.nodeName) def startPrefixMapping(self, prefix, nodeValue): # Ignore the prefix for now. pass def endPrefixMapping(self, prefix): # Ignore the prefix for now. # handler.endPrefixMapping(prefix) pass def characters(self, content): self.soup.handle_data(content) def startDocument(self): pass def endDocument(self): pass class HTMLTreeBuilder(TreeBuilder): """This TreeBuilder knows facts about HTML. Such as which tags are empty-element tags. """ preserve_whitespace_tags = HTMLAwareEntitySubstitution.preserve_whitespace_tags empty_element_tags = set([ # These are from HTML5. 'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr', # These are from earlier versions of HTML and are removed in HTML5. 'basefont', 'bgsound', 'command', 'frame', 'image', 'isindex', 'nextid', 'spacer' ]) # The HTML standard defines these as block-level elements. Beautiful # Soup does not treat these elements differently from other elements, # but it may do so eventually, and this information is available if # you need to use it. block_elements = set(["address", "article", "aside", "blockquote", "canvas", "dd", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "noscript", "ol", "output", "p", "pre", "section", "table", "tfoot", "ul", "video"]) # The HTML standard defines these attributes as containing a # space-separated list of values, not a single value. That is, # class="foo bar" means that the 'class' attribute has two values, # 'foo' and 'bar', not the single value 'foo bar'. When we # encounter one of these attributes, we will parse its value into # a list of values if possible. Upon output, the list will be # converted back into a string. cdata_list_attributes = { "*" : ['class', 'accesskey', 'dropzone'], "a" : ['rel', 'rev'], "link" : ['rel', 'rev'], "td" : ["headers"], "th" : ["headers"], "td" : ["headers"], "form" : ["accept-charset"], "object" : ["archive"], # These are HTML5 specific, as are *.accesskey and *.dropzone above. "area" : ["rel"], "icon" : ["sizes"], "iframe" : ["sandbox"], "output" : ["for"], } def set_up_substitutions(self, tag): # We are only interested in <meta> tags if tag.name != 'meta': return False http_equiv = tag.get('http-equiv') content = tag.get('content') charset = tag.get('charset') # We are interested in <meta> tags that say what encoding the # document was originally in. This means HTML 5-style <meta> # tags that provide the "charset" attribute. It also means # HTML 4-style <meta> tags that provide the "content" # attribute and have "http-equiv" set to "content-type". # # In both cases we will replace the value of the appropriate # attribute with a standin object that can take on any # encoding. meta_encoding = None if charset is not None: # HTML 5 style: # <meta charset="utf8"> meta_encoding = charset tag['charset'] = CharsetMetaAttributeValue(charset) elif (content is not None and http_equiv is not None and http_equiv.lower() == 'content-type'): # HTML 4 style: # <meta http-equiv="content-type" content="text/html; charset=utf8"> tag['content'] = ContentMetaAttributeValue(content) return (meta_encoding is not None) def register_treebuilders_from(module): """Copy TreeBuilders from the given module into this module.""" # I'm fairly sure this is not the best way to do this. this_module = sys.modules['bs4.builder'] for name in module.__all__: obj = getattr(module, name) if issubclass(obj, TreeBuilder): setattr(this_module, name, obj) this_module.__all__.append(name) # Register the builder while we're at it. this_module.builder_registry.register(obj) class ParserRejectedMarkup(Exception): pass # Builders are registered in reverse order of priority, so that custom # builder registrations will take precedence. In general, we want lxml # to take precedence over html5lib, because it's faster. And we only # want to use HTMLParser as a last result. from . import _htmlparser register_treebuilders_from(_htmlparser) try: from . import _html5lib register_treebuilders_from(_html5lib) except ImportError: # They don't have html5lib installed. pass try: from . import _lxml register_treebuilders_from(_lxml) except ImportError: # They don't have lxml installed. pass builder/_html5lib.py 0000644 00000040460 15246412105 0010427 0 ustar 00 # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. __all__ = [ 'HTML5TreeBuilder', ] import warnings import re from bs4.builder import ( PERMISSIVE, HTML, HTML_5, HTMLTreeBuilder, ) from bs4.element import ( NamespacedAttribute, whitespace_re, ) import html5lib from html5lib.constants import ( namespaces, prefixes, ) from bs4.element import ( Comment, Doctype, NavigableString, Tag, ) try: # Pre-0.99999999 from html5lib.treebuilders import _base as treebuilder_base new_html5lib = False except ImportError as e: # 0.99999999 and up from html5lib.treebuilders import base as treebuilder_base new_html5lib = True class HTML5TreeBuilder(HTMLTreeBuilder): """Use html5lib to build a tree.""" NAME = "html5lib" features = [NAME, PERMISSIVE, HTML_5, HTML] def prepare_markup(self, markup, user_specified_encoding, document_declared_encoding=None, exclude_encodings=None): # Store the user-specified encoding for use later on. self.user_specified_encoding = user_specified_encoding # document_declared_encoding and exclude_encodings aren't used # ATM because the html5lib TreeBuilder doesn't use # UnicodeDammit. if exclude_encodings: warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.") yield (markup, None, None, False) # These methods are defined by Beautiful Soup. def feed(self, markup): if self.soup.parse_only is not None: warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.") parser = html5lib.HTMLParser(tree=self.create_treebuilder) extra_kwargs = dict() if not isinstance(markup, str): if new_html5lib: extra_kwargs['override_encoding'] = self.user_specified_encoding else: extra_kwargs['encoding'] = self.user_specified_encoding doc = parser.parse(markup, **extra_kwargs) # Set the character encoding detected by the tokenizer. if isinstance(markup, str): # We need to special-case this because html5lib sets # charEncoding to UTF-8 if it gets Unicode input. doc.original_encoding = None else: original_encoding = parser.tokenizer.stream.charEncoding[0] if not isinstance(original_encoding, str): # In 0.99999999 and up, the encoding is an html5lib # Encoding object. We want to use a string for compatibility # with other tree builders. original_encoding = original_encoding.name doc.original_encoding = original_encoding def create_treebuilder(self, namespaceHTMLElements): self.underlying_builder = TreeBuilderForHtml5lib( namespaceHTMLElements, self.soup) return self.underlying_builder def test_fragment_to_document(self, fragment): """See `TreeBuilder`.""" return '<html><head></head><body>%s</body></html>' % fragment class TreeBuilderForHtml5lib(treebuilder_base.TreeBuilder): def __init__(self, namespaceHTMLElements, soup=None): if soup: self.soup = soup else: from bs4 import BeautifulSoup self.soup = BeautifulSoup("", "html.parser") super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements) def documentClass(self): self.soup.reset() return Element(self.soup, self.soup, None) def insertDoctype(self, token): name = token["name"] publicId = token["publicId"] systemId = token["systemId"] doctype = Doctype.for_name_and_ids(name, publicId, systemId) self.soup.object_was_parsed(doctype) def elementClass(self, name, namespace): tag = self.soup.new_tag(name, namespace) return Element(tag, self.soup, namespace) def commentClass(self, data): return TextNode(Comment(data), self.soup) def fragmentClass(self): from bs4 import BeautifulSoup self.soup = BeautifulSoup("", "html.parser") self.soup.name = "[document_fragment]" return Element(self.soup, self.soup, None) def appendChild(self, node): # XXX This code is not covered by the BS4 tests. self.soup.append(node.element) def getDocument(self): return self.soup def getFragment(self): return treebuilder_base.TreeBuilder.getFragment(self).element def testSerializer(self, element): from bs4 import BeautifulSoup rv = [] doctype_re = re.compile(r'^(.*?)(?: PUBLIC "(.*?)"(?: "(.*?)")?| SYSTEM "(.*?)")?$') def serializeElement(element, indent=0): if isinstance(element, BeautifulSoup): pass if isinstance(element, Doctype): m = doctype_re.match(element) if m: name = m.group(1) if m.lastindex > 1: publicId = m.group(2) or "" systemId = m.group(3) or m.group(4) or "" rv.append("""|%s<!DOCTYPE %s "%s" "%s">""" % (' ' * indent, name, publicId, systemId)) else: rv.append("|%s<!DOCTYPE %s>" % (' ' * indent, name)) else: rv.append("|%s<!DOCTYPE >" % (' ' * indent,)) elif isinstance(element, Comment): rv.append("|%s<!-- %s -->" % (' ' * indent, element)) elif isinstance(element, NavigableString): rv.append("|%s\"%s\"" % (' ' * indent, element)) else: if element.namespace: name = "%s %s" % (prefixes[element.namespace], element.name) else: name = element.name rv.append("|%s<%s>" % (' ' * indent, name)) if element.attrs: attributes = [] for name, value in list(element.attrs.items()): if isinstance(name, NamespacedAttribute): name = "%s %s" % (prefixes[name.namespace], name.name) if isinstance(value, list): value = " ".join(value) attributes.append((name, value)) for name, value in sorted(attributes): rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value)) indent += 2 for child in element.children: serializeElement(child, indent) serializeElement(element, 0) return "\n".join(rv) class AttrList(object): def __init__(self, element): self.element = element self.attrs = dict(self.element.attrs) def __iter__(self): return list(self.attrs.items()).__iter__() def __setitem__(self, name, value): # If this attribute is a multi-valued attribute for this element, # turn its value into a list. list_attr = HTML5TreeBuilder.cdata_list_attributes if (name in list_attr['*'] or (self.element.name in list_attr and name in list_attr[self.element.name])): # A node that is being cloned may have already undergone # this procedure. if not isinstance(value, list): value = whitespace_re.split(value) self.element[name] = value def items(self): return list(self.attrs.items()) def keys(self): return list(self.attrs.keys()) def __len__(self): return len(self.attrs) def __getitem__(self, name): return self.attrs[name] def __contains__(self, name): return name in list(self.attrs.keys()) class Element(treebuilder_base.Node): def __init__(self, element, soup, namespace): treebuilder_base.Node.__init__(self, element.name) self.element = element self.soup = soup self.namespace = namespace def appendChild(self, node): string_child = child = None if isinstance(node, str): # Some other piece of code decided to pass in a string # instead of creating a TextElement object to contain the # string. string_child = child = node elif isinstance(node, Tag): # Some other piece of code decided to pass in a Tag # instead of creating an Element object to contain the # Tag. child = node elif node.element.__class__ == NavigableString: string_child = child = node.element node.parent = self else: child = node.element node.parent = self if not isinstance(child, str) and child.parent is not None: node.element.extract() if (string_child and self.element.contents and self.element.contents[-1].__class__ == NavigableString): # We are appending a string onto another string. # TODO This has O(n^2) performance, for input like # "a</a>a</a>a</a>..." old_element = self.element.contents[-1] new_element = self.soup.new_string(old_element + string_child) old_element.replace_with(new_element) self.soup._most_recent_element = new_element else: if isinstance(node, str): # Create a brand new NavigableString from this string. child = self.soup.new_string(node) # Tell Beautiful Soup to act as if it parsed this element # immediately after the parent's last descendant. (Or # immediately after the parent, if it has no children.) if self.element.contents: most_recent_element = self.element._last_descendant(False) elif self.element.next_element is not None: # Something from further ahead in the parse tree is # being inserted into this earlier element. This is # very annoying because it means an expensive search # for the last element in the tree. most_recent_element = self.soup._last_descendant() else: most_recent_element = self.element self.soup.object_was_parsed( child, parent=self.element, most_recent_element=most_recent_element) def getAttributes(self): if isinstance(self.element, Comment): return {} return AttrList(self.element) def setAttributes(self, attributes): if attributes is not None and len(attributes) > 0: converted_attributes = [] for name, value in list(attributes.items()): if isinstance(name, tuple): new_name = NamespacedAttribute(*name) del attributes[name] attributes[new_name] = value self.soup.builder._replace_cdata_list_attribute_values( self.name, attributes) for name, value in list(attributes.items()): self.element[name] = value # The attributes may contain variables that need substitution. # Call set_up_substitutions manually. # # The Tag constructor called this method when the Tag was created, # but we just set/changed the attributes, so call it again. self.soup.builder.set_up_substitutions(self.element) attributes = property(getAttributes, setAttributes) def insertText(self, data, insertBefore=None): text = TextNode(self.soup.new_string(data), self.soup) if insertBefore: self.insertBefore(text, insertBefore) else: self.appendChild(text) def insertBefore(self, node, refNode): index = self.element.index(refNode.element) if (node.element.__class__ == NavigableString and self.element.contents and self.element.contents[index-1].__class__ == NavigableString): # (See comments in appendChild) old_node = self.element.contents[index-1] new_str = self.soup.new_string(old_node + node.element) old_node.replace_with(new_str) else: self.element.insert(index, node.element) node.parent = self def removeChild(self, node): node.element.extract() def reparentChildren(self, new_parent): """Move all of this tag's children into another tag.""" # print "MOVE", self.element.contents # print "FROM", self.element # print "TO", new_parent.element element = self.element new_parent_element = new_parent.element # Determine what this tag's next_element will be once all the children # are removed. final_next_element = element.next_sibling new_parents_last_descendant = new_parent_element._last_descendant(False, False) if len(new_parent_element.contents) > 0: # The new parent already contains children. We will be # appending this tag's children to the end. new_parents_last_child = new_parent_element.contents[-1] new_parents_last_descendant_next_element = new_parents_last_descendant.next_element else: # The new parent contains no children. new_parents_last_child = None new_parents_last_descendant_next_element = new_parent_element.next_element to_append = element.contents if len(to_append) > 0: # Set the first child's previous_element and previous_sibling # to elements within the new parent first_child = to_append[0] if new_parents_last_descendant: first_child.previous_element = new_parents_last_descendant else: first_child.previous_element = new_parent_element first_child.previous_sibling = new_parents_last_child if new_parents_last_descendant: new_parents_last_descendant.next_element = first_child else: new_parent_element.next_element = first_child if new_parents_last_child: new_parents_last_child.next_sibling = first_child # Find the very last element being moved. It is now the # parent's last descendant. It has no .next_sibling and # its .next_element is whatever the previous last # descendant had. last_childs_last_descendant = to_append[-1]._last_descendant(False, True) last_childs_last_descendant.next_element = new_parents_last_descendant_next_element if new_parents_last_descendant_next_element: # TODO: This code has no test coverage and I'm not sure # how to get html5lib to go through this path, but it's # just the other side of the previous line. new_parents_last_descendant_next_element.previous_element = last_childs_last_descendant last_childs_last_descendant.next_sibling = None for child in to_append: child.parent = new_parent_element new_parent_element.contents.append(child) # Now that this element has no children, change its .next_element. element.contents = [] element.next_element = final_next_element # print "DONE WITH MOVE" # print "FROM", self.element # print "TO", new_parent_element def cloneNode(self): tag = self.soup.new_tag(self.element.name, self.namespace) node = Element(tag, self.soup, self.namespace) for key,value in self.attributes: node.attributes[key] = value return node def hasContent(self): return self.element.contents def getNameTuple(self): if self.namespace == None: return namespaces["html"], self.name else: return self.namespace, self.name nameTuple = property(getNameTuple) class TextNode(Element): def __init__(self, element, soup): treebuilder_base.Node.__init__(self, None) self.element = element self.soup = soup def cloneNode(self): raise NotImplementedError builder/__pycache__/_htmlparser.cpython-36.pyc 0000644 00000021054 15246412105 0015352 0 ustar 00 3 6]�3 � @ s� d Z dgZddlmZ yddlmZ W n2 ek rX Z zG dd� de�ZW Y ddZ[X nX ddlZddl Z ej dd� \Z Z Z e dko�e d ko�e dkZe dko�e dkZe dko�e d kZdd lmZmZmZmZmZ dd lmZmZ dd lmZm Z m Z dZ G dd� de�Z G dd� de �Z e dk�r�e d k�r�e �r�ddl!Z!e!j"d�Z#e#e _#e!j"de!j$�Z%e%e _%ddlm&Z&m'Z' dd� Z(dd� Z)e(e _(e)e _)dZdS )zCUse the HTMLParser library to parse HTML files that aren't too bad.�HTMLParserTreeBuilder� )� HTMLParser)�HTMLParseErrorc @ s e Zd ZdS )r N)�__name__� __module__� __qualname__� r r �!/usr/lib/python3.6/_htmlparser.pyr s r N� � � )�CData�Comment� Declaration�Doctype�ProcessingInstruction)�EntitySubstitution� UnicodeDammit)�HTML�HTMLTreeBuilder�STRICTz html.parserc @ sp e Zd Zdd� Zdd� Zdd� Zddd �Zd d d �Zd d � Zdd� Z dd� Z dd� Z dd� Z dd� Z dd� ZdS ) �BeautifulSoupHTMLParserc O s t j| f|�|� g | _d S )N)r �__init__� already_closed_empty_element)�self�args�kwargsr r r r 9 s z BeautifulSoupHTMLParser.__init__c C s t j|� dS )ai In Python 3, HTMLParser subclasses must implement error(), although this requirement doesn't appear to be documented. In Python 2, HTMLParser implements error() as raising an exception. In any event, this method is called only on very strange markup and our best strategy is to pretend it didn't happen and keep going. N)�warnings�warn)r �msgr r r �errorE s z BeautifulSoupHTMLParser.errorc C s | j ||dd�}| j|� d S )NF)�handle_empty_element)�handle_starttag� handle_endtag)r �name�attrs�tagr r r �handle_startendtagP s z*BeautifulSoupHTMLParser.handle_startendtagTc C sl i }x(|D ] \}}|d kr d}|||< d}q W | j j|d d |�}|rh|jrh|rh| j|dd� | jj|� d S )N� z""F)�check_already_closed)�soupr" Zis_empty_elementr# r �append) r r$ r% r! Z attr_dict�key�value� attrvaluer& r r r r" [ s z'BeautifulSoupHTMLParser.handle_starttagc C s, |r || j kr | j j|� n | jj|� d S )N)r �remover* r# )r r$ r) r r r r# w s z%BeautifulSoupHTMLParser.handle_endtagc C s | j j|� d S )N)r* � handle_data)r �datar r r r0 � s z#BeautifulSoupHTMLParser.handle_datac C s� |j d�r t|jd�d�}n$|j d�r8t|jd�d�}nt|�}d }|dk r�xR| jjdfD ]B}|sdqZyt|g�j|�}W qZ tk r� } zW Y d d }~X qZX qZW |s�y t|�}W n& t t fk r� } zW Y d d }~X nX |p�d}| j |� d S )N�x� �X� z windows-1252u �) � startswith�int�lstripr* �original_encoding� bytearray�decode�UnicodeDecodeError�chr� ValueError� OverflowErrorr0 )r r$ Z real_namer1 �encoding�er r r �handle_charref� s* z&BeautifulSoupHTMLParser.handle_charrefc C s0 t jj|�}|d k r|}nd| }| j|� d S )Nz&%s)r ZHTML_ENTITY_TO_CHARACTER�getr0 )r r$ � characterr1 r r r �handle_entityref� s z(BeautifulSoupHTMLParser.handle_entityrefc C s& | j j� | j j|� | j jt� d S )N)r* �endDatar0 r )r r1 r r r �handle_comment� s z&BeautifulSoupHTMLParser.handle_commentc C sN | j j� |jd�r&|td�d � }n |dkr2d}| j j|� | j jt� d S )NzDOCTYPE ZDOCTYPEr( )r* rF r6 �lenr0 r )r r1 r r r � handle_decl� s z#BeautifulSoupHTMLParser.handle_declc C sN |j � jd�r$t}|td�d � }nt}| jj� | jj|� | jj|� d S )NzCDATA[)�upperr6 r rH r r* rF r0 )r r1 �clsr r r � unknown_decl� s z$BeautifulSoupHTMLParser.unknown_declc C s& | j j� | j j|� | j jt� d S )N)r* rF r0 r )r r1 r r r � handle_pi� s z!BeautifulSoupHTMLParser.handle_piN)T)T)r r r r r r' r" r# r0 rB rE rG rI rL rM r r r r r 7 s ! r c @ s< e Zd ZdZdZeZeeegZ dd� Z d dd�Z dd � Z dS ) r FTc O s, t rt rd|d< tr d|d< ||f| _d S )NF�strictZconvert_charrefs)�CONSTRUCTOR_TAKES_STRICT� CONSTRUCTOR_STRICT_IS_DEPRECATED�"CONSTRUCTOR_TAKES_CONVERT_CHARREFS� parser_args)r r r r r r r � s z HTMLParserTreeBuilder.__init__Nc c sN t |t�r |dddfV dS ||g}t||d|d�}|j|j|j|jfV dS )z� :return: A 4-tuple (markup, original encoding, encoding declared within markup, whether any characters had to be replaced with REPLACEMENT CHARACTER). NFT)Zis_html�exclude_encodings)� isinstance�strr �markupr9 Zdeclared_html_encodingZ contains_replacement_characters)r rV Zuser_specified_encodingZdocument_declared_encodingrS Z try_encodingsZdammitr r r �prepare_markup� s z$HTMLParserTreeBuilder.prepare_markupc C sr | j \}}t||�}| j|_y|j|� |j� W n4 tk rf } ztjtd�� |�W Y d d }~X nX g |_ d S )Na* Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help.) rR r r* �feed�closer r r �RuntimeWarningr )r rV r r �parserrA r r r rX � s zHTMLParserTreeBuilder.feed)NNN) r r r Zis_xmlZ picklable� HTMLPARSER�NAMEr r Zfeaturesr rW rX r r r r r � s zQ\s*((?<=[\'"\s])[^\s/>][^\s/=>]*)(\s*=+\s*(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?a� <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name (?:\s+ # whitespace before attribute name (?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name (?:\s*=\s* # value indicator (?:'[^']*' # LITA-enclosed value |\"[^\"]*\" # LIT-enclosed value |[^'\">\s]+ # bare value ) )? ) )* \s* # trailing whitespace )�tagfind�attrfindc C s8 d | _ | j|�}|dk r |S | j}|||� | _ g }tj||d �}|sPtd��|j� }||d |� j� | _}x�||k �rP| j r�t j||�}n t j||�}|s�P |j ddd�\} } } | s�d } nb| d d� d ko�| d d � kn �s| d d� d k�o| dd � kn �r$| dd� } | �r4| j | �} |j| j� | f� |j� }qtW |||� j� } | dk�r�| j� \} }d | j k�r�| | j jd � } t| j �| j jd � }n|t| j � }| j �r�| jd |||� d d � f � | j|||� � |S | jd ��r| j||� n"| j||� || jk�r4| j|� |S )Nr � z#unexpected call to parse_starttag()r r �'�"�>�/>� z junk characters in start tag: %r� ���rg rg )rc rd )Z__starttag_textZcheck_for_whole_start_tag�rawdatar^ �match�AssertionError�end�lowerZlasttagrN r_ �attrfind_tolerant�groupZunescaper+ �stripZgetpos�countrH �rfindr r0 �endswithr' r" ZCDATA_CONTENT_ELEMENTS�set_cdata_mode)r �i�endposrh r% ri �kr& �mZattrname�restr. rk �lineno�offsetr r r �parse_starttag s\ *, r{ c C s$ |j � | _tjd| j tj�| _d S )Nz </\s*%s\s*>)rl Z cdata_elem�re�compile�IZ interesting)r �elemr r r rs T s rs T)*�__doc__�__all__Z html.parserr r � ImportErrorrA � Exception�sysr � version_info�major�minor�releaserO rP rQ Z bs4.elementr r r r r Z bs4.dammitr r Z bs4.builderr r r r\ r r r| r} rm �VERBOSEZlocatestarttagendr^ r_ r{ rs r r r r �<module> sB " 2 7 builder/__pycache__/_html5lib.cpython-36.pyc 0000644 00000026326 15246412105 0014720 0 ustar 00 3 6]0A � @ s d gZ ddlZddlZddlmZmZmZmZ ddlm Z m Z ddl Z ddl m Z mZ ddlmZmZmZmZ yddlmZ dZW n2 ek r� Z zdd lmZ d ZW Y ddZ[X nX G d d � d e�ZG d d � d ej�Z G dd� de �Z G dd� dej �Z G dd� de �Z!dS )�HTML5TreeBuilder� N)� PERMISSIVE�HTML�HTML_5�HTMLTreeBuilder)�NamespacedAttribute� whitespace_re)� namespaces�prefixes)�Comment�Doctype�NavigableString�Tag)�_baseF)�baseTc @ sB e Zd ZdZdZeeeegZd dd�Z dd� Z dd � Z d d � Z dS ) r z Use html5lib to build a tree.�html5libNc c s& || _ |rtjd� |d d dfV d S )NzjYou provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.F)�user_specified_encoding�warnings�warn)�self�markupr Zdocument_declared_encodingZexclude_encodings� r � /usr/lib/python3.6/_html5lib.py�prepare_markup0 s z HTML5TreeBuilder.prepare_markupc C s� | j jd k rtjd� tj| jd�}t� }t|t �sNt rD| j |d< n | j |d< |j |f|�}t|t �rnd |_ n$|jjjd }t|t �s�|j}||_ d S )Nz�You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.)ZtreeZoverride_encoding�encodingr )�soupZ parse_onlyr r r Z HTMLParser�create_treebuilder�dict� isinstance�str� new_html5libr �parse�original_encodingZ tokenizer�streamZ charEncoding�name)r r �parserZ extra_kwargs�docr" r r r �feed= s zHTML5TreeBuilder.feedc C s t || j�| _| jS )N)�TreeBuilderForHtml5libr Zunderlying_builder)r �namespaceHTMLElementsr r r r X s z#HTML5TreeBuilder.create_treebuilderc C s d| S )zSee `TreeBuilder`.z)<html><head></head><body>%s</body></html>r )r Zfragmentr r r �test_fragment_to_document] s z*HTML5TreeBuilder.test_fragment_to_document)NN) �__name__� __module__� __qualname__�__doc__�NAMEr r r Zfeaturesr r' r r* r r r r r ) s c sf e Zd Zd� fdd� Zdd� Zdd� Zdd � Zd d � Zd d � Zdd� Z dd� Z dd� Z dd� Z � Z S )r( Nc s8 |r || _ nddlm} |dd�| _ tt| �j|� d S )Nr )� BeautifulSoup� z html.parser)r �bs4r0 �superr( �__init__)r r) r r0 )� __class__r r r4 d s z TreeBuilderForHtml5lib.__init__c C s | j j� t| j | j d �S )N)r �reset�Element)r r r r � documentClassl s z$TreeBuilderForHtml5lib.documentClassc C s6 |d }|d }|d }t j|||�}| jj|� d S )Nr$ �publicId�systemId)r Zfor_name_and_idsr �object_was_parsed)r �tokenr$ r9 r: Zdoctyper r r � insertDoctypep s z$TreeBuilderForHtml5lib.insertDoctypec C s | j j||�}t|| j |�S )N)r �new_tagr7 )r r$ � namespace�tagr r r � elementClassx s z#TreeBuilderForHtml5lib.elementClassc C s t t|�| j�S )N)�TextNoder r )r �datar r r � commentClass| s z#TreeBuilderForHtml5lib.commentClassc C s0 ddl m} |dd�| _d| j_t| j| jd �S )Nr )r0 r1 z html.parserz[document_fragment])r2 r0 r r$ r7 )r r0 r r r � fragmentClass s z$TreeBuilderForHtml5lib.fragmentClassc C s | j j|j� d S )N)r �append�element)r �noder r r � appendChild� s z"TreeBuilderForHtml5lib.appendChildc C s | j S )N)r )r r r r � getDocument� s z"TreeBuilderForHtml5lib.getDocumentc C s t jj| �jS )N)�treebuilder_base� TreeBuilder� getFragmentrG )r r r r rM � s z"TreeBuilderForHtml5lib.getFragmentc sB ddl m� g �tjd��d� ���fdd� ��|d� dj��S )Nr )r0 z8^(.*?)(?: PUBLIC "(.*?)"(?: "(.*?)")?| SYSTEM "(.*?)")?$c s� t | � �r t | t�r��j| �}|r�|jd�}|jdkrx|jd�pBd}|jd�pZ|jd�pZd}�jdd| |||f � q��jdd| |f � n�jd d| f � �nHt | t�r̈jd d| | f � �n$t | t�r�jd d| | f � �n | j�rd t | j | j f }n| j }�jd d| |f � | j �r�g }x`t | j j � �D ]N\}}t |t��rnd t |j |j f }t |t ��r�dj|�}|j||f� �qFW x2t|�D ]&\}}�jdd|d ||f � �q�W |d7 }x| jD ]}�||� �q�W d S )N� � r1 � � z|%s<!DOCTYPE %s "%s" "%s">� z|%s<!DOCTYPE %s>z|%s<!DOCTYPE >z|%s<!-- %s -->z|%s"%s"z%s %sz|%s<%s>z |%s%s="%s")r r �match�group� lastindexrF r r r? r r$ �attrs�list�itemsr �join�sortedZchildren) rG �indent�mr$ r9 r: � attributes�value�child)r0 � doctype_re�rv�serializeElementr r rb � sD " z?TreeBuilderForHtml5lib.testSerializer.<locals>.serializeElement� )r )r2 r0 �re�compilerY )r rG r )r0 r` ra rb r �testSerializer� s ) z%TreeBuilderForHtml5lib.testSerializer)N)r+ r, r- r4 r8 r= rA rD rE rI rJ rM rf � __classcell__r r )r5 r r( b s r( c @ sL e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dS )�AttrListc C s || _ t| j j�| _d S )N)rG r rV )r rG r r r r4 � s zAttrList.__init__c C s t | jj� �j� S )N)rW rV rX �__iter__)r r r r ri � s zAttrList.__iter__c C sP t j}||d ks.| jj|krB||| jj krBt|t�sBtj|�}|| j|< d S )N�*)r Zcdata_list_attributesrG r$ r rW r �split)r r$ r^ Z list_attrr r r � __setitem__� s zAttrList.__setitem__c C s t | jj� �S )N)rW rV rX )r r r r rX � s zAttrList.itemsc C s t | jj� �S )N)rW rV �keys)r r r r rm � s z AttrList.keysc C s t | j�S )N)�lenrV )r r r r �__len__� s zAttrList.__len__c C s | j | S )N)rV )r r$ r r r � __getitem__� s zAttrList.__getitem__c C s |t | jj� �kS )N)rW rV rm )r r$ r r r � __contains__� s zAttrList.__contains__N) r+ r, r- r4 ri rl rX rm ro rp rq r r r r rh � s rh c @ sx e Zd Zdd� Zdd� Zdd� Zdd� Zeee�Zdd d �Z d d � Z dd� Z dd� Z dd� Z dd� Zdd� Zee�Zd S )r7 c C s& t jj| |j� || _|| _|| _d S )N)rK �Noder4 r$ rG r r? )r rG r r? r r r r4 � s zElement.__init__c C s( d }}t |t�r | }}n:t |t�r,|}n*|jjtkrJ|j }}| |_n |j}| |_t |t� rv|jd k rv|jj� |r�| jjr�| jjd jtkr�| jjd }| j j || �}|j |� || j _ n`t |t�r�| j j |�}| jjr�| jj d�}n | jjd k �r | j j � }n| j}| j j|| j|d� d S )NrN F)�parent�most_recent_element���ru )r r r rG r5 r rs �extract�contentsr � new_string� replace_withZ_most_recent_element�_last_descendant� next_elementr; )r rH Z string_childr_ Z old_elementZ new_elementrt r r r rI � s8 zElement.appendChildc C s t | jt�ri S t| j�S )N)r rG r rh )r r r r � getAttributes s zElement.getAttributesc C s� |d k r�t |�dkr�g }x8t|j� �D ](\}}t|t�r&t|� }||= |||< q&W | jjj| j |� x"t|j� �D ]\}}|| j |< qrW | jjj | j � d S )Nr ) rn rW rX r �tupler r ZbuilderZ$_replace_cdata_list_attribute_valuesr$ rG Zset_up_substitutions)r r] Zconverted_attributesr$ r^ �new_namer r r � setAttributes! s zElement.setAttributesNc C s4 t | jj|�| j�}|r&| j||� n | j|� d S )N)rB r rx � insertBeforerI )r rC r� �textr r r � insertText9 s zElement.insertTextc C s� | j j|j �}|j jtkrf| j jrf| j j|d jtkrf| j j|d }| jj||j �}|j|� n| j j||j � | |_ d S )NrN ) rG �indexr5 r rw r rx ry �insertrs )r rH ZrefNoder� Zold_nodeZnew_strr r r r� @ s zElement.insertBeforec C s |j j� d S )N)rG rv )r rH r r r � removeChildL s zElement.removeChildc C s� | j }|j }|j}|jdd�}t|j�dkr>|jd }|j}n d}|j}|j}t|�dkr�|d } |rn|| _n|| _|| _|r�| |_n| |_|r�| |_|d jdd�} || _|r�| |_d| _x |D ]} || _|jj | � q�W g |_||_dS )z1Move all of this tag's children into another tag.Fr rN NTru ru ) rG Z next_siblingrz rn rw r{ Zprevious_elementZprevious_siblingrs rF ) r Z new_parentrG Znew_parent_elementZfinal_next_elementZnew_parents_last_descendantZnew_parents_last_childZ(new_parents_last_descendant_next_elementZ to_appendZ first_childZlast_childs_last_descendantr_ r r r �reparentChildrenO s> zElement.reparentChildrenc C sF | j j| jj| j�}t|| j | j�}x | jD ]\}}||j|< q,W |S )N)r r> rG r$ r? r7 r] )r r@ rH �keyr^ r r r � cloneNode� s zElement.cloneNodec C s | j jS )N)rG rw )r r r r � hasContent� s zElement.hasContentc C s( | j d krtd | jfS | j | jfS d S )NZhtml)r? r r$ )r r r r � getNameTuple� s zElement.getNameTuple)N)r+ r, r- r4 rI r| r �propertyr] r� r� r� r� r� r� r� Z nameTupler r r r r7 � s 6 Br7 c @ s e Zd Zdd� Zdd� ZdS )rB c C s t jj| d � || _|| _d S )N)rK rr r4 rG r )r rG r r r r r4 � s zTextNode.__init__c C s t �d S )N)�NotImplementedError)r r r r r� � s zTextNode.cloneNodeN)r+ r, r- r4 r� r r r r rB � s rB )"�__all__r rd Z bs4.builderr r r r Z bs4.elementr r r Zhtml5lib.constantsr r r r r r Zhtml5lib.treebuildersr rK r � ImportError�er r rL r( �objectrh rr r7 rB r r r r �<module> s& 9_ E builder/__pycache__/__init__.cpython-36.pyc 0000644 00000021460 15246412105 0014572 0 ustar 00 3 6]�/ � @ s, d dl mZ d dlZd dlZd dlmZmZmZmZ ddddgZ dZ d Z d Z d Z d Zd ZG dd� de�Ze� ZG dd� de�ZG dd� de�ZG dd� de�Zdd� ZG dd� de�ZddlmZ ee� yddlmZ ee� W n e k r� Y nX yddlm Z ee � W n e k �r& Y nX dS )� )� defaultdictN)�CharsetMetaAttributeValue�ContentMetaAttributeValue�HTMLAwareEntitySubstitution� whitespace_re�HTMLTreeBuilder�SAXTreeBuilder� TreeBuilder�TreeBuilderRegistryZfastZ permissive�strictZxmlZhtmlZhtml5c @ s$ e Zd Zdd� Zdd� Zdd� ZdS )r c C s t t�| _g | _d S )N)r �list�builders_for_feature�builders)�self� r � /usr/lib/python3.6/__init__.py�__init__ s z TreeBuilderRegistry.__init__c C s4 x |j D ]}| j| jd|� qW | jjd|� dS )z8Register a treebuilder based on its advertised features.r N)�featuresr �insertr )r Ztreebuilder_class�featurer r r �register$ s z TreeBuilderRegistry.registerc G s� t | j�dkrd S t |�dkr(| jd S t|�}|j� d }d }xVt |�dkr�|j� }| jj|g �}t |�dkrB|d kr�|}t|�}qB|jt|��}qBW |d kr�d S x|D ]}||kr�|S q�W d S )Nr ) �lenr r �reverse�popr �get�set� intersection)r r Z candidatesZ candidate_setr Zwe_have_the_feature� candidater r r �lookup* s. zTreeBuilderRegistry.lookupN)�__name__� __module__� __qualname__r r r r r r r r s c @ st e Zd ZdZdZg Zg ZdZdZe � Z dZ i Z dd� Z dd� Zd d � Zd d � Zdd d�Zdd� Zdd� Zdd� ZdS )r z2Turn a document into a Beautiful Soup object tree.z[Unknown tree builder]FNc C s d | _ d S )N)�soup)r r r r r f s zTreeBuilder.__init__c C s d S )Nr )r r r r �reseti s zTreeBuilder.resetc C s | j dkrdS || j kS )a� Might a tag with this name be an empty-element tag? The final markup may or may not actually present this tag as self-closing. For instance: an HTMLBuilder does not consider a <p> tag to be an empty-element tag (it's not in HTMLBuilder.empty_element_tags). This means an empty <p> tag will be presented as "<p></p>", not "<p />". The default implementation has no opinion about which tags are empty-element tags, so a tag will be presented as an empty-element tag if and only if it has no contents. "<foo></foo>" will become "<foo />", and "<foo>bar</foo>" will be left alone. NT)�empty_element_tags)r �tag_namer r r �can_be_empty_elementl s z TreeBuilder.can_be_empty_elementc C s t � �d S )N)�NotImplementedError)r �markupr r r �feed� s zTreeBuilder.feedc C s |d d dfS )NFr )r r( Zuser_specified_encodingZdocument_declared_encodingr r r �prepare_markup� s zTreeBuilder.prepare_markupc C s |S )a� Wrap an HTML fragment to make it look like a document. Different parsers do this differently. For instance, lxml introduces an empty <head> tag, and html5lib doesn't. Abstracting this away lets us write simple tests which run HTML fragments through the parser and compare the results against other HTML fragments. This method should not be used outside of tests. r )r Zfragmentr r r �test_fragment_to_document� s z%TreeBuilder.test_fragment_to_documentc C s dS )NFr )r �tagr r r �set_up_substitutions� s z TreeBuilder.set_up_substitutionsc C s� |s|S | j r�| j jdg �}| j j|j� d�}xRt|j� �D ]B}||ksT|r<||kr<|| }t|t�rrtj|�}n|}|||< q<W |S )zaReplaces class="foo bar" with class=["foo", "bar"] Modifies its input in place. �*N) �cdata_list_attributesr �lowerr �keys� isinstance�strr �split)r r% �attrsZ universalZ tag_specific�attr�value�valuesr r r �$_replace_cdata_list_attribute_values� s z0TreeBuilder._replace_cdata_list_attribute_values)NN)r r r! �__doc__�NAMEZALTERNATE_NAMESr Zis_xmlZ picklabler �preserve_whitespace_tagsr$ r/ r r# r& r) r* r+ r- r9 r r r r r T s$ c @ sh e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dd� Z dd� ZdS )r z9A Beautiful Soup treebuilder that listens for SAX events.c C s t � �d S )N)r' )r r( r r r r) � s zSAXTreeBuilder.feedc C s d S )Nr )r r r r �close� s zSAXTreeBuilder.closec C s, t dd� t|j� �D ��}| jj||� d S )Nc s s | ]\}}|d |fV qdS )� Nr )�.0�keyr7 r r r � <genexpr>� s z.SAXTreeBuilder.startElement.<locals>.<genexpr>)�dictr �itemsr" Zhandle_starttag)r �namer5 r r r � startElement� s zSAXTreeBuilder.startElementc C s | j j|� d S )N)r" Z handle_endtag)r rD r r r � endElement� s zSAXTreeBuilder.endElementc C s | j ||� d S )N)rE )r �nsTuple�nodeNamer5 r r r �startElementNS� s z SAXTreeBuilder.startElementNSc C s | j |� d S )N)rF )r rG rH r r r � endElementNS� s zSAXTreeBuilder.endElementNSc C s d S )Nr )r �prefixZ nodeValuer r r �startPrefixMapping� s z!SAXTreeBuilder.startPrefixMappingc C s d S )Nr )r rK r r r �endPrefixMapping� s z SAXTreeBuilder.endPrefixMappingc C s | j j|� d S )N)r" Z handle_data)r �contentr r r � characters� s zSAXTreeBuilder.charactersc C s d S )Nr )r r r r � startDocument� s z SAXTreeBuilder.startDocumentc C s d S )Nr )r r r r � endDocument� s zSAXTreeBuilder.endDocumentN)r r r! r: r) r= rE rF rI rJ rL rM rO rP rQ r r r r r � s c $ @ s� e Zd ZdZejZedddddddd d d d d ddddddddddddg�Zeddd d d d d d!d"d#d$d%d&d'd(d)d*d+d,d-d.dd/d0d1d2d3d4d5d6d7d8d9d:d;g#�Zd<d=d>gd?d@gd?d@gdAgdAgdAgdBgdCgd?gdDgdEgdFgdG� Z dHdI� Z dJS )Kr z]This TreeBuilder knows facts about HTML. Such as which tags are empty-element tags. �area�base�br�colZembedZhrZimg�inputZkeygen�linkZmenuitem�metaZparam�sourceZtrackZwbrZbasefontZbgsoundZcommand�frameZimageZisindexZnextidZspacerZaddressZarticleZasideZ blockquoteZcanvasZddZdivZdlZdtZfieldsetZ figcaptionZfigureZfooter�formZh1Zh2Zh3Zh4Zh5Zh6�headerZli�mainZnavZnoscriptZol�output�pZpreZsection�tableZtfootZulZvideo�classZ accesskeyZdropzoneZrelZrevZheaderszaccept-charset�archiveZsizesZsandbox�for) r. �arW �tdZthre r[ �objectrR ZiconZiframer^ c C sz |j dkrdS |jd�}|jd�}|jd�}d }|d k rJ|}t|�|d< n(|d k rr|d k rr|j� dkrrt|�|d< |d k S )NrX Fz http-equivrN �charsetz content-type)rD r r r0 r )r r, Z http_equivrN rg Z meta_encodingr r r r- s z$HTMLTreeBuilder.set_up_substitutionsN) r r r! r: r r< r r$ Zblock_elementsr/ r- r r r r r � s& N c C sV t jd }xF| jD ]<}t| |�}t|t�rt|||� |jj|� |jj |� qW dS )z9Copy TreeBuilders from the given module into this module.z bs4.builderN) �sys�modules�__all__�getattr� issubclassr �setattr�append�builder_registryr )�moduleZ this_modulerD �objr r r �register_treebuilders_from2 s rr c @ s e Zd ZdS )�ParserRejectedMarkupN)r r r! r r r r rs ? s rs r> )� _htmlparser)� _html5lib)�_lxml) � collectionsr � itertoolsrh Z bs4.elementr r r r rj ZFASTZ PERMISSIVEZSTRICTZXMLZHTMLZHTML_5rf r ro r r r rr � Exceptionrs � rt ru � ImportErrorrv r r r r �<module> s@ 4b.N builder/__pycache__/__init__.cpython-36.opt-1.pyc 0000644 00000021460 15246412105 0015531 0 ustar 00 3 6]�/ � @ s, d dl mZ d dlZd dlZd dlmZmZmZmZ ddddgZ dZ d Z d Z d Z d Zd ZG dd� de�Ze� ZG dd� de�ZG dd� de�ZG dd� de�Zdd� ZG dd� de�ZddlmZ ee� yddlmZ ee� W n e k r� Y nX yddlm Z ee � W n e k �r& Y nX dS )� )� defaultdictN)�CharsetMetaAttributeValue�ContentMetaAttributeValue�HTMLAwareEntitySubstitution� whitespace_re�HTMLTreeBuilder�SAXTreeBuilder� TreeBuilder�TreeBuilderRegistryZfastZ permissive�strictZxmlZhtmlZhtml5c @ s$ e Zd Zdd� Zdd� Zdd� ZdS )r c C s t t�| _g | _d S )N)r �list�builders_for_feature�builders)�self� r � /usr/lib/python3.6/__init__.py�__init__ s z TreeBuilderRegistry.__init__c C s4 x |j D ]}| j| jd|� qW | jjd|� dS )z8Register a treebuilder based on its advertised features.r N)�featuresr �insertr )r Ztreebuilder_class�featurer r r �register$ s z TreeBuilderRegistry.registerc G s� t | j�dkrd S t |�dkr(| jd S t|�}|j� d }d }xVt |�dkr�|j� }| jj|g �}t |�dkrB|d kr�|}t|�}qB|jt|��}qBW |d kr�d S x|D ]}||kr�|S q�W d S )Nr ) �lenr r �reverse�popr �get�set� intersection)r r Z candidatesZ candidate_setr Zwe_have_the_feature� candidater r r �lookup* s. zTreeBuilderRegistry.lookupN)�__name__� __module__� __qualname__r r r r r r r r s c @ st e Zd ZdZdZg Zg ZdZdZe � Z dZ i Z dd� Z dd� Zd d � Zd d � Zdd d�Zdd� Zdd� Zdd� ZdS )r z2Turn a document into a Beautiful Soup object tree.z[Unknown tree builder]FNc C s d | _ d S )N)�soup)r r r r r f s zTreeBuilder.__init__c C s d S )Nr )r r r r �reseti s zTreeBuilder.resetc C s | j dkrdS || j kS )a� Might a tag with this name be an empty-element tag? The final markup may or may not actually present this tag as self-closing. For instance: an HTMLBuilder does not consider a <p> tag to be an empty-element tag (it's not in HTMLBuilder.empty_element_tags). This means an empty <p> tag will be presented as "<p></p>", not "<p />". The default implementation has no opinion about which tags are empty-element tags, so a tag will be presented as an empty-element tag if and only if it has no contents. "<foo></foo>" will become "<foo />", and "<foo>bar</foo>" will be left alone. NT)�empty_element_tags)r �tag_namer r r �can_be_empty_elementl s z TreeBuilder.can_be_empty_elementc C s t � �d S )N)�NotImplementedError)r �markupr r r �feed� s zTreeBuilder.feedc C s |d d dfS )NFr )r r( Zuser_specified_encodingZdocument_declared_encodingr r r �prepare_markup� s zTreeBuilder.prepare_markupc C s |S )a� Wrap an HTML fragment to make it look like a document. Different parsers do this differently. For instance, lxml introduces an empty <head> tag, and html5lib doesn't. Abstracting this away lets us write simple tests which run HTML fragments through the parser and compare the results against other HTML fragments. This method should not be used outside of tests. r )r Zfragmentr r r �test_fragment_to_document� s z%TreeBuilder.test_fragment_to_documentc C s dS )NFr )r �tagr r r �set_up_substitutions� s z TreeBuilder.set_up_substitutionsc C s� |s|S | j r�| j jdg �}| j j|j� d�}xRt|j� �D ]B}||ksT|r<||kr<|| }t|t�rrtj|�}n|}|||< q<W |S )zaReplaces class="foo bar" with class=["foo", "bar"] Modifies its input in place. �*N) �cdata_list_attributesr �lowerr �keys� isinstance�strr �split)r r% �attrsZ universalZ tag_specific�attr�value�valuesr r r �$_replace_cdata_list_attribute_values� s z0TreeBuilder._replace_cdata_list_attribute_values)NN)r r r! �__doc__�NAMEZALTERNATE_NAMESr Zis_xmlZ picklabler �preserve_whitespace_tagsr$ r/ r r# r& r) r* r+ r- r9 r r r r r T s$ c @ sh e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dd� Z dd� ZdS )r z9A Beautiful Soup treebuilder that listens for SAX events.c C s t � �d S )N)r' )r r( r r r r) � s zSAXTreeBuilder.feedc C s d S )Nr )r r r r �close� s zSAXTreeBuilder.closec C s, t dd� t|j� �D ��}| jj||� d S )Nc s s | ]\}}|d |fV qdS )� Nr )�.0�keyr7 r r r � <genexpr>� s z.SAXTreeBuilder.startElement.<locals>.<genexpr>)�dictr �itemsr" Zhandle_starttag)r �namer5 r r r � startElement� s zSAXTreeBuilder.startElementc C s | j j|� d S )N)r" Z handle_endtag)r rD r r r � endElement� s zSAXTreeBuilder.endElementc C s | j ||� d S )N)rE )r �nsTuple�nodeNamer5 r r r �startElementNS� s z SAXTreeBuilder.startElementNSc C s | j |� d S )N)rF )r rG rH r r r � endElementNS� s zSAXTreeBuilder.endElementNSc C s d S )Nr )r �prefixZ nodeValuer r r �startPrefixMapping� s z!SAXTreeBuilder.startPrefixMappingc C s d S )Nr )r rK r r r �endPrefixMapping� s z SAXTreeBuilder.endPrefixMappingc C s | j j|� d S )N)r" Z handle_data)r �contentr r r � characters� s zSAXTreeBuilder.charactersc C s d S )Nr )r r r r � startDocument� s z SAXTreeBuilder.startDocumentc C s d S )Nr )r r r r � endDocument� s zSAXTreeBuilder.endDocumentN)r r r! r: r) r= rE rF rI rJ rL rM rO rP rQ r r r r r � s c $ @ s� e Zd ZdZejZedddddddd d d d d ddddddddddddg�Zeddd d d d d d!d"d#d$d%d&d'd(d)d*d+d,d-d.dd/d0d1d2d3d4d5d6d7d8d9d:d;g#�Zd<d=d>gd?d@gd?d@gdAgdAgdAgdBgdCgd?gdDgdEgdFgdG� Z dHdI� Z dJS )Kr z]This TreeBuilder knows facts about HTML. Such as which tags are empty-element tags. �area�base�br�colZembedZhrZimg�inputZkeygen�linkZmenuitem�metaZparam�sourceZtrackZwbrZbasefontZbgsoundZcommand�frameZimageZisindexZnextidZspacerZaddressZarticleZasideZ blockquoteZcanvasZddZdivZdlZdtZfieldsetZ figcaptionZfigureZfooter�formZh1Zh2Zh3Zh4Zh5Zh6�headerZli�mainZnavZnoscriptZol�output�pZpreZsection�tableZtfootZulZvideo�classZ accesskeyZdropzoneZrelZrevZheaderszaccept-charset�archiveZsizesZsandbox�for) r. �arW �tdZthre r[ �objectrR ZiconZiframer^ c C sz |j dkrdS |jd�}|jd�}|jd�}d }|d k rJ|}t|�|d< n(|d k rr|d k rr|j� dkrrt|�|d< |d k S )NrX Fz http-equivrN �charsetz content-type)rD r r r0 r )r r, Z http_equivrN rg Z meta_encodingr r r r- s z$HTMLTreeBuilder.set_up_substitutionsN) r r r! r: r r< r r$ Zblock_elementsr/ r- r r r r r � s& N c C sV t jd }xF| jD ]<}t| |�}t|t�rt|||� |jj|� |jj |� qW dS )z9Copy TreeBuilders from the given module into this module.z bs4.builderN) �sys�modules�__all__�getattr� issubclassr �setattr�append�builder_registryr )�moduleZ this_modulerD �objr r r �register_treebuilders_from2 s rr c @ s e Zd ZdS )�ParserRejectedMarkupN)r r r! r r r r rs ? s rs r> )� _htmlparser)� _html5lib)�_lxml) � collectionsr � itertoolsrh Z bs4.elementr r r r rj ZFASTZ PERMISSIVEZSTRICTZXMLZHTMLZHTML_5rf r ro r r r rr � Exceptionrs � rt ru � ImportErrorrv r r r r �<module> s@ 4b.N builder/__pycache__/_htmlparser.cpython-36.opt-1.pyc 0000644 00000020751 15246412105 0016314 0 ustar 00 3 6]�3 � @ s� d Z dgZddlmZ yddlmZ W n2 ek rX Z zG dd� de�ZW Y ddZ[X nX ddlZddl Z ej dd� \Z Z Z e dko�e d ko�e dkZe dko�e dkZe dko�e d kZdd lmZmZmZmZmZ dd lmZmZ dd lmZm Z m Z dZ G dd� de�Z G dd� de �Z e dk�r�e d k�r�e �r�ddl!Z!e!j"d�Z#e#e _#e!j"de!j$�Z%e%e _%ddlm&Z&m'Z' dd� Z(dd� Z)e(e _(e)e _)dZdS )zCUse the HTMLParser library to parse HTML files that aren't too bad.�HTMLParserTreeBuilder� )� HTMLParser)�HTMLParseErrorc @ s e Zd ZdS )r N)�__name__� __module__� __qualname__� r r �!/usr/lib/python3.6/_htmlparser.pyr s r N� � � )�CData�Comment� Declaration�Doctype�ProcessingInstruction)�EntitySubstitution� UnicodeDammit)�HTML�HTMLTreeBuilder�STRICTz html.parserc @ sp e Zd Zdd� Zdd� Zdd� Zddd �Zd d d �Zd d � Zdd� Z dd� Z dd� Z dd� Z dd� Z dd� ZdS ) �BeautifulSoupHTMLParserc O s t j| f|�|� g | _d S )N)r �__init__� already_closed_empty_element)�self�args�kwargsr r r r 9 s z BeautifulSoupHTMLParser.__init__c C s t j|� dS )ai In Python 3, HTMLParser subclasses must implement error(), although this requirement doesn't appear to be documented. In Python 2, HTMLParser implements error() as raising an exception. In any event, this method is called only on very strange markup and our best strategy is to pretend it didn't happen and keep going. N)�warnings�warn)r �msgr r r �errorE s z BeautifulSoupHTMLParser.errorc C s | j ||dd�}| j|� d S )NF)�handle_empty_element)�handle_starttag� handle_endtag)r �name�attrs�tagr r r �handle_startendtagP s z*BeautifulSoupHTMLParser.handle_startendtagTc C sl i }x(|D ] \}}|d kr d}|||< d}q W | j j|d d |�}|rh|jrh|rh| j|dd� | jj|� d S )N� z""F)�check_already_closed)�soupr" Zis_empty_elementr# r �append) r r$ r% r! Z attr_dict�key�value� attrvaluer& r r r r" [ s z'BeautifulSoupHTMLParser.handle_starttagc C s, |r || j kr | j j|� n | jj|� d S )N)r �remover* r# )r r$ r) r r r r# w s z%BeautifulSoupHTMLParser.handle_endtagc C s | j j|� d S )N)r* � handle_data)r �datar r r r0 � s z#BeautifulSoupHTMLParser.handle_datac C s� |j d�r t|jd�d�}n$|j d�r8t|jd�d�}nt|�}d }|dk r�xR| jjdfD ]B}|sdqZyt|g�j|�}W qZ tk r� } zW Y d d }~X qZX qZW |s�y t|�}W n& t t fk r� } zW Y d d }~X nX |p�d}| j |� d S )N�x� �X� z windows-1252u �) � startswith�int�lstripr* �original_encoding� bytearray�decode�UnicodeDecodeError�chr� ValueError� OverflowErrorr0 )r r$ Z real_namer1 �encoding�er r r �handle_charref� s* z&BeautifulSoupHTMLParser.handle_charrefc C s0 t jj|�}|d k r|}nd| }| j|� d S )Nz&%s)r ZHTML_ENTITY_TO_CHARACTER�getr0 )r r$ � characterr1 r r r �handle_entityref� s z(BeautifulSoupHTMLParser.handle_entityrefc C s& | j j� | j j|� | j jt� d S )N)r* �endDatar0 r )r r1 r r r �handle_comment� s z&BeautifulSoupHTMLParser.handle_commentc C sN | j j� |jd�r&|td�d � }n |dkr2d}| j j|� | j jt� d S )NzDOCTYPE ZDOCTYPEr( )r* rF r6 �lenr0 r )r r1 r r r � handle_decl� s z#BeautifulSoupHTMLParser.handle_declc C sN |j � jd�r$t}|td�d � }nt}| jj� | jj|� | jj|� d S )NzCDATA[)�upperr6 r rH r r* rF r0 )r r1 �clsr r r � unknown_decl� s z$BeautifulSoupHTMLParser.unknown_declc C s& | j j� | j j|� | j jt� d S )N)r* rF r0 r )r r1 r r r � handle_pi� s z!BeautifulSoupHTMLParser.handle_piN)T)T)r r r r r r' r" r# r0 rB rE rG rI rL rM r r r r r 7 s ! r c @ s< e Zd ZdZdZeZeeegZ dd� Z d dd�Z dd � Z dS ) r FTc O s, t rt rd|d< tr d|d< ||f| _d S )NF�strictZconvert_charrefs)�CONSTRUCTOR_TAKES_STRICT� CONSTRUCTOR_STRICT_IS_DEPRECATED�"CONSTRUCTOR_TAKES_CONVERT_CHARREFS� parser_args)r r r r r r r � s z HTMLParserTreeBuilder.__init__Nc c sN t |t�r |dddfV dS ||g}t||d|d�}|j|j|j|jfV dS )z� :return: A 4-tuple (markup, original encoding, encoding declared within markup, whether any characters had to be replaced with REPLACEMENT CHARACTER). NFT)Zis_html�exclude_encodings)� isinstance�strr �markupr9 Zdeclared_html_encodingZ contains_replacement_characters)r rV Zuser_specified_encodingZdocument_declared_encodingrS Z try_encodingsZdammitr r r �prepare_markup� s z$HTMLParserTreeBuilder.prepare_markupc C sr | j \}}t||�}| j|_y|j|� |j� W n4 tk rf } ztjtd�� |�W Y d d }~X nX g |_ d S )Na* Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help.) rR r r* �feed�closer r r �RuntimeWarningr )r rV r r �parserrA r r r rX � s zHTMLParserTreeBuilder.feed)NNN) r r r Zis_xmlZ picklable� HTMLPARSER�NAMEr r Zfeaturesr rW rX r r r r r � s zQ\s*((?<=[\'"\s])[^\s/>][^\s/=>]*)(\s*=+\s*(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?a� <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name (?:\s+ # whitespace before attribute name (?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name (?:\s*=\s* # value indicator (?:'[^']*' # LITA-enclosed value |\"[^\"]*\" # LIT-enclosed value |[^'\">\s]+ # bare value ) )? ) )* \s* # trailing whitespace )�tagfind�attrfindc C s, d | _ | j|�}|dk r |S | j}|||� | _ g }tj||d �}|j� }||d |� j� | _}x�||k �rD| jr�t j||�}n t j||�}|s�P |j ddd�\} } } | s�d } nb| d d� d ko�| d d � kn �s | d d� d k�o| d d � kn �r| dd� } | �r(| j | �} |j | j� | f� |j� }qhW |||� j� } | dk�r�| j� \} }d | j k�r�| | j jd � } t| j �| j jd � }n|t| j � }| j�r�| jd |||� d d � f � | j|||� � |S | jd��r| j||� n"| j||� || jk�r(| j|� |S )Nr � r r �'�"�>�/>� z junk characters in start tag: %r� ���rg rg )rc rd )Z__starttag_textZcheck_for_whole_start_tag�rawdatar^ �match�end�lowerZlasttagrN r_ �attrfind_tolerant�groupZunescaper+ �stripZgetpos�countrH �rfindr r0 �endswithr' r" ZCDATA_CONTENT_ELEMENTS�set_cdata_mode)r �i�endposrh r% ri �kr& �mZattrname�restr. rj �lineno�offsetr r r �parse_starttag sZ *, rz c C s$ |j � | _tjd| j tj�| _d S )Nz </\s*%s\s*>)rk Z cdata_elem�re�compile�IZ interesting)r �elemr r r rr T s rr T)*�__doc__�__all__Z html.parserr r � ImportErrorrA � Exception�sysr � version_info�major�minor�releaserO rP rQ Z bs4.elementr r r r r Z bs4.dammitr r Z bs4.builderr r r r\ r r r{ r| rl �VERBOSEZlocatestarttagendr^ r_ rz rr r r r r �<module> sB " 2 7 builder/__pycache__/_lxml.cpython-36.pyc 0000644 00000015446 15246412105 0014155 0 ustar 00 3 6]}% � @ s� d dgZ yddlmZ W n. ek rF Z zddlmZ W Y ddZ[X nX ddlmZ ddlmZ ddl m Z ddl m Z m Z mZmZmZ dd lmZmZmZmZmZmZmZ dd lmZ d ZG d d � d e�Z G d d� dee �Z dS )�LXMLTreeBuilderForXML�LXMLTreeBuilder� )�CallableN)�BytesIO)�StringIO)�etree)�Comment�Doctype�NamespacedAttribute�ProcessingInstruction�XMLProcessingInstruction)�FAST�HTML�HTMLTreeBuilder� PERMISSIVE�ParserRejectedMarkup� TreeBuilder�XML)�EncodingDetector�lxmlc @ s� e Zd ZejZdZeZdZ dgZ e e e e egZdZddiZdd� Zdd � Zd%d d �Zd d� Zd&dd�Zdd� Zdd� Zi fdd�Zdd� Zdd� Zdd � Z d d � Z d d � Z d!d"� Z d#d$� Z d S )'r Tzlxml-xmlZxmli z$http://www.w3.org/XML/1998/namespacec C s" | j d k r| j S tj| dd|d�S )NFT)�target� strip_cdataZrecover�encoding)�_default_parserr � XMLParser)�selfr � r �/usr/lib/python3.6/_lxml.py�default_parser5 s z$LXMLTreeBuilderForXML.default_parserc C s& | j |�}t|t�r"|| d|d�}|S )NF)r r r )r � isinstancer )r r �parserr r r � parser_for= s z LXMLTreeBuilderForXML.parser_forNc C s, || _ |d k rt|�| _d | _| jg| _d S )N)r �set�empty_element_tags�soup�DEFAULT_NSMAPS�nsmaps)r r r# r r r �__init__F s z LXMLTreeBuilderForXML.__init__c C s0 |d dkr$t |dd � jdd��S d |fS d S )Nr �{� �})�tuple�split)r �tagr r r � _getNsTagP s z LXMLTreeBuilderForXML._getNsTagc c s� | j }|rt| _nt| _t|t�r2|d|dfV t|t�rP|jd�d|dfV ||g}t||||�}x |jD ]}|j ||dfV qnW dS )z� :yield: A series of 4-tuples. (markup, encoding, declared encoding, has undergone character replacement) Each 4-tuple represents a strategy for parsing the document. NF�utf8) �is_xmlr � processing_instruction_classr r �str�encoder Z encodings�markup) r r4 Zuser_specified_encodingZexclude_encodingsZdocument_declared_encodingZis_htmlZ try_encodingsZdetectorr r r r �prepare_markupX s z$LXMLTreeBuilderForXML.prepare_markupc C s� t |t�rt|�}nt |t�r&t|�}|j| j�}y`| j| jj �| _ | j j |� x4t |�dkr�|j| j�}t |�dkrR| j j |� qRW | j j � W n6 tttjfk r� } ztt|���W Y d d }~X nX d S )Nr )r �bytesr r2 r �read� CHUNK_SIZEr! r$ �original_encodingr �feed�len�close�UnicodeDecodeError� LookupErrorr � ParserErrorr )r r4 �data�er r r r: } s zLXMLTreeBuilderForXML.feedc C s | j g| _d S )N)r% r& )r r r r r< � s zLXMLTreeBuilderForXML.closec C s* t |�}d }t|�dkr4t| j�dkr4| jjd � nht|�dkr�t dd� t|j� �D ��}| jj|� |j� }x,t|j� �D ] \}}td|d�}|||< q|W i } xVt|j� �D ]F\} } | j| �\}} |d kr�| | | < q�| j |�}t|| |�} | | | < q�W | }| j|�\}}| j |�}| j j ||||� d S )Nr r) c s s | ]\}}||fV qd S )Nr )�.0�key�valuer r r � <genexpr>� s z.LXMLTreeBuilderForXML.start.<locals>.<genexpr>Zxmlnsz http://www.w3.org/2000/xmlns/) �dictr; r& �append�list�items�copyr r. �_prefix_for_namespacer$ Zhandle_starttag) r �nameZattrsZnsmap�nsprefix�inverted_nsmap�prefix� namespaceZ attributeZ new_attrs�attrrD r r r �start� s0 zLXMLTreeBuilderForXML.startc C s< |dkr dS x*t | j�D ] }|dk r||kr|| S qW dS )z9Find the currently active prefix for the given namespace.N)�reversedr& )r rP rN r r r rK � s z+LXMLTreeBuilderForXML._prefix_for_namespacec C s� | j j� | j jd }| j|�\}}d }|d k r^x,t| j�D ] }|d k r<||kr<|| }P q<W | j j||� t| j�dkr�| jj� d S )Nr) ���) r$ �endDataZtagStackr. rS r& Z handle_endtagr; �pop)r rL Z completed_tagrP rM rN r r r �end� s zLXMLTreeBuilderForXML.endc C s0 | j j� | j j|d | � | j j| j� d S )N� )r$ rU � handle_datar1 )r r r@ r r r �pi� s zLXMLTreeBuilderForXML.pic C s | j j|� d S )N)r$ rY )r �contentr r r r@ � s zLXMLTreeBuilderForXML.datac C s( | j j� tj|||�}| j j|� d S )N)r$ rU r Zfor_name_and_idsZobject_was_parsed)r rL Zpubid�system�doctyper r r r] � s z LXMLTreeBuilderForXML.doctypec C s& | j j� | j j|� | j jt� dS )z#Handle comments as Comment objects.N)r$ rU rY r )r r[ r r r �comment� s z LXMLTreeBuilderForXML.commentc C s d| S )zSee `TreeBuilder`.z)<?xml version="1.0" encoding="utf-8"?> %sr )r �fragmentr r r �test_fragment_to_document� s z/LXMLTreeBuilderForXML.test_fragment_to_document)NN)NNN)!�__name__� __module__� __qualname__r r ZDEFAULT_PARSER_CLASSr0 r r1 �NAME�ALTERNATE_NAMES�LXMLr r r �featuresr8 r% r r! r' r. r5 r: r< rR rK rW rZ r@ r] r^ r` r r r r r # s2 # ( c @ sF e Zd ZeZdgZeeeeeg Z dZ e Z dd� Z dd� Zdd� Zd S ) r z lxml-htmlFc C s t jS )N)r Z HTMLParser)r r r r r r � s z LXMLTreeBuilder.default_parserc C sj | j j}y&| j|�| _| jj|� | jj� W n6 tttj fk rd } zt t |���W Y d d }~X nX d S )N) r$ r9 r! r r: r< r= r> r r? r r2 )r r4 r rA r r r r: � s zLXMLTreeBuilder.feedc C s d| S )zSee `TreeBuilder`.z <html><body>%s</body></html>r )r r_ r r r r` s z)LXMLTreeBuilder.test_fragment_to_documentN)ra rb rc rf rd re r r r rg r0 r r1 r r: r` r r r r r � s ) �__all__�collections.abcr � ImportErrorrA � collections�ior r r r Z bs4.elementr r r r r Z bs4.builderr r r r r r r Z bs4.dammitr rf r r r r r r �<module> s $ L builder/__pycache__/_lxml.cpython-36.opt-1.pyc 0000644 00000015446 15246412105 0015114 0 ustar 00 3 6]}% � @ s� d dgZ yddlmZ W n. ek rF Z zddlmZ W Y ddZ[X nX ddlmZ ddlmZ ddl m Z ddl m Z m Z mZmZmZ dd lmZmZmZmZmZmZmZ dd lmZ d ZG d d � d e�Z G d d� dee �Z dS )�LXMLTreeBuilderForXML�LXMLTreeBuilder� )�CallableN)�BytesIO)�StringIO)�etree)�Comment�Doctype�NamespacedAttribute�ProcessingInstruction�XMLProcessingInstruction)�FAST�HTML�HTMLTreeBuilder� PERMISSIVE�ParserRejectedMarkup� TreeBuilder�XML)�EncodingDetector�lxmlc @ s� e Zd ZejZdZeZdZ dgZ e e e e egZdZddiZdd� Zdd � Zd%d d �Zd d� Zd&dd�Zdd� Zdd� Zi fdd�Zdd� Zdd� Zdd � Z d d � Z d d � Z d!d"� Z d#d$� Z d S )'r Tzlxml-xmlZxmli z$http://www.w3.org/XML/1998/namespacec C s" | j d k r| j S tj| dd|d�S )NFT)�target� strip_cdataZrecover�encoding)�_default_parserr � XMLParser)�selfr � r �/usr/lib/python3.6/_lxml.py�default_parser5 s z$LXMLTreeBuilderForXML.default_parserc C s& | j |�}t|t�r"|| d|d�}|S )NF)r r r )r � isinstancer )r r �parserr r r � parser_for= s z LXMLTreeBuilderForXML.parser_forNc C s, || _ |d k rt|�| _d | _| jg| _d S )N)r �set�empty_element_tags�soup�DEFAULT_NSMAPS�nsmaps)r r r# r r r �__init__F s z LXMLTreeBuilderForXML.__init__c C s0 |d dkr$t |dd � jdd��S d |fS d S )Nr �{� �})�tuple�split)r �tagr r r � _getNsTagP s z LXMLTreeBuilderForXML._getNsTagc c s� | j }|rt| _nt| _t|t�r2|d|dfV t|t�rP|jd�d|dfV ||g}t||||�}x |jD ]}|j ||dfV qnW dS )z� :yield: A series of 4-tuples. (markup, encoding, declared encoding, has undergone character replacement) Each 4-tuple represents a strategy for parsing the document. NF�utf8) �is_xmlr � processing_instruction_classr r �str�encoder Z encodings�markup) r r4 Zuser_specified_encodingZexclude_encodingsZdocument_declared_encodingZis_htmlZ try_encodingsZdetectorr r r r �prepare_markupX s z$LXMLTreeBuilderForXML.prepare_markupc C s� t |t�rt|�}nt |t�r&t|�}|j| j�}y`| j| jj �| _ | j j |� x4t |�dkr�|j| j�}t |�dkrR| j j |� qRW | j j � W n6 tttjfk r� } ztt|���W Y d d }~X nX d S )Nr )r �bytesr r2 r �read� CHUNK_SIZEr! r$ �original_encodingr �feed�len�close�UnicodeDecodeError� LookupErrorr � ParserErrorr )r r4 �data�er r r r: } s zLXMLTreeBuilderForXML.feedc C s | j g| _d S )N)r% r& )r r r r r< � s zLXMLTreeBuilderForXML.closec C s* t |�}d }t|�dkr4t| j�dkr4| jjd � nht|�dkr�t dd� t|j� �D ��}| jj|� |j� }x,t|j� �D ] \}}td|d�}|||< q|W i } xVt|j� �D ]F\} } | j| �\}} |d kr�| | | < q�| j |�}t|| |�} | | | < q�W | }| j|�\}}| j |�}| j j ||||� d S )Nr r) c s s | ]\}}||fV qd S )Nr )�.0�key�valuer r r � <genexpr>� s z.LXMLTreeBuilderForXML.start.<locals>.<genexpr>Zxmlnsz http://www.w3.org/2000/xmlns/) �dictr; r& �append�list�items�copyr r. �_prefix_for_namespacer$ Zhandle_starttag) r �nameZattrsZnsmap�nsprefix�inverted_nsmap�prefix� namespaceZ attributeZ new_attrs�attrrD r r r �start� s0 zLXMLTreeBuilderForXML.startc C s< |dkr dS x*t | j�D ] }|dk r||kr|| S qW dS )z9Find the currently active prefix for the given namespace.N)�reversedr& )r rP rN r r r rK � s z+LXMLTreeBuilderForXML._prefix_for_namespacec C s� | j j� | j jd }| j|�\}}d }|d k r^x,t| j�D ] }|d k r<||kr<|| }P q<W | j j||� t| j�dkr�| jj� d S )Nr) ���) r$ �endDataZtagStackr. rS r& Z handle_endtagr; �pop)r rL Z completed_tagrP rM rN r r r �end� s zLXMLTreeBuilderForXML.endc C s0 | j j� | j j|d | � | j j| j� d S )N� )r$ rU � handle_datar1 )r r r@ r r r �pi� s zLXMLTreeBuilderForXML.pic C s | j j|� d S )N)r$ rY )r �contentr r r r@ � s zLXMLTreeBuilderForXML.datac C s( | j j� tj|||�}| j j|� d S )N)r$ rU r Zfor_name_and_idsZobject_was_parsed)r rL Zpubid�system�doctyper r r r] � s z LXMLTreeBuilderForXML.doctypec C s& | j j� | j j|� | j jt� dS )z#Handle comments as Comment objects.N)r$ rU rY r )r r[ r r r �comment� s z LXMLTreeBuilderForXML.commentc C s d| S )zSee `TreeBuilder`.z)<?xml version="1.0" encoding="utf-8"?> %sr )r �fragmentr r r �test_fragment_to_document� s z/LXMLTreeBuilderForXML.test_fragment_to_document)NN)NNN)!�__name__� __module__� __qualname__r r ZDEFAULT_PARSER_CLASSr0 r r1 �NAME�ALTERNATE_NAMES�LXMLr r r �featuresr8 r% r r! r' r. r5 r: r< rR rK rW rZ r@ r] r^ r` r r r r r # s2 # ( c @ sF e Zd ZeZdgZeeeeeg Z dZ e Z dd� Z dd� Zdd� Zd S ) r z lxml-htmlFc C s t jS )N)r Z HTMLParser)r r r r r r � s z LXMLTreeBuilder.default_parserc C sj | j j}y&| j|�| _| jj|� | jj� W n6 tttj fk rd } zt t |���W Y d d }~X nX d S )N) r$ r9 r! r r: r< r= r> r r? r r2 )r r4 r rA r r r r: � s zLXMLTreeBuilder.feedc C s d| S )zSee `TreeBuilder`.z <html><body>%s</body></html>r )r r_ r r r r` s z)LXMLTreeBuilder.test_fragment_to_documentN)ra rb rc rf rd re r r r rg r0 r r1 r r: r` r r r r r � s ) �__all__�collections.abcr � ImportErrorrA � collections�ior r r r Z bs4.elementr r r r r Z bs4.builderr r r r r r r Z bs4.dammitr rf r r r r r r �<module> s $ L builder/__pycache__/_html5lib.cpython-36.opt-1.pyc 0000644 00000026326 15246412105 0015657 0 ustar 00 3 6]0A � @ s d gZ ddlZddlZddlmZmZmZmZ ddlm Z m Z ddl Z ddl m Z mZ ddlmZmZmZmZ yddlmZ dZW n2 ek r� Z zdd lmZ d ZW Y ddZ[X nX G d d � d e�ZG d d � d ej�Z G dd� de �Z G dd� dej �Z G dd� de �Z!dS )�HTML5TreeBuilder� N)� PERMISSIVE�HTML�HTML_5�HTMLTreeBuilder)�NamespacedAttribute� whitespace_re)� namespaces�prefixes)�Comment�Doctype�NavigableString�Tag)�_baseF)�baseTc @ sB e Zd ZdZdZeeeegZd dd�Z dd� Z dd � Z d d � Z dS ) r z Use html5lib to build a tree.�html5libNc c s& || _ |rtjd� |d d dfV d S )NzjYou provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.F)�user_specified_encoding�warnings�warn)�self�markupr Zdocument_declared_encodingZexclude_encodings� r � /usr/lib/python3.6/_html5lib.py�prepare_markup0 s z HTML5TreeBuilder.prepare_markupc C s� | j jd k rtjd� tj| jd�}t� }t|t �sNt rD| j |d< n | j |d< |j |f|�}t|t �rnd |_ n$|jjjd }t|t �s�|j}||_ d S )Nz�You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.)ZtreeZoverride_encoding�encodingr )�soupZ parse_onlyr r r Z HTMLParser�create_treebuilder�dict� isinstance�str� new_html5libr �parse�original_encodingZ tokenizer�streamZ charEncoding�name)r r �parserZ extra_kwargs�docr" r r r �feed= s zHTML5TreeBuilder.feedc C s t || j�| _| jS )N)�TreeBuilderForHtml5libr Zunderlying_builder)r �namespaceHTMLElementsr r r r X s z#HTML5TreeBuilder.create_treebuilderc C s d| S )zSee `TreeBuilder`.z)<html><head></head><body>%s</body></html>r )r Zfragmentr r r �test_fragment_to_document] s z*HTML5TreeBuilder.test_fragment_to_document)NN) �__name__� __module__� __qualname__�__doc__�NAMEr r r Zfeaturesr r' r r* r r r r r ) s c sf e Zd Zd� fdd� Zdd� Zdd� Zdd � Zd d � Zd d � Zdd� Z dd� Z dd� Z dd� Z � Z S )r( Nc s8 |r || _ nddlm} |dd�| _ tt| �j|� d S )Nr )� BeautifulSoup� z html.parser)r �bs4r0 �superr( �__init__)r r) r r0 )� __class__r r r4 d s z TreeBuilderForHtml5lib.__init__c C s | j j� t| j | j d �S )N)r �reset�Element)r r r r � documentClassl s z$TreeBuilderForHtml5lib.documentClassc C s6 |d }|d }|d }t j|||�}| jj|� d S )Nr$ �publicId�systemId)r Zfor_name_and_idsr �object_was_parsed)r �tokenr$ r9 r: Zdoctyper r r � insertDoctypep s z$TreeBuilderForHtml5lib.insertDoctypec C s | j j||�}t|| j |�S )N)r �new_tagr7 )r r$ � namespace�tagr r r � elementClassx s z#TreeBuilderForHtml5lib.elementClassc C s t t|�| j�S )N)�TextNoder r )r �datar r r � commentClass| s z#TreeBuilderForHtml5lib.commentClassc C s0 ddl m} |dd�| _d| j_t| j| jd �S )Nr )r0 r1 z html.parserz[document_fragment])r2 r0 r r$ r7 )r r0 r r r � fragmentClass s z$TreeBuilderForHtml5lib.fragmentClassc C s | j j|j� d S )N)r �append�element)r �noder r r � appendChild� s z"TreeBuilderForHtml5lib.appendChildc C s | j S )N)r )r r r r � getDocument� s z"TreeBuilderForHtml5lib.getDocumentc C s t jj| �jS )N)�treebuilder_base� TreeBuilder� getFragmentrG )r r r r rM � s z"TreeBuilderForHtml5lib.getFragmentc sB ddl m� g �tjd��d� ���fdd� ��|d� dj��S )Nr )r0 z8^(.*?)(?: PUBLIC "(.*?)"(?: "(.*?)")?| SYSTEM "(.*?)")?$c s� t | � �r t | t�r��j| �}|r�|jd�}|jdkrx|jd�pBd}|jd�pZ|jd�pZd}�jdd| |||f � q��jdd| |f � n�jd d| f � �nHt | t�r̈jd d| | f � �n$t | t�r�jd d| | f � �n | j�rd t | j | j f }n| j }�jd d| |f � | j �r�g }x`t | j j � �D ]N\}}t |t��rnd t |j |j f }t |t ��r�dj|�}|j||f� �qFW x2t|�D ]&\}}�jdd|d ||f � �q�W |d7 }x| jD ]}�||� �q�W d S )N� � r1 � � z|%s<!DOCTYPE %s "%s" "%s">� z|%s<!DOCTYPE %s>z|%s<!DOCTYPE >z|%s<!-- %s -->z|%s"%s"z%s %sz|%s<%s>z |%s%s="%s")r r �match�group� lastindexrF r r r? r r$ �attrs�list�itemsr �join�sortedZchildren) rG �indent�mr$ r9 r: � attributes�value�child)r0 � doctype_re�rv�serializeElementr r rb � sD " z?TreeBuilderForHtml5lib.testSerializer.<locals>.serializeElement� )r )r2 r0 �re�compilerY )r rG r )r0 r` ra rb r �testSerializer� s ) z%TreeBuilderForHtml5lib.testSerializer)N)r+ r, r- r4 r8 r= rA rD rE rI rJ rM rf � __classcell__r r )r5 r r( b s r( c @ sL e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dS )�AttrListc C s || _ t| j j�| _d S )N)rG r rV )r rG r r r r4 � s zAttrList.__init__c C s t | jj� �j� S )N)rW rV rX �__iter__)r r r r ri � s zAttrList.__iter__c C sP t j}||d ks.| jj|krB||| jj krBt|t�sBtj|�}|| j|< d S )N�*)r Zcdata_list_attributesrG r$ r rW r �split)r r$ r^ Z list_attrr r r � __setitem__� s zAttrList.__setitem__c C s t | jj� �S )N)rW rV rX )r r r r rX � s zAttrList.itemsc C s t | jj� �S )N)rW rV �keys)r r r r rm � s z AttrList.keysc C s t | j�S )N)�lenrV )r r r r �__len__� s zAttrList.__len__c C s | j | S )N)rV )r r$ r r r � __getitem__� s zAttrList.__getitem__c C s |t | jj� �kS )N)rW rV rm )r r$ r r r � __contains__� s zAttrList.__contains__N) r+ r, r- r4 ri rl rX rm ro rp rq r r r r rh � s rh c @ sx e Zd Zdd� Zdd� Zdd� Zdd� Zeee�Zdd d �Z d d � Z dd� Z dd� Z dd� Z dd� Zdd� Zee�Zd S )r7 c C s& t jj| |j� || _|| _|| _d S )N)rK �Noder4 r$ rG r r? )r rG r r? r r r r4 � s zElement.__init__c C s( d }}t |t�r | }}n:t |t�r,|}n*|jjtkrJ|j }}| |_n |j}| |_t |t� rv|jd k rv|jj� |r�| jjr�| jjd jtkr�| jjd }| j j || �}|j |� || j _ n`t |t�r�| j j |�}| jjr�| jj d�}n | jjd k �r | j j � }n| j}| j j|| j|d� d S )NrN F)�parent�most_recent_element���ru )r r r rG r5 r rs �extract�contentsr � new_string� replace_withZ_most_recent_element�_last_descendant� next_elementr; )r rH Z string_childr_ Z old_elementZ new_elementrt r r r rI � s8 zElement.appendChildc C s t | jt�ri S t| j�S )N)r rG r rh )r r r r � getAttributes s zElement.getAttributesc C s� |d k r�t |�dkr�g }x8t|j� �D ](\}}t|t�r&t|� }||= |||< q&W | jjj| j |� x"t|j� �D ]\}}|| j |< qrW | jjj | j � d S )Nr ) rn rW rX r �tupler r ZbuilderZ$_replace_cdata_list_attribute_valuesr$ rG Zset_up_substitutions)r r] Zconverted_attributesr$ r^ �new_namer r r � setAttributes! s zElement.setAttributesNc C s4 t | jj|�| j�}|r&| j||� n | j|� d S )N)rB r rx � insertBeforerI )r rC r� �textr r r � insertText9 s zElement.insertTextc C s� | j j|j �}|j jtkrf| j jrf| j j|d jtkrf| j j|d }| jj||j �}|j|� n| j j||j � | |_ d S )NrN ) rG �indexr5 r rw r rx ry �insertrs )r rH ZrefNoder� Zold_nodeZnew_strr r r r� @ s zElement.insertBeforec C s |j j� d S )N)rG rv )r rH r r r � removeChildL s zElement.removeChildc C s� | j }|j }|j}|jdd�}t|j�dkr>|jd }|j}n d}|j}|j}t|�dkr�|d } |rn|| _n|| _|| _|r�| |_n| |_|r�| |_|d jdd�} || _|r�| |_d| _x |D ]} || _|jj | � q�W g |_||_dS )z1Move all of this tag's children into another tag.Fr rN NTru ru ) rG Z next_siblingrz rn rw r{ Zprevious_elementZprevious_siblingrs rF ) r Z new_parentrG Znew_parent_elementZfinal_next_elementZnew_parents_last_descendantZnew_parents_last_childZ(new_parents_last_descendant_next_elementZ to_appendZ first_childZlast_childs_last_descendantr_ r r r �reparentChildrenO s> zElement.reparentChildrenc C sF | j j| jj| j�}t|| j | j�}x | jD ]\}}||j|< q,W |S )N)r r> rG r$ r? r7 r] )r r@ rH �keyr^ r r r � cloneNode� s zElement.cloneNodec C s | j jS )N)rG rw )r r r r � hasContent� s zElement.hasContentc C s( | j d krtd | jfS | j | jfS d S )NZhtml)r? r r$ )r r r r � getNameTuple� s zElement.getNameTuple)N)r+ r, r- r4 rI r| r �propertyr] r� r� r� r� r� r� r� Z nameTupler r r r r7 � s 6 Br7 c @ s e Zd Zdd� Zdd� ZdS )rB c C s t jj| d � || _|| _d S )N)rK rr r4 rG r )r rG r r r r r4 � s zTextNode.__init__c C s t �d S )N)�NotImplementedError)r r r r r� � s zTextNode.cloneNodeN)r+ r, r- r4 r� r r r r rB � s rB )"�__all__r rd Z bs4.builderr r r r Z bs4.elementr r r Zhtml5lib.constantsr r r r r r Zhtml5lib.treebuildersr rK r � ImportError�er r rL r( �objectrh rr r7 rB r r r r �<module> s& 9_ E __init__.py 0000644 00000054530 15246412105 0006664 0 ustar 00 """Beautiful Soup Elixir and Tonic "The Screen-Scraper's Friend" http://www.crummy.com/software/BeautifulSoup/ Beautiful Soup uses a pluggable XML or HTML parser to parse a (possibly invalid) document into a tree representation. Beautiful Soup provides methods and Pythonic idioms that make it easy to navigate, search, and modify the parse tree. Beautiful Soup works with Python 2.7 and up. It works better if lxml and/or html5lib is installed. For more than you ever wanted to know about Beautiful Soup, see the documentation: http://www.crummy.com/software/BeautifulSoup/bs4/doc/ """ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. __author__ = "Leonard Richardson (leonardr@segfault.org)" __version__ = "4.6.3" __copyright__ = "Copyright (c) 2004-2018 Leonard Richardson" __license__ = "MIT" __all__ = ['BeautifulSoup'] import os import re import sys import traceback import warnings from .builder import builder_registry, ParserRejectedMarkup from .dammit import UnicodeDammit from .element import ( CData, Comment, DEFAULT_OUTPUT_ENCODING, Declaration, Doctype, NavigableString, PageElement, ProcessingInstruction, ResultSet, SoupStrainer, Tag, ) # The very first thing we do is give a useful error if someone is # running this code under Python 3 without converting it. 'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work.'!='You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).' class BeautifulSoup(Tag): """ This class defines the basic interface called by the tree builders. These methods will be called by the parser: reset() feed(markup) The tree builder may call these methods from its feed() implementation: handle_starttag(name, attrs) # See note about return value handle_endtag(name) handle_data(data) # Appends to the current data node endData(containerClass=NavigableString) # Ends the current data node No matter how complicated the underlying parser is, you should be able to build a tree using 'start tag' events, 'end tag' events, 'data' events, and "done with data" events. If you encounter an empty-element tag (aka a self-closing tag, like HTML's <br> tag), call handle_starttag and then handle_endtag. """ ROOT_TAG_NAME = '[document]' # If the end-user gives no indication which tree builder they # want, look for one with these features. DEFAULT_BUILDER_FEATURES = ['html', 'fast'] ASCII_SPACES = '\x20\x0a\x09\x0c\x0d' NO_PARSER_SPECIFIED_WARNING = "No parser was explicitly specified, so I'm using the best available %(markup_type)s parser for this system (\"%(parser)s\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n\nThe code that caused this warning is on line %(line_number)s of the file %(filename)s. To get rid of this warning, pass the additional argument 'features=\"%(parser)s\"' to the BeautifulSoup constructor.\n" def __init__(self, markup="", features=None, builder=None, parse_only=None, from_encoding=None, exclude_encodings=None, **kwargs): """Constructor. :param markup: A string or a file-like object representing markup to be parsed. :param features: Desirable features of the parser to be used. This may be the name of a specific parser ("lxml", "lxml-xml", "html.parser", or "html5lib") or it may be the type of markup to be used ("html", "html5", "xml"). It's recommended that you name a specific parser, so that Beautiful Soup gives you the same results across platforms and virtual environments. :param builder: A specific TreeBuilder to use instead of looking one up based on `features`. You shouldn't need to use this. :param parse_only: A SoupStrainer. Only parts of the document matching the SoupStrainer will be considered. This is useful when parsing part of a document that would otherwise be too large to fit into memory. :param from_encoding: A string indicating the encoding of the document to be parsed. Pass this in if Beautiful Soup is guessing wrongly about the document's encoding. :param exclude_encodings: A list of strings indicating encodings known to be wrong. Pass this in if you don't know the document's encoding but you know Beautiful Soup's guess is wrong. :param kwargs: For backwards compatibility purposes, the constructor accepts certain keyword arguments used in Beautiful Soup 3. None of these arguments do anything in Beautiful Soup 4 and there's no need to actually pass keyword arguments into the constructor. """ if 'convertEntities' in kwargs: warnings.warn( "BS4 does not respect the convertEntities argument to the " "BeautifulSoup constructor. Entities are always converted " "to Unicode characters.") if 'markupMassage' in kwargs: del kwargs['markupMassage'] warnings.warn( "BS4 does not respect the markupMassage argument to the " "BeautifulSoup constructor. The tree builder is responsible " "for any necessary markup massage.") if 'smartQuotesTo' in kwargs: del kwargs['smartQuotesTo'] warnings.warn( "BS4 does not respect the smartQuotesTo argument to the " "BeautifulSoup constructor. Smart quotes are always converted " "to Unicode characters.") if 'selfClosingTags' in kwargs: del kwargs['selfClosingTags'] warnings.warn( "BS4 does not respect the selfClosingTags argument to the " "BeautifulSoup constructor. The tree builder is responsible " "for understanding self-closing tags.") if 'isHTML' in kwargs: del kwargs['isHTML'] warnings.warn( "BS4 does not respect the isHTML argument to the " "BeautifulSoup constructor. Suggest you use " "features='lxml' for HTML and features='lxml-xml' for " "XML.") def deprecated_argument(old_name, new_name): if old_name in kwargs: warnings.warn( 'The "%s" argument to the BeautifulSoup constructor ' 'has been renamed to "%s."' % (old_name, new_name)) value = kwargs[old_name] del kwargs[old_name] return value return None parse_only = parse_only or deprecated_argument( "parseOnlyThese", "parse_only") from_encoding = from_encoding or deprecated_argument( "fromEncoding", "from_encoding") if from_encoding and isinstance(markup, str): warnings.warn("You provided Unicode markup but also provided a value for from_encoding. Your from_encoding will be ignored.") from_encoding = None if len(kwargs) > 0: arg = list(kwargs.keys()).pop() raise TypeError( "__init__() got an unexpected keyword argument '%s'" % arg) if builder is None: original_features = features if isinstance(features, str): features = [features] if features is None or len(features) == 0: features = self.DEFAULT_BUILDER_FEATURES builder_class = builder_registry.lookup(*features) if builder_class is None: raise FeatureNotFound( "Couldn't find a tree builder with the features you " "requested: %s. Do you need to install a parser library?" % ",".join(features)) builder = builder_class() if not (original_features == builder.NAME or original_features in builder.ALTERNATE_NAMES): if builder.is_xml: markup_type = "XML" else: markup_type = "HTML" # This code adapted from warnings.py so that we get the same line # of code as our warnings.warn() call gets, even if the answer is wrong # (as it may be in a multithreading situation). caller = None try: caller = sys._getframe(1) except ValueError: pass if caller: globals = caller.f_globals line_number = caller.f_lineno else: globals = sys.__dict__ line_number= 1 filename = globals.get('__file__') if filename: fnl = filename.lower() if fnl.endswith((".pyc", ".pyo")): filename = filename[:-1] if filename: # If there is no filename at all, the user is most likely in a REPL, # and the warning is not necessary. values = dict( filename=filename, line_number=line_number, parser=builder.NAME, markup_type=markup_type ) warnings.warn(self.NO_PARSER_SPECIFIED_WARNING % values, stacklevel=2) self.builder = builder self.is_xml = builder.is_xml self.known_xml = self.is_xml self.builder.soup = self self.parse_only = parse_only if hasattr(markup, 'read'): # It's a file-type object. markup = markup.read() elif len(markup) <= 256 and ( (isinstance(markup, bytes) and not b'<' in markup) or (isinstance(markup, str) and not '<' in markup) ): # Print out warnings for a couple beginner problems # involving passing non-markup to Beautiful Soup. # Beautiful Soup will still parse the input as markup, # just in case that's what the user really wants. if (isinstance(markup, str) and not os.path.supports_unicode_filenames): possible_filename = markup.encode("utf8") else: possible_filename = markup is_file = False try: is_file = os.path.exists(possible_filename) except Exception as e: # This is almost certainly a problem involving # characters not valid in filenames on this # system. Just let it go. pass if is_file: if isinstance(markup, str): markup = markup.encode("utf8") warnings.warn( '"%s" looks like a filename, not markup. You should' ' probably open this file and pass the filehandle into' ' Beautiful Soup.' % markup) self._check_markup_is_url(markup) for (self.markup, self.original_encoding, self.declared_html_encoding, self.contains_replacement_characters) in ( self.builder.prepare_markup( markup, from_encoding, exclude_encodings=exclude_encodings)): self.reset() try: self._feed() break except ParserRejectedMarkup: pass # Clear out the markup and remove the builder's circular # reference to this object. self.markup = None self.builder.soup = None def __copy__(self): copy = type(self)( self.encode('utf-8'), builder=self.builder, from_encoding='utf-8' ) # Although we encoded the tree to UTF-8, that may not have # been the encoding of the original markup. Set the copy's # .original_encoding to reflect the original object's # .original_encoding. copy.original_encoding = self.original_encoding return copy def __getstate__(self): # Frequently a tree builder can't be pickled. d = dict(self.__dict__) if 'builder' in d and not self.builder.picklable: d['builder'] = None return d @staticmethod def _check_markup_is_url(markup): """ Check if markup looks like it's actually a url and raise a warning if so. Markup can be unicode or str (py2) / bytes (py3). """ if isinstance(markup, bytes): space = b' ' cant_start_with = (b"http:", b"https:") elif isinstance(markup, str): space = ' ' cant_start_with = ("http:", "https:") else: return if any(markup.startswith(prefix) for prefix in cant_start_with): if not space in markup: if isinstance(markup, bytes): decoded_markup = markup.decode('utf-8', 'replace') else: decoded_markup = markup warnings.warn( '"%s" looks like a URL. Beautiful Soup is not an' ' HTTP client. You should probably use an HTTP client like' ' requests to get the document behind the URL, and feed' ' that document to Beautiful Soup.' % decoded_markup ) def _feed(self): # Convert the document to Unicode. self.builder.reset() self.builder.feed(self.markup) # Close out any unfinished strings and close all the open tags. self.endData() while self.currentTag.name != self.ROOT_TAG_NAME: self.popTag() def reset(self): Tag.__init__(self, self, self.builder, self.ROOT_TAG_NAME) self.hidden = 1 self.builder.reset() self.current_data = [] self.currentTag = None self.tagStack = [] self.preserve_whitespace_tag_stack = [] self.pushTag(self) def new_tag(self, name, namespace=None, nsprefix=None, attrs={}, **kwattrs): """Create a new tag associated with this soup.""" kwattrs.update(attrs) return Tag(None, self.builder, name, namespace, nsprefix, kwattrs) def new_string(self, s, subclass=NavigableString): """Create a new NavigableString associated with this soup.""" return subclass(s) def insert_before(self, successor): raise NotImplementedError("BeautifulSoup objects don't support insert_before().") def insert_after(self, successor): raise NotImplementedError("BeautifulSoup objects don't support insert_after().") def popTag(self): tag = self.tagStack.pop() if self.preserve_whitespace_tag_stack and tag == self.preserve_whitespace_tag_stack[-1]: self.preserve_whitespace_tag_stack.pop() #print "Pop", tag.name if self.tagStack: self.currentTag = self.tagStack[-1] return self.currentTag def pushTag(self, tag): #print "Push", tag.name if self.currentTag: self.currentTag.contents.append(tag) self.tagStack.append(tag) self.currentTag = self.tagStack[-1] if tag.name in self.builder.preserve_whitespace_tags: self.preserve_whitespace_tag_stack.append(tag) def endData(self, containerClass=NavigableString): if self.current_data: current_data = ''.join(self.current_data) # If whitespace is not preserved, and this string contains # nothing but ASCII spaces, replace it with a single space # or newline. if not self.preserve_whitespace_tag_stack: strippable = True for i in current_data: if i not in self.ASCII_SPACES: strippable = False break if strippable: if '\n' in current_data: current_data = '\n' else: current_data = ' ' # Reset the data collector. self.current_data = [] # Should we add this string to the tree at all? if self.parse_only and len(self.tagStack) <= 1 and \ (not self.parse_only.text or \ not self.parse_only.search(current_data)): return o = containerClass(current_data) self.object_was_parsed(o) def object_was_parsed(self, o, parent=None, most_recent_element=None): """Add an object to the parse tree.""" parent = parent or self.currentTag previous_element = most_recent_element or self._most_recent_element next_element = previous_sibling = next_sibling = None if isinstance(o, Tag): next_element = o.next_element next_sibling = o.next_sibling previous_sibling = o.previous_sibling if not previous_element: previous_element = o.previous_element o.setup(parent, previous_element, next_element, previous_sibling, next_sibling) self._most_recent_element = o parent.contents.append(o) if parent.next_sibling: # This node is being inserted into an element that has # already been parsed. Deal with any dangling references. index = len(parent.contents)-1 while index >= 0: if parent.contents[index] is o: break index -= 1 else: raise ValueError( "Error building tree: supposedly %r was inserted " "into %r after the fact, but I don't see it!" % ( o, parent ) ) if index == 0: previous_element = parent previous_sibling = None else: previous_element = previous_sibling = parent.contents[index-1] if index == len(parent.contents)-1: next_element = parent.next_sibling next_sibling = None else: next_element = next_sibling = parent.contents[index+1] o.previous_element = previous_element if previous_element: previous_element.next_element = o o.next_element = next_element if next_element: next_element.previous_element = o o.next_sibling = next_sibling if next_sibling: next_sibling.previous_sibling = o o.previous_sibling = previous_sibling if previous_sibling: previous_sibling.next_sibling = o def _popToTag(self, name, nsprefix=None, inclusivePop=True): """Pops the tag stack up to and including the most recent instance of the given tag. If inclusivePop is false, pops the tag stack up to but *not* including the most recent instqance of the given tag.""" #print "Popping to %s" % name if name == self.ROOT_TAG_NAME: # The BeautifulSoup object itself can never be popped. return most_recently_popped = None stack_size = len(self.tagStack) for i in range(stack_size - 1, 0, -1): t = self.tagStack[i] if (name == t.name and nsprefix == t.prefix): if inclusivePop: most_recently_popped = self.popTag() break most_recently_popped = self.popTag() return most_recently_popped def handle_starttag(self, name, namespace, nsprefix, attrs): """Push a start tag on to the stack. If this method returns None, the tag was rejected by the SoupStrainer. You should proceed as if the tag had not occurred in the document. For instance, if this was a self-closing tag, don't call handle_endtag. """ # print "Start tag %s: %s" % (name, attrs) self.endData() if (self.parse_only and len(self.tagStack) <= 1 and (self.parse_only.text or not self.parse_only.search_tag(name, attrs))): return None tag = Tag(self, self.builder, name, namespace, nsprefix, attrs, self.currentTag, self._most_recent_element) if tag is None: return tag if self._most_recent_element: self._most_recent_element.next_element = tag self._most_recent_element = tag self.pushTag(tag) return tag def handle_endtag(self, name, nsprefix=None): #print "End tag: " + name self.endData() self._popToTag(name, nsprefix) def handle_data(self, data): self.current_data.append(data) def decode(self, pretty_print=False, eventual_encoding=DEFAULT_OUTPUT_ENCODING, formatter="minimal"): """Returns a string or Unicode representation of this document. To get Unicode, pass None for encoding.""" if self.is_xml: # Print the XML declaration encoding_part = '' if eventual_encoding != None: encoding_part = ' encoding="%s"' % eventual_encoding prefix = '<?xml version="1.0"%s?>\n' % encoding_part else: prefix = '' if not pretty_print: indent_level = None else: indent_level = 0 return prefix + super(BeautifulSoup, self).decode( indent_level, eventual_encoding, formatter) # Alias to make it easier to type import: 'from bs4 import _soup' _s = BeautifulSoup _soup = BeautifulSoup class BeautifulStoneSoup(BeautifulSoup): """Deprecated interface to an XML parser.""" def __init__(self, *args, **kwargs): kwargs['features'] = 'xml' warnings.warn( 'The BeautifulStoneSoup class is deprecated. Instead of using ' 'it, pass features="xml" into the BeautifulSoup constructor.') super(BeautifulStoneSoup, self).__init__(*args, **kwargs) class StopParsing(Exception): pass class FeatureNotFound(ValueError): pass #By default, act as an HTML pretty-printer. if __name__ == '__main__': import sys soup = BeautifulSoup(sys.stdin) print((soup.prettify())) diagnose.py 0000644 00000015604 15246412105 0006715 0 ustar 00 """Diagnostic functions, mainly for use when doing tech support.""" # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. __license__ = "MIT" import cProfile from io import StringIO from html.parser import HTMLParser import bs4 from bs4 import BeautifulSoup, __version__ from bs4.builder import builder_registry import os import pstats import random import tempfile import time import traceback import sys import cProfile def diagnose(data): """Diagnostic suite for isolating common problems.""" print(("Diagnostic running on Beautiful Soup %s" % __version__)) print(("Python version %s" % sys.version)) basic_parsers = ["html.parser", "html5lib", "lxml"] for name in basic_parsers: for builder in builder_registry.builders: if name in builder.features: break else: basic_parsers.remove(name) print(( "I noticed that %s is not installed. Installing it may help." % name)) if 'lxml' in basic_parsers: basic_parsers.append("lxml-xml") try: from lxml import etree print(("Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION)))) except ImportError as e: print ( "lxml is not installed or couldn't be imported.") if 'html5lib' in basic_parsers: try: import html5lib print(("Found html5lib version %s" % html5lib.__version__)) except ImportError as e: print ( "html5lib is not installed or couldn't be imported.") if hasattr(data, 'read'): data = data.read() elif data.startswith("http:") or data.startswith("https:"): print(('"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data)) print("You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup.") return else: try: if os.path.exists(data): print(('"%s" looks like a filename. Reading data from the file.' % data)) with open(data) as fp: data = fp.read() except ValueError: # This can happen on some platforms when the 'filename' is # too long. Assume it's data and not a filename. pass print() for parser in basic_parsers: print(("Trying to parse your markup with %s" % parser)) success = False try: soup = BeautifulSoup(data, features=parser) success = True except Exception as e: print(("%s could not parse the markup." % parser)) traceback.print_exc() if success: print(("Here's what %s did with the markup:" % parser)) print((soup.prettify())) print(("-" * 80)) def lxml_trace(data, html=True, **kwargs): """Print out the lxml events that occur during parsing. This lets you see how lxml parses a document when no Beautiful Soup code is running. """ from lxml import etree for event, element in etree.iterparse(StringIO(data), html=html, **kwargs): print(("%s, %4s, %s" % (event, element.tag, element.text))) class AnnouncingParser(HTMLParser): """Announces HTMLParser parse events, without doing anything else.""" def _p(self, s): print(s) def handle_starttag(self, name, attrs): self._p("%s START" % name) def handle_endtag(self, name): self._p("%s END" % name) def handle_data(self, data): self._p("%s DATA" % data) def handle_charref(self, name): self._p("%s CHARREF" % name) def handle_entityref(self, name): self._p("%s ENTITYREF" % name) def handle_comment(self, data): self._p("%s COMMENT" % data) def handle_decl(self, data): self._p("%s DECL" % data) def unknown_decl(self, data): self._p("%s UNKNOWN-DECL" % data) def handle_pi(self, data): self._p("%s PI" % data) def htmlparser_trace(data): """Print out the HTMLParser events that occur during parsing. This lets you see how HTMLParser parses a document when no Beautiful Soup code is running. """ parser = AnnouncingParser() parser.feed(data) _vowels = "aeiou" _consonants = "bcdfghjklmnpqrstvwxyz" def rword(length=5): "Generate a random word-like string." s = '' for i in range(length): if i % 2 == 0: t = _consonants else: t = _vowels s += random.choice(t) return s def rsentence(length=4): "Generate a random sentence-like string." return " ".join(rword(random.randint(4,9)) for i in range(length)) def rdoc(num_elements=1000): """Randomly generate an invalid HTML document.""" tag_names = ['p', 'div', 'span', 'i', 'b', 'script', 'table'] elements = [] for i in range(num_elements): choice = random.randint(0,3) if choice == 0: # New tag. tag_name = random.choice(tag_names) elements.append("<%s>" % tag_name) elif choice == 1: elements.append(rsentence(random.randint(1,4))) elif choice == 2: # Close a tag. tag_name = random.choice(tag_names) elements.append("</%s>" % tag_name) return "<html>" + "\n".join(elements) + "</html>" def benchmark_parsers(num_elements=100000): """Very basic head-to-head performance benchmark.""" print(("Comparative parser benchmark on Beautiful Soup %s" % __version__)) data = rdoc(num_elements) print(("Generated a large invalid HTML document (%d bytes)." % len(data))) for parser in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]: success = False try: a = time.time() soup = BeautifulSoup(data, parser) b = time.time() success = True except Exception as e: print(("%s could not parse the markup." % parser)) traceback.print_exc() if success: print(("BS4+%s parsed the markup in %.2fs." % (parser, b-a))) from lxml import etree a = time.time() etree.HTML(data) b = time.time() print(("Raw lxml parsed the markup in %.2fs." % (b-a))) import html5lib parser = html5lib.HTMLParser() a = time.time() parser.parse(data) b = time.time() print(("Raw html5lib parsed the markup in %.2fs." % (b-a))) def profile(num_elements=100000, parser="lxml"): filehandle = tempfile.NamedTemporaryFile() filename = filehandle.name data = rdoc(num_elements) vars = dict(bs4=bs4, data=data, parser=parser) cProfile.runctx('bs4.BeautifulSoup(data, parser)' , vars, vars, filename) stats = pstats.Stats(filename) # stats.strip_dirs() stats.sort_stats("cumulative") stats.print_stats('_html5lib|bs4', 50) if __name__ == '__main__': diagnose(sys.stdin.read()) dammit.py 0000644 00000072335 15246412105 0006403 0 ustar 00 # -*- coding: utf-8 -*- """Beautiful Soup bonus library: Unicode, Dammit This library converts a bytestream to Unicode through any means necessary. It is heavily based on code from Mark Pilgrim's Universal Feed Parser. It works best on XML and HTML, but it does not rewrite the XML or HTML to reflect a new encoding; that's the tree builder's job. """ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. __license__ = "MIT" import codecs from html.entities import codepoint2name import re import logging import string # Import a library to autodetect character encodings. chardet_type = None try: # First try the fast C implementation. # PyPI package: cchardet import cchardet def chardet_dammit(s): return cchardet.detect(s)['encoding'] except ImportError: try: # Fall back to the pure Python implementation # Debian package: python-chardet # PyPI package: chardet import chardet def chardet_dammit(s): return chardet.detect(s)['encoding'] #import chardet.constants #chardet.constants._debug = 1 except ImportError: # No chardet available. def chardet_dammit(s): return None # Available from http://cjkpython.i18n.org/. try: import iconv_codec except ImportError: pass xml_encoding_re = re.compile( '^<\\?.*encoding=[\'"](.*?)[\'"].*\\?>'.encode(), re.I) html_meta_re = re.compile( '<\\s*meta[^>]+charset\\s*=\\s*["\']?([^>]*?)[ /;\'">]'.encode(), re.I) class EntitySubstitution(object): """Substitute XML or HTML entities for the corresponding characters.""" def _populate_class_variables(): lookup = {} reverse_lookup = {} characters_for_re = [] for codepoint, name in list(codepoint2name.items()): character = chr(codepoint) if codepoint != 34: # There's no point in turning the quotation mark into # ", unless it happens within an attribute value, which # is handled elsewhere. characters_for_re.append(character) lookup[character] = name # But we do want to turn " into the quotation mark. reverse_lookup[name] = character re_definition = "[%s]" % "".join(characters_for_re) return lookup, reverse_lookup, re.compile(re_definition) (CHARACTER_TO_HTML_ENTITY, HTML_ENTITY_TO_CHARACTER, CHARACTER_TO_HTML_ENTITY_RE) = _populate_class_variables() CHARACTER_TO_XML_ENTITY = { "'": "apos", '"': "quot", "&": "amp", "<": "lt", ">": "gt", } BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|" "&(?!#\\d+;|#x[0-9a-fA-F]+;|\\w+;)" ")") AMPERSAND_OR_BRACKET = re.compile("([<>&])") @classmethod def _substitute_html_entity(cls, matchobj): entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0)) return "&%s;" % entity @classmethod def _substitute_xml_entity(cls, matchobj): """Used with a regular expression to substitute the appropriate XML entity for an XML special character.""" entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)] return "&%s;" % entity @classmethod def quoted_attribute_value(self, value): """Make a value into a quoted XML attribute, possibly escaping it. Most strings will be quoted using double quotes. Bob's Bar -> "Bob's Bar" If a string contains double quotes, it will be quoted using single quotes. Welcome to "my bar" -> 'Welcome to "my bar"' If a string contains both single and double quotes, the double quotes will be escaped, and the string will be quoted using double quotes. Welcome to "Bob's Bar" -> "Welcome to "Bob's bar" """ quote_with = '"' if '"' in value: if "'" in value: # The string contains both single and double # quotes. Turn the double quotes into # entities. We quote the double quotes rather than # the single quotes because the entity name is # """ whether this is HTML or XML. If we # quoted the single quotes, we'd have to decide # between ' and &squot;. replace_with = """ value = value.replace('"', replace_with) else: # There are double quotes but no single quotes. # We can use single quotes to quote the attribute. quote_with = "'" return quote_with + value + quote_with @classmethod def substitute_xml(cls, value, make_quoted_attribute=False): """Substitute XML entities for special XML characters. :param value: A string to be substituted. The less-than sign will become <, the greater-than sign will become >, and any ampersands will become &. If you want ampersands that appear to be part of an entity definition to be left alone, use substitute_xml_containing_entities() instead. :param make_quoted_attribute: If True, then the string will be quoted, as befits an attribute value. """ # Escape angle brackets and ampersands. value = cls.AMPERSAND_OR_BRACKET.sub( cls._substitute_xml_entity, value) if make_quoted_attribute: value = cls.quoted_attribute_value(value) return value @classmethod def substitute_xml_containing_entities( cls, value, make_quoted_attribute=False): """Substitute XML entities for special XML characters. :param value: A string to be substituted. The less-than sign will become <, the greater-than sign will become >, and any ampersands that are not part of an entity defition will become &. :param make_quoted_attribute: If True, then the string will be quoted, as befits an attribute value. """ # Escape angle brackets, and ampersands that aren't part of # entities. value = cls.BARE_AMPERSAND_OR_BRACKET.sub( cls._substitute_xml_entity, value) if make_quoted_attribute: value = cls.quoted_attribute_value(value) return value @classmethod def substitute_html(cls, s): """Replace certain Unicode characters with named HTML entities. This differs from data.encode(encoding, 'xmlcharrefreplace') in that the goal is to make the result more readable (to those with ASCII displays) rather than to recover from errors. There's absolutely nothing wrong with a UTF-8 string containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that character with "é" will make it more readable to some people. """ return cls.CHARACTER_TO_HTML_ENTITY_RE.sub( cls._substitute_html_entity, s) class EncodingDetector: """Suggests a number of possible encodings for a bytestring. Order of precedence: 1. Encodings you specifically tell EncodingDetector to try first (the override_encodings argument to the constructor). 2. An encoding declared within the bytestring itself, either in an XML declaration (if the bytestring is to be interpreted as an XML document), or in a <meta> tag (if the bytestring is to be interpreted as an HTML document.) 3. An encoding detected through textual analysis by chardet, cchardet, or a similar external library. 4. UTF-8. 5. Windows-1252. """ def __init__(self, markup, override_encodings=None, is_html=False, exclude_encodings=None): self.override_encodings = override_encodings or [] exclude_encodings = exclude_encodings or [] self.exclude_encodings = set([x.lower() for x in exclude_encodings]) self.chardet_encoding = None self.is_html = is_html self.declared_encoding = None # First order of business: strip a byte-order mark. self.markup, self.sniffed_encoding = self.strip_byte_order_mark(markup) def _usable(self, encoding, tried): if encoding is not None: encoding = encoding.lower() if encoding in self.exclude_encodings: return False if encoding not in tried: tried.add(encoding) return True return False @property def encodings(self): """Yield a number of encodings that might work for this markup.""" tried = set() for e in self.override_encodings: if self._usable(e, tried): yield e # Did the document originally start with a byte-order mark # that indicated its encoding? if self._usable(self.sniffed_encoding, tried): yield self.sniffed_encoding # Look within the document for an XML or HTML encoding # declaration. if self.declared_encoding is None: self.declared_encoding = self.find_declared_encoding( self.markup, self.is_html) if self._usable(self.declared_encoding, tried): yield self.declared_encoding # Use third-party character set detection to guess at the # encoding. if self.chardet_encoding is None: self.chardet_encoding = chardet_dammit(self.markup) if self._usable(self.chardet_encoding, tried): yield self.chardet_encoding # As a last-ditch effort, try utf-8 and windows-1252. for e in ('utf-8', 'windows-1252'): if self._usable(e, tried): yield e @classmethod def strip_byte_order_mark(cls, data): """If a byte-order mark is present, strip it and return the encoding it implies.""" encoding = None if isinstance(data, str): # Unicode data cannot have a byte-order mark. return data, encoding if (len(data) >= 4) and (data[:2] == b'\xfe\xff') \ and (data[2:4] != '\x00\x00'): encoding = 'utf-16be' data = data[2:] elif (len(data) >= 4) and (data[:2] == b'\xff\xfe') \ and (data[2:4] != '\x00\x00'): encoding = 'utf-16le' data = data[2:] elif data[:3] == b'\xef\xbb\xbf': encoding = 'utf-8' data = data[3:] elif data[:4] == b'\x00\x00\xfe\xff': encoding = 'utf-32be' data = data[4:] elif data[:4] == b'\xff\xfe\x00\x00': encoding = 'utf-32le' data = data[4:] return data, encoding @classmethod def find_declared_encoding(cls, markup, is_html=False, search_entire_document=False): """Given a document, tries to find its declared encoding. An XML encoding is declared at the beginning of the document. An HTML encoding is declared in a <meta> tag, hopefully near the beginning of the document. """ if search_entire_document: xml_endpos = html_endpos = len(markup) else: xml_endpos = 1024 html_endpos = max(2048, int(len(markup) * 0.05)) declared_encoding = None declared_encoding_match = xml_encoding_re.search(markup, endpos=xml_endpos) if not declared_encoding_match and is_html: declared_encoding_match = html_meta_re.search(markup, endpos=html_endpos) if declared_encoding_match is not None: declared_encoding = declared_encoding_match.groups()[0].decode( 'ascii', 'replace') if declared_encoding: return declared_encoding.lower() return None class UnicodeDammit: """A class for detecting the encoding of a *ML document and converting it to a Unicode string. If the source encoding is windows-1252, can replace MS smart quotes with their HTML or XML equivalents.""" # This dictionary maps commonly seen values for "charset" in HTML # meta tags to the corresponding Python codec names. It only covers # values that aren't in Python's aliases and can't be determined # by the heuristics in find_codec. CHARSET_ALIASES = {"macintosh": "mac-roman", "x-sjis": "shift-jis"} ENCODINGS_WITH_SMART_QUOTES = [ "windows-1252", "iso-8859-1", "iso-8859-2", ] def __init__(self, markup, override_encodings=[], smart_quotes_to=None, is_html=False, exclude_encodings=[]): self.smart_quotes_to = smart_quotes_to self.tried_encodings = [] self.contains_replacement_characters = False self.is_html = is_html self.log = logging.getLogger(__name__) self.detector = EncodingDetector( markup, override_encodings, is_html, exclude_encodings) # Short-circuit if the data is in Unicode to begin with. if isinstance(markup, str) or markup == '': self.markup = markup self.unicode_markup = str(markup) self.original_encoding = None return # The encoding detector may have stripped a byte-order mark. # Use the stripped markup from this point on. self.markup = self.detector.markup u = None for encoding in self.detector.encodings: markup = self.detector.markup u = self._convert_from(encoding) if u is not None: break if not u: # None of the encodings worked. As an absolute last resort, # try them again with character replacement. for encoding in self.detector.encodings: if encoding != "ascii": u = self._convert_from(encoding, "replace") if u is not None: self.log.warning( "Some characters could not be decoded, and were " "replaced with REPLACEMENT CHARACTER." ) self.contains_replacement_characters = True break # If none of that worked, we could at this point force it to # ASCII, but that would destroy so much data that I think # giving up is better. self.unicode_markup = u if not u: self.original_encoding = None def _sub_ms_char(self, match): """Changes a MS smart quote character to an XML or HTML entity, or an ASCII character.""" orig = match.group(1) if self.smart_quotes_to == 'ascii': sub = self.MS_CHARS_TO_ASCII.get(orig).encode() else: sub = self.MS_CHARS.get(orig) if type(sub) == tuple: if self.smart_quotes_to == 'xml': sub = '&#x'.encode() + sub[1].encode() + ';'.encode() else: sub = '&'.encode() + sub[0].encode() + ';'.encode() else: sub = sub.encode() return sub def _convert_from(self, proposed, errors="strict"): proposed = self.find_codec(proposed) if not proposed or (proposed, errors) in self.tried_encodings: return None self.tried_encodings.append((proposed, errors)) markup = self.markup # Convert smart quotes to HTML if coming from an encoding # that might have them. if (self.smart_quotes_to is not None and proposed in self.ENCODINGS_WITH_SMART_QUOTES): smart_quotes_re = b"([\x80-\x9f])" smart_quotes_compiled = re.compile(smart_quotes_re) markup = smart_quotes_compiled.sub(self._sub_ms_char, markup) try: #print "Trying to convert document to %s (errors=%s)" % ( # proposed, errors) u = self._to_unicode(markup, proposed, errors) self.markup = u self.original_encoding = proposed except Exception as e: #print "That didn't work!" #print e return None #print "Correct encoding: %s" % proposed return self.markup def _to_unicode(self, data, encoding, errors="strict"): '''Given a string and its encoding, decodes the string into Unicode. %encoding is a string recognized by encodings.aliases''' return str(data, encoding, errors) @property def declared_html_encoding(self): if not self.is_html: return None return self.detector.declared_encoding def find_codec(self, charset): value = (self._codec(self.CHARSET_ALIASES.get(charset, charset)) or (charset and self._codec(charset.replace("-", ""))) or (charset and self._codec(charset.replace("-", "_"))) or (charset and charset.lower()) or charset ) if value: return value.lower() return None def _codec(self, charset): if not charset: return charset codec = None try: codecs.lookup(charset) codec = charset except (LookupError, ValueError): pass return codec # A partial mapping of ISO-Latin-1 to HTML entities/XML numeric entities. MS_CHARS = {b'\x80': ('euro', '20AC'), b'\x81': ' ', b'\x82': ('sbquo', '201A'), b'\x83': ('fnof', '192'), b'\x84': ('bdquo', '201E'), b'\x85': ('hellip', '2026'), b'\x86': ('dagger', '2020'), b'\x87': ('Dagger', '2021'), b'\x88': ('circ', '2C6'), b'\x89': ('permil', '2030'), b'\x8A': ('Scaron', '160'), b'\x8B': ('lsaquo', '2039'), b'\x8C': ('OElig', '152'), b'\x8D': '?', b'\x8E': ('#x17D', '17D'), b'\x8F': '?', b'\x90': '?', b'\x91': ('lsquo', '2018'), b'\x92': ('rsquo', '2019'), b'\x93': ('ldquo', '201C'), b'\x94': ('rdquo', '201D'), b'\x95': ('bull', '2022'), b'\x96': ('ndash', '2013'), b'\x97': ('mdash', '2014'), b'\x98': ('tilde', '2DC'), b'\x99': ('trade', '2122'), b'\x9a': ('scaron', '161'), b'\x9b': ('rsaquo', '203A'), b'\x9c': ('oelig', '153'), b'\x9d': '?', b'\x9e': ('#x17E', '17E'), b'\x9f': ('Yuml', ''),} # A parochial partial mapping of ISO-Latin-1 to ASCII. Contains # horrors like stripping diacritical marks to turn á into a, but also # contains non-horrors like turning “ into ". MS_CHARS_TO_ASCII = { b'\x80' : 'EUR', b'\x81' : ' ', b'\x82' : ',', b'\x83' : 'f', b'\x84' : ',,', b'\x85' : '...', b'\x86' : '+', b'\x87' : '++', b'\x88' : '^', b'\x89' : '%', b'\x8a' : 'S', b'\x8b' : '<', b'\x8c' : 'OE', b'\x8d' : '?', b'\x8e' : 'Z', b'\x8f' : '?', b'\x90' : '?', b'\x91' : "'", b'\x92' : "'", b'\x93' : '"', b'\x94' : '"', b'\x95' : '*', b'\x96' : '-', b'\x97' : '--', b'\x98' : '~', b'\x99' : '(TM)', b'\x9a' : 's', b'\x9b' : '>', b'\x9c' : 'oe', b'\x9d' : '?', b'\x9e' : 'z', b'\x9f' : 'Y', b'\xa0' : ' ', b'\xa1' : '!', b'\xa2' : 'c', b'\xa3' : 'GBP', b'\xa4' : '$', #This approximation is especially parochial--this is the #generic currency symbol. b'\xa5' : 'YEN', b'\xa6' : '|', b'\xa7' : 'S', b'\xa8' : '..', b'\xa9' : '', b'\xaa' : '(th)', b'\xab' : '<<', b'\xac' : '!', b'\xad' : ' ', b'\xae' : '(R)', b'\xaf' : '-', b'\xb0' : 'o', b'\xb1' : '+-', b'\xb2' : '2', b'\xb3' : '3', b'\xb4' : ("'", 'acute'), b'\xb5' : 'u', b'\xb6' : 'P', b'\xb7' : '*', b'\xb8' : ',', b'\xb9' : '1', b'\xba' : '(th)', b'\xbb' : '>>', b'\xbc' : '1/4', b'\xbd' : '1/2', b'\xbe' : '3/4', b'\xbf' : '?', b'\xc0' : 'A', b'\xc1' : 'A', b'\xc2' : 'A', b'\xc3' : 'A', b'\xc4' : 'A', b'\xc5' : 'A', b'\xc6' : 'AE', b'\xc7' : 'C', b'\xc8' : 'E', b'\xc9' : 'E', b'\xca' : 'E', b'\xcb' : 'E', b'\xcc' : 'I', b'\xcd' : 'I', b'\xce' : 'I', b'\xcf' : 'I', b'\xd0' : 'D', b'\xd1' : 'N', b'\xd2' : 'O', b'\xd3' : 'O', b'\xd4' : 'O', b'\xd5' : 'O', b'\xd6' : 'O', b'\xd7' : '*', b'\xd8' : 'O', b'\xd9' : 'U', b'\xda' : 'U', b'\xdb' : 'U', b'\xdc' : 'U', b'\xdd' : 'Y', b'\xde' : 'b', b'\xdf' : 'B', b'\xe0' : 'a', b'\xe1' : 'a', b'\xe2' : 'a', b'\xe3' : 'a', b'\xe4' : 'a', b'\xe5' : 'a', b'\xe6' : 'ae', b'\xe7' : 'c', b'\xe8' : 'e', b'\xe9' : 'e', b'\xea' : 'e', b'\xeb' : 'e', b'\xec' : 'i', b'\xed' : 'i', b'\xee' : 'i', b'\xef' : 'i', b'\xf0' : 'o', b'\xf1' : 'n', b'\xf2' : 'o', b'\xf3' : 'o', b'\xf4' : 'o', b'\xf5' : 'o', b'\xf6' : 'o', b'\xf7' : '/', b'\xf8' : 'o', b'\xf9' : 'u', b'\xfa' : 'u', b'\xfb' : 'u', b'\xfc' : 'u', b'\xfd' : 'y', b'\xfe' : 'b', b'\xff' : 'y', } # A map used when removing rogue Windows-1252/ISO-8859-1 # characters in otherwise UTF-8 documents. # # Note that \x81, \x8d, \x8f, \x90, and \x9d are undefined in # Windows-1252. WINDOWS_1252_TO_UTF8 = { 0x80 : b'\xe2\x82\xac', # € 0x82 : b'\xe2\x80\x9a', # ‚ 0x83 : b'\xc6\x92', # ƒ 0x84 : b'\xe2\x80\x9e', # „ 0x85 : b'\xe2\x80\xa6', # … 0x86 : b'\xe2\x80\xa0', # † 0x87 : b'\xe2\x80\xa1', # ‡ 0x88 : b'\xcb\x86', # ˆ 0x89 : b'\xe2\x80\xb0', # ‰ 0x8a : b'\xc5\xa0', # Š 0x8b : b'\xe2\x80\xb9', # ‹ 0x8c : b'\xc5\x92', # Œ 0x8e : b'\xc5\xbd', # Ž 0x91 : b'\xe2\x80\x98', # ‘ 0x92 : b'\xe2\x80\x99', # ’ 0x93 : b'\xe2\x80\x9c', # “ 0x94 : b'\xe2\x80\x9d', # ” 0x95 : b'\xe2\x80\xa2', # • 0x96 : b'\xe2\x80\x93', # – 0x97 : b'\xe2\x80\x94', # — 0x98 : b'\xcb\x9c', # ˜ 0x99 : b'\xe2\x84\xa2', # ™ 0x9a : b'\xc5\xa1', # š 0x9b : b'\xe2\x80\xba', # › 0x9c : b'\xc5\x93', # œ 0x9e : b'\xc5\xbe', # ž 0x9f : b'\xc5\xb8', # Ÿ 0xa0 : b'\xc2\xa0', # 0xa1 : b'\xc2\xa1', # ¡ 0xa2 : b'\xc2\xa2', # ¢ 0xa3 : b'\xc2\xa3', # £ 0xa4 : b'\xc2\xa4', # ¤ 0xa5 : b'\xc2\xa5', # ¥ 0xa6 : b'\xc2\xa6', # ¦ 0xa7 : b'\xc2\xa7', # § 0xa8 : b'\xc2\xa8', # ¨ 0xa9 : b'\xc2\xa9', # © 0xaa : b'\xc2\xaa', # ª 0xab : b'\xc2\xab', # « 0xac : b'\xc2\xac', # ¬ 0xad : b'\xc2\xad', # 0xae : b'\xc2\xae', # ® 0xaf : b'\xc2\xaf', # ¯ 0xb0 : b'\xc2\xb0', # ° 0xb1 : b'\xc2\xb1', # ± 0xb2 : b'\xc2\xb2', # ² 0xb3 : b'\xc2\xb3', # ³ 0xb4 : b'\xc2\xb4', # ´ 0xb5 : b'\xc2\xb5', # µ 0xb6 : b'\xc2\xb6', # ¶ 0xb7 : b'\xc2\xb7', # · 0xb8 : b'\xc2\xb8', # ¸ 0xb9 : b'\xc2\xb9', # ¹ 0xba : b'\xc2\xba', # º 0xbb : b'\xc2\xbb', # » 0xbc : b'\xc2\xbc', # ¼ 0xbd : b'\xc2\xbd', # ½ 0xbe : b'\xc2\xbe', # ¾ 0xbf : b'\xc2\xbf', # ¿ 0xc0 : b'\xc3\x80', # À 0xc1 : b'\xc3\x81', # Á 0xc2 : b'\xc3\x82', #  0xc3 : b'\xc3\x83', # à 0xc4 : b'\xc3\x84', # Ä 0xc5 : b'\xc3\x85', # Å 0xc6 : b'\xc3\x86', # Æ 0xc7 : b'\xc3\x87', # Ç 0xc8 : b'\xc3\x88', # È 0xc9 : b'\xc3\x89', # É 0xca : b'\xc3\x8a', # Ê 0xcb : b'\xc3\x8b', # Ë 0xcc : b'\xc3\x8c', # Ì 0xcd : b'\xc3\x8d', # Í 0xce : b'\xc3\x8e', # Î 0xcf : b'\xc3\x8f', # Ï 0xd0 : b'\xc3\x90', # Ð 0xd1 : b'\xc3\x91', # Ñ 0xd2 : b'\xc3\x92', # Ò 0xd3 : b'\xc3\x93', # Ó 0xd4 : b'\xc3\x94', # Ô 0xd5 : b'\xc3\x95', # Õ 0xd6 : b'\xc3\x96', # Ö 0xd7 : b'\xc3\x97', # × 0xd8 : b'\xc3\x98', # Ø 0xd9 : b'\xc3\x99', # Ù 0xda : b'\xc3\x9a', # Ú 0xdb : b'\xc3\x9b', # Û 0xdc : b'\xc3\x9c', # Ü 0xdd : b'\xc3\x9d', # Ý 0xde : b'\xc3\x9e', # Þ 0xdf : b'\xc3\x9f', # ß 0xe0 : b'\xc3\xa0', # à 0xe1 : b'\xa1', # á 0xe2 : b'\xc3\xa2', # â 0xe3 : b'\xc3\xa3', # ã 0xe4 : b'\xc3\xa4', # ä 0xe5 : b'\xc3\xa5', # å 0xe6 : b'\xc3\xa6', # æ 0xe7 : b'\xc3\xa7', # ç 0xe8 : b'\xc3\xa8', # è 0xe9 : b'\xc3\xa9', # é 0xea : b'\xc3\xaa', # ê 0xeb : b'\xc3\xab', # ë 0xec : b'\xc3\xac', # ì 0xed : b'\xc3\xad', # í 0xee : b'\xc3\xae', # î 0xef : b'\xc3\xaf', # ï 0xf0 : b'\xc3\xb0', # ð 0xf1 : b'\xc3\xb1', # ñ 0xf2 : b'\xc3\xb2', # ò 0xf3 : b'\xc3\xb3', # ó 0xf4 : b'\xc3\xb4', # ô 0xf5 : b'\xc3\xb5', # õ 0xf6 : b'\xc3\xb6', # ö 0xf7 : b'\xc3\xb7', # ÷ 0xf8 : b'\xc3\xb8', # ø 0xf9 : b'\xc3\xb9', # ù 0xfa : b'\xc3\xba', # ú 0xfb : b'\xc3\xbb', # û 0xfc : b'\xc3\xbc', # ü 0xfd : b'\xc3\xbd', # ý 0xfe : b'\xc3\xbe', # þ } MULTIBYTE_MARKERS_AND_SIZES = [ (0xc2, 0xdf, 2), # 2-byte characters start with a byte C2-DF (0xe0, 0xef, 3), # 3-byte characters start with E0-EF (0xf0, 0xf4, 4), # 4-byte characters start with F0-F4 ] FIRST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[0][0] LAST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[-1][1] @classmethod def detwingle(cls, in_bytes, main_encoding="utf8", embedded_encoding="windows-1252"): """Fix characters from one encoding embedded in some other encoding. Currently the only situation supported is Windows-1252 (or its subset ISO-8859-1), embedded in UTF-8. The input must be a bytestring. If you've already converted the document to Unicode, you're too late. The output is a bytestring in which `embedded_encoding` characters have been converted to their `main_encoding` equivalents. """ if embedded_encoding.replace('_', '-').lower() not in ( 'windows-1252', 'windows_1252'): raise NotImplementedError( "Windows-1252 and ISO-8859-1 are the only currently supported " "embedded encodings.") if main_encoding.lower() not in ('utf8', 'utf-8'): raise NotImplementedError( "UTF-8 is the only currently supported main encoding.") byte_chunks = [] chunk_start = 0 pos = 0 while pos < len(in_bytes): byte = in_bytes[pos] if not isinstance(byte, int): # Python 2.x byte = ord(byte) if (byte >= cls.FIRST_MULTIBYTE_MARKER and byte <= cls.LAST_MULTIBYTE_MARKER): # This is the start of a UTF-8 multibyte character. Skip # to the end. for start, end, size in cls.MULTIBYTE_MARKERS_AND_SIZES: if byte >= start and byte <= end: pos += size break elif byte >= 0x80 and byte in cls.WINDOWS_1252_TO_UTF8: # We found a Windows-1252 character! # Save the string up to this point as a chunk. byte_chunks.append(in_bytes[chunk_start:pos]) # Now translate the Windows-1252 character into UTF-8 # and add it as another, one-byte chunk. byte_chunks.append(cls.WINDOWS_1252_TO_UTF8[byte]) pos += 1 chunk_start = pos else: # Go on to the next character. pos += 1 if chunk_start == 0: # The string is unchanged. return in_bytes else: # Store the final chunk. byte_chunks.append(in_bytes[chunk_start:]) return b''.join(byte_chunks) element.py 0000644 00000215044 15246412105 0006555 0 ustar 00 # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. __license__ = "MIT" try: from collections.abc import Callable # Python 3.6 except ImportError as e: from collections import Callable import re import shlex import sys import warnings from bs4.dammit import EntitySubstitution DEFAULT_OUTPUT_ENCODING = "utf-8" PY3K = (sys.version_info[0] > 2) whitespace_re = re.compile(r"\s+") def _alias(attr): """Alias one attribute name to another for backward compatibility""" @property def alias(self): return getattr(self, attr) @alias.setter def alias(self): return setattr(self, attr) return alias class NamespacedAttribute(str): def __new__(cls, prefix, name, namespace=None): if name is None: obj = str.__new__(cls, prefix) elif prefix is None: # Not really namespaced. obj = str.__new__(cls, name) else: obj = str.__new__(cls, prefix + ":" + name) obj.prefix = prefix obj.name = name obj.namespace = namespace return obj class AttributeValueWithCharsetSubstitution(str): """A stand-in object for a character encoding specified in HTML.""" class CharsetMetaAttributeValue(AttributeValueWithCharsetSubstitution): """A generic stand-in for the value of a meta tag's 'charset' attribute. When Beautiful Soup parses the markup '<meta charset="utf8">', the value of the 'charset' attribute will be one of these objects. """ def __new__(cls, original_value): obj = str.__new__(cls, original_value) obj.original_value = original_value return obj def encode(self, encoding): return encoding class ContentMetaAttributeValue(AttributeValueWithCharsetSubstitution): """A generic stand-in for the value of a meta tag's 'content' attribute. When Beautiful Soup parses the markup: <meta http-equiv="content-type" content="text/html; charset=utf8"> The value of the 'content' attribute will be one of these objects. """ CHARSET_RE = re.compile(r"((^|;)\s*charset=)([^;]*)", re.M) def __new__(cls, original_value): match = cls.CHARSET_RE.search(original_value) if match is None: # No substitution necessary. return str.__new__(str, original_value) obj = str.__new__(cls, original_value) obj.original_value = original_value return obj def encode(self, encoding): def rewrite(match): return match.group(1) + encoding return self.CHARSET_RE.sub(rewrite, self.original_value) class HTMLAwareEntitySubstitution(EntitySubstitution): """Entity substitution rules that are aware of some HTML quirks. Specifically, the contents of <script> and <style> tags should not undergo entity substitution. Incoming NavigableString objects are checked to see if they're the direct children of a <script> or <style> tag. """ cdata_containing_tags = set(["script", "style"]) preformatted_tags = set(["pre"]) preserve_whitespace_tags = set(['pre', 'textarea']) @classmethod def _substitute_if_appropriate(cls, ns, f): if (isinstance(ns, NavigableString) and ns.parent is not None and ns.parent.name in cls.cdata_containing_tags): # Do nothing. return ns # Substitute. return f(ns) @classmethod def substitute_html(cls, ns): return cls._substitute_if_appropriate( ns, EntitySubstitution.substitute_html) @classmethod def substitute_xml(cls, ns): return cls._substitute_if_appropriate( ns, EntitySubstitution.substitute_xml) class Formatter(object): """Contains information about how to format a parse tree.""" # By default, represent void elements as <tag/> rather than <tag> void_element_close_prefix = '/' def substitute_entities(self, *args, **kwargs): """Transform certain characters into named entities.""" raise NotImplementedError() class HTMLFormatter(Formatter): """The default HTML formatter.""" def substitute(self, *args, **kwargs): return HTMLAwareEntitySubstitution.substitute_html(*args, **kwargs) class MinimalHTMLFormatter(Formatter): """A minimal HTML formatter.""" def substitute(self, *args, **kwargs): return HTMLAwareEntitySubstitution.substitute_xml(*args, **kwargs) class HTML5Formatter(HTMLFormatter): """An HTML formatter that omits the slash in a void tag.""" void_element_close_prefix = None class XMLFormatter(Formatter): """Substitute only the essential XML entities.""" def substitute(self, *args, **kwargs): return EntitySubstitution.substitute_xml(*args, **kwargs) class HTMLXMLFormatter(Formatter): """Format XML using HTML rules.""" def substitute(self, *args, **kwargs): return HTMLAwareEntitySubstitution.substitute_html(*args, **kwargs) class PageElement(object): """Contains the navigational information for some part of the page (either a tag or a piece of text)""" # There are five possible values for the "formatter" argument passed in # to methods like encode() and prettify(): # # "html" - All Unicode characters with corresponding HTML entities # are converted to those entities on output. # "html5" - The same as "html", but empty void tags are represented as # <tag> rather than <tag/> # "minimal" - Bare ampersands and angle brackets are converted to # XML entities: & < > # None - The null formatter. Unicode characters are never # converted to entities. This is not recommended, but it's # faster than "minimal". # A callable function - it will be called on every string that needs to undergo entity substitution. # A Formatter instance - Formatter.substitute(string) will be called on every string that # needs to undergo entity substitution. # # In an HTML document, the default "html", "html5", and "minimal" # functions will leave the contents of <script> and <style> tags # alone. For an XML document, all tags will be given the same # treatment. HTML_FORMATTERS = { "html" : HTMLFormatter(), "html5" : HTML5Formatter(), "minimal" : MinimalHTMLFormatter(), None : None } XML_FORMATTERS = { "html" : HTMLXMLFormatter(), "minimal" : XMLFormatter(), None : None } def format_string(self, s, formatter='minimal'): """Format the given string using the given formatter.""" if isinstance(formatter, str): formatter = self._formatter_for_name(formatter) if formatter is None: output = s else: if callable(formatter): # Backwards compatibility -- you used to pass in a formatting method. output = formatter(s) else: output = formatter.substitute(s) return output @property def _is_xml(self): """Is this element part of an XML tree or an HTML tree? This is used when mapping a formatter name ("minimal") to an appropriate function (one that performs entity-substitution on the contents of <script> and <style> tags, or not). It can be inefficient, but it should be called very rarely. """ if self.known_xml is not None: # Most of the time we will have determined this when the # document is parsed. return self.known_xml # Otherwise, it's likely that this element was created by # direct invocation of the constructor from within the user's # Python code. if self.parent is None: # This is the top-level object. It should have .known_xml set # from tree creation. If not, take a guess--BS is usually # used on HTML markup. return getattr(self, 'is_xml', False) return self.parent._is_xml def _formatter_for_name(self, name): "Look up a formatter function based on its name and the tree." if self._is_xml: return self.XML_FORMATTERS.get(name, XMLFormatter()) else: return self.HTML_FORMATTERS.get(name, HTMLFormatter()) def setup(self, parent=None, previous_element=None, next_element=None, previous_sibling=None, next_sibling=None): """Sets up the initial relations between this element and other elements.""" self.parent = parent self.previous_element = previous_element if previous_element is not None: self.previous_element.next_element = self self.next_element = next_element if self.next_element: self.next_element.previous_element = self self.next_sibling = next_sibling if self.next_sibling: self.next_sibling.previous_sibling = self if (not previous_sibling and self.parent is not None and self.parent.contents): previous_sibling = self.parent.contents[-1] self.previous_sibling = previous_sibling if previous_sibling: self.previous_sibling.next_sibling = self nextSibling = _alias("next_sibling") # BS3 previousSibling = _alias("previous_sibling") # BS3 def replace_with(self, replace_with): if not self.parent: raise ValueError( "Cannot replace one element with another when the" "element to be replaced is not part of a tree.") if replace_with is self: return if replace_with is self.parent: raise ValueError("Cannot replace a Tag with its parent.") old_parent = self.parent my_index = self.parent.index(self) self.extract() old_parent.insert(my_index, replace_with) return self replaceWith = replace_with # BS3 def unwrap(self): my_parent = self.parent if not self.parent: raise ValueError( "Cannot replace an element with its contents when that" "element is not part of a tree.") my_index = self.parent.index(self) self.extract() for child in reversed(self.contents[:]): my_parent.insert(my_index, child) return self replace_with_children = unwrap replaceWithChildren = unwrap # BS3 def wrap(self, wrap_inside): me = self.replace_with(wrap_inside) wrap_inside.append(me) return wrap_inside def extract(self): """Destructively rips this element out of the tree.""" if self.parent is not None: del self.parent.contents[self.parent.index(self)] #Find the two elements that would be next to each other if #this element (and any children) hadn't been parsed. Connect #the two. last_child = self._last_descendant() next_element = last_child.next_element if (self.previous_element is not None and self.previous_element is not next_element): self.previous_element.next_element = next_element if next_element is not None and next_element is not self.previous_element: next_element.previous_element = self.previous_element self.previous_element = None last_child.next_element = None self.parent = None if (self.previous_sibling is not None and self.previous_sibling is not self.next_sibling): self.previous_sibling.next_sibling = self.next_sibling if (self.next_sibling is not None and self.next_sibling is not self.previous_sibling): self.next_sibling.previous_sibling = self.previous_sibling self.previous_sibling = self.next_sibling = None return self def _last_descendant(self, is_initialized=True, accept_self=True): "Finds the last element beneath this object to be parsed." if is_initialized and self.next_sibling: last_child = self.next_sibling.previous_element else: last_child = self while isinstance(last_child, Tag) and last_child.contents: last_child = last_child.contents[-1] if not accept_self and last_child is self: last_child = None return last_child # BS3: Not part of the API! _lastRecursiveChild = _last_descendant def insert(self, position, new_child): if new_child is None: raise ValueError("Cannot insert None into a tag.") if new_child is self: raise ValueError("Cannot insert a tag into itself.") if (isinstance(new_child, str) and not isinstance(new_child, NavigableString)): new_child = NavigableString(new_child) from bs4 import BeautifulSoup if isinstance(new_child, BeautifulSoup): # We don't want to end up with a situation where one BeautifulSoup # object contains another. Insert the children one at a time. for subchild in list(new_child.contents): self.insert(position, subchild) position += 1 return position = min(position, len(self.contents)) if hasattr(new_child, 'parent') and new_child.parent is not None: # We're 'inserting' an element that's already one # of this object's children. if new_child.parent is self: current_index = self.index(new_child) if current_index < position: # We're moving this element further down the list # of this object's children. That means that when # we extract this element, our target index will # jump down one. position -= 1 new_child.extract() new_child.parent = self previous_child = None if position == 0: new_child.previous_sibling = None new_child.previous_element = self else: previous_child = self.contents[position - 1] new_child.previous_sibling = previous_child new_child.previous_sibling.next_sibling = new_child new_child.previous_element = previous_child._last_descendant(False) if new_child.previous_element is not None: new_child.previous_element.next_element = new_child new_childs_last_element = new_child._last_descendant(False) if position >= len(self.contents): new_child.next_sibling = None parent = self parents_next_sibling = None while parents_next_sibling is None and parent is not None: parents_next_sibling = parent.next_sibling parent = parent.parent if parents_next_sibling is not None: # We found the element that comes next in the document. break if parents_next_sibling is not None: new_childs_last_element.next_element = parents_next_sibling else: # The last element of this tag is the last element in # the document. new_childs_last_element.next_element = None else: next_child = self.contents[position] new_child.next_sibling = next_child if new_child.next_sibling is not None: new_child.next_sibling.previous_sibling = new_child new_childs_last_element.next_element = next_child if new_childs_last_element.next_element is not None: new_childs_last_element.next_element.previous_element = new_childs_last_element self.contents.insert(position, new_child) def append(self, tag): """Appends the given tag to the contents of this tag.""" self.insert(len(self.contents), tag) def insert_before(self, predecessor): """Makes the given element the immediate predecessor of this one. The two elements will have the same parent, and the given element will be immediately before this one. """ if self is predecessor: raise ValueError("Can't insert an element before itself.") parent = self.parent if parent is None: raise ValueError( "Element has no parent, so 'before' has no meaning.") # Extract first so that the index won't be screwed up if they # are siblings. if isinstance(predecessor, PageElement): predecessor.extract() index = parent.index(self) parent.insert(index, predecessor) def insert_after(self, successor): """Makes the given element the immediate successor of this one. The two elements will have the same parent, and the given element will be immediately after this one. """ if self is successor: raise ValueError("Can't insert an element after itself.") parent = self.parent if parent is None: raise ValueError( "Element has no parent, so 'after' has no meaning.") # Extract first so that the index won't be screwed up if they # are siblings. if isinstance(successor, PageElement): successor.extract() index = parent.index(self) parent.insert(index+1, successor) def find_next(self, name=None, attrs={}, text=None, **kwargs): """Returns the first item that matches the given criteria and appears after this Tag in the document.""" return self._find_one(self.find_all_next, name, attrs, text, **kwargs) findNext = find_next # BS3 def find_all_next(self, name=None, attrs={}, text=None, limit=None, **kwargs): """Returns all items that match the given criteria and appear after this Tag in the document.""" return self._find_all(name, attrs, text, limit, self.next_elements, **kwargs) findAllNext = find_all_next # BS3 def find_next_sibling(self, name=None, attrs={}, text=None, **kwargs): """Returns the closest sibling to this Tag that matches the given criteria and appears after this Tag in the document.""" return self._find_one(self.find_next_siblings, name, attrs, text, **kwargs) findNextSibling = find_next_sibling # BS3 def find_next_siblings(self, name=None, attrs={}, text=None, limit=None, **kwargs): """Returns the siblings of this Tag that match the given criteria and appear after this Tag in the document.""" return self._find_all(name, attrs, text, limit, self.next_siblings, **kwargs) findNextSiblings = find_next_siblings # BS3 fetchNextSiblings = find_next_siblings # BS2 def find_previous(self, name=None, attrs={}, text=None, **kwargs): """Returns the first item that matches the given criteria and appears before this Tag in the document.""" return self._find_one( self.find_all_previous, name, attrs, text, **kwargs) findPrevious = find_previous # BS3 def find_all_previous(self, name=None, attrs={}, text=None, limit=None, **kwargs): """Returns all items that match the given criteria and appear before this Tag in the document.""" return self._find_all(name, attrs, text, limit, self.previous_elements, **kwargs) findAllPrevious = find_all_previous # BS3 fetchPrevious = find_all_previous # BS2 def find_previous_sibling(self, name=None, attrs={}, text=None, **kwargs): """Returns the closest sibling to this Tag that matches the given criteria and appears before this Tag in the document.""" return self._find_one(self.find_previous_siblings, name, attrs, text, **kwargs) findPreviousSibling = find_previous_sibling # BS3 def find_previous_siblings(self, name=None, attrs={}, text=None, limit=None, **kwargs): """Returns the siblings of this Tag that match the given criteria and appear before this Tag in the document.""" return self._find_all(name, attrs, text, limit, self.previous_siblings, **kwargs) findPreviousSiblings = find_previous_siblings # BS3 fetchPreviousSiblings = find_previous_siblings # BS2 def find_parent(self, name=None, attrs={}, **kwargs): """Returns the closest parent of this Tag that matches the given criteria.""" # NOTE: We can't use _find_one because findParents takes a different # set of arguments. r = None l = self.find_parents(name, attrs, 1, **kwargs) if l: r = l[0] return r findParent = find_parent # BS3 def find_parents(self, name=None, attrs={}, limit=None, **kwargs): """Returns the parents of this Tag that match the given criteria.""" return self._find_all(name, attrs, None, limit, self.parents, **kwargs) findParents = find_parents # BS3 fetchParents = find_parents # BS2 @property def next(self): return self.next_element @property def previous(self): return self.previous_element #These methods do the real heavy lifting. def _find_one(self, method, name, attrs, text, **kwargs): r = None l = method(name, attrs, text, 1, **kwargs) if l: r = l[0] return r def _find_all(self, name, attrs, text, limit, generator, **kwargs): "Iterates over a generator looking for things that match." if text is None and 'string' in kwargs: text = kwargs['string'] del kwargs['string'] if isinstance(name, SoupStrainer): strainer = name else: strainer = SoupStrainer(name, attrs, text, **kwargs) if text is None and not limit and not attrs and not kwargs: if name is True or name is None: # Optimization to find all tags. result = (element for element in generator if isinstance(element, Tag)) return ResultSet(strainer, result) elif isinstance(name, str): # Optimization to find all tags with a given name. if name.count(':') == 1: # This is a name with a prefix. If this is a namespace-aware document, # we need to match the local name against tag.name. If not, # we need to match the fully-qualified name against tag.name. prefix, local_name = name.split(':', 1) else: prefix = None local_name = name result = (element for element in generator if isinstance(element, Tag) and ( element.name == name ) or ( element.name == local_name and (prefix is None or element.prefix == prefix) ) ) return ResultSet(strainer, result) results = ResultSet(strainer) while True: try: i = next(generator) except StopIteration: break if i: found = strainer.search(i) if found: results.append(found) if limit and len(results) >= limit: break return results #These generators can be used to navigate starting from both #NavigableStrings and Tags. @property def next_elements(self): i = self.next_element while i is not None: yield i i = i.next_element @property def next_siblings(self): i = self.next_sibling while i is not None: yield i i = i.next_sibling @property def previous_elements(self): i = self.previous_element while i is not None: yield i i = i.previous_element @property def previous_siblings(self): i = self.previous_sibling while i is not None: yield i i = i.previous_sibling @property def parents(self): i = self.parent while i is not None: yield i i = i.parent # Methods for supporting CSS selectors. tag_name_re = re.compile('^[a-zA-Z0-9][-.a-zA-Z0-9:_]*$') # /^([a-zA-Z0-9][-.a-zA-Z0-9:_]*)\[(\w+)([=~\|\^\$\*]?)=?"?([^\]"]*)"?\]$/ # \---------------------------/ \---/\-------------/ \-------/ # | | | | # | | | The value # | | ~,|,^,$,* or = # | Attribute # Tag attribselect_re = re.compile( r'^(?P<tag>[a-zA-Z0-9][-.a-zA-Z0-9:_]*)?\[(?P<attribute>[\w-]+)(?P<operator>[=~\|\^\$\*]?)' + r'=?"?(?P<value>[^\]"]*)"?\]$' ) def _attr_value_as_string(self, value, default=None): """Force an attribute value into a string representation. A multi-valued attribute will be converted into a space-separated stirng. """ value = self.get(value, default) if isinstance(value, list) or isinstance(value, tuple): value =" ".join(value) return value def _tag_name_matches_and(self, function, tag_name): if not tag_name: return function else: def _match(tag): return tag.name == tag_name and function(tag) return _match def _attribute_checker(self, operator, attribute, value=''): """Create a function that performs a CSS selector operation. Takes an operator, attribute and optional value. Returns a function that will return True for elements that match that combination. """ if operator == '=': # string representation of `attribute` is equal to `value` return lambda el: el._attr_value_as_string(attribute) == value elif operator == '~': # space-separated list representation of `attribute` # contains `value` def _includes_value(element): attribute_value = element.get(attribute, []) if not isinstance(attribute_value, list): attribute_value = attribute_value.split() return value in attribute_value return _includes_value elif operator == '^': # string representation of `attribute` starts with `value` return lambda el: el._attr_value_as_string( attribute, '').startswith(value) elif operator == '$': # string representation of `attribute` ends with `value` return lambda el: el._attr_value_as_string( attribute, '').endswith(value) elif operator == '*': # string representation of `attribute` contains `value` return lambda el: value in el._attr_value_as_string(attribute, '') elif operator == '|': # string representation of `attribute` is either exactly # `value` or starts with `value` and then a dash. def _is_or_starts_with_dash(element): attribute_value = element._attr_value_as_string(attribute, '') return (attribute_value == value or attribute_value.startswith( value + '-')) return _is_or_starts_with_dash else: return lambda el: el.has_attr(attribute) # Old non-property versions of the generators, for backwards # compatibility with BS3. def nextGenerator(self): return self.next_elements def nextSiblingGenerator(self): return self.next_siblings def previousGenerator(self): return self.previous_elements def previousSiblingGenerator(self): return self.previous_siblings def parentGenerator(self): return self.parents class NavigableString(str, PageElement): PREFIX = '' SUFFIX = '' # We can't tell just by looking at a string whether it's contained # in an XML document or an HTML document. known_xml = None def __new__(cls, value): """Create a new NavigableString. When unpickling a NavigableString, this method is called with the string in DEFAULT_OUTPUT_ENCODING. That encoding needs to be passed in to the superclass's __new__ or the superclass won't know how to handle non-ASCII characters. """ if isinstance(value, str): u = str.__new__(cls, value) else: u = str.__new__(cls, value, DEFAULT_OUTPUT_ENCODING) u.setup() return u def __copy__(self): """A copy of a NavigableString has the same contents and class as the original, but it is not connected to the parse tree. """ return type(self)(self) def __getnewargs__(self): return (str(self),) def __getattr__(self, attr): """text.string gives you text. This is for backwards compatibility for Navigable*String, but for CData* it lets you get the string without the CData wrapper.""" if attr == 'string': return self else: raise AttributeError( "'%s' object has no attribute '%s'" % ( self.__class__.__name__, attr)) def output_ready(self, formatter="minimal"): output = self.format_string(self, formatter) return self.PREFIX + output + self.SUFFIX @property def name(self): return None @name.setter def name(self, name): raise AttributeError("A NavigableString cannot be given a name.") class PreformattedString(NavigableString): """A NavigableString not subject to the normal formatting rules. The string will be passed into the formatter (to trigger side effects), but the return value will be ignored. """ def output_ready(self, formatter="minimal"): """CData strings are passed into the formatter. But the return value is ignored.""" self.format_string(self, formatter) return self.PREFIX + self + self.SUFFIX class CData(PreformattedString): PREFIX = '<![CDATA[' SUFFIX = ']]>' class ProcessingInstruction(PreformattedString): """A SGML processing instruction.""" PREFIX = '<?' SUFFIX = '>' class XMLProcessingInstruction(ProcessingInstruction): """An XML processing instruction.""" PREFIX = '<?' SUFFIX = '?>' class Comment(PreformattedString): PREFIX = '<!--' SUFFIX = '-->' class Declaration(PreformattedString): PREFIX = '<?' SUFFIX = '?>' class Doctype(PreformattedString): @classmethod def for_name_and_ids(cls, name, pub_id, system_id): value = name or '' if pub_id is not None: value += ' PUBLIC "%s"' % pub_id if system_id is not None: value += ' "%s"' % system_id elif system_id is not None: value += ' SYSTEM "%s"' % system_id return Doctype(value) PREFIX = '<!DOCTYPE ' SUFFIX = '>\n' class Tag(PageElement): """Represents a found HTML tag with its attributes and contents.""" def __init__(self, parser=None, builder=None, name=None, namespace=None, prefix=None, attrs=None, parent=None, previous=None, is_xml=None): "Basic constructor." if parser is None: self.parser_class = None else: # We don't actually store the parser object: that lets extracted # chunks be garbage-collected. self.parser_class = parser.__class__ if name is None: raise ValueError("No value provided for new tag's name.") self.name = name self.namespace = namespace self.prefix = prefix if builder is not None: preserve_whitespace_tags = builder.preserve_whitespace_tags else: if is_xml: preserve_whitespace_tags = [] else: preserve_whitespace_tags = HTMLAwareEntitySubstitution.preserve_whitespace_tags self.preserve_whitespace_tags = preserve_whitespace_tags if attrs is None: attrs = {} elif attrs: if builder is not None and builder.cdata_list_attributes: attrs = builder._replace_cdata_list_attribute_values( self.name, attrs) else: attrs = dict(attrs) else: attrs = dict(attrs) # If possible, determine ahead of time whether this tag is an # XML tag. if builder: self.known_xml = builder.is_xml else: self.known_xml = is_xml self.attrs = attrs self.contents = [] self.setup(parent, previous) self.hidden = False # Set up any substitutions, such as the charset in a META tag. if builder is not None: builder.set_up_substitutions(self) self.can_be_empty_element = builder.can_be_empty_element(name) else: self.can_be_empty_element = False parserClass = _alias("parser_class") # BS3 def __copy__(self): """A copy of a Tag is a new Tag, unconnected to the parse tree. Its contents are a copy of the old Tag's contents. """ clone = type(self)(None, self.builder, self.name, self.namespace, self.prefix, self.attrs, is_xml=self._is_xml) for attr in ('can_be_empty_element', 'hidden'): setattr(clone, attr, getattr(self, attr)) for child in self.contents: clone.append(child.__copy__()) return clone @property def is_empty_element(self): """Is this tag an empty-element tag? (aka a self-closing tag) A tag that has contents is never an empty-element tag. A tag that has no contents may or may not be an empty-element tag. It depends on the builder used to create the tag. If the builder has a designated list of empty-element tags, then only a tag whose name shows up in that list is considered an empty-element tag. If the builder has no designated list of empty-element tags, then any tag with no contents is an empty-element tag. """ return len(self.contents) == 0 and self.can_be_empty_element isSelfClosing = is_empty_element # BS3 @property def string(self): """Convenience property to get the single string within this tag. :Return: If this tag has a single string child, return value is that string. If this tag has no children, or more than one child, return value is None. If this tag has one child tag, return value is the 'string' attribute of the child tag, recursively. """ if len(self.contents) != 1: return None child = self.contents[0] if isinstance(child, NavigableString): return child return child.string @string.setter def string(self, string): self.clear() self.append(string.__class__(string)) def _all_strings(self, strip=False, types=(NavigableString, CData)): """Yield all strings of certain classes, possibly stripping them. By default, yields only NavigableString and CData objects. So no comments, processing instructions, etc. """ for descendant in self.descendants: if ( (types is None and not isinstance(descendant, NavigableString)) or (types is not None and type(descendant) not in types)): continue if strip: descendant = descendant.strip() if len(descendant) == 0: continue yield descendant strings = property(_all_strings) @property def stripped_strings(self): for string in self._all_strings(True): yield string def get_text(self, separator="", strip=False, types=(NavigableString, CData)): """ Get all child strings, concatenated using the given separator. """ return separator.join([s for s in self._all_strings( strip, types=types)]) getText = get_text text = property(get_text) def decompose(self): """Recursively destroys the contents of this tree.""" self.extract() i = self while i is not None: next = i.next_element i.__dict__.clear() i.contents = [] i = next def clear(self, decompose=False): """ Extract all children. If decompose is True, decompose instead. """ if decompose: for element in self.contents[:]: if isinstance(element, Tag): element.decompose() else: element.extract() else: for element in self.contents[:]: element.extract() def index(self, element): """ Find the index of a child by identity, not value. Avoids issues with tag.contents.index(element) getting the index of equal elements. """ for i, child in enumerate(self.contents): if child is element: return i raise ValueError("Tag.index: element not in tag") def get(self, key, default=None): """Returns the value of the 'key' attribute for the tag, or the value given for 'default' if it doesn't have that attribute.""" return self.attrs.get(key, default) def get_attribute_list(self, key, default=None): """The same as get(), but always returns a list.""" value = self.get(key, default) if not isinstance(value, list): value = [value] return value def has_attr(self, key): return key in self.attrs def __hash__(self): return str(self).__hash__() def __getitem__(self, key): """tag[key] returns the value of the 'key' attribute for the tag, and throws an exception if it's not there.""" return self.attrs[key] def __iter__(self): "Iterating over a tag iterates over its contents." return iter(self.contents) def __len__(self): "The length of a tag is the length of its list of contents." return len(self.contents) def __contains__(self, x): return x in self.contents def __bool__(self): "A tag is non-None even if it has no contents." return True def __setitem__(self, key, value): """Setting tag[key] sets the value of the 'key' attribute for the tag.""" self.attrs[key] = value def __delitem__(self, key): "Deleting tag[key] deletes all 'key' attributes for the tag." self.attrs.pop(key, None) def __call__(self, *args, **kwargs): """Calling a tag like a function is the same as calling its find_all() method. Eg. tag('a') returns a list of all the A tags found within this tag.""" return self.find_all(*args, **kwargs) def __getattr__(self, tag): #print "Getattr %s.%s" % (self.__class__, tag) if len(tag) > 3 and tag.endswith('Tag'): # BS3: soup.aTag -> "soup.find("a") tag_name = tag[:-3] warnings.warn( '.%(name)sTag is deprecated, use .find("%(name)s") instead. If you really were looking for a tag called %(name)sTag, use .find("%(name)sTag")' % dict( name=tag_name ) ) return self.find(tag_name) # We special case contents to avoid recursion. elif not tag.startswith("__") and not tag == "contents": return self.find(tag) raise AttributeError( "'%s' object has no attribute '%s'" % (self.__class__, tag)) def __eq__(self, other): """Returns true iff this tag has the same name, the same attributes, and the same contents (recursively) as the given tag.""" if self is other: return True if (not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other)): return False for i, my_child in enumerate(self.contents): if my_child != other.contents[i]: return False return True def __ne__(self, other): """Returns true iff this tag is not identical to the other tag, as defined in __eq__.""" return not self == other def __repr__(self, encoding="unicode-escape"): """Renders this tag as a string.""" if PY3K: # "The return value must be a string object", i.e. Unicode return self.decode() else: # "The return value must be a string object", i.e. a bytestring. # By convention, the return value of __repr__ should also be # an ASCII string. return self.encode(encoding) def __unicode__(self): return self.decode() def __str__(self): if PY3K: return self.decode() else: return self.encode() if PY3K: __str__ = __repr__ = __unicode__ def encode(self, encoding=DEFAULT_OUTPUT_ENCODING, indent_level=None, formatter="minimal", errors="xmlcharrefreplace"): # Turn the data structure into Unicode, then encode the # Unicode. u = self.decode(indent_level, encoding, formatter) return u.encode(encoding, errors) def _should_pretty_print(self, indent_level): """Should this tag be pretty-printed?""" return ( indent_level is not None and self.name not in self.preserve_whitespace_tags ) def decode(self, indent_level=None, eventual_encoding=DEFAULT_OUTPUT_ENCODING, formatter="minimal"): """Returns a Unicode representation of this tag and its contents. :param eventual_encoding: The tag is destined to be encoded into this encoding. This method is _not_ responsible for performing that encoding. This information is passed in so that it can be substituted in if the document contains a <META> tag that mentions the document's encoding. """ # First off, turn a string formatter into a Formatter object. This # will stop the lookup from happening over and over again. if not isinstance(formatter, Formatter) and not callable(formatter): formatter = self._formatter_for_name(formatter) attrs = [] if self.attrs: for key, val in sorted(self.attrs.items()): if val is None: decoded = key else: if isinstance(val, list) or isinstance(val, tuple): val = ' '.join(val) elif not isinstance(val, str): val = str(val) elif ( isinstance(val, AttributeValueWithCharsetSubstitution) and eventual_encoding is not None): val = val.encode(eventual_encoding) text = self.format_string(val, formatter) decoded = ( str(key) + '=' + EntitySubstitution.quoted_attribute_value(text)) attrs.append(decoded) close = '' closeTag = '' prefix = '' if self.prefix: prefix = self.prefix + ":" if self.is_empty_element: close = '' if isinstance(formatter, Formatter): close = formatter.void_element_close_prefix or close else: closeTag = '</%s%s>' % (prefix, self.name) pretty_print = self._should_pretty_print(indent_level) space = '' indent_space = '' if indent_level is not None: indent_space = (' ' * (indent_level - 1)) if pretty_print: space = indent_space indent_contents = indent_level + 1 else: indent_contents = None contents = self.decode_contents( indent_contents, eventual_encoding, formatter) if self.hidden: # This is the 'document root' object. s = contents else: s = [] attribute_string = '' if attrs: attribute_string = ' ' + ' '.join(attrs) if indent_level is not None: # Even if this particular tag is not pretty-printed, # we should indent up to the start of the tag. s.append(indent_space) s.append('<%s%s%s%s>' % ( prefix, self.name, attribute_string, close)) if pretty_print: s.append("\n") s.append(contents) if pretty_print and contents and contents[-1] != "\n": s.append("\n") if pretty_print and closeTag: s.append(space) s.append(closeTag) if indent_level is not None and closeTag and self.next_sibling: # Even if this particular tag is not pretty-printed, # we're now done with the tag, and we should add a # newline if appropriate. s.append("\n") s = ''.join(s) return s def prettify(self, encoding=None, formatter="minimal"): if encoding is None: return self.decode(True, formatter=formatter) else: return self.encode(encoding, True, formatter=formatter) def decode_contents(self, indent_level=None, eventual_encoding=DEFAULT_OUTPUT_ENCODING, formatter="minimal"): """Renders the contents of this tag as a Unicode string. :param indent_level: Each line of the rendering will be indented this many spaces. :param eventual_encoding: The tag is destined to be encoded into this encoding. This method is _not_ responsible for performing that encoding. This information is passed in so that it can be substituted in if the document contains a <META> tag that mentions the document's encoding. :param formatter: The output formatter responsible for converting entities to Unicode characters. """ # First off, turn a string formatter into a Formatter object. This # will stop the lookup from happening over and over again. if not isinstance(formatter, Formatter) and not callable(formatter): formatter = self._formatter_for_name(formatter) pretty_print = (indent_level is not None) s = [] for c in self: text = None if isinstance(c, NavigableString): text = c.output_ready(formatter) elif isinstance(c, Tag): s.append(c.decode(indent_level, eventual_encoding, formatter)) if text and indent_level and not self.name == 'pre': text = text.strip() if text: if pretty_print and not self.name == 'pre': s.append(" " * (indent_level - 1)) s.append(text) if pretty_print and not self.name == 'pre': s.append("\n") return ''.join(s) def encode_contents( self, indent_level=None, encoding=DEFAULT_OUTPUT_ENCODING, formatter="minimal"): """Renders the contents of this tag as a bytestring. :param indent_level: Each line of the rendering will be indented this many spaces. :param eventual_encoding: The bytestring will be in this encoding. :param formatter: The output formatter responsible for converting entities to Unicode characters. """ contents = self.decode_contents(indent_level, encoding, formatter) return contents.encode(encoding) # Old method for BS3 compatibility def renderContents(self, encoding=DEFAULT_OUTPUT_ENCODING, prettyPrint=False, indentLevel=0): if not prettyPrint: indentLevel = None return self.encode_contents( indent_level=indentLevel, encoding=encoding) #Soup methods def find(self, name=None, attrs={}, recursive=True, text=None, **kwargs): """Return only the first child of this Tag matching the given criteria.""" r = None l = self.find_all(name, attrs, recursive, text, 1, **kwargs) if l: r = l[0] return r findChild = find def find_all(self, name=None, attrs={}, recursive=True, text=None, limit=None, **kwargs): """Extracts a list of Tag objects that match the given criteria. You can specify the name of the Tag and any attributes you want the Tag to have. The value of a key-value pair in the 'attrs' map can be a string, a list of strings, a regular expression object, or a callable that takes a string and returns whether or not the string matches for some custom definition of 'matches'. The same is true of the tag name.""" generator = self.descendants if not recursive: generator = self.children return self._find_all(name, attrs, text, limit, generator, **kwargs) findAll = find_all # BS3 findChildren = find_all # BS2 #Generator methods @property def children(self): # return iter() to make the purpose of the method clear return iter(self.contents) # XXX This seems to be untested. @property def descendants(self): if not len(self.contents): return stopNode = self._last_descendant().next_element current = self.contents[0] while current is not stopNode: yield current current = current.next_element # CSS selector code _selector_combinators = ['>', '+', '~'] _select_debug = False quoted_colon = re.compile('"[^"]*:[^"]*"') def select_one(self, selector): """Perform a CSS selection operation on the current element.""" value = self.select(selector, limit=1) if value: return value[0] return None def select(self, selector, _candidate_generator=None, limit=None): """Perform a CSS selection operation on the current element.""" # Handle grouping selectors if ',' exists, ie: p,a if ',' in selector: context = [] selectors = [x.strip() for x in selector.split(",")] # If a selector is mentioned multiple times we don't want # to use it more than once. used_selectors = set() # We also don't want to select the same element more than once, # if it's matched by multiple selectors. selected_object_ids = set() for partial_selector in selectors: if partial_selector == '': raise ValueError('Invalid group selection syntax: %s' % selector) if partial_selector in used_selectors: continue used_selectors.add(partial_selector) candidates = self.select(partial_selector, limit=limit) for candidate in candidates: # This lets us distinguish between distinct tags that # represent the same markup. object_id = id(candidate) if object_id not in selected_object_ids: context.append(candidate) selected_object_ids.add(object_id) if limit and len(context) >= limit: break return context tokens = shlex.split(selector) current_context = [self] if tokens[-1] in self._selector_combinators: raise ValueError( 'Final combinator "%s" is missing an argument.' % tokens[-1]) if self._select_debug: print(('Running CSS selector "%s"' % selector)) for index, token in enumerate(tokens): new_context = [] new_context_ids = set([]) if tokens[index-1] in self._selector_combinators: # This token was consumed by the previous combinator. Skip it. if self._select_debug: print(' Token was consumed by the previous combinator.') continue if self._select_debug: print((' Considering token "%s"' % token)) recursive_candidate_generator = None tag_name = None # Each operation corresponds to a checker function, a rule # for determining whether a candidate matches the # selector. Candidates are generated by the active # iterator. checker = None m = self.attribselect_re.match(token) if m is not None: # Attribute selector tag_name, attribute, operator, value = m.groups() checker = self._attribute_checker(operator, attribute, value) elif '#' in token: # ID selector tag_name, tag_id = token.split('#', 1) def id_matches(tag): return tag.get('id', None) == tag_id checker = id_matches elif '.' in token: # Class selector tag_name, klass = token.split('.', 1) classes = set(klass.split('.')) def classes_match(candidate): return classes.issubset(candidate.get('class', [])) checker = classes_match elif ':' in token and not self.quoted_colon.search(token): # Pseudo-class tag_name, pseudo = token.split(':', 1) if tag_name == '': raise ValueError( "A pseudo-class must be prefixed with a tag name.") pseudo_attributes = re.match(r'([a-zA-Z\d-]+)\(([a-zA-Z\d]+)\)', pseudo) found = [] if pseudo_attributes is None: pseudo_type = pseudo pseudo_value = None else: pseudo_type, pseudo_value = pseudo_attributes.groups() if pseudo_type == 'nth-of-type': try: pseudo_value = int(pseudo_value) except: raise NotImplementedError( 'Only numeric values are currently supported for the nth-of-type pseudo-class.') if pseudo_value < 1: raise ValueError( 'nth-of-type pseudo-class value must be at least 1.') class Counter(object): def __init__(self, destination): self.count = 0 self.destination = destination def nth_child_of_type(self, tag): self.count += 1 if self.count == self.destination: return True else: return False checker = Counter(pseudo_value).nth_child_of_type else: raise NotImplementedError( 'Only the following pseudo-classes are implemented: nth-of-type.') elif token == '*': # Star selector -- matches everything pass elif token == '>': # Run the next token as a CSS selector against the # direct children of each tag in the current context. recursive_candidate_generator = lambda tag: tag.children elif token == '~': # Run the next token as a CSS selector against the # siblings of each tag in the current context. recursive_candidate_generator = lambda tag: tag.next_siblings elif token == '+': # For each tag in the current context, run the next # token as a CSS selector against the tag's next # sibling that's a tag. def next_tag_sibling(tag): yield tag.find_next_sibling(True) recursive_candidate_generator = next_tag_sibling elif self.tag_name_re.match(token): # Just a tag name. tag_name = token else: raise ValueError( 'Unsupported or invalid CSS selector: "%s"' % token) if recursive_candidate_generator: # This happens when the selector looks like "> foo". # # The generator calls select() recursively on every # member of the current context, passing in a different # candidate generator and a different selector. # # In the case of "> foo", the candidate generator is # one that yields a tag's direct children (">"), and # the selector is "foo". next_token = tokens[index+1] def recursive_select(tag): if self._select_debug: print((' Calling select("%s") recursively on %s %s' % (next_token, tag.name, tag.attrs))) print(('-' * 40)) for i in tag.select(next_token, recursive_candidate_generator): if self._select_debug: print(('(Recursive select picked up candidate %s %s)' % (i.name, i.attrs))) yield i if self._select_debug: print(('-' * 40)) _use_candidate_generator = recursive_select elif _candidate_generator is None: # By default, a tag's candidates are all of its # children. If tag_name is defined, only yield tags # with that name. if self._select_debug: if tag_name: check = "[any]" else: check = tag_name print((' Default candidate generator, tag name="%s"' % check)) if self._select_debug: # This is redundant with later code, but it stops # a bunch of bogus tags from cluttering up the # debug log. def default_candidate_generator(tag): for child in tag.descendants: if not isinstance(child, Tag): continue if tag_name and not child.name == tag_name: continue yield child _use_candidate_generator = default_candidate_generator else: _use_candidate_generator = lambda tag: tag.descendants else: _use_candidate_generator = _candidate_generator count = 0 for tag in current_context: if self._select_debug: print((" Running candidate generator on %s %s" % ( tag.name, repr(tag.attrs)))) for candidate in _use_candidate_generator(tag): if not isinstance(candidate, Tag): continue if tag_name and candidate.name != tag_name: continue if checker is not None: try: result = checker(candidate) except StopIteration: # The checker has decided we should no longer # run the generator. break if checker is None or result: if self._select_debug: print((" SUCCESS %s %s" % (candidate.name, repr(candidate.attrs)))) if id(candidate) not in new_context_ids: # If a tag matches a selector more than once, # don't include it in the context more than once. new_context.append(candidate) new_context_ids.add(id(candidate)) elif self._select_debug: print((" FAILURE %s %s" % (candidate.name, repr(candidate.attrs)))) current_context = new_context if limit and len(current_context) >= limit: current_context = current_context[:limit] if self._select_debug: print("Final verdict:") for i in current_context: print((" %s %s" % (i.name, i.attrs))) return current_context # Old names for backwards compatibility def childGenerator(self): return self.children def recursiveChildGenerator(self): return self.descendants def has_key(self, key): """This was kind of misleading because has_key() (attributes) was different from __in__ (contents). has_key() is gone in Python 3, anyway.""" warnings.warn('has_key is deprecated. Use has_attr("%s") instead.' % ( key)) return self.has_attr(key) # Next, a couple classes to represent queries and their results. class SoupStrainer(object): """Encapsulates a number of ways of matching a markup element (tag or text).""" def __init__(self, name=None, attrs={}, text=None, **kwargs): self.name = self._normalize_search_value(name) if not isinstance(attrs, dict): # Treat a non-dict value for attrs as a search for the 'class' # attribute. kwargs['class'] = attrs attrs = None if 'class_' in kwargs: # Treat class_="foo" as a search for the 'class' # attribute, overriding any non-dict value for attrs. kwargs['class'] = kwargs['class_'] del kwargs['class_'] if kwargs: if attrs: attrs = attrs.copy() attrs.update(kwargs) else: attrs = kwargs normalized_attrs = {} for key, value in list(attrs.items()): normalized_attrs[key] = self._normalize_search_value(value) self.attrs = normalized_attrs self.text = self._normalize_search_value(text) def _normalize_search_value(self, value): # Leave it alone if it's a Unicode string, a callable, a # regular expression, a boolean, or None. if (isinstance(value, str) or callable(value) or hasattr(value, 'match') or isinstance(value, bool) or value is None): return value # If it's a bytestring, convert it to Unicode, treating it as UTF-8. if isinstance(value, bytes): return value.decode("utf8") # If it's listlike, convert it into a list of strings. if hasattr(value, '__iter__'): new_value = [] for v in value: if (hasattr(v, '__iter__') and not isinstance(v, bytes) and not isinstance(v, str)): # This is almost certainly the user's mistake. In the # interests of avoiding infinite loops, we'll let # it through as-is rather than doing a recursive call. new_value.append(v) else: new_value.append(self._normalize_search_value(v)) return new_value # Otherwise, convert it into a Unicode string. # The unicode(str()) thing is so this will do the same thing on Python 2 # and Python 3. return str(str(value)) def __str__(self): if self.text: return self.text else: return "%s|%s" % (self.name, self.attrs) def search_tag(self, markup_name=None, markup_attrs={}): found = None markup = None if isinstance(markup_name, Tag): markup = markup_name markup_attrs = markup call_function_with_tag_data = ( isinstance(self.name, Callable) and not isinstance(markup_name, Tag)) if ((not self.name) or call_function_with_tag_data or (markup and self._matches(markup, self.name)) or (not markup and self._matches(markup_name, self.name))): if call_function_with_tag_data: match = self.name(markup_name, markup_attrs) else: match = True markup_attr_map = None for attr, match_against in list(self.attrs.items()): if not markup_attr_map: if hasattr(markup_attrs, 'get'): markup_attr_map = markup_attrs else: markup_attr_map = {} for k, v in markup_attrs: markup_attr_map[k] = v attr_value = markup_attr_map.get(attr) if not self._matches(attr_value, match_against): match = False break if match: if markup: found = markup else: found = markup_name if found and self.text and not self._matches(found.string, self.text): found = None return found searchTag = search_tag def search(self, markup): # print 'looking for %s in %s' % (self, markup) found = None # If given a list of items, scan it for a text element that # matches. if hasattr(markup, '__iter__') and not isinstance(markup, (Tag, str)): for element in markup: if isinstance(element, NavigableString) \ and self.search(element): found = element break # If it's a Tag, make sure its name or attributes match. # Don't bother with Tags if we're searching for text. elif isinstance(markup, Tag): if not self.text or self.name or self.attrs: found = self.search_tag(markup) # If it's text, make sure the text matches. elif isinstance(markup, NavigableString) or \ isinstance(markup, str): if not self.name and not self.attrs and self._matches(markup, self.text): found = markup else: raise Exception( "I don't know how to match against a %s" % markup.__class__) return found def _matches(self, markup, match_against, already_tried=None): # print u"Matching %s against %s" % (markup, match_against) result = False if isinstance(markup, list) or isinstance(markup, tuple): # This should only happen when searching a multi-valued attribute # like 'class'. for item in markup: if self._matches(item, match_against): return True # We didn't match any particular value of the multivalue # attribute, but maybe we match the attribute value when # considered as a string. if self._matches(' '.join(markup), match_against): return True return False if match_against is True: # True matches any non-None value. return markup is not None if isinstance(match_against, Callable): return match_against(markup) # Custom callables take the tag as an argument, but all # other ways of matching match the tag name as a string. original_markup = markup if isinstance(markup, Tag): markup = markup.name # Ensure that `markup` is either a Unicode string, or None. markup = self._normalize_search_value(markup) if markup is None: # None matches None, False, an empty string, an empty list, and so on. return not match_against if (hasattr(match_against, '__iter__') and not isinstance(match_against, str)): # We're asked to match against an iterable of items. # The markup must be match at least one item in the # iterable. We'll try each one in turn. # # To avoid infinite recursion we need to keep track of # items we've already seen. if not already_tried: already_tried = set() for item in match_against: if item.__hash__: key = item else: key = id(item) if key in already_tried: continue else: already_tried.add(key) if self._matches(original_markup, item, already_tried): return True else: return False # Beyond this point we might need to run the test twice: once against # the tag's name and once against its prefixed name. match = False if not match and isinstance(match_against, str): # Exact string match match = markup == match_against if not match and hasattr(match_against, 'search'): # Regexp match return match_against.search(markup) if (not match and isinstance(original_markup, Tag) and original_markup.prefix): # Try the whole thing again with the prefixed tag name. return self._matches( original_markup.prefix + ':' + original_markup.name, match_against ) return match class ResultSet(list): """A ResultSet is just a list that keeps track of the SoupStrainer that created it.""" def __init__(self, source, result=()): super(ResultSet, self).__init__(result) self.source = source def __getattr__(self, key): raise AttributeError( "ResultSet object has no attribute '%s'. You're probably treating a list of items like a single item. Did you call find_all() when you meant to call find()?" % key ) __pycache__/element.cpython-36.opt-1.pyc 0000644 00000142406 15246412105 0014001 0 ustar 00 3 &6]$ � @ s d Z yddlmZ W n. ek rB Z zddlmZ W Y ddZ[X nX ddlZddlZddlZddl Z ddl m Z dZ ej d dkZejd�Zdd � ZG d d � d e�ZG d d � d e�ZG dd� de�ZG dd� de�ZG dd� de �ZG dd� de�ZG dd� de�ZG dd� de�ZG dd� de�Z G d d � d e�Z G d d � d e�Z G d d!� d!e�Z G d"d#� d#ee �Z G d$d%� d%e �Z!G d&d'� d'e!�Z"G d(d)� d)e!�Z#G d*d+� d+e#�Z$G d,d-� d-e!�Z%G d.d/� d/e!�Z&G d0d1� d1e!�Z'G d2d3� d3e �Z(G d4d5� d5e�Z)G d6d7� d7e*�Z+dS )8ZMIT� )�CallableN)�EntitySubstitutionzutf-8� z\s+c s& t � fdd��}|j� fdd��}|S )z>Alias one attribute name to another for backward compatibilityc s t | � �S )N)�getattr)�self)�attr� � /usr/lib/python3.6/element.py�alias s z_alias.<locals>.aliasc s t | � �S )N)�setattr)r )r r r r s )�property�setter)r r r )r r �_alias s r c @ s e Zd Zddd�ZdS )�NamespacedAttributeNc C sV |d krt j| |�}n*|d kr,t j| |�}nt j| |d | �}||_||_||_|S )N�:)�str�__new__�prefix�name� namespace)�clsr r r �objr r r r " s zNamespacedAttribute.__new__)N)�__name__� __module__� __qualname__r r r r r r s r c @ s e Zd ZdZdS )�%AttributeValueWithCharsetSubstitutionz=A stand-in object for a character encoding specified in HTML.N)r r r �__doc__r r r r r / s r c @ s e Zd ZdZdd� Zdd� ZdS )�CharsetMetaAttributeValuez�A generic stand-in for the value of a meta tag's 'charset' attribute. When Beautiful Soup parses the markup '<meta charset="utf8">', the value of the 'charset' attribute will be one of these objects. c C s t j| |�}||_|S )N)r r �original_value)r r r r r r r 9 s z!CharsetMetaAttributeValue.__new__c C s |S )Nr )r �encodingr r r �encode> s z CharsetMetaAttributeValue.encodeN)r r r r r r r r r r r 2 s r c @ s. e Zd ZdZejdej�Zdd� Zdd� Z dS )�ContentMetaAttributeValuea A generic stand-in for the value of a meta tag's 'content' attribute. When Beautiful Soup parses the markup: <meta http-equiv="content-type" content="text/html; charset=utf8"> The value of the 'content' attribute will be one of these objects. z((^|;)\s*charset=)([^;]*)c C s6 | j j|�}|d kr tjt|�S tj| |�}||_|S )N)� CHARSET_RE�searchr r r )r r �matchr r r r r M s z!ContentMetaAttributeValue.__new__c s � fdd�}| j j|| j�S )Nc s | j d�� S )N� )�group)r$ )r r r �rewriteX s z1ContentMetaAttributeValue.encode.<locals>.rewrite)r" �subr )r r r' r )r r r W s z ContentMetaAttributeValue.encodeN) r r r r �re�compile�Mr" r r r r r r r! B s r! c @ sV e Zd ZdZeddg�Zedg�Zeddg�Zedd� �Z edd � �Z ed d � �Z d S ) �HTMLAwareEntitySubstitutiona% Entity substitution rules that are aware of some HTML quirks. Specifically, the contents of <script> and <style> tags should not undergo entity substitution. Incoming NavigableString objects are checked to see if they're the direct children of a <script> or <style> tag. ZscriptZstyle�preZtextareac C s. t |t�r&|jd k r&|jj| jkr&|S ||�S )N)� isinstance�NavigableString�parentr �cdata_containing_tags)r �ns�fr r r �_substitute_if_appropriatem s z6HTMLAwareEntitySubstitution._substitute_if_appropriatec C s | j |tj�S )N)r4 r �substitute_html)r r2 r r r r5 w s z+HTMLAwareEntitySubstitution.substitute_htmlc C s | j |tj�S )N)r4 r �substitute_xml)r r2 r r r r6 | s z*HTMLAwareEntitySubstitution.substitute_xmlN) r r r r �setr1 Zpreformatted_tags�preserve_whitespace_tags� classmethodr4 r5 r6 r r r r r, \ s r, c @ s e Zd ZdZdZdd� ZdS )� Formatterz6Contains information about how to format a parse tree.�/c O s t � �dS )z1Transform certain characters into named entities.N)�NotImplementedError)r �args�kwargsr r r �substitute_entities� s z Formatter.substitute_entitiesN)r r r r �void_element_close_prefixr? r r r r r: � s r: c @ s e Zd ZdZdd� ZdS )� HTMLFormatterzThe default HTML formatter.c O s t j||�S )N)r, r5 )r r= r> r r r � substitute� s zHTMLFormatter.substituteN)r r r r rB r r r r rA � s rA c @ s e Zd ZdZdd� ZdS )�MinimalHTMLFormatterzA minimal HTML formatter.c O s t j||�S )N)r, r6 )r r= r> r r r rB � s z MinimalHTMLFormatter.substituteN)r r r r rB r r r r rC � s rC c @ s e Zd ZdZdZdS )�HTML5Formatterz5An HTML formatter that omits the slash in a void tag.N)r r r r r@ r r r r rD � s rD c @ s e Zd ZdZdd� ZdS )� XMLFormatterz+Substitute only the essential XML entities.c O s t j||�S )N)r r6 )r r= r> r r r rB � s zXMLFormatter.substituteN)r r r r rB r r r r rE � s rE c @ s e Zd ZdZdd� ZdS )�HTMLXMLFormatterz Format XML using HTML rules.c O s t j||�S )N)r, r5 )r r= r> r r r rB � s zHTMLXMLFormatter.substituteN)r r r r rB r r r r rF � s rF c @ s` e Zd ZdZe� e� e� dd�Ze� e � dd�Z d]dd�Z e dd � �Z d d � Zd^d d �Zed�Zed�Zdd� ZeZdd� ZeZeZdd� Zdd� Zd_dd�ZeZdd � Z d d � Z d d � Z d!d"� Z di dfd#d$�Z e Z!di ddfd%d&�Z"e"Z#di dfd'd(�Z$e$Z%di ddfd)d*�Z&e&Z'e&Z(di dfd+d,�Z)e)Z*di ddfd-d.�Z+e+Z,e+Z-di dfd/d0�Z.e.Z/di ddfd1d2�Z0e0Z1e0Z2di fd3d4�Z3e3Z4di dfd5d6�Z5e5Z6e5Z7e d7d8� �Z8e d9d:� �Z9d;d<� Z:d=d>� Z;e d?d@� �Z<e dAdB� �Z=e dCdD� �Z>e dEdF� �Z?e dGdH� �Z@eAjBdI�ZCeAjBd`�ZDdadLdM�ZEdNdO� ZFdbdQdR�ZGdSdT� ZHdUdV� ZIdWdX� ZJdYdZ� ZKd[d\� ZLdS )c� PageElementzeContains the navigational information for some part of the page (either a tag or a piece of text)N)�htmlZhtml5�minimalN)rH rI NrI c C sB t |t�r| j|�}|dkr"|}n t|�r4||�}n |j|�}|S )z2Format the given string using the given formatter.N)r. r �_formatter_for_name�callablerB )r �s� formatter�outputr r r � format_string� s zPageElement.format_stringc C s. | j dk r| j S | jdkr&t| dd�S | jjS )aJ Is this element part of an XML tree or an HTML tree? This is used when mapping a formatter name ("minimal") to an appropriate function (one that performs entity-substitution on the contents of <script> and <style> tags, or not). It can be inefficient, but it should be called very rarely. N�is_xmlF)� known_xmlr0 r �_is_xml)r r r r rR � s zPageElement._is_xmlc C s* | j r| jj|t� �S | jj|t� �S dS )z<Look up a formatter function based on its name and the tree.N)rR �XML_FORMATTERS�getrE �HTML_FORMATTERSrA )r r r r r rJ � s z PageElement._formatter_for_namec C s~ || _ || _|dk r | | j_|| _| jr0| | j_|| _| jrD| | j_| rh| j dk rh| j jrh| j jd }|| _|rz| | j_dS )zNSets up the initial relations between this element and other elements.Nr% ���)r0 �previous_element� next_element� next_sibling�previous_sibling�contents)r r0 rW rX rZ rY r r r �setup� s zPageElement.setuprY rZ c C sV | j std��|| krd S || j kr,td��| j }| j j| �}| j� |j||� | S )Nz]Cannot replace one element with another when theelement to be replaced is not part of a tree.z%Cannot replace a Tag with its parent.)r0 � ValueError�index�extract�insert)r � replace_withZ old_parent�my_indexr r r ra s zPageElement.replace_withc C sT | j }| j std��| j j| �}| j� x&t| jd d � �D ]}|j||� q<W | S )NzSCannot replace an element with its contents when thatelement is not part of a tree.)r0 r] r^ r_ �reversedr[ r` )r Z my_parentrb �childr r r �unwrap% s zPageElement.unwrapc C s | j |�}|j|� |S )N)ra �append)r Z wrap_inside�mer r r �wrap3 s zPageElement.wrapc C s� | j dk r | j j| j j| �= | j� }|j}| jdk rF| j|k rF|| j_|dk r`|| jk r`| j|_d| _d|_d| _ | jdk r�| j| jk r�| j| j_| jdk r�| j| jk r�| j| j_d | _| _| S )z0Destructively rips this element out of the tree.N)r0 r[ r^ �_last_descendantrX rW rZ rY )r � last_childrX r r r r_ 8 s( zPageElement.extractTc C sN |r| j r| j j}n$| }x t|t�r6|jr6|jd }qW | rJ|| krJd}|S )z8Finds the last element beneath this object to be parsed.r% NrV )rY rW r. �Tagr[ )r Zis_initializedZ accept_selfrj r r r ri U s z PageElement._last_descendantc C s� |d krt d��|| kr t d��t|t�r>t|t� r>t|�}ddlm} t||�r�x&t|j�D ]}| j||� |d7 }q`W d S t |t | j��}t |d�r�|j d k r�|j | kr�| j |�}||k r�|d8 }|j� | |_ d }|dkr�d |_| |_n(| j|d }||_||j_|jd�|_|jd k �r,||j_|jd�}|t | j�k�r�d |_| }d } x2| d k�r�|d k �r�|j} |j }| d k �rVP �qVW | d k �r�| |_nd |_n*| j| } | |_|jd k �r�||j_| |_|jd k �r�||j_| jj||� d S )Nz Cannot insert None into a tag.z Cannot insert a tag into itself.r )� BeautifulSoupr% r0 F)r] r. r r/ Zbs4rl �listr[ r` �min�len�hasattrr0 r^ r_ rZ rW rY ri rX ) r Zposition� new_childrl ZsubchildZ current_indexZprevious_childZnew_childs_last_elementr0 Zparents_next_siblingZ next_childr r r r` c sh zPageElement.insertc C s | j t| j�|� dS )z2Appends the given tag to the contents of this tag.N)r` ro r[ )r �tagr r r rf � s zPageElement.appendc C sR | |krt d��| j}|dkr&t d��t|t�r8|j� |j| �}|j||� dS )z�Makes the given element the immediate predecessor of this one. The two elements will have the same parent, and the given element will be immediately before this one. z&Can't insert an element before itself.Nz2Element has no parent, so 'before' has no meaning.)r] r0 r. rG r_ r^ r` )r Z predecessorr0 r^ r r r � insert_before� s zPageElement.insert_beforec C sV | |krt d��| j}|dkr&t d��t|t�r8|j� |j| �}|j|d |� dS )z�Makes the given element the immediate successor of this one. The two elements will have the same parent, and the given element will be immediately after this one. z%Can't insert an element after itself.Nz1Element has no parent, so 'after' has no meaning.r% )r] r0 r. rG r_ r^ r` )r Z successorr0 r^ r r r � insert_after� s zPageElement.insert_afterc K s | j | j|||f|�S )zjReturns the first item that matches the given criteria and appears after this Tag in the document.)� _find_one� find_all_next)r r �attrs�textr> r r r � find_next� s zPageElement.find_nextc K s | j ||||| jf|�S )zbReturns all items that match the given criteria and appear after this Tag in the document.)� _find_all� next_elements)r r rw rx �limitr> r r r rv � s zPageElement.find_all_nextc K s | j | j|||f|�S )z{Returns the closest sibling to this Tag that matches the given criteria and appears after this Tag in the document.)ru �find_next_siblings)r r rw rx r> r r r �find_next_sibling� s z PageElement.find_next_siblingc K s | j ||||| jf|�S )zqReturns the siblings of this Tag that match the given criteria and appear after this Tag in the document.)rz � next_siblings)r r rw rx r| r> r r r r} � s z PageElement.find_next_siblingsc K s | j | j|||f|�S )zkReturns the first item that matches the given criteria and appears before this Tag in the document.)ru �find_all_previous)r r rw rx r> r r r � find_previous� s zPageElement.find_previousc K s | j ||||| jf|�S )zcReturns all items that match the given criteria and appear before this Tag in the document.)rz �previous_elements)r r rw rx r| r> r r r r� � s z PageElement.find_all_previousc K s | j | j|||f|�S )z|Returns the closest sibling to this Tag that matches the given criteria and appears before this Tag in the document.)ru �find_previous_siblings)r r rw rx r> r r r �find_previous_sibling s z!PageElement.find_previous_siblingc K s | j ||||| jf|�S )zrReturns the siblings of this Tag that match the given criteria and appear before this Tag in the document.)rz �previous_siblings)r r rw rx r| r> r r r r� s z"PageElement.find_previous_siblingsc K s& d}| j ||df|�}|r"|d }|S )zOReturns the closest parent of this Tag that matches the given criteria.Nr% r )� find_parents)r r rw r> �r�lr r r � find_parent s zPageElement.find_parentc K s | j ||d|| jf|�S )zFReturns the parents of this Tag that match the given criteria.N)rz �parents)r r rw r| r> r r r r� ! s zPageElement.find_parentsc C s | j S )N)rX )r r r r �next* s zPageElement.nextc C s | j S )N)rW )r r r r �previous. s zPageElement.previousc K s& d }||||df|�}|r"|d }|S )Nr% r r )r �methodr rw rx r> r� r� r r r ru 4 s zPageElement._find_onec s8 |dkr d|kr |d }|d= t �t�r.�}nt�||f|�}|dkr�| r�| r�| r҈dksh�dkr�dd� |D �}t||�S t �t�r҈jd�dkr��jdd�\�� nd��� � ��fdd�|D �}t||�S t|�} xXy t|�} W n tk �r P Y nX | r�|j| �} | r�| j | � |r�t | �|kr�P q�W | S ) z8Iterates over a generator looking for things that match.N�stringTc s s | ]}t |t�r|V qd S )N)r. rk )�.0�elementr r r � <genexpr>J s z(PageElement._find_all.<locals>.<genexpr>r r% c 3 sB | ]:}t |t�r|j�ks6|j� kr�d ks6|j�kr|V qd S )N)r. rk r r )r� r� )� local_namer r r r r� W s ) r. � SoupStrainer� ResultSetr �count�splitr� � StopIterationr# rf ro ) r r rw rx r| � generatorr> Zstrainer�result�results�i�foundr )r� r r r rz ; s< zPageElement._find_allc c s$ | j }x|d k r |V |j }qW d S )N)rX )r r� r r r r{ q s zPageElement.next_elementsc c s$ | j }x|d k r |V |j }qW d S )N)rY )r r� r r r r x s zPageElement.next_siblingsc c s$ | j }x|d k r |V |j }qW d S )N)rW )r r� r r r r� s z PageElement.previous_elementsc c s$ | j }x|d k r |V |j }qW d S )N)rZ )r r� r r r r� � s z PageElement.previous_siblingsc c s$ | j }x|d k r |V |j }qW d S )N)r0 )r r� r r r r� � s zPageElement.parentsz ^[a-zA-Z0-9][-.a-zA-Z0-9:_]*$zX^(?P<tag>[a-zA-Z0-9][-.a-zA-Z0-9:_]*)?\[(?P<attribute>[\w-]+)(?P<operator>[=~\|\^\$\*]?)z=?"?(?P<value>[^\]"]*)"?\]$c C s. | j ||�}t|t�s t|t�r*dj|�}|S )z�Force an attribute value into a string representation. A multi-valued attribute will be converted into a space-separated stirng. � )rT r. rm �tuple�join)r �value�defaultr r r �_attr_value_as_string� s z!PageElement._attr_value_as_stringc s �s� S � �fdd�}|S d S )Nc s | j �ko� | �S )N)r )rr )�function�tag_namer r �_match� s z1PageElement._tag_name_matches_and.<locals>._matchr )r r� r� r� r )r� r� r �_tag_name_matches_and� s z!PageElement._tag_name_matches_and� c s� |dkr� �fdd�S |dkr0� �fdd�}|S |dkrF� �fdd�S |d kr\� �fd d�S |d krr� �fd d�S |d kr�� �fdd�}|S � fdd�S dS )z�Create a function that performs a CSS selector operation. Takes an operator, attribute and optional value. Returns a function that will return True for elements that match that combination. �=c s | j � ��kS )N)r� )�el)� attributer� r r �<lambda>� s z0PageElement._attribute_checker.<locals>.<lambda>�~c s& | j � g �}t|t�s |j� }�|kS )N)rT r. rm r� )r� �attribute_value)r� r� r r �_includes_value� s z7PageElement._attribute_checker.<locals>._includes_value�^c s | j � d�j��S )Nr� )r� � startswith)r� )r� r� r r r� � s �$c s | j � d�j��S )Nr� )r� �endswith)r� )r� r� r r r� � s �*c s �| j � d�kS )Nr� )r� )r� )r� r� r r r� � s �|c s" | j � d�}|�kp |j�d �S )Nr� �-)r� r� )r� r� )r� r� r r �_is_or_starts_with_dash� s z?PageElement._attribute_checker.<locals>._is_or_starts_with_dashc s | j � �S )N)�has_attr)r� )r� r r r� � s Nr )r �operatorr� r� r� r� r )r� r� r �_attribute_checker� s z PageElement._attribute_checkerc C s | j S )N)r{ )r r r r � nextGenerator� s zPageElement.nextGeneratorc C s | j S )N)r )r r r r �nextSiblingGenerator� s z PageElement.nextSiblingGeneratorc C s | j S )N)r� )r r r r �previousGenerator� s z PageElement.previousGeneratorc C s | j S )N)r� )r r r r �previousSiblingGenerator� s z$PageElement.previousSiblingGeneratorc C s | j S )N)r� )r r r r �parentGenerator� s zPageElement.parentGenerator)rI )NNNNN)TTzs^(?P<tag>[a-zA-Z0-9][-.a-zA-Z0-9:_]*)?\[(?P<attribute>[\w-]+)(?P<operator>[=~\|\^\$\*]?)=?"?(?P<value>[^\]"]*)"?\]$)N)r� )Mr r r r rA rD rC rU rF rE rS rO r rR rJ r\ r Z nextSiblingZpreviousSiblingra Z replaceWithre Zreplace_with_childrenZreplaceWithChildrenrh r_ ri Z_lastRecursiveChildr` rf rs rt ry ZfindNextrv Z findAllNextr~ ZfindNextSiblingr} ZfindNextSiblingsZfetchNextSiblingsr� Z findPreviousr� ZfindAllPreviousZ fetchPreviousr� ZfindPreviousSiblingr� ZfindPreviousSiblingsZfetchPreviousSiblingsr� Z findParentr� Z findParentsZ fetchParentsr� r� ru rz r{ r r� r� r� r) r* � tag_name_re�attribselect_rer� r� r� r� r� r� r� r� r r r r rG � s� J 6 +rG c @ s\ e Zd ZdZdZdZdd� Zdd� Zdd� Zd d � Z dd d �Z e dd� �Z e j dd� �Z dS )r/ r� Nc C s2 t |t�rtj| |�}ntj| |t�}|j� |S )a- Create a new NavigableString. When unpickling a NavigableString, this method is called with the string in DEFAULT_OUTPUT_ENCODING. That encoding needs to be passed in to the superclass's __new__ or the superclass won't know how to handle non-ASCII characters. )r. r r �DEFAULT_OUTPUT_ENCODINGr\ )r r� �ur r r r � s zNavigableString.__new__c C s t | �| �S )z�A copy of a NavigableString has the same contents and class as the original, but it is not connected to the parse tree. )�type)r r r r �__copy__ s zNavigableString.__copy__c C s t | �fS )N)r )r r r r �__getnewargs__ s z NavigableString.__getnewargs__c C s$ |dkr | S t d| jj|f ��dS )z�text.string gives you text. This is for backwards compatibility for Navigable*String, but for CData* it lets you get the string without the CData wrapper.r� z!'%s' object has no attribute '%s'N)�AttributeError� __class__r )r r r r r � __getattr__ s zNavigableString.__getattr__rI c C s | j | |�}| j| | j S )N)rO �PREFIX�SUFFIX)r rM rN r r r � output_ready s z NavigableString.output_readyc C s d S )Nr )r r r r r # s zNavigableString.namec C s t d��d S )Nz)A NavigableString cannot be given a name.)r� )r r r r r r ' s )rI )r r r r� r� rQ r r� r� r� r� r r r r r r r r/ � s r/ c @ s e Zd ZdZddd�ZdS )�PreformattedStringz�A NavigableString not subject to the normal formatting rules. The string will be passed into the formatter (to trigger side effects), but the return value will be ignored. rI c C s | j | |� | j| | j S )zUCData strings are passed into the formatter. But the return value is ignored.)rO r� r� )r rM r r r r� 2 s z PreformattedString.output_readyN)rI )r r r r r� r r r r r� + s r� c @ s e Zd ZdZdZdS )�CDataz <![CDATA[z]]>N)r r r r� r� r r r r r� 8 s r� c @ s e Zd ZdZdZdZdS )�ProcessingInstructionz A SGML processing instruction.z<?�>N)r r r r r� r� r r r r r� = s r� c @ s e Zd ZdZdZdZdS )�XMLProcessingInstructionz An XML processing instruction.z<?z?>N)r r r r r� r� r r r r r� C s r� c @ s e Zd ZdZdZdS )�Commentz<!--z-->N)r r r r� r� r r r r r� H s r� c @ s e Zd ZdZdZdS )� Declarationz<?z?>N)r r r r� r� r r r r r� N s r� c @ s e Zd Zedd� �ZdZdZdS )�Doctypec C sN |pd}|d k r2|d| 7 }|d k rF|d| 7 }n|d k rF|d| 7 }t |�S )Nr� z PUBLIC "%s"z "%s"z SYSTEM "%s")r� )r r Zpub_idZ system_idr� r r r �for_name_and_idsU s zDoctype.for_name_and_idsz <!DOCTYPE z> N)r r r r9 r� r� r� r r r r r� S s r� c @ sH e Zd ZdZdhdd�Zed�Zdd� Zedd � �Z e Z ed d � �Z e j d d � �Z d e effdd�Zee�Zedd� �Zdd e effdd�ZeZee�Zdd� Zdidd�Zdd� Zdjdd �Zdkd d �Zd d � Zd!d"� Zd#d$� Z d%d&� Z d'd(� Z d)d*� Z d+d,� Z d-d.� Z!d/d0� Z"d1d2� Z#d3d4� Z$d5d6� Z%d7d8� Z&dld:d;�Z'd<d=� Z(d>d?� Z)e*�rTe( Z)Z'e+dd@dAfdBdC�Z,dDdE� Z-de+d@fdFdG�Z.dmdHdI�Z/de+d@fdJdK�Z0de+d@fdLdM�Z1e+d dNfdOdP�Z2di dQdfdRdS�Z3e3Z4di dQddfdTdU�Z5e5Z6e5Z7edVdW� �Z8edXdY� �Z9dZd[d\gZ:d Z;e<j=d]�Z>d^d_� Z?dnd`da�Z@dbdc� ZAddde� ZBdfdg� ZCdS )ork z=Represents a found HTML tag with its attributes and contents.Nc C s� |dkrd| _ n|j| _ |dkr(td��|| _|| _|| _|dk rJ|j} n| rTg } ntj} | | _|dkrni }n4|r�|dk r�|jr�|j | j|�}q�t |�}nt |�}|r�|j | _ n| | _ || _ g | _| j||� d| _|dk r�|j| � |j|�| _nd| _dS )zBasic constructor.Nz%No value provided for new tag's name.F)� parser_classr� r] r r r r8 r, Zcdata_list_attributesZ$_replace_cdata_list_attribute_values�dictrP rQ rw r[ r\ �hiddenZset_up_substitutions�can_be_empty_element) r �parser�builderr r r rw r0 r� rP r8 r r r �__init__i sB z Tag.__init__r� c C sh t | �d| j| j| j| j| j| jd�}x dD ]}t||t| |�� q,W x | j D ]}|j |j � � qNW |S )z�A copy of a Tag is a new Tag, unconnected to the parse tree. Its contents are a copy of the old Tag's contents. N)rP r� r� )r� r� ) r� r� r r r rw rR r r r[ rf r� )r Zcloner rd r r r r� � s z Tag.__copy__c C s t | j�dko| jS )a7 Is this tag an empty-element tag? (aka a self-closing tag) A tag that has contents is never an empty-element tag. A tag that has no contents may or may not be an empty-element tag. It depends on the builder used to create the tag. If the builder has a designated list of empty-element tags, then only a tag whose name shows up in that list is considered an empty-element tag. If the builder has no designated list of empty-element tags, then any tag with no contents is an empty-element tag. r )ro r[ r� )r r r r �is_empty_element� s zTag.is_empty_elementc C s0 t | j�dkrdS | jd }t|t�r*|S |jS )aq Convenience property to get the single string within this tag. :Return: If this tag has a single string child, return value is that string. If this tag has no children, or more than one child, return value is None. If this tag has one child tag, return value is the 'string' attribute of the child tag, recursively. r% Nr )ro r[ r. r/ r� )r rd r r r r� � s z Tag.stringc C s | j � | j|j|�� d S )N)�clearrf r� )r r� r r r r� � s Fc c s^ xX| j D ]N}|dkr t|t� s|dk r6t|�|kr6q|rP|j� }t|�dkrPq|V qW dS )z�Yield all strings of certain classes, possibly stripping them. By default, yields only NavigableString and CData objects. So no comments, processing instructions, etc. Nr )� descendantsr. r/ r� �stripro )r r� �typesZ descendantr r r � _all_strings� s zTag._all_stringsc c s x| j d�D ] }|V q W d S )NT)r� )r r� r r r �stripped_strings� s zTag.stripped_stringsr� c C s |j dd� | j||d�D ��S )zP Get all child strings, concatenated using the given separator. c S s g | ]}|�qS r r )r� rL r r r � <listcomp>� s z Tag.get_text.<locals>.<listcomp>)r� )r� r� )r Z separatorr� r� r r r �get_text� s z Tag.get_textc C s8 | j � | }x&|dk r2|j}|jj� g |_|}qW dS )z/Recursively destroys the contents of this tree.N)r_ rX �__dict__r� r[ )r r� r� r r r � decompose� s z Tag.decomposec C s^ |r:xT| j dd� D ] }t|t�r,|j� q|j� qW n x | j dd� D ] }|j� qJW dS )zP Extract all children. If decompose is True, decompose instead. N)r[ r. rk r� r_ )r r� r� r r r r� s z Tag.clearc C s0 x"t | j�D ]\}}||kr |S q W td��dS )z� Find the index of a child by identity, not value. Avoids issues with tag.contents.index(element) getting the index of equal elements. z Tag.index: element not in tagN)� enumerater[ r] )r r� r� rd r r r r^ s z Tag.indexc C s | j j||�S )z�Returns the value of the 'key' attribute for the tag, or the value given for 'default' if it doesn't have that attribute.)rw rT )r �keyr� r r r rT s zTag.getc C s | j ||�}t|t�s |g}|S )z-The same as get(), but always returns a list.)rT r. rm )r r� r� r� r r r �get_attribute_list s zTag.get_attribute_listc C s || j kS )N)rw )r r� r r r r� & s z Tag.has_attrc C s t | �j� S )N)r �__hash__)r r r r r� ) s z Tag.__hash__c C s | j | S )zqtag[key] returns the value of the 'key' attribute for the tag, and throws an exception if it's not there.)rw )r r� r r r � __getitem__, s zTag.__getitem__c C s t | j�S )z0Iterating over a tag iterates over its contents.)�iterr[ )r r r r �__iter__1 s z Tag.__iter__c C s t | j�S )z:The length of a tag is the length of its list of contents.)ro r[ )r r r r �__len__5 s z Tag.__len__c C s || j kS )N)r[ )r �xr r r � __contains__9 s zTag.__contains__c C s dS )z-A tag is non-None even if it has no contents.Tr )r r r r �__bool__< s z Tag.__bool__c C s || j |< dS )zKSetting tag[key] sets the value of the 'key' attribute for the tag.N)rw )r r� r� r r r � __setitem__@ s zTag.__setitem__c C s | j j|d� dS )z;Deleting tag[key] deletes all 'key' attributes for the tag.N)rw �pop)r r� r r r � __delitem__E s zTag.__delitem__c O s | j ||�S )z�Calling a tag like a function is the same as calling its find_all() method. Eg. tag('a') returns a list of all the A tags found within this tag.)�find_all)r r= r> r r r �__call__I s z Tag.__call__c C sv t |�dkr@|jd�r@|d d� }tjdt|d� � | j|�S |jd� r`|dk r`| j|�S td| j|f ��d S ) N� rk z�.%(name)sTag is deprecated, use .find("%(name)s") instead. If you really were looking for a tag called %(name)sTag, use .find("%(name)sTag"))r �__r[ z!'%s' object has no attribute '%s'���) ro r� �warnings�warnr� �findr� r� r� )r rr r� r r r r� O s zTag.__getattr__c C s� | |kr dS t |d� sXt |d� sXt |d� sX| j|jksX| j|jksXt| �t|�kr\dS x(t| j�D ]\}}||j| krhdS qhW dS )zReturns true iff this tag has the same name, the same attributes, and the same contents (recursively) as the given tag.Tr rw r[ F)rp r rw ro r� r[ )r �otherr� Zmy_childr r r �__eq__` s z Tag.__eq__c C s | |k S )zZReturns true iff this tag is not identical to the other tag, as defined in __eq__.r )r r� r r r �__ne__q s z Tag.__ne__�unicode-escapec C s t r | j� S | j|�S dS )z Renders this tag as a string.N)�PY3K�decoder )r r r r r �__repr__v s z Tag.__repr__c C s | j � S )N)r )r r r r � __unicode__� s zTag.__unicode__c C s t r | j� S | j� S d S )N)r r r )r r r r �__str__� s z Tag.__str__rI �xmlcharrefreplacec C s | j |||�}|j||�S )N)r r )r r � indent_levelrM �errorsr� r r r r � s z Tag.encodec C s |dk o| j | jkS )z"Should this tag be pretty-printed?N)r r8 )r r r r r �_should_pretty_print� s zTag._should_pretty_printc C sR t |t� r t|� r | j|�}g }| jr�x�t| jj� �D ]�\}}|dkrP|}nrt |t�sdt |t�rpdj |�}n0t |t �s�t |�}n t |t �r�|dk r�|j |�}| j ||�}t |�d tj|� }|j|� q:W d} d} d} | jr�| jd } | j�rd} t |t��r |j�p| } nd| | jf } | j|�} d} d}|dk �rHd|d }| �r\|} |d }nd}| j|||�}| j�r||}n�g }d}|�r�ddj |� }|dk �r�|j|� |jd| | j|| f � | �r�|jd � |j|� | �r|�r|d d k�r|jd � | �r| �r|j| � |j| � |dk �rD| �rD| j�rD|jd � dj |�}|S ) a� Returns a Unicode representation of this tag and its contents. :param eventual_encoding: The tag is destined to be encoded into this encoding. This method is _not_ responsible for performing that encoding. This information is passed in so that it can be substituted in if the document contains a <META> tag that mentions the document's encoding. Nr� r� r� r z</%s%s>r% z <%s%s%s%s>� rV )r. r: rK rJ rw �sorted�itemsrm r� r� r r r rO r Zquoted_attribute_valuerf r r� r@ r r �decode_contentsr� rY )r r �eventual_encodingrM rw r� �valZdecodedrx �closeZcloseTagr � pretty_printZspaceZ indent_spaceZindent_contentsr[ rL Zattribute_stringr r r r � sx z Tag.decodec C s* |d kr| j d|d�S | j|d|d�S d S )NT)rM )r r )r r rM r r r �prettify� s z Tag.prettifyc C s� t |t� r t|� r | j|�}|dk }g }x�| D ]�}d}t |t�rP|j|�}n t |t�rn|j|j|||�� |r�|r�| j dk r�|j � }|r2|r�| j dk r�|jd|d � |j|� |r2| j dk r2|jd� q2W dj |�S )av Renders the contents of this tag as a Unicode string. :param indent_level: Each line of the rendering will be indented this many spaces. :param eventual_encoding: The tag is destined to be encoded into this encoding. This method is _not_ responsible for performing that encoding. This information is passed in so that it can be substituted in if the document contains a <META> tag that mentions the document's encoding. :param formatter: The output formatter responsible for converting entities to Unicode characters. Nr- r� r% r r� ) r. r: rK rJ r/ r� rk rf r r r� r� )r r r rM r rL �crx r r r r s( zTag.decode_contentsc C s | j |||�}|j|�S )ac Renders the contents of this tag as a bytestring. :param indent_level: Each line of the rendering will be indented this many spaces. :param eventual_encoding: The bytestring will be in this encoding. :param formatter: The output formatter responsible for converting entities to Unicode characters. )r r )r r r rM r[ r r r �encode_contents+ s zTag.encode_contentsr c C s |sd }| j ||d�S )N)r r )r )r r Z prettyPrintZ indentLevelr r r �renderContents= s zTag.renderContentsTc K s* d}| j ||||df|�}|r&|d }|S )zLReturn only the first child of this Tag matching the given criteria.Nr% r )r� )r r rw � recursiverx r> r� r� r r r r� F s zTag.findc K s& | j }|s| j}| j|||||f|�S )a� Extracts a list of Tag objects that match the given criteria. You can specify the name of the Tag and any attributes you want the Tag to have. The value of a key-value pair in the 'attrs' map can be a string, a list of strings, a regular expression object, or a callable that takes a string and returns whether or not the string matches for some custom definition of 'matches'. The same is true of the tag name.)r� �childrenrz )r r rw r rx r| r> r� r r r r� Q s z Tag.find_allc C s t | j�S )N)r� r[ )r r r r r e s z Tag.childrenc c s@ t | j�sd S | j� j}| jd }x||k r:|V |j}q$W d S )Nr )ro r[ ri rX )r ZstopNodeZcurrentr r r r� j s zTag.descendantsr� �+r� z "[^"]*:[^"]*"c C s | j |dd�}|r|d S dS )z9Perform a CSS selection operation on the current element.r% )r| r N)�select)r �selectorr� r r r � select_oney s zTag.select_onec ) s: d|kr�g }dd� |j d�D �}t� }t� }x�|D ]~}|dkrJtd| ��||krTq2|j|� �j||d�} x0| D ](} t| �} | |krr|j| � |j| � qrW |r2t|�|kr2P q2W |S tj |�} �g} | d3 �j kr�td| d4 ���j r�t d | � �x�t | �D �]�\}}g }tg �}| |d �j k�rB�j �rt d � �q�j �rVt d | � d �d �d }�j j|�}|d k �r�|j� \�}}}�j|||�}�n�d |k�r�|j d d�\���fdd�}|}�nxd|k�r|j dd�\�}t|j d��� � fdd�}|}�n<d|k�rԈjj|� �r�|j dd�\�}�dk�r@td��tjd|�}g } |d k�rd|} d } n |j� \} } | dk�r�y t| �} W n td��Y nX | dk �r�td��G dd� dt�} | | �j}ntd��nl|d k�r�n`|d k�r�d d � �nL|d k�rd!d � �n8|d"k�r d#d$� } | �n �jj|��r4|�n td%| ����rh| |d ����fd&d'�}!|!}"nV|d k�r��j �r���r�d(}#n�}#t d)|# � �j �r��fd*d+�}$|$}"nd,d � }"n|}"d-}%�x| D �]}&�j �r�t d.|&jt|&j�f � x�|"|&�D ]�} t | t ��s�q���r$| j�k�r$�q�|d k �rTy || �}'W n t k �rR P Y nX |d k�sd|'�r��j �r�t d/| jt| j�f � t| �|k�r�|j| � |jt| �� n �j �r�t d0| jt| j�f � �q�W �q�W |} �qW |�rt| �|k�r| d |� } �j �r6t d1� x"| D ]}(t d2|(j|(jf � �qW | S )5z9Perform a CSS selection operation on the current element.�,c S s g | ] }|j � �qS r )r� )r� r� r r r r� � s z Tag.select.<locals>.<listcomp>r� z"Invalid group selection syntax: %s)r| r% z-Final combinator "%s" is missing an argument.zRunning CSS selector "%s"z0 Token was consumed by the previous combinator.z Considering token "%s"N�#c s | j dd �� kS )N�id)rT )rr )�tag_idr r � id_matches� s z Tag.select.<locals>.id_matches�.c s � j | jdg ��S )N�class)�issubsetrT )� candidate)�classesr r � classes_match� s z!Tag.select.<locals>.classes_matchr z0A pseudo-class must be prefixed with a tag name.z ([a-zA-Z\d-]+)\(([a-zA-Z\d]+)\)z nth-of-typezMOnly numeric values are currently supported for the nth-of-type pseudo-class.z2nth-of-type pseudo-class value must be at least 1.c @ s e Zd Zdd� Zdd� ZdS )zTag.select.<locals>.Counterc S s d| _ || _d S )Nr )r� � destination)r r' r r r r� � s z$Tag.select.<locals>.Counter.__init__c S s& | j d7 _ | j | jkr dS dS d S )Nr% TF)r� r' )r rr r r r �nth_child_of_type� s z-Tag.select.<locals>.Counter.nth_child_of_typeN)r r r r� r( r r r r �Counter� s r) z?Only the following pseudo-classes are implemented: nth-of-type.r� r� c S s | j S )N)r )rr r r r r� s z Tag.select.<locals>.<lambda>r� c S s | j S )N)r )rr r r r r� s r c s s | j d�V d S )NT)r~ )rr r r r �next_tag_sibling s z$Tag.select.<locals>.next_tag_siblingz)Unsupported or invalid CSS selector: "%s"c 3 st �j r(td� | j| jf � tdd � x4| j� ��D ]$}�j rTtd|j|jf � |V q6W �j rptdd � d S )Nz- Calling select("%s") recursively on %s %sr� �( z,(Recursive select picked up candidate %s %s))� _select_debug�printr rw r )rr r� )� next_token� recursive_candidate_generatorr r r �recursive_select s z$Tag.select.<locals>.recursive_selectz[any]z- Default candidate generator, tag name="%s"c 3 s8 x2| j D ](}t|t�sq� r*|j� k r*q|V qW d S )N)r� r. rk r )rr rd )r� r r �default_candidate_generator7 s z/Tag.select.<locals>.default_candidate_generatorc S s | j S )N)r� )rr r r r r� @ s r z( Running candidate generator on %s %sz SUCCESS %s %sz FAILURE %s %szFinal verdict:z %s %srV rV ) r� r7 r] �addr r rf ro �shlex�_selector_combinatorsr, r- r� r� r$ �groupsr� � quoted_colonr# r) �intr< �objectr( r� r �reprrw r. rk r� ))r r Z_candidate_generatorr| �contextZ selectorsZused_selectorsZselected_object_idsZpartial_selectorZ candidatesr$ Z object_id�tokensZcurrent_contextr^ �tokenZ new_contextZnew_context_idsZchecker�mr� r� r� r �klassr& ZpseudoZpseudo_attributesr� Z pseudo_typeZ pseudo_valuer) r* r0 Z_use_candidate_generatorZcheckr1 r� rr r� r� r )r% r. r/ r r r� r r � s $ z Tag.selectc C s | j S )N)r )r r r r �childGeneratork s zTag.childGeneratorc C s | j S )N)r� )r r r r �recursiveChildGeneratorn s zTag.recursiveChildGeneratorc C s t jd| � | j|�S )z�This was kind of misleading because has_key() (attributes) was different from __in__ (contents). has_key() is gone in Python 3, anyway.z2has_key is deprecated. Use has_attr("%s") instead.)r� r� r� )r r� r r r �has_keyq s z Tag.has_key) NNNNNNNNN)F)N)N)r )NrI )NN)Dr r r r r� r Z parserClassr� r r� Z isSelfClosingr� r r/ r� r� Zstringsr� r� ZgetTextrx r� r� r^ rT r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r r� r r r r r r r r� Z findChildr� ZfindAllZ findChildrenr r� r4 r, r) r* r6 r r r? r@ rA r r r r rk e s� 3 \ ) lrk c @ sT e Zd ZdZdi dfdd�Zdd� Zdd� Zdi fd d �ZeZd d � Z dd d�Z dS )r� zMEncapsulates a number of ways of matching a markup element (tag or text).Nc K s� | j |�| _t|t�s"||d< d }d|kr<|d |d< |d= |r\|rX|j� }|j|� n|}i }x&t|j� �D ]\}}| j |�||< qnW || _| j |�| _ d S )Nr" Zclass_) �_normalize_search_valuer r. r� �copy�updaterm r rw rx )r r rw rx r> Znormalized_attrsr� r� r r r r� ~ s" zSoupStrainer.__init__c C s� t |t�s.t|�s.t|d�s.t |t�s.|d kr2|S t |t�rF|jd�S t|d�r�g }xJ|D ]B}t|d�r�t |t� r�t |t� r�|j|� qZ|j| j|�� qZW |S tt|��S )Nr$ �utf8r� ) r. r rK rp �bool�bytesr rf rB )r r� Z new_value�vr r r rB � s z$SoupStrainer._normalize_search_valuec C s | j r | j S d| j| jf S d S )Nz%s|%s)rx r rw )r r r r r � s zSoupStrainer.__str__c C s$ d }d }t |t�r|}|}t | jt�o0t |t� }| j sd|sd|rP| j|| j�sd| r�| j|| j�r�|rv| j||�}nrd}d }xht| jj� �D ]V\}} |s�t|d�r�|}n i }x|D ]\} } | || < q�W |j |�} | j| | �s�d}P q�W |r�|r�|}n|}|�r | j �r | j|j | j � �r d }|S )NTrT F) r. rk r r �_matchesrm rw r rp rT rx r� ) r Z markup_nameZ markup_attrsr� �markupZcall_function_with_tag_datar$ Zmarkup_attr_mapr � match_against�krH Z attr_valuer r r � search_tag� sB "zSoupStrainer.search_tagc C s� d }t |d�rHt|ttf� rHx�|D ] }t|t�r$| j|�r$|}P q$W npt|t�rr| j sf| jsf| jr�| j |�}nFt|t�s�t|t�r�| j r�| j r�| j || j�r�|}nt d|j ��|S )Nr� z&I don't know how to match against a %s) rp r. rk r r/ r# rx r rw rM rI � Exceptionr� )r rJ r� r� r r r r# � s" zSoupStrainer.searchc C s� d}t |t�st |t�rPx |D ]}| j||�r dS q W | jdj|�|�rLdS dS |dkr`|d k S t |t�rr||�S |}t |t�r�|j}| j|�}|d kr�| S t |d�o�t |t � �r|s�t � }xL|D ]@}|j r�|}nt |�}||kr�q�q�|j|� | j|||�r�dS q�W dS d}| �r.t |t ��r.||k}| �rLt |d��rL|j|�S | �r�t |t��r�|j�r�| j|jd |j |�S |S )NFTr� r� r# r )r. rm r� rI r� r rk r rB rp r r7 r� r r2 r# r ) r rJ rK Z already_triedr� �itemZoriginal_markupr� r$ r r r rI sV zSoupStrainer._matches)N) r r r r r� rB r rM Z searchTagr# rI r r r r r� z s 'r� c s, e Zd ZdZf f� fdd� Zdd� Z� ZS )r� zTA ResultSet is just a list that keeps track of the SoupStrainer that created it.c s t t| �j|� || _d S )N)�superr� r� �source)r rQ r� )r� r r r� V s zResultSet.__init__c C s t d| ��d S )Nz�ResultSet object has no attribute '%s'. You're probably treating a list of items like a single item. Did you call find_all() when you meant to call find()?)r� )r r� r r r r� Z s zResultSet.__getattr__)r r r r r� r� � __classcell__r r )r� r r� S s r� ),Z __license__�collections.abcr � ImportError�e� collectionsr) r3 �sysr� Z bs4.dammitr r� � version_infor r* Z whitespace_rer r r r r r! r, r8 r: rA rC rD rE rF rG r/ r� r� r� r� r� r� r� rk r� rm r� r r r r �<module> s^ % R9 Z __pycache__/__init__.cpython-36.pyc 0000644 00000037377 15246412105 0013162 0 ustar 00 3 (6]XY � @ s d Z dZdZdZdZdgZddlZddlZddlZddl Z ddl Z dd l m Z m Z dd lmZ dd lmZmZmZmZmZmZmZmZmZmZmZ d d k G dd� de�Z e Z e Z G dd� de �Z G dd� de �Z!G dd� de"�Z#e$dk�r ddlZe ej%�Z&e'e&j(� � dS )aH Beautiful Soup Elixir and Tonic "The Screen-Scraper's Friend" http://www.crummy.com/software/BeautifulSoup/ Beautiful Soup uses a pluggable XML or HTML parser to parse a (possibly invalid) document into a tree representation. Beautiful Soup provides methods and Pythonic idioms that make it easy to navigate, search, and modify the parse tree. Beautiful Soup works with Python 2.7 and up. It works better if lxml and/or html5lib is installed. For more than you ever wanted to know about Beautiful Soup, see the documentation: http://www.crummy.com/software/BeautifulSoup/bs4/doc/ z*Leonard Richardson (leonardr@segfault.org)z4.6.3z*Copyright (c) 2004-2018 Leonard RichardsonZMIT� BeautifulSoup� N� )�builder_registry�ParserRejectedMarkup)� UnicodeDammit) �CData�Comment�DEFAULT_OUTPUT_ENCODING� Declaration�Doctype�NavigableString� PageElement�ProcessingInstruction� ResultSet� SoupStrainer�Tagz`You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work.zuYou need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).c s� e Zd ZdZdZddgZdZdZd2d d �Zd d � Z d d� Z e dd� �Z dd� Z dd� Zddi fdd�Zefdd�Zdd� Zdd � Zd d � Zd d � Zefd!d"�Zd3d#d$�Zd4d&d'�Zd(d)� Zd5d*d+�Zd,d-� Zd.e d/f� fd0d1� Z � Z S )6r a This class defines the basic interface called by the tree builders. These methods will be called by the parser: reset() feed(markup) The tree builder may call these methods from its feed() implementation: handle_starttag(name, attrs) # See note about return value handle_endtag(name) handle_data(data) # Appends to the current data node endData(containerClass=NavigableString) # Ends the current data node No matter how complicated the underlying parser is, you should be able to build a tree using 'start tag' events, 'end tag' events, 'data' events, and "done with data" events. If you encounter an empty-element tag (aka a self-closing tag, like HTML's <br> tag), call handle_starttag and then handle_endtag. z [document]ZhtmlZfastz a� No parser was explicitly specified, so I'm using the best available %(markup_type)s parser for this system ("%(parser)s"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently. The code that caused this warning is on line %(line_number)s of the file %(filename)s. To get rid of this warning, pass the additional argument 'features="%(parser)s"' to the BeautifulSoup constructor. � Nc " s� d� krt jd� d� kr*� d= t jd� d� krB� d= t jd� d� krZ� d= t jd� d � krr� d = t jd � � fd d �}|p�|d d�}|p�|dd�}|r�t|t�r�t jd� d}t� �dkr�t� j� �j� } td| ��|dk�r|} t|t�r�|g}|dk�st|�dk�r| j }t j |� } | dk�r@t ddj |� ��| � }| |jk�pZ| |jk�s|j�rld} nd} d} ytjd�} W n tk �r� Y nX | �r�| j}| j}n tj}d}|jd�}|�r�|j� }|jd(��r�|dd)� }|�rt|||j| d �}t j| j| d d � || _ |j| _| j| _ | | j _ || _ t |d ��rN|j!� }n�t|�d!k�r t|t"��rrd"|k�s�t|t��r d#|k�r t|t��r�t#j$j% �r�|j&d$�}n|}d%}yt#j$j'|�}W n$ t(k �r� } zW Y dd}~X nX |�rt|t��r|j&d$�}t jd&| � | j)|� xZ| j j*|||d'�D ]D\| _+| _,| _-| _.| j/� y| j0� P W n t1k �rr Y nX �q2W d| _+d| j _ dS )*a_ Constructor. :param markup: A string or a file-like object representing markup to be parsed. :param features: Desirable features of the parser to be used. This may be the name of a specific parser ("lxml", "lxml-xml", "html.parser", or "html5lib") or it may be the type of markup to be used ("html", "html5", "xml"). It's recommended that you name a specific parser, so that Beautiful Soup gives you the same results across platforms and virtual environments. :param builder: A specific TreeBuilder to use instead of looking one up based on `features`. You shouldn't need to use this. :param parse_only: A SoupStrainer. Only parts of the document matching the SoupStrainer will be considered. This is useful when parsing part of a document that would otherwise be too large to fit into memory. :param from_encoding: A string indicating the encoding of the document to be parsed. Pass this in if Beautiful Soup is guessing wrongly about the document's encoding. :param exclude_encodings: A list of strings indicating encodings known to be wrong. Pass this in if you don't know the document's encoding but you know Beautiful Soup's guess is wrong. :param kwargs: For backwards compatibility purposes, the constructor accepts certain keyword arguments used in Beautiful Soup 3. None of these arguments do anything in Beautiful Soup 4 and there's no need to actually pass keyword arguments into the constructor. ZconvertEntitiesz�BS4 does not respect the convertEntities argument to the BeautifulSoup constructor. Entities are always converted to Unicode characters.Z markupMassagez�BS4 does not respect the markupMassage argument to the BeautifulSoup constructor. The tree builder is responsible for any necessary markup massage.Z smartQuotesToz�BS4 does not respect the smartQuotesTo argument to the BeautifulSoup constructor. Smart quotes are always converted to Unicode characters.ZselfClosingTagsz�BS4 does not respect the selfClosingTags argument to the BeautifulSoup constructor. The tree builder is responsible for understanding self-closing tags.ZisHTMLz�BS4 does not respect the isHTML argument to the BeautifulSoup constructor. Suggest you use features='lxml' for HTML and features='lxml-xml' for XML.c s0 | � kr,t jd| |f � � | }� | = |S d S )NzLThe "%s" argument to the BeautifulSoup constructor has been renamed to "%s.")�warnings�warn)Zold_name�new_name�value)�kwargs� � /usr/lib/python3.6/__init__.py�deprecated_argument� s z3BeautifulSoup.__init__.<locals>.deprecated_argumentZparseOnlyThese� parse_onlyZ fromEncoding� from_encodingzlYou provided Unicode markup but also provided a value for from_encoding. Your from_encoding will be ignored.Nr z2__init__() got an unexpected keyword argument '%s'zjCouldn't find a tree builder with the features you requested: %s. Do you need to install a parser library?�,ZXMLZHTMLr �__file__�.pyc�.pyo)�filename� line_number�parser� markup_type� )� stacklevel�read� � <�<�utf8Fzw"%s" looks like a filename, not markup. You should probably open this file and pass the filehandle into Beautiful Soup.)�exclude_encodings)r r ���)2r r � isinstance�str�len�list�keys�pop� TypeError�DEFAULT_BUILDER_FEATURESr �lookup�FeatureNotFound�join�NAMEZALTERNATE_NAMES�is_xml�sys� _getframe� ValueError� f_globals�f_lineno�__dict__�get�lower�endswith�dict�NO_PARSER_SPECIFIED_WARNING�builderZ known_xml�soupr �hasattrr' �bytes�os�path�supports_unicode_filenames�encode�exists� Exception�_check_markup_is_urlZprepare_markup�markup�original_encodingZdeclared_html_encodingZ contains_replacement_characters�reset�_feedr )�selfrQ �featuresrF r r r, r r �argZoriginal_featuresZ builder_classr$ Zcaller�globalsr" r! Zfnl�valuesZpossible_filename�is_file�er )r r �__init__X s� ' zBeautifulSoup.__init__c C s& t | �| jd�| jdd�}| j|_|S )Nzutf-8)rF r )�typerM rF rR )rU �copyr r r �__copy__$ s zBeautifulSoup.__copy__c C s( t | j�}d|kr$| jj r$d |d< |S )NrF )rD r@ rF Z picklable)rU �dr r r � __getstate__0 s zBeautifulSoup.__getstate__c sx t � t�rd}d }nt � t�r(d}d}ndS t� fdd �|D ��rt|� krtt � t�rb� jd d �}n� }tjd | � dS )z� Check if markup looks like it's actually a url and raise a warning if so. Markup can be unicode or str (py2) / bytes (py3). � � http:� https:� �http:�https:Nc 3 s | ]}� j |�V qd S )N)� startswith)�.0�prefix)rQ r r � <genexpr>F s z5BeautifulSoup._check_markup_is_url.<locals>.<genexpr>zutf-8�replacez�"%s" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client like requests to get the document behind the URL, and feed that document to Beautiful Soup.)rc rd )rf rg )r. rI r/ �any�decoder r )rQ ZspaceZcant_start_withZdecoded_markupr )rQ r rP 7 s z"BeautifulSoup._check_markup_is_urlc C s@ | j j� | j j| j� | j� x| jj| jkr:| j� q"W d S )N) rF rS ZfeedrQ �endData� currentTag�name� ROOT_TAG_NAME�popTag)rU r r r rT S s zBeautifulSoup._feedc C sJ t j| | | j| j� d| _| jj� g | _d | _g | _g | _ | j | � d S )Nr ) r r\ rF rr ZhiddenrS � current_datarp �tagStack� preserve_whitespace_tag_stack�pushTag)rU r r r rS ] s zBeautifulSoup.resetc K s |j |� td| j||||�S )z+Create a new tag associated with this soup.N)�updater rF )rU rq � namespace�nsprefix�attrsZkwattrsr r r �new_tagg s zBeautifulSoup.new_tagc C s ||�S )z7Create a new NavigableString associated with this soup.r )rU �s�subclassr r r � new_stringl s zBeautifulSoup.new_stringc C s t d��d S )Nz4BeautifulSoup objects don't support insert_before().)�NotImplementedError)rU � successorr r r � insert_beforep s zBeautifulSoup.insert_beforec C s t d��d S )Nz3BeautifulSoup objects don't support insert_after().)r� )rU r� r r r � insert_afters s zBeautifulSoup.insert_afterc C s@ | j j� }| jr(|| jd kr(| jj� | j r:| j d | _| jS )Nr r- r- )ru r3 rv rp )rU �tagr r r rs v s zBeautifulSoup.popTagc C sJ | j r| j jj|� | jj|� | jd | _ |j| jjkrF| jj|� d S )Nr r- )rp �contents�appendru rq rF Zpreserve_whitespace_tagsrv )rU r� r r r rw s zBeautifulSoup.pushTagc C s� | j r�dj| j �}| jsPd}x |D ]}|| jkr"d}P q"W |rPd|krLd}nd}g | _ | jr�t| j�dkr�| jj s�| jj|� r�d S ||�}| j |� d S )Nr TF� re r ) rt r8 rv � ASCII_SPACESr r0 ru �text�search�object_was_parsed)rU ZcontainerClassrt Z strippable�i�or r r ro � s& zBeautifulSoup.endDatac C sV |p| j }|p| j}d } }}t|t�rF|j}|j}|j}|sF|j}|j|||||� || _|j j |� |j�rRt |j �d }x4|dkr�|j | |kr�P |d8 }q�W t d||f ��|dkr�|}d}n|j |d }}|t |j �d kr�|j}d}n|j |d }}||_|�r ||_||_|�r.||_||_|�r@||_||_|�rR||_dS )z Add an object to the parse tree.Nr r z[Error building tree: supposedly %r was inserted into %r after the fact, but I don't see it!) rp �_most_recent_elementr. r � next_element� next_sibling�previous_sibling�previous_elementZsetupr� r� r0 r= ) rU r� �parentZmost_recent_elementr� r� r� r� �indexr r r r� � sR z BeautifulSoup.object_was_parsedTc C sn || j krdS d}t| j�}xLt|d dd�D ]8}| j| }||jkr^||jkr^|r\| j� }P | j� }q.W |S )z�Pops the tag stack up to and including the most recent instance of the given tag. If inclusivePop is false, pops the tag stack up to but *not* including the most recent instqance of the given tag.Nr r r- )rr r0 ru �rangerq rj rs )rU rq rz Z inclusivePopZmost_recently_popped� stack_sizer� �tr r r � _popToTag� s zBeautifulSoup._popToTagc C s� | j � | jr8t| j�dkr8| jjs4| jj||� r8dS t| | j||||| j| j �}|dkr`|S | j rn|| j _ || _ | j |� |S )a Push a start tag on to the stack. If this method returns None, the tag was rejected by the SoupStrainer. You should proceed as if the tag had not occurred in the document. For instance, if this was a self-closing tag, don't call handle_endtag. r N) ro r r0 ru r� Z search_tagr rF rp r� r� rw )rU rq ry rz r{ r� r r r �handle_starttag� s z BeautifulSoup.handle_starttagc C s | j � | j||� d S )N)ro r� )rU rq rz r r r � handle_endtag s zBeautifulSoup.handle_endtagc C s | j j|� d S )N)rt r� )rU �datar r r � handle_data s zBeautifulSoup.handle_dataFZminimalc sN | j r$d}|dkrd| }d| }nd}|s2d}nd}|tt| �j|||� S )zlReturns a string or Unicode representation of this document. To get Unicode, pass None for encoding.r Nz encoding="%s"z<?xml version="1.0"%s?> r )r: �superr rn )rU Z pretty_printZeventual_encodingZ formatterZ encoding_partrj Z indent_level)� __class__r r rn s zBeautifulSoup.decode)r NNNNN)NN)NT)N) �__name__� __module__� __qualname__�__doc__rr r5 r� rE r\ r_ ra � staticmethodrP rT rS r| r r r� r� rs rw ro r� r� r� r� r� r rn � __classcell__r r )r� r r 8 s8 L 9 c s e Zd ZdZ� fdd�Z� ZS )�BeautifulStoneSoupz&Deprecated interface to an XML parser.c s( d|d< t jd� tt| �j||� d S )NZxmlrV zxThe BeautifulStoneSoup class is deprecated. Instead of using it, pass features="xml" into the BeautifulSoup constructor.)r r r� r� r\ )rU �argsr )r� r r r\ 5 s zBeautifulStoneSoup.__init__)r� r� r� r� r\ r� r r )r� r r� 2 s r� c @ s e Zd ZdS )� StopParsingN)r� r� r� r r r r r� = s r� c @ s e Zd ZdS )r7 N)r� r� r� r r r r r7 @ s r7 �__main__))r� � __author__� __version__Z __copyright__Z __license__�__all__rJ �rer; � tracebackr rF r r Zdammitr �elementr r r r r r r r r r r r Z_sZ_soupr� rO r� r= r7 r� �stdinrG �printZprettifyr r r r �<module> s6 4 z __pycache__/testing.cpython-36.opt-1.pyc 0000644 00000077524 15246412105 0014035 0 ustar 00 3 "6]�~ � @ s� d Z dZddlZddlZddlZddlZddlmZ ddlmZ ddl m Z m Z m Z m Z mZ ddlmZ eZG dd � d ej�ZG d d � d e�ZG d d � d e�ZG dd� de�Zdd� ZdS )zHelper classes for tests.ZMIT� N)�TestCase)� BeautifulSoup)�CharsetMetaAttributeValue�Comment�ContentMetaAttributeValue�Doctype� SoupStrainer)�HTMLParserTreeBuilderc @ s: e Zd Zedd� �Zdd� Zdd� Zd dd �Zd d � ZdS ) �SoupTestc C s t � S )N)�default_builder)�self� r � /usr/lib/python3.6/testing.pyr s zSoupTest.default_builderc K s" |j d| j�}t|fd|i|��S )z*Build a Beautiful Soup object from markup.�builder)�popr r )r �markup�kwargsr r r r �soup s z SoupTest.soupc C s | j j|�S )z[Turn an HTML fragment into a document. The details depend on the builder. )r Ztest_fragment_to_document)r r r r r � document_for% s zSoupTest.document_forNc C s8 | j }t||d�}|d kr |}| j|j� | j|�� d S )N)r )r r � assertEqual�decoder )r Zto_parseZcompare_parsed_tor �objr r r �assertSoupEquals, s zSoupTest.assertSoupEqualsc C s< d}x2|j D ](}|r0| j||j� | j||j� |}q W dS )zyEnsure that next_element and previous_element are properly set for all descendants of the given element. N)Z descendantsr � next_element�previous_element)r �elementZearlier�er r r �assertConnectedness4 s z SoupTest.assertConnectedness)N) �__name__� __module__� __qualname__�propertyr r r r r r r r r r s r c @ s� e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd� Zd d � Zd d � Zd d!� Zd"d#� Zd$d%� Zd&d'� Zd(d)� Zd*d+� Zd,d-� Zd.d/� Zd0d1� Zd2d3� Z d4d5� Z d6d7� Z d8d9� Z d:d;� Z d<d=� Z!d>d?� Z"d@dA� Z#dBdC� Z$dDdE� Z%dFd� ZdGdH� Z&dIdJ� Z'dKdL� Z(dMdN� Z)dOdP� Z*dQdR� Z+dSdT� Z,dUdV� Z-dWdX� Z.dYdZ� Z/d[d\� Z0d]d^� Z1d_d`� Z2dadb� Z3dcdd� Z4dedf� Z5dgdh� Z6didj� Z7dkdl� Z8dmdn� Z9dodp� Z:dqS )r�HTMLTreeBuilderSmokeTestaC A basic test of a treebuilder's competence. Any HTML treebuilder, present or future, should be able to pass these tests. With invalid markup, there's room for interpretation, and different parsers can handle it differently. But with the markup in these tests, there's not much room for interpretation. c C s4 x.dD ]&}| j d�}|j|�}| jd|j� qW dS )zmVerify that all HTML4 and HTML5 empty element (aka void element) tags are handled correctly. �area�base�br�col�embed�hr�img�input�keygen�link�menuitem�meta�param�source�track�wbr�spacer�frame� TN)r# r$ r% r& r' r( r) r* r+ r, r- r. r/ r0 r1 r2 r3 r4 )r �new_tagr �is_empty_element)r �namer r6 r r r �test_empty_element_tagsI s z0HTMLTreeBuilderSmokeTest.test_empty_element_tagsc C sF | j d�}tj|d�}tj|�}| j|jt� | j|j� |j� � d S )Nz <a><b>foo</a>� )r �pickle�dumps�loadsr � __class__r r )r �tree�dumped�loadedr r r �!test_pickle_and_unpickle_identityU s z:HTMLTreeBuilderSmokeTest.test_pickle_and_unpickle_identityc C sf | j |�\}}|jd }| j|jt� | j||� | jt|�dt|�� |� | j|jjd d� dS )z8Assert that a given doctype string is handled correctly.r N�foo)�_document_with_doctype�contentsr r> r �str�len�p)r �doctype_fragmentZ doctype_strr �doctyper r r �assertDoctypeHandled^ s z-HTMLTreeBuilderSmokeTest.assertDoctypeHandledc C s" d| }|d }| j |�}||fS )z5Generate and parse a document with the given doctype.z <!DOCTYPE %s>z <p>foo</p>)r )r rI rJ r r r r r rD l s z/HTMLTreeBuilderSmokeTest._document_with_doctypec C s | j d� | j d� dS )z?Make sure normal, everyday HTML doctypes are handled correctly.�htmlz4html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"N)rK )r r r r �test_normal_doctypess s z-HTMLTreeBuilderSmokeTest.test_normal_doctypesc C s( | j d�}|jd }| jd|j� � d S )Nz <!DOCTYPE>r r5 )r rE r �strip)r r rJ r r r �test_empty_doctypey s z+HTMLTreeBuilderSmokeTest.test_empty_doctypec C s d}| j |� d S )Nznhtml PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd")rK )r rJ r r r � test_public_doctype_with_url~ s z5HTMLTreeBuilderSmokeTest.test_public_doctype_with_urlc C s | j d� d S )Nz$foo SYSTEM "http://www.example.com/")rK )r r r r �test_system_doctype� s z,HTMLTreeBuilderSmokeTest.test_system_doctypec C s | j d� d S )Nz#xsl:stylesheet SYSTEM "htmlent.dtd")rK )r r r r � test_namespaced_system_doctype� s z7HTMLTreeBuilderSmokeTest.test_namespaced_system_doctypec C s | j d� d S )Nz#xsl:stylesheet PUBLIC "htmlent.dtd")rK )r r r r � test_namespaced_public_doctype� s z7HTMLTreeBuilderSmokeTest.test_namespaced_public_doctypec C s4 d}| j |�}| j|jd�jdd�|jdd�� dS )zJA real XHTML document should come out more or less the same as it went in.s� <?xml version="1.0" encoding="utf-8"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"> <html xmlns="http://www.w3.org/1999/xhtml"> <head><title>Hello.</title></head> <body>Goodbye.</body> </html>zutf-8� � N)r r �encode�replace)r r r r r r �test_real_xhtml_document� s z1HTMLTreeBuilderSmokeTest.test_real_xhtml_documentc C s( d}| j |�}| jdt|jd��� dS )ztWhen a namespaced XML document is parsed as HTML it should be treated as HTML with weird tag names. s. <ns1:foo>content</ns1:foo><ns1:foo/><ns2:foo/>r: zns1:fooN)r r rG �find_all)r r r r r r �test_namespaced_html� s z-HTMLTreeBuilderSmokeTest.test_namespaced_htmlc C sB d}| j |�}| j||j� � d}| j |�}| j||jd�� d S )Nz<?PITarget PIContent?>s <?PITarget PIContent?>�utf8)r r r rV )r r r r r r �test_processing_instruction� s z4HTMLTreeBuilderSmokeTest.test_processing_instructionc C s t j| j� dS )z�Make sure you can copy the tree builder. This is important because the builder is part of a BeautifulSoup object, and we want to be able to copy that. N)�copy�deepcopyr )r r r r � test_deepcopy� s z&HTMLTreeBuilderSmokeTest.test_deepcopyc C s. | j d�}| j|jj� | jt|j�d� dS )z�A <p> tag is never designated as an empty-element tag. Even if the markup shows it as an empty-element tag, it shouldn't be presented that way. z<p/>z<p></p>N)r Z assertFalserH r7 r rF )r r r r r �!test_p_tag_is_never_empty_element� s z:HTMLTreeBuilderSmokeTest.test_p_tag_is_never_empty_elementc C s( | j dd� | j dd� | j dd� dS )z�A tag that's not closed by the end of the document should be closed. This applies to all tags except empty-element tags. z<p>z<p></p>z<b>z<b></b>z<br>z<br/>N)r )r r r r � test_unclosed_tags_get_closed� s z6HTMLTreeBuilderSmokeTest.test_unclosed_tags_get_closedc C s. | j d�}| j|jj� | jt|j�d� dS )z�A <br> tag is designated as an empty-element tag. Some parsers treat <br></br> as one <br/> tag, some parsers as two tags, but it should always be an empty-element tag. z <br></br>z<br/>N)r � assertTruer% r7 r rF )r r r r r �#test_br_is_always_empty_element_tag� s z<HTMLTreeBuilderSmokeTest.test_br_is_always_empty_element_tagc C s | j d� d S )Nz<em><em></em></em>)r )r r r r � test_nested_formatting_elements� s z8HTMLTreeBuilderSmokeTest.test_nested_formatting_elementsc C s( d}| j |�}| jd|jd�d � d S )Nz�<!DOCTYPE html> <html> <head> <title>Ordinary HEAD element test</title> </head> <script type="text/javascript"> alert("Help!"); </script> <body> Hello, world! </body> </html> ztext/javascript�script�type)r r �find)r rL r r r r �test_double_head� s z)HTMLTreeBuilderSmokeTest.test_double_headc C sj d}| j |� | j|�}|jdd�}| j|jt� |jdd�}| j||j� |jdd�}| j||j� d S )Nz<p>foo<!--foobar-->baz</p>Zfoobar)�textrC �baz)r r rg r r> r r r )r r r ZcommentrC rj r r r � test_comment� s z%HTMLTreeBuilderSmokeTest.test_commentc C st d}d}| j |� | j |� | j|�}| j|jj� |� | j|�}| j|jj� |� | jd�}| j|jj� d� dS )zWhitespace must be preserved in <pre> and <textarea> tags, even if that would mean not prettifying the markup. z<pre> </pre>z <textarea> woo woo </textarea>z<textarea></textarea>N)r r r ZpreZprettifyZtextarea)r Z pre_markupZtextarea_markupr r r r �-test_preserved_whitespace_in_pre_and_textarea� s zFHTMLTreeBuilderSmokeTest.test_preserved_whitespace_in_pre_and_textareac C s. d}| j |� d}| j |� d}| j |� dS )z+Inline elements can be nested indefinitely.z<b>Inside a B tag</b>z!<p>A <i>nested <b>tag</b></i></p>z/<p>A <a>doubly <i>nested <b>tag</b></i></a></p>N)r )r Zb_tagZ nested_b_tagZdouble_nested_b_tagr r r �test_nested_inline_elements s z4HTMLTreeBuilderSmokeTest.test_nested_inline_elementsc C s6 | j d�}|j}| j|jjjd� | j|jjd� dS )z Block elements can be nested.z*<blockquote><p><b>Foo</b></p></blockquote>ZFooN)r � blockquoter rH �b�string)r r rn r r r � test_nested_block_level_elements s z9HTMLTreeBuilderSmokeTest.test_nested_block_level_elementsc C s d}| j |d� | j d� dS )z$One table can go inside another one.z[<table id="1"><tr><td>Here's another table:<table id="2"><tr><td>foo</td></tr></table></td>zh<table id="1"><tr><td>Here's another table:<table id="2"><tr><td>foo</td></tr></table></td></tr></table>z{<table><thead><tr><td>Foo</td></tr></thead><tbody><tr><td>Bar</td></tr></tbody><tfoot><tr><td>Baz</td></tr></tfoot></table>N)r )r r r r r � test_correctly_nested_tables s z5HTMLTreeBuilderSmokeTest.test_correctly_nested_tablesc C s( d}| j |�}| jdg|jjd � d S )Nz1<table><div><div class="css"></div></div></table>Zcss�class)r r Zdiv)r r r r r r �(test_deeply_nested_multivalued_attribute0 s zAHTMLTreeBuilderSmokeTest.test_deeply_nested_multivalued_attributec C s( d}| j |�}| jddg|jd � d S )Nz<html class="a b"></html>�aro rs )r r rL )r r r r r r �"test_multivalued_attribute_on_html8 s z;HTMLTreeBuilderSmokeTest.test_multivalued_attribute_on_htmlc C s | j dd� d S )Nz<a b="<a>"></a>z<a b="<a>"></a>)r )r r r r �3test_angle_brackets_in_attribute_values_are_escaped@ s zLHTMLTreeBuilderSmokeTest.test_angle_brackets_in_attribute_values_are_escapedc C s | j dd� d S )Nz$<p>• AT&T is in the s&p 500</p>z,<p>\u2022 AT&T is in the s&p 500</p>)r )r r r r �3test_strings_resembling_character_entity_referencesC s zLHTMLTreeBuilderSmokeTest.test_strings_resembling_character_entity_referencesc C s" d}| j |�}| jd|jj� d S )Nz%<p>“Hello” -☃</p>u “Hello” -☃)r r rH rp )r r r r r r �*test_entities_in_foreign_document_encodingK s zCHTMLTreeBuilderSmokeTest.test_entities_in_foreign_document_encodingc C s8 d}| j d|� | j d|� | j d|� | j d|� d S )Nu <p id="piñata"></p>z<p id="piñata"></p>z<p id="piñata"></p>z<p id="piñata"></p>z<p id="piñata"></p>)r )r �expectr r r �0test_entities_in_attributes_converted_to_unicodeW s zIHTMLTreeBuilderSmokeTest.test_entities_in_attributes_converted_to_unicodec C s8 d}| j d|� | j d|� | j d|� | j d|� d S )Nu <p>piñata</p>z<p>piñata</p>z<p>piñata</p>z<p>piñata</p>z<p>piñata</p>)r )r rz r r r �*test_entities_in_text_converted_to_unicode^ s zCHTMLTreeBuilderSmokeTest.test_entities_in_text_converted_to_unicodec C s | j dd� d S )Nz#<p>I said "good day!"</p>z<p>I said "good day!"</p>)r )r r r r �,test_quot_entity_converted_to_quotation_marke s zEHTMLTreeBuilderSmokeTest.test_quot_entity_converted_to_quotation_markc C s, d}| j d|� | j d|� | j d|� d S )Nu �z�z�z �)r )r rz r r r �test_out_of_range_entityi s z1HTMLTreeBuilderSmokeTest.test_out_of_range_entityc C s< | j d�}| jd|jjjj� | jd|jj� | j|� dS )zDMostly to prevent a recurrence of a bug in the html5lib treebuilder.z!<html><h2> foo</h2><p></p></html>rH N)r r Zh2rp r r8 rH r )r r r r r �test_multipart_stringso s z/HTMLTreeBuilderSmokeTest.test_multipart_stringsc C s | j dd� | j dd� dS )zqVerify consistent handling of empty-element tags, no matter how they come in through the markup. z<br/><br/><br/>z<br /><br /><br />N)r )r r r r r9 v s c C s, d}| j |�}| jd|jj� | j|� dS )z8Prevent recurrence of a bug in the html5lib treebuilder.z?<html><head></head> <link></link> <body>foo</body> </html> N)r ZassertNotEqualrL �bodyr )r �contentr r r r �#test_head_tag_between_head_and_body} s z<HTMLTreeBuilderSmokeTest.test_head_tag_between_head_and_bodyc C s d}| j |�}| j|j� dS )z8Prevent recurrence of a bug in the html5lib treebuilder.z�<!DOCTYPE html> <html> <body> <article id="a" > <div><a href="1"></div> <footer> <a href="2"></a> </footer> </article> </body> </html> N)r r Zarticle)r r� r r r r � test_multiple_copies_of_a_tag� s z6HTMLTreeBuilderSmokeTest.test_multiple_copies_of_a_tagc C s^ d}| j |�}| j||j� � |j}| jd|jd � | jd|jd � | jd|jd � dS ) z�Parsers don't need to *understand* namespaces, but at the very least they should not choke on namespaces or lose data.s� <html xmlns="http://www.w3.org/1999/xhtml" xmlns:mathml="http://www.w3.org/1998/Math/MathML" xmlns:svg="http://www.w3.org/2000/svg"><head></head><body><mathml:msqrt>4</mathml:msqrt><b svg:fill="red"></b></body></html>z http://www.w3.org/1999/xhtmlZxmlnsz"http://www.w3.org/1998/Math/MathMLz xmlns:mathmlzhttp://www.w3.org/2000/svgz xmlns:svgN)r r rV rL )r r r rL r r r �test_basic_namespaces� s z.HTMLTreeBuilderSmokeTest.test_basic_namespacesc C s( d}| j |�}| jddg|jd � d S )Ns <a class="foo bar">rC �barrs )r r ru )r r r r r r �-test_multivalued_attribute_value_becomes_list� s zFHTMLTreeBuilderSmokeTest.test_multivalued_attribute_value_becomes_listc C s" d}| j |�}| jd|jj� d S )NuD <html><head><meta encoding="euc-jp"></head><body>Sacré bleu!</body>u Sacré bleu!)r r r� rp )r r r r r r � test_can_parse_unicode_document� s z8HTMLTreeBuilderSmokeTest.test_can_parse_unicode_documentc C s* t d�}| jd|d�}| j|j� d� dS )z2Parsers should be able to work with SoupStrainers.ro z&A <b>bold</b> <meta/> <i>statement</i>)Z parse_onlyz <b>bold</b>N)r r r r )r Zstrainerr r r r �test_soupstrainer� s z*HTMLTreeBuilderSmokeTest.test_soupstrainerc C s | j dd� d S )Nz<foo attr='bar'></foo>z<foo attr="bar"></foo>)r )r r r r �7test_single_quote_attribute_values_become_double_quotes� s zPHTMLTreeBuilderSmokeTest.test_single_quote_attribute_values_become_double_quotesc C s d}| j |� d S )Nz'<foo attr='bar "brawls" happen'>a</foo>)r )r ri r r r �7test_attribute_values_with_nested_quotes_are_left_alone� s zPHTMLTreeBuilderSmokeTest.test_attribute_values_with_nested_quotes_are_left_alonec C s. d}| j |�}d|jd< | j|jj� d� d S )Nz'<foo attr='bar "brawls" happen'>a</foo>z Brawls happen at "Bob's Bar"�attrz:<foo attr="Brawls happen at "Bob's Bar"">a</foo>)r rC r r )r ri r r r r �:test_attribute_values_with_double_nested_quotes_get_quoted� s zSHTMLTreeBuilderSmokeTest.test_attribute_values_with_double_nested_quotes_get_quotedc C s | j dd� | j dd� d S )Nz+<this is="really messed up & stuff"></this>z/<this is="really messed up & stuff"></this>z.<a href="http://example.org?a=1&b=2;3">foo</a>z2<a href="http://example.org?a=1&b=2;3">foo</a>)r )r r r r �.test_ampersand_in_attribute_value_gets_escaped� s zGHTMLTreeBuilderSmokeTest.test_ampersand_in_attribute_value_gets_escapedc C s | j d� d S )Nz/<a href="http://example.org?a=1&b=2;3"></a>)r )r r r r �7test_escaped_ampersand_in_attribute_value_is_left_alone� s zPHTMLTreeBuilderSmokeTest.test_escaped_ampersand_in_attribute_value_is_left_alonec C s d}d}| j ||� d S )Nz-<p><<sacré bleu!>></p>u# <p><<sacré bleu!>></p>)r )r ri �expectedr r r �1test_entities_in_strings_converted_during_parsing� s zJHTMLTreeBuilderSmokeTest.test_entities_in_strings_converted_during_parsingc C s" d}| j |�}| j|jjd� d S )Ns <p>�Foo�</p>u ‘Foo’)r r rH rp )r Zquoter r r r �)test_smart_quotes_converted_on_the_way_in� s zBHTMLTreeBuilderSmokeTest.test_smart_quotes_converted_on_the_way_inc C s | j d�}| j|jjd� d S )Nz<a> </a>� r: u )r r ru rp )r r r r r �0test_non_breaking_spaces_converted_on_the_way_in� s zIHTMLTreeBuilderSmokeTest.test_non_breaking_spaces_converted_on_the_way_inc C s0 d}dj d�}| j|�}| j|jj d�|� d S )Nz-<p><<sacré bleu!>></p>u# <p><<sacré bleu!>></p>zutf-8)rV r r rH )r ri r� r r r r �&test_entities_converted_on_the_way_out� s z?HTMLTreeBuilderSmokeTest.test_entities_converted_on_the_way_outc C sH d}|j d�}| j|�}|j d�}|jdd�}|j d�}| j||� d S )Nu� <html><head><meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type"/></head><body><p>Sacré bleu!</p></body></html>z iso-8859-1zutf-8z ISO-Latin-1)rV r rW r )r � unicode_htmlZiso_latin_htmlr �resultr� r r r � test_real_iso_latin_document� s z5HTMLTreeBuilderSmokeTest.test_real_iso_latin_documentc C sL d}|j d�}| j|�}| j|jd�|jd�� | j|jd�|jd�� d S )Nsk <html><head></head><body><pre>�����Shift-JIS�ŃR�[�f�B���O���ꂽ���{��̃t�@�C���ł��B</pre></body></html>z shift-jiszutf-8�euc_jp)r r r rV )r �shift_jis_htmlr� r r r r � test_real_shift_jis_document s z5HTMLTreeBuilderSmokeTest.test_real_shift_jis_documentc C s4 d}| j |dd�}| j|jd�|jd�jd�� d S )Ns� <html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>���</body></html>z iso8859-8)Z from_encodingzutf-8)r r rV r )r Zhebrew_documentr r r r �test_real_hebrew_document$ s z2HTMLTreeBuilderSmokeTest.test_real_hebrew_documentc C s` d}d| }| j |�}|jdddi�}|d }| jd|� | jt|t�� | jd|jd �� d S ) NzE<meta content="text/html; charset=x-sjis" http-equiv="Content-type"/>zj<html><head> %s <meta http-equiv="Content-language" content="ja"/></head><body>Shift-JIS markup goes here.r. z http-equivz Content-typer� ztext/html; charset=x-sjisztext/html; charset=utf8r[ )r rg r rb � isinstancer rV )r �meta_tagr� r � parsed_metar� r r r �'test_meta_tag_reflects_current_encoding1 s z@HTMLTreeBuilderSmokeTest.test_meta_tag_reflects_current_encodingc C s^ d}d| }| j |�}|jddd�}|d }| jd|� | jt|t�� | jd|jd�� d S ) Nz'<meta id="encoding" charset="x-sjis" />zj<html><head> %s <meta http-equiv="Content-language" content="ja"/></head><body>Shift-JIS markup goes here.r. �encoding)�id�charsetzx-sjisr[ )r rg r rb r� r rV )r r� r� r r� r� r r r �3test_html5_style_meta_tag_reflects_current_encodingM s zLHTMLTreeBuilderSmokeTest.test_html5_style_meta_tag_reflects_current_encodingc C s* | j d�}d|jd< | jd|jj� � d S )Nz <a>text</a>r� rC z<a foo="bar">text</a>)r ru r r )r �datar r r �5test_tag_with_no_attributes_can_have_attributes_addede s zNHTMLTreeBuilderSmokeTest.test_tag_with_no_attributes_can_have_attributes_addedN);r r r �__doc__r9 rB rK rD rM rO rP rQ rR rS rX rZ r\ r_ r` ra rc rd rh rk rl rm rq rr rt rv rw rx ry r{ r| r} r~ r r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r r" ? sr r" c @ s� e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%S )&�XMLTreeBuilderSmokeTestc C sF | j d�}tj|d�}tj|�}| j|jt� | j|j� |j� � d S )Nz <a><b>foo</a>r: )r r; r< r= r r> r r )r r? r@ rA r r r rB l s z9XMLTreeBuilderSmokeTest.test_pickle_and_unpickle_identityc C s | j d�}| j|j� d� d S )Nz<root/>s. <?xml version="1.0" encoding="utf-8"?> <root/>)r r rV )r r r r r �test_docstring_generatedu s z0XMLTreeBuilderSmokeTest.test_docstring_generatedc C s$ d}| j |�}| j||jd�� d S )Ns, <?xml version="1.0" encoding="utf8"?> <foo/>r[ )r r rV )r r r r r r �test_xml_declarationz s z,XMLTreeBuilderSmokeTest.test_xml_declarationc C s$ d}| j |�}| j||jd�� d S )Ns< <?xml version="1.0" encoding="utf8"?> <?PITarget PIContent?>r[ )r r rV )r r r r r r r\ s z3XMLTreeBuilderSmokeTest.test_processing_instructionc C s$ d}| j |�}| j|jd�|� dS )zGA real XHTML document should come out *exactly* the same as it went in.s� <?xml version="1.0" encoding="utf-8"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"> <html xmlns="http://www.w3.org/1999/xhtml"> <head><title>Hello.</title></head> <body>Goodbye.</body> </html>zutf-8N)r r rV )r r r r r r rX � s z0XMLTreeBuilderSmokeTest.test_real_xhtml_documentc C s" d}| j |�}| j||j� � d S )Ns <?xml version="1.0" encoding="utf-8"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> <parent xmlns="http://ns1/"> <child xmlns="http://ns2/" xmlns:ns3="http://ns3/"> <grandchild ns3:attr="value" xmlns="http://ns4/"/> </child> </parent>)r r rV )r �docr r r r �test_nested_namespaces� s z.XMLTreeBuilderSmokeTest.test_nested_namespacesc C s0 d}t |d�}d|j_|j� }| jd|k� d S )Nz/ <script type="text/javascript"> </script> zlxml-xmlz console.log("< < hey > > ");s < < hey > >)r re rp rV rb )r r� r Zencodedr r r �5test_formatter_processes_script_tag_for_xml_documents� s zMXMLTreeBuilderSmokeTest.test_formatter_processes_script_tag_for_xml_documentsc C s" d}| j |�}| jd|jj� d S )Nu? <?xml version="1.0" encoding="euc-jp"><root>Sacré bleu!</root>u Sacré bleu!)r r �rootrp )r r r r r r r� � s z7XMLTreeBuilderSmokeTest.test_can_parse_unicode_documentc C s$ d}| j |�}| jt|j�|� d S )Nz�<rss xmlns:dc="foo"><dc:creator>b</dc:creator><dc:date>2012-07-02T20:33:42Z</dc:date><dc:rights>c</dc:rights><image>d</image></rss>)r r rF Zrss)r r r r r r �test_popping_namespaced_tag� s z3XMLTreeBuilderSmokeTest.test_popping_namespaced_tagc C s | j d�}| j|jd�d� d S )Nz<root/>�latin1s/ <?xml version="1.0" encoding="latin1"?> <root/>)r r rV )r r r r r �(test_docstring_includes_correct_encoding� s z@XMLTreeBuilderSmokeTest.test_docstring_includes_correct_encodingc C s0 ddd d }| j |�}| j|jd�|� dS ) z<A large XML document should come out the same as it went in.s- <?xml version="1.0" encoding="utf-8"?> <root>� 0r: � s </root>zutf-8Ni )r r rV )r r r r r r �test_large_xml_document� s z/XMLTreeBuilderSmokeTest.test_large_xml_documentc C s | j dd� | j d� d S )Nz<p>z<p/>z <p>foo</p>)r )r r r r �9test_tags_are_empty_element_if_and_only_if_they_are_empty� s zQXMLTreeBuilderSmokeTest.test_tags_are_empty_element_if_and_only_if_they_are_emptyc C s8 d}| j |�}|j}| jd|d � | jd|d � d S )Nz�<root xmlns:a="http://example.com/" xmlns:b="http://example.net/"><a:foo>This tag is in the a namespace</a:foo><b:foo>This tag is in the b namespace</b:foo></root>zhttp://example.com/zxmlns:azhttp://example.net/zxmlns:b)r r� r )r r r r� r r r � test_namespaces_are_preserved� s z5XMLTreeBuilderSmokeTest.test_namespaces_are_preservedc C s$ d}| j |�}| jt|j�|� d S )NzN<p xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:date>20010504</dc:date></p>)r r rF rH )r r r r r r �test_closing_namespaced_tag� s z3XMLTreeBuilderSmokeTest.test_closing_namespaced_tagc C s$ d}| j |�}| jt|j�|� d S )Nzs<foo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><bar xsi:schemaLocation="http://www.example.com"/></foo>)r r rF rC )r r r r r r �test_namespaced_attributes� s z2XMLTreeBuilderSmokeTest.test_namespaced_attributesc C s$ d}| j |�}| jt|j�|� d S )Nz <foo xml:lang="fr">bar</foo>)r r rF rC )r r r r r r �(test_namespaced_attributes_xml_namespace� s z@XMLTreeBuilderSmokeTest.test_namespaced_attributes_xml_namespacec C s� d}| j |�}| jdt|jd��� | jdt|jd��� | jdt|jd��� | jdt|jddd ��� | jdt|jddg��� d S ) Na <?xml version="1.0" encoding="utf-8"?> <Document xmlns="http://example.com/ns0" xmlns:ns1="http://example.com/ns1" xmlns:ns2="http://example.com/ns2" <ns1:tag>foo</ns1:tag> <ns1:tag>bar</ns1:tag> <ns2:tag key="value">baz</ns2:tag> </Document> � �tagr: zns1:tag� zns2:tag�value)�key)r r rG rY )r r� r r r r �test_find_by_prefixed_name� s z2XMLTreeBuilderSmokeTest.test_find_by_prefixed_namec C s2 d}| j |�}|j}tj|�}| j|j|j� d S )Nzf<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <w:document xmlns:w="http://example.com/ns0"/>)r Zdocumentr] r �prefix)r Zxmlr r� Z duplicater r r �!test_copy_tag_preserves_namespace� s z9XMLTreeBuilderSmokeTest.test_copy_tag_preserves_namespaceN)r r r rB r� r� r\ rX r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r r� j s$ r� c @ s8 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd S ) �HTML5TreeBuilderSmokeTestz2Smoke test for a tree builder that supports HTML5.c C s d S )Nr )r r r r rX s z2HTML5TreeBuilderSmokeTest.test_real_xhtml_documentc C s" d}| j |�}| jd|jj� d S )Nz<a>z http://www.w3.org/1999/xhtml)r r ru � namespace)r r r r r r � test_html_tags_have_namespace s z7HTML5TreeBuilderSmokeTest.test_html_tags_have_namespacec C s6 d}| j |�}d}| j||jj� | j||jj� d S )Nz<svg><circle/></svg>zhttp://www.w3.org/2000/svg)r r Zsvgr� Zcircle)r r r r� r r r � test_svg_tags_have_namespace s z6HTML5TreeBuilderSmokeTest.test_svg_tags_have_namespacec C s6 d}| j |�}d}| j||jj� | j||jj� d S )Nz <math><msqrt>5</msqrt></math>z"http://www.w3.org/1998/Math/MathML)r r Zmathr� Zmsqrt)r r r r� r r r � test_mathml_tags_have_namespace s z9HTML5TreeBuilderSmokeTest.test_mathml_tags_have_namespacec C sP d}| j |�}| jt|jd t�� | j|jd d� | jd|jd jj� d S )Nz3<?xml version="1.0" encoding="utf-8"?><html></html>r z$?xml version="1.0" encoding="utf-8"?rL )r rb r� rE r r r r8 )r r r r r r �$test_xml_declaration_becomes_comment s z>HTML5TreeBuilderSmokeTest.test_xml_declaration_becomes_commentN) r r r r� rX r� r� r� r� r r r r r� � s r� c s dd� �� �fdd�}|S )Nc _ s d S )Nr )Ztest�argsr r r r �nothing! s zskipIf.<locals>.nothingc s � r�S | S d S )Nr )Z test_item)� conditionr� r r � decorator$ s zskipIf.<locals>.decoratorr )r� �reasonr� r )r� r� r �skipIf s r� )r� Z __license__r; r] � functoolsZunittestr Zbs4r Z bs4.elementr r r r r Z bs4.builderr r r �objectr" r� r� r� r r r r �<module> s( % / # __pycache__/__init__.cpython-36.opt-1.pyc 0000644 00000037377 15246412105 0014121 0 ustar 00 3 (6]XY � @ s d Z dZdZdZdZdgZddlZddlZddlZddl Z ddl Z dd l m Z m Z dd lmZ dd lmZmZmZmZmZmZmZmZmZmZmZ d d k G dd� de�Z e Z e Z G dd� de �Z G dd� de �Z!G dd� de"�Z#e$dk�r ddlZe ej%�Z&e'e&j(� � dS )aH Beautiful Soup Elixir and Tonic "The Screen-Scraper's Friend" http://www.crummy.com/software/BeautifulSoup/ Beautiful Soup uses a pluggable XML or HTML parser to parse a (possibly invalid) document into a tree representation. Beautiful Soup provides methods and Pythonic idioms that make it easy to navigate, search, and modify the parse tree. Beautiful Soup works with Python 2.7 and up. It works better if lxml and/or html5lib is installed. For more than you ever wanted to know about Beautiful Soup, see the documentation: http://www.crummy.com/software/BeautifulSoup/bs4/doc/ z*Leonard Richardson (leonardr@segfault.org)z4.6.3z*Copyright (c) 2004-2018 Leonard RichardsonZMIT� BeautifulSoup� N� )�builder_registry�ParserRejectedMarkup)� UnicodeDammit) �CData�Comment�DEFAULT_OUTPUT_ENCODING� Declaration�Doctype�NavigableString� PageElement�ProcessingInstruction� ResultSet� SoupStrainer�Tagz`You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work.zuYou need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).c s� e Zd ZdZdZddgZdZdZd2d d �Zd d � Z d d� Z e dd� �Z dd� Z dd� Zddi fdd�Zefdd�Zdd� Zdd � Zd d � Zd d � Zefd!d"�Zd3d#d$�Zd4d&d'�Zd(d)� Zd5d*d+�Zd,d-� Zd.e d/f� fd0d1� Z � Z S )6r a This class defines the basic interface called by the tree builders. These methods will be called by the parser: reset() feed(markup) The tree builder may call these methods from its feed() implementation: handle_starttag(name, attrs) # See note about return value handle_endtag(name) handle_data(data) # Appends to the current data node endData(containerClass=NavigableString) # Ends the current data node No matter how complicated the underlying parser is, you should be able to build a tree using 'start tag' events, 'end tag' events, 'data' events, and "done with data" events. If you encounter an empty-element tag (aka a self-closing tag, like HTML's <br> tag), call handle_starttag and then handle_endtag. z [document]ZhtmlZfastz a� No parser was explicitly specified, so I'm using the best available %(markup_type)s parser for this system ("%(parser)s"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently. The code that caused this warning is on line %(line_number)s of the file %(filename)s. To get rid of this warning, pass the additional argument 'features="%(parser)s"' to the BeautifulSoup constructor. � Nc " s� d� krt jd� d� kr*� d= t jd� d� krB� d= t jd� d� krZ� d= t jd� d � krr� d = t jd � � fd d �}|p�|d d�}|p�|dd�}|r�t|t�r�t jd� d}t� �dkr�t� j� �j� } td| ��|dk�r|} t|t�r�|g}|dk�st|�dk�r| j }t j |� } | dk�r@t ddj |� ��| � }| |jk�pZ| |jk�s|j�rld} nd} d} ytjd�} W n tk �r� Y nX | �r�| j}| j}n tj}d}|jd�}|�r�|j� }|jd(��r�|dd)� }|�rt|||j| d �}t j| j| d d � || _ |j| _| j| _ | | j _ || _ t |d ��rN|j!� }n�t|�d!k�r t|t"��rrd"|k�s�t|t��r d#|k�r t|t��r�t#j$j% �r�|j&d$�}n|}d%}yt#j$j'|�}W n$ t(k �r� } zW Y dd}~X nX |�rt|t��r|j&d$�}t jd&| � | j)|� xZ| j j*|||d'�D ]D\| _+| _,| _-| _.| j/� y| j0� P W n t1k �rr Y nX �q2W d| _+d| j _ dS )*a_ Constructor. :param markup: A string or a file-like object representing markup to be parsed. :param features: Desirable features of the parser to be used. This may be the name of a specific parser ("lxml", "lxml-xml", "html.parser", or "html5lib") or it may be the type of markup to be used ("html", "html5", "xml"). It's recommended that you name a specific parser, so that Beautiful Soup gives you the same results across platforms and virtual environments. :param builder: A specific TreeBuilder to use instead of looking one up based on `features`. You shouldn't need to use this. :param parse_only: A SoupStrainer. Only parts of the document matching the SoupStrainer will be considered. This is useful when parsing part of a document that would otherwise be too large to fit into memory. :param from_encoding: A string indicating the encoding of the document to be parsed. Pass this in if Beautiful Soup is guessing wrongly about the document's encoding. :param exclude_encodings: A list of strings indicating encodings known to be wrong. Pass this in if you don't know the document's encoding but you know Beautiful Soup's guess is wrong. :param kwargs: For backwards compatibility purposes, the constructor accepts certain keyword arguments used in Beautiful Soup 3. None of these arguments do anything in Beautiful Soup 4 and there's no need to actually pass keyword arguments into the constructor. ZconvertEntitiesz�BS4 does not respect the convertEntities argument to the BeautifulSoup constructor. Entities are always converted to Unicode characters.Z markupMassagez�BS4 does not respect the markupMassage argument to the BeautifulSoup constructor. The tree builder is responsible for any necessary markup massage.Z smartQuotesToz�BS4 does not respect the smartQuotesTo argument to the BeautifulSoup constructor. Smart quotes are always converted to Unicode characters.ZselfClosingTagsz�BS4 does not respect the selfClosingTags argument to the BeautifulSoup constructor. The tree builder is responsible for understanding self-closing tags.ZisHTMLz�BS4 does not respect the isHTML argument to the BeautifulSoup constructor. Suggest you use features='lxml' for HTML and features='lxml-xml' for XML.c s0 | � kr,t jd| |f � � | }� | = |S d S )NzLThe "%s" argument to the BeautifulSoup constructor has been renamed to "%s.")�warnings�warn)Zold_name�new_name�value)�kwargs� � /usr/lib/python3.6/__init__.py�deprecated_argument� s z3BeautifulSoup.__init__.<locals>.deprecated_argumentZparseOnlyThese� parse_onlyZ fromEncoding� from_encodingzlYou provided Unicode markup but also provided a value for from_encoding. Your from_encoding will be ignored.Nr z2__init__() got an unexpected keyword argument '%s'zjCouldn't find a tree builder with the features you requested: %s. Do you need to install a parser library?�,ZXMLZHTMLr �__file__�.pyc�.pyo)�filename� line_number�parser� markup_type� )� stacklevel�read� � <�<�utf8Fzw"%s" looks like a filename, not markup. You should probably open this file and pass the filehandle into Beautiful Soup.)�exclude_encodings)r r ���)2r r � isinstance�str�len�list�keys�pop� TypeError�DEFAULT_BUILDER_FEATURESr �lookup�FeatureNotFound�join�NAMEZALTERNATE_NAMES�is_xml�sys� _getframe� ValueError� f_globals�f_lineno�__dict__�get�lower�endswith�dict�NO_PARSER_SPECIFIED_WARNING�builderZ known_xml�soupr �hasattrr' �bytes�os�path�supports_unicode_filenames�encode�exists� Exception�_check_markup_is_urlZprepare_markup�markup�original_encodingZdeclared_html_encodingZ contains_replacement_characters�reset�_feedr )�selfrQ �featuresrF r r r, r r �argZoriginal_featuresZ builder_classr$ Zcaller�globalsr" r! Zfnl�valuesZpossible_filename�is_file�er )r r �__init__X s� ' zBeautifulSoup.__init__c C s& t | �| jd�| jdd�}| j|_|S )Nzutf-8)rF r )�typerM rF rR )rU �copyr r r �__copy__$ s zBeautifulSoup.__copy__c C s( t | j�}d|kr$| jj r$d |d< |S )NrF )rD r@ rF Z picklable)rU �dr r r � __getstate__0 s zBeautifulSoup.__getstate__c sx t � t�rd}d }nt � t�r(d}d}ndS t� fdd �|D ��rt|� krtt � t�rb� jd d �}n� }tjd | � dS )z� Check if markup looks like it's actually a url and raise a warning if so. Markup can be unicode or str (py2) / bytes (py3). � � http:� https:� �http:�https:Nc 3 s | ]}� j |�V qd S )N)� startswith)�.0�prefix)rQ r r � <genexpr>F s z5BeautifulSoup._check_markup_is_url.<locals>.<genexpr>zutf-8�replacez�"%s" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client like requests to get the document behind the URL, and feed that document to Beautiful Soup.)rc rd )rf rg )r. rI r/ �any�decoder r )rQ ZspaceZcant_start_withZdecoded_markupr )rQ r rP 7 s z"BeautifulSoup._check_markup_is_urlc C s@ | j j� | j j| j� | j� x| jj| jkr:| j� q"W d S )N) rF rS ZfeedrQ �endData� currentTag�name� ROOT_TAG_NAME�popTag)rU r r r rT S s zBeautifulSoup._feedc C sJ t j| | | j| j� d| _| jj� g | _d | _g | _g | _ | j | � d S )Nr ) r r\ rF rr ZhiddenrS � current_datarp �tagStack� preserve_whitespace_tag_stack�pushTag)rU r r r rS ] s zBeautifulSoup.resetc K s |j |� td| j||||�S )z+Create a new tag associated with this soup.N)�updater rF )rU rq � namespace�nsprefix�attrsZkwattrsr r r �new_tagg s zBeautifulSoup.new_tagc C s ||�S )z7Create a new NavigableString associated with this soup.r )rU �s�subclassr r r � new_stringl s zBeautifulSoup.new_stringc C s t d��d S )Nz4BeautifulSoup objects don't support insert_before().)�NotImplementedError)rU � successorr r r � insert_beforep s zBeautifulSoup.insert_beforec C s t d��d S )Nz3BeautifulSoup objects don't support insert_after().)r� )rU r� r r r � insert_afters s zBeautifulSoup.insert_afterc C s@ | j j� }| jr(|| jd kr(| jj� | j r:| j d | _| jS )Nr r- r- )ru r3 rv rp )rU �tagr r r rs v s zBeautifulSoup.popTagc C sJ | j r| j jj|� | jj|� | jd | _ |j| jjkrF| jj|� d S )Nr r- )rp �contents�appendru rq rF Zpreserve_whitespace_tagsrv )rU r� r r r rw s zBeautifulSoup.pushTagc C s� | j r�dj| j �}| jsPd}x |D ]}|| jkr"d}P q"W |rPd|krLd}nd}g | _ | jr�t| j�dkr�| jj s�| jj|� r�d S ||�}| j |� d S )Nr TF� re r ) rt r8 rv � ASCII_SPACESr r0 ru �text�search�object_was_parsed)rU ZcontainerClassrt Z strippable�i�or r r ro � s& zBeautifulSoup.endDatac C sV |p| j }|p| j}d } }}t|t�rF|j}|j}|j}|sF|j}|j|||||� || _|j j |� |j�rRt |j �d }x4|dkr�|j | |kr�P |d8 }q�W t d||f ��|dkr�|}d}n|j |d }}|t |j �d kr�|j}d}n|j |d }}||_|�r ||_||_|�r.||_||_|�r@||_||_|�rR||_dS )z Add an object to the parse tree.Nr r z[Error building tree: supposedly %r was inserted into %r after the fact, but I don't see it!) rp �_most_recent_elementr. r � next_element� next_sibling�previous_sibling�previous_elementZsetupr� r� r0 r= ) rU r� �parentZmost_recent_elementr� r� r� r� �indexr r r r� � sR z BeautifulSoup.object_was_parsedTc C sn || j krdS d}t| j�}xLt|d dd�D ]8}| j| }||jkr^||jkr^|r\| j� }P | j� }q.W |S )z�Pops the tag stack up to and including the most recent instance of the given tag. If inclusivePop is false, pops the tag stack up to but *not* including the most recent instqance of the given tag.Nr r r- )rr r0 ru �rangerq rj rs )rU rq rz Z inclusivePopZmost_recently_popped� stack_sizer� �tr r r � _popToTag� s zBeautifulSoup._popToTagc C s� | j � | jr8t| j�dkr8| jjs4| jj||� r8dS t| | j||||| j| j �}|dkr`|S | j rn|| j _ || _ | j |� |S )a Push a start tag on to the stack. If this method returns None, the tag was rejected by the SoupStrainer. You should proceed as if the tag had not occurred in the document. For instance, if this was a self-closing tag, don't call handle_endtag. r N) ro r r0 ru r� Z search_tagr rF rp r� r� rw )rU rq ry rz r{ r� r r r �handle_starttag� s z BeautifulSoup.handle_starttagc C s | j � | j||� d S )N)ro r� )rU rq rz r r r � handle_endtag s zBeautifulSoup.handle_endtagc C s | j j|� d S )N)rt r� )rU �datar r r � handle_data s zBeautifulSoup.handle_dataFZminimalc sN | j r$d}|dkrd| }d| }nd}|s2d}nd}|tt| �j|||� S )zlReturns a string or Unicode representation of this document. To get Unicode, pass None for encoding.r Nz encoding="%s"z<?xml version="1.0"%s?> r )r: �superr rn )rU Z pretty_printZeventual_encodingZ formatterZ encoding_partrj Z indent_level)� __class__r r rn s zBeautifulSoup.decode)r NNNNN)NN)NT)N) �__name__� __module__� __qualname__�__doc__rr r5 r� rE r\ r_ ra � staticmethodrP rT rS r| r r r� r� rs rw ro r� r� r� r� r� r rn � __classcell__r r )r� r r 8 s8 L 9 c s e Zd ZdZ� fdd�Z� ZS )�BeautifulStoneSoupz&Deprecated interface to an XML parser.c s( d|d< t jd� tt| �j||� d S )NZxmlrV zxThe BeautifulStoneSoup class is deprecated. Instead of using it, pass features="xml" into the BeautifulSoup constructor.)r r r� r� r\ )rU �argsr )r� r r r\ 5 s zBeautifulStoneSoup.__init__)r� r� r� r� r\ r� r r )r� r r� 2 s r� c @ s e Zd ZdS )� StopParsingN)r� r� r� r r r r r� = s r� c @ s e Zd ZdS )r7 N)r� r� r� r r r r r7 @ s r7 �__main__))r� � __author__� __version__Z __copyright__Z __license__�__all__rJ �rer; � tracebackr rF r r Zdammitr �elementr r r r r r r r r r r r Z_sZ_soupr� rO r� r= r7 r� �stdinrG �printZprettifyr r r r �<module> s6 4 z __pycache__/diagnose.cpython-36.pyc 0000644 00000016740 15246412105 0013203 0 ustar 00 3 #6]� � @ s d Z dZddlZddlmZ ddlmZ ddlZddlmZm Z ddl m Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZdd � Zd#d d �ZG d d� de�Zdd� ZdZdZd$dd�Zd%dd�Zd&dd�Zd'd d �Z d(d d!�Z e d"k�reej j � � dS ))z=Diagnostic functions, mainly for use when doing tech support.ZMIT� N)�StringIO)� HTMLParser)� BeautifulSoup� __version__)�builder_registryc ; C sX t dt � t dtj � dddg}x>|D ]6}x0tjD ]}||jkr6P q6W |j|� t d| � q*W d|kr�|jd� y*dd l m } t d d j t t |j�� � W n* tk r� } zt d � W Y d d }~X nX d|k�r ydd l}t d|j � W n, tk �r } zt d� W Y d d }~X nX t| d��r4| j� } n�| jd��sL| jd��rdt d| � t d� d S y:tjj| ��r�t d| � t| ��}|j� } W d Q R X W n tk �r� Y nX t � x�|D ]�}t d| � d} yt| |d�} d} W n8 tk �r" } zt d| � tj � W Y d d }~X nX | �rBt d| � t | j � � t d d � �q�W d S ) z/Diagnostic suite for isolating common problems.z'Diagnostic running on Beautiful Soup %szPython version %sz html.parser�html5lib�lxmlz;I noticed that %s is not installed. Installing it may help.zlxml-xmlr )�etreezFound lxml version %s�.z.lxml is not installed or couldn't be imported.NzFound html5lib version %sz2html5lib is not installed or couldn't be imported.�readzhttp:zhttps:z<"%s" looks like a URL. Beautiful Soup is not an HTTP client.zpYou need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup.z7"%s" looks like a filename. Reading data from the file.z#Trying to parse your markup with %sF)�featuresTz %s could not parse the markup.z#Here's what %s did with the markup:�-�P ) �printr �sys�versionr Zbuildersr �remove�appendr r �join�map�strZ LXML_VERSION� ImportErrorr �hasattrr � startswith�os�path�exists�open� ValueErrorr � Exception� traceback� print_excZprettify) �dataZ basic_parsers�nameZbuilderr �er �fp�parser�success�soup� r) � /usr/lib/python3.6/diagnose.py�diagnose sj r+ Tc K sN ddl m} x<|jt| �fd|i|��D ] \}}td||j|jf � q(W dS )z�Print out the lxml events that occur during parsing. This lets you see how lxml parses a document when no Beautiful Soup code is running. r )r �htmlz %s, %4s, %sN)r r Z iterparser r �tag�text)r" r, �kwargsr Zevent�elementr) r) r* � lxml_traceZ s $r1 c @ s` e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dd� Z dS )�AnnouncingParserz?Announces HTMLParser parse events, without doing anything else.c C s t |� d S )N)r )�self�sr) r) r* �_pg s zAnnouncingParser._pc C s | j d| � d S )Nz%s START)r5 )r3 r# Zattrsr) r) r* �handle_starttagj s z AnnouncingParser.handle_starttagc C s | j d| � d S )Nz%s END)r5 )r3 r# r) r) r* � handle_endtagm s z AnnouncingParser.handle_endtagc C s | j d| � d S )Nz%s DATA)r5 )r3 r" r) r) r* � handle_datap s z AnnouncingParser.handle_datac C s | j d| � d S )Nz %s CHARREF)r5 )r3 r# r) r) r* �handle_charrefs s z AnnouncingParser.handle_charrefc C s | j d| � d S )Nz %s ENTITYREF)r5 )r3 r# r) r) r* �handle_entityrefv s z!AnnouncingParser.handle_entityrefc C s | j d| � d S )Nz %s COMMENT)r5 )r3 r" r) r) r* �handle_commenty s z AnnouncingParser.handle_commentc C s | j d| � d S )Nz%s DECL)r5 )r3 r" r) r) r* � handle_decl| s z AnnouncingParser.handle_declc C s | j d| � d S )Nz%s UNKNOWN-DECL)r5 )r3 r" r) r) r* � unknown_decl s z AnnouncingParser.unknown_declc C s | j d| � d S )Nz%s PI)r5 )r3 r" r) r) r* � handle_pi� s zAnnouncingParser.handle_piN)�__name__� __module__� __qualname__�__doc__r5 r6 r7 r8 r9 r: r; r< r= r> r) r) r) r* r2 d s r2 c C s t � }|j| � dS )z�Print out the HTMLParser events that occur during parsing. This lets you see how HTMLParser parses a document when no Beautiful Soup code is running. N)r2 Zfeed)r" r& r) r) r* �htmlparser_trace� s rC ZaeiouZbcdfghjklmnpqrstvwxyz� c C s> d}x4t | �D ](}|d dkr$t}nt}|tj|�7 }qW |S )z#Generate a random word-like string.� � r )�range� _consonants�_vowels�random�choice)�lengthr4 �i�tr) r) r* �rword� s rO � c C s dj dd� t| �D ��S )z'Generate a random sentence-like string.� c s s | ]}t tjd d��V qdS )rP � N)rO rJ �randint)�.0rM r) r) r* � <genexpr>� s z rsentence.<locals>.<genexpr>)r rG )rL r) r) r* � rsentence� s rV �� c C s� dddddddg}g }x~t | �D ]r}tjdd �}|dkrRtj|�}|jd | � q |d krr|jttjd d ��� q |d kr tj|�}|jd| � q W ddj|� d S )z+Randomly generate an invalid HTML document.�pZdiv�spanrM �bZscript�tabler � z<%s>� rP rF z</%s>z<html>� z</html>)rG rJ rS rK r rV r )� num_elementsZ tag_names�elementsrM rK Ztag_namer) r) r* �rdoc� s ra 順 c C s( t dt � t| �}t dt|� � x�dddgddgD ]z}d}y"tj� }t||�}tj� }d}W n6 tk r� } zt d | � tj� W Y d d }~X nX |r6t d ||| f � q6W d d l m } tj� }|j |� tj� }t d|| � d d l } | j � }tj� }|j|� tj� }t d|| � d S )z.Very basic head-to-head performance benchmark.z1Comparative parser benchmark on Beautiful Soup %sz3Generated a large invalid HTML document (%d bytes).r r, r z html.parserFTz %s could not parse the markup.Nz"BS4+%s parsed the markup in %.2fs.r )r z$Raw lxml parsed the markup in %.2fs.z(Raw html5lib parsed the markup in %.2fs.)r r ra �len�timer r r r! r r ZHTMLr r �parse) r_ r" r& r' �ar( rZ r$ r r r) r) r* �benchmark_parsers� s4 rg r c C sX t j� }|j}t| �}tt||d�}tjd|||� tj |�}|j d� |j dd� d S )N)�bs4r" r& z bs4.BeautifulSoup(data, parser)Z cumulativez _html5lib|bs4�2 ) �tempfileZNamedTemporaryFiler# ra �dictrh �cProfileZrunctx�pstatsZStatsZ sort_statsZ print_stats)r_ r& Z filehandle�filenamer" �varsZstatsr) r) r* �profile� s rp �__main__)T)rD )rP )rW )rb )rb r )!rB Z __license__rl �ior Z html.parserr rh r r Z bs4.builderr r rm rJ rj rd r r r+ r1 r2 rC rI rH rO rV ra rg rp r? �stdinr r) r) r) r* �<module> s8 C ! __pycache__/element.cpython-36.pyc 0000644 00000142406 15246412105 0013042 0 ustar 00 3 &6]$ � @ s d Z yddlmZ W n. ek rB Z zddlmZ W Y ddZ[X nX ddlZddlZddlZddl Z ddl m Z dZ ej d dkZejd�Zdd � ZG d d � d e�ZG d d � d e�ZG dd� de�ZG dd� de�ZG dd� de �ZG dd� de�ZG dd� de�ZG dd� de�ZG dd� de�Z G d d � d e�Z G d d � d e�Z G d d!� d!e�Z G d"d#� d#ee �Z G d$d%� d%e �Z!G d&d'� d'e!�Z"G d(d)� d)e!�Z#G d*d+� d+e#�Z$G d,d-� d-e!�Z%G d.d/� d/e!�Z&G d0d1� d1e!�Z'G d2d3� d3e �Z(G d4d5� d5e�Z)G d6d7� d7e*�Z+dS )8ZMIT� )�CallableN)�EntitySubstitutionzutf-8� z\s+c s& t � fdd��}|j� fdd��}|S )z>Alias one attribute name to another for backward compatibilityc s t | � �S )N)�getattr)�self)�attr� � /usr/lib/python3.6/element.py�alias s z_alias.<locals>.aliasc s t | � �S )N)�setattr)r )r r r r s )�property�setter)r r r )r r �_alias s r c @ s e Zd Zddd�ZdS )�NamespacedAttributeNc C sV |d krt j| |�}n*|d kr,t j| |�}nt j| |d | �}||_||_||_|S )N�:)�str�__new__�prefix�name� namespace)�clsr r r �objr r r r " s zNamespacedAttribute.__new__)N)�__name__� __module__� __qualname__r r r r r r s r c @ s e Zd ZdZdS )�%AttributeValueWithCharsetSubstitutionz=A stand-in object for a character encoding specified in HTML.N)r r r �__doc__r r r r r / s r c @ s e Zd ZdZdd� Zdd� ZdS )�CharsetMetaAttributeValuez�A generic stand-in for the value of a meta tag's 'charset' attribute. When Beautiful Soup parses the markup '<meta charset="utf8">', the value of the 'charset' attribute will be one of these objects. c C s t j| |�}||_|S )N)r r �original_value)r r r r r r r 9 s z!CharsetMetaAttributeValue.__new__c C s |S )Nr )r �encodingr r r �encode> s z CharsetMetaAttributeValue.encodeN)r r r r r r r r r r r 2 s r c @ s. e Zd ZdZejdej�Zdd� Zdd� Z dS )�ContentMetaAttributeValuea A generic stand-in for the value of a meta tag's 'content' attribute. When Beautiful Soup parses the markup: <meta http-equiv="content-type" content="text/html; charset=utf8"> The value of the 'content' attribute will be one of these objects. z((^|;)\s*charset=)([^;]*)c C s6 | j j|�}|d kr tjt|�S tj| |�}||_|S )N)� CHARSET_RE�searchr r r )r r �matchr r r r r M s z!ContentMetaAttributeValue.__new__c s � fdd�}| j j|| j�S )Nc s | j d�� S )N� )�group)r$ )r r r �rewriteX s z1ContentMetaAttributeValue.encode.<locals>.rewrite)r" �subr )r r r' r )r r r W s z ContentMetaAttributeValue.encodeN) r r r r �re�compile�Mr" r r r r r r r! B s r! c @ sV e Zd ZdZeddg�Zedg�Zeddg�Zedd� �Z edd � �Z ed d � �Z d S ) �HTMLAwareEntitySubstitutiona% Entity substitution rules that are aware of some HTML quirks. Specifically, the contents of <script> and <style> tags should not undergo entity substitution. Incoming NavigableString objects are checked to see if they're the direct children of a <script> or <style> tag. ZscriptZstyle�preZtextareac C s. t |t�r&|jd k r&|jj| jkr&|S ||�S )N)� isinstance�NavigableString�parentr �cdata_containing_tags)r �ns�fr r r �_substitute_if_appropriatem s z6HTMLAwareEntitySubstitution._substitute_if_appropriatec C s | j |tj�S )N)r4 r �substitute_html)r r2 r r r r5 w s z+HTMLAwareEntitySubstitution.substitute_htmlc C s | j |tj�S )N)r4 r �substitute_xml)r r2 r r r r6 | s z*HTMLAwareEntitySubstitution.substitute_xmlN) r r r r �setr1 Zpreformatted_tags�preserve_whitespace_tags� classmethodr4 r5 r6 r r r r r, \ s r, c @ s e Zd ZdZdZdd� ZdS )� Formatterz6Contains information about how to format a parse tree.�/c O s t � �dS )z1Transform certain characters into named entities.N)�NotImplementedError)r �args�kwargsr r r �substitute_entities� s z Formatter.substitute_entitiesN)r r r r �void_element_close_prefixr? r r r r r: � s r: c @ s e Zd ZdZdd� ZdS )� HTMLFormatterzThe default HTML formatter.c O s t j||�S )N)r, r5 )r r= r> r r r � substitute� s zHTMLFormatter.substituteN)r r r r rB r r r r rA � s rA c @ s e Zd ZdZdd� ZdS )�MinimalHTMLFormatterzA minimal HTML formatter.c O s t j||�S )N)r, r6 )r r= r> r r r rB � s z MinimalHTMLFormatter.substituteN)r r r r rB r r r r rC � s rC c @ s e Zd ZdZdZdS )�HTML5Formatterz5An HTML formatter that omits the slash in a void tag.N)r r r r r@ r r r r rD � s rD c @ s e Zd ZdZdd� ZdS )� XMLFormatterz+Substitute only the essential XML entities.c O s t j||�S )N)r r6 )r r= r> r r r rB � s zXMLFormatter.substituteN)r r r r rB r r r r rE � s rE c @ s e Zd ZdZdd� ZdS )�HTMLXMLFormatterz Format XML using HTML rules.c O s t j||�S )N)r, r5 )r r= r> r r r rB � s zHTMLXMLFormatter.substituteN)r r r r rB r r r r rF � s rF c @ s` e Zd ZdZe� e� e� dd�Ze� e � dd�Z d]dd�Z e dd � �Z d d � Zd^d d �Zed�Zed�Zdd� ZeZdd� ZeZeZdd� Zdd� Zd_dd�ZeZdd � Z d d � Z d d � Z d!d"� Z di dfd#d$�Z e Z!di ddfd%d&�Z"e"Z#di dfd'd(�Z$e$Z%di ddfd)d*�Z&e&Z'e&Z(di dfd+d,�Z)e)Z*di ddfd-d.�Z+e+Z,e+Z-di dfd/d0�Z.e.Z/di ddfd1d2�Z0e0Z1e0Z2di fd3d4�Z3e3Z4di dfd5d6�Z5e5Z6e5Z7e d7d8� �Z8e d9d:� �Z9d;d<� Z:d=d>� Z;e d?d@� �Z<e dAdB� �Z=e dCdD� �Z>e dEdF� �Z?e dGdH� �Z@eAjBdI�ZCeAjBd`�ZDdadLdM�ZEdNdO� ZFdbdQdR�ZGdSdT� ZHdUdV� ZIdWdX� ZJdYdZ� ZKd[d\� ZLdS )c� PageElementzeContains the navigational information for some part of the page (either a tag or a piece of text)N)�htmlZhtml5�minimalN)rH rI NrI c C sB t |t�r| j|�}|dkr"|}n t|�r4||�}n |j|�}|S )z2Format the given string using the given formatter.N)r. r �_formatter_for_name�callablerB )r �s� formatter�outputr r r � format_string� s zPageElement.format_stringc C s. | j dk r| j S | jdkr&t| dd�S | jjS )aJ Is this element part of an XML tree or an HTML tree? This is used when mapping a formatter name ("minimal") to an appropriate function (one that performs entity-substitution on the contents of <script> and <style> tags, or not). It can be inefficient, but it should be called very rarely. N�is_xmlF)� known_xmlr0 r �_is_xml)r r r r rR � s zPageElement._is_xmlc C s* | j r| jj|t� �S | jj|t� �S dS )z<Look up a formatter function based on its name and the tree.N)rR �XML_FORMATTERS�getrE �HTML_FORMATTERSrA )r r r r r rJ � s z PageElement._formatter_for_namec C s~ || _ || _|dk r | | j_|| _| jr0| | j_|| _| jrD| | j_| rh| j dk rh| j jrh| j jd }|| _|rz| | j_dS )zNSets up the initial relations between this element and other elements.Nr% ���)r0 �previous_element� next_element� next_sibling�previous_sibling�contents)r r0 rW rX rZ rY r r r �setup� s zPageElement.setuprY rZ c C sV | j std��|| krd S || j kr,td��| j }| j j| �}| j� |j||� | S )Nz]Cannot replace one element with another when theelement to be replaced is not part of a tree.z%Cannot replace a Tag with its parent.)r0 � ValueError�index�extract�insert)r � replace_withZ old_parent�my_indexr r r ra s zPageElement.replace_withc C sT | j }| j std��| j j| �}| j� x&t| jd d � �D ]}|j||� q<W | S )NzSCannot replace an element with its contents when thatelement is not part of a tree.)r0 r] r^ r_ �reversedr[ r` )r Z my_parentrb �childr r r �unwrap% s zPageElement.unwrapc C s | j |�}|j|� |S )N)ra �append)r Z wrap_inside�mer r r �wrap3 s zPageElement.wrapc C s� | j dk r | j j| j j| �= | j� }|j}| jdk rF| j|k rF|| j_|dk r`|| jk r`| j|_d| _d|_d| _ | jdk r�| j| jk r�| j| j_| jdk r�| j| jk r�| j| j_d | _| _| S )z0Destructively rips this element out of the tree.N)r0 r[ r^ �_last_descendantrX rW rZ rY )r � last_childrX r r r r_ 8 s( zPageElement.extractTc C sN |r| j r| j j}n$| }x t|t�r6|jr6|jd }qW | rJ|| krJd}|S )z8Finds the last element beneath this object to be parsed.r% NrV )rY rW r. �Tagr[ )r Zis_initializedZ accept_selfrj r r r ri U s z PageElement._last_descendantc C s� |d krt d��|| kr t d��t|t�r>t|t� r>t|�}ddlm} t||�r�x&t|j�D ]}| j||� |d7 }q`W d S t |t | j��}t |d�r�|j d k r�|j | kr�| j |�}||k r�|d8 }|j� | |_ d }|dkr�d |_| |_n(| j|d }||_||j_|jd�|_|jd k �r,||j_|jd�}|t | j�k�r�d |_| }d } x2| d k�r�|d k �r�|j} |j }| d k �rVP �qVW | d k �r�| |_nd |_n*| j| } | |_|jd k �r�||j_| |_|jd k �r�||j_| jj||� d S )Nz Cannot insert None into a tag.z Cannot insert a tag into itself.r )� BeautifulSoupr% r0 F)r] r. r r/ Zbs4rl �listr[ r` �min�len�hasattrr0 r^ r_ rZ rW rY ri rX ) r Zposition� new_childrl ZsubchildZ current_indexZprevious_childZnew_childs_last_elementr0 Zparents_next_siblingZ next_childr r r r` c sh zPageElement.insertc C s | j t| j�|� dS )z2Appends the given tag to the contents of this tag.N)r` ro r[ )r �tagr r r rf � s zPageElement.appendc C sR | |krt d��| j}|dkr&t d��t|t�r8|j� |j| �}|j||� dS )z�Makes the given element the immediate predecessor of this one. The two elements will have the same parent, and the given element will be immediately before this one. z&Can't insert an element before itself.Nz2Element has no parent, so 'before' has no meaning.)r] r0 r. rG r_ r^ r` )r Z predecessorr0 r^ r r r � insert_before� s zPageElement.insert_beforec C sV | |krt d��| j}|dkr&t d��t|t�r8|j� |j| �}|j|d |� dS )z�Makes the given element the immediate successor of this one. The two elements will have the same parent, and the given element will be immediately after this one. z%Can't insert an element after itself.Nz1Element has no parent, so 'after' has no meaning.r% )r] r0 r. rG r_ r^ r` )r Z successorr0 r^ r r r � insert_after� s zPageElement.insert_afterc K s | j | j|||f|�S )zjReturns the first item that matches the given criteria and appears after this Tag in the document.)� _find_one� find_all_next)r r �attrs�textr> r r r � find_next� s zPageElement.find_nextc K s | j ||||| jf|�S )zbReturns all items that match the given criteria and appear after this Tag in the document.)� _find_all� next_elements)r r rw rx �limitr> r r r rv � s zPageElement.find_all_nextc K s | j | j|||f|�S )z{Returns the closest sibling to this Tag that matches the given criteria and appears after this Tag in the document.)ru �find_next_siblings)r r rw rx r> r r r �find_next_sibling� s z PageElement.find_next_siblingc K s | j ||||| jf|�S )zqReturns the siblings of this Tag that match the given criteria and appear after this Tag in the document.)rz � next_siblings)r r rw rx r| r> r r r r} � s z PageElement.find_next_siblingsc K s | j | j|||f|�S )zkReturns the first item that matches the given criteria and appears before this Tag in the document.)ru �find_all_previous)r r rw rx r> r r r � find_previous� s zPageElement.find_previousc K s | j ||||| jf|�S )zcReturns all items that match the given criteria and appear before this Tag in the document.)rz �previous_elements)r r rw rx r| r> r r r r� � s z PageElement.find_all_previousc K s | j | j|||f|�S )z|Returns the closest sibling to this Tag that matches the given criteria and appears before this Tag in the document.)ru �find_previous_siblings)r r rw rx r> r r r �find_previous_sibling s z!PageElement.find_previous_siblingc K s | j ||||| jf|�S )zrReturns the siblings of this Tag that match the given criteria and appear before this Tag in the document.)rz �previous_siblings)r r rw rx r| r> r r r r� s z"PageElement.find_previous_siblingsc K s& d}| j ||df|�}|r"|d }|S )zOReturns the closest parent of this Tag that matches the given criteria.Nr% r )� find_parents)r r rw r> �r�lr r r � find_parent s zPageElement.find_parentc K s | j ||d|| jf|�S )zFReturns the parents of this Tag that match the given criteria.N)rz �parents)r r rw r| r> r r r r� ! s zPageElement.find_parentsc C s | j S )N)rX )r r r r �next* s zPageElement.nextc C s | j S )N)rW )r r r r �previous. s zPageElement.previousc K s& d }||||df|�}|r"|d }|S )Nr% r r )r �methodr rw rx r> r� r� r r r ru 4 s zPageElement._find_onec s8 |dkr d|kr |d }|d= t �t�r.�}nt�||f|�}|dkr�| r�| r�| r҈dksh�dkr�dd� |D �}t||�S t �t�r҈jd�dkr��jdd�\�� nd��� � ��fdd�|D �}t||�S t|�} xXy t|�} W n tk �r P Y nX | r�|j| �} | r�| j | � |r�t | �|kr�P q�W | S ) z8Iterates over a generator looking for things that match.N�stringTc s s | ]}t |t�r|V qd S )N)r. rk )�.0�elementr r r � <genexpr>J s z(PageElement._find_all.<locals>.<genexpr>r r% c 3 sB | ]:}t |t�r|j�ks6|j� kr�d ks6|j�kr|V qd S )N)r. rk r r )r� r� )� local_namer r r r r� W s ) r. � SoupStrainer� ResultSetr �count�splitr� � StopIterationr# rf ro ) r r rw rx r| � generatorr> Zstrainer�result�results�i�foundr )r� r r r rz ; s< zPageElement._find_allc c s$ | j }x|d k r |V |j }qW d S )N)rX )r r� r r r r{ q s zPageElement.next_elementsc c s$ | j }x|d k r |V |j }qW d S )N)rY )r r� r r r r x s zPageElement.next_siblingsc c s$ | j }x|d k r |V |j }qW d S )N)rW )r r� r r r r� s z PageElement.previous_elementsc c s$ | j }x|d k r |V |j }qW d S )N)rZ )r r� r r r r� � s z PageElement.previous_siblingsc c s$ | j }x|d k r |V |j }qW d S )N)r0 )r r� r r r r� � s zPageElement.parentsz ^[a-zA-Z0-9][-.a-zA-Z0-9:_]*$zX^(?P<tag>[a-zA-Z0-9][-.a-zA-Z0-9:_]*)?\[(?P<attribute>[\w-]+)(?P<operator>[=~\|\^\$\*]?)z=?"?(?P<value>[^\]"]*)"?\]$c C s. | j ||�}t|t�s t|t�r*dj|�}|S )z�Force an attribute value into a string representation. A multi-valued attribute will be converted into a space-separated stirng. � )rT r. rm �tuple�join)r �value�defaultr r r �_attr_value_as_string� s z!PageElement._attr_value_as_stringc s �s� S � �fdd�}|S d S )Nc s | j �ko� | �S )N)r )rr )�function�tag_namer r �_match� s z1PageElement._tag_name_matches_and.<locals>._matchr )r r� r� r� r )r� r� r �_tag_name_matches_and� s z!PageElement._tag_name_matches_and� c s� |dkr� �fdd�S |dkr0� �fdd�}|S |dkrF� �fdd�S |d kr\� �fd d�S |d krr� �fd d�S |d kr�� �fdd�}|S � fdd�S dS )z�Create a function that performs a CSS selector operation. Takes an operator, attribute and optional value. Returns a function that will return True for elements that match that combination. �=c s | j � ��kS )N)r� )�el)� attributer� r r �<lambda>� s z0PageElement._attribute_checker.<locals>.<lambda>�~c s& | j � g �}t|t�s |j� }�|kS )N)rT r. rm r� )r� �attribute_value)r� r� r r �_includes_value� s z7PageElement._attribute_checker.<locals>._includes_value�^c s | j � d�j��S )Nr� )r� � startswith)r� )r� r� r r r� � s �$c s | j � d�j��S )Nr� )r� �endswith)r� )r� r� r r r� � s �*c s �| j � d�kS )Nr� )r� )r� )r� r� r r r� � s �|c s" | j � d�}|�kp |j�d �S )Nr� �-)r� r� )r� r� )r� r� r r �_is_or_starts_with_dash� s z?PageElement._attribute_checker.<locals>._is_or_starts_with_dashc s | j � �S )N)�has_attr)r� )r� r r r� � s Nr )r �operatorr� r� r� r� r )r� r� r �_attribute_checker� s z PageElement._attribute_checkerc C s | j S )N)r{ )r r r r � nextGenerator� s zPageElement.nextGeneratorc C s | j S )N)r )r r r r �nextSiblingGenerator� s z PageElement.nextSiblingGeneratorc C s | j S )N)r� )r r r r �previousGenerator� s z PageElement.previousGeneratorc C s | j S )N)r� )r r r r �previousSiblingGenerator� s z$PageElement.previousSiblingGeneratorc C s | j S )N)r� )r r r r �parentGenerator� s zPageElement.parentGenerator)rI )NNNNN)TTzs^(?P<tag>[a-zA-Z0-9][-.a-zA-Z0-9:_]*)?\[(?P<attribute>[\w-]+)(?P<operator>[=~\|\^\$\*]?)=?"?(?P<value>[^\]"]*)"?\]$)N)r� )Mr r r r rA rD rC rU rF rE rS rO r rR rJ r\ r Z nextSiblingZpreviousSiblingra Z replaceWithre Zreplace_with_childrenZreplaceWithChildrenrh r_ ri Z_lastRecursiveChildr` rf rs rt ry ZfindNextrv Z findAllNextr~ ZfindNextSiblingr} ZfindNextSiblingsZfetchNextSiblingsr� Z findPreviousr� ZfindAllPreviousZ fetchPreviousr� ZfindPreviousSiblingr� ZfindPreviousSiblingsZfetchPreviousSiblingsr� Z findParentr� Z findParentsZ fetchParentsr� r� ru rz r{ r r� r� r� r) r* � tag_name_re�attribselect_rer� r� r� r� r� r� r� r� r r r r rG � s� J 6 +rG c @ s\ e Zd ZdZdZdZdd� Zdd� Zdd� Zd d � Z dd d �Z e dd� �Z e j dd� �Z dS )r/ r� Nc C s2 t |t�rtj| |�}ntj| |t�}|j� |S )a- Create a new NavigableString. When unpickling a NavigableString, this method is called with the string in DEFAULT_OUTPUT_ENCODING. That encoding needs to be passed in to the superclass's __new__ or the superclass won't know how to handle non-ASCII characters. )r. r r �DEFAULT_OUTPUT_ENCODINGr\ )r r� �ur r r r � s zNavigableString.__new__c C s t | �| �S )z�A copy of a NavigableString has the same contents and class as the original, but it is not connected to the parse tree. )�type)r r r r �__copy__ s zNavigableString.__copy__c C s t | �fS )N)r )r r r r �__getnewargs__ s z NavigableString.__getnewargs__c C s$ |dkr | S t d| jj|f ��dS )z�text.string gives you text. This is for backwards compatibility for Navigable*String, but for CData* it lets you get the string without the CData wrapper.r� z!'%s' object has no attribute '%s'N)�AttributeError� __class__r )r r r r r � __getattr__ s zNavigableString.__getattr__rI c C s | j | |�}| j| | j S )N)rO �PREFIX�SUFFIX)r rM rN r r r � output_ready s z NavigableString.output_readyc C s d S )Nr )r r r r r # s zNavigableString.namec C s t d��d S )Nz)A NavigableString cannot be given a name.)r� )r r r r r r ' s )rI )r r r r� r� rQ r r� r� r� r� r r r r r r r r/ � s r/ c @ s e Zd ZdZddd�ZdS )�PreformattedStringz�A NavigableString not subject to the normal formatting rules. The string will be passed into the formatter (to trigger side effects), but the return value will be ignored. rI c C s | j | |� | j| | j S )zUCData strings are passed into the formatter. But the return value is ignored.)rO r� r� )r rM r r r r� 2 s z PreformattedString.output_readyN)rI )r r r r r� r r r r r� + s r� c @ s e Zd ZdZdZdS )�CDataz <![CDATA[z]]>N)r r r r� r� r r r r r� 8 s r� c @ s e Zd ZdZdZdZdS )�ProcessingInstructionz A SGML processing instruction.z<?�>N)r r r r r� r� r r r r r� = s r� c @ s e Zd ZdZdZdZdS )�XMLProcessingInstructionz An XML processing instruction.z<?z?>N)r r r r r� r� r r r r r� C s r� c @ s e Zd ZdZdZdS )�Commentz<!--z-->N)r r r r� r� r r r r r� H s r� c @ s e Zd ZdZdZdS )� Declarationz<?z?>N)r r r r� r� r r r r r� N s r� c @ s e Zd Zedd� �ZdZdZdS )�Doctypec C sN |pd}|d k r2|d| 7 }|d k rF|d| 7 }n|d k rF|d| 7 }t |�S )Nr� z PUBLIC "%s"z "%s"z SYSTEM "%s")r� )r r Zpub_idZ system_idr� r r r �for_name_and_idsU s zDoctype.for_name_and_idsz <!DOCTYPE z> N)r r r r9 r� r� r� r r r r r� S s r� c @ sH e Zd ZdZdhdd�Zed�Zdd� Zedd � �Z e Z ed d � �Z e j d d � �Z d e effdd�Zee�Zedd� �Zdd e effdd�ZeZee�Zdd� Zdidd�Zdd� Zdjdd �Zdkd d �Zd d � Zd!d"� Zd#d$� Z d%d&� Z d'd(� Z d)d*� Z d+d,� Z d-d.� Z!d/d0� Z"d1d2� Z#d3d4� Z$d5d6� Z%d7d8� Z&dld:d;�Z'd<d=� Z(d>d?� Z)e*�rTe( Z)Z'e+dd@dAfdBdC�Z,dDdE� Z-de+d@fdFdG�Z.dmdHdI�Z/de+d@fdJdK�Z0de+d@fdLdM�Z1e+d dNfdOdP�Z2di dQdfdRdS�Z3e3Z4di dQddfdTdU�Z5e5Z6e5Z7edVdW� �Z8edXdY� �Z9dZd[d\gZ:d Z;e<j=d]�Z>d^d_� Z?dnd`da�Z@dbdc� ZAddde� ZBdfdg� ZCdS )ork z=Represents a found HTML tag with its attributes and contents.Nc C s� |dkrd| _ n|j| _ |dkr(td��|| _|| _|| _|dk rJ|j} n| rTg } ntj} | | _|dkrni }n4|r�|dk r�|jr�|j | j|�}q�t |�}nt |�}|r�|j | _ n| | _ || _ g | _| j||� d| _|dk r�|j| � |j|�| _nd| _dS )zBasic constructor.Nz%No value provided for new tag's name.F)� parser_classr� r] r r r r8 r, Zcdata_list_attributesZ$_replace_cdata_list_attribute_values�dictrP rQ rw r[ r\ �hiddenZset_up_substitutions�can_be_empty_element) r �parser�builderr r r rw r0 r� rP r8 r r r �__init__i sB z Tag.__init__r� c C sh t | �d| j| j| j| j| j| jd�}x dD ]}t||t| |�� q,W x | j D ]}|j |j � � qNW |S )z�A copy of a Tag is a new Tag, unconnected to the parse tree. Its contents are a copy of the old Tag's contents. N)rP r� r� )r� r� ) r� r� r r r rw rR r r r[ rf r� )r Zcloner rd r r r r� � s z Tag.__copy__c C s t | j�dko| jS )a7 Is this tag an empty-element tag? (aka a self-closing tag) A tag that has contents is never an empty-element tag. A tag that has no contents may or may not be an empty-element tag. It depends on the builder used to create the tag. If the builder has a designated list of empty-element tags, then only a tag whose name shows up in that list is considered an empty-element tag. If the builder has no designated list of empty-element tags, then any tag with no contents is an empty-element tag. r )ro r[ r� )r r r r �is_empty_element� s zTag.is_empty_elementc C s0 t | j�dkrdS | jd }t|t�r*|S |jS )aq Convenience property to get the single string within this tag. :Return: If this tag has a single string child, return value is that string. If this tag has no children, or more than one child, return value is None. If this tag has one child tag, return value is the 'string' attribute of the child tag, recursively. r% Nr )ro r[ r. r/ r� )r rd r r r r� � s z Tag.stringc C s | j � | j|j|�� d S )N)�clearrf r� )r r� r r r r� � s Fc c s^ xX| j D ]N}|dkr t|t� s|dk r6t|�|kr6q|rP|j� }t|�dkrPq|V qW dS )z�Yield all strings of certain classes, possibly stripping them. By default, yields only NavigableString and CData objects. So no comments, processing instructions, etc. Nr )� descendantsr. r/ r� �stripro )r r� �typesZ descendantr r r � _all_strings� s zTag._all_stringsc c s x| j d�D ] }|V q W d S )NT)r� )r r� r r r �stripped_strings� s zTag.stripped_stringsr� c C s |j dd� | j||d�D ��S )zP Get all child strings, concatenated using the given separator. c S s g | ]}|�qS r r )r� rL r r r � <listcomp>� s z Tag.get_text.<locals>.<listcomp>)r� )r� r� )r Z separatorr� r� r r r �get_text� s z Tag.get_textc C s8 | j � | }x&|dk r2|j}|jj� g |_|}qW dS )z/Recursively destroys the contents of this tree.N)r_ rX �__dict__r� r[ )r r� r� r r r � decompose� s z Tag.decomposec C s^ |r:xT| j dd� D ] }t|t�r,|j� q|j� qW n x | j dd� D ] }|j� qJW dS )zP Extract all children. If decompose is True, decompose instead. N)r[ r. rk r� r_ )r r� r� r r r r� s z Tag.clearc C s0 x"t | j�D ]\}}||kr |S q W td��dS )z� Find the index of a child by identity, not value. Avoids issues with tag.contents.index(element) getting the index of equal elements. z Tag.index: element not in tagN)� enumerater[ r] )r r� r� rd r r r r^ s z Tag.indexc C s | j j||�S )z�Returns the value of the 'key' attribute for the tag, or the value given for 'default' if it doesn't have that attribute.)rw rT )r �keyr� r r r rT s zTag.getc C s | j ||�}t|t�s |g}|S )z-The same as get(), but always returns a list.)rT r. rm )r r� r� r� r r r �get_attribute_list s zTag.get_attribute_listc C s || j kS )N)rw )r r� r r r r� & s z Tag.has_attrc C s t | �j� S )N)r �__hash__)r r r r r� ) s z Tag.__hash__c C s | j | S )zqtag[key] returns the value of the 'key' attribute for the tag, and throws an exception if it's not there.)rw )r r� r r r � __getitem__, s zTag.__getitem__c C s t | j�S )z0Iterating over a tag iterates over its contents.)�iterr[ )r r r r �__iter__1 s z Tag.__iter__c C s t | j�S )z:The length of a tag is the length of its list of contents.)ro r[ )r r r r �__len__5 s z Tag.__len__c C s || j kS )N)r[ )r �xr r r � __contains__9 s zTag.__contains__c C s dS )z-A tag is non-None even if it has no contents.Tr )r r r r �__bool__< s z Tag.__bool__c C s || j |< dS )zKSetting tag[key] sets the value of the 'key' attribute for the tag.N)rw )r r� r� r r r � __setitem__@ s zTag.__setitem__c C s | j j|d� dS )z;Deleting tag[key] deletes all 'key' attributes for the tag.N)rw �pop)r r� r r r � __delitem__E s zTag.__delitem__c O s | j ||�S )z�Calling a tag like a function is the same as calling its find_all() method. Eg. tag('a') returns a list of all the A tags found within this tag.)�find_all)r r= r> r r r �__call__I s z Tag.__call__c C sv t |�dkr@|jd�r@|d d� }tjdt|d� � | j|�S |jd� r`|dk r`| j|�S td| j|f ��d S ) N� rk z�.%(name)sTag is deprecated, use .find("%(name)s") instead. If you really were looking for a tag called %(name)sTag, use .find("%(name)sTag"))r �__r[ z!'%s' object has no attribute '%s'���) ro r� �warnings�warnr� �findr� r� r� )r rr r� r r r r� O s zTag.__getattr__c C s� | |kr dS t |d� sXt |d� sXt |d� sX| j|jksX| j|jksXt| �t|�kr\dS x(t| j�D ]\}}||j| krhdS qhW dS )zReturns true iff this tag has the same name, the same attributes, and the same contents (recursively) as the given tag.Tr rw r[ F)rp r rw ro r� r[ )r �otherr� Zmy_childr r r �__eq__` s z Tag.__eq__c C s | |k S )zZReturns true iff this tag is not identical to the other tag, as defined in __eq__.r )r r� r r r �__ne__q s z Tag.__ne__�unicode-escapec C s t r | j� S | j|�S dS )z Renders this tag as a string.N)�PY3K�decoder )r r r r r �__repr__v s z Tag.__repr__c C s | j � S )N)r )r r r r � __unicode__� s zTag.__unicode__c C s t r | j� S | j� S d S )N)r r r )r r r r �__str__� s z Tag.__str__rI �xmlcharrefreplacec C s | j |||�}|j||�S )N)r r )r r � indent_levelrM �errorsr� r r r r � s z Tag.encodec C s |dk o| j | jkS )z"Should this tag be pretty-printed?N)r r8 )r r r r r �_should_pretty_print� s zTag._should_pretty_printc C sR t |t� r t|� r | j|�}g }| jr�x�t| jj� �D ]�\}}|dkrP|}nrt |t�sdt |t�rpdj |�}n0t |t �s�t |�}n t |t �r�|dk r�|j |�}| j ||�}t |�d tj|� }|j|� q:W d} d} d} | jr�| jd } | j�rd} t |t��r |j�p| } nd| | jf } | j|�} d} d}|dk �rHd|d }| �r\|} |d }nd}| j|||�}| j�r||}n�g }d}|�r�ddj |� }|dk �r�|j|� |jd| | j|| f � | �r�|jd � |j|� | �r|�r|d d k�r|jd � | �r| �r|j| � |j| � |dk �rD| �rD| j�rD|jd � dj |�}|S ) a� Returns a Unicode representation of this tag and its contents. :param eventual_encoding: The tag is destined to be encoded into this encoding. This method is _not_ responsible for performing that encoding. This information is passed in so that it can be substituted in if the document contains a <META> tag that mentions the document's encoding. Nr� r� r� r z</%s%s>r% z <%s%s%s%s>� rV )r. r: rK rJ rw �sorted�itemsrm r� r� r r r rO r Zquoted_attribute_valuerf r r� r@ r r �decode_contentsr� rY )r r �eventual_encodingrM rw r� �valZdecodedrx �closeZcloseTagr � pretty_printZspaceZ indent_spaceZindent_contentsr[ rL Zattribute_stringr r r r � sx z Tag.decodec C s* |d kr| j d|d�S | j|d|d�S d S )NT)rM )r r )r r rM r r r �prettify� s z Tag.prettifyc C s� t |t� r t|� r | j|�}|dk }g }x�| D ]�}d}t |t�rP|j|�}n t |t�rn|j|j|||�� |r�|r�| j dk r�|j � }|r2|r�| j dk r�|jd|d � |j|� |r2| j dk r2|jd� q2W dj |�S )av Renders the contents of this tag as a Unicode string. :param indent_level: Each line of the rendering will be indented this many spaces. :param eventual_encoding: The tag is destined to be encoded into this encoding. This method is _not_ responsible for performing that encoding. This information is passed in so that it can be substituted in if the document contains a <META> tag that mentions the document's encoding. :param formatter: The output formatter responsible for converting entities to Unicode characters. Nr- r� r% r r� ) r. r: rK rJ r/ r� rk rf r r r� r� )r r r rM r rL �crx r r r r s( zTag.decode_contentsc C s | j |||�}|j|�S )ac Renders the contents of this tag as a bytestring. :param indent_level: Each line of the rendering will be indented this many spaces. :param eventual_encoding: The bytestring will be in this encoding. :param formatter: The output formatter responsible for converting entities to Unicode characters. )r r )r r r rM r[ r r r �encode_contents+ s zTag.encode_contentsr c C s |sd }| j ||d�S )N)r r )r )r r Z prettyPrintZ indentLevelr r r �renderContents= s zTag.renderContentsTc K s* d}| j ||||df|�}|r&|d }|S )zLReturn only the first child of this Tag matching the given criteria.Nr% r )r� )r r rw � recursiverx r> r� r� r r r r� F s zTag.findc K s& | j }|s| j}| j|||||f|�S )a� Extracts a list of Tag objects that match the given criteria. You can specify the name of the Tag and any attributes you want the Tag to have. The value of a key-value pair in the 'attrs' map can be a string, a list of strings, a regular expression object, or a callable that takes a string and returns whether or not the string matches for some custom definition of 'matches'. The same is true of the tag name.)r� �childrenrz )r r rw r rx r| r> r� r r r r� Q s z Tag.find_allc C s t | j�S )N)r� r[ )r r r r r e s z Tag.childrenc c s@ t | j�sd S | j� j}| jd }x||k r:|V |j}q$W d S )Nr )ro r[ ri rX )r ZstopNodeZcurrentr r r r� j s zTag.descendantsr� �+r� z "[^"]*:[^"]*"c C s | j |dd�}|r|d S dS )z9Perform a CSS selection operation on the current element.r% )r| r N)�select)r �selectorr� r r r � select_oney s zTag.select_onec ) s: d|kr�g }dd� |j d�D �}t� }t� }x�|D ]~}|dkrJtd| ��||krTq2|j|� �j||d�} x0| D ](} t| �} | |krr|j| � |j| � qrW |r2t|�|kr2P q2W |S tj |�} �g} | d3 �j kr�td| d4 ���j r�t d | � �x�t | �D �]�\}}g }tg �}| |d �j k�rB�j �rt d � �q�j �rVt d | � d �d �d }�j j|�}|d k �r�|j� \�}}}�j|||�}�n�d |k�r�|j d d�\���fdd�}|}�nxd|k�r|j dd�\�}t|j d��� � fdd�}|}�n<d|k�rԈjj|� �r�|j dd�\�}�dk�r@td��tjd|�}g } |d k�rd|} d } n |j� \} } | dk�r�y t| �} W n td��Y nX | dk �r�td��G dd� dt�} | | �j}ntd��nl|d k�r�n`|d k�r�d d � �nL|d k�rd!d � �n8|d"k�r d#d$� } | �n �jj|��r4|�n td%| ����rh| |d ����fd&d'�}!|!}"nV|d k�r��j �r���r�d(}#n�}#t d)|# � �j �r��fd*d+�}$|$}"nd,d � }"n|}"d-}%�x| D �]}&�j �r�t d.|&jt|&j�f � x�|"|&�D ]�} t | t ��s�q���r$| j�k�r$�q�|d k �rTy || �}'W n t k �rR P Y nX |d k�sd|'�r��j �r�t d/| jt| j�f � t| �|k�r�|j| � |jt| �� n �j �r�t d0| jt| j�f � �q�W �q�W |} �qW |�rt| �|k�r| d |� } �j �r6t d1� x"| D ]}(t d2|(j|(jf � �qW | S )5z9Perform a CSS selection operation on the current element.�,c S s g | ] }|j � �qS r )r� )r� r� r r r r� � s z Tag.select.<locals>.<listcomp>r� z"Invalid group selection syntax: %s)r| r% z-Final combinator "%s" is missing an argument.zRunning CSS selector "%s"z0 Token was consumed by the previous combinator.z Considering token "%s"N�#c s | j dd �� kS )N�id)rT )rr )�tag_idr r � id_matches� s z Tag.select.<locals>.id_matches�.c s � j | jdg ��S )N�class)�issubsetrT )� candidate)�classesr r � classes_match� s z!Tag.select.<locals>.classes_matchr z0A pseudo-class must be prefixed with a tag name.z ([a-zA-Z\d-]+)\(([a-zA-Z\d]+)\)z nth-of-typezMOnly numeric values are currently supported for the nth-of-type pseudo-class.z2nth-of-type pseudo-class value must be at least 1.c @ s e Zd Zdd� Zdd� ZdS )zTag.select.<locals>.Counterc S s d| _ || _d S )Nr )r� � destination)r r' r r r r� � s z$Tag.select.<locals>.Counter.__init__c S s& | j d7 _ | j | jkr dS dS d S )Nr% TF)r� r' )r rr r r r �nth_child_of_type� s z-Tag.select.<locals>.Counter.nth_child_of_typeN)r r r r� r( r r r r �Counter� s r) z?Only the following pseudo-classes are implemented: nth-of-type.r� r� c S s | j S )N)r )rr r r r r� s z Tag.select.<locals>.<lambda>r� c S s | j S )N)r )rr r r r r� s r c s s | j d�V d S )NT)r~ )rr r r r �next_tag_sibling s z$Tag.select.<locals>.next_tag_siblingz)Unsupported or invalid CSS selector: "%s"c 3 st �j r(td� | j| jf � tdd � x4| j� ��D ]$}�j rTtd|j|jf � |V q6W �j rptdd � d S )Nz- Calling select("%s") recursively on %s %sr� �( z,(Recursive select picked up candidate %s %s))� _select_debug�printr rw r )rr r� )� next_token� recursive_candidate_generatorr r r �recursive_select s z$Tag.select.<locals>.recursive_selectz[any]z- Default candidate generator, tag name="%s"c 3 s8 x2| j D ](}t|t�sq� r*|j� k r*q|V qW d S )N)r� r. rk r )rr rd )r� r r �default_candidate_generator7 s z/Tag.select.<locals>.default_candidate_generatorc S s | j S )N)r� )rr r r r r� @ s r z( Running candidate generator on %s %sz SUCCESS %s %sz FAILURE %s %szFinal verdict:z %s %srV rV ) r� r7 r] �addr r rf ro �shlex�_selector_combinatorsr, r- r� r� r$ �groupsr� � quoted_colonr# r) �intr< �objectr( r� r �reprrw r. rk r� ))r r Z_candidate_generatorr| �contextZ selectorsZused_selectorsZselected_object_idsZpartial_selectorZ candidatesr$ Z object_id�tokensZcurrent_contextr^ �tokenZ new_contextZnew_context_idsZchecker�mr� r� r� r �klassr& ZpseudoZpseudo_attributesr� Z pseudo_typeZ pseudo_valuer) r* r0 Z_use_candidate_generatorZcheckr1 r� rr r� r� r )r% r. r/ r r r� r r � s $ z Tag.selectc C s | j S )N)r )r r r r �childGeneratork s zTag.childGeneratorc C s | j S )N)r� )r r r r �recursiveChildGeneratorn s zTag.recursiveChildGeneratorc C s t jd| � | j|�S )z�This was kind of misleading because has_key() (attributes) was different from __in__ (contents). has_key() is gone in Python 3, anyway.z2has_key is deprecated. Use has_attr("%s") instead.)r� r� r� )r r� r r r �has_keyq s z Tag.has_key) NNNNNNNNN)F)N)N)r )NrI )NN)Dr r r r r� r Z parserClassr� r r� Z isSelfClosingr� r r/ r� r� Zstringsr� r� ZgetTextrx r� r� r^ rT r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r r� r r r r r r r r� Z findChildr� ZfindAllZ findChildrenr r� r4 r, r) r* r6 r r r? r@ rA r r r r rk e s� 3 \ ) lrk c @ sT e Zd ZdZdi dfdd�Zdd� Zdd� Zdi fd d �ZeZd d � Z dd d�Z dS )r� zMEncapsulates a number of ways of matching a markup element (tag or text).Nc K s� | j |�| _t|t�s"||d< d }d|kr<|d |d< |d= |r\|rX|j� }|j|� n|}i }x&t|j� �D ]\}}| j |�||< qnW || _| j |�| _ d S )Nr" Zclass_) �_normalize_search_valuer r. r� �copy�updaterm r rw rx )r r rw rx r> Znormalized_attrsr� r� r r r r� ~ s" zSoupStrainer.__init__c C s� t |t�s.t|�s.t|d�s.t |t�s.|d kr2|S t |t�rF|jd�S t|d�r�g }xJ|D ]B}t|d�r�t |t� r�t |t� r�|j|� qZ|j| j|�� qZW |S tt|��S )Nr$ �utf8r� ) r. r rK rp �bool�bytesr rf rB )r r� Z new_value�vr r r rB � s z$SoupStrainer._normalize_search_valuec C s | j r | j S d| j| jf S d S )Nz%s|%s)rx r rw )r r r r r � s zSoupStrainer.__str__c C s$ d }d }t |t�r|}|}t | jt�o0t |t� }| j sd|sd|rP| j|| j�sd| r�| j|| j�r�|rv| j||�}nrd}d }xht| jj� �D ]V\}} |s�t|d�r�|}n i }x|D ]\} } | || < q�W |j |�} | j| | �s�d}P q�W |r�|r�|}n|}|�r | j �r | j|j | j � �r d }|S )NTrT F) r. rk r r �_matchesrm rw r rp rT rx r� ) r Z markup_nameZ markup_attrsr� �markupZcall_function_with_tag_datar$ Zmarkup_attr_mapr � match_against�krH Z attr_valuer r r � search_tag� sB "zSoupStrainer.search_tagc C s� d }t |d�rHt|ttf� rHx�|D ] }t|t�r$| j|�r$|}P q$W npt|t�rr| j sf| jsf| jr�| j |�}nFt|t�s�t|t�r�| j r�| j r�| j || j�r�|}nt d|j ��|S )Nr� z&I don't know how to match against a %s) rp r. rk r r/ r# rx r rw rM rI � Exceptionr� )r rJ r� r� r r r r# � s" zSoupStrainer.searchc C s� d}t |t�st |t�rPx |D ]}| j||�r dS q W | jdj|�|�rLdS dS |dkr`|d k S t |t�rr||�S |}t |t�r�|j}| j|�}|d kr�| S t |d�o�t |t � �r|s�t � }xL|D ]@}|j r�|}nt |�}||kr�q�q�|j|� | j|||�r�dS q�W dS d}| �r.t |t ��r.||k}| �rLt |d��rL|j|�S | �r�t |t��r�|j�r�| j|jd |j |�S |S )NFTr� r� r# r )r. rm r� rI r� r rk r rB rp r r7 r� r r2 r# r ) r rJ rK Z already_triedr� �itemZoriginal_markupr� r$ r r r rI sV zSoupStrainer._matches)N) r r r r r� rB r rM Z searchTagr# rI r r r r r� z s 'r� c s, e Zd ZdZf f� fdd� Zdd� Z� ZS )r� zTA ResultSet is just a list that keeps track of the SoupStrainer that created it.c s t t| �j|� || _d S )N)�superr� r� �source)r rQ r� )r� r r r� V s zResultSet.__init__c C s t d| ��d S )Nz�ResultSet object has no attribute '%s'. You're probably treating a list of items like a single item. Did you call find_all() when you meant to call find()?)r� )r r� r r r r� Z s zResultSet.__getattr__)r r r r r� r� � __classcell__r r )r� r r� S s r� ),Z __license__�collections.abcr � ImportError�e� collectionsr) r3 �sysr� Z bs4.dammitr r� � version_infor r* Z whitespace_rer r r r r r! r, r8 r: rA rC rD rE rF rG r/ r� r� r� r� r� r� r� rk r� rm r� r r r r �<module> s^ % R9 Z __pycache__/dammit.cpython-36.opt-1.pyc 0000644 00000044336 15246412105 0013626 0 ustar 00 3 6]�t � @ s d Z dZddlZddlmZ ddlZddlZddlZdZyddl Z dd� Z W nF e k r� yddl Z dd� Z W n e k r� dd� Z Y nX Y nX y ddl Z W n e k r� Y nX ejd j� ej�Zejd j� ej�ZG d d � d e�ZG d d� d�ZG dd� d�ZdS )aB Beautiful Soup bonus library: Unicode, Dammit This library converts a bytestream to Unicode through any means necessary. It is heavily based on code from Mark Pilgrim's Universal Feed Parser. It works best on XML and HTML, but it does not rewrite the XML or HTML to reflect a new encoding; that's the tree builder's job. ZMIT� N)�codepoint2namec C s t j| �d S )N�encoding)�cchardet�detect)�s� r � /usr/lib/python3.6/dammit.py�chardet_dammit s r c C s t j| �d S )Nr )�chardetr )r r r r r ! s c C s d S )Nr )r r r r r ' s z!^<\?.*encoding=['"](.*?)['"].*\?>z0<\s*meta[^>]+charset\s*=\s*["']?([^>]*?)[ /;'">]c @ s� e Zd ZdZdd� Ze� \ZZZdddddd �Ze j d �Z e j d �Z e d d � �Ze dd� �Ze dd� �Ze ddd��Ze ddd��Ze dd� �ZdS ) �EntitySubstitutionzASubstitute XML or HTML entities for the corresponding characters.c C sn i } i }g }xBt tj� �D ]2\}}t|�}|dkrD|j|� || |< |||< qW ddj|� }| |tj|�fS )N�" z[%s]� )�listr �items�chr�append�join�re�compile)�lookupZreverse_lookupZcharacters_for_reZ codepoint�name� characterZ re_definitionr r r �_populate_class_variables9 s z,EntitySubstitution._populate_class_variablesZaposZquotZamp�lt�gt)�'�"�&�<�>z&([<>]|&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;))z([<>&])c C s | j j|jd��}d| S )Nr z&%s;)�CHARACTER_TO_HTML_ENTITY�get�group)�cls�matchobj�entityr r r �_substitute_html_entityZ s z*EntitySubstitution._substitute_html_entityc C s | j |jd� }d| S )zmUsed with a regular expression to substitute the appropriate XML entity for an XML special character.r z&%s;)�CHARACTER_TO_XML_ENTITYr" )r# r$ r% r r r �_substitute_xml_entity_ s z)EntitySubstitution._substitute_xml_entityc C s6 d}d|kr*d|kr&d}|j d|�}nd}|| | S )a* Make a value into a quoted XML attribute, possibly escaping it. Most strings will be quoted using double quotes. Bob's Bar -> "Bob's Bar" If a string contains double quotes, it will be quoted using single quotes. Welcome to "my bar" -> 'Welcome to "my bar"' If a string contains both single and double quotes, the double quotes will be escaped, and the string will be quoted using double quotes. Welcome to "Bob's Bar" -> "Welcome to "Bob's bar" r r z")�replace)�self�valueZ quote_withZ replace_withr r r �quoted_attribute_valuef s z)EntitySubstitution.quoted_attribute_valueFc C s" | j j| j|�}|r | j|�}|S )a Substitute XML entities for special XML characters. :param value: A string to be substituted. The less-than sign will become <, the greater-than sign will become >, and any ampersands will become &. If you want ampersands that appear to be part of an entity definition to be left alone, use substitute_xml_containing_entities() instead. :param make_quoted_attribute: If True, then the string will be quoted, as befits an attribute value. )�AMPERSAND_OR_BRACKET�subr( r, )r# r+ �make_quoted_attributer r r �substitute_xml� s z!EntitySubstitution.substitute_xmlc C s" | j j| j|�}|r | j|�}|S )a� Substitute XML entities for special XML characters. :param value: A string to be substituted. The less-than sign will become <, the greater-than sign will become >, and any ampersands that are not part of an entity defition will become &. :param make_quoted_attribute: If True, then the string will be quoted, as befits an attribute value. )�BARE_AMPERSAND_OR_BRACKETr. r( r, )r# r+ r/ r r r �"substitute_xml_containing_entities� s z5EntitySubstitution.substitute_xml_containing_entitiesc C s | j j| j|�S )a� Replace certain Unicode characters with named HTML entities. This differs from data.encode(encoding, 'xmlcharrefreplace') in that the goal is to make the result more readable (to those with ASCII displays) rather than to recover from errors. There's absolutely nothing wrong with a UTF-8 string containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that character with "é" will make it more readable to some people. )�CHARACTER_TO_HTML_ENTITY_REr. r& )r# r r r r �substitute_html� s z"EntitySubstitution.substitute_htmlN)F)F)�__name__� __module__� __qualname__�__doc__r r ZHTML_ENTITY_TO_CHARACTERr3 r' r r r1 r- � classmethodr& r( r, r0 r2 r4 r r r r r 5 s$ % r c @ sH e Zd ZdZddd�Zdd� Zedd � �Zed d � �Z edd d ��Z dS )�EncodingDetectora^ Suggests a number of possible encodings for a bytestring. Order of precedence: 1. Encodings you specifically tell EncodingDetector to try first (the override_encodings argument to the constructor). 2. An encoding declared within the bytestring itself, either in an XML declaration (if the bytestring is to be interpreted as an XML document), or in a <meta> tag (if the bytestring is to be interpreted as an HTML document.) 3. An encoding detected through textual analysis by chardet, cchardet, or a similar external library. 4. UTF-8. 5. Windows-1252. NFc C sN |pg | _ |pg }tdd� |D ��| _d | _|| _d | _| j|�\| _| _d S )Nc S s g | ] }|j � �qS r )�lower)�.0�xr r r � <listcomp>� s z-EncodingDetector.__init__.<locals>.<listcomp>) �override_encodings�set�exclude_encodings�chardet_encoding�is_html�declared_encoding�strip_byte_order_mark�markup�sniffed_encoding)r* rF r? rC rA r r r �__init__� s zEncodingDetector.__init__c C s8 |d k r4|j � }|| jkr dS ||kr4|j|� dS dS )NFT)r; rA �add)r* r �triedr r r �_usable� s zEncodingDetector._usablec c s� t � }x | jD ]}| j||�r|V qW | j| j|�r>| jV | jdkrZ| j| j| j�| _| j| j|�rp| jV | jdkr�t | j�| _| j| j|�r�| jV x dD ]}| j||�r�|V q�W dS )z<Yield a number of encodings that might work for this markup.N�utf-8� windows-1252)rL rM ) r@ r? rK rG rD �find_declared_encodingrF rC rB r )r* rJ �er r r � encodings� s$ zEncodingDetector.encodingsc C s� d}t |t�r||fS t|�dkrT|dd� dkrT|dd� dkrTd}|dd� }n�t|�dkr�|dd� dkr�|dd� dkr�d}|dd� }nd|dd � d kr�d }|d d� }nB|dd� d kr�d }|dd� }n |dd� dkr�d}|dd� }||fS )zMIf a byte-order mark is present, strip it and return the encoding it implies.N� � s ��z zutf-16bes ��zutf-16le� s zutf-8s ��zutf-32bes �� zutf-32le)� isinstance�str�len)r# �datar r r r rE s* z&EncodingDetector.strip_byte_order_markc C s� |rt |� }}nd}tdtt |�d ��}d}tj||d�}| rV|rVtj||d�}|dk rr|j� d jdd�}|r~|j� S dS ) z�Given a document, tries to find its declared encoding. An XML encoding is declared at the beginning of the document. An HTML encoding is declared in a <meta> tag, hopefully near the beginning of the document. i i g�������?N)�endposr �asciir) ) rV �max�int�xml_encoding_re�search� html_meta_re�groups�decoder; )r# rF rC Zsearch_entire_documentZ xml_endposZ html_endposrD Zdeclared_encoding_matchr r r rN + s z'EncodingDetector.find_declared_encoding)NFN)FF) r5 r6 r7 r8 rH rK �propertyrP r9 rE rN r r r r r: � s ! r: c � @ sR e Zd ZdZddd�ZdddgZg dd g fd d �Zd d � Z�d dd�Z�d dd�Z e dd� �Z dd� Z dd� Z �d d�d �d!�d"�d#�d$�d%�d&�d'�d(�d)�d*d2�d+d2d2�d,�d-�d.�d/�d0�d1�d2�d3�d4�d5�d6�d7d2�d8�d9dQ� ZdRddSdTdUdVdWdXdYdZd[d\d]d2d^d2d2d_d_d`d`dadbdcdddedfdgdhd2didjddkdldmdndodpd[dqdPdrdsdkddtdbdudvdwdx�d:dzd{dadSd|drd}d~dd�d2d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�dad�d�d�d�d�djd�d�d�d�d�d�d�d�d�dld�d�d�d�d�d�d�d�dud�dududududud�dudzdzdzdzd�d�d�d���Zd�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d��d �d�d�d�d�d�d�d�d�d �d �d �d �d �d�zZ�d;�d<�d=gZe�d �d Ze�d> �d Ze�d?�d�d ��ZdS (@ � UnicodeDammitz�A class for detecting the encoding of a *ML document and converting it to a Unicode string. If the source encoding is windows-1252, can replace MS smart quotes with their HTML or XML equivalents.z mac-romanz shift-jis)� macintoshzx-sjis� windows-1252z iso-8859-1z iso-8859-2NFc C s� || _ g | _d| _|| _tjt�| _t||||�| _ t |t �sF|dkr`|| _ t |�| _ d | _d S | j j | _ d }x,| j jD ] }| j j }| j|�}|d k rxP qxW |s�x@| j jD ]4}|dkr�| j|d�}|d k r�| jjd� d| _P q�W || _ |s�d | _d S )NFr rY r) zSSome characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.T)�smart_quotes_to�tried_encodingsZ contains_replacement_charactersrC �loggingZ getLoggerr5 �logr: �detectorrT rU rF Zunicode_markup�original_encodingrP � _convert_fromZwarning)r* rF r? re rC rA �ur r r r rH X s> zUnicodeDammit.__init__c C s� |j d�}| jdkr&| jj|�j� }nf| jj|�}t|�tkr�| jdkrfdj� |d j� dj� }q�dj� |d j� dj� }n|j� }|S )z[Changes a MS smart quote character to an XML or HTML entity, or an ASCII character.� rY Zxmlz&#x�;r r )r" re �MS_CHARS_TO_ASCIIr! �encode�MS_CHARS�type�tuple)r* �matchZorigr. r r r � _sub_ms_char� s zUnicodeDammit._sub_ms_char�strictc C s� | j |�}| s ||f| jkr"d S | jj||f� | j}| jd k rh|| jkrhd}tj|�}|j| j |�}y | j |||�}|| _|| _ W n t k r� } zd S d }~X nX | jS )Ns ([�-�])) � find_codecrf r rF re �ENCODINGS_WITH_SMART_QUOTESr r r. ru � _to_unicoderj � Exception)r* Zproposed�errorsrF Zsmart_quotes_reZsmart_quotes_compiledrl rO r r r rk � s" zUnicodeDammit._convert_fromc C s t |||�S )zGiven a string and its encoding, decodes the string into Unicode. %encoding is a string recognized by encodings.aliases)rU )r* rW r r{ r r r ry � s zUnicodeDammit._to_unicodec C s | j s d S | jjS )N)rC ri rD )r* r r r �declared_html_encoding� s z$UnicodeDammit.declared_html_encodingc C s` | j | jj||��pN|r*| j |jdd��pN|r@| j |jdd��pN|rL|j� pN|}|r\|j� S d S )N�-r �_)�_codec�CHARSET_ALIASESr! r) r; )r* �charsetr+ r r r rw � s zUnicodeDammit.find_codecc C s< |s|S d }yt j|� |}W n ttfk r6 Y nX |S )N)�codecsr � LookupError� ValueError)r* r� �codecr r r r � s zUnicodeDammit._codec�euro�20AC� �sbquo�201A�fnof�192�bdquo�201E�hellip�2026�dagger�2020�Dagger�2021�circ�2C6�permil�2030�Scaron�160�lsaquo�2039�OElig�152�?�#x17D�17D�lsquo�2018�rsquo�2019�ldquo�201C�rdquo�201D�bull�2022�ndash�2013�mdash�2014�tilde�2DC�trade�2122�scaron�161�rsaquo�203A�oelig�153�#x17E�17E�Yumlr ) � �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �ZEUR�,�fz,,z...�+z++�^�%�Sr ZOE�Zr r �*r} z--�~z(TM)r r Zoe�z�Y�!�cZGBP�$ZYEN�|z..z(th)z<<z(R)�oz+-�2�3�acuterl �P�1z>>z1/4z1/2z3/4�AZAE�C�E�I�D�N�O�U�b�B�aZaerO �i�n�/�y)�r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� � �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �s €s ‚s ƒs „s …s †s ‡s ˆs ‰s Šs ‹s Œs Žs ‘s ’s “s ”s •s –s —s ˜s ™s šs ›s œs žs Ÿs s ¡s ¢s £s ¤s ¥s ¦s §s ¨s ©s ªs «s ¬s s ®s ¯s °s ±s ²s ³s ´s µs ¶s ·s ¸s ¹s ºs »s ¼s ½s ¾s ¿s Às Ás Âs Ãs Äs Ås Æs Çs Ès És Ês Ës Ìs Ís Îs Ïs Ðs Ñs Òs Ós Ôs Õs Ös ×s Øs Ùs Ús Ûs Üs Ýs Þs ßs àr s âs ãs äs ås æs çs ès és ês ës ìs ís îs ïs ðs ñs òs ós ôs õs ös ÷s øs ùs ús ûs üs ýs þ)z� � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� � � � � � � � � � � r� r� rR r� r� rS r� r� rQ r rm �utf8c C s$ |j dd�j� dkr td��|j� dkr0td��g }d }d }x�|t|�k � r�|| }t|t�sft|�}|| jkr�|| jkr�xz| j D ]$\}} } ||kr�|| kr�|| 7 }P q�W q>|d kr�|| j kr�|j |||� � |j | j | � |d 7 }|}q>|d 7 }q>W |d k�r|S |j ||d � � d j |�S )a� Fix characters from one encoding embedded in some other encoding. Currently the only situation supported is Windows-1252 (or its subset ISO-8859-1), embedded in UTF-8. The input must be a bytestring. If you've already converted the document to Unicode, you're too late. The output is a bytestring in which `embedded_encoding` characters have been converted to their `main_encoding` equivalents. r~ r} � windows-1252� windows_1252zPWindows-1252 and ISO-8859-1 are the only currently supported embedded encodings.r� �utf-8z4UTF-8 is the only currently supported main encoding.r ra rm N� )r� r� )r� r� ) r) r; �NotImplementedErrorrV rT r[ �ord�FIRST_MULTIBYTE_MARKER�LAST_MULTIBYTE_MARKER�MULTIBYTE_MARKERS_AND_SIZES�WINDOWS_1252_TO_UTF8r r ) r# Zin_bytesZ main_encodingZembedded_encodingZ byte_chunksZ chunk_start�posZbyte�start�end�sizer r r � detwingle s< zUnicodeDammit.detwingle)rv )rv )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r )r r� )r� r� rR )r� r� rS )r� r� rQ ���)r� rd )r5 r6 r7 r8 r� rx rH ru rk ry ra r| rw r rq ro r� r� r� r� r9 r� r r r r rb E s` 1 rb )r8 Z __license__r� Z html.entitiesr r rg �stringZ chardet_typer r � ImportErrorr Z iconv_codecr rp r� r\ r^ �objectr r: rb r r r r �<module> s8 __pycache__/diagnose.cpython-36.opt-1.pyc 0000644 00000016740 15246412105 0014142 0 ustar 00 3 #6]� � @ s d Z dZddlZddlmZ ddlmZ ddlZddlmZm Z ddl m Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZdd � Zd#d d �ZG d d� de�Zdd� ZdZdZd$dd�Zd%dd�Zd&dd�Zd'd d �Z d(d d!�Z e d"k�reej j � � dS ))z=Diagnostic functions, mainly for use when doing tech support.ZMIT� N)�StringIO)� HTMLParser)� BeautifulSoup� __version__)�builder_registryc ; C sX t dt � t dtj � dddg}x>|D ]6}x0tjD ]}||jkr6P q6W |j|� t d| � q*W d|kr�|jd� y*dd l m } t d d j t t |j�� � W n* tk r� } zt d � W Y d d }~X nX d|k�r ydd l}t d|j � W n, tk �r } zt d� W Y d d }~X nX t| d��r4| j� } n�| jd��sL| jd��rdt d| � t d� d S y:tjj| ��r�t d| � t| ��}|j� } W d Q R X W n tk �r� Y nX t � x�|D ]�}t d| � d} yt| |d�} d} W n8 tk �r" } zt d| � tj � W Y d d }~X nX | �rBt d| � t | j � � t d d � �q�W d S ) z/Diagnostic suite for isolating common problems.z'Diagnostic running on Beautiful Soup %szPython version %sz html.parser�html5lib�lxmlz;I noticed that %s is not installed. Installing it may help.zlxml-xmlr )�etreezFound lxml version %s�.z.lxml is not installed or couldn't be imported.NzFound html5lib version %sz2html5lib is not installed or couldn't be imported.�readzhttp:zhttps:z<"%s" looks like a URL. Beautiful Soup is not an HTTP client.zpYou need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup.z7"%s" looks like a filename. Reading data from the file.z#Trying to parse your markup with %sF)�featuresTz %s could not parse the markup.z#Here's what %s did with the markup:�-�P ) �printr �sys�versionr Zbuildersr �remove�appendr r �join�map�strZ LXML_VERSION� ImportErrorr �hasattrr � startswith�os�path�exists�open� ValueErrorr � Exception� traceback� print_excZprettify) �dataZ basic_parsers�nameZbuilderr �er �fp�parser�success�soup� r) � /usr/lib/python3.6/diagnose.py�diagnose sj r+ Tc K sN ddl m} x<|jt| �fd|i|��D ] \}}td||j|jf � q(W dS )z�Print out the lxml events that occur during parsing. This lets you see how lxml parses a document when no Beautiful Soup code is running. r )r �htmlz %s, %4s, %sN)r r Z iterparser r �tag�text)r" r, �kwargsr Zevent�elementr) r) r* � lxml_traceZ s $r1 c @ s` e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dd� Z dS )�AnnouncingParserz?Announces HTMLParser parse events, without doing anything else.c C s t |� d S )N)r )�self�sr) r) r* �_pg s zAnnouncingParser._pc C s | j d| � d S )Nz%s START)r5 )r3 r# Zattrsr) r) r* �handle_starttagj s z AnnouncingParser.handle_starttagc C s | j d| � d S )Nz%s END)r5 )r3 r# r) r) r* � handle_endtagm s z AnnouncingParser.handle_endtagc C s | j d| � d S )Nz%s DATA)r5 )r3 r" r) r) r* � handle_datap s z AnnouncingParser.handle_datac C s | j d| � d S )Nz %s CHARREF)r5 )r3 r# r) r) r* �handle_charrefs s z AnnouncingParser.handle_charrefc C s | j d| � d S )Nz %s ENTITYREF)r5 )r3 r# r) r) r* �handle_entityrefv s z!AnnouncingParser.handle_entityrefc C s | j d| � d S )Nz %s COMMENT)r5 )r3 r" r) r) r* �handle_commenty s z AnnouncingParser.handle_commentc C s | j d| � d S )Nz%s DECL)r5 )r3 r" r) r) r* � handle_decl| s z AnnouncingParser.handle_declc C s | j d| � d S )Nz%s UNKNOWN-DECL)r5 )r3 r" r) r) r* � unknown_decl s z AnnouncingParser.unknown_declc C s | j d| � d S )Nz%s PI)r5 )r3 r" r) r) r* � handle_pi� s zAnnouncingParser.handle_piN)�__name__� __module__� __qualname__�__doc__r5 r6 r7 r8 r9 r: r; r< r= r> r) r) r) r* r2 d s r2 c C s t � }|j| � dS )z�Print out the HTMLParser events that occur during parsing. This lets you see how HTMLParser parses a document when no Beautiful Soup code is running. N)r2 Zfeed)r" r& r) r) r* �htmlparser_trace� s rC ZaeiouZbcdfghjklmnpqrstvwxyz� c C s> d}x4t | �D ](}|d dkr$t}nt}|tj|�7 }qW |S )z#Generate a random word-like string.� � r )�range� _consonants�_vowels�random�choice)�lengthr4 �i�tr) r) r* �rword� s rO � c C s dj dd� t| �D ��S )z'Generate a random sentence-like string.� c s s | ]}t tjd d��V qdS )rP � N)rO rJ �randint)�.0rM r) r) r* � <genexpr>� s z rsentence.<locals>.<genexpr>)r rG )rL r) r) r* � rsentence� s rV �� c C s� dddddddg}g }x~t | �D ]r}tjdd �}|dkrRtj|�}|jd | � q |d krr|jttjd d ��� q |d kr tj|�}|jd| � q W ddj|� d S )z+Randomly generate an invalid HTML document.�pZdiv�spanrM �bZscript�tabler � z<%s>� rP rF z</%s>z<html>� z</html>)rG rJ rS rK r rV r )� num_elementsZ tag_names�elementsrM rK Ztag_namer) r) r* �rdoc� s ra 順 c C s( t dt � t| �}t dt|� � x�dddgddgD ]z}d}y"tj� }t||�}tj� }d}W n6 tk r� } zt d | � tj� W Y d d }~X nX |r6t d ||| f � q6W d d l m } tj� }|j |� tj� }t d|| � d d l } | j � }tj� }|j|� tj� }t d|| � d S )z.Very basic head-to-head performance benchmark.z1Comparative parser benchmark on Beautiful Soup %sz3Generated a large invalid HTML document (%d bytes).r r, r z html.parserFTz %s could not parse the markup.Nz"BS4+%s parsed the markup in %.2fs.r )r z$Raw lxml parsed the markup in %.2fs.z(Raw html5lib parsed the markup in %.2fs.)r r ra �len�timer r r r! r r ZHTMLr r �parse) r_ r" r& r' �ar( rZ r$ r r r) r) r* �benchmark_parsers� s4 rg r c C sX t j� }|j}t| �}tt||d�}tjd|||� tj |�}|j d� |j dd� d S )N)�bs4r" r& z bs4.BeautifulSoup(data, parser)Z cumulativez _html5lib|bs4�2 ) �tempfileZNamedTemporaryFiler# ra �dictrh �cProfileZrunctx�pstatsZStatsZ sort_statsZ print_stats)r_ r& Z filehandle�filenamer" �varsZstatsr) r) r* �profile� s rp �__main__)T)rD )rP )rW )rb )rb r )!rB Z __license__rl �ior Z html.parserr rh r r Z bs4.builderr r rm rJ rj rd r r r+ r1 r2 rC rI rH rO rV ra rg rp r? �stdinr r) r) r) r* �<module> s8 C ! __pycache__/testing.cpython-36.pyc 0000644 00000077637 15246412105 0013103 0 ustar 00 3 "6]�~ � @ s� d Z dZddlZddlZddlZddlZddlmZ ddlmZ ddl m Z m Z m Z m Z mZ ddlmZ eZG dd � d ej�ZG d d � d e�ZG d d � d e�ZG dd� de�Zdd� ZdS )zHelper classes for tests.ZMIT� N)�TestCase)� BeautifulSoup)�CharsetMetaAttributeValue�Comment�ContentMetaAttributeValue�Doctype� SoupStrainer)�HTMLParserTreeBuilderc @ s: e Zd Zedd� �Zdd� Zdd� Zd dd �Zd d � ZdS ) �SoupTestc C s t � S )N)�default_builder)�self� r � /usr/lib/python3.6/testing.pyr s zSoupTest.default_builderc K s" |j d| j�}t|fd|i|��S )z*Build a Beautiful Soup object from markup.�builder)�popr r )r �markup�kwargsr r r r �soup s z SoupTest.soupc C s | j j|�S )z[Turn an HTML fragment into a document. The details depend on the builder. )r Ztest_fragment_to_document)r r r r r � document_for% s zSoupTest.document_forNc C s8 | j }t||d�}|d kr |}| j|j� | j|�� d S )N)r )r r � assertEqual�decoder )r Zto_parseZcompare_parsed_tor �objr r r �assertSoupEquals, s zSoupTest.assertSoupEqualsc C s< d}x2|j D ](}|r0| j||j� | j||j� |}q W dS )zyEnsure that next_element and previous_element are properly set for all descendants of the given element. N)Z descendantsr � next_element�previous_element)r �elementZearlier�er r r �assertConnectedness4 s z SoupTest.assertConnectedness)N) �__name__� __module__� __qualname__�propertyr r r r r r r r r r s r c @ s� e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd� Zd d � Zd d � Zd d!� Zd"d#� Zd$d%� Zd&d'� Zd(d)� Zd*d+� Zd,d-� Zd.d/� Zd0d1� Zd2d3� Z d4d5� Z d6d7� Z d8d9� Z d:d;� Z d<d=� Z!d>d?� Z"d@dA� Z#dBdC� Z$dDdE� Z%dFd� ZdGdH� Z&dIdJ� Z'dKdL� Z(dMdN� Z)dOdP� Z*dQdR� Z+dSdT� Z,dUdV� Z-dWdX� Z.dYdZ� Z/d[d\� Z0d]d^� Z1d_d`� Z2dadb� Z3dcdd� Z4dedf� Z5dgdh� Z6didj� Z7dkdl� Z8dmdn� Z9dodp� Z:dqS )r�HTMLTreeBuilderSmokeTestaC A basic test of a treebuilder's competence. Any HTML treebuilder, present or future, should be able to pass these tests. With invalid markup, there's room for interpretation, and different parsers can handle it differently. But with the markup in these tests, there's not much room for interpretation. c C s4 x.dD ]&}| j d�}|j|�}| jd|j� qW dS )zmVerify that all HTML4 and HTML5 empty element (aka void element) tags are handled correctly. �area�base�br�col�embed�hr�img�input�keygen�link�menuitem�meta�param�source�track�wbr�spacer�frame� TN)r# r$ r% r& r' r( r) r* r+ r, r- r. r/ r0 r1 r2 r3 r4 )r �new_tagr �is_empty_element)r �namer r6 r r r �test_empty_element_tagsI s z0HTMLTreeBuilderSmokeTest.test_empty_element_tagsc C sF | j d�}tj|d�}tj|�}| j|jt� | j|j� |j� � d S )Nz <a><b>foo</a>� )r �pickle�dumps�loadsr � __class__r r )r �tree�dumped�loadedr r r �!test_pickle_and_unpickle_identityU s z:HTMLTreeBuilderSmokeTest.test_pickle_and_unpickle_identityc C sf | j |�\}}|jd }| j|jt� | j||� | jt|�dt|�� |� | j|jjd d� dS )z8Assert that a given doctype string is handled correctly.r N�foo)�_document_with_doctype�contentsr r> r �str�len�p)r �doctype_fragmentZ doctype_strr �doctyper r r �assertDoctypeHandled^ s z-HTMLTreeBuilderSmokeTest.assertDoctypeHandledc C s" d| }|d }| j |�}||fS )z5Generate and parse a document with the given doctype.z <!DOCTYPE %s>z <p>foo</p>)r )r rI rJ r r r r r rD l s z/HTMLTreeBuilderSmokeTest._document_with_doctypec C s | j d� | j d� dS )z?Make sure normal, everyday HTML doctypes are handled correctly.�htmlz4html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"N)rK )r r r r �test_normal_doctypess s z-HTMLTreeBuilderSmokeTest.test_normal_doctypesc C s( | j d�}|jd }| jd|j� � d S )Nz <!DOCTYPE>r r5 )r rE r �strip)r r rJ r r r �test_empty_doctypey s z+HTMLTreeBuilderSmokeTest.test_empty_doctypec C s d}| j |� d S )Nznhtml PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd")rK )r rJ r r r � test_public_doctype_with_url~ s z5HTMLTreeBuilderSmokeTest.test_public_doctype_with_urlc C s | j d� d S )Nz$foo SYSTEM "http://www.example.com/")rK )r r r r �test_system_doctype� s z,HTMLTreeBuilderSmokeTest.test_system_doctypec C s | j d� d S )Nz#xsl:stylesheet SYSTEM "htmlent.dtd")rK )r r r r � test_namespaced_system_doctype� s z7HTMLTreeBuilderSmokeTest.test_namespaced_system_doctypec C s | j d� d S )Nz#xsl:stylesheet PUBLIC "htmlent.dtd")rK )r r r r � test_namespaced_public_doctype� s z7HTMLTreeBuilderSmokeTest.test_namespaced_public_doctypec C s4 d}| j |�}| j|jd�jdd�|jdd�� dS )zJA real XHTML document should come out more or less the same as it went in.s� <?xml version="1.0" encoding="utf-8"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"> <html xmlns="http://www.w3.org/1999/xhtml"> <head><title>Hello.</title></head> <body>Goodbye.</body> </html>zutf-8� � N)r r �encode�replace)r r r r r r �test_real_xhtml_document� s z1HTMLTreeBuilderSmokeTest.test_real_xhtml_documentc C s( d}| j |�}| jdt|jd��� dS )ztWhen a namespaced XML document is parsed as HTML it should be treated as HTML with weird tag names. s. <ns1:foo>content</ns1:foo><ns1:foo/><ns2:foo/>r: zns1:fooN)r r rG �find_all)r r r r r r �test_namespaced_html� s z-HTMLTreeBuilderSmokeTest.test_namespaced_htmlc C sB d}| j |�}| j||j� � d}| j |�}| j||jd�� d S )Nz<?PITarget PIContent?>s <?PITarget PIContent?>�utf8)r r r rV )r r r r r r �test_processing_instruction� s z4HTMLTreeBuilderSmokeTest.test_processing_instructionc C s t j| j� dS )z�Make sure you can copy the tree builder. This is important because the builder is part of a BeautifulSoup object, and we want to be able to copy that. N)�copy�deepcopyr )r r r r � test_deepcopy� s z&HTMLTreeBuilderSmokeTest.test_deepcopyc C s. | j d�}| j|jj� | jt|j�d� dS )z�A <p> tag is never designated as an empty-element tag. Even if the markup shows it as an empty-element tag, it shouldn't be presented that way. z<p/>z<p></p>N)r Z assertFalserH r7 r rF )r r r r r �!test_p_tag_is_never_empty_element� s z:HTMLTreeBuilderSmokeTest.test_p_tag_is_never_empty_elementc C s( | j dd� | j dd� | j dd� dS )z�A tag that's not closed by the end of the document should be closed. This applies to all tags except empty-element tags. z<p>z<p></p>z<b>z<b></b>z<br>z<br/>N)r )r r r r � test_unclosed_tags_get_closed� s z6HTMLTreeBuilderSmokeTest.test_unclosed_tags_get_closedc C s. | j d�}| j|jj� | jt|j�d� dS )z�A <br> tag is designated as an empty-element tag. Some parsers treat <br></br> as one <br/> tag, some parsers as two tags, but it should always be an empty-element tag. z <br></br>z<br/>N)r � assertTruer% r7 r rF )r r r r r �#test_br_is_always_empty_element_tag� s z<HTMLTreeBuilderSmokeTest.test_br_is_always_empty_element_tagc C s | j d� d S )Nz<em><em></em></em>)r )r r r r � test_nested_formatting_elements� s z8HTMLTreeBuilderSmokeTest.test_nested_formatting_elementsc C s( d}| j |�}| jd|jd�d � d S )Nz�<!DOCTYPE html> <html> <head> <title>Ordinary HEAD element test</title> </head> <script type="text/javascript"> alert("Help!"); </script> <body> Hello, world! </body> </html> ztext/javascript�script�type)r r �find)r rL r r r r �test_double_head� s z)HTMLTreeBuilderSmokeTest.test_double_headc C sj d}| j |� | j|�}|jdd�}| j|jt� |jdd�}| j||j� |jdd�}| j||j� d S )Nz<p>foo<!--foobar-->baz</p>Zfoobar)�textrC �baz)r r rg r r> r r r )r r r ZcommentrC rj r r r � test_comment� s z%HTMLTreeBuilderSmokeTest.test_commentc C st d}d}| j |� | j |� | j|�}| j|jj� |� | j|�}| j|jj� |� | jd�}| j|jj� d� dS )zWhitespace must be preserved in <pre> and <textarea> tags, even if that would mean not prettifying the markup. z<pre> </pre>z <textarea> woo woo </textarea>z<textarea></textarea>N)r r r ZpreZprettifyZtextarea)r Z pre_markupZtextarea_markupr r r r �-test_preserved_whitespace_in_pre_and_textarea� s zFHTMLTreeBuilderSmokeTest.test_preserved_whitespace_in_pre_and_textareac C s. d}| j |� d}| j |� d}| j |� dS )z+Inline elements can be nested indefinitely.z<b>Inside a B tag</b>z!<p>A <i>nested <b>tag</b></i></p>z/<p>A <a>doubly <i>nested <b>tag</b></i></a></p>N)r )r Zb_tagZ nested_b_tagZdouble_nested_b_tagr r r �test_nested_inline_elements s z4HTMLTreeBuilderSmokeTest.test_nested_inline_elementsc C s6 | j d�}|j}| j|jjjd� | j|jjd� dS )z Block elements can be nested.z*<blockquote><p><b>Foo</b></p></blockquote>ZFooN)r � blockquoter rH �b�string)r r rn r r r � test_nested_block_level_elements s z9HTMLTreeBuilderSmokeTest.test_nested_block_level_elementsc C s d}| j |d� | j d� dS )z$One table can go inside another one.z[<table id="1"><tr><td>Here's another table:<table id="2"><tr><td>foo</td></tr></table></td>zh<table id="1"><tr><td>Here's another table:<table id="2"><tr><td>foo</td></tr></table></td></tr></table>z{<table><thead><tr><td>Foo</td></tr></thead><tbody><tr><td>Bar</td></tr></tbody><tfoot><tr><td>Baz</td></tr></tfoot></table>N)r )r r r r r � test_correctly_nested_tables s z5HTMLTreeBuilderSmokeTest.test_correctly_nested_tablesc C s( d}| j |�}| jdg|jjd � d S )Nz1<table><div><div class="css"></div></div></table>Zcss�class)r r Zdiv)r r r r r r �(test_deeply_nested_multivalued_attribute0 s zAHTMLTreeBuilderSmokeTest.test_deeply_nested_multivalued_attributec C s( d}| j |�}| jddg|jd � d S )Nz<html class="a b"></html>�aro rs )r r rL )r r r r r r �"test_multivalued_attribute_on_html8 s z;HTMLTreeBuilderSmokeTest.test_multivalued_attribute_on_htmlc C s | j dd� d S )Nz<a b="<a>"></a>z<a b="<a>"></a>)r )r r r r �3test_angle_brackets_in_attribute_values_are_escaped@ s zLHTMLTreeBuilderSmokeTest.test_angle_brackets_in_attribute_values_are_escapedc C s | j dd� d S )Nz$<p>• AT&T is in the s&p 500</p>z,<p>\u2022 AT&T is in the s&p 500</p>)r )r r r r �3test_strings_resembling_character_entity_referencesC s zLHTMLTreeBuilderSmokeTest.test_strings_resembling_character_entity_referencesc C s" d}| j |�}| jd|jj� d S )Nz%<p>“Hello” -☃</p>u “Hello” -☃)r r rH rp )r r r r r r �*test_entities_in_foreign_document_encodingK s zCHTMLTreeBuilderSmokeTest.test_entities_in_foreign_document_encodingc C s8 d}| j d|� | j d|� | j d|� | j d|� d S )Nu <p id="piñata"></p>z<p id="piñata"></p>z<p id="piñata"></p>z<p id="piñata"></p>z<p id="piñata"></p>)r )r �expectr r r �0test_entities_in_attributes_converted_to_unicodeW s zIHTMLTreeBuilderSmokeTest.test_entities_in_attributes_converted_to_unicodec C s8 d}| j d|� | j d|� | j d|� | j d|� d S )Nu <p>piñata</p>z<p>piñata</p>z<p>piñata</p>z<p>piñata</p>z<p>piñata</p>)r )r rz r r r �*test_entities_in_text_converted_to_unicode^ s zCHTMLTreeBuilderSmokeTest.test_entities_in_text_converted_to_unicodec C s | j dd� d S )Nz#<p>I said "good day!"</p>z<p>I said "good day!"</p>)r )r r r r �,test_quot_entity_converted_to_quotation_marke s zEHTMLTreeBuilderSmokeTest.test_quot_entity_converted_to_quotation_markc C s, d}| j d|� | j d|� | j d|� d S )Nu �z�z�z �)r )r rz r r r �test_out_of_range_entityi s z1HTMLTreeBuilderSmokeTest.test_out_of_range_entityc C s< | j d�}| jd|jjjj� | jd|jj� | j|� dS )zDMostly to prevent a recurrence of a bug in the html5lib treebuilder.z!<html><h2> foo</h2><p></p></html>rH N)r r Zh2rp r r8 rH r )r r r r r �test_multipart_stringso s z/HTMLTreeBuilderSmokeTest.test_multipart_stringsc C s | j dd� | j dd� dS )zqVerify consistent handling of empty-element tags, no matter how they come in through the markup. z<br/><br/><br/>z<br /><br /><br />N)r )r r r r r9 v s c C s, d}| j |�}| jd|jj� | j|� dS )z8Prevent recurrence of a bug in the html5lib treebuilder.z?<html><head></head> <link></link> <body>foo</body> </html> N)r ZassertNotEqualrL �bodyr )r �contentr r r r �#test_head_tag_between_head_and_body} s z<HTMLTreeBuilderSmokeTest.test_head_tag_between_head_and_bodyc C s d}| j |�}| j|j� dS )z8Prevent recurrence of a bug in the html5lib treebuilder.z�<!DOCTYPE html> <html> <body> <article id="a" > <div><a href="1"></div> <footer> <a href="2"></a> </footer> </article> </body> </html> N)r r Zarticle)r r� r r r r � test_multiple_copies_of_a_tag� s z6HTMLTreeBuilderSmokeTest.test_multiple_copies_of_a_tagc C s^ d}| j |�}| j||j� � |j}| jd|jd � | jd|jd � | jd|jd � dS ) z�Parsers don't need to *understand* namespaces, but at the very least they should not choke on namespaces or lose data.s� <html xmlns="http://www.w3.org/1999/xhtml" xmlns:mathml="http://www.w3.org/1998/Math/MathML" xmlns:svg="http://www.w3.org/2000/svg"><head></head><body><mathml:msqrt>4</mathml:msqrt><b svg:fill="red"></b></body></html>z http://www.w3.org/1999/xhtmlZxmlnsz"http://www.w3.org/1998/Math/MathMLz xmlns:mathmlzhttp://www.w3.org/2000/svgz xmlns:svgN)r r rV rL )r r r rL r r r �test_basic_namespaces� s z.HTMLTreeBuilderSmokeTest.test_basic_namespacesc C s( d}| j |�}| jddg|jd � d S )Ns <a class="foo bar">rC �barrs )r r ru )r r r r r r �-test_multivalued_attribute_value_becomes_list� s zFHTMLTreeBuilderSmokeTest.test_multivalued_attribute_value_becomes_listc C s" d}| j |�}| jd|jj� d S )NuD <html><head><meta encoding="euc-jp"></head><body>Sacré bleu!</body>u Sacré bleu!)r r r� rp )r r r r r r � test_can_parse_unicode_document� s z8HTMLTreeBuilderSmokeTest.test_can_parse_unicode_documentc C s* t d�}| jd|d�}| j|j� d� dS )z2Parsers should be able to work with SoupStrainers.ro z&A <b>bold</b> <meta/> <i>statement</i>)Z parse_onlyz <b>bold</b>N)r r r r )r Zstrainerr r r r �test_soupstrainer� s z*HTMLTreeBuilderSmokeTest.test_soupstrainerc C s | j dd� d S )Nz<foo attr='bar'></foo>z<foo attr="bar"></foo>)r )r r r r �7test_single_quote_attribute_values_become_double_quotes� s zPHTMLTreeBuilderSmokeTest.test_single_quote_attribute_values_become_double_quotesc C s d}| j |� d S )Nz'<foo attr='bar "brawls" happen'>a</foo>)r )r ri r r r �7test_attribute_values_with_nested_quotes_are_left_alone� s zPHTMLTreeBuilderSmokeTest.test_attribute_values_with_nested_quotes_are_left_alonec C s. d}| j |�}d|jd< | j|jj� d� d S )Nz'<foo attr='bar "brawls" happen'>a</foo>z Brawls happen at "Bob's Bar"�attrz:<foo attr="Brawls happen at "Bob's Bar"">a</foo>)r rC r r )r ri r r r r �:test_attribute_values_with_double_nested_quotes_get_quoted� s zSHTMLTreeBuilderSmokeTest.test_attribute_values_with_double_nested_quotes_get_quotedc C s | j dd� | j dd� d S )Nz+<this is="really messed up & stuff"></this>z/<this is="really messed up & stuff"></this>z.<a href="http://example.org?a=1&b=2;3">foo</a>z2<a href="http://example.org?a=1&b=2;3">foo</a>)r )r r r r �.test_ampersand_in_attribute_value_gets_escaped� s zGHTMLTreeBuilderSmokeTest.test_ampersand_in_attribute_value_gets_escapedc C s | j d� d S )Nz/<a href="http://example.org?a=1&b=2;3"></a>)r )r r r r �7test_escaped_ampersand_in_attribute_value_is_left_alone� s zPHTMLTreeBuilderSmokeTest.test_escaped_ampersand_in_attribute_value_is_left_alonec C s d}d}| j ||� d S )Nz-<p><<sacré bleu!>></p>u# <p><<sacré bleu!>></p>)r )r ri �expectedr r r �1test_entities_in_strings_converted_during_parsing� s zJHTMLTreeBuilderSmokeTest.test_entities_in_strings_converted_during_parsingc C s" d}| j |�}| j|jjd� d S )Ns <p>�Foo�</p>u ‘Foo’)r r rH rp )r Zquoter r r r �)test_smart_quotes_converted_on_the_way_in� s zBHTMLTreeBuilderSmokeTest.test_smart_quotes_converted_on_the_way_inc C s | j d�}| j|jjd� d S )Nz<a> </a>� r: u )r r ru rp )r r r r r �0test_non_breaking_spaces_converted_on_the_way_in� s zIHTMLTreeBuilderSmokeTest.test_non_breaking_spaces_converted_on_the_way_inc C s0 d}dj d�}| j|�}| j|jj d�|� d S )Nz-<p><<sacré bleu!>></p>u# <p><<sacré bleu!>></p>zutf-8)rV r r rH )r ri r� r r r r �&test_entities_converted_on_the_way_out� s z?HTMLTreeBuilderSmokeTest.test_entities_converted_on_the_way_outc C sH d}|j d�}| j|�}|j d�}|jdd�}|j d�}| j||� d S )Nu� <html><head><meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type"/></head><body><p>Sacré bleu!</p></body></html>z iso-8859-1zutf-8z ISO-Latin-1)rV r rW r )r � unicode_htmlZiso_latin_htmlr �resultr� r r r � test_real_iso_latin_document� s z5HTMLTreeBuilderSmokeTest.test_real_iso_latin_documentc C sL d}|j d�}| j|�}| j|jd�|jd�� | j|jd�|jd�� d S )Nsk <html><head></head><body><pre>�����Shift-JIS�ŃR�[�f�B���O���ꂽ���{��̃t�@�C���ł��B</pre></body></html>z shift-jiszutf-8�euc_jp)r r r rV )r �shift_jis_htmlr� r r r r � test_real_shift_jis_document s z5HTMLTreeBuilderSmokeTest.test_real_shift_jis_documentc C sB d}| j |dd�}|jdks t�| j|jd�|jd�jd�� d S )Ns� <html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>���</body></html>� iso8859-8)Z from_encoding� iso-8859-8zutf-8)r� r� )r Zoriginal_encoding�AssertionErrorr rV r )r Zhebrew_documentr r r r �test_real_hebrew_document$ s z2HTMLTreeBuilderSmokeTest.test_real_hebrew_documentc C s` d}d| }| j |�}|jdddi�}|d }| jd|� | jt|t�� | jd|jd �� d S ) NzE<meta content="text/html; charset=x-sjis" http-equiv="Content-type"/>zj<html><head> %s <meta http-equiv="Content-language" content="ja"/></head><body>Shift-JIS markup goes here.r. z http-equivz Content-typer� ztext/html; charset=x-sjisztext/html; charset=utf8r[ )r rg r rb � isinstancer rV )r �meta_tagr� r � parsed_metar� r r r �'test_meta_tag_reflects_current_encoding1 s z@HTMLTreeBuilderSmokeTest.test_meta_tag_reflects_current_encodingc C s^ d}d| }| j |�}|jddd�}|d }| jd|� | jt|t�� | jd|jd�� d S ) Nz'<meta id="encoding" charset="x-sjis" />zj<html><head> %s <meta http-equiv="Content-language" content="ja"/></head><body>Shift-JIS markup goes here.r. �encoding)�id�charsetzx-sjisr[ )r rg r rb r� r rV )r r� r� r r� r� r r r �3test_html5_style_meta_tag_reflects_current_encodingM s zLHTMLTreeBuilderSmokeTest.test_html5_style_meta_tag_reflects_current_encodingc C s* | j d�}d|jd< | jd|jj� � d S )Nz <a>text</a>r� rC z<a foo="bar">text</a>)r ru r r )r �datar r r �5test_tag_with_no_attributes_can_have_attributes_addede s zNHTMLTreeBuilderSmokeTest.test_tag_with_no_attributes_can_have_attributes_addedN);r r r �__doc__r9 rB rK rD rM rO rP rQ rR rS rX rZ r\ r_ r` ra rc rd rh rk rl rm rq rr rt rv rw rx ry r{ r| r} r~ r r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r r" ? sr r" c @ s� e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%S )&�XMLTreeBuilderSmokeTestc C sF | j d�}tj|d�}tj|�}| j|jt� | j|j� |j� � d S )Nz <a><b>foo</a>r: )r r; r< r= r r> r r )r r? r@ rA r r r rB l s z9XMLTreeBuilderSmokeTest.test_pickle_and_unpickle_identityc C s | j d�}| j|j� d� d S )Nz<root/>s. <?xml version="1.0" encoding="utf-8"?> <root/>)r r rV )r r r r r �test_docstring_generatedu s z0XMLTreeBuilderSmokeTest.test_docstring_generatedc C s$ d}| j |�}| j||jd�� d S )Ns, <?xml version="1.0" encoding="utf8"?> <foo/>r[ )r r rV )r r r r r r �test_xml_declarationz s z,XMLTreeBuilderSmokeTest.test_xml_declarationc C s$ d}| j |�}| j||jd�� d S )Ns< <?xml version="1.0" encoding="utf8"?> <?PITarget PIContent?>r[ )r r rV )r r r r r r r\ s z3XMLTreeBuilderSmokeTest.test_processing_instructionc C s$ d}| j |�}| j|jd�|� dS )zGA real XHTML document should come out *exactly* the same as it went in.s� <?xml version="1.0" encoding="utf-8"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"> <html xmlns="http://www.w3.org/1999/xhtml"> <head><title>Hello.</title></head> <body>Goodbye.</body> </html>zutf-8N)r r rV )r r r r r r rX � s z0XMLTreeBuilderSmokeTest.test_real_xhtml_documentc C s" d}| j |�}| j||j� � d S )Ns <?xml version="1.0" encoding="utf-8"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> <parent xmlns="http://ns1/"> <child xmlns="http://ns2/" xmlns:ns3="http://ns3/"> <grandchild ns3:attr="value" xmlns="http://ns4/"/> </child> </parent>)r r rV )r �docr r r r �test_nested_namespaces� s z.XMLTreeBuilderSmokeTest.test_nested_namespacesc C s0 d}t |d�}d|j_|j� }| jd|k� d S )Nz/ <script type="text/javascript"> </script> zlxml-xmlz console.log("< < hey > > ");s < < hey > >)r re rp rV rb )r r� r Zencodedr r r �5test_formatter_processes_script_tag_for_xml_documents� s zMXMLTreeBuilderSmokeTest.test_formatter_processes_script_tag_for_xml_documentsc C s" d}| j |�}| jd|jj� d S )Nu? <?xml version="1.0" encoding="euc-jp"><root>Sacré bleu!</root>u Sacré bleu!)r r �rootrp )r r r r r r r� � s z7XMLTreeBuilderSmokeTest.test_can_parse_unicode_documentc C s$ d}| j |�}| jt|j�|� d S )Nz�<rss xmlns:dc="foo"><dc:creator>b</dc:creator><dc:date>2012-07-02T20:33:42Z</dc:date><dc:rights>c</dc:rights><image>d</image></rss>)r r rF Zrss)r r r r r r �test_popping_namespaced_tag� s z3XMLTreeBuilderSmokeTest.test_popping_namespaced_tagc C s | j d�}| j|jd�d� d S )Nz<root/>�latin1s/ <?xml version="1.0" encoding="latin1"?> <root/>)r r rV )r r r r r �(test_docstring_includes_correct_encoding� s z@XMLTreeBuilderSmokeTest.test_docstring_includes_correct_encodingc C s0 ddd d }| j |�}| j|jd�|� dS ) z<A large XML document should come out the same as it went in.s- <?xml version="1.0" encoding="utf-8"?> <root>� 0r: � s </root>zutf-8Ni )r r rV )r r r r r r �test_large_xml_document� s z/XMLTreeBuilderSmokeTest.test_large_xml_documentc C s | j dd� | j d� d S )Nz<p>z<p/>z <p>foo</p>)r )r r r r �9test_tags_are_empty_element_if_and_only_if_they_are_empty� s zQXMLTreeBuilderSmokeTest.test_tags_are_empty_element_if_and_only_if_they_are_emptyc C s8 d}| j |�}|j}| jd|d � | jd|d � d S )Nz�<root xmlns:a="http://example.com/" xmlns:b="http://example.net/"><a:foo>This tag is in the a namespace</a:foo><b:foo>This tag is in the b namespace</b:foo></root>zhttp://example.com/zxmlns:azhttp://example.net/zxmlns:b)r r� r )r r r r� r r r � test_namespaces_are_preserved� s z5XMLTreeBuilderSmokeTest.test_namespaces_are_preservedc C s$ d}| j |�}| jt|j�|� d S )NzN<p xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:date>20010504</dc:date></p>)r r rF rH )r r r r r r �test_closing_namespaced_tag� s z3XMLTreeBuilderSmokeTest.test_closing_namespaced_tagc C s$ d}| j |�}| jt|j�|� d S )Nzs<foo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><bar xsi:schemaLocation="http://www.example.com"/></foo>)r r rF rC )r r r r r r �test_namespaced_attributes� s z2XMLTreeBuilderSmokeTest.test_namespaced_attributesc C s$ d}| j |�}| jt|j�|� d S )Nz <foo xml:lang="fr">bar</foo>)r r rF rC )r r r r r r �(test_namespaced_attributes_xml_namespace� s z@XMLTreeBuilderSmokeTest.test_namespaced_attributes_xml_namespacec C s� d}| j |�}| jdt|jd��� | jdt|jd��� | jdt|jd��� | jdt|jddd ��� | jdt|jddg��� d S ) Na <?xml version="1.0" encoding="utf-8"?> <Document xmlns="http://example.com/ns0" xmlns:ns1="http://example.com/ns1" xmlns:ns2="http://example.com/ns2" <ns1:tag>foo</ns1:tag> <ns1:tag>bar</ns1:tag> <ns2:tag key="value">baz</ns2:tag> </Document> � �tagr: zns1:tag� zns2:tag�value)�key)r r rG rY )r r� r r r r �test_find_by_prefixed_name� s z2XMLTreeBuilderSmokeTest.test_find_by_prefixed_namec C s2 d}| j |�}|j}tj|�}| j|j|j� d S )Nzf<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <w:document xmlns:w="http://example.com/ns0"/>)r Zdocumentr] r �prefix)r Zxmlr r� Z duplicater r r �!test_copy_tag_preserves_namespace� s z9XMLTreeBuilderSmokeTest.test_copy_tag_preserves_namespaceN)r r r rB r� r� r\ rX r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r r� j s$ r� c @ s8 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd S ) �HTML5TreeBuilderSmokeTestz2Smoke test for a tree builder that supports HTML5.c C s d S )Nr )r r r r rX s z2HTML5TreeBuilderSmokeTest.test_real_xhtml_documentc C s" d}| j |�}| jd|jj� d S )Nz<a>z http://www.w3.org/1999/xhtml)r r ru � namespace)r r r r r r � test_html_tags_have_namespace s z7HTML5TreeBuilderSmokeTest.test_html_tags_have_namespacec C s6 d}| j |�}d}| j||jj� | j||jj� d S )Nz<svg><circle/></svg>zhttp://www.w3.org/2000/svg)r r Zsvgr� Zcircle)r r r r� r r r � test_svg_tags_have_namespace s z6HTML5TreeBuilderSmokeTest.test_svg_tags_have_namespacec C s6 d}| j |�}d}| j||jj� | j||jj� d S )Nz <math><msqrt>5</msqrt></math>z"http://www.w3.org/1998/Math/MathML)r r Zmathr� Zmsqrt)r r r r� r r r � test_mathml_tags_have_namespace s z9HTML5TreeBuilderSmokeTest.test_mathml_tags_have_namespacec C sP d}| j |�}| jt|jd t�� | j|jd d� | jd|jd jj� d S )Nz3<?xml version="1.0" encoding="utf-8"?><html></html>r z$?xml version="1.0" encoding="utf-8"?rL )r rb r� rE r r r r8 )r r r r r r �$test_xml_declaration_becomes_comment s z>HTML5TreeBuilderSmokeTest.test_xml_declaration_becomes_commentN) r r r r� rX r� r� r� r� r r r r r� � s r� c s dd� �� �fdd�}|S )Nc _ s d S )Nr )Ztest�argsr r r r �nothing! s zskipIf.<locals>.nothingc s � r�S | S d S )Nr )Z test_item)� conditionr� r r � decorator$ s zskipIf.<locals>.decoratorr )r� �reasonr� r )r� r� r �skipIf s r� )r� Z __license__r; r] � functoolsZunittestr Zbs4r Z bs4.elementr r r r r Z bs4.builderr r r �objectr" r� r� r� r r r r �<module> s( % / # __pycache__/dammit.cpython-36.pyc 0000644 00000044336 15246412105 0012667 0 ustar 00 3 6]�t � @ s d Z dZddlZddlmZ ddlZddlZddlZdZyddl Z dd� Z W nF e k r� yddl Z dd� Z W n e k r� dd� Z Y nX Y nX y ddl Z W n e k r� Y nX ejd j� ej�Zejd j� ej�ZG d d � d e�ZG d d� d�ZG dd� d�ZdS )aB Beautiful Soup bonus library: Unicode, Dammit This library converts a bytestream to Unicode through any means necessary. It is heavily based on code from Mark Pilgrim's Universal Feed Parser. It works best on XML and HTML, but it does not rewrite the XML or HTML to reflect a new encoding; that's the tree builder's job. ZMIT� N)�codepoint2namec C s t j| �d S )N�encoding)�cchardet�detect)�s� r � /usr/lib/python3.6/dammit.py�chardet_dammit s r c C s t j| �d S )Nr )�chardetr )r r r r r ! s c C s d S )Nr )r r r r r ' s z!^<\?.*encoding=['"](.*?)['"].*\?>z0<\s*meta[^>]+charset\s*=\s*["']?([^>]*?)[ /;'">]c @ s� e Zd ZdZdd� Ze� \ZZZdddddd �Ze j d �Z e j d �Z e d d � �Ze dd� �Ze dd� �Ze ddd��Ze ddd��Ze dd� �ZdS ) �EntitySubstitutionzASubstitute XML or HTML entities for the corresponding characters.c C sn i } i }g }xBt tj� �D ]2\}}t|�}|dkrD|j|� || |< |||< qW ddj|� }| |tj|�fS )N�" z[%s]� )�listr �items�chr�append�join�re�compile)�lookupZreverse_lookupZcharacters_for_reZ codepoint�name� characterZ re_definitionr r r �_populate_class_variables9 s z,EntitySubstitution._populate_class_variablesZaposZquotZamp�lt�gt)�'�"�&�<�>z&([<>]|&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;))z([<>&])c C s | j j|jd��}d| S )Nr z&%s;)�CHARACTER_TO_HTML_ENTITY�get�group)�cls�matchobj�entityr r r �_substitute_html_entityZ s z*EntitySubstitution._substitute_html_entityc C s | j |jd� }d| S )zmUsed with a regular expression to substitute the appropriate XML entity for an XML special character.r z&%s;)�CHARACTER_TO_XML_ENTITYr" )r# r$ r% r r r �_substitute_xml_entity_ s z)EntitySubstitution._substitute_xml_entityc C s6 d}d|kr*d|kr&d}|j d|�}nd}|| | S )a* Make a value into a quoted XML attribute, possibly escaping it. Most strings will be quoted using double quotes. Bob's Bar -> "Bob's Bar" If a string contains double quotes, it will be quoted using single quotes. Welcome to "my bar" -> 'Welcome to "my bar"' If a string contains both single and double quotes, the double quotes will be escaped, and the string will be quoted using double quotes. Welcome to "Bob's Bar" -> "Welcome to "Bob's bar" r r z")�replace)�self�valueZ quote_withZ replace_withr r r �quoted_attribute_valuef s z)EntitySubstitution.quoted_attribute_valueFc C s" | j j| j|�}|r | j|�}|S )a Substitute XML entities for special XML characters. :param value: A string to be substituted. The less-than sign will become <, the greater-than sign will become >, and any ampersands will become &. If you want ampersands that appear to be part of an entity definition to be left alone, use substitute_xml_containing_entities() instead. :param make_quoted_attribute: If True, then the string will be quoted, as befits an attribute value. )�AMPERSAND_OR_BRACKET�subr( r, )r# r+ �make_quoted_attributer r r �substitute_xml� s z!EntitySubstitution.substitute_xmlc C s" | j j| j|�}|r | j|�}|S )a� Substitute XML entities for special XML characters. :param value: A string to be substituted. The less-than sign will become <, the greater-than sign will become >, and any ampersands that are not part of an entity defition will become &. :param make_quoted_attribute: If True, then the string will be quoted, as befits an attribute value. )�BARE_AMPERSAND_OR_BRACKETr. r( r, )r# r+ r/ r r r �"substitute_xml_containing_entities� s z5EntitySubstitution.substitute_xml_containing_entitiesc C s | j j| j|�S )a� Replace certain Unicode characters with named HTML entities. This differs from data.encode(encoding, 'xmlcharrefreplace') in that the goal is to make the result more readable (to those with ASCII displays) rather than to recover from errors. There's absolutely nothing wrong with a UTF-8 string containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that character with "é" will make it more readable to some people. )�CHARACTER_TO_HTML_ENTITY_REr. r& )r# r r r r �substitute_html� s z"EntitySubstitution.substitute_htmlN)F)F)�__name__� __module__� __qualname__�__doc__r r ZHTML_ENTITY_TO_CHARACTERr3 r' r r r1 r- � classmethodr& r( r, r0 r2 r4 r r r r r 5 s$ % r c @ sH e Zd ZdZddd�Zdd� Zedd � �Zed d � �Z edd d ��Z dS )�EncodingDetectora^ Suggests a number of possible encodings for a bytestring. Order of precedence: 1. Encodings you specifically tell EncodingDetector to try first (the override_encodings argument to the constructor). 2. An encoding declared within the bytestring itself, either in an XML declaration (if the bytestring is to be interpreted as an XML document), or in a <meta> tag (if the bytestring is to be interpreted as an HTML document.) 3. An encoding detected through textual analysis by chardet, cchardet, or a similar external library. 4. UTF-8. 5. Windows-1252. NFc C sN |pg | _ |pg }tdd� |D ��| _d | _|| _d | _| j|�\| _| _d S )Nc S s g | ] }|j � �qS r )�lower)�.0�xr r r � <listcomp>� s z-EncodingDetector.__init__.<locals>.<listcomp>) �override_encodings�set�exclude_encodings�chardet_encoding�is_html�declared_encoding�strip_byte_order_mark�markup�sniffed_encoding)r* rF r? rC rA r r r �__init__� s zEncodingDetector.__init__c C s8 |d k r4|j � }|| jkr dS ||kr4|j|� dS dS )NFT)r; rA �add)r* r �triedr r r �_usable� s zEncodingDetector._usablec c s� t � }x | jD ]}| j||�r|V qW | j| j|�r>| jV | jdkrZ| j| j| j�| _| j| j|�rp| jV | jdkr�t | j�| _| j| j|�r�| jV x dD ]}| j||�r�|V q�W dS )z<Yield a number of encodings that might work for this markup.N�utf-8� windows-1252)rL rM ) r@ r? rK rG rD �find_declared_encodingrF rC rB r )r* rJ �er r r � encodings� s$ zEncodingDetector.encodingsc C s� d}t |t�r||fS t|�dkrT|dd� dkrT|dd� dkrTd}|dd� }n�t|�dkr�|dd� dkr�|dd� dkr�d}|dd� }nd|dd � d kr�d }|d d� }nB|dd� d kr�d }|dd� }n |dd� dkr�d}|dd� }||fS )zMIf a byte-order mark is present, strip it and return the encoding it implies.N� � s ��z zutf-16bes ��zutf-16le� s zutf-8s ��zutf-32bes �� zutf-32le)� isinstance�str�len)r# �datar r r r rE s* z&EncodingDetector.strip_byte_order_markc C s� |rt |� }}nd}tdtt |�d ��}d}tj||d�}| rV|rVtj||d�}|dk rr|j� d jdd�}|r~|j� S dS ) z�Given a document, tries to find its declared encoding. An XML encoding is declared at the beginning of the document. An HTML encoding is declared in a <meta> tag, hopefully near the beginning of the document. i i g�������?N)�endposr �asciir) ) rV �max�int�xml_encoding_re�search� html_meta_re�groups�decoder; )r# rF rC Zsearch_entire_documentZ xml_endposZ html_endposrD Zdeclared_encoding_matchr r r rN + s z'EncodingDetector.find_declared_encoding)NFN)FF) r5 r6 r7 r8 rH rK �propertyrP r9 rE rN r r r r r: � s ! r: c � @ sR e Zd ZdZddd�ZdddgZg dd g fd d �Zd d � Z�d dd�Z�d dd�Z e dd� �Z dd� Z dd� Z �d d�d �d!�d"�d#�d$�d%�d&�d'�d(�d)�d*d2�d+d2d2�d,�d-�d.�d/�d0�d1�d2�d3�d4�d5�d6�d7d2�d8�d9dQ� ZdRddSdTdUdVdWdXdYdZd[d\d]d2d^d2d2d_d_d`d`dadbdcdddedfdgdhd2didjddkdldmdndodpd[dqdPdrdsdkddtdbdudvdwdx�d:dzd{dadSd|drd}d~dd�d2d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�dad�d�d�d�d�djd�d�d�d�d�d�d�d�d�dld�d�d�d�d�d�d�d�dud�dududududud�dudzdzdzdzd�d�d�d���Zd�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d�d��d �d�d�d�d�d�d�d�d�d �d �d �d �d �d�zZ�d;�d<�d=gZe�d �d Ze�d> �d Ze�d?�d�d ��ZdS (@ � UnicodeDammitz�A class for detecting the encoding of a *ML document and converting it to a Unicode string. If the source encoding is windows-1252, can replace MS smart quotes with their HTML or XML equivalents.z mac-romanz shift-jis)� macintoshzx-sjis� windows-1252z iso-8859-1z iso-8859-2NFc C s� || _ g | _d| _|| _tjt�| _t||||�| _ t |t �sF|dkr`|| _ t |�| _ d | _d S | j j | _ d }x,| j jD ] }| j j }| j|�}|d k rxP qxW |s�x@| j jD ]4}|dkr�| j|d�}|d k r�| jjd� d| _P q�W || _ |s�d | _d S )NFr rY r) zSSome characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.T)�smart_quotes_to�tried_encodingsZ contains_replacement_charactersrC �loggingZ getLoggerr5 �logr: �detectorrT rU rF Zunicode_markup�original_encodingrP � _convert_fromZwarning)r* rF r? re rC rA �ur r r r rH X s> zUnicodeDammit.__init__c C s� |j d�}| jdkr&| jj|�j� }nf| jj|�}t|�tkr�| jdkrfdj� |d j� dj� }q�dj� |d j� dj� }n|j� }|S )z[Changes a MS smart quote character to an XML or HTML entity, or an ASCII character.� rY Zxmlz&#x�;r r )r" re �MS_CHARS_TO_ASCIIr! �encode�MS_CHARS�type�tuple)r* �matchZorigr. r r r � _sub_ms_char� s zUnicodeDammit._sub_ms_char�strictc C s� | j |�}| s ||f| jkr"d S | jj||f� | j}| jd k rh|| jkrhd}tj|�}|j| j |�}y | j |||�}|| _|| _ W n t k r� } zd S d }~X nX | jS )Ns ([�-�])) � find_codecrf r rF re �ENCODINGS_WITH_SMART_QUOTESr r r. ru � _to_unicoderj � Exception)r* Zproposed�errorsrF Zsmart_quotes_reZsmart_quotes_compiledrl rO r r r rk � s" zUnicodeDammit._convert_fromc C s t |||�S )zGiven a string and its encoding, decodes the string into Unicode. %encoding is a string recognized by encodings.aliases)rU )r* rW r r{ r r r ry � s zUnicodeDammit._to_unicodec C s | j s d S | jjS )N)rC ri rD )r* r r r �declared_html_encoding� s z$UnicodeDammit.declared_html_encodingc C s` | j | jj||��pN|r*| j |jdd��pN|r@| j |jdd��pN|rL|j� pN|}|r\|j� S d S )N�-r �_)�_codec�CHARSET_ALIASESr! r) r; )r* �charsetr+ r r r rw � s zUnicodeDammit.find_codecc C s< |s|S d }yt j|� |}W n ttfk r6 Y nX |S )N)�codecsr � LookupError� ValueError)r* r� �codecr r r r � s zUnicodeDammit._codec�euro�20AC� �sbquo�201A�fnof�192�bdquo�201E�hellip�2026�dagger�2020�Dagger�2021�circ�2C6�permil�2030�Scaron�160�lsaquo�2039�OElig�152�?�#x17D�17D�lsquo�2018�rsquo�2019�ldquo�201C�rdquo�201D�bull�2022�ndash�2013�mdash�2014�tilde�2DC�trade�2122�scaron�161�rsaquo�203A�oelig�153�#x17E�17E�Yumlr ) � �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �ZEUR�,�fz,,z...�+z++�^�%�Sr ZOE�Zr r �*r} z--�~z(TM)r r Zoe�z�Y�!�cZGBP�$ZYEN�|z..z(th)z<<z(R)�oz+-�2�3�acuterl �P�1z>>z1/4z1/2z3/4�AZAE�C�E�I�D�N�O�U�b�B�aZaerO �i�n�/�y)�r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� � �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �s €s ‚s ƒs „s …s †s ‡s ˆs ‰s Šs ‹s Œs Žs ‘s ’s “s ”s •s –s —s ˜s ™s šs ›s œs žs Ÿs s ¡s ¢s £s ¤s ¥s ¦s §s ¨s ©s ªs «s ¬s s ®s ¯s °s ±s ²s ³s ´s µs ¶s ·s ¸s ¹s ºs »s ¼s ½s ¾s ¿s Às Ás Âs Ãs Äs Ås Æs Çs Ès És Ês Ës Ìs Ís Îs Ïs Ðs Ñs Òs Ós Ôs Õs Ös ×s Øs Ùs Ús Ûs Üs Ýs Þs ßs àr s âs ãs äs ås æs çs ès és ês ës ìs ís îs ïs ðs ñs òs ós ôs õs ös ÷s øs ùs ús ûs üs ýs þ)z� � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � � �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� �� � � � � � � � � � � r� r� rR r� r� rS r� r� rQ r rm �utf8c C s$ |j dd�j� dkr td��|j� dkr0td��g }d }d }x�|t|�k � r�|| }t|t�sft|�}|| jkr�|| jkr�xz| j D ]$\}} } ||kr�|| kr�|| 7 }P q�W q>|d kr�|| j kr�|j |||� � |j | j | � |d 7 }|}q>|d 7 }q>W |d k�r|S |j ||d � � d j |�S )a� Fix characters from one encoding embedded in some other encoding. Currently the only situation supported is Windows-1252 (or its subset ISO-8859-1), embedded in UTF-8. The input must be a bytestring. If you've already converted the document to Unicode, you're too late. The output is a bytestring in which `embedded_encoding` characters have been converted to their `main_encoding` equivalents. r~ r} � windows-1252� windows_1252zPWindows-1252 and ISO-8859-1 are the only currently supported embedded encodings.r� �utf-8z4UTF-8 is the only currently supported main encoding.r ra rm N� )r� r� )r� r� ) r) r; �NotImplementedErrorrV rT r[ �ord�FIRST_MULTIBYTE_MARKER�LAST_MULTIBYTE_MARKER�MULTIBYTE_MARKERS_AND_SIZES�WINDOWS_1252_TO_UTF8r r ) r# Zin_bytesZ main_encodingZembedded_encodingZ byte_chunksZ chunk_start�posZbyte�start�end�sizer r r � detwingle s< zUnicodeDammit.detwingle)rv )rv )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r� )r� r )r r� )r� r� rR )r� r� rS )r� r� rQ ���)r� rd )r5 r6 r7 r8 r� rx rH ru rk ry ra r| rw r rq ro r� r� r� r� r9 r� r r r r rb E s` 1 rb )r8 Z __license__r� Z html.entitiesr r rg �stringZ chardet_typer r � ImportErrorr Z iconv_codecr rp r� r\ r^ �objectr r: rb r r r r �<module> s8
Free Space : 745713037312 Byte