Ross Teixeira rosstex
rosstex
/ soup_xpath_gen.py
Last active
June 16, 2023 18:59
deterministic xpath generation of BeautifulSoup elements for web crawling
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import html | |
| BAD_CHARS = set(["\"", "'", "[", "]"]) | |
| # generates an xpath string for a given BeautifulSoup element | |
| def soup_xpath_gen(element): | |
| xpath = "" | |
| while element.name != 'document': | |
| if element.name == 'html': | |
| xpath = "/html/" + xpath |