|
| 1 | +"""A parser for HTML and XHTML.""" |
| 2 | +# This file is a modified Version from htmlparser.HTMLParser |
| 3 | +# in this version tag and attribute names aren't converted to |
| 4 | +# lowercase |
| 5 | + |
| 6 | +# This file is based on sgmllib.py, but the API is slightly different. |
| 7 | + |
| 8 | +# XXX There should be a way to distinguish between PCDATA (parsed |
| 9 | +# character data -- the normal case), RCDATA (replaceable character |
| 10 | +# data -- only char and entity references and end tags are special) |
| 11 | +# and CDATA (character data -- only end tags are special). |
| 12 | + |
| 13 | + |
| 14 | +import markupbase |
| 15 | +import re |
| 16 | + |
| 17 | +# Regular expressions used for parsing |
| 18 | + |
| 19 | +interesting_normal = re.compile('[&<]') |
| 20 | +interesting_cdata = re.compile(r'<(/|\Z)') |
| 21 | +incomplete = re.compile('&[a-zA-Z#]') |
| 22 | + |
| 23 | +entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]') |
| 24 | +charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]') |
| 25 | + |
| 26 | +starttagopen = re.compile('<[a-zA-Z]') |
| 27 | +piclose = re.compile('>') |
| 28 | +commentclose = re.compile(r'--\s*>') |
| 29 | +tagfind = re.compile('[a-zA-Z][-.a-zA-Z0-9:_]*') |
| 30 | +attrfind = re.compile( |
| 31 | + r'\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\s*=\s*' |
| 32 | + r'(\'[^\']*\'|"[^"]*"|[^\s"\'=<>`]*))?') |
| 33 | + |
| 34 | +locatestarttagend = re.compile(r""" |
| 35 | + <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name |
| 36 | + (?:\s+ # whitespace before attribute name |
| 37 | + (?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name |
| 38 | + (?:\s*=\s* # value indicator |
| 39 | + (?:'[^']*' # LITA-enclosed value |
| 40 | + |\"[^\"]*\" # LIT-enclosed value |
| 41 | + |[^'\">\s]+ # bare value |
| 42 | + ) |
| 43 | + )? |
| 44 | + ) |
| 45 | + )* |
| 46 | + \s* # trailing whitespace |
| 47 | +""", re.VERBOSE) |
| 48 | +endendtag = re.compile('>') |
| 49 | +endtagfind = re.compile('</\s*([a-zA-Z][-.a-zA-Z0-9:_]*)\s*>') |
| 50 | + |
| 51 | + |
| 52 | +class HTMLParseError(Exception): |
| 53 | + """Exception raised for all parse errors.""" |
| 54 | + |
| 55 | + def __init__(self, msg, position=(None, None)): |
| 56 | + assert msg |
| 57 | + self.msg = msg |
| 58 | + self.lineno = position[0] |
| 59 | + self.offset = position[1] |
| 60 | + |
| 61 | + def __str__(self): |
| 62 | + result = self.msg |
| 63 | + if self.lineno is not None: |
| 64 | + result = result + ", at line %d" % self.lineno |
| 65 | + if self.offset is not None: |
| 66 | + result = result + ", column %d" % (self.offset + 1) |
| 67 | + return result |
| 68 | + |
| 69 | + |
| 70 | +class HTMLParser(markupbase.ParserBase): |
| 71 | + """Find tags and other markup and call handler functions. |
| 72 | +
|
| 73 | + Usage: |
| 74 | + p = HTMLParser() |
| 75 | + p.feed(data) |
| 76 | + ... |
| 77 | + p.close() |
| 78 | +
|
| 79 | + Start tags are handled by calling self.handle_starttag() or |
| 80 | + self.handle_startendtag(); end tags by self.handle_endtag(). The |
| 81 | + data between tags is passed from the parser to the derived class |
| 82 | + by calling self.handle_data() with the data as argument (the data |
| 83 | + may be split up in arbitrary chunks). Entity references are |
| 84 | + passed by calling self.handle_entityref() with the entity |
| 85 | + reference as the argument. Numeric character references are |
| 86 | + passed to self.handle_charref() with the string containing the |
| 87 | + reference as the argument. |
| 88 | + """ |
| 89 | + |
| 90 | + CDATA_CONTENT_ELEMENTS = ("script", "style") |
| 91 | + |
| 92 | + |
| 93 | + def __init__(self): |
| 94 | + """Initialize and reset this instance.""" |
| 95 | + self.reset() |
| 96 | + |
| 97 | + def reset(self): |
| 98 | + """Reset this instance. Loses all unprocessed data.""" |
| 99 | + self.rawdata = '' |
| 100 | + self.lasttag = '???' |
| 101 | + self.interesting = interesting_normal |
| 102 | + markupbase.ParserBase.reset(self) |
| 103 | + |
| 104 | + def feed(self, data): |
| 105 | + r"""Feed data to the parser. |
| 106 | +
|
| 107 | + Call this as often as you want, with as little or as much text |
| 108 | + as you want (may include '\n'). |
| 109 | + """ |
| 110 | + self.rawdata = self.rawdata + data |
| 111 | + self.goahead(0) |
| 112 | + |
| 113 | + def close(self): |
| 114 | + """Handle any buffered data.""" |
| 115 | + self.goahead(1) |
| 116 | + |
| 117 | + def error(self, message): |
| 118 | + raise HTMLParseError(message, self.getpos()) |
| 119 | + |
| 120 | + __starttag_text = None |
| 121 | + |
| 122 | + def get_starttag_text(self): |
| 123 | + """Return full source of start tag: '<...>'.""" |
| 124 | + return self.__starttag_text |
| 125 | + |
| 126 | + def set_cdata_mode(self): |
| 127 | + self.interesting = interesting_cdata |
| 128 | + |
| 129 | + def clear_cdata_mode(self): |
| 130 | + self.interesting = interesting_normal |
| 131 | + |
| 132 | + # Internal -- handle data as far as reasonable. May leave state |
| 133 | + # and data to be processed by a subsequent call. If 'end' is |
| 134 | + # true, force handling all data as if followed by EOF marker. |
| 135 | + def goahead(self, end): |
| 136 | + rawdata = self.rawdata |
| 137 | + i = 0 |
| 138 | + n = len(rawdata) |
| 139 | + while i < n: |
| 140 | + match = self.interesting.search(rawdata, i) # < or & |
| 141 | + if match: |
| 142 | + j = match.start() |
| 143 | + else: |
| 144 | + j = n |
| 145 | + if i < j: self.handle_data(rawdata[i:j]) |
| 146 | + i = self.updatepos(i, j) |
| 147 | + if i == n: break |
| 148 | + startswith = rawdata.startswith |
| 149 | + if startswith('<', i): |
| 150 | + if starttagopen.match(rawdata, i): # < + letter |
| 151 | + k = self.parse_starttag(i) |
| 152 | + elif startswith("</", i): |
| 153 | + k = self.parse_endtag(i) |
| 154 | + elif startswith("<!--", i): |
| 155 | + k = self.parse_comment(i) |
| 156 | + elif startswith("<?", i): |
| 157 | + k = self.parse_pi(i) |
| 158 | + elif startswith("<!", i): |
| 159 | + k = self.parse_declaration(i) |
| 160 | + elif (i + 1) < n: |
| 161 | + self.handle_data("<") |
| 162 | + k = i + 1 |
| 163 | + else: |
| 164 | + break |
| 165 | + if k < 0: |
| 166 | + if end: |
| 167 | + self.error("EOF in middle of construct") |
| 168 | + break |
| 169 | + i = self.updatepos(i, k) |
| 170 | + elif startswith("&#", i): |
| 171 | + match = charref.match(rawdata, i) |
| 172 | + if match: |
| 173 | + name = match.group()[2:-1] |
| 174 | + self.handle_charref(name) |
| 175 | + k = match.end() |
| 176 | + if not startswith(';', k-1): |
| 177 | + k = k - 1 |
| 178 | + i = self.updatepos(i, k) |
| 179 | + continue |
| 180 | + else: |
| 181 | + if ";" in rawdata[i:]: #bail by consuming &# |
| 182 | + self.handle_data(rawdata[0:2]) |
| 183 | + i = self.updatepos(i, 2) |
| 184 | + break |
| 185 | + elif startswith('&', i): |
| 186 | + match = entityref.match(rawdata, i) |
| 187 | + if match: |
| 188 | + name = match.group(1) |
| 189 | + self.handle_entityref(name) |
| 190 | + k = match.end() |
| 191 | + if not startswith(';', k-1): |
| 192 | + k = k - 1 |
| 193 | + i = self.updatepos(i, k) |
| 194 | + continue |
| 195 | + match = incomplete.match(rawdata, i) |
| 196 | + if match: |
| 197 | + # match.group() will contain at least 2 chars |
| 198 | + if end and match.group() == rawdata[i:]: |
| 199 | + self.error("EOF in middle of entity or char ref") |
| 200 | + # incomplete |
| 201 | + break |
| 202 | + elif (i + 1) < n: |
| 203 | + # not the end of the buffer, and can't be confused |
| 204 | + # with some other construct |
| 205 | + self.handle_data("&") |
| 206 | + i = self.updatepos(i, i + 1) |
| 207 | + else: |
| 208 | + break |
| 209 | + else: |
| 210 | + assert 0, "interesting.search() lied" |
| 211 | + # end while |
| 212 | + if end and i < n: |
| 213 | + self.handle_data(rawdata[i:n]) |
| 214 | + i = self.updatepos(i, n) |
| 215 | + self.rawdata = rawdata[i:] |
| 216 | + |
| 217 | + # Internal -- parse processing instr, return end or -1 if not terminated |
| 218 | + def parse_pi(self, i): |
| 219 | + rawdata = self.rawdata |
| 220 | + assert rawdata[i:i+2] == '<?', 'unexpected call to parse_pi()' |
| 221 | + match = piclose.search(rawdata, i+2) # > |
| 222 | + if not match: |
| 223 | + return -1 |
| 224 | + j = match.start() |
| 225 | + self.handle_pi(rawdata[i+2: j]) |
| 226 | + j = match.end() |
| 227 | + return j |
| 228 | + |
| 229 | + # Internal -- handle starttag, return end or -1 if not terminated |
| 230 | + def parse_starttag(self, i): |
| 231 | + self.__starttag_text = None |
| 232 | + endpos = self.check_for_whole_start_tag(i) |
| 233 | + if endpos < 0: |
| 234 | + return endpos |
| 235 | + rawdata = self.rawdata |
| 236 | + self.__starttag_text = rawdata[i:endpos] |
| 237 | + |
| 238 | + # Now parse the data between i+1 and j into a tag and attrs |
| 239 | + attrs = [] |
| 240 | + match = tagfind.match(rawdata, i+1) |
| 241 | + assert match, 'unexpected call to parse_starttag()' |
| 242 | + k = match.end() |
| 243 | + self.lasttag = tag = rawdata[i+1:k] |
| 244 | + |
| 245 | + while k < endpos: |
| 246 | + m = attrfind.match(rawdata, k) |
| 247 | + if not m: |
| 248 | + break |
| 249 | + attrname, rest, attrvalue = m.group(1, 2, 3) |
| 250 | + if not rest: |
| 251 | + attrvalue = None |
| 252 | + elif attrvalue[:1] == '\'' == attrvalue[-1:] or \ |
| 253 | + attrvalue[:1] == '"' == attrvalue[-1:]: |
| 254 | + attrvalue = attrvalue[1:-1] |
| 255 | + attrvalue = self.unescape(attrvalue) |
| 256 | + attrs.append((attrname, attrvalue)) |
| 257 | + k = m.end() |
| 258 | + |
| 259 | + end = rawdata[k:endpos].strip() |
| 260 | + if end not in (">", "/>"): |
| 261 | + lineno, offset = self.getpos() |
| 262 | + if "\n" in self.__starttag_text: |
| 263 | + lineno = lineno + self.__starttag_text.count("\n") |
| 264 | + offset = len(self.__starttag_text) \ |
| 265 | + - self.__starttag_text.rfind("\n") |
| 266 | + else: |
| 267 | + offset = offset + len(self.__starttag_text) |
| 268 | + self.error("junk characters in start tag: %r" |
| 269 | + % (rawdata[k:endpos][:20],)) |
| 270 | + if end.endswith('/>'): |
| 271 | + # XHTML-style empty tag: <span attr="value" /> |
| 272 | + self.handle_startendtag(tag, attrs) |
| 273 | + else: |
| 274 | + self.handle_starttag(tag, attrs) |
| 275 | + if tag in self.CDATA_CONTENT_ELEMENTS: |
| 276 | + self.set_cdata_mode() |
| 277 | + return endpos |
| 278 | + |
| 279 | + # Internal -- check to see if we have a complete starttag; return end |
| 280 | + # or -1 if incomplete. |
| 281 | + def check_for_whole_start_tag(self, i): |
| 282 | + rawdata = self.rawdata |
| 283 | + m = locatestarttagend.match(rawdata, i) |
| 284 | + if m: |
| 285 | + j = m.end() |
| 286 | + next = rawdata[j:j+1] |
| 287 | + if next == ">": |
| 288 | + return j + 1 |
| 289 | + if next == "/": |
| 290 | + if rawdata.startswith("/>", j): |
| 291 | + return j + 2 |
| 292 | + if rawdata.startswith("/", j): |
| 293 | + # buffer boundary |
| 294 | + return -1 |
| 295 | + # else bogus input |
| 296 | + self.updatepos(i, j + 1) |
| 297 | + self.error("malformed empty start tag") |
| 298 | + if next == "": |
| 299 | + # end of input |
| 300 | + return -1 |
| 301 | + if next in ("abcdefghijklmnopqrstuvwxyz=/" |
| 302 | + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"): |
| 303 | + # end of input in or before attribute value, or we have the |
| 304 | + # '/' from a '/>' ending |
| 305 | + return -1 |
| 306 | + self.updatepos(i, j) |
| 307 | + self.error("malformed start tag") |
| 308 | + raise AssertionError("we should not get here!") |
| 309 | + |
| 310 | + # Internal -- parse endtag, return end or -1 if incomplete |
| 311 | + def parse_endtag(self, i): |
| 312 | + rawdata = self.rawdata |
| 313 | + assert rawdata[i:i+2] == "</", "unexpected call to parse_endtag" |
| 314 | + match = endendtag.search(rawdata, i+1) # > |
| 315 | + if not match: |
| 316 | + return -1 |
| 317 | + j = match.end() |
| 318 | + match = endtagfind.match(rawdata, i) # </ + tag + > |
| 319 | + if not match: |
| 320 | + self.error("bad end tag: %r" % (rawdata[i:j],)) |
| 321 | + tag = match.group(1) |
| 322 | + self.handle_endtag(tag) |
| 323 | + self.clear_cdata_mode() |
| 324 | + return j |
| 325 | + |
| 326 | + # Overridable -- finish processing of start+end tag: <tag.../> |
| 327 | + def handle_startendtag(self, tag, attrs): |
| 328 | + self.handle_starttag(tag, attrs) |
| 329 | + self.handle_endtag(tag) |
| 330 | + |
| 331 | + # Overridable -- handle start tag |
| 332 | + def handle_starttag(self, tag, attrs): |
| 333 | + pass |
| 334 | + |
| 335 | + # Overridable -- handle end tag |
| 336 | + def handle_endtag(self, tag): |
| 337 | + pass |
| 338 | + |
| 339 | + # Overridable -- handle character reference |
| 340 | + def handle_charref(self, name): |
| 341 | + pass |
| 342 | + |
| 343 | + # Overridable -- handle entity reference |
| 344 | + def handle_entityref(self, name): |
| 345 | + pass |
| 346 | + |
| 347 | + # Overridable -- handle data |
| 348 | + def handle_data(self, data): |
| 349 | + pass |
| 350 | + |
| 351 | + # Overridable -- handle comment |
| 352 | + def handle_comment(self, data): |
| 353 | + pass |
| 354 | + |
| 355 | + # Overridable -- handle declaration |
| 356 | + def handle_decl(self, decl): |
| 357 | + pass |
| 358 | + |
| 359 | + # Overridable -- handle processing instruction |
| 360 | + def handle_pi(self, data): |
| 361 | + pass |
| 362 | + |
| 363 | + def unknown_decl(self, data): |
| 364 | + self.error("unknown declaration: %r" % (data,)) |
| 365 | + |
| 366 | + # Internal -- helper to remove special character quoting |
| 367 | + entitydefs = None |
| 368 | + def unescape(self, s): |
| 369 | + if '&' not in s: |
| 370 | + return s |
| 371 | + def replaceEntities(s): |
| 372 | + s = s.groups()[0] |
| 373 | + try: |
| 374 | + if s[0] == "#": |
| 375 | + s = s[1:] |
| 376 | + if s[0] in ['x','X']: |
| 377 | + c = int(s[1:], 16) |
| 378 | + else: |
| 379 | + c = int(s) |
| 380 | + return unichr(c) |
| 381 | + except ValueError: |
| 382 | + return '&#'+s+';' |
| 383 | + else: |
| 384 | + # Cannot use name2codepoint directly, because HTMLParser supports apos, |
| 385 | + # which is not part of HTML 4 |
| 386 | + import htmlentitydefs |
| 387 | + if HTMLParser.entitydefs is None: |
| 388 | + entitydefs = HTMLParser.entitydefs = {'apos':u"'"} |
| 389 | + for k, v in htmlentitydefs.name2codepoint.iteritems(): |
| 390 | + entitydefs[k] = unichr(v) |
| 391 | + try: |
| 392 | + return self.entitydefs[s] |
| 393 | + except KeyError: |
| 394 | + return '&'+s+';' |
| 395 | + |
| 396 | + return re.sub(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));", replaceEntities, s) |
0 commit comments