/usr/lib/python2.7/dist-packages/docutils/utils/punctuation_chars.py is in python-docutils 0.12+dfsg-1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 | #!/usr/bin/env python
# -*- coding: utf-8 -*-
# :Copyright: © 2011 Günter Milde.
# :License: Released under the terms of the `2-Clause BSD license`_, in short:
#
# Copying and distribution of this file, with or without modification,
# are permitted in any medium without royalty provided the copyright
# notice and this notice are preserved.
# This file is offered as-is, without any warranty.
#
# .. _2-Clause BSD license: http://www.spdx.org/licenses/BSD-2-Clause
# :Id: $Id: punctuation_chars.py 7668 2013-06-04 12:46:30Z milde $
import sys, re
import unicodedata
# punctuation characters around inline markup
# ===========================================
#
# This module provides the lists of characters for the implementation of
# the `inline markup recognition rules`_ in the reStructuredText parser
# (states.py)
#
# .. _inline markup recognition rules:
# ../../docs/ref/rst/restructuredtext.html#inline-markup
# Docutils punctuation category sample strings
# --------------------------------------------
#
# The sample strings are generated by punctuation_samples() and put here
# literal to avoid the time-consuming generation with every Docutils run.
# As the samples are used inside ``[ ]`` in regular expressions, hyphen and
# square brackets are escaped. ::
openers = (u'"\'(<\\[{\u0f3a\u0f3c\u169b\u2045\u207d\u208d\u2329\u2768'
u'\u276a\u276c\u276e\u2770\u2772\u2774\u27c5\u27e6\u27e8\u27ea'
u'\u27ec\u27ee\u2983\u2985\u2987\u2989\u298b\u298d\u298f\u2991'
u'\u2993\u2995\u2997\u29d8\u29da\u29fc\u2e22\u2e24\u2e26\u2e28'
u'\u3008\u300a\u300c\u300e\u3010\u3014\u3016\u3018\u301a\u301d'
u'\u301d\ufd3e\ufe17\ufe35\ufe37\ufe39\ufe3b\ufe3d\ufe3f\ufe41'
u'\ufe43\ufe47\ufe59\ufe5b\ufe5d\uff08\uff3b\uff5b\uff5f\uff62'
u'\xab\u2018\u201c\u2039\u2e02\u2e04\u2e09\u2e0c\u2e1c\u2e20'
u'\u201a\u201e\xbb\u2019\u201d\u203a\u2e03\u2e05\u2e0a\u2e0d'
u'\u2e1d\u2e21\u201b\u201f')
closers = (u'"\')>\\]}\u0f3b\u0f3d\u169c\u2046\u207e\u208e\u232a\u2769'
u'\u276b\u276d\u276f\u2771\u2773\u2775\u27c6\u27e7\u27e9\u27eb'
u'\u27ed\u27ef\u2984\u2986\u2988\u298a\u298c\u298e\u2990\u2992'
u'\u2994\u2996\u2998\u29d9\u29db\u29fd\u2e23\u2e25\u2e27\u2e29'
u'\u3009\u300b\u300d\u300f\u3011\u3015\u3017\u3019\u301b\u301e'
u'\u301f\ufd3f\ufe18\ufe36\ufe38\ufe3a\ufe3c\ufe3e\ufe40\ufe42'
u'\ufe44\ufe48\ufe5a\ufe5c\ufe5e\uff09\uff3d\uff5d\uff60\uff63'
u'\xbb\u2019\u201d\u203a\u2e03\u2e05\u2e0a\u2e0d\u2e1d\u2e21'
u'\u201b\u201f\xab\u2018\u201c\u2039\u2e02\u2e04\u2e09\u2e0c'
u'\u2e1c\u2e20\u201a\u201e')
delimiters = (u'\\-/:\u058a\xa1\xb7\xbf\u037e\u0387\u055a-\u055f\u0589'
u'\u05be\u05c0\u05c3\u05c6\u05f3\u05f4\u0609\u060a\u060c'
u'\u060d\u061b\u061e\u061f\u066a-\u066d\u06d4\u0700-\u070d'
u'\u07f7-\u07f9\u0830-\u083e\u0964\u0965\u0970\u0df4\u0e4f'
u'\u0e5a\u0e5b\u0f04-\u0f12\u0f85\u0fd0-\u0fd4\u104a-\u104f'
u'\u10fb\u1361-\u1368\u1400\u166d\u166e\u16eb-\u16ed\u1735'
u'\u1736\u17d4-\u17d6\u17d8-\u17da\u1800-\u180a\u1944\u1945'
u'\u19de\u19df\u1a1e\u1a1f\u1aa0-\u1aa6\u1aa8-\u1aad\u1b5a-'
u'\u1b60\u1c3b-\u1c3f\u1c7e\u1c7f\u1cd3\u2010-\u2017\u2020-'
u'\u2027\u2030-\u2038\u203b-\u203e\u2041-\u2043\u2047-'
u'\u2051\u2053\u2055-\u205e\u2cf9-\u2cfc\u2cfe\u2cff\u2e00'
u'\u2e01\u2e06-\u2e08\u2e0b\u2e0e-\u2e1b\u2e1e\u2e1f\u2e2a-'
u'\u2e2e\u2e30\u2e31\u3001-\u3003\u301c\u3030\u303d\u30a0'
u'\u30fb\ua4fe\ua4ff\ua60d-\ua60f\ua673\ua67e\ua6f2-\ua6f7'
u'\ua874-\ua877\ua8ce\ua8cf\ua8f8-\ua8fa\ua92e\ua92f\ua95f'
u'\ua9c1-\ua9cd\ua9de\ua9df\uaa5c-\uaa5f\uaade\uaadf\uabeb'
u'\ufe10-\ufe16\ufe19\ufe30-\ufe32\ufe45\ufe46\ufe49-\ufe4c'
u'\ufe50-\ufe52\ufe54-\ufe58\ufe5f-\ufe61\ufe63\ufe68\ufe6a'
u'\ufe6b\uff01-\uff03\uff05-\uff07\uff0a\uff0c-\uff0f\uff1a'
u'\uff1b\uff1f\uff20\uff3c\uff61\uff64\uff65')
if sys.maxunicode >= 0x10FFFF: # "wide" build
delimiters += (u'\U00010100\U00010101\U0001039f\U000103d0\U00010857'
u'\U0001091f\U0001093f\U00010a50-\U00010a58\U00010a7f'
u'\U00010b39-\U00010b3f\U000110bb\U000110bc\U000110be-'
u'\U000110c1\U00012470-\U00012473')
closing_delimiters = u'\\\\.,;!?'
# Matching open/close quotes
# --------------------------
# Rule (5) requires determination of matching open/close pairs. However,
# the pairing of open/close quotes is ambigue due to different typographic
# conventions in different languages.
quote_pairs = {u'\xbb': u'\xbb', # Swedish
u'\u2018': u'\u201a', # Greek
u'\u2019': u'\u2019', # Swedish
u'\u201a': u'\u2018\u2019', # German, Polish
u'\u201c': u'\u201e', # German
u'\u201e': u'\u201c\u201d',
u'\u201d': u'\u201d', # Swedish
u'\u203a': u'\u203a', # Swedish
}
def match_chars(c1, c2):
try:
i = openers.index(c1)
except ValueError: # c1 not in openers
return False
return c2 == closers[i] or c2 in quote_pairs.get(c1, '')
# Running this file as a standalone module checks the definitions against a
# re-calculation::
if __name__ == '__main__':
# Unicode punctuation character categories
# ----------------------------------------
unicode_punctuation_categories = {
# 'Pc': 'Connector', # not used in Docutils inline markup recognition
'Pd': 'Dash',
'Ps': 'Open',
'Pe': 'Close',
'Pi': 'Initial quote', # may behave like Ps or Pe depending on usage
'Pf': 'Final quote', # may behave like Ps or Pe depending on usage
'Po': 'Other'
}
"""Unicode character categories for punctuation"""
# generate character pattern strings
# ==================================
def unicode_charlists(categories, cp_min=0, cp_max=None):
"""Return dictionary of Unicode character lists.
For each of the `catagories`, an item contains a list with all Unicode
characters with `cp_min` <= code-point <= `cp_max` that belong to
the category.
The default values check every code-point supported by Python
(`sys.maxint` is 0x10FFFF in a "wide" build and 0xFFFF in a "narrow"
build, i.e. ucs4 and ucs2 respectively).
"""
# Determine highest code point with one of the given categories
# (may shorten the search time considerably if there are many
# categories with not too high characters):
if cp_max is None:
cp_max = max(x for x in xrange(sys.maxunicode+1)
if unicodedata.category(unichr(x)) in categories)
# print cp_max # => 74867 for unicode_punctuation_categories
charlists = {}
for cat in categories:
charlists[cat] = [unichr(x) for x in xrange(cp_min, cp_max+1)
if unicodedata.category(unichr(x)) == cat]
return charlists
# Character categories in Docutils
# --------------------------------
def punctuation_samples():
"""Docutils punctuation category sample strings.
Return list of sample strings for the categories "Open", "Close",
"Delimiters" and "Closing-Delimiters" used in the `inline markup
recognition rules`_.
"""
# Lists with characters in Unicode punctuation character categories
cp_min = 160 # ASCII chars have special rules for backwards compatibility
ucharlists = unicode_charlists(unicode_punctuation_categories, cp_min)
# match opening/closing characters
# --------------------------------
# Rearange the lists to ensure matching characters at the same
# index position.
# low quotation marks are also used as closers (e.g. in Greek)
# move them to category Pi:
ucharlists['Ps'].remove(u'‚') # 201A SINGLE LOW-9 QUOTATION MARK
ucharlists['Ps'].remove(u'„') # 201E DOUBLE LOW-9 QUOTATION MARK
ucharlists['Pi'] += [u'‚', u'„']
ucharlists['Pi'].remove(u'‛') # 201B SINGLE HIGH-REVERSED-9 QUOTATION MARK
ucharlists['Pi'].remove(u'‟') # 201F DOUBLE HIGH-REVERSED-9 QUOTATION MARK
ucharlists['Pf'] += [u'‛', u'‟']
# 301F LOW DOUBLE PRIME QUOTATION MARK misses the opening pendant:
ucharlists['Ps'].insert(ucharlists['Pe'].index(u'\u301f'), u'\u301d')
# print u''.join(ucharlists['Ps']).encode('utf8')
# print u''.join(ucharlists['Pe']).encode('utf8')
# print u''.join(ucharlists['Pi']).encode('utf8')
# print u''.join(ucharlists['Pf']).encode('utf8')
# The Docutils character categories
# ---------------------------------
#
# The categorization of ASCII chars is non-standard to reduce
# both false positives and need for escaping. (see `inline markup
# recognition rules`_)
# allowed before markup if there is a matching closer
openers = [u'"\'(<\\[{']
for cat in ('Ps', 'Pi', 'Pf'):
openers.extend(ucharlists[cat])
# allowed after markup if there is a matching opener
closers = [u'"\')>\\]}']
for cat in ('Pe', 'Pf', 'Pi'):
closers.extend(ucharlists[cat])
# non-matching, allowed on both sides
delimiters = [u'\\-/:']
for cat in ('Pd', 'Po'):
delimiters.extend(ucharlists[cat])
# non-matching, after markup
closing_delimiters = [r'\\.,;!?']
# # Test open/close matching:
# for i in range(min(len(openers),len(closers))):
# print '%4d %s %s' % (i, openers[i].encode('utf8'),
# closers[i].encode('utf8'))
return [u''.join(chars) for chars in (openers, closers, delimiters,
closing_delimiters)]
def separate_wide_chars(s):
"""Return (s1,s2) with characters above 0xFFFF in s2"""
maxunicode_narrow = 0xFFFF
l1 = [ch for ch in s if ord(ch) <= maxunicode_narrow]
l2 = [ch for ch in s if ord(ch) > maxunicode_narrow]
return ''.join(l1), ''.join(l2)
def mark_intervals(s):
"""Return s with shortcut notation for runs of consecutive characters
Sort string and replace 'cdef' by 'c-f' and similar.
"""
l =[]
s = [ord(ch) for ch in s]
s.sort()
for n in s:
try:
if l[-1][-1]+1 == n:
l[-1].append(n)
else:
l.append([n])
except IndexError:
l.append([n])
l2 = []
for i in l:
i = [unichr(n) for n in i]
if len(i) > 2:
i = i[0], u'-', i[-1]
l2.extend(i)
return ''.join(l2)
def wrap_string(s, startstring= "(",
endstring = ")", wrap=65):
"""Line-wrap a unicode string literal definition."""
c = len(startstring)
contstring = "'\n" + ' ' * len(startstring) + "u'"
l = [startstring]
for ch in s:
c += 1
if ch == '\\' and c > wrap:
c = len(startstring)
ch = contstring + ch
l.append(ch)
l.append(endstring)
return ''.join(l)
# print results
# =============
# (re) create and compare the samples:
(o, c, d, cd) = punctuation_samples()
o, o_wide = separate_wide_chars(o)
c, c_wide = separate_wide_chars(c)
d, d_wide = separate_wide_chars(d)
d = d[:5] + mark_intervals(d[5:])
d_wide = mark_intervals(d_wide)
if sys.maxunicode >= 0x10FFFF: # "wide" build
d += d_wide
if o != openers:
print '- openers = ur"""%s"""' % openers.encode('utf8')
print '+ openers = ur"""%s"""' % o.encode('utf8')
if o_wide:
print '+ openers-wide = ur"""%s"""' % o_wide.encode('utf8')
if c != closers:
print '- closers = ur"""%s"""' % closers.encode('utf8')
print '+ closers = ur"""%s"""' % c.encode('utf8')
if c_wide:
print '+ closers-wide = ur"""%s"""' % c_wide.encode('utf8')
if d != delimiters:
print '- delimiters = ur"%s"' % delimiters.encode('utf8')
print '+ delimiters = ur"%s"' % d.encode('utf8')
if cd != closing_delimiters:
print '- closing_delimiters = ur"%s"' % closing_delimiters.encode('utf8')
print '+ closing_delimiters = ur"%s"' % cd.encode('utf8')
# closing_delimiters are all ASCII characters
# Print literal code to define the character sets:
# `openers` and `closers` must be verbose and keep order because they are
# also used in `match_chars()`.
print wrap_string(repr(o), startstring='openers = (')
print wrap_string(repr(c), startstring='closers = (')
# delimiters: sort and use shortcut for intervals (saves ~150 characters):
print wrap_string(repr(d), startstring='delimiters = (')
# add characters in the upper plane only in a "wide" build:
print 'if sys.maxunicode >= 0x10FFFF: # "wide" build'
print wrap_string(repr(d_wide), startstring=' delimiters += (')
print 'closing_delimiters =', repr(cd)
# test prints
# print "wide" Unicode characters:
# ucharlists = unicode_charlists(unicode_punctuation_categories)
# for key in ucharlists:
# if key.endswith('wide'):
# print key, ucharlists[key]
# print 'openers = ', repr(openers)
# print 'closers = ', repr(closers)
# print 'delimiters = ', repr(delimiters)
# print 'closing_delimiters = ', repr(closing_delimiters)
# ucharlists = unicode_charlists(unicode_punctuation_categories)
# for cat, chars in ucharlists.items():
# # print cat, chars
# # compact output (visible with a comprehensive font):
# print (u":%s: %s" % (cat, u''.join(chars))).encode('utf8')
# verbose print
# print 'openers:'
# for ch in openers:
# print ch.encode('utf8'), unicodedata.name(ch)
# print 'closers:'
# for ch in closers:
# print ch.encode('utf8'), unicodedata.name(ch)
# print 'delimiters:'
# for ch in delimiters:
# print ch.encode('utf8'), unicodedata.name(ch)
# print 'closing_delimiters:'
# for ch in closing_delimiters:
# print ch.encode('utf8'), unicodedata.name(ch)
|