mirror of
https://gerrit.wikimedia.org/r/mediawiki/extensions/VisualEditor
synced 2024-11-30 00:55:00 +00:00
6f2090aac6
This is a prerequisite to browser-based grapheme cluster handling, which is needed so left/right cursoring and backspace behave as users expect. modules/ve/ve.js modules/ve/ce/ve.ce.Document.js modules/ve/ce/ve.ce.js * Revert cluster-aware splitting to trivial javascript code unit splitting * Rewrite ve.splitClusters as a trivial compatibility method (remove soon) * getClusterOffset/getByteOffset use unicodeJS.graphemebreak.splitClusters modules/unicodejs/tools/unicodejs-properties.py modules/unicodejs/unicodejs.graphemebreakproperties.js modules/unicodejs/unicodejs.js * Allow grapheme break tests to work with surrogate pairs demos/ve/pages/minimal.html demos/ve/pages/multibyte.html demos/ve/pages/unicode.html * replace file with more precise tests modules/ve/test/ve.test.js * Remove reference to grapheme-based splitting (which is no longer used) * Correct typo Bug: 53757 Bug: 51472 Bug: 51596 Bug: 51846 Change-Id: Ife34c87ebe40bc1689298b592eec5c0cdc2f7589
52 lines
1.7 KiB
Python
52 lines
1.7 KiB
Python
#!/usr/bin/env python
|
|
|
|
"""Generates unicodejs.*properties.js from Unicode data"""
|
|
|
|
import re, urllib2
|
|
|
|
for breaktype in ['Grapheme', 'Word']:
|
|
# a list of property name strings like "Extend", "Format" etc
|
|
properties = []
|
|
|
|
# range[property] -> character range list e.g. [0x0040, [0x0060-0x0070], 0x00A3, ...]
|
|
ranges = {}
|
|
|
|
# Analyse unicode data file
|
|
url = "http://www.unicode.org/Public/UNIDATA/auxiliary/" + breaktype + "BreakProperty.txt"
|
|
for line in urllib2.urlopen( url ):
|
|
line = line.strip()
|
|
# Ignore comment or blank lines
|
|
if re.search( r"^\s*(#|$)", line ): continue
|
|
# Find things like one of the following:
|
|
# XXXX ; propertyname
|
|
# XXXX..YYYY ; propertyname
|
|
m = re.search( r"^([0-9A-F]{4,5})(?:\.\.([0-9A-F]{4,5}))?\s*;\s*(\w+)\s*#", line )
|
|
if not m:
|
|
raise ValueError( "Bad line: %r" % line )
|
|
start, end, prop = m.groups()
|
|
if breaktype == 'Grapheme' and start == 'D800' and end == 'DFFF':
|
|
continue # raw surrogates are not treated
|
|
|
|
if not ranges.has_key( prop ):
|
|
properties.append( prop )
|
|
ranges.setdefault( prop, [] ).append( (start, end) )
|
|
|
|
# Translate ranges into js fragments
|
|
fragments = []
|
|
for prop in properties:
|
|
rangeStrings = []
|
|
for start, end in ranges[prop]:
|
|
if not end:
|
|
rangeStrings.append( "0x" + start )
|
|
else:
|
|
rangeStrings.append( "[0x" + start + ", 0x" + end + "]" )
|
|
fragments.append( "'" + prop + "': [" + ", ".join( rangeStrings ) + "]" )
|
|
|
|
# Write js file
|
|
js = "unicodeJS." + breaktype.lower() + "breakproperties = {\n\t"
|
|
js += ",\n\t".join( fragments )
|
|
js += "\n};\n"
|
|
jsFilename = "../unicodejs." + breaktype.lower() + "breakproperties.js"
|
|
open( jsFilename, "w" ).write( js )
|
|
print "wrote " + jsFilename
|