Merge pull request microsoft#4825 from be5invis/be5invis-patch-kinsok…

…u-shori

Add CJK-aware line wrapping, and basic Kinsoku Shori (禁則処理) to Visual Studio Code

src/vs/editor/common/config/defaultConfig.ts

@@ -54,8 +54,8 @@ class ConfigClass implements IConfiguration {
 			automaticLayout: false,
 			wrappingColumn: 300,
 			wrappingIndent: 'same',
-			wordWrapBreakBeforeCharacters: '{([+',
-			wordWrapBreakAfterCharacters: ' \t})]?|&,;',
+			wordWrapBreakBeforeCharacters: '([{‘“〈《「『【〔（［｛｢£¥＄￡￥+＋',
+			wordWrapBreakAfterCharacters: ' \t})]?|&,;¢°′″‰℃、。｡､￠，．：；？！％・･ゝゞヽヾーァィゥェォッャュョヮヵヶぁぃぅぇぉっゃゅょゎゕゖㇰㇱㇲㇳㇴㇵㇶㇷㇸㇹㇺㇻㇼㇽㇾㇿ々〻ｧｨｩｪｫｬｭｮｯｰ’”〉》」』】〕）］｝｣',
 			wordWrapBreakObtrusiveCharacters: '.',
 			tabFocusMode: false,
 			// stopLineTokenizationAfter
@@ -88,4 +88,4 @@ class ConfigClass implements IConfiguration {
 	}
 }
 
-export var DefaultConfig: IConfiguration = new ConfigClass();
+export var DefaultConfig: IConfiguration = new ConfigClass();

src/vs/editor/common/viewModel/characterHardWrappingLineMapper.ts

@@ -12,6 +12,7 @@ import {ILineMapperFactory, ILineMapping, OutputPosition} from 'vs/editor/common
 var BREAK_BEFORE_CLASS = 1;
 var BREAK_AFTER_CLASS = 2;
 var BREAK_OBTRUSIVE_CLASS = 3;
+var BREAK_IDEOGRAPHIC = 4; // for Han and Kana.
 
 function buildCharacterClassesMap(BREAK_BEFORE:string, BREAK_AFTER:string, BREAK_OBTRUSIVE:string): number[] {
 	var result:number[] = [],
@@ -26,6 +27,20 @@ function buildCharacterClassesMap(BREAK_BEFORE:string, BREAK_AFTER:string, BREAK
 	for (i = 0; i <= maxCharCode; i++) {
 		result[i] = 0;
 	}
+
+	// Initialize BREAK_IDEOGRAPHIC for these Unicode ranges:
+	// 1. CJK Unified Ideographs (0x4E00 -- 0x9FFF)
+	// 2. CJK Unified Ideographs Extension A (0x3400 -- 0x4DBF)
+	// 3. Hiragana and Katakana (0x3040 -- 0x30FF)
+	for (i = 0x3040; i <= 0x30FF; i++) {
+		result[i] = BREAK_IDEOGRAPHIC;
+	}
+	for (i = 0x3400; i <= 0x4DBF; i++) {
+		result[i] = BREAK_IDEOGRAPHIC;
+	}
+	for (i = 0x4E00; i <= 0x9FFF; i++) {
+		result[i] = BREAK_IDEOGRAPHIC;
+	}
 
 	for (i = 0; i < BREAK_BEFORE.length; i++) {
 		result[BREAK_BEFORE.charCodeAt(i)] = BREAK_BEFORE_CLASS;
@@ -106,7 +121,11 @@ export class CharacterHardWrappingLineMapperFactory implements ILineMapperFactor
 			charCodeIsTab:boolean,
 			charCodeClass:number,
 			breakBeforeOffset:number, // 0-based offset in the lineText before which breaking
-			restoreVisibleColumnFrom:number; // visible column used to re-establish a correct `visibleColumn`
+			restoreVisibleColumnFrom:number, // visible column used to re-establish a correct `visibleColumn`
+			prevCode:number, // CJK kinsoku shori: character and class before current one
+			prevClass:number,
+			nextCode:number, // CJK kinsoku shori: character and class after current one
+			nextClass:number;
 
 		var niceBreakOffset = -1, // Last index of a character that indicates a break should happen before it (more desirable)
 			niceBreakVisibleColumn = 0, // visible column if a break were to be later introduced before `niceBreakOffset`
@@ -130,6 +149,16 @@ export class CharacterHardWrappingLineMapperFactory implements ILineMapperFactor
 				niceBreakVisibleColumn = 0;
 			}
 
+			// CJK breaking : before break
+			if (charCodeClass === BREAK_IDEOGRAPHIC && i > 0) {
+				prevCode = lineText.charCodeAt(i - 1);
+				prevClass = prevCode < characterClasses.length ? characterClasses[prevCode] : 0;
+				if (prevClass !== BREAK_BEFORE_CLASS) { // Kinsoku Shori: Don't break after a leading character, like an open bracket
+					niceBreakOffset = i;
+					niceBreakVisibleColumn = 0;
+				}
+			}
+
 			var charColumnSize = 1;
 			if (strings.isFullWidthCharacter(charCode)) {
 				charColumnSize = columnsForFullWidthChar;
@@ -194,7 +223,17 @@ export class CharacterHardWrappingLineMapperFactory implements ILineMapperFactor
 				niceBreakOffset = i + 1;
 				niceBreakVisibleColumn = 0;
 			}
-
+
+			// CJK breaking : after break
+			if (charCodeClass === BREAK_IDEOGRAPHIC && i < len - 1) {
+				nextCode = lineText.charCodeAt(i + 1);
+				nextClass = nextCode < characterClasses.length ? characterClasses[nextCode] : 0;
+				if (nextClass !== BREAK_AFTER_CLASS) { // Kinsoku Shori: Don't break before a trailing character, like a period
+					niceBreakOffset = i + 1;
+					niceBreakVisibleColumn = 0;
+				}
+			}
+
 			if (charCodeClass === BREAK_OBTRUSIVE_CLASS) {
 				// This is an obtrusive character that indicates that a break should happen after it
 				obtrusiveBreakOffset = i + 1;
@@ -243,4 +282,4 @@ export class CharacterHardWrappingLineMapping implements ILineMapping {
 		let r = this._prefixSums.getIndexOf(inputOffset);
 		return new OutputPosition(r.index, r.remainder);
 	}
-}
+}

src/vs/editor/test/common/viewModel/characterHardWrappingLineMapper.test.ts

@@ -55,7 +55,7 @@ function assertLineMapping(factory:ILineMapperFactory, tabSize:number, breakAfte
 		}
 	}
 
-	let mapper = factory.createLineMapping(rawText, tabSize, breakAfter, 1, WrappingIndent.None);
+	var mapper = factory.createLineMapping(rawText, tabSize, breakAfter, 2, WrappingIndent.None);
 
 	assert.equal(safeGetOutputLineCount(mapper), (lineIndices.length > 0 ? lineIndices[lineIndices.length - 1] + 1 : 1));
 	for (let i = 0, len = rawText.length; i < len; i++) {
@@ -105,4 +105,13 @@ suite('Editor ViewModel - CharacterHardWrappingLineMapper', () => {
 		assertLineMapping(factory, 4, 5, 'aa.(|()|.aaa');
 		assertLineMapping(factory, 4, 5, 'aa.|(.)|.aaa');
 	});
+	test('CharacterHardWrappingLineMapper - CJK and Kinsoku Shori', () => {
+		var factory = new CharacterHardWrappingLineMapperFactory('(', ')', '.');
+		assertLineMapping(factory, 4, 5, 'aa \u5b89|\u5b89');
+		assertLineMapping(factory, 4, 5, '\u3042 \u5b89|\u5b89');
+		assertLineMapping(factory, 4, 5, '\u3042\u3042|\u5b89\u5b89');
+		assertLineMapping(factory, 4, 5, 'aa |\u5b89)\u5b89|\u5b89');
+		assertLineMapping(factory, 4, 5, 'aa \u3042|\u5b89\u3042)|\u5b89');
+		assertLineMapping(factory, 4, 5, 'aa |(\u5b89aa|\u5b89');
+	});
 });