CodeMirrorModeMediaWiki: add highlighting for <nowiki> and <pre>

Since <nowiki> and <pre> ignore wikitext, the CM5 implementation
cleverly leveraged the tagModes system so that only HTML entities are
processed. We're effectively doing the same here, only we don't need to
register them as proper TagModes. A FIXME is left to remove the entries
from extension.json after the CM6 upgrade is complete.

Note that line-level styling is still missing, see T351686#9431669.
As a result, multi-line content in a <nowiki> or <pre> may emit JS
warnings, but this is expected until T351686 is resolved.

Bug: T348684
Change-Id: Ia834c4609faf38af3c8f6b791544a7441b5cfb0a

src/codemirror.mode.mediawiki.config.js

@@ -122,13 +122,21 @@ class CodeMirrorModeMediaWikiConfig {
 			em: 'mw-em',
 			error: 'mw-error',
 			extGround: 'mw-ext-ground',
+			extNowiki: 'mw-ext-nowiki',
+			extPre: 'mw-ext-pre',
+			extTag: 'mw-exttag',
+			extTagAttribute: 'mw-exttag-attribute',
+			extTagBracket: 'mw-exttag-bracket',
+			extTagName: 'mw-exttag-name',
 			freeExtLink: 'mw-free-extlink',
 			freeExtLinkProtocol: 'mw-free-extlink-protocol',
 			htmlEntity: 'mw-html-entity',
 			link: 'mw-link',
 			linkGround: 'mw-link-ground',
 			linkPageName: 'mw-link-pagename',
+			nowiki: 'mw-tag-nowiki',
 			pageName: 'mw-pagename',
+			pre: 'mw-tag-pre',
 			skipFormatting: 'mw-skipformatting',
 			strong: 'mw-strong',
 			tableCaption: 'mw-table-caption',
@@ -156,13 +164,21 @@ class CodeMirrorModeMediaWikiConfig {
 			[ this.tags.em ]: Tag.define(),
 			[ this.tags.error ]: Tag.define(),
 			[ this.tags.extGround ]: Tag.define(),
+			[ this.tags.extNowiki ]: Tag.define(),
+			[ this.tags.extPre ]: Tag.define(),
+			[ this.tags.extTag ]: Tag.define(),
+			[ this.tags.extTagAttribute ]: Tag.define(),
+			[ this.tags.extTagBracket ]: Tag.define(),
+			[ this.tags.extTagName ]: Tag.define(),
 			[ this.tags.freeExtLink ]: Tag.define(),
 			[ this.tags.freeExtLinkProtocol ]: Tag.define(),
 			[ this.tags.htmlEntity ]: Tag.define(),
 			[ this.tags.link ]: Tag.define(),
 			[ this.tags.linkGround ]: Tag.define(),
 			[ this.tags.linkPageName ]: Tag.define(),
+			[ this.tags.nowiki ]: Tag.define(),
 			[ this.tags.pageName ]: Tag.define(),
+			[ this.tags.pre ]: Tag.define(),
 			[ this.tags.skipFormatting ]: Tag.define(),
 			[ this.tags.strong ]: Tag.define(),
 			[ this.tags.tableCaption ]: Tag.define(),
@@ -372,6 +388,30 @@ class CodeMirrorModeMediaWikiConfig {
 				tag: context.tokenTable[ this.tags.extGround ],
 				class: 'cm-mw-ext-ground'
 			},
+			{
+				tag: context.tokenTable[ this.tags.extNowiki ],
+				class: 'cm-mw-ext-nowiki'
+			},
+			{
+				tag: context.tokenTable[ this.tags.extPre ],
+				class: 'cm-mw-ext-pre'
+			},
+			{
+				tag: context.tokenTable[ this.tags.extTagBracket ],
+				class: 'cm-mw-exttag-bracket'
+			},
+			{
+				tag: context.tokenTable[ this.tags.extTag ],
+				class: 'cm-mw-exttag'
+			},
+			{
+				tag: context.tokenTable[ this.tags.extTagAttribute ],
+				class: 'cm-mw-exttag-attribute'
+			},
+			{
+				tag: context.tokenTable[ this.tags.extTagName ],
+				class: 'cm-mw-exttag-name'
+			},
 			{
 				tag: context.tokenTable[ this.tags.freeExtLink ],
 				class: 'cm-mw-free-extlink'
@@ -392,10 +432,18 @@ class CodeMirrorModeMediaWikiConfig {
 				tag: context.tokenTable[ this.tags.linkPageName ],
 				class: 'cm-mw-link-pagename'
 			},
+			{
+				tag: context.tokenTable[ this.tags.nowiki ],
+				class: 'cm-mw-tag-nowiki'
+			},
 			{
 				tag: context.tokenTable[ this.tags.pageName ],
 				class: 'cm-mw-pagename'
 			},
+			{
+				tag: context.tokenTable[ this.tags.pre ],
+				class: 'cm-mw-tag-pre'
+			},
 			{
 				tag: context.tokenTable[ this.tags.skipFormatting ],
 				class: 'cm-mw-skipformatting'

src/codemirror.mode.mediawiki.js

@@ -393,7 +393,8 @@ class CodeMirrorModeMediaWiki {
 			state.tokenize = state.stack.pop();
 			return;
 		}
-		if ( stream.match( /^[^|\]&~{}]+/ ) ) { // FIXME '{{' brokes Link, sample [[z{{page]]
+		// FIXME '{{' brokes Link, sample [[z{{page]]
+		if ( stream.match( /^[^|\]&~{}]+/ ) ) {
 			return this.makeLocalStyle( modeConfig.tags.linkToSection, state );
 		}
 		if ( stream.eat( '|' ) ) {
@@ -462,16 +463,17 @@ class CodeMirrorModeMediaWiki {
 					state.tokenize = this.eatHtmlTagAttribute( name );
 				}
 				return this.makeLocalStyle( modeConfig.tags.htmlTagName, state );
-			} // it is the extension tag
+			}
+			// it is the extension tag
 			if ( isCloseTag ) {
 				state.tokenize = this.eatChar(
 					'>',
-					`${ modeConfig.tags.extLinkBracket } mw-ext-${ name }`
+					`${ modeConfig.tags.extTagBracket } mw-ext-${ name }`
 				);
 			} else {
 				state.tokenize = this.eatExtTagAttribute( name );
 			}
-			return this.makeLocalStyle( 'mw-exttag-name mw-ext-' + name, state );
+			return this.makeLocalStyle( `${ modeConfig.tags.extTagName } mw-ext-${ name }`, state );
 		};
 	}
 
@@ -496,15 +498,37 @@ class CodeMirrorModeMediaWiki {
 		};
 	}
 
+	eatNowiki( style ) {
+		return ( stream ) => {
+			if ( stream.match( /^[^&]+/ ) ) {
+				return style;
+			}
+			// eat &
+			stream.next();
+			return this.eatHtmlEntity( stream, style, style );
+		};
+	}
+
 	eatExtTagAttribute( name ) {
 		return ( stream, state ) => {
 			// eslint-disable-next-line security/detect-unsafe-regex
 			if ( stream.match( /^(?:"[^">]*"|'[^'>]*'|[^>/<{&~])+/ ) ) {
-				return this.makeLocalStyle( `mw-exttag-attribute mw-ext-${ name }`, state );
+				return this.makeLocalStyle( `${ modeConfig.tags.extTagAttribute } mw-ext-${ name }`, state );
 			}
 			if ( stream.eat( '>' ) ) {
 				state.extName = name;
-				if ( name in this.config.tagModes ) {
+
+				// FIXME: remove 'nowiki' and 'pre' from TagModes in extension.json after CM6 upgrade
+				// leverage the tagModes system for <nowiki> and <pre>
+				if ( name === 'nowiki' || name === 'pre' ) {
+					// There's no actual processing within these tags (apart from HTML entities),
+					// so startState and copyState can be no-ops.
+					state.extMode = {
+						startState: () => {},
+						copyState: () => {},
+						token: this.eatNowiki( modeConfig.tags[ name ] )
+					};
+				} else if ( name in this.config.tagModes ) {
 					// FIXME: tracked at T348684
 					// state.extMode = CodeMirror.getMode(
 					//  this.config,
@@ -512,14 +536,15 @@ class CodeMirrorModeMediaWiki {
 					// );
 					// state.extState = CodeMirror.startState( state.extMode );
 				}
+
 				state.tokenize = this.eatExtTagArea( name );
-				return this.makeLocalStyle( 'mw-exttag-bracket mw-ext-' + name, state );
+				return this.makeLocalStyle( `${ modeConfig.tags.extTagBracket } mw-ext-${ name }`, state );
 			}
 			if ( stream.match( '/>' ) ) {
 				state.tokenize = state.stack.pop();
-				return this.makeLocalStyle( 'mw-exttag-bracket mw-ext-' + name, state );
+				return this.makeLocalStyle( `${ modeConfig.tags.extTagBracket } mw-ext-${ name }`, state );
 			}
-			return this.eatWikiText( 'mw-exttag-attribute mw-ext-' + name )( stream, state );
+			return this.eatWikiText( `${ modeConfig.tags.extTagAttribute } mw-ext-${ name }` )( stream, state );
 		};
 	}
 
@@ -558,7 +583,7 @@ class CodeMirrorModeMediaWiki {
 			stream.next(); // eat <
 			stream.next(); // eat /
 			state.tokenize = this.eatTagName( name.length, true, false );
-			return this.makeLocalStyle( 'mw-exttag-bracket mw-ext-' + name, state );
+			return this.makeLocalStyle( `${ modeConfig.tags.extTagBracket } mw-ext-${ name }`, state );
 		};
 	}
 
@@ -566,7 +591,7 @@ class CodeMirrorModeMediaWiki {
 		return ( stream, state ) => {
 			let ret;
 			if ( state.extMode === false ) {
-				ret = ( origString === false && stream.sol() ? 'line-cm-mw-exttag' : 'mw-exttag' );
+				ret = origString === false && stream.sol() ? 'line-cm-mw-exttag' : modeConfig.tags.extTag;
 				stream.skipToEnd();
 			} else {
 				ret = (
@@ -891,7 +916,7 @@ class CodeMirrorModeMediaWiki {
 					}
 					if ( tagname ) {
 						tagname = tagname[ 0 ].toLowerCase();
-						if ( this.config.tags && tagname in this.config.tags ) {
+						if ( tagname in this.config.tags ) {
 							// Parser function
 							if ( isCloseTag === true ) {
 								return modeConfig.tags.error;
@@ -899,7 +924,7 @@ class CodeMirrorModeMediaWiki {
 							stream.backUp( tagname.length );
 							state.stack.push( state.tokenize );
 							state.tokenize = this.eatTagName( tagname.length, isCloseTag, false );
-							return this.makeLocalStyle( 'mw-exttag-bracket mw-ext-' + tagname, state );
+							return this.makeLocalStyle( `${ modeConfig.tags.extTagBracket } mw-ext-${ tagname }`, state );
 						}
 						if ( tagname in modeConfig.permittedHtmlTags ) {
 							// Html tag

tests/jest/codemirror.mode.mediawiki.test.js

@@ -47,11 +47,11 @@ const testCases = [
 		input: '__NOTOC__',
 		output: '<div class="cm-line"><span class="cm-mw-double-underscore">__NOTOC__</span></div>'
 	},
-	// {
-	//  title: 'nowiki',
-	//  input: '<nowiki>{{foo}}<p> </div> {{{</nowiki>',
-	//  output: '<div class="cm-line"><span class="cm-mw-exttag-bracket cm-mw-ext-nowiki">&lt;</span><span class="cm-mw-exttag-name cm-mw-ext-nowiki">nowiki</span><span class="cm-mw-exttag-bracket cm-mw-ext-nowiki">&gt;</span><span class="cm-mw-tag-nowiki cm-mw-tag-nowiki">{{foo}}&lt;p&gt; &lt;/div&gt; {{{</span><span class="cm-mw-exttag-bracket cm-mw-ext-nowiki">&lt;/</span><span class="cm-mw-exttag-name cm-mw-ext-nowiki">nowiki</span><span class="cm-mw-exttag-bracket cm-mw-ext-nowiki">&gt;</span></div>'
-	// },
+	{
+		title: 'nowiki',
+		input: '<nowiki>{{foo}}<p> </div> {{{</nowiki>\n<nowiki/><pre class="foo">\n\n {{bar}}</pre>',
+		output: '<div class="cm-line"><span class="cm-mw-exttag-bracket cm-mw-ext-nowiki">&lt;</span><span class="cm-mw-exttag-name cm-mw-ext-nowiki">nowiki</span><span class="cm-mw-exttag-bracket cm-mw-ext-nowiki">&gt;</span><span class="cm-mw-tag-nowiki cm-mw-tag-nowiki">{{foo}}&lt;p&gt; &lt;/div&gt; {{{</span><span class="cm-mw-exttag-bracket cm-mw-ext-nowiki">&lt;/</span><span class="cm-mw-exttag-name cm-mw-ext-nowiki">nowiki</span><span class="cm-mw-exttag-bracket cm-mw-ext-nowiki">&gt;</span></div><div class="cm-line"><span class="cm-mw-exttag-bracket cm-mw-ext-nowiki">&lt;</span><span class="cm-mw-exttag-name cm-mw-ext-nowiki">nowiki</span><span class="cm-mw-exttag-bracket cm-mw-ext-nowiki">/&gt;</span><span class="cm-mw-exttag-bracket cm-mw-ext-pre">&lt;</span><span class="cm-mw-exttag-name cm-mw-ext-pre">pre </span><span class="cm-mw-exttag-attribute cm-mw-ext-pre">class="foo"</span><span class="cm-mw-exttag-bracket cm-mw-ext-pre">&gt;</span></div><div class="cm-line"><br></div><div class="cm-line"><span class="cm-mw-tag-pre cm-mw-tag-pre"> {{bar}}</span><span class="cm-mw-exttag-bracket cm-mw-ext-pre">&lt;/</span><span class="cm-mw-exttag-name cm-mw-ext-pre">pre</span><span class="cm-mw-exttag-bracket cm-mw-ext-pre">&gt;</span></div>'
+	},
 	// {
 	//  title: 'ref tag with cite web, extraneous curly braces',
 	//  input: '<ref>{{cite web|2=foo}}}}</ref>',
@@ -145,7 +145,11 @@ const mwLang = mediaWikiLang( {
 	} ],
 	functionSynonyms: [ {}, {
 		'!': '!'
-	} ]
+	} ],
+	tags: {
+		nowiki: true,
+		pre: true
+	}
 } );
 cm.initialize( [ ...cm.defaultExtensions, mwLang ] );
 
@@ -213,13 +217,21 @@ describe( 'CodeMirrorModeMediaWiki', () => {
 			'em',
 			'error',
 			'extGround',
+			'extNowiki',
+			'extPre',
+			'extTag',
+			'extTagAttribute',
+			'extTagBracket',
+			'extTagName',
 			'freeExtLink',
 			'freeExtLinkProtocol',
 			'htmlEntity',
 			'link',
 			'linkGround',
 			'linkPageName',
+			'nowiki',
 			'pageName',
+			'pre',
 			'skipFormatting',
 			'strong',
 			'tableCaption',