From 6ad61d4ddb022d951426078f7bcb4833741f76c0 Mon Sep 17 00:00:00 2001 From: Ed Sanders Date: Wed, 17 Apr 2013 18:53:26 +0100 Subject: [PATCH] Add data model support for MediaWiki references So far just read-only. Bug: 39599 Change-Id: I6daff5c5969e5fdc871f8f346cf790b4302ae080 --- .docs/categories.json | 4 + VisualEditor.php | 9 + demos/ve/index.php | 9 + modules/ve/ce/nodes/ve.ce.AlienNode.js | 4 +- .../ve/ce/nodes/ve.ce.GeneratedContentNode.js | 2 +- modules/ve/ce/nodes/ve.ce.InternalItemNode.js | 36 ++++ modules/ve/ce/nodes/ve.ce.InternalListNode.js | 36 ++++ .../ve/ce/nodes/ve.ce.MWReferenceListNode.js | 47 ++++++ modules/ve/ce/nodes/ve.ce.MWReferenceNode.js | 67 ++++++++ modules/ve/ce/nodes/ve.ce.MWTemplateNode.js | 2 +- modules/ve/ce/ve.ce.ContentBranchNode.js | 2 +- modules/ve/dm/nodes/ve.dm.InternalItemNode.js | 38 +++++ modules/ve/dm/nodes/ve.dm.InternalListNode.js | 38 +++++ modules/ve/dm/nodes/ve.dm.MWHeadingNode.js | 2 +- .../ve/dm/nodes/ve.dm.MWPreformattedNode.js | 2 +- .../ve/dm/nodes/ve.dm.MWReferenceListNode.js | 51 ++++++ modules/ve/dm/nodes/ve.dm.MWReferenceNode.js | 96 +++++++++++ modules/ve/dm/ve.dm.Converter.js | 146 +++++++++------- modules/ve/dm/ve.dm.Document.js | 11 +- modules/ve/dm/ve.dm.InternalList.js | 130 ++++++++++++++ modules/ve/dm/ve.dm.Model.js | 2 +- modules/ve/dm/ve.dm.Node.js | 9 + modules/ve/dm/ve.dm.NodeFactory.js | 15 ++ modules/ve/dm/ve.dm.Transaction.js | 38 ++++- .../mw/targets/ve.init.mw.ViewPageTarget.js | 12 +- modules/ve/init/mw/ve.init.mw.Target.js | 15 +- modules/ve/test/dm/ve.dm.Converter.test.js | 15 +- modules/ve/test/dm/ve.dm.Transaction.test.js | 78 ++++++++- modules/ve/test/dm/ve.dm.example.js | 159 ++++++++++++++++++ modules/ve/test/index.php | 9 + 30 files changed, 984 insertions(+), 100 deletions(-) create mode 100644 modules/ve/ce/nodes/ve.ce.InternalItemNode.js create mode 100644 modules/ve/ce/nodes/ve.ce.InternalListNode.js create mode 100644 modules/ve/ce/nodes/ve.ce.MWReferenceListNode.js create mode 100644 modules/ve/ce/nodes/ve.ce.MWReferenceNode.js create mode 100644 modules/ve/dm/nodes/ve.dm.InternalItemNode.js create mode 100644 modules/ve/dm/nodes/ve.dm.InternalListNode.js create mode 100644 modules/ve/dm/nodes/ve.dm.MWReferenceListNode.js create mode 100644 modules/ve/dm/nodes/ve.dm.MWReferenceNode.js create mode 100644 modules/ve/dm/ve.dm.InternalList.js diff --git a/.docs/categories.json b/.docs/categories.json index 65df31cb9e..4ec5aae915 100644 --- a/.docs/categories.json +++ b/.docs/categories.json @@ -65,6 +65,10 @@ "name": "Meta items", "classes": ["ve.dm.*MetaItem", "ve.dm.MetaList"] }, + { + "name": "Internal list", + "classes": ["ve.dm.InternalList"] + }, { "name": "Nodes", "classes": ["ve.dm.Document", "ve.dm.*Node"] diff --git a/VisualEditor.php b/VisualEditor.php index 7a4393a648..e2036c860f 100644 --- a/VisualEditor.php +++ b/VisualEditor.php @@ -240,6 +240,7 @@ $wgResourceModules += array( 've/dm/ve.dm.BranchNode.js', 've/dm/ve.dm.LeafNode.js', 've/dm/ve.dm.Annotation.js', + 've/dm/ve.dm.InternalList.js', 've/dm/ve.dm.MetaItem.js', 've/dm/ve.dm.MetaList.js', 've/dm/ve.dm.TransactionProcessor.js', @@ -266,6 +267,8 @@ $wgResourceModules += array( 've/dm/nodes/ve.dm.DocumentNode.js', 've/dm/nodes/ve.dm.HeadingNode.js', 've/dm/nodes/ve.dm.ImageNode.js', + 've/dm/nodes/ve.dm.InternalItemNode.js', + 've/dm/nodes/ve.dm.InternalListNode.js', 've/dm/nodes/ve.dm.ListItemNode.js', 've/dm/nodes/ve.dm.ListNode.js', 've/dm/nodes/ve.dm.ParagraphNode.js', @@ -279,6 +282,8 @@ $wgResourceModules += array( 've/dm/nodes/ve.dm.MWEntityNode.js', 've/dm/nodes/ve.dm.MWHeadingNode.js', 've/dm/nodes/ve.dm.MWPreformattedNode.js', + 've/dm/nodes/ve.dm.MWReferenceListNode.js', + 've/dm/nodes/ve.dm.MWReferenceNode.js', 've/dm/annotations/ve.dm.LinkAnnotation.js', 've/dm/annotations/ve.dm.MWExternalLinkAnnotation.js', @@ -317,6 +322,8 @@ $wgResourceModules += array( 've/ce/nodes/ve.ce.DocumentNode.js', 've/ce/nodes/ve.ce.HeadingNode.js', 've/ce/nodes/ve.ce.ImageNode.js', + 've/ce/nodes/ve.ce.InternalItemNode.js', + 've/ce/nodes/ve.ce.InternalListNode.js', 've/ce/nodes/ve.ce.ListItemNode.js', 've/ce/nodes/ve.ce.ListNode.js', 've/ce/nodes/ve.ce.ParagraphNode.js', @@ -330,6 +337,8 @@ $wgResourceModules += array( 've/ce/nodes/ve.ce.MWEntityNode.js', 've/ce/nodes/ve.ce.MWHeadingNode.js', 've/ce/nodes/ve.ce.MWPreformattedNode.js', + 've/ce/nodes/ve.ce.MWReferenceListNode.js', + 've/ce/nodes/ve.ce.MWReferenceNode.js', 've/ce/annotations/ve.ce.LinkAnnotation.js', 've/ce/annotations/ve.ce.MWExternalLinkAnnotation.js', diff --git a/demos/ve/index.php b/demos/ve/index.php index 427cdd2d9f..b7c1e86c66 100644 --- a/demos/ve/index.php +++ b/demos/ve/index.php @@ -124,6 +124,7 @@ $html = file_get_contents( $page ); + @@ -148,6 +149,8 @@ $html = file_get_contents( $page ); + + @@ -161,6 +164,8 @@ $html = file_get_contents( $page ); + + @@ -195,6 +200,8 @@ $html = file_get_contents( $page ); + + @@ -208,6 +215,8 @@ $html = file_get_contents( $page ); + + diff --git a/modules/ve/ce/nodes/ve.ce.AlienNode.js b/modules/ve/ce/nodes/ve.ce.AlienNode.js index b0705ea3b2..9cccccaeca 100644 --- a/modules/ve/ce/nodes/ve.ce.AlienNode.js +++ b/modules/ve/ce/nodes/ve.ce.AlienNode.js @@ -1,5 +1,5 @@ /*! - * VisualEditor ContentEditable AlienNode class. + * VisualEditor ContentEditable AlienNode, AlienBlockNode and AlienInlineNode classes. * * @copyright 2011-2013 VisualEditor Team and others; see AUTHORS.txt * @license The MIT License (MIT); see LICENSE.txt @@ -119,7 +119,7 @@ ve.ce.AlienNode.prototype.onSurfaceMouseMove = function ( e ) { * @param {jQuery.Event} e */ ve.ce.AlienNode.prototype.onSurfaceMouseOut = function ( e ) { - if ( e.toElement === null) { + if ( e.toElement === null ) { this.clearPhantoms(); } }; diff --git a/modules/ve/ce/nodes/ve.ce.GeneratedContentNode.js b/modules/ve/ce/nodes/ve.ce.GeneratedContentNode.js index 7d54b08d04..63bb61ba6f 100644 --- a/modules/ve/ce/nodes/ve.ce.GeneratedContentNode.js +++ b/modules/ve/ce/nodes/ve.ce.GeneratedContentNode.js @@ -1,5 +1,5 @@ /*! - * VisualEditor ContentEditable GeneratedContent class. + * VisualEditor ContentEditable GeneratedContentNode class. * * @copyright 2011-2013 VisualEditor Team and others; see AUTHORS.txt * @license The MIT License (MIT); see LICENSE.txt diff --git a/modules/ve/ce/nodes/ve.ce.InternalItemNode.js b/modules/ve/ce/nodes/ve.ce.InternalItemNode.js new file mode 100644 index 0000000000..c2109700fa --- /dev/null +++ b/modules/ve/ce/nodes/ve.ce.InternalItemNode.js @@ -0,0 +1,36 @@ +/*! + * VisualEditor InternalItemNode class. + * + * @copyright 2011-2013 VisualEditor Team and others; see AUTHORS.txt + * @license The MIT License (MIT); see LICENSE.txt + */ + +/** + * ContentEditable internal item node. + * + * @class + * @extends ve.ce.BranchNode + * @constructor + * @param {ve.dm.InternalItemNode} model Model to observe + */ +ve.ce.InternalItemNode = function VeCeInternalItemNode( model ) { + // Parent constructor + ve.ce.BranchNode.call( + this, model, $( '' ) + ); + + // TODO: render nothing + this.$.hide(); +}; + +/* Inheritance */ + +ve.inheritClass( ve.ce.InternalItemNode, ve.ce.BranchNode ); + +/* Static Properties */ + +ve.ce.InternalItemNode.static.name = 'internalItem'; + +/* Registration */ + +ve.ce.nodeFactory.register( ve.ce.InternalItemNode ); diff --git a/modules/ve/ce/nodes/ve.ce.InternalListNode.js b/modules/ve/ce/nodes/ve.ce.InternalListNode.js new file mode 100644 index 0000000000..edb3e9e1fc --- /dev/null +++ b/modules/ve/ce/nodes/ve.ce.InternalListNode.js @@ -0,0 +1,36 @@ +/*! + * VisualEditor InternalListNode class. + * + * @copyright 2011-2013 VisualEditor Team and others; see AUTHORS.txt + * @license The MIT License (MIT); see LICENSE.txt + */ + +/** + * ContentEditable internal list node. + * + * @class + * @extends ve.ce.BranchNode + * @constructor + * @param {ve.dm.InternalListNode} model Model to observe + */ +ve.ce.InternalListNode = function VeCeInternalListNode( model ) { + // Parent constructor + ve.ce.BranchNode.call( + this, model, $( '' ) + ); + + // TODO: render nothing + this.$.hide(); +}; + +/* Inheritance */ + +ve.inheritClass( ve.ce.InternalListNode, ve.ce.BranchNode ); + +/* Static Properties */ + +ve.ce.InternalListNode.static.name = 'internalList'; + +/* Registration */ + +ve.ce.nodeFactory.register( ve.ce.InternalListNode ); diff --git a/modules/ve/ce/nodes/ve.ce.MWReferenceListNode.js b/modules/ve/ce/nodes/ve.ce.MWReferenceListNode.js new file mode 100644 index 0000000000..ca8c73767a --- /dev/null +++ b/modules/ve/ce/nodes/ve.ce.MWReferenceListNode.js @@ -0,0 +1,47 @@ +/*! + * VisualEditor ContentEditable MWReferenceListNode class. + * + * @copyright 2011-2013 VisualEditor Team and others; see AUTHORS.txt + * @license The MIT License (MIT); see LICENSE.txt + */ + +/** + * ContentEditable MediaWiki reference list node. + * + * @class + * @extends ve.ce.LeafNode + * @constructor + * @param {ve.dm.MWReferenceListNode} model Model to observe + */ +ve.ce.MWReferenceListNode = function VeCeMWReferenceListNode( model ) { + // Parent constructor + ve.ce.LeafNode.call( this, model, $( '
' ) ); + + // DOM Changes + this.$.addClass( 've-ce-MWreferenceListNode', 'reference' ) + .attr( 'contenteditable', false ); + + // Events + this.model.addListenerMethod( this, 'update', 'onUpdate' ); + + // Initialization + this.onUpdate(); +}; + +/* Inheritance */ + +ve.inheritClass( ve.ce.MWReferenceListNode, ve.ce.LeafNode ); + +/* Static Properties */ + +ve.ce.MWReferenceListNode.static.name = 'MWreferenceList'; + +/* Methods */ + +ve.ce.MWReferenceListNode.prototype.onUpdate = function () { + this.$.html( this.model.getAttribute( 'html' ) ); +}; + +/* Registration */ + +ve.ce.nodeFactory.register( ve.ce.MWReferenceListNode ); diff --git a/modules/ve/ce/nodes/ve.ce.MWReferenceNode.js b/modules/ve/ce/nodes/ve.ce.MWReferenceNode.js new file mode 100644 index 0000000000..b5f413b233 --- /dev/null +++ b/modules/ve/ce/nodes/ve.ce.MWReferenceNode.js @@ -0,0 +1,67 @@ +/*! + * VisualEditor ContentEditable MWReferenceNode class. + * + * @copyright 2011-2013 VisualEditor Team and others; see AUTHORS.txt + * @license The MIT License (MIT); see LICENSE.txt + */ + +/** + * ContentEditable MediaWiki reference node. + * + * @class + * @extends ve.ce.LeafNode + * @constructor + * @param {ve.dm.MWReferenceNode} model Model to observe + */ +ve.ce.MWReferenceNode = function VeCeMWReferenceNode( model ) { + // Parent constructor + ve.ce.LeafNode.call( this, model, $( '' ) ); + + // DOM Changes + this.$link = $( '' ).attr( 'href', '#' ); + this.$.addClass( 've-ce-MWreferenceNode', 'reference' ) + .attr( 'contenteditable', false ) + .append( this.$link ); + + // Events + this.model.addListenerMethod( this, 'update', 'onUpdate' ); + this.$link.click( ve.bind( this.onClick, this ) ); + + // Initialization + this.onUpdate(); +}; + +/* Inheritance */ + +ve.inheritClass( ve.ce.MWReferenceNode, ve.ce.LeafNode ); + +/* Static Properties */ + +ve.ce.MWReferenceNode.static.name = 'MWreference'; + +/* Methods */ + +/** + * Handle update events. + * + * @method + */ +ve.ce.MWReferenceNode.prototype.onUpdate = function () { + // TODO: auto-generate this number properly + this.$link.text( '[' + ( this.model.getAttribute( 'listIndex' ) + 1 ) + ']' ); +}; + +/** + * Handle the reference being clicked. + * + * @method + */ +ve.ce.MWReferenceNode.prototype.onClick = function ( e ) { + // TODO: Start editing. Internal item dm node can be accessed using: + // var itemNode = this.model.getInternalItem(); + e.preventDefault(); +}; + +/* Registration */ + +ve.ce.nodeFactory.register( ve.ce.MWReferenceNode ); diff --git a/modules/ve/ce/nodes/ve.ce.MWTemplateNode.js b/modules/ve/ce/nodes/ve.ce.MWTemplateNode.js index 59c560cb9e..48e66d46f5 100644 --- a/modules/ve/ce/nodes/ve.ce.MWTemplateNode.js +++ b/modules/ve/ce/nodes/ve.ce.MWTemplateNode.js @@ -1,5 +1,5 @@ /*! - * VisualEditor ContentEditable MWTemplate class. + * VisualEditor ContentEditable MWTemplateNode class. * * @copyright 2011-2013 VisualEditor Team and others; see AUTHORS.txt * @license The MIT License (MIT); see LICENSE.txt diff --git a/modules/ve/ce/ve.ce.ContentBranchNode.js b/modules/ve/ce/ve.ce.ContentBranchNode.js index 540d962a68..6676c00c88 100644 --- a/modules/ve/ce/ve.ce.ContentBranchNode.js +++ b/modules/ve/ce/ve.ce.ContentBranchNode.js @@ -129,7 +129,7 @@ ve.ce.ContentBranchNode.prototype.renderContents = function () { // Detach all child nodes from this.$ // We can't use this.$.empty() because that destroys .data() and event handlers this.$.contents().each( function () { - $(this).detach(); + $( this ).detach(); } ); // Reattach child nodes with the right annotations diff --git a/modules/ve/dm/nodes/ve.dm.InternalItemNode.js b/modules/ve/dm/nodes/ve.dm.InternalItemNode.js new file mode 100644 index 0000000000..7ebed4b8ec --- /dev/null +++ b/modules/ve/dm/nodes/ve.dm.InternalItemNode.js @@ -0,0 +1,38 @@ +/*! + * VisualEditor DataModel InternalItemNode class. + * + * @copyright 2011-2013 VisualEditor Team and others; see AUTHORS.txt + * @license The MIT License (MIT); see LICENSE.txt + */ + +/** + * DataModel internal item node. + * + * @class + * @extends ve.dm.BranchNode + * @constructor + * @param {ve.dm.BranchNode[]} [children] Child nodes to attach + * @param {Object} [element] Reference to element in linear model + */ +ve.dm.InternalItemNode = function VeDmInternalItemNode( children, element ) { + // Parent constructor + ve.dm.BranchNode.call( this, children, element ); +}; + +/* Inheritance */ + +ve.inheritClass( ve.dm.InternalItemNode, ve.dm.BranchNode ); + +/* Static members */ + +ve.dm.InternalItemNode.static.name = 'internalItem'; + +ve.dm.InternalItemNode.static.matchTagNames = []; + +ve.dm.InternalItemNode.static.handlesOwnChildren = true; + +ve.dm.InternalItemNode.static.isInternal = true; + +/* Registration */ + +ve.dm.modelRegistry.register( ve.dm.InternalItemNode ); \ No newline at end of file diff --git a/modules/ve/dm/nodes/ve.dm.InternalListNode.js b/modules/ve/dm/nodes/ve.dm.InternalListNode.js new file mode 100644 index 0000000000..35ffd29bbc --- /dev/null +++ b/modules/ve/dm/nodes/ve.dm.InternalListNode.js @@ -0,0 +1,38 @@ +/*! + * VisualEditor DataModel InternalListNode class. + * + * @copyright 2011-2013 VisualEditor Team and others; see AUTHORS.txt + * @license The MIT License (MIT); see LICENSE.txt + */ + +/** + * DataModel internal list node. + * + * @class + * @extends ve.dm.BranchNode + * @constructor + * @param {ve.dm.BranchNode[]} [children] Child nodes to attach + * @param {Object} [element] Reference to element in linear model + */ +ve.dm.InternalListNode = function VeDmInternalListNode( children, element ) { + // Parent constructor + ve.dm.BranchNode.call( this, children, element ); +}; + +/* Inheritance */ + +ve.inheritClass( ve.dm.InternalListNode, ve.dm.BranchNode ); + +/* Static members */ + +ve.dm.InternalListNode.static.name = 'internalList'; + +ve.dm.InternalListNode.static.childNodeTypes = [ 'internalItem' ]; + +ve.dm.InternalListNode.static.matchTagNames = []; + +ve.dm.InternalListNode.static.isInternal = true; + +/* Registration */ + +ve.dm.modelRegistry.register( ve.dm.InternalListNode ); \ No newline at end of file diff --git a/modules/ve/dm/nodes/ve.dm.MWHeadingNode.js b/modules/ve/dm/nodes/ve.dm.MWHeadingNode.js index 5fba677704..320e331d6c 100644 --- a/modules/ve/dm/nodes/ve.dm.MWHeadingNode.js +++ b/modules/ve/dm/nodes/ve.dm.MWHeadingNode.js @@ -6,7 +6,7 @@ */ /** - * DataModel MW heading node. + * DataModel MediaWiki heading node. * * @class * @extends ve.dm.HeadingNode diff --git a/modules/ve/dm/nodes/ve.dm.MWPreformattedNode.js b/modules/ve/dm/nodes/ve.dm.MWPreformattedNode.js index 210d18f244..2585b454f1 100644 --- a/modules/ve/dm/nodes/ve.dm.MWPreformattedNode.js +++ b/modules/ve/dm/nodes/ve.dm.MWPreformattedNode.js @@ -6,7 +6,7 @@ */ /** - * DataModel MW preformatted node. + * DataModel MediaWiki preformatted node. * * @class * @extends ve.dm.PreformattedNode diff --git a/modules/ve/dm/nodes/ve.dm.MWReferenceListNode.js b/modules/ve/dm/nodes/ve.dm.MWReferenceListNode.js new file mode 100644 index 0000000000..4a387ead44 --- /dev/null +++ b/modules/ve/dm/nodes/ve.dm.MWReferenceListNode.js @@ -0,0 +1,51 @@ +/*! + * VisualEditor DataModel MWReferenceListNode class. + * + * @copyright 2011-2013 VisualEditor Team and others; see AUTHORS.txt + * @license The MIT License (MIT); see LICENSE.txt + */ + +/** + * DataModel MediaWiki reference list node. + * + * @class + * @extends ve.dm.LeafNode + * @constructor + * @param {number} [length] Length of content data in document; ignored and overridden to 0 + * @param {Object} [element] Reference to element in linear model + */ +ve.dm.MWReferenceListNode = function VeDmMWReferenceListNode( length, element ) { + // Parent constructor + ve.dm.LeafNode.call( this, 0, element ); +}; + +/* Inheritance */ + +ve.inheritClass( ve.dm.MWReferenceListNode, ve.dm.LeafNode ); + +/* Static members */ + +ve.dm.MWReferenceListNode.static.name = 'MWreferenceList'; + +ve.dm.MWReferenceListNode.static.matchTagNames = null; + +ve.dm.MWReferenceListNode.static.matchRdfaTypes = [ 'mw:Object/References' ]; + +ve.dm.MWReferenceListNode.static.toDataElement = function ( domElements ) { + var html = $( '
', domElements[0].ownerDocument ).append( $( domElements ).clone() ).html(); + + return { + 'type': this.name, + 'attributes': { + 'html': html + } + }; +}; + +ve.dm.MWReferenceListNode.static.toDomElements = function ( dataElement, doc ) { + return [ doc.createElement( 'ol' ) ]; +}; + +/* Registration */ + +ve.dm.modelRegistry.register( ve.dm.MWReferenceListNode ); \ No newline at end of file diff --git a/modules/ve/dm/nodes/ve.dm.MWReferenceNode.js b/modules/ve/dm/nodes/ve.dm.MWReferenceNode.js new file mode 100644 index 0000000000..466030163c --- /dev/null +++ b/modules/ve/dm/nodes/ve.dm.MWReferenceNode.js @@ -0,0 +1,96 @@ +/*! + * VisualEditor DataModel MWReferenceNode class. + * + * @copyright 2011-2013 VisualEditor Team and others; see AUTHORS.txt + * @license The MIT License (MIT); see LICENSE.txt + */ + +/** + * DataModel MediaWiki reference node. + * + * @class + * @extends ve.dm.LeafNode + * @constructor + * @param {number} [length] Length of content data in document; ignored and overridden to 0 + * @param {Object} [element] Reference to element in linear model + */ +ve.dm.MWReferenceNode = function VeDmMWReferenceNode( length, element ) { + // Parent constructor + ve.dm.LeafNode.call( this, 0, element ); +}; + +/* Inheritance */ + +ve.inheritClass( ve.dm.MWReferenceNode, ve.dm.LeafNode ); + +/* Static members */ + +ve.dm.MWReferenceNode.static.name = 'MWreference'; + +ve.dm.MWReferenceNode.static.matchTagNames = null; + +ve.dm.MWReferenceNode.static.matchRdfaTypes = [ 'mw:Object/Ext/Ref' ]; + +ve.dm.MWReferenceNode.static.isContent = true; + +ve.dm.MWReferenceNode.static.toDataElement = function ( domElements, converter ) { + var dataElement, listIndex, + about = domElements[0].getAttribute( 'about' ), + // TODO: this is always-present in the new spec, so "|| '{}'" can be removed later + mw = JSON.parse( domElements[0].getAttribute( 'data-mw' ) || '{}' ), + // TODO: this will be stored in mw.body.html in the new spec + body = JSON.parse( domElements[0].getAttribute( 'data-parsoid' ) ).src, + // TODO: this will be stored in mw.name in the new spec + name = $( body ).attr( 'name' ), + key = name !== null ? name : ve.getHash( body ); + + listIndex = converter.internalList.addItem( key, body ); + + dataElement = { + 'type': this.name, + 'attributes': { + 'mw': mw, + 'about': about, + 'listIndex': listIndex + } + }; + return dataElement; +}; + +ve.dm.MWReferenceNode.static.toDomElements = function ( dataElement, doc, converter ) { + var itemNodeHtml, + span = doc.createElement( 'span' ), + itemNodeWrapper = doc.createElement( 'div' ), + itemNode = converter.internalList.getItemNode( dataElement.attributes.listIndex ), + itemNodeRange = itemNode.getRange(); + + span.setAttribute( 'about', dataElement.attributes.about ); + span.setAttribute( 'typeof', 'mw:Object/Ext/Ref' ); + + converter.getDomSubtreeFromData( + converter.documentData.slice( itemNodeRange.start, itemNodeRange.end ), + itemNodeWrapper + ), + itemNodeHtml = $( itemNodeWrapper ).html(); + + // TODO: store internalNodeHtml in data.mw: + // dataElement.attributes.mw.body.html = itemNodeHtml; + // span.setAttribute( 'data-mw', JSON.stringify( dataElement.attributes.mw ) ); + + return [ span ]; +}; + +/* Methods */ + +/** + * Gets the internal item node associated with this node + * @method + * @returns {ve.dm.InternalItemNode} Item node + */ +ve.dm.MWReferenceNode.prototype.getInternalItem = function () { + return this.doc.getInternalList().getItemNode( this.getAttribute( 'listIndex' ) ); +}; + +/* Registration */ + +ve.dm.modelRegistry.register( ve.dm.MWReferenceNode ); \ No newline at end of file diff --git a/modules/ve/dm/ve.dm.Converter.js b/modules/ve/dm/ve.dm.Converter.js index 39bb09731b..4db4fe94bb 100644 --- a/modules/ve/dm/ve.dm.Converter.js +++ b/modules/ve/dm/ve.dm.Converter.js @@ -23,7 +23,9 @@ ve.dm.Converter = function VeDmConverter( modelRegistry, nodeFactory, annotation this.annotationFactory = annotationFactory; this.metaItemFactory = metaItemFactory; this.doc = null; + this.documentData = null; this.store = null; + this.internalList = null; this.contextStack = null; }; @@ -205,6 +207,9 @@ ve.dm.Converter.prototype.getDomElementsFromDataElement = function ( dataElement if ( !nodeClass ) { throw new Error( 'Attempting to convert unknown data element type ' + dataElement.type ); } + if ( nodeClass.static.isInternal ) { + return false; + } domElements = nodeClass.static.toDomElements( dataElements, doc, this ); if ( !domElements || !domElements.length ) { throw new Error( 'toDomElements() failed to return an array when converting element of type ' + dataElement.type ); @@ -285,26 +290,33 @@ ve.dm.Converter.prototype.getDomElementFromDataAnnotation = function ( dataAnnot /** * Convert an HTML document to a linear model. - * @param {ve.dm.IndexValueStore} store Index-value store * @param {HTMLDocument} doc HTML document to convert + * @param {ve.dm.IndexValueStore} store Index-value store + * @param {ve.dm.InternalList} internalList Internal list * @returns {ve.dm.ElementLinearData} Linear model data */ -ve.dm.Converter.prototype.getDataFromDom = function ( store, doc ) { - var result; +ve.dm.Converter.prototype.getDataFromDom = function ( doc, store, internalList ) { + var linearData, refData; // Set up the converter state this.doc = doc; this.store = store; + this.internalList = internalList; this.contextStack = []; // Possibly do things with doc and the head in the future - result = new ve.dm.ElementLinearData( + + linearData = new ve.dm.ElementLinearData( store, this.getDataFromDomRecursion( doc.body ) ); + refData = this.internalList.getDataFromDom( this ); + linearData.batchSplice( linearData.getLength(), 0, refData ); + // Clear the state this.doc = null; this.store = null; + this.internalList = null; this.contextStack = null; - return result; + return linearData; }; /** @@ -764,15 +776,25 @@ ve.dm.Converter.prototype.getDataFromDomRecursion = function ( domElement, wrapp * Convert linear model data to an HTML DOM * * @method + * @param {Array} documentData Linear model data * @param {ve.dm.IndexValueStore} store Index-value store - * @param {Array} data Linear model data + * @param {ve.dm.InternalList} internalList Internal list * @returns {HTMLDocument} Document containing the resulting HTML */ -ve.dm.Converter.prototype.getDomFromData = function ( store, data ) { +ve.dm.Converter.prototype.getDomFromData = function ( documentData, store, internalList ) { var doc = ve.createDocumentFromHTML( '' ); + // Set up the converter state + this.documentData = documentData; this.store = store; - this.getDomSubtreeFromData( data, doc.body ); + this.internalList = internalList; + + this.getDomSubtreeFromData( documentData, doc.body ); + + // Clear the state + this.documentData = null; this.store = null; + this.internalList = null; + return doc; }; @@ -1072,60 +1094,62 @@ ve.dm.Converter.prototype.getDomSubtreeFromData = function ( data, container ) { // Create node from data dataElementOrSlice = getDataElementOrSlice(); childDomElements = this.getDomElementsFromDataElement( dataElementOrSlice, doc ); - // Add clone of internal data; we use a clone rather than a reference because - // we modify .veInternal.whitespace[1] in some cases - childDomElements[0].veInternal = ve.extendObject( - { 'childDomElements': childDomElements }, - ve.copyObject( dataElement.internal || {} ) - ); - // Add elements - for ( j = 0; j < childDomElements.length; j++ ) { - domElement.appendChild( childDomElements[j] ); - } - // Descend into the first child node - parentDomElement = domElement; - domElement = childDomElements[0]; - - // Process outer whitespace - // Every piece of outer whitespace is duplicated somewhere: - // each node's outerPost is duplicated as the next node's - // outerPre, the first node's outerPre is the parent's - // innerPre, and the last node's outerPost is the parent's - // innerPost. For each piece of whitespace, we verify that - // the duplicate matches. If it doesn't, we take that to - // mean the user has messed with it and don't output any - // whitespace. - if ( domElement.veInternal && domElement.veInternal.whitespace ) { - // Process this node's outerPre - ours = domElement.veInternal.whitespace[0]; - theirs = undefined; - if ( domElement.previousSibling ) { - // Get previous sibling's outerPost - theirs = parentDomElement.lastOuterPost; - } else if ( parentDomElement === container ) { - // outerPre of the very first node in the document, this one - // has no duplicate - theirs = ours; - } else { - // First child, get parent's innerPre - if ( - parentDomElement.veInternal && - parentDomElement.veInternal.whitespace - ) { - theirs = parentDomElement.veInternal.whitespace[1]; - // Clear after use so it's not used twice - parentDomElement.veInternal.whitespace[1] = undefined; - } - // else theirs=undefined + if ( childDomElements ) { + // Add clone of internal data; we use a clone rather than a reference because + // we modify .veInternal.whitespace[1] in some cases + childDomElements[0].veInternal = ve.extendObject( + { 'childDomElements': childDomElements }, + ve.copyObject( dataElement.internal || {} ) + ); + // Add elements + for ( j = 0; j < childDomElements.length; j++ ) { + domElement.appendChild( childDomElements[j] ); } - if ( ours && ours === theirs ) { - // Matches the duplicate, insert a TextNode - textNode = doc.createTextNode( ours ); - textNode.veIsWhitespace = true; - parentDomElement.insertBefore( - textNode, - domElement - ); + // Descend into the first child node + parentDomElement = domElement; + domElement = childDomElements[0]; + + // Process outer whitespace + // Every piece of outer whitespace is duplicated somewhere: + // each node's outerPost is duplicated as the next node's + // outerPre, the first node's outerPre is the parent's + // innerPre, and the last node's outerPost is the parent's + // innerPost. For each piece of whitespace, we verify that + // the duplicate matches. If it doesn't, we take that to + // mean the user has messed with it and don't output any + // whitespace. + if ( domElement.veInternal && domElement.veInternal.whitespace ) { + // Process this node's outerPre + ours = domElement.veInternal.whitespace[0]; + theirs = undefined; + if ( domElement.previousSibling ) { + // Get previous sibling's outerPost + theirs = parentDomElement.lastOuterPost; + } else if ( parentDomElement === container ) { + // outerPre of the very first node in the document, this one + // has no duplicate + theirs = ours; + } else { + // First child, get parent's innerPre + if ( + parentDomElement.veInternal && + parentDomElement.veInternal.whitespace + ) { + theirs = parentDomElement.veInternal.whitespace[1]; + // Clear after use so it's not used twice + parentDomElement.veInternal.whitespace[1] = undefined; + } + // else theirs=undefined + } + if ( ours && ours === theirs ) { + // Matches the duplicate, insert a TextNode + textNode = doc.createTextNode( ours ); + textNode.veIsWhitespace = true; + parentDomElement.insertBefore( + textNode, + domElement + ); + } } } diff --git a/modules/ve/dm/ve.dm.Document.js b/modules/ve/dm/ve.dm.Document.js index 218512a86f..ef06419330 100644 --- a/modules/ve/dm/ve.dm.Document.js +++ b/modules/ve/dm/ve.dm.Document.js @@ -41,6 +41,7 @@ ve.dm.Document = function VeDmDocument( documentOrData, parentDocument ) { currentNode = this.documentNode; this.documentNode.setDocument( doc ); this.documentNode.setRoot( root ); + this.internalList = new ve.dm.InternalList( this ); // Properties this.parentDocument = parentDocument; @@ -48,7 +49,7 @@ ve.dm.Document = function VeDmDocument( documentOrData, parentDocument ) { if ( documentOrData instanceof ve.dm.LinearData ) { this.data = documentOrData; } else if ( !ve.isArray( documentOrData ) && typeof documentOrData === 'object' ) { - this.data = ve.dm.converter.getDataFromDom( new ve.dm.IndexValueStore(), documentOrData ); + this.data = ve.dm.converter.getDataFromDom( documentOrData, new ve.dm.IndexValueStore(), this.getInternalList() ); } else { this.data = new ve.dm.ElementLinearData( new ve.dm.IndexValueStore(), @@ -274,6 +275,14 @@ ve.dm.Document.prototype.getStore = function () { return this.store; }; +/** + * Get the document's internal list + * @returns {ve.dm.InternalList} The document's internal list + */ +ve.dm.Document.prototype.getInternalList = function () { + return this.internalList; +}; + /** * Get the metadata replace operation required to keep data & metadata in sync after a splice * diff --git a/modules/ve/dm/ve.dm.InternalList.js b/modules/ve/dm/ve.dm.InternalList.js new file mode 100644 index 0000000000..6060bfd58b --- /dev/null +++ b/modules/ve/dm/ve.dm.InternalList.js @@ -0,0 +1,130 @@ +/*! + * VisualEditor DataModel InternalList class. + * + * @copyright 2011-2013 VisualEditor Team and others; see AUTHORS.txt + * @license The MIT License (MIT); see LICENSE.txt + */ + +/** + * DataModel meta item. + * + * @class + * @extends ve.EventEmitter + * @constructor + * @param {ve.dm.Document} doc Document model + */ +ve.dm.InternalList = function VeDmInternalList( doc ) { + // Parent constructor + ve.EventEmitter.call( this ); + + // Properties + this.document = doc; + this.store = new ve.dm.IndexValueStore(); + this.itemsHtml = []; + this.listNode = null; + + // Event handlers + //this.document.on( 'transact', ve.bind( this.onTransact, this ) ); +}; + +/* Inheritance */ + +ve.inheritClass( ve.dm.InternalList, ve.EventEmitter ); + +/* Methods */ + +/** + * Add an item to the list. + * + * If an item with this key already exists it will be ignored. + * + * @method + * @param {string} key Item key + * @param {string} body Item contents + * @returns {number} Index of the item in the index-value store, and also the list + */ +ve.dm.InternalList.prototype.addItem = function ( key, body ) { + var index = this.store.indexOfHash( key ); + if ( index === null ) { + index = this.store.index( body, key ); + this.itemsHtml.push( index ); + } + return index; +}; + +/** + * Gets all the item's HTML strings + * @method + * @returns {Object} Name-indexed object containing HTMLElements + */ +ve.dm.InternalList.prototype.getItemsHtml = function () { + return this.store.values( this.itemsHtml ); +}; + +/** + * Gets the internal list's document model + * @method + * @returns {ve.dm.Document} Document model + */ +ve.dm.InternalList.prototype.getDocument = function () { + return this.document; +}; + +/** + * Get the list node + * @method + * @returns {ve.dm.InternalListNode} List node + */ +ve.dm.InternalList.prototype.getListNode = function () { + var i, nodes; + // find listNode if not set, or unattached + if ( !this.listNode || !this.listNode.doc ) { + nodes = this.getDocument().documentNode.children; + for ( i = nodes.length; i >= 0; i-- ) { + if ( nodes[i] instanceof ve.dm.InternalListNode ) { + this.listNode = nodes[i]; + break; + } + } + } + return this.listNode; +}; + +/** + * Get the item node from a specific index + * @method + * @param {number} index Item's index + * @returns {ve.dm.InternalItemNode} Item node + */ +ve.dm.InternalList.prototype.getItemNode = function ( index ) { + return this.getListNode().children[index]; +}; + +/** + * Gets linear model data for all the stored item's HTML. + * + * Each item is an InternalItem, and they are wrapped in an InternalList. + * If there are no items an empty array is returned. + * + * @method + * @param {ve.dm.Converter} converter Converter object + * @returns {Array} Linear model data + */ +ve.dm.InternalList.prototype.getDataFromDom = function ( converter ) { + var i, length, itemData, + itemsHtml = this.getItemsHtml(), list = []; + + if ( itemsHtml.length ) { + list.push( { 'type': 'internalList' } ); + for ( i = 0, length = itemsHtml.length; i < length; i++ ) { + itemData = converter.getDataFromDomRecursion( $( itemsHtml[i] )[0] ); + list = list.concat( + [{ 'type': 'internalItem' }], + itemData, + [{ 'type': '/internalItem' }] + ); + } + list.push( { 'type': '/internalList' } ); + } + return list; +}; \ No newline at end of file diff --git a/modules/ve/dm/ve.dm.Model.js b/modules/ve/dm/ve.dm.Model.js index 550fc83f49..77043fad5b 100644 --- a/modules/ve/dm/ve.dm.Model.js +++ b/modules/ve/dm/ve.dm.Model.js @@ -130,7 +130,7 @@ ve.dm.Model.static.toDataElement = function ( /*domElements, converter*/ ) { * If this model is a node with .handlesOwnChildren set to true, dataElement will be an array of * the linear model data of this node and all of its children, rather than a single element. * In this case, this function way want to recursively convert linear model data to DOM, which can - * be done with converter.getDomSubtreeFromData( store, data, containerElement ); + * be done with converter#getDomSubtreeFromData. * * NOTE: If this function returns multiple DOM elements, the DOM elements produced by the children * of this model (if it's a node and has children) will be attached to the first DOM element in the array. diff --git a/modules/ve/dm/ve.dm.Node.js b/modules/ve/dm/ve.dm.Node.js index 3ca243b0d2..35ce57a5a3 100644 --- a/modules/ve/dm/ve.dm.Node.js +++ b/modules/ve/dm/ve.dm.Node.js @@ -65,6 +65,15 @@ ve.mixinClass( ve.dm.Node, ve.Node ); */ ve.dm.Node.static.handlesOwnChildren = false; +/** + * Whether this node type is internal. Internal node types are ignored by the converter. + * + * @static + * @property {boolean} static.isInternal + * @inheritable + */ +ve.dm.Node.static.isInternal = false; + /** * Whether this node type has a wrapping element in the linear model. Most node types are wrapped, * only special node types are not wrapped. diff --git a/modules/ve/dm/ve.dm.NodeFactory.js b/modules/ve/dm/ve.dm.NodeFactory.js index 89e3373e4b..bcbbfc5711 100644 --- a/modules/ve/dm/ve.dm.NodeFactory.js +++ b/modules/ve/dm/ve.dm.NodeFactory.js @@ -216,6 +216,21 @@ ve.dm.NodeFactory.prototype.doesNodeHandleOwnChildren = function ( type ) { throw new Error( 'Unknown node type: ' + type ); }; +/** + * Check if the node is internal. + * + * @method + * @param {string} type Node type + * @returns {boolean} Whether the node is internal + * @throws {Error} Unknown node type + */ +ve.dm.NodeFactory.prototype.isNodeInternal = function ( type ) { + if ( type in this.registry ) { + return this.registry[type].static.isInternal; + } + throw new Error( 'Unknown node type: ' + type ); +}; + /* Initialization */ ve.dm.nodeFactory = new ve.dm.NodeFactory(); diff --git a/modules/ve/dm/ve.dm.Transaction.js b/modules/ve/dm/ve.dm.Transaction.js index fad9b6e729..5adcd0d195 100644 --- a/modules/ve/dm/ve.dm.Transaction.js +++ b/modules/ve/dm/ve.dm.Transaction.js @@ -102,7 +102,7 @@ ve.dm.Transaction.newFromRemoval = function ( doc, range ) { removeEnd = ( last.range || last.nodeRange ).end; } tx.pushRetain( removeStart ); - tx.pushReplace( doc, removeStart, removeEnd - removeStart, [] ); + tx.addSafeRemoveOps( doc, removeStart, removeEnd ); tx.pushRetain( data.length - removeEnd ); // All done return tx; @@ -135,7 +135,7 @@ ve.dm.Transaction.newFromRemoval = function ( doc, range ) { // Push the previous removal first tx.pushRetain( removeStart - offset ); - tx.pushReplace( doc, removeStart, removeEnd - removeStart, [] ); + tx.addSafeRemoveOps( doc, removeStart, removeEnd ); offset = removeEnd; // Now start this removal @@ -146,7 +146,7 @@ ve.dm.Transaction.newFromRemoval = function ( doc, range ) { // Apply the last removal, if any if ( removeEnd !== null ) { tx.pushRetain( removeStart - offset ); - tx.pushReplace( doc, removeStart, removeEnd - removeStart, [] ); + tx.addSafeRemoveOps( doc, removeStart, removeEnd ); offset = removeEnd; } // Retain up to the end of the document @@ -802,6 +802,38 @@ ve.dm.Transaction.prototype.pushRetainMetadata = function ( length ) { } }; +/** + * Adds a replace op to remove the desired range and, where required, splices in retain ops + * to prevent the deletion of internal data. + * + * @param {ve.dm.Document} doc Document + * @param {number} removeStart Offset to start removing from + * @param {number} removeEnd Offset to remove to + */ +ve.dm.Transaction.prototype.addSafeRemoveOps = function ( doc, removeStart, removeEnd ) { + var i, retainStart, internalStackDepth = 0; + // Iterate over removal range and use a stack counter to determine if + // we are inside an internal node + for ( i = removeStart; i <= removeEnd; i++ ) { + if ( doc.data.isElementData( i ) && ve.dm.nodeFactory.isNodeInternal( doc.data.getType( i ) ) ) { + if ( !doc.data.isCloseElementData( i ) ) { + if ( internalStackDepth === 0 ) { + this.pushReplace( doc, removeStart, i - removeStart, [] ); + retainStart = i; + } + internalStackDepth++; + } else { + internalStackDepth--; + if ( internalStackDepth === 0 ) { + this.pushRetain( i + 1 - retainStart ); + removeStart = i + 1; + } + } + } + } + this.pushReplace( doc, removeStart, removeEnd - removeStart, [] ); +}; + /** * Add a replace operation, keeping metadata in sync if required * diff --git a/modules/ve/init/mw/targets/ve.init.mw.ViewPageTarget.js b/modules/ve/init/mw/targets/ve.init.mw.ViewPageTarget.js index 3c27ae67e4..d9067580de 100644 --- a/modules/ve/init/mw/targets/ve.init.mw.ViewPageTarget.js +++ b/modules/ve/init/mw/targets/ve.init.mw.ViewPageTarget.js @@ -433,10 +433,11 @@ ve.init.mw.ViewPageTarget.prototype.onShowChangesError = function ( jqXHR, statu * @method */ ve.init.mw.ViewPageTarget.prototype.onEditConflict = function () { + var doc = this.surface.getDocumentModel(); if ( confirm( ve.msg( 'visualeditor-editconflict', status ) ) ) { // Get Wikitext from the DOM, and setup a submit call when it's done this.serialize( - ve.dm.converter.getDomFromData( this.surface.getDocumentModel().getStore(), this.surface.getDocumentModel().getFullData() ), + ve.dm.converter.getDomFromData( doc.getFullData(), doc.getStore(), doc.getInternalList() ), ve.bind( function ( wikitext ) { this.submit( wikitext, this.getSaveOptions() ); }, this ) @@ -569,10 +570,11 @@ ve.init.mw.ViewPageTarget.prototype.onSurfaceModelHistory = function () { * @method */ ve.init.mw.ViewPageTarget.prototype.onSaveDialogSaveButtonClick = function () { + var doc = this.surface.getDocumentModel(); this.saveDialogSaveButton.setDisabled( true ); this.$saveDialogLoadingIcon.show(); this.save( - ve.dm.converter.getDomFromData( this.surface.getDocumentModel().getStore(), this.surface.getDocumentModel().getFullData() ), + ve.dm.converter.getDomFromData( doc.getFullData(), doc.getStore(), doc.getInternalList() ), this.getSaveOptions() ); }; @@ -1206,7 +1208,7 @@ ve.init.mw.ViewPageTarget.prototype.resetSaveDialog = function () { * @throws {Error} Unknown saveDialog slide */ ve.init.mw.ViewPageTarget.prototype.swapSaveDialog = function ( slide ) { - var $slide, $viewer; + var $slide, $viewer, doc = this.surface.getDocumentModel(); if ( ve.indexOf( slide, [ 'review', 'report', 'save'] ) === -1 ) { throw new Error( 'Unknown saveDialog slide: ' + slide ); } @@ -1239,11 +1241,11 @@ ve.init.mw.ViewPageTarget.prototype.swapSaveDialog = function ( slide ) { if ( this.pageExists ) { // Has no callback, handled via viewPage.onShowChanges this.showChanges( - ve.dm.converter.getDomFromData( this.surface.getDocumentModel().getStore(), this.surface.getDocumentModel().getFullData() ) + ve.dm.converter.getDomFromData( doc.getFullData(), doc.getStore(), doc.getInternalList() ) ); } else { this.serialize( - ve.dm.converter.getDomFromData( this.surface.getDocumentModel().getStore(), this.surface.getDocumentModel().getFullData() ), + ve.dm.converter.getDomFromData( doc.getFullData(), doc.getStore(), doc.getInternalList() ), function ( wikitext ) { $viewer.empty().append( $( '
' ).text( wikitext ) );
 
diff --git a/modules/ve/init/mw/ve.init.mw.Target.js b/modules/ve/init/mw/ve.init.mw.Target.js
index a51582f480..5193aac4c7 100644
--- a/modules/ve/init/mw/ve.init.mw.Target.js
+++ b/modules/ve/init/mw/ve.init.mw.Target.js
@@ -582,8 +582,10 @@ ve.init.mw.Target.prototype.serialize = function ( doc, callback ) {
 ve.init.mw.Target.prototype.reportProblem = function ( message ) {
 	// Gather reporting information
 	var now = new Date(),
-		editedData = this.surface.getDocumentModel().getFullData(),
-		store = this.surface.getDocumentModel().getStore(),
+		doc = this.surface.getDocumentModel(),
+		editedData = doc.getFullData(),
+		store = doc.getStore(),
+		internalList = doc.getInternalList(),
 		report = {
 			'title': this.pageName,
 			'oldid': this.oldid,
@@ -594,11 +596,12 @@ ve.init.mw.Target.prototype.reportProblem = function ( message ) {
 			'originalData':
 				// originalHTML only has the body's HTML for now, see TODO comment in ve.init.mw.ViewPageTarget.prototype.setUpSurface
 				// FIXME: need to expand this data before sending it, see bug 47319
-				ve.dm.converter.getDataFromDom( store,
-					ve.createDocumentFromHTML( '' + this.originalHtml  + '' )
-			),
+				ve.dm.converter.getDataFromDom(
+					ve.createDocumentFromHTML( '' + this.originalHtml  + '' ),
+					store, internalList
+				),
 			'editedData': editedData,
-			'editedHtml': ve.properInnerHTML( ve.dm.converter.getDomFromData( store, editedData ).body ),
+			'editedHtml': ve.properInnerHTML( ve.dm.converter.getDomFromData( editedData, store, internalList ).body ),
 			'wiki': mw.config.get( 'wgDBname' )
 		};
 	$.post(
diff --git a/modules/ve/test/dm/ve.dm.Converter.test.js b/modules/ve/test/dm/ve.dm.Converter.test.js
index bec9131a63..ca0c100eeb 100644
--- a/modules/ve/test/dm/ve.dm.Converter.test.js
+++ b/modules/ve/test/dm/ve.dm.Converter.test.js
@@ -40,7 +40,7 @@ QUnit.test( 'getDomElementsFromDataElement', 20, function ( assert ) {
 } );
 
 QUnit.test( 'getDataFromDom', function ( assert ) {
-	var msg, store, i, length, hash, n = 0,
+	var msg, store, internalList, i, length, hash, n = 0,
 		cases = ve.copyObject( ve.dm.example.domToDataCases );
 
 	// TODO: this is a hack to make normal heading/preformatted
@@ -61,9 +61,10 @@ QUnit.test( 'getDataFromDom', function ( assert ) {
 	for ( msg in cases ) {
 		if ( cases[msg].html !== null ) {
 			store = new ve.dm.IndexValueStore();
+			internalList = new ve.dm.InternalList();
 			ve.dm.example.preprocessAnnotations( cases[msg].data, store );
 			assert.deepEqual(
-				ve.dm.converter.getDataFromDom( store, ve.createDocumentFromHTML( cases[msg].html ) ).getData(),
+				ve.dm.converter.getDataFromDom( ve.createDocumentFromHTML( cases[msg].html ), store, internalList ).getData(),
 				cases[msg].data,
 				msg
 			);
@@ -83,7 +84,7 @@ QUnit.test( 'getDataFromDom', function ( assert ) {
 } );
 
 QUnit.test( 'getDomFromData', function ( assert ) {
-	var msg, originalData, store, i, length, n = 0,
+	var msg, originalData, doc, store, i, length, n = 0,
 		cases = ve.copyObject( ve.dm.example.domToDataCases );
 
 	for ( msg in cases ) {
@@ -103,13 +104,13 @@ QUnit.test( 'getDomFromData', function ( assert ) {
 		if( ve.dm.example.domToDataCases[msg].modify ) {
 			ve.dm.example.domToDataCases[msg].modify( cases[msg].data );
 		}
-		ve.dm.example.preprocessAnnotations( cases[msg].data, store );
-		originalData = ve.copyArray( cases[msg].data );
+		doc = new ve.dm.Document( ve.dm.example.preprocessAnnotations( cases[msg].data, store ) );
+		originalData = ve.copyArray( doc.getFullData() );
 		assert.equalDomElement(
-			ve.dm.converter.getDomFromData( store, cases[msg].data ),
+			ve.dm.converter.getDomFromData( doc.getFullData(), doc.getStore(), doc.getInternalList() ),
 			ve.createDocumentFromHTML( cases[msg].normalizedHtml || cases[msg].html ),
 			msg
 		);
-		assert.deepEqual( cases[msg].data, originalData, msg + ' (data hasn\'t changed)' );
+		assert.deepEqual( doc.getFullData(), originalData, msg + ' (data hasn\'t changed)' );
 	}
 } );
diff --git a/modules/ve/test/dm/ve.dm.Transaction.test.js b/modules/ve/test/dm/ve.dm.Transaction.test.js
index eae2620005..0e16dc48ad 100644
--- a/modules/ve/test/dm/ve.dm.Transaction.test.js
+++ b/modules/ve/test/dm/ve.dm.Transaction.test.js
@@ -221,11 +221,11 @@ QUnit.test( 'newFromInsertion', function ( assert ) {
 } );
 
 QUnit.test( 'newFromRemoval', function ( assert ) {
-	var i, key,
-		store = new ve.dm.IndexValueStore(),
-		doc = ve.dm.example.createExampleDocument( 'data', store ),
-		alienDoc = ve.dm.example.createExampleDocument( 'alienData', store ),
-		metaDoc = ve.dm.example.createExampleDocument( 'withMeta', store ),
+	var i, key, store,
+		doc = ve.dm.example.createExampleDocument( 'data' ),
+		alienDoc = ve.dm.example.createExampleDocument( 'alienData' ),
+		metaDoc = ve.dm.example.createExampleDocument( 'withMeta' ),
+		internalDoc = ve.dm.example.createExampleDocument( 'internalData' ),
 		cases = {
 			'content in first element': {
 				'args': [doc, new ve.Range( 1, 3 )],
@@ -471,11 +471,70 @@ QUnit.test( 'newFromRemoval', function ( assert ) {
 					},
 					{ 'type': 'retain', 'length': 2 }
 				]
+			},
+			'selection including internal nodes doesn\'t remove them': {
+				'args': [internalDoc, new ve.Range( 2, 24 )],
+				'ops': [
+					{ 'type': 'retain', 'length': 2 },
+					{
+						'type': 'replace',
+						'remove': [
+							'o', 'o',
+							{ 'type': '/paragraph' }
+						],
+						'insert': []
+					},
+					{ 'type': 'retain', 'length': 16 },
+					{
+						'type': 'replace',
+						'remove': [
+							{ 'type': 'paragraph' },
+							'Q', 'u'
+						],
+						'insert': []
+					},
+					{ 'type': 'retain', 'length': 3 }
+				]
+			},
+			'selection ending with internal nodes': {
+				'args': [internalDoc, new ve.Range( 2, 21 )],
+				'ops': [
+					{ 'type': 'retain', 'length': 2 },
+					{
+						'type': 'replace',
+						'remove': [
+							'o', 'o'
+						],
+						'insert': []
+					},
+					{ 'type': 'retain', 'length': 23 },
+				]
+			},
+			'selection starting with internal nodes': {
+				'args': [internalDoc, new ve.Range( 5, 24 )],
+				'ops': [
+					{ 'type': 'retain', 'length': 22 },
+					{
+						'type': 'replace',
+						'remove': [
+							'Q', 'u'
+						],
+						'insert': []
+					},
+					{ 'type': 'retain', 'length': 3 },
+				]
+			},
+			'selection of just internal nodes returns a no-op transaction': {
+				'args': [internalDoc, new ve.Range( 5, 21 )],
+				'ops': [
+					{ 'type': 'retain', 'length': 27 },
+				]
 			}
 		};
 	QUnit.expect( ve.getObjectKeys( cases ).length );
 	for ( key in cases ) {
 		for ( i = 0; i < cases[key].ops.length; i++ ) {
+			store = cases[key].args[0].getStore();
 			if ( cases[key].ops[i].remove ) {
 				ve.dm.example.preprocessAnnotations( cases[key].ops[i].remove, store );
 			}
@@ -632,8 +691,8 @@ QUnit.test( 'newFromAnnotation', function ( assert ) {
 } );
 
 QUnit.test( 'newFromContentBranchConversion', function ( assert ) {
-	var doc = ve.dm.example.createExampleDocument(),
-		i, key,
+	var i, key, store,
+		doc = ve.dm.example.createExampleDocument(),
 		cases = {
 			'range inside a heading, convert to paragraph': {
 				'args': [doc, new ve.Range( 1, 2 ), 'paragraph'],
@@ -686,11 +745,12 @@ QUnit.test( 'newFromContentBranchConversion', function ( assert ) {
 	QUnit.expect( ve.getObjectKeys( cases ).length );
 	for ( key in cases ) {
 		for ( i = 0; i < cases[key].ops.length; i++ ) {
+			store = cases[key].args[0].getStore();
 			if ( cases[key].ops[i].remove ) {
-				ve.dm.example.preprocessAnnotations( cases[key].ops[i].remove, doc.getStore() );
+				ve.dm.example.preprocessAnnotations( cases[key].ops[i].remove, store );
 			}
 			if ( cases[key].ops[i].insert ) {
-				ve.dm.example.preprocessAnnotations( cases[key].ops[i].insert, doc.getStore() );
+				ve.dm.example.preprocessAnnotations( cases[key].ops[i].insert, store );
 			}
 		}
 	}
diff --git a/modules/ve/test/dm/ve.dm.example.js b/modules/ve/test/dm/ve.dm.example.js
index 92fa9b8275..3584967537 100644
--- a/modules/ve/test/dm/ve.dm.example.js
+++ b/modules/ve/test/dm/ve.dm.example.js
@@ -328,6 +328,27 @@ ve.dm.example.alienData = [
 	// 10 - End of document
 ];
 
+ve.dm.example.internalData = [
+	{ 'type': 'paragraph' },
+	'F', 'o', 'o',
+	{ 'type': '/paragraph' },
+	{ 'type': 'internalList' },
+	{ 'type': 'internalItem' },
+	{ 'type': 'paragraph', 'internal': { 'generated': 'wrapper' } },
+	'B', 'a', 'r',
+	{ 'type': '/paragraph' },
+	{ 'type': '/internalItem' },
+	{ 'type': 'internalItem' },
+	{ 'type': 'paragraph', 'internal': { 'generated': 'wrapper' } },
+	'B', 'a', 'z',
+	{ 'type': '/paragraph' },
+	{ 'type': '/internalItem' },
+	{ 'type': '/internalList' },
+	{ 'type': 'paragraph' },
+	'Q', 'u', 'u', 'x',
+	{ 'type': '/paragraph' }
+];
+
 ve.dm.example.withMeta = [
 	{
 		'type': 'alienMeta',
@@ -898,6 +919,144 @@ ve.dm.example.domToDataCases = {
 		},
 		'normalizedHtml': ve.dm.example.MWTemplate.inlineOpenModified + ve.dm.example.MWTemplate.inlineClose
 	},
+	'mw:Reference': {
+		'html':
+			'' +
+				'
Foo' +
+					''+
+						'[1]' +
+					'' +
+					' Baz' +
+					'' +
+						'[2]' +
+					'' +
+					' Whee' +
+					'' +
+						'[1]' +
+					'' +
+					' Yay' +
+					'' +
+						'[3]' +
+					'' +
+				'
' +
+				'
    ' +
+					'
  1. u2191Quux
    2. ' +
+				'
' +
+			'',
+		'data': [
+			{ 'type': 'paragraph' },
+			'F', 'o', 'o',
+			{
+				'type': 'MWreference',
+				'attributes': {
+					'about': '#mwt5',
+					'listIndex': 0,
+					'mw': {},
+					'html/0/about': '#mwt5',
+					'html/0/class': 'reference',
+					'html/0/data-parsoid': '{"src":"Bar"}',
+					'html/0/id': 'cite_ref-bar-1-0',
+					'html/0/typeof': 'mw:Object/Ext/Ref'
+				}
+			},
+			{ 'type': '/MWreference' },
+			' ', 'B', 'a', 'z',
+			{
+				'type': 'MWreference',
+				'attributes': {
+					'about': '#mwt6',
+					'listIndex': 1,
+					'mw': {},
+					'html/0/about': '#mwt6',
+					'html/0/class': 'reference',
+					'html/0/data-parsoid': '{"src":"Quux"}',
+					'html/0/id': 'cite_ref-quux-2-0',
+					'html/0/typeof': 'mw:Object/Ext/Ref'
+				}
+			},
+			{ 'type': '/MWreference' },
+			' ', 'W', 'h', 'e', 'e',
+			{
+				'type': 'MWreference',
+				'attributes': {
+					'about': '#mwt7',
+					'listIndex': 0,
+					'mw': {},
+					'html/0/about': '#mwt7',
+					'html/0/class': 'reference',
+					'html/0/data-parsoid': '{"src":""}',
+					'html/0/id': 'cite_ref-bar-1-1',
+					'html/0/typeof': 'mw:Object/Ext/Ref'
+				}
+			},
+			{ 'type': '/MWreference' },
+			' ', 'Y', 'a', 'y',
+			{
+				'type': 'MWreference',
+				'attributes': {
+					'about': '#mwt8',
+					'listIndex': 2,
+					'mw': {},
+					'html/0/about': '#mwt8',
+					'html/0/class': 'reference',
+					'html/0/data-parsoid': '{"src":"No name"}',
+					'html/0/id': 'cite_ref-3-0',
+					'html/0/typeof': 'mw:Object/Ext/Ref'
+				}
+			},
+			{ 'type': '/MWreference' },
+			{ 'type': '/paragraph' },
+			{
+				'type': 'MWreferenceList',
+				'attributes': {
+					'html': '
  1. u2191Quux
',
+					'html/0/class': 'references',
+					'html/0/typeof': 'mw:Object/References'
+				}
+			},
+			{ 'type': '/MWreferenceList' },
+			{ 'type': 'internalList' },
+			{ 'type': 'internalItem' },
+			{ 'type': 'paragraph', 'internal': { 'generated': 'wrapper' } },
+			'B', 'a', 'r',
+			{ 'type': '/paragraph' },
+			{ 'type': '/internalItem' },
+			{ 'type': 'internalItem' },
+			{ 'type': 'paragraph', 'internal': { 'generated': 'wrapper' } },
+			'Q', 'u', 'u', 'x',
+			{ 'type': '/paragraph' },
+			{ 'type': '/internalItem' },
+			{ 'type': 'internalItem' },
+			{ 'type': 'paragraph', 'internal': { 'generated': 'wrapper' } },
+			'N', 'o', ' ', 'n', 'a', 'm', 'e',
+			{ 'type': '/paragraph' },
+			{ 'type': '/internalItem' },
+			{ 'type': '/internalList' }
+		],
+		'normalizedHtml':
+			'
Foo' +
+				''+
+				'' +
+				' Baz' +
+				'' +
+				'' +
+				' Whee' +
+				'' +
+				'' +
+				' Yay' +
+				'' +
+				'' +
+			'
' +
+			'
    '
+	},
 	'paragraph with alienInline inside': {
 		'html': '
    abc
    ',
 		'data': [
diff --git a/modules/ve/test/index.php b/modules/ve/test/index.php
index 068dabdd2b..273b863e05 100644
--- a/modules/ve/test/index.php
+++ b/modules/ve/test/index.php
@@ -67,6 +67,7 @@
 		
 		
 		
+		
 		
 		
 		
@@ -91,6 +92,8 @@
 		
 		
 		
+		
+		
 		
 		
 		
@@ -104,6 +107,8 @@
 		
 		
 		
+		
+		
 		
 		
 		
@@ -138,6 +143,8 @@
 		
 		
 		
+		
+		
 		
 		
 		
@@ -151,6 +158,8 @@
 		
 		
 		
+		
+