Merge pull request #7602 from ckeditor/i/7556

Feature (engine): Introduced new marker conversion helpers that produce semantic HTML data output. See `DowncastHelpers#markerToData()` and `UpcastHelpers#dataToMarker()`. Closes #7556.  
Other (engine): Added `model.Schema` instance to downcast conversion API, available under `conversionApi.schema`.  
Other (engine): `UpcastHelpers#elementToMarker()` is now deprecated. Use `UpcastHelpers#dataToMarker()` instead. `DowncastHelpers#markerToElement()` should be used only for editing downcast.

BREAKING CHANGE (engine): Marker names with "," character are now disallowed.

docs/framework/guides/architecture/editing-engine.md

@@ -159,9 +159,11 @@ Markers are a special type of ranges.
 * They can only be created and changed through the {@link module:engine/model/writer~Writer model writer}.
 * They can be synchronized over the network with other collaborating clients.
 * They are automatically updated when the document's structure is changed.
-* They can be converted to attributes or elements in the [view](#view).
+* They can be converted to the editing view, to show them in the editor (as {@link module:engine/conversion/downcasthelpers~DowncastHelpers#markerToHighlight highlights} or {@link module:engine/conversion/downcasthelpers~DowncastHelpers#markerToElement elements}).
+* They can be {@link module:engine/conversion/downcasthelpers~DowncastHelpers#markerToData converted to the data view}, to store them with the document data.
+* They can be {@link module:engine/conversion/upcasthelpers~UpcastHelpers#dataToMarker loaded with the document data}.
 
-This makes them ideal for storing and maintaining additional data in the model — such as comments, selections of other users, etc.
+Markers are ideal for storing and maintaining additional data related to portions of the document — such as comments or selections of other users.
 
 ### Schema
 

packages/ckeditor5-engine/src/controller/datacontroller.js

@@ -90,7 +90,8 @@ export default class DataController {
 		 * @member {module:engine/conversion/downcastdispatcher~DowncastDispatcher}
 		 */
 		this.downcastDispatcher = new DowncastDispatcher( {
-			mapper: this.mapper
+			mapper: this.mapper,
+			schema: model.schema
 		} );
 		this.downcastDispatcher.on( 'insert:$text', insertText(), { priority: 'lowest' } );
 

packages/ckeditor5-engine/src/controller/editingcontroller.js

@@ -65,7 +65,8 @@ export default class EditingController {
 		 * @member {module:engine/conversion/downcastdispatcher~DowncastDispatcher} #downcastDispatcher
 		 */
 		this.downcastDispatcher = new DowncastDispatcher( {
-			mapper: this.mapper
+			mapper: this.mapper,
+			schema: model.schema
 		} );
 
 		const doc = this.model.document;

packages/ckeditor5-engine/src/conversion/downcastdispatcher.js

@@ -658,6 +658,12 @@ function shouldMarkerChangeBeConverted( modelPosition, marker, mapper ) {
  * @member {module:engine/conversion/mapper~Mapper} #mapper
  */
 
+/**
+ * The {@link module:engine/model/schema~Schema} instance set for the model that is downcast.
+ *
+ * @member {module:engine/model/schema~Schema} #schema
+ */
+
 /**
  * The {@link module:engine/view/downcastwriter~DowncastWriter} instance used to manipulate data during conversion.
  *

packages/ckeditor5-engine/src/conversion/downcasthelpers.js

@@ -239,22 +239,25 @@ export default class DowncastHelpers extends ConversionHelpers {
 	/**
 	 * Model marker to view element conversion helper.
 	 *
+	 * **Note**: This method should be used only for editing downcast. For data downcast, use
+	 * {@link #markerToData `#markerToData()`} that produces valid HTML data.
+	 *
 	 * This conversion results in creating a view element on the boundaries of the converted marker. If the converted marker
 	 * is collapsed, only one element is created. For example, model marker set like this: `<paragraph>F[oo b]ar</paragraph>`
 	 * becomes `<p>F<span data-marker="search"></span>oo b<span data-marker="search"></span>ar</p>` in the view.
 	 *
-	 *		editor.conversion.for( 'downcast' ).markerToElement( {
+	 *		editor.conversion.for( 'editingDowncast' ).markerToElement( {
 	 *			model: 'search',
 	 *			view: 'marker-search'
 	 *		} );
 	 *
-	 *		editor.conversion.for( 'downcast' ).markerToElement( {
+	 *		editor.conversion.for( 'editingDowncast' ).markerToElement( {
 	 *			model: 'search',
 	 *			view: 'search-result',
 	 *			converterPriority: 'high'
 	 *		} );
 	 *
-	 *		editor.conversion.for( 'downcast' ).markerToElement( {
+	 *		editor.conversion.for( 'editingDowncast' ).markerToElement( {
 	 *			model: 'search',
 	 *			view: {
 	 *				name: 'span',
@@ -264,7 +267,7 @@ export default class DowncastHelpers extends ConversionHelpers {
 	 *			}
 	 *		} );
 	 *
-	 *		editor.conversion.for( 'downcast' ).markerToElement( {
+	 *		editor.conversion.for( 'editingDowncast' ).markerToElement( {
 	 *			model: 'search',
 	 *			view: ( markerData, viewWriter ) => {
 	 *				return viewWriter.createUIElement( 'span', {
@@ -282,8 +285,6 @@ export default class DowncastHelpers extends ConversionHelpers {
 	 * the `data.isOpening` parameter is passed, which is set to `true` for the marker start boundary element, and `false` to
 	 * the marker end boundary element.
 	 *
-	 * This kind of conversion is useful for saving data into the database, so it should be used in the data conversion pipeline.
-	 *
 	 * See {@link module:engine/conversion/conversion~Conversion#for `conversion.for()`} to learn how to add a converter
 	 * to the conversion process.
 	 *
@@ -357,6 +358,119 @@ export default class DowncastHelpers extends ConversionHelpers {
 	markerToHighlight( config ) {
 		return this.add( downcastMarkerToHighlight( config ) );
 	}
+
+	/**
+	 * Model marker converter for data downcast.
+	 *
+	 * This conversion creates a representation for model marker boundaries in the view:
+	 *
+	 * * if the marker boundary is at a position where text nodes are allowed, then a view element with specified tag name
+	 * and `name` attribute is added at that position,
+	 * * in other cases, a specified attribute is set on a view element that is before/after marker boundary.
+	 *
+	 * Typically, the marker names use `group:uniqueId:otherData` convention. For example: `comment:e34zfk9k2n459df53sjl34:zx32c`.
+	 * The default configuration for this conversion is that the first part is `group` part and the rest of
+	 * the marker name becomes `name` part.
+	 *
+	 * Tag and attribute names and values are generated from the marker name:
+	 *
+	 * * templates for attributes are `data-[group]-start-before="[name]"`, `data-[group]-start-after="[name]"`,
+	 * `data-[group]-end-before="[name]"` and `data-[group]-end-after="[name]"`,
+	 * * templates for view elements are `<[group]-start name="[name]">` and `<[group]-end name="[name]">`.
+	 *
+	 * Attributes mark whether given marker start or end boundary is before or after given element.
+	 * Attributes `data-[group]-start-before` and `data-[group]-end-after` are favored.
+	 * The other two are used when the former two cannot be used.
+	 *
+	 * The conversion configuration can take a function that will generate different group and name parts.
+	 * If such function is set as the `config.view` parameter, it is passed a marker name and it is expected to return an object with two
+	 * properties: `group` and `name`. If the function returns falsy value, the conversion will not take place.
+	 *
+	 * Basic usage:
+	 *
+	 *		// Using the default conversion.
+	 *		// In this case, all markers which name starts with 'comment:' will be converted.
+	 *		// The `group` parameter will be set to `comment`.
+	 *		// The `name` parameter will be the rest of the marker name (without `:`).
+	 *		editor.conversion.for( 'dataDowncast' ).markerToData( {
+	 *			model: 'comment'
+	 *		} );
+	 *
+	 * An example of a view that may be generated by this conversion (assuming a marker with name `comment:commentId:uid` marked by `[]`):
+	 *
+	 *		// Model:
+	 *		<paragraph>Foo[bar</paragraph>
+	 *		<image src="abc.jpg"></image>]
+	 *
+	 *		// View:
+	 *		<p>Foo<comment-start name="commentId:uid"></comment-start>bar</p>
+	 *		<figure data-comment-end-after="commentId:uid" class="image"><img src="abc.jpg" /></figure>
+	 *
+	 * In the example above, the comment starts before "bar" and ends after the image.
+	 *
+	 * If `name` part is empty, following view may be generated:
+	 *
+	 *		<p>Foo <myMarker-start></myMarker-start>bar</p>
+	 *		<figure data-myMarker-end-after="" class="image"><img src="abc.jpg" /></figure>
+	 *
+	 * **Note:** situation when some markers have `name` part and some don't is incorrect and should be avoided.
+	 *
+	 * Examples where `data-group-start-after` and `data-group-end-before` are used:
+	 *
+	 *		// Model:
+	 *		<blockQuote>[]<paragraph>Foo</paragraph></blockQuote>
+	 *
+	 * 		// View:
+	 *		<blockquote><p data-group-end-before="name" data-group-start-before="name">Foo</p></blockquote>
+	 *
+	 * Similarly, when marker is collapsed after the last element:
+	 *
+	 *		// Model:
+	 *		<blockQuote><paragraph>Foo</paragraph>[]</blockQuote>
+	 *
+	 *		// View:
+	 *		<blockquote><p data-group-end-after="name" data-group-start-after="name">Foo</p></blockquote>
+	 *
+	 * When there are multiple markers from the same group stored in the same attribute of the same element, their
+	 * name parts are put together in the attribute value, for example: `data-group-start-before="name1,name2,name3"`.
+	 *
+	 * Other examples of usage:
+	 *
+	 *		// Using custom function which is the same as the default conversion:
+	 *		editor.conversion.for( 'dataDowncast' ).markerToData( {
+	 *			model: 'comment'
+	 *			view: markerName => ( {
+	 *				group: 'comment',
+	 *				name: markerName.substr( 8 ) // Removes 'comment:' part.
+	 *			} )
+	 *		} );
+	 *
+	 *		// Using converter priority:
+	 *		editor.conversion.for( 'dataDowncast' ).markerToData( {
+	 *			model: 'comment'
+	 *			view: markerName => ( {
+	 *				group: 'comment',
+	 *				name: markerName.substr( 8 ) // Removes 'comment:' part.
+	 *			} ),
+	 *			converterPriority: 'high'
+	 *		} );
+	 *
+	 * This kind of conversion is useful for saving data into the database, so it should be used in the data conversion pipeline.
+	 *
+	 * See {@link module:engine/conversion/conversion~Conversion#for `conversion.for()`} to learn how to add a converter
+	 * to the conversion process.
+	 *
+	 * @method #markerToData
+	 * @param {Object} config Conversion configuration.
+	 * @param {String} config.model The name of the model marker (or model marker group) to convert.
+	 * @param {Function} [config.view] Function that takes the model marker name as a parameter and returns an object with `group`
+	 * and `name` properties.
+	 * @param {module:utils/priorities~PriorityString} [config.converterPriority='normal'] Converter priority.
+	 * @returns {module:engine/conversion/downcasthelpers~DowncastHelpers}
+	 */
+	markerToData( config ) {
+		return this.add( downcastMarkerToData( config ) );
+	}
 }
 
 /**
@@ -755,6 +869,141 @@ function removeUIElement() {
 	};
 }
 
+// Function factory that creates a default converter for model markers.
+//
+// See {@link DowncastHelpers#markerToData} for more information what type of view is generated.
+//
+// This converter binds created UI elements and affected view elements with the marker name
+// using {@link module:engine/conversion/mapper~Mapper#bindElementToMarker}.
+//
+// @returns {Function} Add marker converter.
+function insertMarkerData( viewCreator ) {
+	return ( evt, data, conversionApi ) => {
+		const viewMarkerData = viewCreator( data.markerName );
+
+		if ( !viewMarkerData ) {
+			return;
+		}
+
+		const markerRange = data.markerRange;
+
+		if ( !conversionApi.consumable.consume( markerRange, evt.name ) ) {
+			return;
+		}
+
+		// Adding closing data first to keep the proper order in the view.
+		handleMarkerBoundary( markerRange, false, conversionApi, data, viewMarkerData );
+		handleMarkerBoundary( markerRange, true, conversionApi, data, viewMarkerData );
+
+		evt.stop();
+	};
+}
+
+// Helper function for `insertMarkerData()` that marks a marker boundary at the beginning or end of given `range`.
+function handleMarkerBoundary( range, isStart, conversionApi, data, viewMarkerData ) {
+	const modelPosition = isStart ? range.start : range.end;
+	const canInsertElement = conversionApi.schema.checkChild( modelPosition, '$text' );
+
+	if ( canInsertElement ) {
+		const viewPosition = conversionApi.mapper.toViewPosition( modelPosition );
+
+		insertMarkerAsElement( viewPosition, isStart, conversionApi, data, viewMarkerData );
+	} else {
+		let modelElement;
+		let isBefore;
+
+		// If possible, we want to add `data-group-start-before` and `data-group-end-after` attributes.
+		// Below `if` is constructed in a way that will favor adding these attributes.
+		//
+		// Also, I assume that there will be always an element either after or before the position.
+		// If not, then it is a case when we are not in a position where text is allowed and also there are no elements around...
+		if ( isStart && modelPosition.nodeAfter || !isStart && !modelPosition.nodeBefore ) {
+			modelElement = modelPosition.nodeAfter;
+			isBefore = true;
+		} else {
+			modelElement = modelPosition.nodeBefore;
+			isBefore = false;
+		}
+
+		const viewElement = conversionApi.mapper.toViewElement( modelElement );
+
+		insertMarkerAsAttribute( viewElement, isStart, isBefore, conversionApi, data, viewMarkerData );
+	}
+}
+
+// Helper function for `insertMarkerData()` that marks a marker boundary in the view as an attribute on a view element.
+function insertMarkerAsAttribute( viewElement, isStart, isBefore, conversionApi, data, viewMarkerData ) {
+	const attributeName = `data-${ viewMarkerData.group }-${ isStart ? 'start' : 'end' }-${ isBefore ? 'before' : 'after' }`;
+
+	const markerNames = viewElement.hasAttribute( attributeName ) ? viewElement.getAttribute( attributeName ).split( ',' ) : [];
+
+	// Adding marker name at the beginning to have the same order in the attribute as there is with marker elements.
+	markerNames.unshift( viewMarkerData.name );
+
+	conversionApi.writer.setAttribute( attributeName, markerNames.join( ',' ), viewElement );
+	conversionApi.mapper.bindElementToMarker( viewElement, data.markerName );
+}
+
+// Helper function for `insertMarkerData()` that marks a marker boundary in the view as a separate view ui element.
+function insertMarkerAsElement( position, isStart, conversionApi, data, viewMarkerData ) {
+	const viewElementName = `${ viewMarkerData.group }-${ isStart ? 'start' : 'end' }`;
+
+	const attrs = viewMarkerData.name ? { 'name': viewMarkerData.name } : null;
+	const viewElement = conversionApi.writer.createUIElement( viewElementName, attrs );
+
+	conversionApi.writer.insert( position, viewElement );
+	conversionApi.mapper.bindElementToMarker( viewElement, data.markerName );
+}
+
+// Function factory that creates a converter for removing a model marker data added by the {@link #insertMarkerData} converter.
+//
+// @returns {Function} Remove marker converter.
+function removeMarkerData( viewCreator ) {
+	return ( evt, data, conversionApi ) => {
+		const viewData = viewCreator( data.markerName );
+
+		if ( !viewData ) {
+			return;
+		}
+
+		const elements = conversionApi.mapper.markerNameToElements( data.markerName );
+
+		if ( !elements ) {
+			return;
+		}
+
+		for ( const element of elements ) {
+			conversionApi.mapper.unbindElementFromMarkerName( element, data.markerName );
+
+			if ( element.is( 'containerElement' ) ) {
+				removeMarkerFromAttribute( `data-${ viewData.group }-start-before`, element );
+				removeMarkerFromAttribute( `data-${ viewData.group }-start-after`, element );
+				removeMarkerFromAttribute( `data-${ viewData.group }-end-before`, element );
+				removeMarkerFromAttribute( `data-${ viewData.group }-end-after`, element );
+			} else {
+				conversionApi.writer.clear( conversionApi.writer.createRangeOn( element ), element );
+			}
+		}
+
+		conversionApi.writer.clearClonedElementsGroup( data.markerName );
+
+		evt.stop();
+
+		function removeMarkerFromAttribute( attributeName, element ) {
+			if ( element.hasAttribute( attributeName ) ) {
+				const markerNames = new Set( element.getAttribute( attributeName ).split( ',' ) );
+				markerNames.delete( viewData.name );
+
+				if ( markerNames.size == 0 ) {
+					conversionApi.writer.removeAttribute( attributeName, element );
+				} else {
+					conversionApi.writer.setAttribute( attributeName, Array.from( markerNames ).join( ',' ), element );
+				}
+			}
+		}
+	};
+}
+
 // Function factory that creates a converter which converts set/change/remove attribute changes from the model to the view.
 //
 // Attributes from the model are converted to the view element attributes in the view. You may provide a custom function to generate
@@ -1179,6 +1428,34 @@ function downcastMarkerToElement( config ) {
 	};
 }
 
+// Model marker to view data conversion helper.
+//
+// See {@link ~DowncastHelpers#markerToData `markerToData()` downcast helper} to learn more.
+//
+// @param {Object} config
+// @param {String} config.model
+// @param {Function} [config.view]
+// @param {module:utils/priorities~PriorityString} [config.converterPriority='normal']
+// @returns {Function} Conversion helper.
+function downcastMarkerToData( config ) {
+	config = cloneDeep( config );
+
+	const group = config.model;
+
+	// Default conversion.
+	if ( !config.view ) {
+		config.view = markerName => ( {
+			group,
+			name: markerName.substr( config.model.length + 1 )
+		} );
+	}
+
+	return dispatcher => {
+		dispatcher.on( 'addMarker:' + group, insertMarkerData( config.view ), { priority: config.converterPriority || 'normal' } );
+		dispatcher.on( 'removeMarker:' + group, removeMarkerData( config.view ), { priority: config.converterPriority || 'normal' } );
+	};
+}
+
 // Model marker to highlight conversion helper.
 //
 // See {@link ~DowncastHelpers#markerToElement `.markerToElement()` downcast helper} for examples.