chore(vscode): update to 1.53.2

These conflicts will be resolved in the following commits. We do it this way so
that PR review is possible.

lib/vscode/src/vs/editor/common/config/commonEditorConfig.ts

@@ -272,7 +272,7 @@ function migrateOptions(options: IEditorOptions): void {
 	}
 }
 
-function deepCloneAndMigrateOptions(_options: IEditorOptions): IEditorOptions {
+function deepCloneAndMigrateOptions(_options: Readonly<IEditorOptions>): IEditorOptions {
 	const options = objects.deepClone(_options);
 	migrateOptions(options);
 	return options;
@@ -298,7 +298,7 @@ export abstract class CommonEditorConfiguration extends Disposable implements IC
 	private _readOptions: RawEditorOptions;
 	protected _validatedOptions: ValidatedEditorOptions;
 
-	constructor(isSimpleWidget: boolean, _options: IEditorOptions) {
+	constructor(isSimpleWidget: boolean, _options: Readonly<IEditorOptions>) {
 		super();
 		this.isSimpleWidget = isSimpleWidget;
 
@@ -318,8 +318,7 @@ export abstract class CommonEditorConfiguration extends Disposable implements IC
 	public observeReferenceElement(dimension?: IDimension): void {
 	}
 
-	public dispose(): void {
-		super.dispose();
+	public updatePixelRatio(): void {
 	}
 
 	protected _recomputeOptions(): void {
@@ -348,7 +347,7 @@ export abstract class CommonEditorConfiguration extends Disposable implements IC
 
 	private _computeInternalOptions(): ComputedEditorOptions {
 		const partialEnv = this._getEnvConfiguration();
-		const bareFontInfo = BareFontInfo.createFromValidatedSettings(this._validatedOptions, partialEnv.zoomLevel, this.isSimpleWidget);
+		const bareFontInfo = BareFontInfo.createFromValidatedSettings(this._validatedOptions, partialEnv.zoomLevel, partialEnv.pixelRatio, this.isSimpleWidget);
 		const env: IEnvironmentalOptions = {
 			memory: this._computeOptionsMemory,
 			outerWidth: partialEnv.outerWidth,
@@ -394,7 +393,7 @@ export abstract class CommonEditorConfiguration extends Disposable implements IC
 		return true;
 	}
 
-	public updateOptions(_newOptions: IEditorOptions): void {
+	public updateOptions(_newOptions: Readonly<IEditorOptions>): void {
 		if (typeof _newOptions === 'undefined') {
 			return;
 		}

lib/vscode/src/vs/editor/common/config/editorOptions.ts

@@ -625,6 +625,10 @@ export interface IEditorOptions {
 	 * Controls strikethrough deprecated variables.
 	 */
 	showDeprecated?: boolean;
+	/**
+	 * Control the behavior and rendering of the inline hints.
+	 */
+	inlineHints?: IEditorInlineHintsOptions;
 }
 
 /**
@@ -677,6 +681,11 @@ export interface IDiffEditorOptions extends IEditorOptions {
 	 * Defaults to false
 	 */
 	isInEmbeddedEditor?: boolean;
+	/**
+	 * Is the diff editor should render overview ruler
+	 * Defaults to true
+	 */
+	renderOverviewRuler?: boolean;
 	/**
 	 * Control the wrapping of the diff editor.
 	 */
@@ -2364,6 +2373,74 @@ class EditorLightbulb extends BaseEditorOption<EditorOption.lightbulb, EditorLig
 
 //#endregion
 
+//#region inlineHints
+
+/**
+ * Configuration options for editor inlineHints
+ */
+export interface IEditorInlineHintsOptions {
+	/**
+	 * Enable the inline hints.
+	 * Defaults to true.
+	 */
+	enabled?: boolean;
+
+	/**
+	 * Font size of inline hints.
+	 * Default to 90% of the editor font size.
+	 */
+	fontSize?: number;
+
+	/**
+	 * Font family of inline hints.
+	 * Defaults to editor font family.
+	 */
+	fontFamily?: string;
+}
+
+export type EditorInlineHintsOptions = Readonly<Required<IEditorInlineHintsOptions>>;
+
+class EditorInlineHints extends BaseEditorOption<EditorOption.inlineHints, EditorInlineHintsOptions> {
+
+	constructor() {
+		const defaults: EditorInlineHintsOptions = { enabled: true, fontSize: 0, fontFamily: EDITOR_FONT_DEFAULTS.fontFamily };
+		super(
+			EditorOption.inlineHints, 'inlineHints', defaults,
+			{
+				'editor.inlineHints.enabled': {
+					type: 'boolean',
+					default: defaults.enabled,
+					description: nls.localize('inlineHints.enable', "Enables the inline hints in the editor.")
+				},
+				'editor.inlineHints.fontSize': {
+					type: 'number',
+					default: defaults.fontSize,
+					description: nls.localize('inlineHints.fontSize', "Controls font size of inline hints in the editor. When set to `0`, the 90% of `#editor.fontSize#` is used.")
+				},
+				'editor.inlineHints.fontFamily': {
+					type: 'string',
+					default: defaults.fontFamily,
+					description: nls.localize('inlineHints.fontFamily', "Controls font family of inline hints in the editor.")
+				},
+			}
+		);
+	}
+
+	public validate(_input: any): EditorInlineHintsOptions {
+		if (!_input || typeof _input !== 'object') {
+			return this.defaultValue;
+		}
+		const input = _input as IEditorInlineHintsOptions;
+		return {
+			enabled: boolean(input.enabled, this.defaultValue.enabled),
+			fontSize: EditorIntOption.clampedInt(input.fontSize, this.defaultValue.fontSize, 0, 100),
+			fontFamily: EditorStringOption.string(input.fontFamily, this.defaultValue.fontFamily)
+		};
+	}
+}
+
+//#endregion
+
 //#region lineHeight
 
 class EditorLineHeight extends EditorIntOption<EditorOption.lineHeight> {
@@ -3273,7 +3350,7 @@ class EditorSuggest extends BaseEditorOption<EditorOption.suggest, InternalSugge
 				'editor.suggest.showInlineDetails': {
 					type: 'boolean',
 					default: defaults.showInlineDetails,
-					description: nls.localize('suggest.showInlineDetails', "Controls whether sugget details show inline with the label or only in the details widget")
+					description: nls.localize('suggest.showInlineDetails', "Controls whether suggest details show inline with the label or only in the details widget")
 				},
 				'editor.suggest.maxVisibleSuggestions': {
 					type: 'number',
@@ -3754,7 +3831,7 @@ export const enum EditorOption {
 	wrappingIndent,
 	wrappingStrategy,
 	showDeprecated,
-
+	inlineHints,
 	// Leave these at the end (because they have dependencies!)
 	editorClassName,
 	pixelRatio,
@@ -4258,6 +4335,7 @@ export const EditorOptions = {
 		EditorOption.showDeprecated, 'showDeprecated', true,
 		{ description: nls.localize('showDeprecated', "Controls strikethrough deprecated variables.") }
 	)),
+	inlineHints: register(new EditorInlineHints()),
 	snippetSuggestions: register(new EditorStringEnumOption(
 		EditorOption.snippetSuggestions, 'snippetSuggestions',
 		'inline' as 'top' | 'bottom' | 'inline' | 'none',

lib/vscode/src/vs/editor/common/config/fontInfo.ts

@@ -24,33 +24,33 @@ export class BareFontInfo {
 	/**
 	 * @internal
 	 */
-	public static createFromValidatedSettings(options: ValidatedEditorOptions, zoomLevel: number, ignoreEditorZoom: boolean): BareFontInfo {
+	public static createFromValidatedSettings(options: ValidatedEditorOptions, zoomLevel: number, pixelRatio: number, ignoreEditorZoom: boolean): BareFontInfo {
 		const fontFamily = options.get(EditorOption.fontFamily);
 		const fontWeight = options.get(EditorOption.fontWeight);
 		const fontSize = options.get(EditorOption.fontSize);
 		const fontFeatureSettings = options.get(EditorOption.fontLigatures);
 		const lineHeight = options.get(EditorOption.lineHeight);
 		const letterSpacing = options.get(EditorOption.letterSpacing);
-		return BareFontInfo._create(fontFamily, fontWeight, fontSize, fontFeatureSettings, lineHeight, letterSpacing, zoomLevel, ignoreEditorZoom);
+		return BareFontInfo._create(fontFamily, fontWeight, fontSize, fontFeatureSettings, lineHeight, letterSpacing, zoomLevel, pixelRatio, ignoreEditorZoom);
 	}
 
 	/**
 	 * @internal
 	 */
-	public static createFromRawSettings(opts: { fontFamily?: string; fontWeight?: string; fontSize?: number; fontLigatures?: boolean | string; lineHeight?: number; letterSpacing?: number; }, zoomLevel: number, ignoreEditorZoom: boolean = false): BareFontInfo {
+	public static createFromRawSettings(opts: { fontFamily?: string; fontWeight?: string; fontSize?: number; fontLigatures?: boolean | string; lineHeight?: number; letterSpacing?: number; }, zoomLevel: number, pixelRatio: number, ignoreEditorZoom: boolean = false): BareFontInfo {
 		const fontFamily = EditorOptions.fontFamily.validate(opts.fontFamily);
 		const fontWeight = EditorOptions.fontWeight.validate(opts.fontWeight);
 		const fontSize = EditorOptions.fontSize.validate(opts.fontSize);
 		const fontFeatureSettings = EditorOptions.fontLigatures2.validate(opts.fontLigatures);
 		const lineHeight = EditorOptions.lineHeight.validate(opts.lineHeight);
 		const letterSpacing = EditorOptions.letterSpacing.validate(opts.letterSpacing);
-		return BareFontInfo._create(fontFamily, fontWeight, fontSize, fontFeatureSettings, lineHeight, letterSpacing, zoomLevel, ignoreEditorZoom);
+		return BareFontInfo._create(fontFamily, fontWeight, fontSize, fontFeatureSettings, lineHeight, letterSpacing, zoomLevel, pixelRatio, ignoreEditorZoom);
 	}
 
 	/**
 	 * @internal
 	 */
-	private static _create(fontFamily: string, fontWeight: string, fontSize: number, fontFeatureSettings: string, lineHeight: number, letterSpacing: number, zoomLevel: number, ignoreEditorZoom: boolean): BareFontInfo {
+	private static _create(fontFamily: string, fontWeight: string, fontSize: number, fontFeatureSettings: string, lineHeight: number, letterSpacing: number, zoomLevel: number, pixelRatio: number, ignoreEditorZoom: boolean): BareFontInfo {
 		if (lineHeight === 0) {
 			lineHeight = Math.round(GOLDEN_LINE_HEIGHT_RATIO * fontSize);
 		} else if (lineHeight < MINIMUM_LINE_HEIGHT) {
@@ -63,6 +63,7 @@ export class BareFontInfo {
 
 		return new BareFontInfo({
 			zoomLevel: zoomLevel,
+			pixelRatio: pixelRatio,
 			fontFamily: fontFamily,
 			fontWeight: fontWeight,
 			fontSize: fontSize,
@@ -73,6 +74,7 @@ export class BareFontInfo {
 	}
 
 	readonly zoomLevel: number;
+	readonly pixelRatio: number;
 	readonly fontFamily: string;
 	readonly fontWeight: string;
 	readonly fontSize: number;
@@ -85,6 +87,7 @@ export class BareFontInfo {
 	 */
 	protected constructor(opts: {
 		zoomLevel: number;
+		pixelRatio: number;
 		fontFamily: string;
 		fontWeight: string;
 		fontSize: number;
@@ -93,6 +96,7 @@ export class BareFontInfo {
 		letterSpacing: number;
 	}) {
 		this.zoomLevel = opts.zoomLevel;
+		this.pixelRatio = opts.pixelRatio;
 		this.fontFamily = String(opts.fontFamily);
 		this.fontWeight = String(opts.fontWeight);
 		this.fontSize = opts.fontSize;
@@ -105,7 +109,7 @@ export class BareFontInfo {
 	 * @internal
 	 */
 	public getId(): string {
-		return this.zoomLevel + '-' + this.fontFamily + '-' + this.fontWeight + '-' + this.fontSize + '-' + this.fontFeatureSettings + '-' + this.lineHeight + '-' + this.letterSpacing;
+		return this.zoomLevel + '-' + this.pixelRatio + '-' + this.fontFamily + '-' + this.fontWeight + '-' + this.fontSize + '-' + this.fontFeatureSettings + '-' + this.lineHeight + '-' + this.letterSpacing;
 	}
 
 	/**
@@ -125,9 +129,13 @@ export class BareFontInfo {
 	}
 }
 
+// change this whenever `FontInfo` members are changed
+export const SERIALIZED_FONT_INFO_VERSION = 1;
+
 export class FontInfo extends BareFontInfo {
 	readonly _editorStylingBrand: void;
 
+	readonly version: number = SERIALIZED_FONT_INFO_VERSION;
 	readonly isTrusted: boolean;
 	readonly isMonospace: boolean;
 	readonly typicalHalfwidthCharacterWidth: number;
@@ -143,6 +151,7 @@ export class FontInfo extends BareFontInfo {
 	 */
 	constructor(opts: {
 		zoomLevel: number;
+		pixelRatio: number;
 		fontFamily: string;
 		fontWeight: string;
 		fontSize: number;

lib/vscode/src/vs/editor/common/controller/cursor.ts

@@ -393,8 +393,8 @@ export class Cursor extends Disposable {
 		return this._cursors.getPrimaryCursor().modelState.position;
 	}
 
-	public setSelections(eventsCollector: ViewModelEventsCollector, source: string | null | undefined, selections: readonly ISelection[]): void {
-		this.setStates(eventsCollector, source, CursorChangeReason.NotSet, CursorState.fromModelSelections(selections));
+	public setSelections(eventsCollector: ViewModelEventsCollector, source: string | null | undefined, selections: readonly ISelection[], reason: CursorChangeReason): void {
+		this.setStates(eventsCollector, source, reason, CursorState.fromModelSelections(selections));
 	}
 
 	public getPrevEditOperationType(): EditOperationType {
@@ -584,7 +584,7 @@ export class Cursor extends Disposable {
 		});
 		if (selections) {
 			this._isHandling = false;
-			this.setSelections(eventsCollector, source, selections);
+			this.setSelections(eventsCollector, source, selections, CursorChangeReason.NotSet);
 		}
 		if (autoClosedCharactersRanges.length > 0) {
 			this._pushAutoClosedAction(autoClosedCharactersRanges, autoClosedEnclosingRanges);

lib/vscode/src/vs/editor/common/controller/cursorMoveOperations.ts

@@ -39,11 +39,11 @@ export class MoveOperations {
 	public static leftPositionAtomicSoftTabs(model: ICursorSimpleModel, lineNumber: number, column: number, tabSize: number): Position {
 		const minColumn = model.getLineMinColumn(lineNumber);
 		const lineContent = model.getLineContent(lineNumber);
-		const newPosition = AtomicTabMoveOperations.atomicPosition(lineContent, column - minColumn, tabSize, Direction.Left);
-		if (newPosition === -1) {
+		const newPosition = AtomicTabMoveOperations.atomicPosition(lineContent, column - 1, tabSize, Direction.Left);
+		if (newPosition === -1 || newPosition + 1 < minColumn) {
 			return this.leftPosition(model, lineNumber, column);
 		}
-		return new Position(lineNumber, minColumn + newPosition);
+		return new Position(lineNumber, newPosition + 1);
 	}
 
 	public static left(config: CursorConfiguration, model: ICursorSimpleModel, lineNumber: number, column: number): CursorPosition {
@@ -81,13 +81,12 @@ export class MoveOperations {
 	}
 
 	public static rightPositionAtomicSoftTabs(model: ICursorSimpleModel, lineNumber: number, column: number, tabSize: number, indentSize: number): Position {
-		const minColumn = model.getLineMinColumn(lineNumber);
 		const lineContent = model.getLineContent(lineNumber);
-		const newPosition = AtomicTabMoveOperations.atomicPosition(lineContent, column - minColumn, tabSize, Direction.Right);
+		const newPosition = AtomicTabMoveOperations.atomicPosition(lineContent, column - 1, tabSize, Direction.Right);
 		if (newPosition === -1) {
 			return this.rightPosition(model, lineNumber, column);
 		}
-		return new Position(lineNumber, minColumn + newPosition);
+		return new Position(lineNumber, newPosition + 1);
 	}
 
 	public static right(config: CursorConfiguration, model: ICursorSimpleModel, lineNumber: number, column: number): CursorPosition {

lib/vscode/src/vs/editor/common/controller/cursorTypeOperations.ts

@@ -351,13 +351,6 @@ export class TypeOperations {
 			if (ir) {
 				let oldEndViewColumn = CursorColumns.visibleColumnFromColumn2(config, model, range.getEndPosition());
 				const oldEndColumn = range.endColumn;
-
-				let beforeText = '\n';
-				if (indentation !== config.normalizeIndentation(ir.beforeEnter)) {
-					beforeText = config.normalizeIndentation(ir.beforeEnter) + lineText.substring(indentation.length, range.startColumn - 1) + '\n';
-					range = new Range(range.startLineNumber, 1, range.endLineNumber, range.endColumn);
-				}
-
 				const newLineContent = model.getLineContent(range.endLineNumber);
 				const firstNonWhitespace = strings.firstNonWhitespaceIndex(newLineContent);
 				if (firstNonWhitespace >= 0) {
@@ -367,7 +360,7 @@ export class TypeOperations {
 				}
 
 				if (keepPosition) {
-					return new ReplaceCommandWithoutChangingPosition(range, beforeText + config.normalizeIndentation(ir.afterEnter), true);
+					return new ReplaceCommandWithoutChangingPosition(range, '\n' + config.normalizeIndentation(ir.afterEnter), true);
 				} else {
 					let offset = 0;
 					if (oldEndColumn <= firstNonWhitespace + 1) {
@@ -376,7 +369,7 @@ export class TypeOperations {
 						}
 						offset = Math.min(oldEndViewColumn + 1 - config.normalizeIndentation(ir.afterEnter).length - 1, 0);
 					}
-					return new ReplaceCommandWithOffsetCursorState(range, beforeText + config.normalizeIndentation(ir.afterEnter), 0, offset, true);
+					return new ReplaceCommandWithOffsetCursorState(range, '\n' + config.normalizeIndentation(ir.afterEnter), 0, offset, true);
 				}
 			}
 		}

lib/vscode/src/vs/editor/common/diff/diffComputer.ts

@@ -313,6 +313,13 @@ export class DiffComputer {
 
 		if (this.original.lines.length === 1 && this.original.lines[0].length === 0) {
 			// empty original => fast path
+			if (this.modified.lines.length === 1 && this.modified.lines[0].length === 0) {
+				return {
+					quitEarly: false,
+					changes: []
+				};
+			}
+
 			return {
 				quitEarly: false,
 				changes: [{

lib/vscode/src/vs/editor/common/editorCommon.ts

@@ -157,9 +157,10 @@ export interface IConfiguration extends IDisposable {
 
 	setMaxLineNumber(maxLineNumber: number): void;
 	setViewLineCount(viewLineCount: number): void;
-	updateOptions(newOptions: IEditorOptions): void;
+	updateOptions(newOptions: Readonly<IEditorOptions>): void;
 	getRawOptions(): IEditorOptions;
 	observeReferenceElement(dimension?: IDimension): void;
+	updatePixelRatio(): void;
 	setIsDominatedByLongLines(isDominatedByLongLines: boolean): void;
 }
 
@@ -605,6 +606,7 @@ export interface IThemeDecorationRenderOptions {
 
 	fontStyle?: string;
 	fontWeight?: string;
+	fontSize?: string;
 	textDecoration?: string;
 	cursor?: string;
 	color?: string | ThemeColor;
@@ -629,13 +631,17 @@ export interface IContentDecorationRenderOptions {
 
 	border?: string;
 	borderColor?: string | ThemeColor;
+	borderRadius?: string;
 	fontStyle?: string;
 	fontWeight?: string;
+	fontSize?: string;
+	fontFamily?: string;
 	textDecoration?: string;
 	color?: string | ThemeColor;
 	backgroundColor?: string | ThemeColor;
 
 	margin?: string;
+	padding?: string;
 	width?: string;
 	height?: string;
 }

lib/vscode/src/vs/editor/common/editorContextKeys.ts

@@ -61,6 +61,7 @@ export namespace EditorContextKeys {
 	export const hasReferenceProvider = new RawContextKey<boolean>('editorHasReferenceProvider', false);
 	export const hasRenameProvider = new RawContextKey<boolean>('editorHasRenameProvider', false);
 	export const hasSignatureHelpProvider = new RawContextKey<boolean>('editorHasSignatureHelpProvider', false);
+	export const hasInlineHintsProvider = new RawContextKey<boolean>('editorHasInlineHintsProvider', false);
 
 	// -- mode context keys: formatting
 	export const hasDocumentFormattingProvider = new RawContextKey<boolean>('editorHasDocumentFormattingProvider', false);

lib/vscode/src/vs/editor/common/model.ts

@@ -600,12 +600,6 @@ export interface ITextModel {
 	 */
 	setValue(newValue: string): void;
 
-	/**
-	 * Replace the entire text buffer value contained in this model.
-	 * @internal
-	 */
-	setValueFromTextBuffer(newValue: ITextBuffer): void;
-
 	/**
 	 * Get the text stored in this model.
 	 * @param eol The end of line character preference. Defaults to `EndOfLinePreference.TextDefined`.
@@ -1276,7 +1270,7 @@ export interface ITextBufferBuilder {
  * @internal
  */
 export interface ITextBufferFactory {
-	create(defaultEOL: DefaultEndOfLine): ITextBuffer;
+	create(defaultEOL: DefaultEndOfLine): { textBuffer: ITextBuffer; disposable: IDisposable; };
 	getFirstLineText(lengthLimit: number): string;
 }
 

lib/vscode/src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder.ts

@@ -4,6 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { CharCode } from 'vs/base/common/charCode';
+import { IDisposable } from 'vs/base/common/lifecycle';
 import * as strings from 'vs/base/common/strings';
 import { DefaultEndOfLine, ITextBuffer, ITextBufferBuilder, ITextBufferFactory } from 'vs/editor/common/model';
 import { StringBuffer, createLineStarts, createLineStartsFast } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase';
@@ -38,7 +39,7 @@ export class PieceTreeTextBufferFactory implements ITextBufferFactory {
 		return '\n';
 	}
 
-	public create(defaultEOL: DefaultEndOfLine): ITextBuffer {
+	public create(defaultEOL: DefaultEndOfLine): { textBuffer: ITextBuffer; disposable: IDisposable; } {
 		const eol = this._getEOL(defaultEOL);
 		let chunks = this._chunks;
 
@@ -54,7 +55,8 @@ export class PieceTreeTextBufferFactory implements ITextBufferFactory {
 			}
 		}
 
-		return new PieceTreeTextBuffer(chunks, this._bom, eol, this._containsRTL, this._containsUnusualLineTerminators, this._isBasicASCII, this._normalizeEOL);
+		const textBuffer = new PieceTreeTextBuffer(chunks, this._bom, eol, this._containsRTL, this._containsUnusualLineTerminators, this._isBasicASCII, this._normalizeEOL);
+		return { textBuffer: textBuffer, disposable: textBuffer };
 	}
 
 	public getFirstLineText(lengthLimit: number): string {

lib/vscode/src/vs/editor/common/model/textModel.ts

@@ -37,6 +37,7 @@ import { EditorTheme } from 'vs/editor/common/view/viewContext';
 import { IUndoRedoService, ResourceEditStackSnapshot } from 'vs/platform/undoRedo/common/undoRedo';
 import { TextChange } from 'vs/editor/common/model/textChange';
 import { Constants } from 'vs/base/common/uint';
+import { PieceTreeTextBuffer } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer';
 
 function createTextBufferBuilder() {
 	return new PieceTreeTextBufferBuilder();
@@ -106,7 +107,7 @@ export function createTextBufferFactoryFromSnapshot(snapshot: model.ITextSnapsho
 	return builder.finish();
 }
 
-export function createTextBuffer(value: string | model.ITextBufferFactory, defaultEOL: model.DefaultEndOfLine): model.ITextBuffer {
+export function createTextBuffer(value: string | model.ITextBufferFactory, defaultEOL: model.DefaultEndOfLine): { textBuffer: model.ITextBuffer; disposable: IDisposable; } {
 	const factory = (typeof value === 'string' ? createTextBufferFactory(value) : value);
 	return factory.create(defaultEOL);
 }
@@ -268,6 +269,7 @@ export class TextModel extends Disposable implements model.ITextModel {
 	private readonly _undoRedoService: IUndoRedoService;
 	private _attachedEditorCount: number;
 	private _buffer: model.ITextBuffer;
+	private _bufferDisposable: IDisposable;
 	private _options: model.TextModelResolvedOptions;
 
 	private _isDisposed: boolean;
@@ -328,7 +330,9 @@ export class TextModel extends Disposable implements model.ITextModel {
 		this._undoRedoService = undoRedoService;
 		this._attachedEditorCount = 0;
 
-		this._buffer = createTextBuffer(source, creationOptions.defaultEOL);
+		const { textBuffer, disposable } = createTextBuffer(source, creationOptions.defaultEOL);
+		this._buffer = textBuffer;
+		this._bufferDisposable = disposable;
 
 		this._options = TextModel.resolveOptions(this._buffer, creationOptions);
 
@@ -386,10 +390,13 @@ export class TextModel extends Disposable implements model.ITextModel {
 		this._tokenization.dispose();
 		this._isDisposed = true;
 		super.dispose();
+		this._bufferDisposable.dispose();
 		this._isDisposing = false;
 		// Manually release reference to previous text buffer to avoid large leaks
 		// in case someone leaks a TextModel reference
-		this._buffer = createTextBuffer('', this._options.defaultEOL);
+		const emptyDisposedTextBuffer = new PieceTreeTextBuffer([], '', '\n', false, false, true, true);
+		emptyDisposedTextBuffer.dispose();
+		this._buffer = emptyDisposedTextBuffer;
 	}
 
 	private _assertNotDisposed(): void {
@@ -423,8 +430,8 @@ export class TextModel extends Disposable implements model.ITextModel {
 			return;
 		}
 
-		const textBuffer = createTextBuffer(value, this._options.defaultEOL);
-		this.setValueFromTextBuffer(textBuffer);
+		const { textBuffer, disposable } = createTextBuffer(value, this._options.defaultEOL);
+		this._setValueFromTextBuffer(textBuffer, disposable);
 	}
 
 	private _createContentChanged2(range: Range, rangeOffset: number, rangeLength: number, text: string, isUndoing: boolean, isRedoing: boolean, isFlush: boolean): IModelContentChangedEvent {
@@ -443,18 +450,16 @@ export class TextModel extends Disposable implements model.ITextModel {
 		};
 	}
 
-	public setValueFromTextBuffer(textBuffer: model.ITextBuffer): void {
+	private _setValueFromTextBuffer(textBuffer: model.ITextBuffer, textBufferDisposable: IDisposable): void {
 		this._assertNotDisposed();
-		if (textBuffer === null) {
-			// There's nothing to do
-			return;
-		}
 		const oldFullModelRange = this.getFullModelRange();
 		const oldModelValueLength = this.getValueLengthInRange(oldFullModelRange);
 		const endLineNumber = this.getLineCount();
 		const endColumn = this.getLineMaxColumn(endLineNumber);
 
 		this._buffer = textBuffer;
+		this._bufferDisposable.dispose();
+		this._bufferDisposable = textBufferDisposable;
 		this._increaseVersionId();
 
 		// Flush all tokens

lib/vscode/src/vs/editor/common/model/textModelTokens.ts

@@ -359,7 +359,7 @@ export class TextModelTokenization extends Disposable {
 			const text = this._textModel.getLineContent(lineIndex + 1);
 			const lineStartState = this._tokenizationStateStore.getBeginState(lineIndex);
 
-			const r = safeTokenize(languageIdentifier, this._tokenizationSupport, text, lineStartState!);
+			const r = safeTokenize(languageIdentifier, this._tokenizationSupport, text, true, lineStartState!);
 			builder.add(lineIndex + 1, r.tokens);
 			this._tokenizationStateStore.setEndState(linesLength, lineIndex, r.endState);
 			lineIndex = this._tokenizationStateStore.invalidLineStartIndex - 1; // -1 because the outer loop increments it
@@ -410,13 +410,13 @@ export class TextModelTokenization extends Disposable {
 		const languageIdentifier = this._textModel.getLanguageIdentifier();
 		let state = initialState;
 		for (let i = fakeLines.length - 1; i >= 0; i--) {
-			let r = safeTokenize(languageIdentifier, this._tokenizationSupport, fakeLines[i], state);
+			let r = safeTokenize(languageIdentifier, this._tokenizationSupport, fakeLines[i], false, state);
 			state = r.endState;
 		}
 
 		for (let lineNumber = startLineNumber; lineNumber <= endLineNumber; lineNumber++) {
 			let text = this._textModel.getLineContent(lineNumber);
-			let r = safeTokenize(languageIdentifier, this._tokenizationSupport, text, state);
+			let r = safeTokenize(languageIdentifier, this._tokenizationSupport, text, true, state);
 			builder.add(lineNumber, r.tokens);
 			this._tokenizationStateStore.setFakeTokens(lineNumber - 1);
 			state = r.endState;
@@ -443,12 +443,12 @@ function initializeTokenization(textModel: TextModel): [ITokenizationSupport | n
 	return [tokenizationSupport, initialState];
 }
 
-function safeTokenize(languageIdentifier: LanguageIdentifier, tokenizationSupport: ITokenizationSupport | null, text: string, state: IState): TokenizationResult2 {
+function safeTokenize(languageIdentifier: LanguageIdentifier, tokenizationSupport: ITokenizationSupport | null, text: string, hasEOL: boolean, state: IState): TokenizationResult2 {
 	let r: TokenizationResult2 | null = null;
 
 	if (tokenizationSupport) {
 		try {
-			r = tokenizationSupport.tokenize2(text, state.clone(), 0);
+			r = tokenizationSupport.tokenize2(text, hasEOL, state.clone(), 0);
 		} catch (e) {
 			onUnexpectedError(e);
 		}

lib/vscode/src/vs/editor/common/modes.ts

@@ -211,9 +211,9 @@ export interface ITokenizationSupport {
 	getInitialState(): IState;
 
 	// add offsetDelta to each of the returned indices
-	tokenize(line: string, state: IState, offsetDelta: number): TokenizationResult;
+	tokenize(line: string, hasEOL: boolean, state: IState, offsetDelta: number): TokenizationResult;
 
-	tokenize2(line: string, state: IState, offsetDelta: number): TokenizationResult2;
+	tokenize2(line: string, hasEOL: boolean, state: IState, offsetDelta: number): TokenizationResult2;
 }
 
 /**
@@ -1659,6 +1659,19 @@ export interface CodeLensProvider {
 	resolveCodeLens?(model: model.ITextModel, codeLens: CodeLens, token: CancellationToken): ProviderResult<CodeLens>;
 }
 
+export interface InlineHint {
+	text: string;
+	range: IRange;
+	description?: string | IMarkdownString;
+	whitespaceBefore?: boolean;
+	whitespaceAfter?: boolean;
+}
+
+export interface InlineHintsProvider {
+	onDidChangeInlineHints?: Event<void> | undefined;
+	provideInlineHints(model: model.ITextModel, range: Range, token: CancellationToken): ProviderResult<InlineHint[]>;
+}
+
 export interface SemanticTokensLegend {
 	readonly tokenTypes: string[];
 	readonly tokenModifiers: string[];
@@ -1764,6 +1777,11 @@ export const TypeDefinitionProviderRegistry = new LanguageFeatureRegistry<TypeDe
  */
 export const CodeLensProviderRegistry = new LanguageFeatureRegistry<CodeLensProvider>();
 
+/**
+ * @internal
+ */
+export const InlineHintsProviderRegistry = new LanguageFeatureRegistry<InlineHintsProvider>();
+
 /**
  * @internal
  */
@@ -1876,3 +1894,14 @@ export interface ITokenizationRegistry {
  * @internal
  */
 export const TokenizationRegistry = new TokenizationRegistryImpl();
+
+
+/**
+ * @internal
+ */
+export enum ExternalUriOpenerPriority {
+	None = 0,
+	Option = 1,
+	Default = 2,
+	Preferred = 3,
+}

lib/vscode/src/vs/editor/common/modes/languageConfiguration.ts

@@ -150,7 +150,7 @@ export interface OnEnterRule {
 	/**
 	 * This rule will only execute if the text above the this line matches this regular expression.
 	 */
-	oneLineAboveText?: RegExp;
+	previousLineText?: RegExp;
 	/**
 	 * The action to execute.
 	 */

lib/vscode/src/vs/editor/common/modes/languageConfigurationRegistry.ts

@@ -101,11 +101,11 @@ export class RichEditSupport {
 		return this._electricCharacter;
 	}
 
-	public onEnter(autoIndent: EditorAutoIndentStrategy, oneLineAboveText: string, beforeEnterText: string, afterEnterText: string): EnterAction | null {
+	public onEnter(autoIndent: EditorAutoIndentStrategy, previousLineText: string, beforeEnterText: string, afterEnterText: string): EnterAction | null {
 		if (!this._onEnterSupport) {
 			return null;
 		}
-		return this._onEnterSupport.onEnter(autoIndent, oneLineAboveText, beforeEnterText, afterEnterText);
+		return this._onEnterSupport.onEnter(autoIndent, previousLineText, beforeEnterText, afterEnterText);
 	}
 
 	private static _mergeConf(prev: LanguageConfiguration | null, current: LanguageConfiguration): LanguageConfiguration {
@@ -700,17 +700,17 @@ export class LanguageConfigurationRegistryImpl {
 			afterEnterText = endScopedLineTokens.getLineContent().substr(range.endColumn - 1 - scopedLineTokens.firstCharOffset);
 		}
 
-		let oneLineAboveText = '';
+		let previousLineText = '';
 		if (range.startLineNumber > 1 && scopedLineTokens.firstCharOffset === 0) {
 			// This is not the first line and the entire line belongs to this mode
 			const oneLineAboveScopedLineTokens = this.getScopedLineTokens(model, range.startLineNumber - 1);
 			if (oneLineAboveScopedLineTokens.languageId === scopedLineTokens.languageId) {
 				// The line above ends with text belonging to the same mode
-				oneLineAboveText = oneLineAboveScopedLineTokens.getLineContent();
+				previousLineText = oneLineAboveScopedLineTokens.getLineContent();
 			}
 		}
 
-		const enterResult = richEditSupport.onEnter(autoIndent, oneLineAboveText, beforeEnterText, afterEnterText);
+		const enterResult = richEditSupport.onEnter(autoIndent, previousLineText, beforeEnterText, afterEnterText);
 		if (!enterResult) {
 			return null;
 		}
@@ -729,6 +729,8 @@ export class LanguageConfigurationRegistryImpl {
 			} else {
 				appendText = '';
 			}
+		} else if (indentAction === IndentAction.Indent) {
+			appendText = '\t' + appendText;
 		}
 
 		let indentation = this.getIndentationAtPosition(model, range.startLineNumber, range.startColumn);

lib/vscode/src/vs/editor/common/modes/modesRegistry.ts

@@ -58,11 +58,12 @@ export const ModesRegistry = new EditorModesRegistry();
 Registry.add(Extensions.ModesRegistry, ModesRegistry);
 
 export const PLAINTEXT_MODE_ID = 'plaintext';
+export const PLAINTEXT_EXTENSION = '.txt';
 export const PLAINTEXT_LANGUAGE_IDENTIFIER = new LanguageIdentifier(PLAINTEXT_MODE_ID, LanguageId.PlainText);
 
 ModesRegistry.registerLanguage({
 	id: PLAINTEXT_MODE_ID,
-	extensions: ['.txt'],
+	extensions: [PLAINTEXT_EXTENSION],
 	aliases: [nls.localize('plainText.alias', "Plain Text"), 'text'],
 	mimetypes: ['text/plain']
 });

lib/vscode/src/vs/editor/common/modes/supports/onEnter.ts

@@ -49,7 +49,7 @@ export class OnEnterSupport {
 		this._regExpRules = opts.onEnterRules || [];
 	}
 
-	public onEnter(autoIndent: EditorAutoIndentStrategy, oneLineAboveText: string, beforeEnterText: string, afterEnterText: string): EnterAction | null {
+	public onEnter(autoIndent: EditorAutoIndentStrategy, previousLineText: string, beforeEnterText: string, afterEnterText: string): EnterAction | null {
 		// (1): `regExpRules`
 		if (autoIndent >= EditorAutoIndentStrategy.Advanced) {
 			for (let i = 0, len = this._regExpRules.length; i < len; i++) {
@@ -61,8 +61,8 @@ export class OnEnterSupport {
 					reg: rule.afterText,
 					text: afterEnterText
 				}, {
-					reg: rule.oneLineAboveText,
-					text: oneLineAboveText
+					reg: rule.previousLineText,
+					text: previousLineText
 				}].every((obj): boolean => {
 					return obj.reg ? obj.reg.test(obj.text) : true;
 				});

lib/vscode/src/vs/editor/common/modes/textToHtmlTokenizer.ts

@@ -12,12 +12,12 @@ import { NULL_STATE, nullTokenize2 } from 'vs/editor/common/modes/nullMode';
 
 export interface IReducedTokenizationSupport {
 	getInitialState(): IState;
-	tokenize2(line: string, state: IState, offsetDelta: number): TokenizationResult2;
+	tokenize2(line: string, hasEOL: boolean, state: IState, offsetDelta: number): TokenizationResult2;
 }
 
 const fallback: IReducedTokenizationSupport = {
 	getInitialState: () => NULL_STATE,
-	tokenize2: (buffer: string, state: IState, deltaOffset: number) => nullTokenize2(LanguageId.Null, buffer, state, deltaOffset)
+	tokenize2: (buffer: string, hasEOL: boolean, state: IState, deltaOffset: number) => nullTokenize2(LanguageId.Null, buffer, state, deltaOffset)
 };
 
 export function tokenizeToString(text: string, tokenizationSupport: IReducedTokenizationSupport = fallback): string {
@@ -110,7 +110,7 @@ function _tokenizeToString(text: string, tokenizationSupport: IReducedTokenizati
 			result += `<br/>`;
 		}
 
-		let tokenizationResult = tokenizationSupport.tokenize2(line, currentState, 0);
+		let tokenizationResult = tokenizationSupport.tokenize2(line, true, currentState, 0);
 		LineTokens.convertToEndOffset(tokenizationResult.tokens, line.length);
 		let lineTokens = new LineTokens(tokenizationResult.tokens, line);
 		let viewLineTokens = lineTokens.inflate();

lib/vscode/src/vs/editor/common/services/getSemanticTokens.ts

@@ -0,0 +1,160 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { CancellationToken } from 'vs/base/common/cancellation';
+import { onUnexpectedExternalError } from 'vs/base/common/errors';
+import { URI } from 'vs/base/common/uri';
+import { ITextModel } from 'vs/editor/common/model';
+import { DocumentSemanticTokensProviderRegistry, DocumentSemanticTokensProvider, SemanticTokens, SemanticTokensEdits, SemanticTokensLegend, DocumentRangeSemanticTokensProviderRegistry, DocumentRangeSemanticTokensProvider } from 'vs/editor/common/modes';
+import { IModelService } from 'vs/editor/common/services/modelService';
+import { CommandsRegistry, ICommandService } from 'vs/platform/commands/common/commands';
+import { assertType } from 'vs/base/common/types';
+import { VSBuffer } from 'vs/base/common/buffer';
+import { encodeSemanticTokensDto } from 'vs/editor/common/services/semanticTokensDto';
+import { Range } from 'vs/editor/common/core/range';
+
+export function isSemanticTokens(v: SemanticTokens | SemanticTokensEdits): v is SemanticTokens {
+	return v && !!((<SemanticTokens>v).data);
+}
+
+export function isSemanticTokensEdits(v: SemanticTokens | SemanticTokensEdits): v is SemanticTokensEdits {
+	return v && Array.isArray((<SemanticTokensEdits>v).edits);
+}
+
+export interface IDocumentSemanticTokensResult {
+	provider: DocumentSemanticTokensProvider;
+	request: Promise<SemanticTokens | SemanticTokensEdits | null | undefined>;
+}
+
+export function getDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): IDocumentSemanticTokensResult | null {
+	const provider = _getDocumentSemanticTokensProvider(model);
+	if (!provider) {
+		return null;
+	}
+	return {
+		provider: provider,
+		request: Promise.resolve(provider.provideDocumentSemanticTokens(model, lastResultId, token))
+	};
+}
+
+function _getDocumentSemanticTokensProvider(model: ITextModel): DocumentSemanticTokensProvider | null {
+	const result = DocumentSemanticTokensProviderRegistry.ordered(model);
+	return (result.length > 0 ? result[0] : null);
+}
+
+export function getDocumentRangeSemanticTokensProvider(model: ITextModel): DocumentRangeSemanticTokensProvider | null {
+	const result = DocumentRangeSemanticTokensProviderRegistry.ordered(model);
+	return (result.length > 0 ? result[0] : null);
+}
+
+CommandsRegistry.registerCommand('_provideDocumentSemanticTokensLegend', async (accessor, ...args): Promise<SemanticTokensLegend | undefined> => {
+	const [uri] = args;
+	assertType(uri instanceof URI);
+
+	const model = accessor.get(IModelService).getModel(uri);
+	if (!model) {
+		return undefined;
+	}
+
+	const provider = _getDocumentSemanticTokensProvider(model);
+	if (!provider) {
+		// there is no provider => fall back to a document range semantic tokens provider
+		return accessor.get(ICommandService).executeCommand('_provideDocumentRangeSemanticTokensLegend', uri);
+	}
+
+	return provider.getLegend();
+});
+
+CommandsRegistry.registerCommand('_provideDocumentSemanticTokens', async (accessor, ...args): Promise<VSBuffer | undefined> => {
+	const [uri] = args;
+	assertType(uri instanceof URI);
+
+	const model = accessor.get(IModelService).getModel(uri);
+	if (!model) {
+		return undefined;
+	}
+
+	const r = getDocumentSemanticTokens(model, null, CancellationToken.None);
+	if (!r) {
+		// there is no provider => fall back to a document range semantic tokens provider
+		return accessor.get(ICommandService).executeCommand('_provideDocumentRangeSemanticTokens', uri, model.getFullModelRange());
+	}
+
+	const { provider, request } = r;
+
+	let result: SemanticTokens | SemanticTokensEdits | null | undefined;
+	try {
+		result = await request;
+	} catch (err) {
+		onUnexpectedExternalError(err);
+		return undefined;
+	}
+
+	if (!result || !isSemanticTokens(result)) {
+		return undefined;
+	}
+
+	const buff = encodeSemanticTokensDto({
+		id: 0,
+		type: 'full',
+		data: result.data
+	});
+	if (result.resultId) {
+		provider.releaseDocumentSemanticTokens(result.resultId);
+	}
+	return buff;
+});
+
+CommandsRegistry.registerCommand('_provideDocumentRangeSemanticTokensLegend', async (accessor, ...args): Promise<SemanticTokensLegend | undefined> => {
+	const [uri] = args;
+	assertType(uri instanceof URI);
+
+	const model = accessor.get(IModelService).getModel(uri);
+	if (!model) {
+		return undefined;
+	}
+
+	const provider = getDocumentRangeSemanticTokensProvider(model);
+	if (!provider) {
+		return undefined;
+	}
+
+	return provider.getLegend();
+});
+
+CommandsRegistry.registerCommand('_provideDocumentRangeSemanticTokens', async (accessor, ...args): Promise<VSBuffer | undefined> => {
+	const [uri, range] = args;
+	assertType(uri instanceof URI);
+	assertType(Range.isIRange(range));
+
+	const model = accessor.get(IModelService).getModel(uri);
+	if (!model) {
+		return undefined;
+	}
+
+	const provider = getDocumentRangeSemanticTokensProvider(model);
+	if (!provider) {
+		// there is no provider
+		return undefined;
+	}
+
+	let result: SemanticTokens | null | undefined;
+	try {
+		result = await provider.provideDocumentRangeSemanticTokens(model, Range.lift(range), CancellationToken.None);
+	} catch (err) {
+		onUnexpectedExternalError(err);
+		return undefined;
+	}
+
+	if (!result || !isSemanticTokens(result)) {
+		return undefined;
+	}
+
+	return encodeSemanticTokensDto({
+		id: 0,
+		type: 'full',
+		data: result.data
+	});
+});

lib/vscode/src/vs/editor/common/services/markerDecorationsServiceImpl.ts

@@ -87,13 +87,13 @@ export class MarkerDecorationsService extends Disposable implements IMarkerDecor
 		this._markerDecorations.clear();
 	}
 
-	getMarker(model: ITextModel, decoration: IModelDecoration): IMarker | null {
-		const markerDecorations = this._markerDecorations.get(MODEL_ID(model.uri));
+	getMarker(uri: URI, decoration: IModelDecoration): IMarker | null {
+		const markerDecorations = this._markerDecorations.get(MODEL_ID(uri));
 		return markerDecorations ? (markerDecorations.getMarker(decoration) || null) : null;
 	}
 
-	getLiveMarkers(model: ITextModel): [Range, IMarker][] {
-		const markerDecorations = this._markerDecorations.get(MODEL_ID(model.uri));
+	getLiveMarkers(uri: URI): [Range, IMarker][] {
+		const markerDecorations = this._markerDecorations.get(MODEL_ID(uri));
 		return markerDecorations ? markerDecorations.getMarkers() : [];
 	}
 

lib/vscode/src/vs/editor/common/services/markersDecorationService.ts

@@ -8,6 +8,7 @@ import { createDecorator } from 'vs/platform/instantiation/common/instantiation'
 import { IMarker } from 'vs/platform/markers/common/markers';
 import { Event } from 'vs/base/common/event';
 import { Range } from 'vs/editor/common/core/range';
+import { URI } from 'vs/base/common/uri';
 
 export const IMarkerDecorationsService = createDecorator<IMarkerDecorationsService>('markerDecorationsService');
 
@@ -16,7 +17,7 @@ export interface IMarkerDecorationsService {
 
 	onDidChangeMarker: Event<ITextModel>;
 
-	getMarker(model: ITextModel, decoration: IModelDecoration): IMarker | null;
+	getMarker(uri: URI, decoration: IModelDecoration): IMarker | null;
 
-	getLiveMarkers(model: ITextModel): [Range, IMarker][];
+	getLiveMarkers(uri: URI): [Range, IMarker][];
 }

lib/vscode/src/vs/editor/common/services/modeService.ts

@@ -50,7 +50,7 @@ export interface IModeService {
 	// --- instantiation
 	create(commaSeparatedMimetypesOrCommaSeparatedIds: string | undefined): ILanguageSelection;
 	createByLanguageName(languageName: string): ILanguageSelection;
-	createByFilepathOrFirstLine(rsource: URI | null, firstLine?: string): ILanguageSelection;
+	createByFilepathOrFirstLine(resource: URI | null, firstLine?: string): ILanguageSelection;
 
 	triggerMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): void;
 }

lib/vscode/src/vs/editor/common/services/modeServiceImpl.ts

@@ -40,23 +40,24 @@ class LanguageSelection extends Disposable implements ILanguageSelection {
 	}
 }
 
-export class ModeServiceImpl implements IModeService {
+export class ModeServiceImpl extends Disposable implements IModeService {
 	public _serviceBrand: undefined;
 
 	private readonly _instantiatedModes: { [modeId: string]: IMode; };
 	private readonly _registry: LanguagesRegistry;
 
-	private readonly _onDidCreateMode = new Emitter<IMode>();
+	private readonly _onDidCreateMode = this._register(new Emitter<IMode>());
 	public readonly onDidCreateMode: Event<IMode> = this._onDidCreateMode.event;
 
-	protected readonly _onLanguagesMaybeChanged = new Emitter<void>();
+	protected readonly _onLanguagesMaybeChanged = this._register(new Emitter<void>());
 	public readonly onLanguagesMaybeChanged: Event<void> = this._onLanguagesMaybeChanged.event;
 
 	constructor(warnOnOverwrite = false) {
+		super();
 		this._instantiatedModes = {};
 
-		this._registry = new LanguagesRegistry(true, warnOnOverwrite);
-		this._registry.onDidChange(() => this._onLanguagesMaybeChanged.fire());
+		this._registry = this._register(new LanguagesRegistry(true, warnOnOverwrite));
+		this._register(this._registry.onDidChange(() => this._onLanguagesMaybeChanged.fire()));
 	}
 
 	protected _onReady(): Promise<boolean> {

lib/vscode/src/vs/editor/common/services/modelServiceImpl.ts

@@ -29,6 +29,7 @@ import { StringSHA1 } from 'vs/base/common/hash';
 import { EditStackElement, isEditStackElement } from 'vs/editor/common/model/editStack';
 import { Schemas } from 'vs/base/common/network';
 import { SemanticTokensProviderStyling, toMultilineTokens2 } from 'vs/editor/common/services/semanticTokensProviderStyling';
+import { getDocumentSemanticTokens, isSemanticTokens, isSemanticTokensEdits } from 'vs/editor/common/services/getSemanticTokens';
 
 export interface IEditorSemanticHighlightingOptions {
 	enabled: true | false | 'configuredByTheme';
@@ -412,10 +413,11 @@ export class ModelServiceImpl extends Disposable implements IModelService {
 
 	public updateModel(model: ITextModel, value: string | ITextBufferFactory): void {
 		const options = this.getCreationOptions(model.getLanguageIdentifier().language, model.uri, model.isForSimpleWidget);
-		const textBuffer = createTextBuffer(value, options.defaultEOL);
+		const { textBuffer, disposable } = createTextBuffer(value, options.defaultEOL);
 
 		// Return early if the text is already set in that form
 		if (model.equalsTextBuffer(textBuffer)) {
+			disposable.dispose();
 			return;
 		}
 
@@ -428,6 +430,7 @@ export class ModelServiceImpl extends Disposable implements IModelService {
 			() => []
 		);
 		model.pushStackElement();
+		disposable.dispose();
 	}
 
 	private static _commonPrefix(a: ILineSequence, aLen: number, aDelta: number, b: ILineSequence, bLen: number, bDelta: number): number {
@@ -513,58 +516,6 @@ export class ModelServiceImpl extends Disposable implements IModelService {
 		if (!modelData) {
 			return;
 		}
-		const model = modelData.model;
-		const sharesUndoRedoStack = (this._undoRedoService.getUriComparisonKey(model.uri) !== model.uri.toString());
-		let maintainUndoRedoStack = false;
-		let heapSize = 0;
-		if (sharesUndoRedoStack || (this._shouldRestoreUndoStack() && schemaShouldMaintainUndoRedoElements(resource))) {
-			const elements = this._undoRedoService.getElements(resource);
-			if (elements.past.length > 0 || elements.future.length > 0) {
-				for (const element of elements.past) {
-					if (isEditStackElement(element) && element.matchesResource(resource)) {
-						maintainUndoRedoStack = true;
-						heapSize += element.heapSize(resource);
-						element.setModel(resource); // remove reference from text buffer instance
-					}
-				}
-				for (const element of elements.future) {
-					if (isEditStackElement(element) && element.matchesResource(resource)) {
-						maintainUndoRedoStack = true;
-						heapSize += element.heapSize(resource);
-						element.setModel(resource); // remove reference from text buffer instance
-					}
-				}
-			}
-		}
-
-		if (!maintainUndoRedoStack) {
-			if (!sharesUndoRedoStack) {
-				const initialUndoRedoSnapshot = modelData.model.getInitialUndoRedoSnapshot();
-				if (initialUndoRedoSnapshot !== null) {
-					this._undoRedoService.restoreSnapshot(initialUndoRedoSnapshot);
-				}
-			}
-			modelData.model.dispose();
-			return;
-		}
-
-		const maxMemory = ModelServiceImpl.MAX_MEMORY_FOR_CLOSED_FILES_UNDO_STACK;
-		if (!sharesUndoRedoStack && heapSize > maxMemory) {
-			// the undo stack for this file would never fit in the configured memory, so don't bother with it.
-			const initialUndoRedoSnapshot = modelData.model.getInitialUndoRedoSnapshot();
-			if (initialUndoRedoSnapshot !== null) {
-				this._undoRedoService.restoreSnapshot(initialUndoRedoSnapshot);
-			}
-			modelData.model.dispose();
-			return;
-		}
-
-		this._ensureDisposedModelsHeapSize(maxMemory - heapSize);
-
-		// We only invalidate the elements, but they remain in the undo-redo service.
-		this._undoRedoService.setElementsValidFlag(resource, false, (element) => (isEditStackElement(element) && element.matchesResource(resource)));
-		this._insertDisposedModel(new DisposedModelInfo(resource, modelData.model.getInitialUndoRedoSnapshot(), Date.now(), sharesUndoRedoStack, heapSize, computeModelSha1(model), model.getVersionId(), model.getAlternativeVersionId()));
-
 		modelData.model.dispose();
 	}
 
@@ -599,6 +550,50 @@ export class ModelServiceImpl extends Disposable implements IModelService {
 		const modelId = MODEL_ID(model.uri);
 		const modelData = this._models[modelId];
 
+		const sharesUndoRedoStack = (this._undoRedoService.getUriComparisonKey(model.uri) !== model.uri.toString());
+		let maintainUndoRedoStack = false;
+		let heapSize = 0;
+		if (sharesUndoRedoStack || (this._shouldRestoreUndoStack() && schemaShouldMaintainUndoRedoElements(model.uri))) {
+			const elements = this._undoRedoService.getElements(model.uri);
+			if (elements.past.length > 0 || elements.future.length > 0) {
+				for (const element of elements.past) {
+					if (isEditStackElement(element) && element.matchesResource(model.uri)) {
+						maintainUndoRedoStack = true;
+						heapSize += element.heapSize(model.uri);
+						element.setModel(model.uri); // remove reference from text buffer instance
+					}
+				}
+				for (const element of elements.future) {
+					if (isEditStackElement(element) && element.matchesResource(model.uri)) {
+						maintainUndoRedoStack = true;
+						heapSize += element.heapSize(model.uri);
+						element.setModel(model.uri); // remove reference from text buffer instance
+					}
+				}
+			}
+		}
+
+		const maxMemory = ModelServiceImpl.MAX_MEMORY_FOR_CLOSED_FILES_UNDO_STACK;
+		if (!maintainUndoRedoStack) {
+			if (!sharesUndoRedoStack) {
+				const initialUndoRedoSnapshot = modelData.model.getInitialUndoRedoSnapshot();
+				if (initialUndoRedoSnapshot !== null) {
+					this._undoRedoService.restoreSnapshot(initialUndoRedoSnapshot);
+				}
+			}
+		} else if (!sharesUndoRedoStack && heapSize > maxMemory) {
+			// the undo stack for this file would never fit in the configured memory, so don't bother with it.
+			const initialUndoRedoSnapshot = modelData.model.getInitialUndoRedoSnapshot();
+			if (initialUndoRedoSnapshot !== null) {
+				this._undoRedoService.restoreSnapshot(initialUndoRedoSnapshot);
+			}
+		} else {
+			this._ensureDisposedModelsHeapSize(maxMemory - heapSize);
+			// We only invalidate the elements, but they remain in the undo-redo service.
+			this._undoRedoService.setElementsValidFlag(model.uri, false, (element) => (isEditStackElement(element) && element.matchesResource(model.uri)));
+			this._insertDisposedModel(new DisposedModelInfo(model.uri, modelData.model.getInitialUndoRedoSnapshot(), Date.now(), sharesUndoRedoStack, heapSize, computeModelSha1(model), model.getVersionId(), model.getAlternativeVersionId()));
+		}
+
 		delete this._models[modelId];
 		modelData.dispose();
 
@@ -718,7 +713,9 @@ class SemanticTokensResponse {
 	}
 }
 
-class ModelSemanticColoring extends Disposable {
+export class ModelSemanticColoring extends Disposable {
+
+	public static FETCH_DOCUMENT_SEMANTIC_TOKENS_DELAY = 300;
 
 	private _isDisposed: boolean;
 	private readonly _model: ITextModel;
@@ -734,7 +731,7 @@ class ModelSemanticColoring extends Disposable {
 		this._isDisposed = false;
 		this._model = model;
 		this._semanticStyling = stylingProvider;
-		this._fetchDocumentSemanticTokens = this._register(new RunOnceScheduler(() => this._fetchDocumentSemanticTokensNow(), 300));
+		this._fetchDocumentSemanticTokens = this._register(new RunOnceScheduler(() => this._fetchDocumentSemanticTokensNow(), ModelSemanticColoring.FETCH_DOCUMENT_SEMANTIC_TOKENS_DELAY));
 		this._currentDocumentResponse = null;
 		this._currentDocumentRequestCancellationTokenSource = null;
 		this._documentProvidersChangeListeners = [];
@@ -788,15 +785,21 @@ class ModelSemanticColoring extends Disposable {
 			// there is already a request running, let it finish...
 			return;
 		}
-		const provider = this._getSemanticColoringProvider();
-		if (!provider) {
+
+		const cancellationTokenSource = new CancellationTokenSource();
+		const lastResultId = this._currentDocumentResponse ? this._currentDocumentResponse.resultId || null : null;
+		const r = getDocumentSemanticTokens(this._model, lastResultId, cancellationTokenSource.token);
+		if (!r) {
+			// there is no provider
 			if (this._currentDocumentResponse) {
 				// there are semantic tokens set
 				this._model.setSemanticTokens(null, false);
 			}
 			return;
 		}
-		this._currentDocumentRequestCancellationTokenSource = new CancellationTokenSource();
+
+		const { provider, request } = r;
+		this._currentDocumentRequestCancellationTokenSource = cancellationTokenSource;
 
 		const pendingChanges: IModelContentChangedEvent[] = [];
 		const contentChangeListener = this._model.onDidChangeContent((e) => {
@@ -805,15 +808,13 @@ class ModelSemanticColoring extends Disposable {
 
 		const styling = this._semanticStyling.get(provider);
 
-		const lastResultId = this._currentDocumentResponse ? this._currentDocumentResponse.resultId || null : null;
-		const request = Promise.resolve(provider.provideDocumentSemanticTokens(this._model, lastResultId, this._currentDocumentRequestCancellationTokenSource.token));
-
 		request.then((res) => {
 			this._currentDocumentRequestCancellationTokenSource = null;
 			contentChangeListener.dispose();
 			this._setDocumentSemanticTokens(provider, res || null, styling, pendingChanges);
 		}, (err) => {
-			if (!err || typeof err.message !== 'string' || err.message.indexOf('busy') === -1) {
+			const isExpectedError = err && (errors.isPromiseCanceledError(err) || (typeof err.message === 'string' && err.message.indexOf('busy') !== -1));
+			if (!isExpectedError) {
 				errors.onUnexpectedError(err);
 			}
 
@@ -831,14 +832,6 @@ class ModelSemanticColoring extends Disposable {
 		});
 	}
 
-	private static _isSemanticTokens(v: SemanticTokens | SemanticTokensEdits): v is SemanticTokens {
-		return v && !!((<SemanticTokens>v).data);
-	}
-
-	private static _isSemanticTokensEdits(v: SemanticTokens | SemanticTokensEdits): v is SemanticTokensEdits {
-		return v && Array.isArray((<SemanticTokensEdits>v).edits);
-	}
-
 	private static _copy(src: Uint32Array, srcOffset: number, dest: Uint32Array, destOffset: number, length: number): void {
 		for (let i = 0; i < length; i++) {
 			dest[destOffset + i] = src[srcOffset + i];
@@ -847,6 +840,12 @@ class ModelSemanticColoring extends Disposable {
 
 	private _setDocumentSemanticTokens(provider: DocumentSemanticTokensProvider | null, tokens: SemanticTokens | SemanticTokensEdits | null, styling: SemanticTokensProviderStyling | null, pendingChanges: IModelContentChangedEvent[]): void {
 		const currentResponse = this._currentDocumentResponse;
+		const rescheduleIfNeeded = () => {
+			if (pendingChanges.length > 0 && !this._fetchDocumentSemanticTokens.isScheduled()) {
+				this._fetchDocumentSemanticTokens.schedule();
+			}
+		};
+
 		if (this._currentDocumentResponse) {
 			this._currentDocumentResponse.dispose();
 			this._currentDocumentResponse = null;
@@ -864,10 +863,11 @@ class ModelSemanticColoring extends Disposable {
 		}
 		if (!tokens) {
 			this._model.setSemanticTokens(null, true);
+			rescheduleIfNeeded();
 			return;
 		}
 
-		if (ModelSemanticColoring._isSemanticTokensEdits(tokens)) {
+		if (isSemanticTokensEdits(tokens)) {
 			if (!currentResponse) {
 				// not possible!
 				this._model.setSemanticTokens(null, true);
@@ -918,7 +918,7 @@ class ModelSemanticColoring extends Disposable {
 			}
 		}
 
-		if (ModelSemanticColoring._isSemanticTokens(tokens)) {
+		if (isSemanticTokens(tokens)) {
 
 			this._currentDocumentResponse = new SemanticTokensResponse(provider, tokens.resultId, tokens.data);
 
@@ -937,21 +937,13 @@ class ModelSemanticColoring extends Disposable {
 						}
 					}
 				}
-
-				if (!this._fetchDocumentSemanticTokens.isScheduled()) {
-					this._fetchDocumentSemanticTokens.schedule();
-				}
 			}
 
 			this._model.setSemanticTokens(result, true);
-			return;
+		} else {
+			this._model.setSemanticTokens(null, true);
 		}
 
-		this._model.setSemanticTokens(null, true);
-	}
-
-	private _getSemanticColoringProvider(): DocumentSemanticTokensProvider | null {
-		const result = DocumentSemanticTokensProviderRegistry.ordered(this._model);
-		return (result.length > 0 ? result[0] : null);
+		rescheduleIfNeeded();
 	}
 }

lib/vscode/src/vs/editor/common/services/semanticTokensDto.ts

@@ -0,0 +1,152 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { VSBuffer } from 'vs/base/common/buffer';
+import * as platform from 'vs/base/common/platform';
+
+export interface IFullSemanticTokensDto {
+	id: number;
+	type: 'full';
+	data: Uint32Array;
+}
+
+export interface IDeltaSemanticTokensDto {
+	id: number;
+	type: 'delta';
+	deltas: { start: number; deleteCount: number; data?: Uint32Array; }[];
+}
+
+export type ISemanticTokensDto = IFullSemanticTokensDto | IDeltaSemanticTokensDto;
+
+const enum EncodedSemanticTokensType {
+	Full = 1,
+	Delta = 2
+}
+
+function reverseEndianness(arr: Uint8Array): void {
+	for (let i = 0, len = arr.length; i < len; i += 4) {
+		// flip bytes 0<->3 and 1<->2
+		const b0 = arr[i + 0];
+		const b1 = arr[i + 1];
+		const b2 = arr[i + 2];
+		const b3 = arr[i + 3];
+		arr[i + 0] = b3;
+		arr[i + 1] = b2;
+		arr[i + 2] = b1;
+		arr[i + 3] = b0;
+	}
+}
+
+function toLittleEndianBuffer(arr: Uint32Array): VSBuffer {
+	const uint8Arr = new Uint8Array(arr.buffer, arr.byteOffset, arr.length * 4);
+	if (!platform.isLittleEndian()) {
+		// the byte order must be changed
+		reverseEndianness(uint8Arr);
+	}
+	return VSBuffer.wrap(uint8Arr);
+}
+
+function fromLittleEndianBuffer(buff: VSBuffer): Uint32Array {
+	const uint8Arr = buff.buffer;
+	if (!platform.isLittleEndian()) {
+		// the byte order must be changed
+		reverseEndianness(uint8Arr);
+	}
+	if (uint8Arr.byteOffset % 4 === 0) {
+		return new Uint32Array(uint8Arr.buffer, uint8Arr.byteOffset, uint8Arr.length / 4);
+	} else {
+		// unaligned memory access doesn't work on all platforms
+		const data = new Uint8Array(uint8Arr.byteLength);
+		data.set(uint8Arr);
+		return new Uint32Array(data.buffer, data.byteOffset, data.length / 4);
+	}
+}
+
+export function encodeSemanticTokensDto(semanticTokens: ISemanticTokensDto): VSBuffer {
+	const dest = new Uint32Array(encodeSemanticTokensDtoSize(semanticTokens));
+	let offset = 0;
+	dest[offset++] = semanticTokens.id;
+	if (semanticTokens.type === 'full') {
+		dest[offset++] = EncodedSemanticTokensType.Full;
+		dest[offset++] = semanticTokens.data.length;
+		dest.set(semanticTokens.data, offset); offset += semanticTokens.data.length;
+	} else {
+		dest[offset++] = EncodedSemanticTokensType.Delta;
+		dest[offset++] = semanticTokens.deltas.length;
+		for (const delta of semanticTokens.deltas) {
+			dest[offset++] = delta.start;
+			dest[offset++] = delta.deleteCount;
+			if (delta.data) {
+				dest[offset++] = delta.data.length;
+				dest.set(delta.data, offset); offset += delta.data.length;
+			} else {
+				dest[offset++] = 0;
+			}
+		}
+	}
+	return toLittleEndianBuffer(dest);
+}
+
+function encodeSemanticTokensDtoSize(semanticTokens: ISemanticTokensDto): number {
+	let result = 0;
+	result += (
+		+ 1 // id
+		+ 1 // type
+	);
+	if (semanticTokens.type === 'full') {
+		result += (
+			+ 1 // data length
+			+ semanticTokens.data.length
+		);
+	} else {
+		result += (
+			+ 1 // delta count
+		);
+		result += (
+			+ 1 // start
+			+ 1 // deleteCount
+			+ 1 // data length
+		) * semanticTokens.deltas.length;
+		for (const delta of semanticTokens.deltas) {
+			if (delta.data) {
+				result += delta.data.length;
+			}
+		}
+	}
+	return result;
+}
+
+export function decodeSemanticTokensDto(_buff: VSBuffer): ISemanticTokensDto {
+	const src = fromLittleEndianBuffer(_buff);
+	let offset = 0;
+	const id = src[offset++];
+	const type: EncodedSemanticTokensType = src[offset++];
+	if (type === EncodedSemanticTokensType.Full) {
+		const length = src[offset++];
+		const data = src.subarray(offset, offset + length); offset += length;
+		return {
+			id: id,
+			type: 'full',
+			data: data
+		};
+	}
+	const deltaCount = src[offset++];
+	let deltas: { start: number; deleteCount: number; data?: Uint32Array; }[] = [];
+	for (let i = 0; i < deltaCount; i++) {
+		const start = src[offset++];
+		const deleteCount = src[offset++];
+		const length = src[offset++];
+		let data: Uint32Array | undefined;
+		if (length > 0) {
+			data = src.subarray(offset, offset + length); offset += length;
+		}
+		deltas[i] = { start, deleteCount, data };
+	}
+	return {
+		id: id,
+		type: 'delta',
+		deltas: deltas
+	};
+}

lib/vscode/src/vs/editor/common/standalone/standaloneEnums.ts

@@ -287,11 +287,12 @@ export enum EditorOption {
 	wrappingIndent = 117,
 	wrappingStrategy = 118,
 	showDeprecated = 119,
-	editorClassName = 120,
-	pixelRatio = 121,
-	tabFocusMode = 122,
-	layoutInfo = 123,
-	wrappingInfo = 124
+	inlineHints = 120,
+	editorClassName = 121,
+	pixelRatio = 122,
+	tabFocusMode = 123,
+	layoutInfo = 124,
+	wrappingInfo = 125
 }
 
 /**

lib/vscode/src/vs/editor/common/viewLayout/viewLayout.ts

@@ -255,7 +255,7 @@ export class ViewLayout extends Disposable implements IViewLayout {
 
 		let result = this._linesLayout.getLinesTotalHeight();
 		if (options.get(EditorOption.scrollBeyondLastLine)) {
-			result += height - options.get(EditorOption.lineHeight);
+			result += Math.max(0, height - options.get(EditorOption.lineHeight) - options.get(EditorOption.padding).bottom);
 		} else {
 			result += this._getHorizontalScrollbarHeight(width, contentWidth);
 		}

lib/vscode/src/vs/editor/common/viewModel/monospaceLineBreaksComputer.ts

@@ -303,6 +303,19 @@ function createLineBreaksFromPreviousLineBreaks(classifier: WrappingCharacterCla
 			breakOffsetVisibleColumn = forcedBreakOffsetVisibleColumn;
 		}
 
+		if (breakOffset <= lastBreakingOffset) {
+			// Make sure that we are advancing (at least one character)
+			const charCode = lineText.charCodeAt(lastBreakingOffset);
+			if (strings.isHighSurrogate(charCode)) {
+				// A surrogate pair must always be considered as a single unit, so it is never to be broken
+				breakOffset = lastBreakingOffset + 2;
+				breakOffsetVisibleColumn = lastBreakingOffsetVisibleColumn + 2;
+			} else {
+				breakOffset = lastBreakingOffset + 1;
+				breakOffsetVisibleColumn = lastBreakingOffsetVisibleColumn + computeCharWidth(charCode, lastBreakingOffsetVisibleColumn, tabSize, columnsForFullWidthChar);
+			}
+		}
+
 		lastBreakingOffset = breakOffset;
 		breakingOffsets[breakingOffsetsCount] = breakOffset;
 		lastBreakingOffsetVisibleColumn = breakOffsetVisibleColumn;
@@ -435,6 +448,10 @@ function computeCharWidth(charCode: number, visibleColumn: number, tabSize: numb
 	if (strings.isFullWidthCharacter(charCode)) {
 		return columnsForFullWidthChar;
 	}
+	if (charCode < 32) {
+		// when using `editor.renderControlCharacters`, the substitutions are often wide
+		return columnsForFullWidthChar;
+	}
 	return 1;
 }
 

lib/vscode/src/vs/editor/common/viewModel/splitLinesCollection.ts

@@ -503,15 +503,8 @@ export class SplitLinesCollection implements IViewModelLinesCollection {
 			return null;
 		}
 
-		let hiddenAreas = this.getHiddenAreas();
-		let isInHiddenArea = false;
-		let testPosition = new Position(fromLineNumber, 1);
-		for (const hiddenArea of hiddenAreas) {
-			if (hiddenArea.containsPosition(testPosition)) {
-				isInHiddenArea = true;
-				break;
-			}
-		}
+		// cannot use this.getHiddenAreas() because those decorations have already seen the effect of this model change
+		const isInHiddenArea = (fromLineNumber > 2 && !this.lines[fromLineNumber - 2].isVisible());
 
 		let outputFromLineNumber = (fromLineNumber === 1 ? 1 : this.prefixSumComputer.getAccumulatedValue(fromLineNumber - 2) + 1);
 

lib/vscode/src/vs/editor/common/viewModel/viewModelImpl.ts

@@ -916,8 +916,8 @@ export class ViewModel extends Disposable implements IViewModel {
 	public getPosition(): Position {
 		return this._cursor.getPrimaryCursorState().modelState.position;
 	}
-	public setSelections(source: string | null | undefined, selections: readonly ISelection[]): void {
-		this._withViewEventsCollector(eventsCollector => this._cursor.setSelections(eventsCollector, source, selections));
+	public setSelections(source: string | null | undefined, selections: readonly ISelection[], reason = CursorChangeReason.NotSet): void {
+		this._withViewEventsCollector(eventsCollector => this._cursor.setSelections(eventsCollector, source, selections, reason));
 	}
 	public saveCursorState(): ICursorState[] {
 		return this._cursor.saveState();