diff --git a/src/vs/editor/contrib/inlayHints/browser/inlayHintsController.ts b/src/vs/editor/contrib/inlayHints/browser/inlayHintsController.ts
index 7bb8d9044d051..ab1a93af43260 100644
--- a/src/vs/editor/contrib/inlayHints/browser/inlayHintsController.ts
+++ b/src/vs/editor/contrib/inlayHints/browser/inlayHintsController.ts
@@ -74,6 +74,10 @@ export class RenderedInlayHintLabelPart {
 	}
 }
 
+class ActiveInlayHintInfo {
+	constructor(readonly part: RenderedInlayHintLabelPart, readonly hasTriggerModifier: boolean) { }
+}
+
 // --- controller
 
 export class InlayHintsController implements IEditorContribution {
@@ -92,7 +96,7 @@ export class InlayHintsController implements IEditorContribution {
 	private readonly _decorationsMetadata = new Map<string, { item: InlayHintItem; classNameRef: IDisposable }>();
 	private readonly _ruleFactory = new DynamicCssRules(this._editor);
 
-	private _activeInlayHintPart?: RenderedInlayHintLabelPart;
+	private _activeInlayHintPart?: ActiveInlayHintInfo;
 
 	constructor(
 		private readonly _editor: ICodeEditor,
@@ -220,28 +224,32 @@ export class InlayHintsController implements IEditorContribution {
 		const store = new DisposableStore();
 		const gesture = store.add(new ClickLinkGesture(this._editor));
 
-		let removeHighlight = () => { };
+		// let removeHighlight = () => { };
+
+		const sessionStore = new DisposableStore();
+		store.add(sessionStore);
 
 		store.add(gesture.onMouseMoveOrRelevantKeyDown(e => {
 			const [mouseEvent] = e;
 			const labelPart = this._getInlayHintLabelPart(mouseEvent);
 			const model = this._editor.getModel();
 
-			if (!labelPart || !mouseEvent.hasTriggerModifier || !model) {
-				removeHighlight();
+			if (!labelPart || !model) {
+				sessionStore.clear();
 				return;
 			}
 
-			// render link => when the modifier is pressed and when there is a command or location
-			if (mouseEvent.hasTriggerModifier && (labelPart.part.command || labelPart.part.location)) {
+			// resolve the item
+			const cts = new CancellationTokenSource();
+			sessionStore.add(toDisposable(() => cts.dispose(true)));
+			labelPart.item.resolve(cts.token);
 
-				// resolve the item
-				const cts = new CancellationTokenSource();
-				labelPart.item.resolve(cts.token);
+			// render link => when the modifier is pressed and when there is a command or location
+			if ((labelPart.part.command || labelPart.part.location)) {
 
-				this._activeInlayHintPart = labelPart;
+				this._activeInlayHintPart = new ActiveInlayHintInfo(labelPart, mouseEvent.hasTriggerModifier);
 
-				const lineNumber = this._activeInlayHintPart.item.hint.position.lineNumber;
+				const lineNumber = labelPart.item.hint.position.lineNumber;
 				const range = new Range(lineNumber, 1, lineNumber, model.getLineMaxColumn(lineNumber));
 				const lineHints = new Set<InlayHintItem>();
 				for (const data of this._decorationsMetadata.values()) {
@@ -250,14 +258,13 @@ export class InlayHintsController implements IEditorContribution {
 					}
 				}
 				this._updateHintsDecorators([range], Array.from(lineHints));
-				removeHighlight = () => {
-					cts.dispose(true);
+				sessionStore.add(toDisposable(() => {
 					this._activeInlayHintPart = undefined;
 					this._updateHintsDecorators([range], Array.from(lineHints));
-				};
+				}));
 			}
 		}));
-		store.add(gesture.onCancel(removeHighlight));
+		store.add(gesture.onCancel(() => sessionStore.clear()));
 		store.add(gesture.onExecute(async e => {
 			const label = this._getInlayHintLabelPart(e);
 			if (label) {
@@ -430,11 +437,13 @@ export class InlayHintsController implements IEditorContribution {
 
 				this._fillInColors(cssProperties, item.hint);
 
-				if ((part.command || part.location) && this._activeInlayHintPart?.item === item && this._activeInlayHintPart.index === i) {
+				if ((part.command || part.location) && this._activeInlayHintPart?.part.item === item && this._activeInlayHintPart.part.index === i) {
 					// active link!
 					cssProperties.textDecoration = 'underline';
-					cssProperties.cursor = 'pointer';
-					cssProperties.color = themeColorFromId(colors.editorActiveLinkForeground);
+					if (this._activeInlayHintPart.hasTriggerModifier) {
+						cssProperties.color = themeColorFromId(colors.editorActiveLinkForeground);
+						cssProperties.cursor = 'pointer';
+					}
 				}
 
 				if (isFirst && isLast) {