Mitigations for Microarchitectural Data Sampling.

Approved by:	so
Security:	FreeBSD-SA-19:07.mds
Security:	CVE-2018-12126
Security:	CVE-2018-12127
Security:	CVE-2018-12130
Security:	CVE-2019-11091

sys/amd64/amd64/exception.S

@@ -487,6 +487,7 @@ fast_syscall_common:
 	testl	$TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax)
 	jne	3f
 	call	handle_ibrs_exit
+	callq	*mds_handler
 	/* Restore preserved registers. */
 	MEXITCOUNT
 	movq	TF_RDI(%rsp),%rdi	/* bonus; preserve arg 1 */
@@ -1121,6 +1122,7 @@ ld_regs:
 	jz	2f			/* keep running with kernel GS.base */
 	cli
 	call	handle_ibrs_exit_rs
+	callq	*mds_handler
 	cmpb	$0,pti
 	je	1f
 	pushq	%rdx

sys/amd64/amd64/genassym.c

@@ -230,6 +230,9 @@ ASSYM(PC_SAVED_UCR3, offsetof(struct pcpu, pc_saved_ucr3));
 ASSYM(PC_PTI_STACK, offsetof(struct pcpu, pc_pti_stack));
 ASSYM(PC_PTI_STACK_SZ, PC_PTI_STACK_SZ);
 ASSYM(PC_IBPB_SET, offsetof(struct pcpu, pc_ibpb_set));
+ASSYM(PC_MDS_TMP, offsetof(struct pcpu, pc_mds_tmp));
+ASSYM(PC_MDS_BUF, offsetof(struct pcpu, pc_mds_buf));
+ASSYM(PC_MDS_BUF64, offsetof(struct pcpu, pc_mds_buf64));
 
 ASSYM(LA_EOI, LAPIC_EOI * LAPIC_MEM_MUL);
 ASSYM(LA_ISR, LAPIC_ISR0 * LAPIC_MEM_MUL);

sys/amd64/amd64/initcpu.c

@@ -254,6 +254,7 @@ initializecpu(void)
 	}
 	hw_ibrs_recalculate();
 	hw_ssb_recalculate(false);
+	hw_mds_recalculate();
 	switch (cpu_vendor_id) {
 	case CPU_VENDOR_INTEL:
 		init_intel();

sys/amd64/amd64/machdep.c

@@ -1851,6 +1851,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 
 	TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable);
 	TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable);
+	TUNABLE_INT_FETCH("hw.mds_disable", &hw_mds_disable);
 
 	/* Location of kernel stack for locore */
 	return ((u_int64_t)thread0.td_pcb);

sys/amd64/amd64/support.S

@@ -1,8 +1,13 @@
 /*-
+ * Copyright (c) 2018-2019 The FreeBSD Foundation
  * Copyright (c) 2003 Peter Wemm.
  * Copyright (c) 1993 The Regents of the University of California.
  * All rights reserved.
  *
+ * Portions of this software were developed by
+ * Konstantin Belousov <[email protected]> under sponsorship from
+ * the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -892,3 +897,246 @@ ENTRY(handle_ibrs_exit_rs)
 END(handle_ibrs_exit_rs)
 
 	.noaltmacro
+
+ENTRY(mds_handler_void)
+	retq
+END(mds_handler_void)
+
+ENTRY(mds_handler_verw)
+	subq	$8, %rsp
+	movw	%ds, (%rsp)
+	verw	(%rsp)
+	addq	$8, %rsp
+	retq
+END(mds_handler_verw)
+
+ENTRY(mds_handler_ivb)
+	pushq	%rax
+	pushq	%rdx
+	pushq	%rcx
+
+	movq	%cr0, %rax
+	testb	$CR0_TS, %al
+	je	1f
+	clts
+1:	movq	PCPU(MDS_BUF), %rdx
+	movdqa	%xmm0, PCPU(MDS_TMP)
+	pxor	%xmm0, %xmm0
+
+	lfence
+	orpd	(%rdx), %xmm0
+	orpd	(%rdx), %xmm0
+	mfence
+	movl	$40, %ecx
+	addq	$16, %rdx
+2:	movntdq	%xmm0, (%rdx)
+	addq	$16, %rdx
+	decl	%ecx
+	jnz	2b
+	mfence
+
+	movdqa	PCPU(MDS_TMP),%xmm0
+	testb	$CR0_TS, %al
+	je	3f
+	movq	%rax, %cr0
+3:	popq	%rcx
+	popq	%rdx
+	popq	%rax
+	retq
+END(mds_handler_ivb)
+
+ENTRY(mds_handler_bdw)
+	pushq	%rax
+	pushq	%rbx
+	pushq	%rcx
+	pushq	%rdi
+	pushq	%rsi
+
+	movq	%cr0, %rax
+	testb	$CR0_TS, %al
+	je	1f
+	clts
+1:	movq	PCPU(MDS_BUF), %rbx
+	movdqa	%xmm0, PCPU(MDS_TMP)
+	pxor	%xmm0, %xmm0
+
+	movq	%rbx, %rdi
+	movq	%rbx, %rsi
+	movl	$40, %ecx
+2:	movntdq	%xmm0, (%rbx)
+	addq	$16, %rbx
+	decl	%ecx
+	jnz	2b
+	mfence
+	movl	$1536, %ecx
+	rep; movsb
+	lfence
+
+	movdqa	PCPU(MDS_TMP),%xmm0
+	testb	$CR0_TS, %al
+	je	3f
+	movq	%rax, %cr0
+3:	popq	%rsi
+	popq	%rdi
+	popq	%rcx
+	popq	%rbx
+	popq	%rax
+	retq
+END(mds_handler_bdw)
+
+ENTRY(mds_handler_skl_sse)
+	pushq	%rax
+	pushq	%rdx
+	pushq	%rcx
+	pushq	%rdi
+
+	movq	%cr0, %rax
+	testb	$CR0_TS, %al
+	je	1f
+	clts
+1:	movq	PCPU(MDS_BUF), %rdi
+	movq	PCPU(MDS_BUF64), %rdx
+	movdqa	%xmm0, PCPU(MDS_TMP)
+	pxor	%xmm0, %xmm0
+
+	lfence
+	orpd	(%rdx), %xmm0
+	orpd	(%rdx), %xmm0
+	xorl	%eax, %eax
+2:	clflushopt	5376(%rdi, %rax, 8)
+	addl	$8, %eax
+	cmpl	$8 * 12, %eax
+	jb	2b
+	sfence
+	movl	$6144, %ecx
+	xorl	%eax, %eax
+	rep; stosb
+	mfence
+
+	movdqa	PCPU(MDS_TMP), %xmm0
+	testb	$CR0_TS, %al
+	je	3f
+	movq	%rax, %cr0
+3:	popq	%rdi
+	popq	%rcx
+	popq	%rdx
+	popq	%rax
+	retq
+END(mds_handler_skl_sse)
+
+ENTRY(mds_handler_skl_avx)
+	pushq	%rax
+	pushq	%rdx
+	pushq	%rcx
+	pushq	%rdi
+
+	movq	%cr0, %rax
+	testb	$CR0_TS, %al
+	je	1f
+	clts
+1:	movq	PCPU(MDS_BUF), %rdi
+	movq	PCPU(MDS_BUF64), %rdx
+	vmovdqa	%ymm0, PCPU(MDS_TMP)
+	vpxor	%ymm0, %ymm0, %ymm0
+
+	lfence
+	vorpd	(%rdx), %ymm0, %ymm0
+	vorpd	(%rdx), %ymm0, %ymm0
+	xorl	%eax, %eax
+2:	clflushopt	5376(%rdi, %rax, 8)
+	addl	$8, %eax
+	cmpl	$8 * 12, %eax
+	jb	2b
+	sfence
+	movl	$6144, %ecx
+	xorl	%eax, %eax
+	rep; stosb
+	mfence
+
+	vmovdqa	PCPU(MDS_TMP), %ymm0
+	testb	$CR0_TS, %al
+	je	3f
+	movq	%rax, %cr0
+3:	popq	%rdi
+	popq	%rcx
+	popq	%rdx
+	popq	%rax
+	retq
+END(mds_handler_skl_avx)
+
+ENTRY(mds_handler_skl_avx512)
+	pushq	%rax
+	pushq	%rdx
+	pushq	%rcx
+	pushq	%rdi
+
+	movq	%cr0, %rax
+	testb	$CR0_TS, %al
+	je	1f
+	clts
+1:	movq	PCPU(MDS_BUF), %rdi
+	movq	PCPU(MDS_BUF64), %rdx
+/*	vmovdqa64	%zmm0, PCPU(MDS_TMP) */
+	.byte	0x65, 0x62, 0xf1, 0xfd, 0x48, 0x7f, 0x04, 0x25
+	.long	PC_MDS_TMP
+/*	vpxor	%zmm0, %zmm0, %zmm0 */
+	.byte	0x62, 0xf1, 0xfd, 0x48, 0xef, 0xc0
+
+	lfence
+/*	vorpd	(%rdx), %zmm0, %zmm0 */
+	.byte	0x62, 0xf1, 0xfd, 0x48, 0x56, 0x02
+/*	vorpd	(%rdx), %zmm0, %zmm0 */
+	.byte	0x62, 0xf1, 0xfd, 0x48, 0x56, 0x02
+	xorl	%eax, %eax
+2:	clflushopt	5376(%rdi, %rax, 8)
+	addl	$8, %eax
+	cmpl	$8 * 12, %eax
+	jb	2b
+	sfence
+	movl	$6144, %ecx
+	xorl	%eax, %eax
+	rep; stosb
+	mfence
+
+/*	vmovdqa64	PCPU(MDS_TMP), %zmm0 */
+	.byte	0x65, 0x62, 0xf1, 0xfd, 0x48, 0x6f, 0x04, 0x25
+	.long	PC_MDS_TMP
+	testb	$CR0_TS, %al
+	je	3f
+	movq	%rax, %cr0
+3:	popq	%rdi
+	popq	%rcx
+	popq	%rdx
+	popq	%rax
+	retq
+END(mds_handler_skl_avx512)
+
+ENTRY(mds_handler_silvermont)
+	pushq	%rax
+	pushq	%rdx
+	pushq	%rcx
+
+	movq	%cr0, %rax
+	testb	$CR0_TS, %al
+	je	1f
+	clts
+1:	movq	PCPU(MDS_BUF), %rdx
+	movdqa	%xmm0, PCPU(MDS_TMP)
+	pxor	%xmm0, %xmm0
+
+	movl	$16, %ecx
+2:	movntdq	%xmm0, (%rdx)
+	addq	$16, %rdx
+	decl	%ecx
+	jnz	2b
+	mfence
+
+	movdqa	PCPU(MDS_TMP),%xmm0
+	testb	$CR0_TS, %al
+	je	3f
+	movq	%rax, %cr0
+3:	popq	%rcx
+	popq	%rdx
+	popq	%rax
+	retq
+END(mds_handler_silvermont)

sys/amd64/include/pcpu.h

@@ -73,7 +73,11 @@
 	uint32_t pc_pcid_gen;						\
 	uint32_t pc_smp_tlb_done;	/* TLB op acknowledgement */	\
 	uint32_t pc_ibpb_set;						\
-	char	__pad[96]		/* be divisor of PAGE_SIZE	\
+	void	*pc_mds_buf;						\
+	void	*pc_mds_buf64;						\
+	uint32_t pc_pad[20];						\
+	uint8_t	pc_mds_tmp[64];						\
+	char	__pad[960]		/* be divisor of PAGE_SIZE	\
 					   after cache alignment */
 
 #define	PC_DBREG_CMD_NONE	0

sys/dev/cpuctl/cpuctl.c

@@ -530,6 +530,7 @@ cpuctl_do_eval_cpu_features(int cpu, struct thread *td)
 	hw_ibrs_recalculate();
 	restore_cpu(oldcpu, is_bound, td);
 	hw_ssb_recalculate(true);
+	hw_mds_recalculate();
 	printcpuinfo();
 	return (0);
 }

sys/i386/i386/exception.s

@@ -406,6 +406,7 @@ doreti_ast:
 	 */
 doreti_exit:
 	MEXITCOUNT
+	call	*mds_handler
 
 	.globl	doreti_popl_fs
 doreti_popl_fs:

sys/i386/i386/genassym.c

@@ -217,6 +217,9 @@ ASSYM(PC_CURRENTLDT, offsetof(struct pcpu, pc_currentldt));
 ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid));
 ASSYM(PC_CURPMAP, offsetof(struct pcpu, pc_curpmap));
 ASSYM(PC_PRIVATE_TSS, offsetof(struct pcpu, pc_private_tss));
+ASSYM(PC_MDS_TMP, offsetof(struct pcpu, pc_mds_tmp));
+ASSYM(PC_MDS_BUF, offsetof(struct pcpu, pc_mds_buf));
+ASSYM(PC_MDS_BUF64, offsetof(struct pcpu, pc_mds_buf64));
 
 #ifdef DEV_APIC
 ASSYM(LA_EOI, LAPIC_EOI * LAPIC_MEM_MUL);

sys/i386/i386/initcpu.c

@@ -768,6 +768,7 @@ initializecpu(void)
 		elf32_nxstack = 1;
 	}
 #endif
+	hw_mds_recalculate();
 }
 
 void