Merge tag 's390-6.1-1' of git://git.kernel.org/pub/scm/linux/kernel/g…

…it/s390/linux

Pull s390 updates from Vasily Gorbik:

 - Make use of the IBM z16 processor activity instrumentation facility
   extension to count neural network processor assist operations: add a
   new PMU device driver so that perf can make use of this.

 - Rework memcpy_real() to avoid DAT-off mode.

 - Rework absolute lowcore access code.

 - Various small fixes and improvements all over the code.

* tag 's390-6.1-1' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux:
  s390/pci: remove unused bus_next field from struct zpci_dev
  s390/cio: remove unused ccw_device_force_console() declaration
  s390/pai: Add support for PAI Extension 1 NNPA counters
  s390/mm: fix no previous prototype warnings in maccess.c
  s390/mm: uninline copy_oldmem_kernel() function
  s390/mm,ptdump: add real memory copy page markers
  s390/mm: rework memcpy_real() to avoid DAT-off mode
  s390/dump: save IPL CPU registers once DAT is available
  s390/pci: convert high_memory to physical address
  s390/smp,ptdump: add absolute lowcore markers
  s390/smp: rework absolute lowcore access
  s390/smp: call smp_reinit_ipl_cpu() before scheduler is available
  s390/ptdump: add missing amode31 markers
  s390/mm: split lowcore pages with set_memory_4k()
  s390/mm: remove unused access parameter from do_fault_error()
  s390/delay: sync comment within __delay() with reality
  s390: move from strlcpy with unused retval to strscpy

arch/s390/boot/startup.c

@@ -10,11 +10,14 @@
 #include <asm/sclp.h>
 #include <asm/diag.h>
 #include <asm/uv.h>
+#include <asm/abs_lowcore.h>
 #include "decompressor.h"
 #include "boot.h"
 #include "uv.h"
 
 unsigned long __bootdata_preserved(__kaslr_offset);
+unsigned long __bootdata_preserved(__abs_lowcore);
+unsigned long __bootdata_preserved(__memcpy_real_area);
 unsigned long __bootdata(__amode31_base);
 unsigned long __bootdata_preserved(VMALLOC_START);
 unsigned long __bootdata_preserved(VMALLOC_END);
@@ -180,7 +183,10 @@ static void setup_kernel_memory_layout(void)
 	/* force vmalloc and modules below kasan shadow */
 	vmax = min(vmax, KASAN_SHADOW_START);
 #endif
-	MODULES_END = vmax;
+	__memcpy_real_area = round_down(vmax - PAGE_SIZE, PAGE_SIZE);
+	__abs_lowcore = round_down(__memcpy_real_area - ABS_LOWCORE_MAP_SIZE,
+				   sizeof(struct lowcore));
+	MODULES_END = round_down(__abs_lowcore, _SEGMENT_SIZE);
 	MODULES_VADDR = MODULES_END - MODULES_LEN;
 	VMALLOC_END = MODULES_VADDR;
 

arch/s390/include/asm/abs_lowcore.h

@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_S390_ABS_LOWCORE_H
+#define _ASM_S390_ABS_LOWCORE_H
+
+#include <asm/lowcore.h>
+
+#define ABS_LOWCORE_MAP_SIZE	(NR_CPUS * sizeof(struct lowcore))
+
+extern unsigned long __abs_lowcore;
+extern bool abs_lowcore_mapped;
+
+struct lowcore *get_abs_lowcore(unsigned long *flags);
+void put_abs_lowcore(struct lowcore *lc, unsigned long flags);
+int abs_lowcore_map(int cpu, struct lowcore *lc, bool alloc);
+void abs_lowcore_unmap(int cpu);
+
+#endif /* _ASM_S390_ABS_LOWCORE_H */

arch/s390/include/asm/ccwdev.h

@@ -214,7 +214,6 @@ extern struct ccw_device *ccw_device_create_console(struct ccw_driver *);
 extern void ccw_device_destroy_console(struct ccw_device *);
 extern int ccw_device_enable_console(struct ccw_device *);
 extern void ccw_device_wait_idle(struct ccw_device *);
-extern int ccw_device_force_console(struct ccw_device *);
 
 extern void *ccw_device_dma_zalloc(struct ccw_device *cdev, size_t size);
 extern void ccw_device_dma_free(struct ccw_device *cdev,

arch/s390/include/asm/ctl_reg.h

@@ -95,7 +95,8 @@ union ctlreg0 {
 					   Interruption-Filtering Override */
 		unsigned long	   : 3;
 		unsigned long ccc  : 1; /* Cryptography counter control */
-		unsigned long	   : 18;
+		unsigned long pec  : 1; /* PAI extension control */
+		unsigned long	   : 17;
 		unsigned long	   : 3;
 		unsigned long lap  : 1; /* Low-address-protection control */
 		unsigned long	   : 4;

arch/s390/include/asm/lowcore.h

@@ -203,7 +203,9 @@ struct lowcore {
 	__u8	pad_0x1400[0x1500-0x1400];	/* 0x1400 */
 	/* Cryptography-counter designation */
 	__u64	ccd;				/* 0x1500 */
-	__u8	pad_0x1508[0x1800-0x1508];	/* 0x1508 */
+	/* AI-extension counter designation */
+	__u64	aicd;				/* 0x1508 */
+	__u8	pad_0x1510[0x1800-0x1510];	/* 0x1510 */
 
 	/* Transaction abort diagnostic block */
 	struct pgm_tdb pgm_tdb;			/* 0x1800 */

arch/s390/include/asm/maccess.h

@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __ASM_S390_MACCESS_H
+#define __ASM_S390_MACCESS_H
+
+#include <linux/types.h>
+
+struct iov_iter;
+
+extern unsigned long __memcpy_real_area;
+void memcpy_real_init(void);
+size_t memcpy_real_iter(struct iov_iter *iter, unsigned long src, size_t count);
+int memcpy_real(void *dest, unsigned long src, size_t count);
+#ifdef CONFIG_CRASH_DUMP
+int copy_oldmem_kernel(void *dst, unsigned long src, size_t count);
+#endif
+
+#endif /* __ASM_S390_MACCESS_H */

arch/s390/include/asm/os_info.h

@@ -41,20 +41,6 @@ u32 os_info_csum(struct os_info *os_info);
 
 #ifdef CONFIG_CRASH_DUMP
 void *os_info_old_entry(int nr, unsigned long *size);
-size_t copy_oldmem_iter(struct iov_iter *iter, unsigned long src, size_t count);
-
-static inline int copy_oldmem_kernel(void *dst, unsigned long src, size_t count)
-{
-	struct iov_iter iter;
-	struct kvec kvec;
-
-	kvec.iov_base = dst;
-	kvec.iov_len = count;
-	iov_iter_kvec(&iter, WRITE, &kvec, 1, count);
-	if (copy_oldmem_iter(&iter, src, count) < count)
-		return -EFAULT;
-	return 0;
-}
 #else
 static inline void *os_info_old_entry(int nr, unsigned long *size)
 {

arch/s390/include/asm/pai.h

@@ -17,7 +17,9 @@ struct qpaci_info_block {
 	struct {
 		u64 : 8;
 		u64 num_cc : 8;	/* # of supported crypto counters */
-		u64 : 48;
+		u64 : 9;
+		u64 num_nnpa : 7;	/* # of supported NNPA counters */
+		u64 : 32;
 	};
 };
 
@@ -42,6 +44,8 @@ static inline int qpaci(struct qpaci_info_block *info)
 #define PAI_CRYPTO_BASE			0x1000	/* First event number */
 #define PAI_CRYPTO_MAXCTR		256	/* Max # of event counters */
 #define PAI_CRYPTO_KERNEL_OFFSET	2048
+#define PAI_NNPA_BASE			0x1800	/* First event number */
+#define PAI_NNPA_MAXCTR			128	/* Max # of event counters */
 
 DECLARE_STATIC_KEY_FALSE(pai_key);
 

arch/s390/include/asm/pci.h

@@ -117,7 +117,6 @@ struct zpci_bus {
 struct zpci_dev {
 	struct zpci_bus *zbus;
 	struct list_head entry;		/* list of all zpci_devices, needed for hotplug, etc. */
-	struct list_head bus_next;
 	struct kref kref;
 	struct hotplug_slot hotplug_slot;
 

arch/s390/include/asm/pgtable.h

@@ -1777,6 +1777,10 @@ static inline swp_entry_t __swp_entry(unsigned long type, unsigned long offset)
 
 extern int vmem_add_mapping(unsigned long start, unsigned long size);
 extern void vmem_remove_mapping(unsigned long start, unsigned long size);
+extern int __vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot, bool alloc);
+extern int vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot);
+extern void vmem_unmap_4k_page(unsigned long addr);
+extern pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc);
 extern int s390_enable_sie(void);
 extern int s390_enable_skey(void);
 extern void s390_reset_cmma(struct mm_struct *mm);

arch/s390/include/asm/processor.h

@@ -306,23 +306,6 @@ static __always_inline void __noreturn disabled_wait(void)
 
 #define ARCH_LOW_ADDRESS_LIMIT	0x7fffffffUL
 
-extern int memcpy_real(void *, unsigned long, size_t);
-extern void memcpy_absolute(void *, void *, size_t);
-
-#define put_abs_lowcore(member, x) do {					\
-	unsigned long __abs_address = offsetof(struct lowcore, member);	\
-	__typeof__(((struct lowcore *)0)->member) __tmp = (x);		\
-									\
-	memcpy_absolute(__va(__abs_address), &__tmp, sizeof(__tmp));	\
-} while (0)
-
-#define get_abs_lowcore(x, member) do {					\
-	unsigned long __abs_address = offsetof(struct lowcore, member);	\
-	__typeof__(((struct lowcore *)0)->member) *__ptr = &(x);	\
-									\
-	memcpy_absolute(__ptr, __va(__abs_address), sizeof(*__ptr));	\
-} while (0)
-
 extern int s390_isolate_bp(void);
 extern int s390_isolate_bp_guest(void);
 

arch/s390/include/asm/smp.h

@@ -30,7 +30,8 @@ extern void smp_emergency_stop(void);
 
 extern int smp_find_processor_id(u16 address);
 extern int smp_store_status(int cpu);
-extern void smp_save_dump_cpus(void);
+extern void smp_save_dump_ipl_cpu(void);
+extern void smp_save_dump_secondary_cpus(void);
 extern void smp_yield_cpu(int cpu);
 extern void smp_cpu_set_polarization(int cpu, int val);
 extern int smp_cpu_get_polarization(int cpu);
@@ -58,6 +59,7 @@ static inline void smp_cpus_done(unsigned int max_cpus)
 {
 }
 
+extern int smp_reinit_ipl_cpu(void);
 extern int smp_rescan_cpus(void);
 extern void __noreturn cpu_die(void);
 extern void __cpu_die(unsigned int cpu);

arch/s390/kernel/Makefile

@@ -40,7 +40,7 @@ obj-y	+= sysinfo.o lgr.o os_info.o machine_kexec.o
 obj-y	+= runtime_instr.o cache.o fpu.o dumpstack.o guarded_storage.o sthyi.o
 obj-y	+= entry.o reipl.o relocate_kernel.o kdebugfs.o alternative.o
 obj-y	+= nospec-branch.o ipl_vmparm.o machine_kexec_reloc.o unwind_bc.o
-obj-y	+= smp.o text_amode31.o stacktrace.o
+obj-y	+= smp.o text_amode31.o stacktrace.o abs_lowcore.o
 
 extra-y				+= head64.o vmlinux.lds
 
@@ -72,7 +72,7 @@ obj-$(CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT)	+= ima_arch.o
 obj-$(CONFIG_PERF_EVENTS)	+= perf_event.o perf_cpum_cf_common.o
 obj-$(CONFIG_PERF_EVENTS)	+= perf_cpum_cf.o perf_cpum_sf.o
 obj-$(CONFIG_PERF_EVENTS)	+= perf_cpum_cf_events.o perf_regs.o
-obj-$(CONFIG_PERF_EVENTS)	+= perf_pai_crypto.o
+obj-$(CONFIG_PERF_EVENTS)	+= perf_pai_crypto.o perf_pai_ext.o
 
 obj-$(CONFIG_TRACEPOINTS)	+= trace.o
 obj-$(findstring y, $(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) $(CONFIG_PGSTE))	+= uv.o

arch/s390/kernel/abs_lowcore.c

@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/pgtable.h>
+#include <asm/abs_lowcore.h>
+
+#define ABS_LOWCORE_UNMAPPED	1
+#define ABS_LOWCORE_LAP_ON	2
+#define ABS_LOWCORE_IRQS_ON	4
+
+unsigned long __bootdata_preserved(__abs_lowcore);
+bool __ro_after_init abs_lowcore_mapped;
+
+int abs_lowcore_map(int cpu, struct lowcore *lc, bool alloc)
+{
+	unsigned long addr = __abs_lowcore + (cpu * sizeof(struct lowcore));
+	unsigned long phys = __pa(lc);
+	int rc, i;
+
+	for (i = 0; i < LC_PAGES; i++) {
+		rc = __vmem_map_4k_page(addr, phys, PAGE_KERNEL, alloc);
+		if (rc) {
+			/*
+			 * Do not unmap allocated page tables in case the
+			 * allocation was not requested. In such a case the
+			 * request is expected coming from an atomic context,
+			 * while the unmap attempt might sleep.
+			 */
+			if (alloc) {
+				for (--i; i >= 0; i--) {
+					addr -= PAGE_SIZE;
+					vmem_unmap_4k_page(addr);
+				}
+			}
+			return rc;
+		}
+		addr += PAGE_SIZE;
+		phys += PAGE_SIZE;
+	}
+	return 0;
+}
+
+void abs_lowcore_unmap(int cpu)
+{
+	unsigned long addr = __abs_lowcore + (cpu * sizeof(struct lowcore));
+	int i;
+
+	for (i = 0; i < LC_PAGES; i++) {
+		vmem_unmap_4k_page(addr);
+		addr += PAGE_SIZE;
+	}
+}
+
+struct lowcore *get_abs_lowcore(unsigned long *flags)
+{
+	unsigned long irq_flags;
+	union ctlreg0 cr0;
+	int cpu;
+
+	*flags = 0;
+	cpu = get_cpu();
+	if (abs_lowcore_mapped) {
+		return ((struct lowcore *)__abs_lowcore) + cpu;
+	} else {
+		if (cpu != 0)
+			panic("Invalid unmapped absolute lowcore access\n");
+		local_irq_save(irq_flags);
+		if (!irqs_disabled_flags(irq_flags))
+			*flags |= ABS_LOWCORE_IRQS_ON;
+		__ctl_store(cr0.val, 0, 0);
+		if (cr0.lap) {
+			*flags |= ABS_LOWCORE_LAP_ON;
+			__ctl_clear_bit(0, 28);
+		}
+		*flags |= ABS_LOWCORE_UNMAPPED;
+		return lowcore_ptr[0];
+	}
+}
+
+void put_abs_lowcore(struct lowcore *lc, unsigned long flags)
+{
+	if (abs_lowcore_mapped) {
+		if (flags)
+			panic("Invalid mapped absolute lowcore release\n");
+	} else {
+		if (smp_processor_id() != 0)
+			panic("Invalid mapped absolute lowcore access\n");
+		if (!(flags & ABS_LOWCORE_UNMAPPED))
+			panic("Invalid unmapped absolute lowcore release\n");
+		if (flags & ABS_LOWCORE_LAP_ON)
+			__ctl_set_bit(0, 28);
+		if (flags & ABS_LOWCORE_IRQS_ON)
+			local_irq_enable();
+	}
+	put_cpu();
+}

arch/s390/kernel/crash_dump.c

@@ -21,6 +21,7 @@
 #include <asm/elf.h>
 #include <asm/ipl.h>
 #include <asm/sclp.h>
+#include <asm/maccess.h>
 
 #define PTR_ADD(x, y) (((char *) (x)) + ((unsigned long) (y)))
 #define PTR_SUB(x, y) (((char *) (x)) - ((unsigned long) (y)))
@@ -53,8 +54,6 @@ struct save_area {
 };
 
 static LIST_HEAD(dump_save_areas);
-static DEFINE_MUTEX(memcpy_real_mutex);
-static char memcpy_real_buf[PAGE_SIZE];
 
 /*
  * Allocate a save area
@@ -116,27 +115,7 @@ void __init save_area_add_vxrs(struct save_area *sa, __vector128 *vxrs)
 	memcpy(sa->vxrs_high, vxrs + 16, 16 * sizeof(__vector128));
 }
 
-static size_t copy_to_iter_real(struct iov_iter *iter, unsigned long src, size_t count)
-{
-	size_t len, copied, res = 0;
-
-	mutex_lock(&memcpy_real_mutex);
-	while (count) {
-		len = min(PAGE_SIZE, count);
-		if (memcpy_real(memcpy_real_buf, src, len))
-			break;
-		copied = copy_to_iter(memcpy_real_buf, len, iter);
-		count -= copied;
-		src += copied;
-		res += copied;
-		if (copied < len)
-			break;
-	}
-	mutex_unlock(&memcpy_real_mutex);
-	return res;
-}
-
-size_t copy_oldmem_iter(struct iov_iter *iter, unsigned long src, size_t count)
+static size_t copy_oldmem_iter(struct iov_iter *iter, unsigned long src, size_t count)
 {
 	size_t len, copied, res = 0;
 
@@ -156,7 +135,7 @@ size_t copy_oldmem_iter(struct iov_iter *iter, unsigned long src, size_t count)
 			} else {
 				len = count;
 			}
-			copied = copy_to_iter_real(iter, src, len);
+			copied = memcpy_real_iter(iter, src, len);
 		}
 		count -= copied;
 		src += copied;
@@ -167,6 +146,19 @@ size_t copy_oldmem_iter(struct iov_iter *iter, unsigned long src, size_t count)
 	return res;
 }
 
+int copy_oldmem_kernel(void *dst, unsigned long src, size_t count)
+{
+	struct iov_iter iter;
+	struct kvec kvec;
+
+	kvec.iov_base = dst;
+	kvec.iov_len = count;
+	iov_iter_kvec(&iter, WRITE, &kvec, 1, count);
+	if (copy_oldmem_iter(&iter, src, count) < count)
+		return -EFAULT;
+	return 0;
+}
+
 /*
  * Copy one page from "oldmem"
  */

arch/s390/kernel/debug.c

@@ -250,7 +250,7 @@ static debug_info_t *debug_info_alloc(const char *name, int pages_per_area,
 	rc->level	   = level;
 	rc->buf_size	   = buf_size;
 	rc->entry_size	   = sizeof(debug_entry_t) + buf_size;
-	strlcpy(rc->name, name, sizeof(rc->name));
+	strscpy(rc->name, name, sizeof(rc->name));
 	memset(rc->views, 0, DEBUG_MAX_VIEWS * sizeof(struct debug_view *));
 	memset(rc->debugfs_entries, 0, DEBUG_MAX_VIEWS * sizeof(struct dentry *));
 	refcount_set(&(rc->ref_count), 0);